[#81] Add config option for localhost links

Problem:
Almost all the time we can't validate localhost links, so we just skip them.
But to run ftp links tests (#47) we need to refer to localhost.

Solution:
Add config option whether to ignore localhost links and provide bats
tests for this new feature.

Co-authored-by: Alexander Bantyev <alexander.bantyev@serokell.io>

.buildkite/pipeline.yml

@@ -1,33 +1,44 @@
-# SPDX-FileCopyrightText: 2019 Serokell <https://serokell.io>
+# SPDX-FileCopyrightText: 2019-2021 Serokell <https://serokell.io>
 #
 # SPDX-License-Identifier: MPL-2.0
 
 steps:
   - command: nix-build ci.nix -A trailing-whitespace-check
     label: Check trailing whitespaces
+
   - command: nix-build ci.nix -A xrefcheck-lib-and-tests
     label: Library and tests
+
+  - command: nix run -f ci.nix pkgs.bats pkgs.diffutils xrefcheck-static -c bash -c "cd tests/golden && bats ./"
+    label: Golden tests (bats)
+
   - command: nix-build ci.nix -A xrefcheck-static
     label: Executable
     artifact_paths:
       - "result/bin/xrefcheck"
+
   - command: nix-build ci.nix -A xrefcheck-windows
     label: Windows executable
     artifact_paths:
       - "result/bin/*"
+
   - command: nix run -f ci.nix xrefcheck-static -c xrefcheck --ignored tests/markdowns
     label: Xrefcheck itself
+
   - command: nix run -f ci.nix pkgs.reuse -c reuse lint
     label: REUSE lint
+
   - command:
       - nix run -f ci.nix pkgs.curl pkgs.gitAndTools.hub -c bash -c "curl https://raw.githubusercontent.com/serokell/scratch/release-binary/scripts/release-binary.sh | bash"
     label: Create a pre-release
     branches: master
+
   - command:
       - nix-build docker
       - nix run -f ci.nix pkgs.skopeo -c ./scripts/upload-docker-image.sh "docker-archive:$(readlink result)" "docker://docker.io/serokell/xrefcheck:latest"
     label: Push to dockerhub
     branches: master
+
   - command:
       - nix-build docker
       - nix run -f ci.nix pkgs.skopeo -c ./scripts/upload-docker-image.sh "docker-archive:$(readlink result)" "docker://docker.io/serokell/xrefcheck:${BUILDKITE_BRANCH}"

.gitmodules

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+[submodule "tests/golden/helpers/bats-assert"]
+	path = tests/golden/helpers/bats-assert
+	url = git@github.com:bats-core/bats-assert.git
+[submodule "tests/golden/helpers/bats-support"]
+	path = tests/golden/helpers/bats-support
+	url = git@github.com:bats-core/bats-support.git

CHANGES.md

@@ -6,6 +6,9 @@
 
 Unreleased
 ==========
+* [#85](https://github.com/serokell/xrefcheck/pull/85)
+  + Make possible to specify whether ignore localhost links, use
+  `check-localhost` CLA argument (by default localhost links will not be checked).
 
 0.2.1
 ==========

ci.nix

@@ -18,7 +18,10 @@ rec {
 
   project-src = pkgs.haskell-nix.haskellLib.cleanGit {
     name = "xrefcheck";
-    src = ./.;
+    src = pkgs.haskell-nix.haskellLib.cleanSourceWith {
+      filter = path: type: !(pkgs.lib.hasInfix "tests/golden/helpers" path);
+      src = ./.;
+    };
   };
 
   xrefcheck-lib-and-tests = (import ./xrefcheck.nix { linux = true; });

exec/Main.hs

@@ -53,8 +53,10 @@ defaultAction Options{..} = do
     when oVerbose $
         fmtLn $ "=== Repository data ===\n\n" <> indentF 2 (build repoInfo)
 
-    verifyRes <- allowRewrite showProgressBar $ \rw ->
-        verifyRepo rw (cVerification config) oMode root repoInfo
+    verifyRes <- allowRewrite showProgressBar $ \rw -> do
+      let fullConfig = addVerifyOptions (cVerification config) oVerifyOptions
+      verifyRepo rw fullConfig oMode root repoInfo
+
     case verifyErrors verifyRes of
         Nothing ->
             fmtLn "All repository links are valid."

src-files/def-config.yaml

@@ -39,6 +39,9 @@ verification:
   ignoreRefs:
     []
 
+  # Check localhost links.
+  checkLocalhost: false
+
 # Parameters of scanners for various file types.
 scanners:
 

src/Xrefcheck/CLI.hs

@@ -11,6 +11,8 @@ module Xrefcheck.CLI
   , shouldCheckExternal
   , Command (..)
   , Options (..)
+  , VerifyOptions (..)
+  , addVerifyOptions
   , TraversalOptions (..)
   , addTraversalOptions
   , defaultConfigPaths
@@ -22,11 +24,12 @@ import qualified Data.List as L
 import qualified Data.Text as T
 import Data.Version (showVersion)
 import Options.Applicative
-  (Parser, ReadM, command, eitherReader, execParser, flag', footerDoc, fullDesc, help, helper,
+  (Parser, ReadM, command, eitherReader, execParser, flag, flag', footerDoc, fullDesc, help, helper,
   hsubparser, info, infoOption, long, metavar, option, progDesc, short, strOption, switch, value)
 import Options.Applicative.Help.Pretty (Doc, displayS, fill, fillSep, indent, renderPretty, text)
 
 import Paths_xrefcheck (version)
+import Xrefcheck.Config (VerifyConfig (..))
 import Xrefcheck.Core
 import Xrefcheck.Scan
 
@@ -56,6 +59,7 @@ data Options = Options
   , oVerbose          :: Bool
   , oShowProgressBar  :: Maybe Bool
   , oTraversalOptions :: TraversalOptions
+  , oVerifyOptions    :: VerifyOptions
   }
 
 data TraversalOptions = TraversalOptions
@@ -69,6 +73,17 @@ addTraversalOptions TraversalConfig{..} (TraversalOptions ignored) =
   , ..
   }
 
+newtype VerifyOptions = VerifyOptions
+  { voCheckLocalhost :: Maybe Bool
+  }
+
+addVerifyOptions :: VerifyConfig -> VerifyOptions -> VerifyConfig
+addVerifyOptions VerifyConfig{..} (VerifyOptions checkLocalhost) =
+  VerifyConfig
+  { vcCheckLocalhost = fromMaybe vcCheckLocalhost checkLocalhost
+  , ..
+  }
+
 -- | Where to try to seek configuration if specific path is not set.
 defaultConfigPaths :: [FilePath]
 defaultConfigPaths = ["./xrefcheck.yaml", "./.xrefcheck.yaml"]
@@ -132,6 +147,7 @@ optionsParser = do
     , pure Nothing
     ]
   oTraversalOptions <- traversalOptionsParser
+  oVerifyOptions <- verifyOptionsParser
   return Options{..}
 
 traversalOptionsParser :: Parser TraversalOptions
@@ -142,6 +158,13 @@ traversalOptionsParser = do
     help "Files and folders which we pretend do not exist."
   return TraversalOptions{..}
 
+verifyOptionsParser :: Parser VerifyOptions
+verifyOptionsParser = do
+  voCheckLocalhost <- flag Nothing (Just True) $
+    long "check-localhost" <>
+    help "Check localhost links."
+  return VerifyOptions{..}
+
 dumpConfigOptions :: Parser Command
 dumpConfigOptions = hsubparser $
   command "dump-config" $

src/Xrefcheck/Config.hs

@@ -50,6 +50,8 @@ data VerifyConfig = VerifyConfig
     -- ^ Prefixes of files, references in which we should not analyze.
   , vcIgnoreRefs                :: Maybe [Regex]
     -- ^ Regular expressions that match external references we should not verify.
+  , vcCheckLocalhost            :: Bool
+    -- ^ If True - we will check localhost links.
   }
 
 -- | Configs for all the supported scanners.

src/Xrefcheck/Verify.hs

@@ -271,17 +271,16 @@ verifyReference
 
 checkExternalResource :: VerifyConfig -> Text -> IO (VerifyResult VerifyError)
 checkExternalResource VerifyConfig{..} link
-  | isIgnored = return mempty
-  | doesReferLocalhost = return mempty
+  | skipCheck = return mempty
   | otherwise = fmap toVerifyRes $ do
       makeRequest HEAD 0.3 >>= \case
         Right () -> return $ Right ()
         Left   _ -> makeRequest GET 0.7
   where
-    isIgnored =
-      let maybeIsIgnored = doesMatchAnyRegex link <$> vcIgnoreRefs
-      in fromMaybe False maybeIsIgnored
-    doesReferLocalhost = any (`T.isInfixOf` link) ["://localhost", "://127.0.0.1"]
+    skipCheck = isIgnored || (not vcCheckLocalhost && isLocalLink)
+      where
+        isIgnored =  maybe False (doesMatchAnyRegex link) vcIgnoreRefs
+        isLocalLink = any (`T.isInfixOf` link) ["://localhost", "://127.0.0.1"]
 
     doesMatchAnyRegex :: Text -> ([Regex] -> Bool)
     doesMatchAnyRegex src = any $ \regex ->

tests/configs/github-config.yaml

@@ -45,6 +45,9 @@ verification:
   ignoreRefs:
     []
 
+  # Check localhost links.
+  checkLocalhost: false
+
 # Parameters of scanners for various file types.
 scanners:
 

tests/golden/check-localhost.bats

@@ -0,0 +1,90 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+load 'helpers/bats-support/load'
+load 'helpers/bats-assert/load'
+
+# this function is used for:
+# - delete all color characters
+# - replace socket port with N
+# - replace multiple connection retry errors with single one
+#   (because at some machine there are one error and at others - two)
+prepare () {
+  sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" \
+    | sed 's/socket: [0-9]*/socket: N/g' \
+    | sed 's/Network.Socket.connect: <socket: N>: does not exist (Connection refused),//g'
+}
+
+@test "Config: check-localhost=false, CLA --check-localhost not provided" {
+  run xrefcheck \
+    -c check-localhost/config-check-disabled.yaml \
+    -r check-localhost
+
+  assert_output --partial "All repository links are valid."
+}
+
+@test "Config: check-localhost=false, CLA --check-localhost provided" {
+  xrefcheck \
+    -c check-localhost/config-check-disabled.yaml \
+    -r check-localhost \
+    --check-localhost \
+    | prepare > /tmp/check-localhost.test || true
+
+  diff /tmp/check-localhost.test check-localhost/expected.gold \
+    --ignore-space-change \
+    --ignore-blank-lines \
+    --new-file # treat absent files as empty
+
+  rm /tmp/check-localhost.test
+}
+
+@test "Config: check-localhost=true, CLA --check-localhost not provided" {
+  xrefcheck \
+    -c check-localhost/config-check-enabled.yaml \
+    -r check-localhost \
+    | prepare > /tmp/check-localhost.test || true
+
+  diff /tmp/check-localhost.test check-localhost/expected.gold \
+    --ignore-space-change \
+    --ignore-blank-lines \
+    --new-file # treat absent files as empty
+
+  rm /tmp/check-localhost.test
+}
+
+@test "Config: check-localhost=true, CLA --check-localhost provided" {
+  xrefcheck \
+    -c check-localhost/config-check-enabled.yaml \
+    -r check-localhost \
+    --check-localhost \
+    | prepare > /tmp/check-localhost.test || true
+
+  diff /tmp/check-localhost.test check-localhost/expected.gold \
+    --ignore-space-change \
+    --ignore-blank-lines \
+    --new-file # treat absent files as empty
+
+  rm /tmp/check-localhost.test
+}
+
+@test "Config: missing, CLA --check-localhost not provided" {
+  run xrefcheck
+
+  assert_output --partial "All repository links are valid."
+}
+
+@test "Config: missing, CLA --check-localhost provided" {
+  xrefcheck \
+    --check-localhost \
+    | prepare > /tmp/check-localhost.test || true
+
+  diff /tmp/check-localhost.test check-localhost/expected.gold \
+    --ignore-space-change \
+    --ignore-blank-lines \
+    --new-file # treat absent files as empty
+
+  rm /tmp/check-localhost.test
+}

tests/golden/check-localhost/check-localhost.md

@@ -0,0 +1,13 @@
+<!--
+ - SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
+ -
+ - SPDX-License-Identifier: MPL-2.0
+ -->
+
+Serokell [web-site](https://localhost/web-site)
+
+Serokell [team](https://127.0.0.1/team)
+
+Serokell [blog](http://localhost/blog)
+
+Serokell [labs](http://127.0.0.1/labs)

tests/golden/check-localhost/config-check-disabled.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
+#
+# SPDX-License-Identifier: Unlicense
+
+traversal:
+  ignored: []
+
+verification:
+  anchorSimilarityThreshold: 0.5
+  externalRefCheckTimeout: 10s
+  notScanned: []
+  virtualFiles: []
+  ignoreRefs: []
+  checkLocalhost: false
+  ignoreAuthFailures: true
+
+scanners:
+  markdown:
+    flavor: GitHub

tests/golden/check-localhost/config-check-enabled.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
+#
+# SPDX-License-Identifier: Unlicense
+
+traversal:
+  ignored: []
+
+verification:
+  anchorSimilarityThreshold: 0.5
+  externalRefCheckTimeout: 10s
+  notScanned: []
+  virtualFiles: []
+  ignoreRefs: []
+  checkLocalhost: true
+  ignoreAuthFailures: true
+
+scanners:
+  markdown:
+    flavor: GitHub

tests/golden/check-localhost/expected.gold

@@ -0,0 +1,43 @@
+=== Invalid references found ===
+
+  ➥  In file ./check-localhost/check-localhost.md
+     bad reference (external) at src:7:10-47:
+       - text: "web-site"
+       - link: https://localhost/web-site
+       - anchor: -
+
+     ⛂  InternalException (HostCannotConnect "localhost" [Network.Socket.connect: <socket: N>: does not exist (Connection refused)])
+
+
+
+  ➥  In file ./check-localhost/check-localhost.md
+     bad reference (external) at src:9:10-39:
+       - text: "team"
+       - link: https://127.0.0.1/team
+       - anchor: -
+
+     ⛂  InternalException (HostCannotConnect "127.0.0.1" [Network.Socket.connect: <socket: N>: does not exist (Connection refused)])
+
+
+
+  ➥  In file ./check-localhost/check-localhost.md
+     bad reference (external) at src:11:10-38:
+       - text: "blog"
+       - link: http://localhost/blog
+       - anchor: -
+
+     ⛂  ConnectionFailure Network.Socket.connect: <socket: N>: does not exist (Connection refused)
+
+
+
+  ➥  In file ./check-localhost/check-localhost.md
+     bad reference (external) at src:13:10-38:
+       - text: "labs"
+       - link: http://127.0.0.1/labs
+       - anchor: -
+
+     ⛂  ConnectionFailure Network.Socket.connect: <socket: N>: does not exist (Connection refused)
+
+
+
+Invalid references dumped, 4 in total.

tests/golden/check-localhost/expected.gold.license

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
+# SPDX-License-Identifier: Unlicense

tests/golden/helpers/bats-assert

@@ -0,0 +1 @@
+Subproject commit 672ad1823a4d2f0c475fdbec0c4497498eec5f41

tests/golden/helpers/bats-support

@@ -0,0 +1 @@
+Subproject commit d140a65044b2d6810381935ae7f0c94c7023c8c3