mirror of
https://github.com/serokell/xrefcheck.git
synced 2024-08-17 09:30:35 +03:00
[#81] Add config option for localhost links
Problem: Almost all the time we can't validate localhost links, so we just skip them. But to run ftp links tests (#47) we need to refer to localhost. Solution: Add config option whether to ignore localhost links and provide bats tests for this new feature. Co-authored-by: Alexander Bantyev <alexander.bantyev@serokell.io>
This commit is contained in:
parent
d58882685f
commit
29d4f12c61
@ -1,33 +1,44 @@
|
||||
# SPDX-FileCopyrightText: 2019 Serokell <https://serokell.io>
|
||||
# SPDX-FileCopyrightText: 2019-2021 Serokell <https://serokell.io>
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
steps:
|
||||
- command: nix-build ci.nix -A trailing-whitespace-check
|
||||
label: Check trailing whitespaces
|
||||
|
||||
- command: nix-build ci.nix -A xrefcheck-lib-and-tests
|
||||
label: Library and tests
|
||||
|
||||
- command: nix run -f ci.nix pkgs.bats pkgs.diffutils xrefcheck-static -c bash -c "cd tests/golden && bats ./"
|
||||
label: Golden tests (bats)
|
||||
|
||||
- command: nix-build ci.nix -A xrefcheck-static
|
||||
label: Executable
|
||||
artifact_paths:
|
||||
- "result/bin/xrefcheck"
|
||||
|
||||
- command: nix-build ci.nix -A xrefcheck-windows
|
||||
label: Windows executable
|
||||
artifact_paths:
|
||||
- "result/bin/*"
|
||||
|
||||
- command: nix run -f ci.nix xrefcheck-static -c xrefcheck --ignored tests/markdowns
|
||||
label: Xrefcheck itself
|
||||
|
||||
- command: nix run -f ci.nix pkgs.reuse -c reuse lint
|
||||
label: REUSE lint
|
||||
|
||||
- command:
|
||||
- nix run -f ci.nix pkgs.curl pkgs.gitAndTools.hub -c bash -c "curl https://raw.githubusercontent.com/serokell/scratch/release-binary/scripts/release-binary.sh | bash"
|
||||
label: Create a pre-release
|
||||
branches: master
|
||||
|
||||
- command:
|
||||
- nix-build docker
|
||||
- nix run -f ci.nix pkgs.skopeo -c ./scripts/upload-docker-image.sh "docker-archive:$(readlink result)" "docker://docker.io/serokell/xrefcheck:latest"
|
||||
label: Push to dockerhub
|
||||
branches: master
|
||||
|
||||
- command:
|
||||
- nix-build docker
|
||||
- nix run -f ci.nix pkgs.skopeo -c ./scripts/upload-docker-image.sh "docker-archive:$(readlink result)" "docker://docker.io/serokell/xrefcheck:${BUILDKITE_BRANCH}"
|
||||
|
10
.gitmodules
vendored
Normal file
10
.gitmodules
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
[submodule "tests/golden/helpers/bats-assert"]
|
||||
path = tests/golden/helpers/bats-assert
|
||||
url = git@github.com:bats-core/bats-assert.git
|
||||
[submodule "tests/golden/helpers/bats-support"]
|
||||
path = tests/golden/helpers/bats-support
|
||||
url = git@github.com:bats-core/bats-support.git
|
@ -6,6 +6,9 @@
|
||||
|
||||
Unreleased
|
||||
==========
|
||||
* [#85](https://github.com/serokell/xrefcheck/pull/85)
|
||||
+ Make possible to specify whether ignore localhost links, use
|
||||
`check-localhost` CLA argument (by default localhost links will not be checked).
|
||||
|
||||
0.2.1
|
||||
==========
|
||||
|
5
ci.nix
5
ci.nix
@ -18,7 +18,10 @@ rec {
|
||||
|
||||
project-src = pkgs.haskell-nix.haskellLib.cleanGit {
|
||||
name = "xrefcheck";
|
||||
src = ./.;
|
||||
src = pkgs.haskell-nix.haskellLib.cleanSourceWith {
|
||||
filter = path: type: !(pkgs.lib.hasInfix "tests/golden/helpers" path);
|
||||
src = ./.;
|
||||
};
|
||||
};
|
||||
|
||||
xrefcheck-lib-and-tests = (import ./xrefcheck.nix { linux = true; });
|
||||
|
@ -53,8 +53,10 @@ defaultAction Options{..} = do
|
||||
when oVerbose $
|
||||
fmtLn $ "=== Repository data ===\n\n" <> indentF 2 (build repoInfo)
|
||||
|
||||
verifyRes <- allowRewrite showProgressBar $ \rw ->
|
||||
verifyRepo rw (cVerification config) oMode root repoInfo
|
||||
verifyRes <- allowRewrite showProgressBar $ \rw -> do
|
||||
let fullConfig = addVerifyOptions (cVerification config) oVerifyOptions
|
||||
verifyRepo rw fullConfig oMode root repoInfo
|
||||
|
||||
case verifyErrors verifyRes of
|
||||
Nothing ->
|
||||
fmtLn "All repository links are valid."
|
||||
|
@ -39,6 +39,9 @@ verification:
|
||||
ignoreRefs:
|
||||
[]
|
||||
|
||||
# Check localhost links.
|
||||
checkLocalhost: false
|
||||
|
||||
# Parameters of scanners for various file types.
|
||||
scanners:
|
||||
|
||||
|
@ -11,6 +11,8 @@ module Xrefcheck.CLI
|
||||
, shouldCheckExternal
|
||||
, Command (..)
|
||||
, Options (..)
|
||||
, VerifyOptions (..)
|
||||
, addVerifyOptions
|
||||
, TraversalOptions (..)
|
||||
, addTraversalOptions
|
||||
, defaultConfigPaths
|
||||
@ -22,11 +24,12 @@ import qualified Data.List as L
|
||||
import qualified Data.Text as T
|
||||
import Data.Version (showVersion)
|
||||
import Options.Applicative
|
||||
(Parser, ReadM, command, eitherReader, execParser, flag', footerDoc, fullDesc, help, helper,
|
||||
(Parser, ReadM, command, eitherReader, execParser, flag, flag', footerDoc, fullDesc, help, helper,
|
||||
hsubparser, info, infoOption, long, metavar, option, progDesc, short, strOption, switch, value)
|
||||
import Options.Applicative.Help.Pretty (Doc, displayS, fill, fillSep, indent, renderPretty, text)
|
||||
|
||||
import Paths_xrefcheck (version)
|
||||
import Xrefcheck.Config (VerifyConfig (..))
|
||||
import Xrefcheck.Core
|
||||
import Xrefcheck.Scan
|
||||
|
||||
@ -56,6 +59,7 @@ data Options = Options
|
||||
, oVerbose :: Bool
|
||||
, oShowProgressBar :: Maybe Bool
|
||||
, oTraversalOptions :: TraversalOptions
|
||||
, oVerifyOptions :: VerifyOptions
|
||||
}
|
||||
|
||||
data TraversalOptions = TraversalOptions
|
||||
@ -69,6 +73,17 @@ addTraversalOptions TraversalConfig{..} (TraversalOptions ignored) =
|
||||
, ..
|
||||
}
|
||||
|
||||
newtype VerifyOptions = VerifyOptions
|
||||
{ voCheckLocalhost :: Maybe Bool
|
||||
}
|
||||
|
||||
addVerifyOptions :: VerifyConfig -> VerifyOptions -> VerifyConfig
|
||||
addVerifyOptions VerifyConfig{..} (VerifyOptions checkLocalhost) =
|
||||
VerifyConfig
|
||||
{ vcCheckLocalhost = fromMaybe vcCheckLocalhost checkLocalhost
|
||||
, ..
|
||||
}
|
||||
|
||||
-- | Where to try to seek configuration if specific path is not set.
|
||||
defaultConfigPaths :: [FilePath]
|
||||
defaultConfigPaths = ["./xrefcheck.yaml", "./.xrefcheck.yaml"]
|
||||
@ -132,6 +147,7 @@ optionsParser = do
|
||||
, pure Nothing
|
||||
]
|
||||
oTraversalOptions <- traversalOptionsParser
|
||||
oVerifyOptions <- verifyOptionsParser
|
||||
return Options{..}
|
||||
|
||||
traversalOptionsParser :: Parser TraversalOptions
|
||||
@ -142,6 +158,13 @@ traversalOptionsParser = do
|
||||
help "Files and folders which we pretend do not exist."
|
||||
return TraversalOptions{..}
|
||||
|
||||
verifyOptionsParser :: Parser VerifyOptions
|
||||
verifyOptionsParser = do
|
||||
voCheckLocalhost <- flag Nothing (Just True) $
|
||||
long "check-localhost" <>
|
||||
help "Check localhost links."
|
||||
return VerifyOptions{..}
|
||||
|
||||
dumpConfigOptions :: Parser Command
|
||||
dumpConfigOptions = hsubparser $
|
||||
command "dump-config" $
|
||||
|
@ -50,6 +50,8 @@ data VerifyConfig = VerifyConfig
|
||||
-- ^ Prefixes of files, references in which we should not analyze.
|
||||
, vcIgnoreRefs :: Maybe [Regex]
|
||||
-- ^ Regular expressions that match external references we should not verify.
|
||||
, vcCheckLocalhost :: Bool
|
||||
-- ^ If True - we will check localhost links.
|
||||
}
|
||||
|
||||
-- | Configs for all the supported scanners.
|
||||
|
@ -271,17 +271,16 @@ verifyReference
|
||||
|
||||
checkExternalResource :: VerifyConfig -> Text -> IO (VerifyResult VerifyError)
|
||||
checkExternalResource VerifyConfig{..} link
|
||||
| isIgnored = return mempty
|
||||
| doesReferLocalhost = return mempty
|
||||
| skipCheck = return mempty
|
||||
| otherwise = fmap toVerifyRes $ do
|
||||
makeRequest HEAD 0.3 >>= \case
|
||||
Right () -> return $ Right ()
|
||||
Left _ -> makeRequest GET 0.7
|
||||
where
|
||||
isIgnored =
|
||||
let maybeIsIgnored = doesMatchAnyRegex link <$> vcIgnoreRefs
|
||||
in fromMaybe False maybeIsIgnored
|
||||
doesReferLocalhost = any (`T.isInfixOf` link) ["://localhost", "://127.0.0.1"]
|
||||
skipCheck = isIgnored || (not vcCheckLocalhost && isLocalLink)
|
||||
where
|
||||
isIgnored = maybe False (doesMatchAnyRegex link) vcIgnoreRefs
|
||||
isLocalLink = any (`T.isInfixOf` link) ["://localhost", "://127.0.0.1"]
|
||||
|
||||
doesMatchAnyRegex :: Text -> ([Regex] -> Bool)
|
||||
doesMatchAnyRegex src = any $ \regex ->
|
||||
|
@ -45,6 +45,9 @@ verification:
|
||||
ignoreRefs:
|
||||
[]
|
||||
|
||||
# Check localhost links.
|
||||
checkLocalhost: false
|
||||
|
||||
# Parameters of scanners for various file types.
|
||||
scanners:
|
||||
|
||||
|
90
tests/golden/check-localhost.bats
Normal file
90
tests/golden/check-localhost.bats
Normal file
@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
|
||||
#
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
load 'helpers/bats-support/load'
|
||||
load 'helpers/bats-assert/load'
|
||||
|
||||
# this function is used for:
|
||||
# - delete all color characters
|
||||
# - replace socket port with N
|
||||
# - replace multiple connection retry errors with single one
|
||||
# (because at some machine there are one error and at others - two)
|
||||
prepare () {
|
||||
sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" \
|
||||
| sed 's/socket: [0-9]*/socket: N/g' \
|
||||
| sed 's/Network.Socket.connect: <socket: N>: does not exist (Connection refused),//g'
|
||||
}
|
||||
|
||||
@test "Config: check-localhost=false, CLA --check-localhost not provided" {
|
||||
run xrefcheck \
|
||||
-c check-localhost/config-check-disabled.yaml \
|
||||
-r check-localhost
|
||||
|
||||
assert_output --partial "All repository links are valid."
|
||||
}
|
||||
|
||||
@test "Config: check-localhost=false, CLA --check-localhost provided" {
|
||||
xrefcheck \
|
||||
-c check-localhost/config-check-disabled.yaml \
|
||||
-r check-localhost \
|
||||
--check-localhost \
|
||||
| prepare > /tmp/check-localhost.test || true
|
||||
|
||||
diff /tmp/check-localhost.test check-localhost/expected.gold \
|
||||
--ignore-space-change \
|
||||
--ignore-blank-lines \
|
||||
--new-file # treat absent files as empty
|
||||
|
||||
rm /tmp/check-localhost.test
|
||||
}
|
||||
|
||||
@test "Config: check-localhost=true, CLA --check-localhost not provided" {
|
||||
xrefcheck \
|
||||
-c check-localhost/config-check-enabled.yaml \
|
||||
-r check-localhost \
|
||||
| prepare > /tmp/check-localhost.test || true
|
||||
|
||||
diff /tmp/check-localhost.test check-localhost/expected.gold \
|
||||
--ignore-space-change \
|
||||
--ignore-blank-lines \
|
||||
--new-file # treat absent files as empty
|
||||
|
||||
rm /tmp/check-localhost.test
|
||||
}
|
||||
|
||||
@test "Config: check-localhost=true, CLA --check-localhost provided" {
|
||||
xrefcheck \
|
||||
-c check-localhost/config-check-enabled.yaml \
|
||||
-r check-localhost \
|
||||
--check-localhost \
|
||||
| prepare > /tmp/check-localhost.test || true
|
||||
|
||||
diff /tmp/check-localhost.test check-localhost/expected.gold \
|
||||
--ignore-space-change \
|
||||
--ignore-blank-lines \
|
||||
--new-file # treat absent files as empty
|
||||
|
||||
rm /tmp/check-localhost.test
|
||||
}
|
||||
|
||||
@test "Config: missing, CLA --check-localhost not provided" {
|
||||
run xrefcheck
|
||||
|
||||
assert_output --partial "All repository links are valid."
|
||||
}
|
||||
|
||||
@test "Config: missing, CLA --check-localhost provided" {
|
||||
xrefcheck \
|
||||
--check-localhost \
|
||||
| prepare > /tmp/check-localhost.test || true
|
||||
|
||||
diff /tmp/check-localhost.test check-localhost/expected.gold \
|
||||
--ignore-space-change \
|
||||
--ignore-blank-lines \
|
||||
--new-file # treat absent files as empty
|
||||
|
||||
rm /tmp/check-localhost.test
|
||||
}
|
13
tests/golden/check-localhost/check-localhost.md
Normal file
13
tests/golden/check-localhost/check-localhost.md
Normal file
@ -0,0 +1,13 @@
|
||||
<!--
|
||||
- SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
|
||||
-
|
||||
- SPDX-License-Identifier: MPL-2.0
|
||||
-->
|
||||
|
||||
Serokell [web-site](https://localhost/web-site)
|
||||
|
||||
Serokell [team](https://127.0.0.1/team)
|
||||
|
||||
Serokell [blog](http://localhost/blog)
|
||||
|
||||
Serokell [labs](http://127.0.0.1/labs)
|
19
tests/golden/check-localhost/config-check-disabled.yaml
Normal file
19
tests/golden/check-localhost/config-check-disabled.yaml
Normal file
@ -0,0 +1,19 @@
|
||||
# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
|
||||
#
|
||||
# SPDX-License-Identifier: Unlicense
|
||||
|
||||
traversal:
|
||||
ignored: []
|
||||
|
||||
verification:
|
||||
anchorSimilarityThreshold: 0.5
|
||||
externalRefCheckTimeout: 10s
|
||||
notScanned: []
|
||||
virtualFiles: []
|
||||
ignoreRefs: []
|
||||
checkLocalhost: false
|
||||
ignoreAuthFailures: true
|
||||
|
||||
scanners:
|
||||
markdown:
|
||||
flavor: GitHub
|
19
tests/golden/check-localhost/config-check-enabled.yaml
Normal file
19
tests/golden/check-localhost/config-check-enabled.yaml
Normal file
@ -0,0 +1,19 @@
|
||||
# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
|
||||
#
|
||||
# SPDX-License-Identifier: Unlicense
|
||||
|
||||
traversal:
|
||||
ignored: []
|
||||
|
||||
verification:
|
||||
anchorSimilarityThreshold: 0.5
|
||||
externalRefCheckTimeout: 10s
|
||||
notScanned: []
|
||||
virtualFiles: []
|
||||
ignoreRefs: []
|
||||
checkLocalhost: true
|
||||
ignoreAuthFailures: true
|
||||
|
||||
scanners:
|
||||
markdown:
|
||||
flavor: GitHub
|
43
tests/golden/check-localhost/expected.gold
Normal file
43
tests/golden/check-localhost/expected.gold
Normal file
@ -0,0 +1,43 @@
|
||||
=== Invalid references found ===
|
||||
|
||||
➥ In file ./check-localhost/check-localhost.md
|
||||
bad reference (external) at src:7:10-47:
|
||||
- text: "web-site"
|
||||
- link: https://localhost/web-site
|
||||
- anchor: -
|
||||
|
||||
⛂ InternalException (HostCannotConnect "localhost" [Network.Socket.connect: <socket: N>: does not exist (Connection refused)])
|
||||
|
||||
|
||||
|
||||
➥ In file ./check-localhost/check-localhost.md
|
||||
bad reference (external) at src:9:10-39:
|
||||
- text: "team"
|
||||
- link: https://127.0.0.1/team
|
||||
- anchor: -
|
||||
|
||||
⛂ InternalException (HostCannotConnect "127.0.0.1" [Network.Socket.connect: <socket: N>: does not exist (Connection refused)])
|
||||
|
||||
|
||||
|
||||
➥ In file ./check-localhost/check-localhost.md
|
||||
bad reference (external) at src:11:10-38:
|
||||
- text: "blog"
|
||||
- link: http://localhost/blog
|
||||
- anchor: -
|
||||
|
||||
⛂ ConnectionFailure Network.Socket.connect: <socket: N>: does not exist (Connection refused)
|
||||
|
||||
|
||||
|
||||
➥ In file ./check-localhost/check-localhost.md
|
||||
bad reference (external) at src:13:10-38:
|
||||
- text: "labs"
|
||||
- link: http://127.0.0.1/labs
|
||||
- anchor: -
|
||||
|
||||
⛂ ConnectionFailure Network.Socket.connect: <socket: N>: does not exist (Connection refused)
|
||||
|
||||
|
||||
|
||||
Invalid references dumped, 4 in total.
|
2
tests/golden/check-localhost/expected.gold.license
Normal file
2
tests/golden/check-localhost/expected.gold.license
Normal file
@ -0,0 +1,2 @@
|
||||
# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
|
||||
# SPDX-License-Identifier: Unlicense
|
1
tests/golden/helpers/bats-assert
Submodule
1
tests/golden/helpers/bats-assert
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 672ad1823a4d2f0c475fdbec0c4497498eec5f41
|
1
tests/golden/helpers/bats-support
Submodule
1
tests/golden/helpers/bats-support
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit d140a65044b2d6810381935ae7f0c94c7023c8c3
|
Loading…
Reference in New Issue
Block a user