Merge pull request #65294 from steshaw/linkchecker-9.4.0

linkchecker: 9.3.1 -> 9.4.0

pkgs/tools/networking/linkchecker/add-no-robots-flag.patch

@@ -1,60 +0,0 @@
-diff --git a/linkcheck/checker/httpurl.py b/linkcheck/checker/httpurl.py
-index 6f207b6..161619c 100644
---- a/linkcheck/checker/httpurl.py
-+++ b/linkcheck/checker/httpurl.py
-@@ -75,7 +75,7 @@ def allows_robots (self, url):
-         @return: True if access is granted, otherwise False
-         @rtype: bool
-         """
--        return self.aggregate.robots_txt.allows_url(self)
-+        return not self.aggregate.config['robotstxt'] or self.aggregate.robots_txt.allows_url(self)
- 
-     def content_allows_robots (self):
-         """
-diff --git a/linkcheck/configuration/__init__.py b/linkcheck/configuration/__init__.py
-index fc2c148..234fa05 100644
---- a/linkcheck/configuration/__init__.py
-+++ b/linkcheck/configuration/__init__.py
-@@ -163,6 +163,7 @@ def __init__ (self):
-         ## checking options
-         self["allowedschemes"] = []
-         self['cookiefile'] = None
-+        self['robotstxt'] = True
-         self["debugmemory"] = False
-         self["localwebroot"] = None
-         self["maxfilesizeparse"] = 1*1024*1024
-diff --git a/linkcheck/configuration/confparse.py b/linkcheck/configuration/confparse.py
-index 67751ed..845fa95 100644
---- a/linkcheck/configuration/confparse.py
-+++ b/linkcheck/configuration/confparse.py
-@@ -149,6 +149,7 @@ def read_checking_config (self):
-                  self.get(section, 'allowedschemes').split(',')]
-         self.read_boolean_option(section, "debugmemory")
-         self.read_string_option(section, "cookiefile")
-+        self.read_boolean_option(section, "robotstxt")
-         self.read_string_option(section, "localwebroot")
-         try:
-             self.read_boolean_option(section, "sslverify")
-diff --git a/linkchecker b/linkchecker
-index 199532c..9e91fa5 100755
---- a/linkchecker
-+++ b/linkchecker
-@@ -321,6 +321,9 @@ group.add_argument("--cookiefile", dest="cookiefile", metavar="FILENAME",
-                  help=_(
- """Read a file with initial cookie data. The cookie data format is
- explained below."""))
-+# const because store_false doesn't detect absent flags
-+group.add_argument("--no-robots", action="store_const", const=False,
-+                   dest="norobotstxt", help=_("Disable robots.txt checks"))
- group.add_argument("--check-extern", action="store_true",
-                  dest="checkextern", help=_("""Check external URLs."""))
- group.add_argument("--ignore-url", action="append", metavar="REGEX",
-@@ -431,6 +434,8 @@ if options.externstrict:
- if options.extern:
-     pats = [linkcheck.get_link_pat(arg) for arg in options.extern]
-     config["externlinks"].extend(pats)
-+if options.norobotstxt is not None:
-+    config['robotstxt'] = options.norobotstxt
- if options.checkextern:
-     config["checkextern"] = True
- elif not config["checkextern"]:

pkgs/tools/networking/linkchecker/default.nix

@@ -1,64 +1,49 @@
-{ stdenv, lib, fetchFromGitHub, fetchpatch, python2, gettext }:
-let
-  # pin requests version until next release.
-  # see: https://github.com/linkcheck/linkchecker/issues/76
-  python2Packages = (python2.override {
-    packageOverrides = self: super: {   
-      requests = super.requests.overridePythonAttrs(oldAttrs: rec {
-        version = "2.14.2";
-        src = oldAttrs.src.override {
-          inherit version;
-          sha256 = "0lyi82a0ijs1m7k9w1mqwbmq1qjsac35fazx7xqyh8ws76xanx52";
-        };
-      });
-    };
-  }).pkgs;
-in
+{ stdenv, lib, fetchFromGitHub, python2Packages, gettext }:
+
 python2Packages.buildPythonApplication rec {
-  pname = "LinkChecker";
-  version = "9.3.1";
+  pname = "linkchecker";
+  version = "9.4.0";
 
-  nativeBuildInputs = [ gettext ];
-  pythonPath = (with python2Packages; [
-    requests
-  ]) ++ [ gettext ];
-
-  checkInputs = with python2Packages; [ pytest ];
-
-  # the original repository is abandoned, development is now happening here:
   src = fetchFromGitHub {
-    owner = "linkcheck";
-    repo = "linkchecker";
+    owner = pname;
+    repo = pname;
     rev = "v${version}";
-    sha256 = "080mv4iwvlsfnm7l9basd6i8p4q8990mdhkwick9s6javrbf1r1d";
+    sha256 = "1vbwl2vb8dyzki27z3sl5yf9dhdd2cpkg10vbgaz868dhpqlshgs";
   };
 
-  # 1. upstream refuses to support ignoring robots.txt
-  # 2. fix build: https://github.com/linkcheck/linkchecker/issues/10
-  patches = 
-    let
-      fix-setup-py = fetchpatch {
-        name = "fix-setup-py.patch";
-        url = https://github.com/linkcheck/linkchecker/commit/e62e630.patch;
-        sha256 = "046q1whg715w2yv33xx6rkj7fspvvz60cl978ax92lnf8j101czx";
-      };
-    in [
-      ./add-no-robots-flag.patch
-      fix-setup-py
-    ];
+  nativeBuildInputs = [ gettext ];
 
-  postInstall = ''
-    rm $out/bin/linkchecker-gui
+  propagatedBuildInputs = with python2Packages; [
+    ConfigArgParse
+    argcomplete
+    dnspython
+    pyxdg
+    requests
+  ];
+
+  checkInputs = with python2Packages; [
+    parameterized
+    pytest
+  ];
+
+  postPatch = ''
+    sed -i 's/^requests.*$/requests>=2.2/' requirements.txt
+    sed -i "s/'request.*'/'requests >= 2.2'/" setup.py
+    sed -i 's~/usr/lib/python2.7/argparse.py~~g' po/Makefile
   '';
 
   checkPhase = ''
+    runHook preCheck
+
     # the mime test fails for me...
     rm tests/test_mimeutil.py
     ${lib.optionalString stdenv.isDarwin ''
-    # network tests fails on darwin
-    rm tests/test_network.py
+      # network tests fails on darwin
+      rm tests/test_network.py
     ''}
     make test PYTESTOPTS="--tb=short" TESTS="tests/test_*.py tests/logger/test_*.py"
+
+    runHook postCheck
   '';
 
   meta = {