diff --git a/pkgs/common-updater/directory-listing-updater.nix b/pkgs/common-updater/directory-listing-updater.nix new file mode 100644 index 000000000000..fb7491ab879a --- /dev/null +++ b/pkgs/common-updater/directory-listing-updater.nix @@ -0,0 +1,19 @@ +{ lib +, genericUpdater +, common-updater-scripts +}: + +{ pname ? null +, version ? null +, attrPath ? null +, ignoredVersions ? "" +, rev-prefix ? "" +, odd-unstable ? false +, patchlevel-unstable ? false +, url ? null +}: + +genericUpdater { + inherit pname version attrPath ignoredVersions rev-prefix odd-unstable patchlevel-unstable; + versionLister = "${common-updater-scripts}/bin/list-directory-versions ${lib.optionalString (url != null) "--url=${lib.escapeShellArg url}"}"; +} diff --git a/pkgs/common-updater/generic-updater.nix b/pkgs/common-updater/generic-updater.nix index 04adcf563814..e75a6dd7ab05 100644 --- a/pkgs/common-updater/generic-updater.nix +++ b/pkgs/common-updater/generic-updater.nix @@ -62,7 +62,7 @@ let return 1 } - tags=$($version_lister --pname=$pname --attr-path=$attr_path --file="${fileForGitCommands}") || exit 1 + tags=$(sh -c "$version_lister --pname=$pname --attr-path=$attr_path --file=\"${fileForGitCommands}\"") || exit 1 # print available tags for tag in $tags; do diff --git a/pkgs/common-updater/scripts.nix b/pkgs/common-updater/scripts.nix index 26c77e876362..d5ee3b58c504 100644 --- a/pkgs/common-updater/scripts.nix +++ b/pkgs/common-updater/scripts.nix @@ -1,9 +1,28 @@ -{ lib, stdenv, makeWrapper, coreutils, gnused, gnugrep, diffutils, nix, git, jq }: +{ lib +, stdenv +, makeWrapper +, coreutils +, diffutils +, git +, gnugrep +, gnused +, jq +, nix +, python3Packages +}: stdenv.mkDerivation { name = "common-updater-scripts"; - nativeBuildInputs = [ makeWrapper ]; + nativeBuildInputs = [ + makeWrapper + python3Packages.wrapPython + ]; + + pythonPath = [ + python3Packages.beautifulsoup4 + python3Packages.requests + ]; dontUnpack = true; @@ -11,8 +30,15 @@ stdenv.mkDerivation { mkdir -p $out/bin cp ${./scripts}/* $out/bin + # wrap non python scripts for f in $out/bin/*; do - wrapProgram $f --prefix PATH : ${lib.makeBinPath [ coreutils gnused gnugrep nix diffutils git jq ]} + if ! (head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'); then + wrapProgram $f --prefix PATH : ${lib.makeBinPath [ coreutils diffutils git gnugrep gnused jq nix ]} + fi done + + # wrap python scripts + makeWrapperArgs+=( --prefix PATH : "${lib.makeBinPath [ nix ]}" ) + wrapPythonPrograms ''; } diff --git a/pkgs/common-updater/scripts/list-directory-versions b/pkgs/common-updater/scripts/list-directory-versions new file mode 100755 index 000000000000..46c9e9d30a5d --- /dev/null +++ b/pkgs/common-updater/scripts/list-directory-versions @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import argparse +import requests +import os +import subprocess +import json +import re +from bs4 import BeautifulSoup + +parser = argparse.ArgumentParser( + description="Get all available versions listed for a package in a site." +) + +parser.add_argument( + "--pname", + default=os.environ.get("UPDATE_NIX_PNAME"), + required="UPDATE_NIX_PNAME" not in os.environ, + help="name of the package", +) +parser.add_argument( + "--attr-path", + default=os.environ.get("UPDATE_NIX_ATTR_PATH"), + help="attribute path of the package", +) +parser.add_argument("--url", help="url of the page that lists the package versions") +parser.add_argument("--file", help="file name for writing debugging information") + + +if __name__ == "__main__": + args = parser.parse_args() + + pname = args.pname + + attr_path = args.attr_path or pname + + url = args.url or json.loads( + subprocess.check_output( + [ + "nix-instantiate", + "--json", + "--eval", + "-E", + f"with import ./. {{}}; dirOf (lib.head {attr_path}.src.urls)", + ], + text=True, + ) + ) + + # print a debugging message + if args.file: + with open(args.file, "a") as f: + f.write(f"# Listing versions for {pname} from {url}\n") + + page = requests.get(url) + soup = BeautifulSoup(page.content, "html.parser") + links = soup.find_all("a") + for link in links: + link_url = link.get("href", None) + if link_url is not None: + match = re.fullmatch( + rf"{args.pname}-([\d.]+?(-[\d\w.-]+?)?)(\.tar)?(\.[^.]*)", link_url + ) + if match: + print(match.group(1)) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 295e50ee13fc..a6dd4b5b1e60 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -126,6 +126,8 @@ with pkgs; _experimental-update-script-combinators = callPackage ../common-updater/combinators.nix { }; + directoryListingUpdater = callPackage ../common-updater/directory-listing-updater.nix { }; + gitUpdater = callPackage ../common-updater/git-updater.nix { }; httpTwoLevelsUpdater = callPackage ../common-updater/http-two-levels-updater.nix { };