From c4bc9c1f27f9a365390c839b2a6196ab9d9ea12c Mon Sep 17 00:00:00 2001 From: DavHau Date: Sun, 1 Oct 2023 00:47:33 +0100 Subject: [PATCH] feat(pdm): build default group Also implement source fetching --- examples/dream2nix-repo-flake-pdm/flake.nix | 11 ++- modules/dream2nix/WIP-python-pdm/default.nix | 72 +++++++++++++++--- .../WIP-python-pdm/fetch-from-pypi.nix | 73 +++++++++++++++++++ .../WIP-python-pdm/fetch-from-pypi.sh | 27 +++++++ modules/dream2nix/groups/group.nix | 6 +- modules/dream2nix/groups/interface.nix | 4 +- modules/flake-parts/examples.nix | 13 ++++ tests/nix-unit/test_groups/default.nix | 4 +- 8 files changed, 190 insertions(+), 20 deletions(-) create mode 100644 modules/dream2nix/WIP-python-pdm/fetch-from-pypi.nix create mode 100644 modules/dream2nix/WIP-python-pdm/fetch-from-pypi.sh diff --git a/examples/dream2nix-repo-flake-pdm/flake.nix b/examples/dream2nix-repo-flake-pdm/flake.nix index dba0790f..20b0a474 100644 --- a/examples/dream2nix-repo-flake-pdm/flake.nix +++ b/examples/dream2nix-repo-flake-pdm/flake.nix @@ -2,7 +2,7 @@ description = "My flake with dream2nix packages"; inputs = { - dream2nix.url = "github:nix-community/dream2nix"; + dream2nix.url = "github:nix-community/dream2nix?dir=modules"; nixpkgs.url = "nixpkgs/nixos-unstable"; }; @@ -27,9 +27,12 @@ pdm.pyproject = ./pyproject.toml; pdm.pythonInterpreter = nixpkgs.legacyPackages.python3; }; - evaled = lib.evalModules {modules = [module];}; - defaultPackage = evaled.config.groups.default.public.packages.my-package; + evaled = lib.evalModules { + modules = [module]; + specialArgs.dream2nix = dream2nix; + specialArgs.packageSets.nixpkgs = nixpkgs.legacyPackages.x86_64-linux; + }; in { - packages.${system}.default = defaultPackage; + packages.${system} = evaled.config.groups.default.public.packages; }; } diff --git a/modules/dream2nix/WIP-python-pdm/default.nix b/modules/dream2nix/WIP-python-pdm/default.nix index 0429b3ac..576f6d1e 100644 --- a/modules/dream2nix/WIP-python-pdm/default.nix +++ b/modules/dream2nix/WIP-python-pdm/default.nix @@ -4,31 +4,81 @@ dream2nix, ... }: let - libpdm = ./lib.nix { + libpdm = import ./lib.nix { inherit lib libpyproject; }; libpyproject = import (dream2nix.inputs.pyproject-nix + "/lib") {inherit lib;}; - lock-data = libpdm.parseLockData { - lock-data = lib.importTOML config.pdm.lock-file; - environ = libpyproject.pep508.mkEnviron config.deps.python3; + lock_data = lib.importTOML config.pdm.lockfile; + environ = libpyproject.pep508.mkEnviron config.deps.python3; + selector = libpdm.preferWheelSelector; + + pyproject = libpdm.loadPdmPyProject (lib.importTOML config.pdm.pyproject); + + groups_with_deps = libpdm.groupsWithDeps { + inherit environ pyproject; + }; + parsed_lock_data = libpdm.parseLockData { + inherit environ lock_data; selector = libpdm.preferWheelSelector; }; + deps_default = libpdm.selectForGroup { + inherit parsed_lock_data groups_with_deps; + groupname = "default"; + }; - pyproject-data = lib.importTOML config.pdm.pyproject; - - pyprojectLoaded = libpdm.loadPdmPyproject pyproject-data; - - build-systems = pyprojectLoaded.build-systems; - dependencies = pyprojectLoaded.dependencies; + fetchFromPypi = import ./fetch-from-pypi.nix { + inherit lib; + inherit (config.deps) curl jq stdenvNoCC; + }; in { imports = [ dream2nix.modules.dream2nix.groups + ../core/deps ./interface.nix ]; + deps = {nixpkgs, ...}: { + inherit + (nixpkgs) + curl + jq + stdenvNoCC + ; + }; commonModule = { - options.sourceSelector = import ./sourceSelectorOption.nix {}; + options.sourceSelector = import ./sourceSelectorOption.nix {inherit lib;}; config.sourceSelector = lib.mkOptionDefault config.pdm.sourceSelector; }; + groups.default = { + packages = lib.flip lib.mapAttrs deps_default (name: pkg: { + inherit name; + version = pkg.version; + imports = [ + dream2nix.modules.dream2nix.buildPythonPackage + dream2nix.modules.dream2nix.mkDerivation + dream2nix.modules.dream2nix.package-func + ]; + buildPythonPackage = { + format = + if lib.hasSuffix ".whl" pkg.source.file + then "wheel" + else "pyproject"; + }; + mkDerivation = { + # required: { pname, file, version, hash, kind, curlOpts ? "" }: + src = fetchFromPypi { + pname = name; + file = pkg.source.file; + version = pkg.version; + hash = pkg.source.hash; + kind = ""; + }; + propagatedBuildInputs = + lib.forEach + parsed_lock_data.${name}.dependencies + (depName: config.groups.default.public.packages.${depName}); + }; + }); + }; } diff --git a/modules/dream2nix/WIP-python-pdm/fetch-from-pypi.nix b/modules/dream2nix/WIP-python-pdm/fetch-from-pypi.nix new file mode 100644 index 00000000..197413e5 --- /dev/null +++ b/modules/dream2nix/WIP-python-pdm/fetch-from-pypi.nix @@ -0,0 +1,73 @@ +# copied from poetry2nix +# LICENSE: https://github.com/nix-community/poetry2nix/blob/master/LICENSE +{ + lib, + curl, + jq, + stdenvNoCC, +}: let + # Predict URL from the PyPI index. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + predictURLFromPypi = lib.makeOverridable ( + { + pname, + file, + hash, + kind, + }: "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}" + ); + + # Fetch from the PyPI index. + # At first we try to fetch the predicated URL but if that fails we + # will use the Pypi API to determine the correct URL. + # Args: + # pname: package name + # file: filename including extension + # version: the version string of the dependency + # hash: SRI hash + # kind: Language implementation and version tag + fetchFromPypi = lib.makeOverridable ( + { + pname, + file, + version, + hash, + kind, + curlOpts ? "", + }: let + predictedURL = predictURLFromPypi {inherit pname file hash kind;}; + in (stdenvNoCC.mkDerivation { + name = file; + nativeBuildInputs = [ + curl + jq + ]; + isWheel = lib.strings.hasSuffix "whl" file; + system = "builtin"; + + preferLocalBuild = true; + impureEnvVars = + lib.fetchers.proxyImpureEnvVars + ++ [ + "NIX_CURL_FLAGS" + ]; + + inherit pname file version curlOpts predictedURL; + + builder = ./fetch-from-pypi.sh; + + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + + passthru = { + urls = [predictedURL]; # retain compatibility with nixpkgs' fetchurl + }; + }) + ); +in + fetchFromPypi diff --git a/modules/dream2nix/WIP-python-pdm/fetch-from-pypi.sh b/modules/dream2nix/WIP-python-pdm/fetch-from-pypi.sh new file mode 100644 index 00000000..627cb938 --- /dev/null +++ b/modules/dream2nix/WIP-python-pdm/fetch-from-pypi.sh @@ -0,0 +1,27 @@ +# copied from poetry2nix +# LICENSE: https://github.com/nix-community/poetry2nix/blob/master/LICENSE + +source $stdenv/setup +set -euo pipefail + +curl="curl \ + --location \ + --max-redirs 20 \ + --retry 2 \ + --disable-epsv \ + --cookie-jar cookies \ + --insecure \ + --speed-time 5 \ + --progress-bar \ + --fail \ + $curlOpts \ + $NIX_CURL_FLAGS" + +echo "Trying to fetch with predicted URL: $predictedURL" + +$curl $predictedURL --output $out && exit 0 + +echo "Predicted URL '$predictedURL' failed, querying pypi.org" +$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url +url=$(cat url) +$curl -k $url --output $out diff --git a/modules/dream2nix/groups/group.nix b/modules/dream2nix/groups/group.nix index e70c6af8..21012e4d 100644 --- a/modules/dream2nix/groups/group.nix +++ b/modules/dream2nix/groups/group.nix @@ -2,6 +2,7 @@ lib, dream2nix, config, + specialArgs, ... }: let t = lib.types; @@ -9,6 +10,7 @@ staticModules = [ dream2nix.modules.dream2nix.core commonModule + {_module.args = specialArgs;} ]; }; in { @@ -32,7 +34,7 @@ in { ''; internal = true; }; - public = lib.mkOption { + public.packages = lib.mkOption { type = t.lazyAttrsOf t.package; description = '' The evaluated packages ready to consume @@ -42,6 +44,6 @@ in { }; config = { packagesEval = config.packages; - public = lib.mapAttrs (name: pkg: pkg.public) config.packagesEval; + public.packages = lib.mapAttrs (name: pkg: pkg.public) config.packagesEval; }; } diff --git a/modules/dream2nix/groups/interface.nix b/modules/dream2nix/groups/interface.nix index 3d9faa44..f0b0b66b 100644 --- a/modules/dream2nix/groups/interface.nix +++ b/modules/dream2nix/groups/interface.nix @@ -7,7 +7,9 @@ }: let t = lib.types; groupType = t.submoduleWith { - modules = [(import ./group.nix {inherit (config) commonModule;})]; + modules = [ + (import ./group.nix {inherit (config) commonModule;}) + ]; inherit specialArgs; }; in { diff --git a/modules/flake-parts/examples.nix b/modules/flake-parts/examples.nix index fc55d3de..9c5e3f67 100644 --- a/modules/flake-parts/examples.nix +++ b/modules/flake-parts/examples.nix @@ -39,6 +39,17 @@ in self'; + importFlakeSmall = flakeFile: let + self' = (import flakeFile).outputs { + dream2nix = modulesFlake; + nixpkgs = inputs.nixpkgs; + self = self'; + }; + in + self'; + + modulesFlake = import (self + /modules) {}; + # Type: [ {${name} = {module, packagePath} ] allExamples = mapAttrsToList (dirName: _: readExamples dirName) packageCategories; @@ -117,6 +128,8 @@ in { .hello; example-repo-flake = (importFlake (self + /examples/dream2nix-repo-flake/flake.nix)).packages.${system}.hello; + example-repo-flake-pdm = + (importFlakeSmall (self + /examples/dream2nix-repo-flake-pdm/flake.nix)).packages.${system}.requests; } ); }; diff --git a/tests/nix-unit/test_groups/default.nix b/tests/nix-unit/test_groups/default.nix index d1c1d792..da65efce 100644 --- a/tests/nix-unit/test_groups/default.nix +++ b/tests/nix-unit/test_groups/default.nix @@ -22,7 +22,7 @@ in { groups.my-group.packages.hello = {...}: fixtures.basic-derivation; }; in { - expr = config.groups.my-group.public.hello ? drvPath; + expr = config.groups.my-group.public.packages.hello ? drvPath; expected = true; }; @@ -32,7 +32,7 @@ in { commonModule = {name = lib.mkForce "hello-mod";}; }; in { - expr = "${config.groups.my-group.public.hello.name}"; + expr = "${config.groups.my-group.public.packages.hello.name}"; expected = "hello-mod"; }; }