python/pdm: improve implementation and UI

- implement per dependency sourceSelector option
- update pyproject.nix
- update nix-unit

dev-flake/flake.lock

@@ -79,11 +79,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1688870561,
+        "lastModified": 1698974481,
-        "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=",
+        "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=",
         "owner": "nix-community",
         "repo": "nix-github-actions",
-        "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301",
+        "rev": "4bb5e752616262457bc7ca5882192a564c0472d2",
         "type": "github"
       },
       "original": {
@@ -98,26 +98,40 @@
           "flake-parts"
         ],
         "nix-github-actions": "nix-github-actions",
-        "nixpkgs": [
+        "nixpkgs": "nixpkgs",
-          "nixpkgs"
-        ],
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1694670962,
+        "lastModified": 1701194455,
-        "narHash": "sha256-HvMq0TJGYSx37zHm4j2d+JUZx4/6X7xKEt/0DeCiwjQ=",
+        "narHash": "sha256-TlCq5DnIlfMFS+b/kqi0vR/8gRgVpqisOr+gjpo1VPc=",
-        "owner": "adisbladis",
+        "owner": "nix-community",
         "repo": "nix-unit",
-        "rev": "3ed2378bddad85257fc508a291408f9ed9673d01",
+        "rev": "2a061b6981c12b1a995740799a4465c3124f60ac",
         "type": "github"
       },
       "original": {
-        "owner": "adisbladis",
+        "owner": "nix-community",
         "repo": "nix-unit",
         "type": "github"
       }
     },
     "nixpkgs": {
+      "locked": {
+        "lastModified": 1701045352,
+        "narHash": "sha256-iWsDbWzBP4gotkRfg/lH2A3O9wFoJc+yVO8CDuHLRe8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "5171694860f185961daff3b1b413dabcab421300",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "master",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
       "locked": {
         "lastModified": 1696604326,
         "narHash": "sha256-YXUNI0kLEcI5g8lqGMb0nh67fY9f2YoJsILafh6zlMo=",
@@ -159,7 +173,7 @@
         "flake-compat": "flake-compat",
         "flake-parts": "flake-parts",
         "nix-unit": "nix-unit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "pre-commit-hooks": "pre-commit-hooks"
       }
     },
@@ -186,11 +200,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1689620039,
+        "lastModified": 1699786194,
-        "narHash": "sha256-BtNwghr05z7k5YMdq+6nbue+nEalvDepuA7qdQMAKoQ=",
+        "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "719c2977f958c41fa60a928e2fbc50af14844114",
+        "rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1",
         "type": "github"
       },
       "original": {

dev-flake/flake.nix

@@ -18,8 +18,8 @@
     pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
     pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs";
 
-    nix-unit.url = "github:adisbladis/nix-unit";
+    nix-unit.url = "github:nix-community/nix-unit";
-    nix-unit.inputs.nixpkgs.follows = "nixpkgs";
+    # nix-unit.inputs.nixpkgs.follows = "nixpkgs";
     nix-unit.inputs.flake-parts.follows = "flake-parts";
 
     devshell = {

modules/dream2nix/WIP-python-pdm/default.nix

@@ -6,6 +6,9 @@
 }: let
   libpdm = import ./lib.nix {
     inherit lib libpyproject;
+    python3 = config.deps.python3;
+    targetPlatform =
+      lib.systems.elaborate config.deps.python3.stdenv.targetPlatform;
   };
 
   libpyproject = import (dream2nix.inputs.pyproject-nix + "/lib") {inherit lib;};
@@ -18,18 +21,8 @@
     stdenvNoCC = config.deps.stdenvNoCC;
   };
 
-  selectWheel = files: let
-    wheelFiles = lib.filter libpyproject.pypa.isWheelFileName files;
-    wheels = map libpyproject.pypa.parseWheelFileName wheelFiles;
-    selected = libpyproject.pypa.selectWheels config.deps.python3.stdenv.targetPlatform config.deps.python3 wheels;
-  in
-    if lib.length selected == 0
-    then null
-    else (lib.head selected).filename;
-
   lock_data = lib.importTOML config.pdm.lockfile;
   environ = libpyproject.pep508.mkEnviron config.deps.python3;
-  selector = config.pdm.sourceSelector;
 
   pyproject = libpdm.loadPdmPyProject (lib.importTOML config.pdm.pyproject);
 
@@ -37,7 +30,7 @@
     inherit environ pyproject;
   };
   parsed_lock_data = libpdm.parseLockData {
-    inherit environ lock_data selector;
+    inherit environ lock_data;
   };
 in {
   imports = [
@@ -47,7 +40,13 @@ in {
     ./interface.nix
   ];
   name = pyproject.pyproject.project.name;
-  version = pyproject.pyproject.project.version;
+  version = lib.mkDefault (
+    if pyproject.pyproject.project ? version
+    then pyproject.pyproject.project.version
+    else if lib.elem "version" pyproject.pyproject.project.dynamic or []
+    then "dynamic"
+    else "unknown"
+  );
   deps = {nixpkgs, ...}: {
     inherit
       (nixpkgs)
@@ -59,14 +58,12 @@ in {
       stdenv
       ;
   };
-  pdm.sourceSelector = lib.mkDefault selectWheel;
+  pdm.sourceSelector = lib.mkDefault libpdm.preferWheelSelector;
   overrideAll = {
-    options.sourceSelector = import ./sourceSelectorOption.nix {inherit lib;};
-    # TODO: per dependency selector isn't yet respected
     config.sourceSelector = lib.mkOptionDefault config.pdm.sourceSelector;
   };
   buildPythonPackage = {
-    format = "pyproject";
+    format = lib.mkDefault "pyproject";
   };
   mkDerivation = {
     propagatedBuildInputs =
@@ -83,27 +80,39 @@ in {
       };
 
       packages = lib.flip lib.mapAttrs deps' (name: pkg: {
-        ${pkg.version}.module = {
+        ${pkg.version}.module = {...} @ depConfig: let
-          inherit name;
+          selector =
-          version = pkg.version;
+            if lib.isFunction depConfig.config.sourceSelector
+            then depConfig.config.sourceSelector
+            else if depConfig.config.sourceSelector == "wheel"
+            then libpdm.preferWheelSelector
+            else if depConfig.config.sourceSelector == "sdist"
+            then libpdm.preferSdistSelector
+            else throw "Invalid sourceSelector: ${depConfig.config.sourceSelector}";
+          source = pkg.sources.${selector (lib.attrNames pkg.sources)};
+        in {
           imports = [
+            ./interface-dependency.nix
             dream2nix.modules.dream2nix.buildPythonPackage
             dream2nix.modules.dream2nix.mkDerivation
             dream2nix.modules.dream2nix.package-func
           ];
+          inherit name;
+          version = pkg.version;
           buildPythonPackage = {
-            format =
+            format = lib.mkDefault (
-              if lib.hasSuffix ".whl" pkg.source.file
+              if lib.hasSuffix ".whl" source.file
               then "wheel"
-              else "pyproject";
+              else "pyproject"
+            );
           };
           mkDerivation = {
             # required: { pname, file, version, hash, kind, curlOpts ? "" }:
             src = libpyproject-fetchers.fetchFromPypi {
               pname = name;
-              file = pkg.source.file;
+              file = source.file;
               version = pkg.version;
-              hash = pkg.source.hash;
+              hash = source.hash;
             };
             propagatedBuildInputs =
               lib.forEach

modules/dream2nix/WIP-python-pdm/interface-dependency.nix

@@ -0,0 +1,5 @@
+{lib, ...}: {
+  options = {
+    sourceSelector = import ./sourceSelectorOption.nix {inherit lib;};
+  };
+}

modules/dream2nix/WIP-python-pdm/lib.nix

@@ -1,6 +1,8 @@
 {
   lib,
   libpyproject,
+  python3,
+  targetPlatform,
 }: rec {
   # Get the filename from an URL.
   # getFilename :: String -> String
@@ -115,8 +117,19 @@
   # Select a single wheel from a list of filenames
   # This assumes filtering on usable wheels has already been performed.
   # selectWheel :: [String] ->  String
-  selectWheel = filenames:
+  selectWheel = files: let
-    lib.findFirst (x: lib.hasSuffix ".whl" x) null filenames;
+    wheelFiles = lib.filter libpyproject.pypa.isWheelFileName files;
+    wheels = map libpyproject.pypa.parseWheelFileName wheelFiles;
+    selected =
+      libpyproject.pypa.selectWheels
+      targetPlatform
+      python3
+      wheels;
+  in (
+    if lib.length selected == 0
+    then null
+    else (lib.head selected).filename
+  );
 
   # Source selectors.
   # Prefer to select a wheel from a list of filenames.
@@ -157,7 +170,6 @@
   parseLockData = {
     lock_data,
     environ, # Output from `libpyproject.pep508.mkEnviron`
-    selector,
   }: let
     # TODO: validate against lock file version.
     parsePackage = item: let
@@ -172,7 +184,6 @@
       parsedDeps = with lib.trivial; (
         map
         ((flip pipe) [
-          lib.strings.toLower
           libpyproject.pep508.parseString
         ])
         item.dependencies or []
@@ -180,7 +191,7 @@
       value = {
         dependencies = requiredDeps environ parsedDeps;
         inherit (item) version;
-        source = sources.${selector (lib.attrNames compatibleSources)};
+        sources = compatibleSources;
         # In the future we could add additional meta data fields
         # such as summary
       };

modules/dream2nix/WIP-python-pdm/sourceSelectorOption.nix

@@ -1,6 +1,9 @@
 {lib}:
 lib.mkOption {
-  type = lib.types.functionTo lib.types.str;
+  type =
+    lib.types.either
+    (lib.types.enum ["wheel" "sdist"])
+    (lib.types.functionTo lib.types.str);
   description = ''
     A selector function which picks a source for a specific dependency
     Python dependencies can have multiple possible sources, like for example:

tests/nix-unit/test_python-pdm-lib/default.nix

@@ -1,18 +1,25 @@
 {
   pkgs ? import <nixpkgs> {},
   lib ? import <nixpkgs/lib>,
-  dream2nix ? (import (../../../modules + "/flake.nix")).outputs inputs,
+  dream2nix ? (import ../../../.),
-  inputs ? (import (../../../modules + "/default.nix")).inputs,
+  inputs ? (import ../../../.).inputs,
 }: let
   libpdm = (import ../../../modules/dream2nix/WIP-python-pdm/lib.nix) {
     inherit lib libpyproject;
+    python3 = pkgs.python3;
+    targetPlatform = lib.systems.elaborate "x86_64-linux";
   };
   pyproject-nix = inputs.pyproject-nix;
   libpyproject = import (pyproject-nix + "/lib") {inherit lib;};
 
   linux_environ = lib.importJSON ./environ.json;
 
-  test_isDependencyRequired = {
+  testIsUsableSdistFilename = filename: let
+    environ = linux_environ;
+  in
+    libpdm.isUsableSdistFilename {inherit environ filename;};
+in {
+  isDependencyRequired = {
     test_isDependencyRequired__not_required = {
       expr =
         libpdm.isDependencyRequired
@@ -29,12 +36,7 @@
     };
   };
 
-  testIsUsableSdistFilename = filename: let
+  isUsableFilename = let
-    environ = linux_environ;
-  in
-    libpdm.isUsableSdistFilename {inherit environ filename;};
-
-  tests_isUsableFilename = let
     testIsUsableWheelFilename = filename: let
       environ = linux_environ;
     in
@@ -51,7 +53,7 @@
     };
   };
 
-  tests_selectExtension = let
+  selectExtension = let
     names = [
       "certifi-2023.7.22-py3-abi3-any.whl"
       "certifi-2023.7.22.tar.gz"
@@ -72,7 +74,7 @@
     };
   };
 
-  tests_selectSdist = let
+  selectSdist = let
     names = [
       "certifi-2023.7.22-py3-abi3-any.whl"
       "certifi-2023.7.22.tar.gz"
@@ -103,7 +105,18 @@
     };
   };
 
-  tests_preferWheelSelector = {
+  selectWheel = {
+    test_empty = {
+      expr = libpdm.selectWheel [];
+      expected = null;
+    };
+    test_simple_call = {
+      expr = libpdm.selectWheel ["charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl"];
+      expected = null;
+    };
+  };
+
+  preferWheelSelector = {
     test_preferWheelSelector__has_wheel = let
       names = [
         "certifi-2023.7.22-py3-abi3-any.whl"
@@ -124,7 +137,8 @@
       expected = "certifi-2023.7.22.tar.gz";
     };
   };
-  tests_preferSdistSelector = {
+
+  preferSdistSelector = {
     test_preferSdistSelector__has_sdist = let
       names = [
         "certifi-2023.7.22-py3-abi3-any.whl"
@@ -145,25 +159,24 @@
       expected = "certifi-2023.7.22.tar.gz";
     };
   };
-  tests_parseLockData = let
+  parseLockData = let
     lock_data = lib.importTOML ./../../../examples/packages/single-language/python-pdm/pdm.lock;
     version = "2.31.0";
     parsed = libpdm.parseLockData {
       inherit lock_data;
       environ = linux_environ;
-      selector = libpdm.preferWheelSelector;
     };
   in {
     test_parseLockData = {
       expr =
         (parsed ? "requests")
         && (parsed.requests.version == version)
-        && (parsed.requests ? source);
+        && (parsed.requests ? sources);
       expected = true;
     };
 
     test_parseLockData_file = {
-      expr = parsed.requests.source.file;
+      expr = libpdm.preferWheelSelector (lib.attrNames parsed.requests.sources);
       expected = "requests-2.31.0-py3-none-any.whl";
     };
 
@@ -177,7 +190,7 @@
       ];
     };
   };
-  tests_groupsWithDeps = let
+  groupsWithDeps = let
     environ = linux_environ;
     pyproject = libpdm.loadPdmPyProject (lib.importTOML ./../../../examples/packages/single-language/python-pdm/pyproject.toml);
     groups_with_deps = libpdm.groupsWithDeps {
@@ -198,12 +211,11 @@
     };
   };
 
-  tests_getDepsRecursively = let
+  getDepsRecursively = let
     environ = linux_environ;
     lock_data = lib.importTOML ./../../../examples/packages/single-language/python-pdm/pdm.lock;
     parsed_lock_data = libpdm.parseLockData {
       inherit environ lock_data;
-      selector = libpdm.preferWheelSelector;
     };
     deps = libpdm.getDepsRecursively parsed_lock_data "requests";
   in {
@@ -221,11 +233,13 @@
         urllib3 = "2.0.5";
       };
     };
-    test_getDepsRecursively_sources = {
+    test_getDepsRecursively_sources = let
-      expr = lib.mapAttrs (key: value: value.source.file) deps;
+      selectFilename = sources: libpdm.preferWheelSelector (lib.attrNames sources);
+    in {
+      expr = lib.mapAttrs (key: value: selectFilename value.sources) deps;
       expected = {
         certifi = "certifi-2023.7.22-py3-none-any.whl";
-        charset-normalizer = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl";
+        charset-normalizer = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl";
         idna = "idna-3.4-py3-none-any.whl";
         requests = "requests-2.31.0-py3-none-any.whl";
         urllib3 = "urllib3-2.0.5-py3-none-any.whl";
@@ -233,7 +247,7 @@
     };
   };
 
-  tests_selectForGroups = let
+  selectForGroups = let
     environ = linux_environ;
     pyproject = libpdm.loadPdmPyProject (lib.importTOML ./../../../examples/packages/single-language/python-pdm/pyproject.toml);
     lock_data = lib.importTOML ./../../../examples/packages/single-language/python-pdm/pdm.lock;
@@ -243,7 +257,6 @@
     parsed_lock_data = libpdm.parseLockData {
       inherit lock_data;
       environ = linux_environ;
-      selector = libpdm.preferWheelSelector;
     };
     deps_default = libpdm.selectForGroups {
       inherit parsed_lock_data groups_with_deps;
@@ -268,11 +281,13 @@
         urllib3 = "2.0.5";
       };
     };
-    test_selectForGroups_sources = {
+    test_selectForGroups_sources = let
-      expr = lib.mapAttrs (key: value: value.source.file) deps_default;
+      selectFilename = sources: libpdm.preferWheelSelector (lib.attrNames sources);
+    in {
+      expr = lib.mapAttrs (key: value: selectFilename value.sources) deps_default;
       expected = {
         certifi = "certifi-2023.7.22-py3-none-any.whl";
-        charset-normalizer = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl";
+        charset-normalizer = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl";
         idna = "idna-3.4-py3-none-any.whl";
         requests = "requests-2.31.0-py3-none-any.whl";
         urllib3 = "urllib3-2.0.5-py3-none-any.whl";
@@ -290,5 +305,4 @@
       };
     };
   };
-in
+}
-  test_isDependencyRequired // tests_isUsableFilename // tests_selectExtension // tests_selectSdist // tests_preferWheelSelector // tests_preferSdistSelector // tests_parseLockData // tests_groupsWithDeps // tests_getDepsRecursively // tests_selectForGroups