fixup/improve python translator and builder

- translator: ensure setup requirements from pyproject.toml are respected
- builder: ensure setup requirements are installed first before the package is built
- simplify the python translator
- dont depend on mach-nix for dependency extraction
- use only pip download to extract dependencies
- allow impure pip translator to cache downloads
- fix dream-lock.json. location was missing

examples/python/flake.nix

@@ -0,0 +1,21 @@
+{
+  inputs = {
+    dream2nix.url = "github:nix-community/dream2nix";
+    src.url = "https://files.pythonhosted.org/packages/5a/86/5f63de7a202550269a617a5d57859a2961f3396ecd1739a70b92224766bc/aiohttp-3.8.1.tar.gz";
+    src.flake = false;
+  };
+
+  outputs = {
+    self,
+    dream2nix,
+    src,
+  } @ inp:
+    (dream2nix.lib.makeFlakeOutputs {
+      systems = ["x86_64-linux"];
+      config.projectRoot = ./.;
+      source = src;
+    })
+    // {
+      checks.x86_64-linux.aiohttp = self.packages.x86_64-linux.aiohttp;
+    };
+}

flake.nix

@@ -146,25 +146,6 @@
         };
       });
 
-    pre-commit-check = forAllSystems (
-      system: pkgs:
-        pre-commit-hooks.lib.${system}.run {
-          src = ./.;
-          hooks = {
-            treefmt = {
-              enable = true;
-              name = "treefmt";
-              pass_filenames = true;
-              entry = l.toString (pkgs.writeScript "treefmt" ''
-                #!${pkgs.bash}/bin/bash
-                export PATH="$PATH:${alejandra.defaultPackage.${system}}/bin"
-                ${pkgs.treefmt}/bin/treefmt --fail-on-change "$@"
-              '');
-            };
-          };
-        }
-    );
-
     docsCli = forAllSystems (
       system: pkgs:
         pkgs.callPackage ./src/docs-cli.nix {
@@ -332,6 +313,17 @@
               ${pkgs.treefmt}/bin/treefmt --clear-cache --fail-on-change
             '');
           };
+          is-cleaned = {
+            enable = true;
+            name = "is-cleaned";
+            entry = l.toString (pkgs.writeScript "is-cleaned" ''
+              #!${pkgs.bash}/bin/bash
+              if find ./examples | grep -q 'flake.lock\|dream2nix-packages'; then
+                echo "./examples should not contain any flake.lock files or dream2nix-packages directories" >&2
+                exit 1
+              fi
+            '');
+          };
         };
       };
     });

src/subsystems/python/builders/simple-builder/default.nix

@@ -34,7 +34,7 @@
       l.flatten
       (l.mapAttrsToList
         (name: versions:
-          if name == defaultPackageName
+          if l.elem name [defaultPackageName "setuptools" "pip"]
           then []
           else l.map (ver: getSource name ver) versions)
         packageVersions);
@@ -51,6 +51,7 @@
       buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
       nativeBuildInputs = [pkgs.autoPatchelfHook];
       doCheck = false;
+      dontStrip = true;
       preBuild = ''
         mkdir dist
         for file in ${builtins.toString allDependencySources}; do
@@ -59,9 +60,6 @@
           fname=$(stripHash $fname)
           cp $file dist/$fname
         done
-      '';
-      installPhase = ''
-        runHook preInstall
         mkdir -p "$out/${python.sitePackages}"
         export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
         ${python}/bin/python -m pip install \
@@ -71,7 +69,6 @@
           --prefix="$out" \
           --no-cache \
           $pipInstallFlags
-        runHook postInstall
       '';
     };
   in {

src/subsystems/python/translators/pip-WIP/default.nix

@@ -10,7 +10,6 @@ in {
   # the input format is specified in /specifications/translator-call-example.json
   # this script receives a json file including the input paths and extraArgs
   translateBin = {
-    # dream2nix
     externalSources,
     utils,
     bash,
@@ -18,84 +17,64 @@ in {
     jq,
     nix,
     python3,
+    toml2json,
     writeScriptBin,
     ...
-  }: let
+  }:
-    machNixExtractor = "${externalSources.mach-nix}/lib/extractor/default.nix";
-
-    setuptools_shim = ''
-      import sys, setuptools, tokenize, os; sys.argv[0] = 'setup.py'; __file__='setup.py';
-      f=getattr(tokenize, 'open', open)(__file__);
-      code=f.read().replace('\r\n', '\n');
-      f.close();
-      exec(compile(code, __file__, 'exec'))
-    '';
-  in
     utils.writePureShellScript
     [
       bash
       coreutils
       jq
       nix
+      toml2json
     ]
     ''
       # accroding to the spec, the translator reads the input from a json file
       jsonInput=$1
 
       # read the json input
-      outputFile=$WORKDIR/$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
+      outputFile=$WORKDIR/$(jq '.outputFile' -c -r $jsonInput)
-      source="$(${jq}/bin/jq '.source' -c -r $jsonInput)/$(${jq}/bin/jq '.project.relPath' -c -r $jsonInput)"
+      source="$(jq '.source' -c -r $jsonInput)/$(jq '.project.relPath' -c -r $jsonInput)"
-      pythonAttr=$(${jq}/bin/jq '.project.subsystemInfo.pythonAttr' -c -r $jsonInput)
+      pythonAttr=$(jq '.project.subsystemInfo.pythonAttr' -c -r $jsonInput)
 
       # build python and pip executables
       tmpBuild=$(mktemp -d)
-      nix build --show-trace --impure --expr \
+      nix build \
-        "
+        --impure \
-          (import ${machNixExtractor} {}).mkPy
+        --expr "(import <nixpkgs> {}).$pythonAttr.withPackages (ps: [ps.pip ps.setuptools])" \
-            (import <nixpkgs> {}).$pythonAttr
+        -o $tmpBuild/pip
-        " \
-        -o $tmpBuild/python
-      nix build --impure --expr "(import <nixpkgs> {}).$pythonAttr.withPackages (ps: [ps.pip ps.setuptools])" -o $tmpBuild/pip
-      python=$tmpBuild/python/bin/python
       pip=$tmpBuild/pip/bin/pip
+      python=$tmpBuild/pip/bin/python
 
       # prepare temporary directory
       tmp=$(mktemp -d)
 
-      # extract python requirements from setup.py
+      # prepare source
-      cp -r $source $tmpBuild/src
+      cp -r $source ./source
-      chmod -R +w $tmpBuild/src
+      chmod +w -R ./source
-      cd $tmpBuild/src
-      chmod +x setup.py || true
-      echo "extracting dependencies"
-      out_file=$tmpBuild/python.json \
-          dump_setup_attrs=y \
-          PYTHONIOENCODING=utf8 \
-          LANG=C.utf8 \
-        $python -c "${setuptools_shim}" install &> $tmpBuild/python.log || true
 
-      # extract requirements from json result
+      # download setup dependencies from pyproject.toml
-      $python -c "
+      toml2json ./source/pyproject.toml | jq '."build-system".requires[]' -r > __setup_reqs.txt \
-      import json
+      && $tmpBuild/pip/bin/pip download \
-      result = json.load(open('$tmpBuild/python.json'))
+        --dest $tmp \
-      for key in ('install_requires', 'setup_requires'):
+        --progress-bar off \
-        if key in result:
+        -r __setup_reqs.txt \
-          print('\n'.join(result[key]))
+      || :
-      " > $tmpBuild/computed_requirements
 
       # download files according to requirements
       $tmpBuild/pip/bin/pip download \
-        --no-cache \
         --dest $tmp \
         --progress-bar off \
-        -r $tmpBuild/computed_requirements
+        -r __setup_reqs.txt \
-        # -r ''${inputFiles/$'\n'/$' -r '}
+        ./source
 
-      cd $WORKDIR
       # generate the dream lock from the downloaded list of files
-      NAME=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
+      cd ./source
-          VERSION=$(${jq}/bin/jq '.version' -c -r $tmpBuild/python.json) \
+      export NAME=$($python ./setup.py --name 2>/dev/null)
-        $tmpBuild/python/bin/python ${./generate-dream-lock.py} $tmp $jsonInput
+      export VERSION=$($python ./setup.py --version 2>/dev/null)
+      cd $WORKDIR
+      $python ${./generate-dream-lock.py} $tmp $jsonInput
 
       rm -rf $tmp $tmpBuild
     '';

src/subsystems/python/translators/pip-WIP/generate-dream-lock.py

@@ -65,6 +65,7 @@ def main():
         os.environ.get('NAME'): os.environ.get('VERSION'),
       },
       "sourcesAggregatedHash": None,
+      "location": "",
     },
     _subsystem={
       "application": jsonInput.get('application', False),