Merge branch 'main' into fetch-pip-metadata

v1/nix/modules/drv-parts/mach-nix-xs/default.nix

@@ -71,7 +71,11 @@
     # Usually references to buildInputs would get lost in the dist output.
     # Patch wheels to ensure build inputs remain dependencies of the `dist` output
     # Those references are needed for the final autoPatchelfHook to find the required deps.
-    patchedWheels = mapAttrs substitutions (name: dist: dist.overridePythonAttrs (old: {postFixup = "ln -s $out $dist/out";}));
+    linkOutToDistOverride = old: {
+      linkOutToDist = "ln -s $out $dist/out";
+      postPhases = ["linkOutToDist"];
+    };
+    patchedWheels = mapAttrs substitutions (name: dist: dist.overridePythonAttrs linkOutToDistOverride);
   in {inherit patchedWheels downloadedWheels builtWheels;};
 
   # The final dists we want to install.
@@ -98,7 +102,7 @@
         ../nixpkgs-overrides
       ];
       config = {
-        nixpkgs-overrides.enable = true;
+        nixpkgs-overrides.enable = l.mkDefault true;
         deps = {nixpkgs, ...}:
           l.mapAttrs (_: l.mkDefault) {
             inherit python;
@@ -106,6 +110,7 @@
               (nixpkgs)
               autoPatchelfHook
               stdenv
+              unzip
               ;
           };
 
@@ -117,16 +122,19 @@
             (map (distDir: "--find-links ${distDir}") manualSetupDeps.${name} or [])
             ++ (
               map (dep: "--find-links ${finalDistsPaths.${dep}}")
-              config.eval-cache.content.mach-nix.dependencyTree.${name}.dependencies or []
+              (getTransitiveDeps name)
             );
         };
         mkDerivation = {
           # distDir will contain a single file which is the src
           preUnpack = ''export src="${distDir}"/*'';
-          nativeBuildInputs = [config.deps.autoPatchelfHook];
+          nativeBuildInputs = [
+            config.deps.unzip
+          ];
           # ensure build inputs are propagated for autopPatchelfHook
-          postFixup = "ln -s $out $dist/out";
+          postPhases = ["linkOutToDist"];
         };
+        env.linkOutToDist = "ln -s $out $dist/out";
         # TODO If setup deps have been specified manually, we need to remove the
         #   propagatedBuildInputs from nixpkgs to prevent collisions.
         #// lib.optionalAttrs (manualSetupDeps ? ${name}) {
@@ -138,6 +146,14 @@
   dependenciesFile = "${cfg.pythonSources}/metadata.json";
   dependencyTree = l.fromJSON (l.readFile dependenciesFile);
 
+  getTransitiveDeps' = name: let
+    directDeps = dependencyTree.${name}.dependencies or [];
+  in
+    directDeps
+    ++ (l.concatMap getTransitiveDeps' directDeps);
+
+  getTransitiveDeps = name: l.unique (getTransitiveDeps' name);
+
   makeModuleFromDerivation = _name: drv:
     drv-parts.lib.makeModule {
       packageFunc = drv;

v1/nix/modules/drv-parts/nixpkgs-overrides/default.nix

@@ -7,8 +7,8 @@
   l = lib // builtins;
   cfg = config.nixpkgs-overrides;
 
-  excludedNixpkgsAttrs =
+  exclude =
-    l.genAttrs cfg.excludedNixpkgsAttrs (name: null);
+    l.genAttrs cfg.exclude (name: null);
 
   extractOverrideAttrs = overrideFunc:
     (overrideFunc (old: {passthru.old = old;}))
@@ -17,7 +17,7 @@
   extractPythonAttrs = pythonPackage: let
     pythonAttrs = extractOverrideAttrs pythonPackage.overridePythonAttrs;
   in
-    l.filterAttrs (name: _: ! excludedNixpkgsAttrs ? ${name}) pythonAttrs;
+    l.filterAttrs (name: _: ! exclude ? ${name}) pythonAttrs;
 
   extracted =
     if config.deps.python.pkgs ? ${config.name}
@@ -58,6 +58,16 @@ in {
     })
     {
       nixpkgs-overrides.lib = {inherit extractOverrideAttrs extractPythonAttrs;};
+      nixpkgs-overrides.exclude = [
+        "all"
+        "args"
+        "builder"
+        "name"
+        "pname"
+        "version"
+        "src"
+        "outputs"
+      ];
     }
   ];
 }

v1/nix/modules/drv-parts/nixpkgs-overrides/interface.nix

@@ -7,21 +7,11 @@
   t = l.types;
 in {
   options.nixpkgs-overrides = {
-    enable = l.mkEnableOption "Whether to copy attributes, except those in `excludedNixpkgsAttrs` from nixpkgs";
+    enable = l.mkEnableOption "Whether to copy attributes, except those in `excluded` from nixpkgs";
 
-    excludedNixpkgsAttrs = l.mkOption {
+    exclude = l.mkOption {
       type = t.listOf t.str;
       description = "Attributes we do not want to copy from nixpkgs";
-      default = [
-        "all"
-        "args"
-        "builder"
-        "name"
-        "pname"
-        "version"
-        "src"
-        "outputs"
-      ];
     };
 
     lib.extractOverrideAttrs = l.mkOption {

v1/nix/modules/drvs/apache-airflow/lock-x86_64-linux.json

@@ -1,3 +1,3 @@
 {
-  "fetchPipHash": "sha256-i+GRzWBaX7DnTKVsUnVW2zU+wANpJwMKVFCS7jtHZrM="
+  "fetchPipHash": "sha256-pAI3j43S4HedxsXIbe2+LQtpzcp1gjwlneohlD3V3LA="
 }

v1/nix/modules/drvs/odoo/lock-x86_64-linux.json

@@ -444,4 +444,5 @@
       "version": "6.0"
     }
   }
+
 }

v1/nix/modules/drvs/pillow/lock-x86_64-linux.json

@@ -1,3 +1,3 @@
 {
-  "fetchPipHash": "sha256-eS81pqSjU6mgBL6tXadSxkGdafsVFThByOQcOf8FkF0="
+  "fetchPipHash": "sha256-9kedXL+AOrMo3oue5IFRrfMlNZQ3UGp7AmhzlPPENsI="
 }

v1/nix/modules/drvs/tensorflow/lock-x86_64-linux.json

@@ -1,3 +1,3 @@
 {
-  "fetchPipHash": "sha256-PDUrECFjoPznqXwqi2e1djx63t+kn/kAyM9JqQrTmd0="
+  "fetchPipHash": "sha256-9TDHKU4cb7UorC3nT5stvraf1HbDvHXmD39P39aPCN0="
 }

v1/nix/modules/flake-parts/lib.evalModules.nix

@@ -32,6 +32,7 @@
             ];
           specialArgs =
             args.specialArgs
+            or {}
             // {
               inherit packageSets;
               dream2nix.modules.drv-parts = self.modules.drv-parts;

v1/nix/pkgs/fetchPip/fetchPip.py

@@ -29,6 +29,7 @@ ONLY_BINARY_FLAGS = os.getenv("onlyBinaryFlags")
 REQUIREMENTS_LIST = os.getenv("requirementsList")
 REQUIREMENTS_FILES = os.getenv("requirementsFiles")
 WRITE_METADATA = os.getenv("writeMetaData")
+TMPDIR = os.getenv("TMPDIR")
 
 
 def get_max_date():
@@ -84,7 +85,10 @@ def wait_for_proxy(proxy_port, cafile):
 # as we only proxy *some* calls, we need to combine upstream
 # ca certificates and the one from mitm proxy
 def generate_ca_bundle(path):
-    with open(HOME / ".mitmproxy/mitmproxy-ca-cert.pem", "r") as f:
+    proxy_cert = HOME / ".mitmproxy/mitmproxy-ca-cert.pem"
+    while not os.path.exists(proxy_cert):
+        time.sleep(0.1)
+    with open(proxy_cert, "r") as f:
         mitmproxy_cacert = f.read()
     with open(certifi.where(), "r") as f:
         certifi_cacert = f.read()
@@ -111,7 +115,7 @@ if __name__ == "__main__":
     names_path = OUT / "names"
     dist_path.mkdir()
     names_path.mkdir()
-    cache_path = Path("/build/pip_cache")
+    cache_path = Path(f"{TMPDIR}/pip_cache")
     cache_path.mkdir()
 
     print(f"selected maximum release date for python packages: {get_max_date()}")
@@ -121,6 +125,10 @@ if __name__ == "__main__":
 
     venv_path = Path(".venv").absolute()
     create_venv(venv_path)
+
+    cafile = generate_ca_bundle(HOME / ".ca-cert.pem")
+    wait_for_proxy(proxy_port, cafile)
+
     pip(
         venv_path,
         "install",
@@ -128,8 +136,17 @@ if __name__ == "__main__":
         f"pip=={PIP_VERSION}",
     )
 
-    cafile = generate_ca_bundle(HOME / ".ca-cert.pem")
+    # some legacy setup.py based packages require wheel in order to be inspected
-    wait_for_proxy(proxy_port, cafile)
+    pip(
+        venv_path,
+        "install",
+        "--proxy",
+        f"https://localhost:{proxy_port}",
+        "--cert",
+        cafile,
+        "--upgrade",
+        f"wheel",
+    )
 
     flags = [
         PIP_FLAGS,
@@ -146,7 +163,7 @@ if __name__ == "__main__":
     if NO_BINARY:
         optional_flags += ["--no-binary " + " --no-binary ".join(NO_BINARY.split())]
     if WRITE_METADATA:
-        metadata_flags = ["--report", "/build/report.json"]
+        metadata_flags = ["--report", f"{TMPDIR}/report.json"]
 
     for req in REQUIREMENTS_LIST.split(" "):
         if req:
@@ -188,7 +205,7 @@ if __name__ == "__main__":
     if WRITE_METADATA:
         packages = dict()
 
-        with open("/build/report.json", "r") as f:
+        with open(f"{TMPDIR}/report.json", "r") as f:
             report = json.load(f)
 
         for install in report["install"]: