Merge branch 'main' into mkOutput_module_args

README.md

@@ -2,7 +2,7 @@
 
 <p align="center">
   <picture>
-    <source width="600" media="(prefers-color-scheme: dark)" srcset="https://gist.githubusercontent.com/DavHau/755fed3774e89c0b9b8953a0a25309fa/raw/e2a12a60ae49aa5eb11b42775abdd1652dbe63c0/dream2nix-01.png">
+    <source width="600" media="(prefers-color-scheme: dark)" srcset="https://gist.githubusercontent.com/DavHau/755fed3774e89c0b9b8953a0a25309fa/raw/0312cc4f785de36212f4303d23298f07c13549dc/dream2nix-dark.png">
     <source width="600" media="(prefers-color-scheme: light)" srcset="https://gist.githubusercontent.com/DavHau/755fed3774e89c0b9b8953a0a25309fa/raw/e2a12a60ae49aa5eb11b42775abdd1652dbe63c0/dream2nix-01.png">
     <img width="600" alt="dream2nix - A framework for automated nix packaging" src="https://gist.githubusercontent.com/DavHau/755fed3774e89c0b9b8953a0a25309fa/raw/e2a12a60ae49aa5eb11b42775abdd1652dbe63c0/dream2nix-01.png">
   </picture>

docs/src/intro/fetchers.md

@@ -90,13 +90,24 @@ Inputs:
 
 ### pypi-sdist
 
-Fetches from pypi registry.
+Fetches sources distributions ("sdists") from pypi registry.
 
 Inputs:
 - pname
 - version
 - hash
 
+### pypi-wheel
+
+Fetches wheels from pypi registry.
+
+Inputs:
+- filename
+- pname
+- version
+- hash
+
+
 ### crates-io
 
 Fetches from crates.io registry.

examples/_d2n-define-projects-manually/flake.nix

@@ -1,8 +1,6 @@
 {
   inputs = {
     dream2nix.url = "github:nix-community/dream2nix";
-    nixpkgs.follows = "dream2nix/nixpkgs";
-    flake-parts.url = "github:hercules-ci/flake-parts";
     src.url = "github:prettier/prettier/2.4.1";
     src.flake = false;
   };
@@ -10,26 +8,21 @@
   outputs = {
     self,
     dream2nix,
-    flake-parts,
     src,
-    ...
   }:
-    flake-parts.lib.mkFlake {inherit self;} {
+    (dream2nix.lib.makeFlakeOutputs {
       systems = ["x86_64-linux"];
-      imports = [dream2nix.flakeModuleBeta];
+      config.projectRoot = ./.;
-
+      source = src;
-      perSystem = {config, ...}: {
+      projects = {
-        # define an input for dream2nix to generate outputs for
+        prettier = {
-        dream2nix.inputs."prettier" = {
+          name = "prettier";
-          source = src;
+          subsystem = "nodejs";
-          projects = {
+          translator = "yarn-lock";
-            prettier = {
-              name = "prettier";
-              subsystem = "nodejs";
-              translator = "yarn-lock";
-            };
-          };
         };
       };
+    })
+    // {
+      # checks = self.packages;
     };
 }

examples/_d2n-flake-parts/flake.nix

@@ -3,7 +3,7 @@
     dream2nix.url = "github:nix-community/dream2nix";
     nixpkgs.follows = "dream2nix/nixpkgs";
     flake-parts.url = "github:hercules-ci/flake-parts";
-    src.url = "github:yusdacra/linemd/v0.4.0";
+    src.url = "github:prettier/prettier/2.4.1";
     src.flake = false;
   };
 
@@ -20,9 +20,15 @@
 
       perSystem = {config, ...}: {
         # define an input for dream2nix to generate outputs for
-        dream2nix.inputs."linemd" = {
+        dream2nix.inputs."prettier" = {
           source = src;
-          settings = [{builder = "crane";}];
+          projects = {
+            prettier = {
+              name = "prettier";
+              subsystem = "nodejs";
+              translator = "yarn-lock";
+            };
+          };
         };
       };
     };

examples/python_poetry/flake.nix

@@ -0,0 +1,40 @@
+{
+  inputs = {
+    dream2nix.url = "github:nix-community/dream2nix";
+    nixpkgs.follows = "dream2nix/nixpkgs";
+    flake-parts.url = "github:hercules-ci/flake-parts";
+    src.url = "github:python-poetry/poetry";
+    src.flake = false;
+  };
+
+  outputs = {
+    self,
+    dream2nix,
+    flake-parts,
+    src,
+    ...
+  }:
+    flake-parts.lib.mkFlake {inherit self;} {
+      systems = ["x86_64-linux"];
+      imports = [dream2nix.flakeModuleBeta];
+
+      perSystem = {
+        config,
+        system,
+        ...
+      }: {
+        # define an input for dream2nix to generate outputs for
+        dream2nix.inputs."my-project" = {
+          source = src;
+          projects.my-project = {
+            name = "my-project";
+            subsystem = "python";
+            translator = "poetry";
+            pythonVersion = "3.10";
+            subsystemInfo.system = system;
+            subsystemInfo.pythonVersion = "3.10";
+          };
+        };
+      };
+    };
+}

flake.lock

@@ -241,16 +241,16 @@
     "poetry2nix": {
       "flake": false,
       "locked": {
-        "lastModified": 1632969109,
+        "lastModified": 1666918719,
-        "narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=",
+        "narHash": "sha256-BkK42fjAku+2WgCOv2/1NrPa754eQPV7gPBmoKQBWlc=",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77",
+        "rev": "289efb187123656a116b915206e66852f038720e",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "1.21.0",
+        "ref": "1.36.0",
         "repo": "poetry2nix",
         "type": "github"
       }

flake.nix

@@ -46,7 +46,7 @@
 
     # required for utils.satisfiesSemver
     poetry2nix = {
-      url = "github:nix-community/poetry2nix/1.21.0";
+      url = "github:nix-community/poetry2nix/1.36.0";
       flake = false;
     };
 

src/fetchers/pypi-sdist/default.nix

@@ -21,20 +21,11 @@
       "https://files.pythonhosted.org/packages/source/"
       + "${firstChar}/${pname}/${pname}-${version}.${extension}";
   in {
-    calcHash = algo:
-      utils.hashPath algo (
-        b.fetchurl {inherit url;}
-      );
-
     fetched = hash: let
-      source =
+      source = pkgs.fetchurl {
-        (pkgs.fetchurl {
+        inherit url;
-          inherit url;
+        sha256 = hash;
-          sha256 = hash;
+      };
-        })
-        .overrideAttrs (old: {
-          outputHashMode = "recursive";
-        });
     in
       utils.extractSource {
         inherit source;

src/fetchers/pypi-wheel/default.nix

@@ -0,0 +1,37 @@
+{
+  pkgs,
+  utils,
+  lib,
+  ...
+}: {
+  inputs = ["filename"];
+
+  versionField = "version";
+
+  defaultUpdater = "pypiNewestReleaseVersion";
+
+  outputs = {
+    filename,
+    pname,
+    version,
+  }: {
+    fetched = hash:
+      pkgs.runCommand
+      filename
+      {
+        buildInputs = [
+          pkgs.curl
+          pkgs.cacert
+          pkgs.jq
+        ];
+        outputHash = hash;
+        outputHashAlgo = "sha256";
+        outputHashMode = "flat";
+        inherit filename pname version;
+      }
+      ''
+        url=$(curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$filename\") | .url")
+        curl $url --output $out
+      '';
+  };
+}

src/specifications/dream-lock-schema.json

@@ -23,6 +23,7 @@
                     "npm",
                     "path",
                     "pypi-sdist",
+                    "pypi-wheel",
                     "crates-io",
                     "unknown"
                   ]
@@ -132,7 +133,22 @@
                     "properties": {
                       "hash": { "type": "string" },
                       "type": { "type": "string" },
-                      "dir": { "type": "string" }
+                      "pname": { "type": "string" },
+                      "version": { "type": "string" }
+                    },
+                    "required": ["type", "pname", "version"],
+                    "additionalProperties": false
+                  }
+                },
+                {
+                  "if": {
+                    "properties": { "type": { "const": "pypi-wheel" } }
+                  },
+                  "then": {
+                    "properties": {
+                      "hash": { "type": "string" },
+                      "type": { "type": "string" },
+                      "filename": { "type": "string" }
                     },
                     "required": ["type"],
                     "additionalProperties": false

src/subsystems/python/builders/simple-python/default.nix

@@ -40,13 +40,20 @@
     package = produceDerivation defaultPackageName (buildFunc {
       name = defaultPackageName;
       src = getSource defaultPackageName defaultPackageVersion;
-      format = "setuptools";
+      format = "other";
       buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
-      nativeBuildInputs = [pkgs.autoPatchelfHook];
+      nativeBuildInputs =
+        [pkgs.autoPatchelfHook]
+        ++ (with python.pkgs; [
+          pip
+          poetry-core
+          wheel
+        ]);
       propagatedBuildInputs = [python.pkgs.setuptools];
       doCheck = false;
       dontStrip = true;
-      preBuild = ''
+
+      buildPhase = ''
         mkdir dist
         for file in ${builtins.toString allDependencySources}; do
           # pick right most element of path
@@ -54,17 +61,18 @@
           fname=$(stripHash $fname)
           cp $file dist/$fname
         done
-        mkdir -p "$out/${python.sitePackages}"
-        export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
         ${python}/bin/python -m pip install \
-          ./dist/*.{whl,tar.gz,zip} \
+          --find-links ./dist/ \
           --no-build-isolation \
           --no-index \
           --no-warn-script-location \
           --prefix="$out" \
           --no-cache \
+          --ignore-installed \
+          . \
           $pipInstallFlags
       '';
+      installPhase = "true";
     });
 
     devShell = pkgs.mkShell {
@@ -72,8 +80,6 @@
         # a drv with all dependencies without the main package
         (package.overrideAttrs (old: {
           src = ".";
-          dontUnpack = true;
-          buildPhase = old.preBuild;
         }))
       ];
     };

src/subsystems/python/translators/poetry/default.nix

@@ -0,0 +1,149 @@
+/*
+This is a pure translator which translates poetry's poetry.lock
+  to a dream2nix dream-lock(.json).
+
+Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry.loc
+*/
+{
+  dlib,
+  lib,
+  inputs,
+  ...
+}: let
+  l = lib // builtins;
+in {
+  type = "pure";
+
+  discoverProject = tree:
+    l.pathExists "${tree.fullPath}/poetry.lock";
+
+  # translate from a given source and a project specification to a dream-lock.
+  translate = {
+    project,
+    tree,
+    pythonVersion,
+    system,
+    ...
+  }: let
+    # get the root source and project source
+    rootSource = tree.fullPath;
+    projectSource = "${tree.fullPath}/${project.relPath}";
+    projectTree = tree.getNodeFromPath project.relPath;
+
+    # initialize poetry2nix libs
+    fakeStdenv = {
+      isLinux = l.hasInfix "-linux" system;
+      isDarwin = l.hasInfix "-darwin" system;
+      targetPlatform.isAarch64 = l.hasPrefix "aarch64-" system;
+      targetPlatform.parsed.cpu.name = l.elemAt (l.splitString "-" system) 0;
+    };
+    fakePython = {
+      version = pythonVersion;
+      passthru.implementation = "cpython";
+    };
+    pep425 = import "${inputs.poetry2nix}/pep425.nix" {
+      inherit lib;
+      python = fakePython;
+      stdenv = fakeStdenv;
+      poetryLib = import "${inputs.poetry2nix}/lib.nix" {
+        inherit lib;
+        pkgs = null;
+        stdenv = fakeStdenv;
+      };
+    };
+
+    # use dream2nix' source tree abstraction to access json content of files
+    sources =
+      (projectTree.getNodeFromPath "poetry.lock").tomlContent;
+
+    pyproject = (projectTree.getNodeFromPath "pyproject.toml").tomlContent;
+    defaultPackageName = pyproject.tool.poetry.name;
+    defaultPackageVersion = pyproject.tool.poetry.version;
+  in
+    # see example in src/specifications/dream-lock-example.json
+    {
+      decompressed = true;
+      # generic fields
+      _generic = {
+        # TODO: specify the default package name
+        defaultPackage = defaultPackageName;
+        # TODO: specify a list of exported packages and their versions
+        packages.${defaultPackageName} = defaultPackageVersion;
+        # TODO: this must be equivalent to the subsystem name
+        subsystem = "python";
+        location = project.relPath;
+      };
+
+      _subsystem = {
+        application = false;
+        pythonAttr = "python3";
+        sourceFormats = {};
+      };
+
+      cyclicDependencies = {};
+
+      dependencies = {};
+
+      sources =
+        l.mapAttrs
+        (sourceName: files: let
+          wheels = pep425.selectWheel files;
+          sdists = builtins.filter (x: !(lib.hasSuffix ".whl" x.file)) files;
+          candidate =
+            if lib.length wheels > 0
+            then builtins.head wheels
+            else builtins.head sdists;
+          isWheel = l.hasSuffix "whl" candidate.file;
+          suffix =
+            if isWheel
+            then ".whl"
+            else ".tar.gz";
+          parts = lib.splitString "-" (lib.removeSuffix suffix candidate.file);
+          last = (lib.length parts) - 1;
+          pname =
+            if isWheel
+            then lib.elemAt parts (last - 4)
+            else builtins.concatStringsSep "-" (lib.init parts);
+          version =
+            if isWheel
+            then lib.elemAt parts (last - 3)
+            else lib.elemAt parts last;
+        in {
+          ${version} =
+            if isWheel
+            then {
+              type = "pypi-wheel";
+              filename = candidate.file;
+              hash = candidate.hash;
+            }
+            else {
+              type = "pypi-sdist";
+              inherit pname version;
+              hash = candidate.hash;
+            };
+        })
+        sources.metadata.files;
+    };
+
+  extraArgs = {
+    system = {
+      description = "System for produced outputs.";
+      # default = "blabla";
+      examples = [
+        "x86_64-linux"
+        "x86_64-darwin"
+      ];
+      type = "argument";
+    };
+    pythonVersion = {
+      description = "python version to translate for";
+      default = "3.10";
+      examples = [
+        "3.8"
+        "3.9"
+        "3.10"
+      ];
+      type = "argument";
+    };
+  };
+}

templates/default.nix

@@ -11,7 +11,7 @@ in {
         default = self.templates.simple;
         simple = {
           description = "Simple dream2nix flake";
-          path = ./templates/simple;
+          path = ./simple;
           welcomeText = ''
             You just created a simple dream2nix package!