diff --git a/README.md b/README.md index 5ac2f742..baf3784d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- + dream2nix - A framework for automated nix packaging diff --git a/docs/src/intro/fetchers.md b/docs/src/intro/fetchers.md index 7f0b1b5a..3c601cf0 100644 --- a/docs/src/intro/fetchers.md +++ b/docs/src/intro/fetchers.md @@ -90,13 +90,24 @@ Inputs: ### pypi-sdist -Fetches from pypi registry. +Fetches sources distributions ("sdists") from pypi registry. Inputs: - pname - version - hash +### pypi-wheel + +Fetches wheels from pypi registry. + +Inputs: +- filename +- pname +- version +- hash + + ### crates-io Fetches from crates.io registry. diff --git a/examples/_d2n-define-projects-manually/flake.nix b/examples/_d2n-define-projects-manually/flake.nix index c612fb1e..05c12e4f 100644 --- a/examples/_d2n-define-projects-manually/flake.nix +++ b/examples/_d2n-define-projects-manually/flake.nix @@ -1,8 +1,6 @@ { inputs = { dream2nix.url = "github:nix-community/dream2nix"; - nixpkgs.follows = "dream2nix/nixpkgs"; - flake-parts.url = "github:hercules-ci/flake-parts"; src.url = "github:prettier/prettier/2.4.1"; src.flake = false; }; @@ -10,26 +8,21 @@ outputs = { self, dream2nix, - flake-parts, src, - ... }: - flake-parts.lib.mkFlake {inherit self;} { + (dream2nix.lib.makeFlakeOutputs { systems = ["x86_64-linux"]; - imports = [dream2nix.flakeModuleBeta]; - - perSystem = {config, ...}: { - # define an input for dream2nix to generate outputs for - dream2nix.inputs."prettier" = { - source = src; - projects = { - prettier = { - name = "prettier"; - subsystem = "nodejs"; - translator = "yarn-lock"; - }; - }; + config.projectRoot = ./.; + source = src; + projects = { + prettier = { + name = "prettier"; + subsystem = "nodejs"; + translator = "yarn-lock"; }; }; + }) + // { + # checks = self.packages; }; } diff --git a/examples/_d2n-flake-parts/flake.nix b/examples/_d2n-flake-parts/flake.nix index 839bc281..c612fb1e 100644 --- a/examples/_d2n-flake-parts/flake.nix +++ b/examples/_d2n-flake-parts/flake.nix @@ -3,7 +3,7 @@ dream2nix.url = "github:nix-community/dream2nix"; nixpkgs.follows = "dream2nix/nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; - src.url = "github:yusdacra/linemd/v0.4.0"; + src.url = "github:prettier/prettier/2.4.1"; src.flake = false; }; @@ -20,9 +20,15 @@ perSystem = {config, ...}: { # define an input for dream2nix to generate outputs for - dream2nix.inputs."linemd" = { + dream2nix.inputs."prettier" = { source = src; - settings = [{builder = "crane";}]; + projects = { + prettier = { + name = "prettier"; + subsystem = "nodejs"; + translator = "yarn-lock"; + }; + }; }; }; }; diff --git a/examples/python_poetry/flake.nix b/examples/python_poetry/flake.nix new file mode 100644 index 00000000..ca06a777 --- /dev/null +++ b/examples/python_poetry/flake.nix @@ -0,0 +1,40 @@ +{ + inputs = { + dream2nix.url = "github:nix-community/dream2nix"; + nixpkgs.follows = "dream2nix/nixpkgs"; + flake-parts.url = "github:hercules-ci/flake-parts"; + src.url = "github:python-poetry/poetry"; + src.flake = false; + }; + + outputs = { + self, + dream2nix, + flake-parts, + src, + ... + }: + flake-parts.lib.mkFlake {inherit self;} { + systems = ["x86_64-linux"]; + imports = [dream2nix.flakeModuleBeta]; + + perSystem = { + config, + system, + ... + }: { + # define an input for dream2nix to generate outputs for + dream2nix.inputs."my-project" = { + source = src; + projects.my-project = { + name = "my-project"; + subsystem = "python"; + translator = "poetry"; + pythonVersion = "3.10"; + subsystemInfo.system = system; + subsystemInfo.pythonVersion = "3.10"; + }; + }; + }; + }; +} diff --git a/flake.lock b/flake.lock index a2321927..56dd1d86 100644 --- a/flake.lock +++ b/flake.lock @@ -241,16 +241,16 @@ "poetry2nix": { "flake": false, "locked": { - "lastModified": 1632969109, - "narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=", + "lastModified": 1666918719, + "narHash": "sha256-BkK42fjAku+2WgCOv2/1NrPa754eQPV7gPBmoKQBWlc=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77", + "rev": "289efb187123656a116b915206e66852f038720e", "type": "github" }, "original": { "owner": "nix-community", - "ref": "1.21.0", + "ref": "1.36.0", "repo": "poetry2nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 8cc764e3..0edbf802 100644 --- a/flake.nix +++ b/flake.nix @@ -46,7 +46,7 @@ # required for utils.satisfiesSemver poetry2nix = { - url = "github:nix-community/poetry2nix/1.21.0"; + url = "github:nix-community/poetry2nix/1.36.0"; flake = false; }; diff --git a/src/fetchers/pypi-sdist/default.nix b/src/fetchers/pypi-sdist/default.nix index a664a7ce..dcd98cc2 100644 --- a/src/fetchers/pypi-sdist/default.nix +++ b/src/fetchers/pypi-sdist/default.nix @@ -21,20 +21,11 @@ "https://files.pythonhosted.org/packages/source/" + "${firstChar}/${pname}/${pname}-${version}.${extension}"; in { - calcHash = algo: - utils.hashPath algo ( - b.fetchurl {inherit url;} - ); - fetched = hash: let - source = - (pkgs.fetchurl { - inherit url; - sha256 = hash; - }) - .overrideAttrs (old: { - outputHashMode = "recursive"; - }); + source = pkgs.fetchurl { + inherit url; + sha256 = hash; + }; in utils.extractSource { inherit source; diff --git a/src/fetchers/pypi-wheel/default.nix b/src/fetchers/pypi-wheel/default.nix new file mode 100644 index 00000000..81ecda49 --- /dev/null +++ b/src/fetchers/pypi-wheel/default.nix @@ -0,0 +1,37 @@ +{ + pkgs, + utils, + lib, + ... +}: { + inputs = ["filename"]; + + versionField = "version"; + + defaultUpdater = "pypiNewestReleaseVersion"; + + outputs = { + filename, + pname, + version, + }: { + fetched = hash: + pkgs.runCommand + filename + { + buildInputs = [ + pkgs.curl + pkgs.cacert + pkgs.jq + ]; + outputHash = hash; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + inherit filename pname version; + } + '' + url=$(curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$filename\") | .url") + curl $url --output $out + ''; + }; +} diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json index 8acb5483..6b7f7240 100644 --- a/src/specifications/dream-lock-schema.json +++ b/src/specifications/dream-lock-schema.json @@ -23,6 +23,7 @@ "npm", "path", "pypi-sdist", + "pypi-wheel", "crates-io", "unknown" ] @@ -132,7 +133,22 @@ "properties": { "hash": { "type": "string" }, "type": { "type": "string" }, - "dir": { "type": "string" } + "pname": { "type": "string" }, + "version": { "type": "string" } + }, + "required": ["type", "pname", "version"], + "additionalProperties": false + } + }, + { + "if": { + "properties": { "type": { "const": "pypi-wheel" } } + }, + "then": { + "properties": { + "hash": { "type": "string" }, + "type": { "type": "string" }, + "filename": { "type": "string" } }, "required": ["type"], "additionalProperties": false diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix index 6312b6ca..d67b809b 100644 --- a/src/subsystems/python/builders/simple-python/default.nix +++ b/src/subsystems/python/builders/simple-python/default.nix @@ -40,13 +40,20 @@ package = produceDerivation defaultPackageName (buildFunc { name = defaultPackageName; src = getSource defaultPackageName defaultPackageVersion; - format = "setuptools"; + format = "other"; buildInputs = pkgs.pythonManylinuxPackages.manylinux1; - nativeBuildInputs = [pkgs.autoPatchelfHook]; + nativeBuildInputs = + [pkgs.autoPatchelfHook] + ++ (with python.pkgs; [ + pip + poetry-core + wheel + ]); propagatedBuildInputs = [python.pkgs.setuptools]; doCheck = false; dontStrip = true; - preBuild = '' + + buildPhase = '' mkdir dist for file in ${builtins.toString allDependencySources}; do # pick right most element of path @@ -54,17 +61,18 @@ fname=$(stripHash $fname) cp $file dist/$fname done - mkdir -p "$out/${python.sitePackages}" - export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" ${python}/bin/python -m pip install \ - ./dist/*.{whl,tar.gz,zip} \ + --find-links ./dist/ \ --no-build-isolation \ --no-index \ --no-warn-script-location \ --prefix="$out" \ --no-cache \ + --ignore-installed \ + . \ $pipInstallFlags ''; + installPhase = "true"; }); devShell = pkgs.mkShell { @@ -72,8 +80,6 @@ # a drv with all dependencies without the main package (package.overrideAttrs (old: { src = "."; - dontUnpack = true; - buildPhase = old.preBuild; })) ]; }; diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix new file mode 100644 index 00000000..8265c12c --- /dev/null +++ b/src/subsystems/python/translators/poetry/default.nix @@ -0,0 +1,149 @@ +/* +This is a pure translator which translates poetry's poetry.lock + to a dream2nix dream-lock(.json). + +Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry.loc +*/ +{ + dlib, + lib, + inputs, + ... +}: let + l = lib // builtins; +in { + type = "pure"; + + discoverProject = tree: + l.pathExists "${tree.fullPath}/poetry.lock"; + + # translate from a given source and a project specification to a dream-lock. + translate = { + project, + tree, + pythonVersion, + system, + ... + }: let + # get the root source and project source + rootSource = tree.fullPath; + projectSource = "${tree.fullPath}/${project.relPath}"; + projectTree = tree.getNodeFromPath project.relPath; + + # initialize poetry2nix libs + fakeStdenv = { + isLinux = l.hasInfix "-linux" system; + isDarwin = l.hasInfix "-darwin" system; + targetPlatform.isAarch64 = l.hasPrefix "aarch64-" system; + targetPlatform.parsed.cpu.name = l.elemAt (l.splitString "-" system) 0; + }; + fakePython = { + version = pythonVersion; + passthru.implementation = "cpython"; + }; + pep425 = import "${inputs.poetry2nix}/pep425.nix" { + inherit lib; + python = fakePython; + stdenv = fakeStdenv; + poetryLib = import "${inputs.poetry2nix}/lib.nix" { + inherit lib; + pkgs = null; + stdenv = fakeStdenv; + }; + }; + + # use dream2nix' source tree abstraction to access json content of files + sources = + (projectTree.getNodeFromPath "poetry.lock").tomlContent; + + pyproject = (projectTree.getNodeFromPath "pyproject.toml").tomlContent; + defaultPackageName = pyproject.tool.poetry.name; + defaultPackageVersion = pyproject.tool.poetry.version; + in + # see example in src/specifications/dream-lock-example.json + { + decompressed = true; + # generic fields + _generic = { + # TODO: specify the default package name + defaultPackage = defaultPackageName; + # TODO: specify a list of exported packages and their versions + packages.${defaultPackageName} = defaultPackageVersion; + # TODO: this must be equivalent to the subsystem name + subsystem = "python"; + location = project.relPath; + }; + + _subsystem = { + application = false; + pythonAttr = "python3"; + sourceFormats = {}; + }; + + cyclicDependencies = {}; + + dependencies = {}; + + sources = + l.mapAttrs + (sourceName: files: let + wheels = pep425.selectWheel files; + sdists = builtins.filter (x: !(lib.hasSuffix ".whl" x.file)) files; + candidate = + if lib.length wheels > 0 + then builtins.head wheels + else builtins.head sdists; + isWheel = l.hasSuffix "whl" candidate.file; + suffix = + if isWheel + then ".whl" + else ".tar.gz"; + parts = lib.splitString "-" (lib.removeSuffix suffix candidate.file); + last = (lib.length parts) - 1; + pname = + if isWheel + then lib.elemAt parts (last - 4) + else builtins.concatStringsSep "-" (lib.init parts); + version = + if isWheel + then lib.elemAt parts (last - 3) + else lib.elemAt parts last; + in { + ${version} = + if isWheel + then { + type = "pypi-wheel"; + filename = candidate.file; + hash = candidate.hash; + } + else { + type = "pypi-sdist"; + inherit pname version; + hash = candidate.hash; + }; + }) + sources.metadata.files; + }; + + extraArgs = { + system = { + description = "System for produced outputs."; + # default = "blabla"; + examples = [ + "x86_64-linux" + "x86_64-darwin" + ]; + type = "argument"; + }; + pythonVersion = { + description = "python version to translate for"; + default = "3.10"; + examples = [ + "3.8" + "3.9" + "3.10" + ]; + type = "argument"; + }; + }; +} diff --git a/templates/default.nix b/templates/default.nix index 37be2d3b..b6c22f1b 100644 --- a/templates/default.nix +++ b/templates/default.nix @@ -11,7 +11,7 @@ in { default = self.templates.simple; simple = { description = "Simple dream2nix flake"; - path = ./templates/simple; + path = ./simple; welcomeText = '' You just created a simple dream2nix package!