From 54aa9bce0996826bdaf3b649200dce2b57c4b097 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 22 Nov 2022 11:28:12 +0100 Subject: [PATCH 01/17] fix schema for pypi-sdist fetcher... pypi-sdist expects pname and version parameters, lets reflect that in the schema. --- src/specifications/dream-lock-schema.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json index 8acb5483..919d3af8 100644 --- a/src/specifications/dream-lock-schema.json +++ b/src/specifications/dream-lock-schema.json @@ -132,8 +132,11 @@ "properties": { "hash": { "type": "string" }, "type": { "type": "string" }, - "dir": { "type": "string" } + "pname": { "type": "string" }, + "version": { "type": "string" } }, + "required": ["type", "pname", "version"], + "additionalProperties": false "required": ["type"], "additionalProperties": false } From 5bed752fbe6ff14e32de8097e97a005b753013eb Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 22 Nov 2022 11:29:57 +0100 Subject: [PATCH 02/17] add pypi-wheel fetcher --- src/fetchers/pypi-wheel/default.nix | 37 +++++++++++++++++++++++ src/specifications/dream-lock-schema.json | 13 ++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/fetchers/pypi-wheel/default.nix diff --git a/src/fetchers/pypi-wheel/default.nix b/src/fetchers/pypi-wheel/default.nix new file mode 100644 index 00000000..902de3bb --- /dev/null +++ b/src/fetchers/pypi-wheel/default.nix @@ -0,0 +1,37 @@ +{ + pkgs, + utils, + lib, + ... +}: { + inputs = ["filename"]; + + versionField = "version"; + + defaultUpdater = "pypiNewestReleaseVersion"; + + outputs = {filename}: let + parts = lib.splitString "-" filename; + pname = builtins.elemAt parts 0; + version = builtins.elemAt parts 1; + in { + fetched = hash: + pkgs.runCommand + "${pname}-pypi-url" + { + buildInputs = [ + pkgs.curl + pkgs.cacert + pkgs.jq + ]; + outputHash = hash; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + inherit filename pname version; + } + '' + url=$(curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$filename\") | .url") + curl $url --output $out + ''; + }; +} diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json index 919d3af8..6b7f7240 100644 --- a/src/specifications/dream-lock-schema.json +++ b/src/specifications/dream-lock-schema.json @@ -23,6 +23,7 @@ "npm", "path", "pypi-sdist", + "pypi-wheel", "crates-io", "unknown" ] @@ -137,6 +138,18 @@ }, "required": ["type", "pname", "version"], "additionalProperties": false + } + }, + { + "if": { + "properties": { "type": { "const": "pypi-wheel" } } + }, + "then": { + "properties": { + "hash": { "type": "string" }, + "type": { "type": "string" }, + "filename": { "type": "string" } + }, "required": ["type"], "additionalProperties": false } From 0a60cec98c78563a4d8a65d21baded8a4646dbf1 Mon Sep 17 00:00:00 2001 From: phaer Date: Mon, 21 Nov 2022 15:55:53 +0100 Subject: [PATCH 03/17] initialize poetry translator from templates --- examples/python_poetry/flake.nix | 33 +++ .../python/translators/poetry/default.nix | 250 ++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 examples/python_poetry/flake.nix create mode 100644 src/subsystems/python/translators/poetry/default.nix diff --git a/examples/python_poetry/flake.nix b/examples/python_poetry/flake.nix new file mode 100644 index 00000000..a649a7d8 --- /dev/null +++ b/examples/python_poetry/flake.nix @@ -0,0 +1,33 @@ +{ + inputs = { + dream2nix.url = "github:nix-community/dream2nix"; + nixpkgs.follows = "dream2nix/nixpkgs"; + flake-parts.url = "github:hercules-ci/flake-parts"; + src.url = "github:nmattia/niv"; + src.flake = false; + }; + + outputs = { + self, + dream2nix, + flake-parts, + src, + ... + }: + flake-parts.lib.mkFlake {inherit self;} { + systems = ["x86_64-linux"]; + imports = [dream2nix.flakeModuleBeta]; + + perSystem = {config, ...}: { + # define an input for dream2nix to generate outputs for + dream2nix.inputs."my-project" = { + source = src; + projects.my-project = { + name = "my-project"; + subsystem = "python"; + translator = "poetry"; + }; + }; + }; + }; +} diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix new file mode 100644 index 00000000..ef1ebdbc --- /dev/null +++ b/src/subsystems/python/translators/poetry/default.nix @@ -0,0 +1,250 @@ +/* +This is an example for a pure translator which translates niv's sources.json + to a dream2nix dream-lock(.json). + +Example sources.json: https://github.com/nmattia/niv/blob/351d8bc316bf901a81885bab5f52687ec8ccab6e/nix/sources.json +*/ +{ + dlib, + lib, + ... +}: let + l = lib // builtins; +in { + type = "pure"; + + /* + Allow dream2nix to detect if a given directory contains a project + which can be translated with this translator. + Usually this can be done by checking for the existence of specific + file names or file endings. + + Alternatively a fully featured discoverer can be implemented under + `src/subsystems/{subsystem}/discoverers`. + This is recommended if more complex project structures need to be + discovered like, for example, workspace projects spanning over multiple + sub-directories + + If a fully featured discoverer exists, do not define `discoverProject`. + */ + discoverProject = tree: + /* + TODO: change this function to identify projects that can be translated + by the current translator. + */ + # checks if the path + l.pathExists "${tree.fullPath}/nix/sources.json"; + + # translate from a given source and a project specification to a dream-lock. + translate = { + /* + A list of projects returned by `discoverProjects` + Example: + { + "dreamLockPath": "packages/optimism/dream-lock.json", + "name": "optimism", + "relPath": "", + "subsystem": "nodejs", + "subsystemInfo": { + "workspaces": [ + "packages/common-ts", + "packages/contracts", + "packages/core-utils", + ] + } + } + */ + project, + /* + Entire source tree represented as nested attribute set. + (produced by `dlib.prepareSourceTree`) + + This has the advantage that files will only be read/parsed once, even + when accessed multiple times or by multiple translators. + + Example: + { + files = { + "package.json" = { + relPath = "package.json" + fullPath = "${source}/package.json" + content = ; + jsonContent = ; + tomlContent = ; + } + }; + + directories = { + "packages" = { + relPath = "packages"; + fullPath = "${source}/packages"; + files = { + ... + }; + directories = { + ... + }; + }; + }; + + # returns the tree object of the given sub-path + getNodeFromPath = path: ... + } + */ + tree, + /* + arguments defined in `extraArgs` specified by user + (see definition for `extraArgs` near the bottom of this file) + */ + noDev, + theAnswer, + ... + }: let + # get the root source and project source + rootSource = tree.fullPath; + projectSource = "${tree.fullPath}/${project.relPath}"; + projectTree = tree.getNodeFromPath project.relPath; + + # use dream2nix' source tree abstraction to access json content of files + sourcesJson = + (projectTree.getNodeFromPath "nix/sources.json").jsonContent; + + /* + + Define the name and version of the top-level package. + If there are multiple top-level packages, just pick any of them. + Dream2nix requires one package to be the default package. + TODO: change this. In the case of niv, we just pick some static values + here because there isn't really a concept of a package in niv. + (Later we just fetch the dependencies and merge them into one output) + */ + defaultPackageName = "my-niv-dependencies"; + defaultPackageVersion = "unknown-version"; + in + # see example in src/specifications/dream-lock-example.json + { + /* + Tell dream2nix that this is the human-readable (decompressed) + representation of the dream-lock. + */ + decompressed = true; + # generic fields + _generic = { + # TODO: specify the default package name + defaultPackage = defaultPackageName; + # TODO: specify a list of exported packages and their versions + packages = { + my-niv-dependencies = "unknown-version"; + }; + # TODO: this must be equivalent to the subsystem name + subsystem = "python"; + }; + + /* + Store subsystem specific data. + This is needed if the subsystem requires extra metadata to be stored. + This will be a free-form field as long as no jsonschema exists for + this subsystem. + TODO: create a jsonschema for the current subsystem under + /src/specifications/{subsystem}/dream-lock-schema.json + */ + _subsystem = { + application = false; + pythonAttr = "python3"; + sourceFormats = {}; + }; + + /* + List dependency edges that need to be removed in order to prevent + infinite recursions in the nix evaluator. + Usually this can be ommitted. + */ + cyclicDependencies = { + }; + + /* + Define the dependency graph. + This can be ommitted, in which case dream2nix assumes that: + - all sources listed in `sources` represent one dependency + - all dependencies are direct dependenceis of the `defaultPackage` + Example: + # foo-1.2.3 depends on bar-2.3.4 and baz-3.4.5 + { + foo."1.2.3" = [ + {name = "bar"; version = "2.3.4"} + {name = "baz"; version = "3.4.5"} + ] + ... + } + */ + dependencies = { + ${defaultPackageName}.${defaultPackageVersion} = + l.mapAttrsToList + ( + sourceName: source: { + name = sourceName; + version = source.rev or "unknown-version"; + } + ) + sourcesJson; + }; + + /* + Define the sources for all dependencies including their checksums. + This allows dream2nix to fetch all sources reproducibly. + Each dependency specified in `dependencies` must have a corresponding + entry in `sources` which describes how the source can be fetched. + Check the `fetchers` section in the docs to see what fetchers are + supported and which arguments they require. + Example: + { + foo."1.2.3" = { + type = "http"; + url = "https://foo.com/tarball.tar.gz"; + hash = "sha256:000000000000000000000000000000000000000"; + }; + bar."2.3.4" = { + ... + } + ... + } + */ + sources = + l.mapAttrs + (sourceName: source: { + ${source.rev or "unknown-version"} = { + type = "archive"; + url = source.url; + hash = "sha256:${source.sha256}"; + }; + }) + sourcesJson; + }; + + # If the translator requires additional arguments, specify them here. + # Users will be able to set these arguments via `settings`. + # There are only two types of arguments: + # - string argument (type = "argument") + # - boolean flag (type = "flag") + # String arguments contain a default value and examples. Flags do not. + # Flags are false by default. + extraArgs = { + # Example: boolean option + # Flags always default to 'false' if not specified by the user + noDev = { + description = "Exclude dev dependencies"; + type = "flag"; + }; + + # Example: string option + theAnswer = { + default = "42"; + description = "The Answer to the Ultimate Question of Life"; + examples = [ + "0" + "1234" + ]; + type = "argument"; + }; + }; +} From 1885ab718d02a7944c478be1677ef74c244e7742 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 22 Nov 2022 19:13:52 +0100 Subject: [PATCH 04/17] initialize poetry translator --- examples/python_poetry/flake.nix | 2 +- src/fetchers/pypi-wheel/default.nix | 10 +- .../python/translators/poetry/default.nix | 91 +++++++++---------- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/examples/python_poetry/flake.nix b/examples/python_poetry/flake.nix index a649a7d8..790c2dd4 100644 --- a/examples/python_poetry/flake.nix +++ b/examples/python_poetry/flake.nix @@ -3,7 +3,7 @@ dream2nix.url = "github:nix-community/dream2nix"; nixpkgs.follows = "dream2nix/nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; - src.url = "github:nmattia/niv"; + src.url = "github:python-poetry/poetry"; src.flake = false; }; diff --git a/src/fetchers/pypi-wheel/default.nix b/src/fetchers/pypi-wheel/default.nix index 902de3bb..68bd1fde 100644 --- a/src/fetchers/pypi-wheel/default.nix +++ b/src/fetchers/pypi-wheel/default.nix @@ -10,11 +10,11 @@ defaultUpdater = "pypiNewestReleaseVersion"; - outputs = {filename}: let - parts = lib.splitString "-" filename; - pname = builtins.elemAt parts 0; - version = builtins.elemAt parts 1; - in { + outputs = { + filename, + pname, + version, + }: { fetched = hash: pkgs.runCommand "${pname}-pypi-url" diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix index ef1ebdbc..e20016f1 100644 --- a/src/subsystems/python/translators/poetry/default.nix +++ b/src/subsystems/python/translators/poetry/default.nix @@ -1,8 +1,8 @@ /* -This is an example for a pure translator which translates niv's sources.json +This is an example for a pure translator which translates poetry's poetry.lock to a dream2nix dream-lock(.json). -Example sources.json: https://github.com/nmattia/niv/blob/351d8bc316bf901a81885bab5f52687ec8ccab6e/nix/sources.json +Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry.loc */ { dlib, @@ -33,7 +33,7 @@ in { by the current translator. */ # checks if the path - l.pathExists "${tree.fullPath}/nix/sources.json"; + l.pathExists "${tree.fullPath}/poetry.lock"; # translate from a given source and a project specification to a dream-lock. translate = { @@ -96,8 +96,6 @@ in { arguments defined in `extraArgs` specified by user (see definition for `extraArgs` near the bottom of this file) */ - noDev, - theAnswer, ... }: let # get the root source and project source @@ -106,11 +104,10 @@ in { projectTree = tree.getNodeFromPath project.relPath; # use dream2nix' source tree abstraction to access json content of files - sourcesJson = - (projectTree.getNodeFromPath "nix/sources.json").jsonContent; + sources = + (projectTree.getNodeFromPath "poetry.lock").tomlContent; /* - Define the name and version of the top-level package. If there are multiple top-level packages, just pick any of them. Dream2nix requires one package to be the default package. @@ -118,7 +115,7 @@ in { here because there isn't really a concept of a package in niv. (Later we just fetch the dependencies and merge them into one output) */ - defaultPackageName = "my-niv-dependencies"; + defaultPackageName = "poetry"; defaultPackageVersion = "unknown-version"; in # see example in src/specifications/dream-lock-example.json @@ -134,10 +131,11 @@ in { defaultPackage = defaultPackageName; # TODO: specify a list of exported packages and their versions packages = { - my-niv-dependencies = "unknown-version"; + poetry = "unknown-version"; }; # TODO: this must be equivalent to the subsystem name subsystem = "python"; + location = project.relPath; }; /* @@ -159,8 +157,7 @@ in { infinite recursions in the nix evaluator. Usually this can be ommitted. */ - cyclicDependencies = { - }; + cyclicDependencies = {}; /* Define the dependency graph. @@ -177,17 +174,7 @@ in { ... } */ - dependencies = { - ${defaultPackageName}.${defaultPackageVersion} = - l.mapAttrsToList - ( - sourceName: source: { - name = sourceName; - version = source.rev or "unknown-version"; - } - ) - sourcesJson; - }; + dependencies = {}; /* Define the sources for all dependencies including their checksums. @@ -211,14 +198,38 @@ in { */ sources = l.mapAttrs - (sourceName: source: { - ${source.rev or "unknown-version"} = { - type = "archive"; - url = source.url; - hash = "sha256:${source.sha256}"; - }; + (sourceName: files: let + candidate = lib.head files; + isWheel = l.hasSuffix "whl" candidate.file; + suffix = + if isWheel + then ".whl" + else ".tar.gz"; + parts = lib.splitString "-" (lib.removeSuffix suffix candidate.file); + last = (lib.length parts) - 1; + pname = + if isWheel + then lib.elemAt parts (last - 4) + else builtins.concatStringsSep "-" (lib.init parts); + version = + if isWheel + then lib.elemAt parts (last - 3) + else lib.elemAt parts last; + in { + ${version} = + if isWheel + then { + type = "pypi-wheel"; + filename = candidate.file; + hash = candidate.hash; + } + else { + type = "pypi-sdist"; + inherit pname version; + hash = candidate.hash; + }; }) - sourcesJson; + sources.metadata.files; }; # If the translator requires additional arguments, specify them here. @@ -228,23 +239,5 @@ in { # - boolean flag (type = "flag") # String arguments contain a default value and examples. Flags do not. # Flags are false by default. - extraArgs = { - # Example: boolean option - # Flags always default to 'false' if not specified by the user - noDev = { - description = "Exclude dev dependencies"; - type = "flag"; - }; - - # Example: string option - theAnswer = { - default = "42"; - description = "The Answer to the Ultimate Question of Life"; - examples = [ - "0" - "1234" - ]; - type = "argument"; - }; - }; + extraArgs = {}; } From 53cfc7610a920f153be5437edc5117afbc723948 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 22 Nov 2022 18:00:29 +0100 Subject: [PATCH 05/17] appease python tools by naming files appropriately --- src/fetchers/pypi-wheel/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fetchers/pypi-wheel/default.nix b/src/fetchers/pypi-wheel/default.nix index 68bd1fde..81ecda49 100644 --- a/src/fetchers/pypi-wheel/default.nix +++ b/src/fetchers/pypi-wheel/default.nix @@ -17,7 +17,7 @@ }: { fetched = hash: pkgs.runCommand - "${pname}-pypi-url" + filename { buildInputs = [ pkgs.curl From 5a28f90c9052c619061f208967ac620e9f02f492 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 22 Nov 2022 19:19:44 +0100 Subject: [PATCH 06/17] poetry translator: use pep425 algo --- .../python/translators/poetry/default.nix | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix index e20016f1..b867a87a 100644 --- a/src/subsystems/python/translators/poetry/default.nix +++ b/src/subsystems/python/translators/poetry/default.nix @@ -7,9 +7,33 @@ Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry. { dlib, lib, + inputs, ... }: let l = lib // builtins; + # initialize poetry2nix libs + pythonVersion = "python3"; + system = "x86_64-linux"; + fakeStdenv = { + isLinux = l.hasInfix "-linux" system; + isDarwin = l.hasInfix "-darwin" system; + targetPlatform.isAarch64 = l.hasPrefix "aarch64-" system; + targetPlatform.parsed.cpu.name = l.elemAt (l.splitString "-" system) 0; + }; + fakePython = { + version = pythonVersion; + passthru.implementation = "cpython"; + }; + pep425 = import "${inputs.poetry2nix}/pep425.nix" { + inherit lib; + python = fakePython; + stdenv = fakeStdenv; + poetryLib = import "${inputs.poetry2nix}/lib.nix" { + inherit lib; + pkgs = null; + stdenv = fakeStdenv; + }; + }; in { type = "pure"; @@ -199,7 +223,12 @@ in { sources = l.mapAttrs (sourceName: files: let - candidate = lib.head files; + wheels = pep425.selectWheel files; + sdists = builtins.filter (x: !(lib.hasSuffix ".whl" x.file)) files; + candidate = + if lib.length wheels > 0 + then builtins.head wheels + else builtins.head sdists; isWheel = l.hasSuffix "whl" candidate.file; suffix = if isWheel From a4cca9b99dee8e7a0da4656e55fd4d5a180e7cc0 Mon Sep 17 00:00:00 2001 From: phaer Date: Tue, 22 Nov 2022 19:21:27 +0100 Subject: [PATCH 07/17] pypi-sdist: use simple hashing --- src/fetchers/pypi-sdist/default.nix | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/fetchers/pypi-sdist/default.nix b/src/fetchers/pypi-sdist/default.nix index a664a7ce..dcd98cc2 100644 --- a/src/fetchers/pypi-sdist/default.nix +++ b/src/fetchers/pypi-sdist/default.nix @@ -21,20 +21,11 @@ "https://files.pythonhosted.org/packages/source/" + "${firstChar}/${pname}/${pname}-${version}.${extension}"; in { - calcHash = algo: - utils.hashPath algo ( - b.fetchurl {inherit url;} - ); - fetched = hash: let - source = - (pkgs.fetchurl { - inherit url; - sha256 = hash; - }) - .overrideAttrs (old: { - outputHashMode = "recursive"; - }); + source = pkgs.fetchurl { + inherit url; + sha256 = hash; + }; in utils.extractSource { inherit source; From 31b0b52dc28979c2ffa73c670e6af0bebf7f6cdf Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 23 Nov 2022 11:07:01 +0100 Subject: [PATCH 08/17] chore: python: poetry: cleanup --- .../python/translators/poetry/default.nix | 145 +----------------- 1 file changed, 1 insertion(+), 144 deletions(-) diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix index b867a87a..1009ee66 100644 --- a/src/subsystems/python/translators/poetry/default.nix +++ b/src/subsystems/python/translators/poetry/default.nix @@ -1,5 +1,5 @@ /* -This is an example for a pure translator which translates poetry's poetry.lock +This is a pure translator which translates poetry's poetry.lock to a dream2nix dream-lock(.json). Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry.loc @@ -37,89 +37,13 @@ Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry. in { type = "pure"; - /* - Allow dream2nix to detect if a given directory contains a project - which can be translated with this translator. - Usually this can be done by checking for the existence of specific - file names or file endings. - - Alternatively a fully featured discoverer can be implemented under - `src/subsystems/{subsystem}/discoverers`. - This is recommended if more complex project structures need to be - discovered like, for example, workspace projects spanning over multiple - sub-directories - - If a fully featured discoverer exists, do not define `discoverProject`. - */ discoverProject = tree: - /* - TODO: change this function to identify projects that can be translated - by the current translator. - */ - # checks if the path l.pathExists "${tree.fullPath}/poetry.lock"; # translate from a given source and a project specification to a dream-lock. translate = { - /* - A list of projects returned by `discoverProjects` - Example: - { - "dreamLockPath": "packages/optimism/dream-lock.json", - "name": "optimism", - "relPath": "", - "subsystem": "nodejs", - "subsystemInfo": { - "workspaces": [ - "packages/common-ts", - "packages/contracts", - "packages/core-utils", - ] - } - } - */ project, - /* - Entire source tree represented as nested attribute set. - (produced by `dlib.prepareSourceTree`) - - This has the advantage that files will only be read/parsed once, even - when accessed multiple times or by multiple translators. - - Example: - { - files = { - "package.json" = { - relPath = "package.json" - fullPath = "${source}/package.json" - content = ; - jsonContent = ; - tomlContent = ; - } - }; - - directories = { - "packages" = { - relPath = "packages"; - fullPath = "${source}/packages"; - files = { - ... - }; - directories = { - ... - }; - }; - }; - - # returns the tree object of the given sub-path - getNodeFromPath = path: ... - } - */ tree, - /* - arguments defined in `extraArgs` specified by user - (see definition for `extraArgs` near the bottom of this file) - */ ... }: let # get the root source and project source @@ -131,23 +55,11 @@ in { sources = (projectTree.getNodeFromPath "poetry.lock").tomlContent; - /* - Define the name and version of the top-level package. - If there are multiple top-level packages, just pick any of them. - Dream2nix requires one package to be the default package. - TODO: change this. In the case of niv, we just pick some static values - here because there isn't really a concept of a package in niv. - (Later we just fetch the dependencies and merge them into one output) - */ defaultPackageName = "poetry"; defaultPackageVersion = "unknown-version"; in # see example in src/specifications/dream-lock-example.json { - /* - Tell dream2nix that this is the human-readable (decompressed) - representation of the dream-lock. - */ decompressed = true; # generic fields _generic = { @@ -162,64 +74,16 @@ in { location = project.relPath; }; - /* - Store subsystem specific data. - This is needed if the subsystem requires extra metadata to be stored. - This will be a free-form field as long as no jsonschema exists for - this subsystem. - TODO: create a jsonschema for the current subsystem under - /src/specifications/{subsystem}/dream-lock-schema.json - */ _subsystem = { application = false; pythonAttr = "python3"; sourceFormats = {}; }; - /* - List dependency edges that need to be removed in order to prevent - infinite recursions in the nix evaluator. - Usually this can be ommitted. - */ cyclicDependencies = {}; - /* - Define the dependency graph. - This can be ommitted, in which case dream2nix assumes that: - - all sources listed in `sources` represent one dependency - - all dependencies are direct dependenceis of the `defaultPackage` - Example: - # foo-1.2.3 depends on bar-2.3.4 and baz-3.4.5 - { - foo."1.2.3" = [ - {name = "bar"; version = "2.3.4"} - {name = "baz"; version = "3.4.5"} - ] - ... - } - */ dependencies = {}; - /* - Define the sources for all dependencies including their checksums. - This allows dream2nix to fetch all sources reproducibly. - Each dependency specified in `dependencies` must have a corresponding - entry in `sources` which describes how the source can be fetched. - Check the `fetchers` section in the docs to see what fetchers are - supported and which arguments they require. - Example: - { - foo."1.2.3" = { - type = "http"; - url = "https://foo.com/tarball.tar.gz"; - hash = "sha256:000000000000000000000000000000000000000"; - }; - bar."2.3.4" = { - ... - } - ... - } - */ sources = l.mapAttrs (sourceName: files: let @@ -261,12 +125,5 @@ in { sources.metadata.files; }; - # If the translator requires additional arguments, specify them here. - # Users will be able to set these arguments via `settings`. - # There are only two types of arguments: - # - string argument (type = "argument") - # - boolean flag (type = "flag") - # String arguments contain a default value and examples. Flags do not. - # Flags are false by default. extraArgs = {}; } From 3dd340290824f910e2c5f9b291184e6e9e462d3c Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 23 Nov 2022 12:52:36 +0100 Subject: [PATCH 09/17] feat: python: poetry: add flags pythonVersion, system --- examples/python_poetry/flake.nix | 9 ++- .../python/translators/poetry/default.nix | 69 ++++++++++++------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/examples/python_poetry/flake.nix b/examples/python_poetry/flake.nix index 790c2dd4..ca06a777 100644 --- a/examples/python_poetry/flake.nix +++ b/examples/python_poetry/flake.nix @@ -18,7 +18,11 @@ systems = ["x86_64-linux"]; imports = [dream2nix.flakeModuleBeta]; - perSystem = {config, ...}: { + perSystem = { + config, + system, + ... + }: { # define an input for dream2nix to generate outputs for dream2nix.inputs."my-project" = { source = src; @@ -26,6 +30,9 @@ name = "my-project"; subsystem = "python"; translator = "poetry"; + pythonVersion = "3.10"; + subsystemInfo.system = system; + subsystemInfo.pythonVersion = "3.10"; }; }; }; diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix index 1009ee66..45ce544d 100644 --- a/src/subsystems/python/translators/poetry/default.nix +++ b/src/subsystems/python/translators/poetry/default.nix @@ -11,29 +11,6 @@ Example poetry.lock: https://github.com/python-poetry/poetry/blob/master/poetry. ... }: let l = lib // builtins; - # initialize poetry2nix libs - pythonVersion = "python3"; - system = "x86_64-linux"; - fakeStdenv = { - isLinux = l.hasInfix "-linux" system; - isDarwin = l.hasInfix "-darwin" system; - targetPlatform.isAarch64 = l.hasPrefix "aarch64-" system; - targetPlatform.parsed.cpu.name = l.elemAt (l.splitString "-" system) 0; - }; - fakePython = { - version = pythonVersion; - passthru.implementation = "cpython"; - }; - pep425 = import "${inputs.poetry2nix}/pep425.nix" { - inherit lib; - python = fakePython; - stdenv = fakeStdenv; - poetryLib = import "${inputs.poetry2nix}/lib.nix" { - inherit lib; - pkgs = null; - stdenv = fakeStdenv; - }; - }; in { type = "pure"; @@ -44,6 +21,8 @@ in { translate = { project, tree, + pythonVersion, + system, ... }: let # get the root source and project source @@ -51,6 +30,28 @@ in { projectSource = "${tree.fullPath}/${project.relPath}"; projectTree = tree.getNodeFromPath project.relPath; + # initialize poetry2nix libs + fakeStdenv = { + isLinux = l.hasInfix "-linux" system; + isDarwin = l.hasInfix "-darwin" system; + targetPlatform.isAarch64 = l.hasPrefix "aarch64-" system; + targetPlatform.parsed.cpu.name = l.elemAt (l.splitString "-" system) 0; + }; + fakePython = { + version = pythonVersion; + passthru.implementation = "cpython"; + }; + pep425 = import "${inputs.poetry2nix}/pep425.nix" { + inherit lib; + python = fakePython; + stdenv = fakeStdenv; + poetryLib = import "${inputs.poetry2nix}/lib.nix" { + inherit lib; + pkgs = null; + stdenv = fakeStdenv; + }; + }; + # use dream2nix' source tree abstraction to access json content of files sources = (projectTree.getNodeFromPath "poetry.lock").tomlContent; @@ -125,5 +126,25 @@ in { sources.metadata.files; }; - extraArgs = {}; + extraArgs = { + system = { + description = "System for produced outputs."; + # default = "blabla"; + examples = [ + "x86_64-linux" + "x86_64-darwin" + ]; + type = "argument"; + }; + pythonVersion = { + description = "python version to translate for"; + default = "3.10"; + examples = [ + "3.8" + "3.9" + "3.10" + ]; + type = "argument"; + }; + }; } From 77d27d08443db484a15b5760554e40467543fda8 Mon Sep 17 00:00:00 2001 From: phaer Date: Wed, 23 Nov 2022 18:17:46 +0100 Subject: [PATCH 10/17] read defaultPackageName from pyproject.toml --- src/subsystems/python/translators/poetry/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/subsystems/python/translators/poetry/default.nix b/src/subsystems/python/translators/poetry/default.nix index 45ce544d..8265c12c 100644 --- a/src/subsystems/python/translators/poetry/default.nix +++ b/src/subsystems/python/translators/poetry/default.nix @@ -56,8 +56,9 @@ in { sources = (projectTree.getNodeFromPath "poetry.lock").tomlContent; - defaultPackageName = "poetry"; - defaultPackageVersion = "unknown-version"; + pyproject = (projectTree.getNodeFromPath "pyproject.toml").tomlContent; + defaultPackageName = pyproject.tool.poetry.name; + defaultPackageVersion = pyproject.tool.poetry.version; in # see example in src/specifications/dream-lock-example.json { @@ -67,9 +68,7 @@ in { # TODO: specify the default package name defaultPackage = defaultPackageName; # TODO: specify a list of exported packages and their versions - packages = { - poetry = "unknown-version"; - }; + packages.${defaultPackageName} = defaultPackageVersion; # TODO: this must be equivalent to the subsystem name subsystem = "python"; location = project.relPath; From f9e6c0e24565b2dfc1881d96e9b29d3d783f2751 Mon Sep 17 00:00:00 2001 From: phaer Date: Wed, 23 Nov 2022 18:18:37 +0100 Subject: [PATCH 11/17] update poetry2nix --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index a2321927..56dd1d86 100644 --- a/flake.lock +++ b/flake.lock @@ -241,16 +241,16 @@ "poetry2nix": { "flake": false, "locked": { - "lastModified": 1632969109, - "narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=", + "lastModified": 1666918719, + "narHash": "sha256-BkK42fjAku+2WgCOv2/1NrPa754eQPV7gPBmoKQBWlc=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77", + "rev": "289efb187123656a116b915206e66852f038720e", "type": "github" }, "original": { "owner": "nix-community", - "ref": "1.21.0", + "ref": "1.36.0", "repo": "poetry2nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 8cc764e3..0edbf802 100644 --- a/flake.nix +++ b/flake.nix @@ -46,7 +46,7 @@ # required for utils.satisfiesSemver poetry2nix = { - url = "github:nix-community/poetry2nix/1.21.0"; + url = "github:nix-community/poetry2nix/1.36.0"; flake = false; }; From c51db6ebeb84914e9f1604fbb38c17d268b9fb06 Mon Sep 17 00:00:00 2001 From: phaer Date: Wed, 23 Nov 2022 20:43:20 +0100 Subject: [PATCH 12/17] docs: add pypi-wheel fetcher --- docs/src/intro/fetchers.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/src/intro/fetchers.md b/docs/src/intro/fetchers.md index 7f0b1b5a..3c601cf0 100644 --- a/docs/src/intro/fetchers.md +++ b/docs/src/intro/fetchers.md @@ -90,13 +90,24 @@ Inputs: ### pypi-sdist -Fetches from pypi registry. +Fetches sources distributions ("sdists") from pypi registry. Inputs: - pname - version - hash +### pypi-wheel + +Fetches wheels from pypi registry. + +Inputs: +- filename +- pname +- version +- hash + + ### crates-io Fetches from crates.io registry. From a5e098038c42528bf4dff2db6cb958b1777b00e3 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 25 Nov 2022 06:02:43 +0300 Subject: [PATCH 13/17] fix: correct simple template path --- templates/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default.nix b/templates/default.nix index 37be2d3b..b6c22f1b 100644 --- a/templates/default.nix +++ b/templates/default.nix @@ -11,7 +11,7 @@ in { default = self.templates.simple; simple = { description = "Simple dream2nix flake"; - path = ./templates/simple; + path = ./simple; welcomeText = '' You just created a simple dream2nix package! From ed42f6fa091ca788d413998a4ddc9bc0de1b42d7 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 25 Nov 2022 11:52:38 +0100 Subject: [PATCH 14/17] wip: adapt python builder for poetry. --- .../python/builders/simple-python/default.nix | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix index 6312b6ca..1901d6b9 100644 --- a/src/subsystems/python/builders/simple-python/default.nix +++ b/src/subsystems/python/builders/simple-python/default.nix @@ -40,13 +40,14 @@ package = produceDerivation defaultPackageName (buildFunc { name = defaultPackageName; src = getSource defaultPackageName defaultPackageVersion; - format = "setuptools"; + format = "other"; buildInputs = pkgs.pythonManylinuxPackages.manylinux1; - nativeBuildInputs = [pkgs.autoPatchelfHook]; + nativeBuildInputs = [pkgs.autoPatchelfHook python.pkgs.pip python.pkgs.poetry-core]; propagatedBuildInputs = [python.pkgs.setuptools]; doCheck = false; dontStrip = true; - preBuild = '' + + buildPhase = '' mkdir dist for file in ${builtins.toString allDependencySources}; do # pick right most element of path @@ -54,17 +55,18 @@ fname=$(stripHash $fname) cp $file dist/$fname done - mkdir -p "$out/${python.sitePackages}" - export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" ${python}/bin/python -m pip install \ - ./dist/*.{whl,tar.gz,zip} \ + --find-links ./dist/ \ --no-build-isolation \ --no-index \ --no-warn-script-location \ --prefix="$out" \ --no-cache \ + --ignore-installed \ + . \ $pipInstallFlags ''; + installPhase = "true"; }); devShell = pkgs.mkShell { @@ -72,8 +74,6 @@ # a drv with all dependencies without the main package (package.overrideAttrs (old: { src = "."; - dontUnpack = true; - buildPhase = old.preBuild; })) ]; }; From 5095de6aa31a8e7be6655141f63be7da21f7eb58 Mon Sep 17 00:00:00 2001 From: DavHau Date: Sun, 27 Nov 2022 13:05:03 +0100 Subject: [PATCH 15/17] fix: python: simple-builder: add wheel to nativeBuildInputs --- src/subsystems/python/builders/simple-python/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix index 1901d6b9..d67b809b 100644 --- a/src/subsystems/python/builders/simple-python/default.nix +++ b/src/subsystems/python/builders/simple-python/default.nix @@ -42,7 +42,13 @@ src = getSource defaultPackageName defaultPackageVersion; format = "other"; buildInputs = pkgs.pythonManylinuxPackages.manylinux1; - nativeBuildInputs = [pkgs.autoPatchelfHook python.pkgs.pip python.pkgs.poetry-core]; + nativeBuildInputs = + [pkgs.autoPatchelfHook] + ++ (with python.pkgs; [ + pip + poetry-core + wheel + ]); propagatedBuildInputs = [python.pkgs.setuptools]; doCheck = false; dontStrip = true; From b6af93946130748f72671dfd2ab84a5aeaf1f191 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 30 Nov 2022 00:43:59 +0700 Subject: [PATCH 16/17] chore: add dark logo to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ac2f742..baf3784d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- + dream2nix - A framework for automated nix packaging From ce047a9ceaf1040dc14e393cc3243d22dd0dbcdc Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 1 Dec 2022 18:55:52 +0700 Subject: [PATCH 17/17] chore: stop promoting flake parts as api is in beta --- .../_d2n-define-projects-manually/flake.nix | 29 +++++++------------ examples/_d2n-flake-parts/flake.nix | 12 ++++++-- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/examples/_d2n-define-projects-manually/flake.nix b/examples/_d2n-define-projects-manually/flake.nix index c612fb1e..05c12e4f 100644 --- a/examples/_d2n-define-projects-manually/flake.nix +++ b/examples/_d2n-define-projects-manually/flake.nix @@ -1,8 +1,6 @@ { inputs = { dream2nix.url = "github:nix-community/dream2nix"; - nixpkgs.follows = "dream2nix/nixpkgs"; - flake-parts.url = "github:hercules-ci/flake-parts"; src.url = "github:prettier/prettier/2.4.1"; src.flake = false; }; @@ -10,26 +8,21 @@ outputs = { self, dream2nix, - flake-parts, src, - ... }: - flake-parts.lib.mkFlake {inherit self;} { + (dream2nix.lib.makeFlakeOutputs { systems = ["x86_64-linux"]; - imports = [dream2nix.flakeModuleBeta]; - - perSystem = {config, ...}: { - # define an input for dream2nix to generate outputs for - dream2nix.inputs."prettier" = { - source = src; - projects = { - prettier = { - name = "prettier"; - subsystem = "nodejs"; - translator = "yarn-lock"; - }; - }; + config.projectRoot = ./.; + source = src; + projects = { + prettier = { + name = "prettier"; + subsystem = "nodejs"; + translator = "yarn-lock"; }; }; + }) + // { + # checks = self.packages; }; } diff --git a/examples/_d2n-flake-parts/flake.nix b/examples/_d2n-flake-parts/flake.nix index 839bc281..c612fb1e 100644 --- a/examples/_d2n-flake-parts/flake.nix +++ b/examples/_d2n-flake-parts/flake.nix @@ -3,7 +3,7 @@ dream2nix.url = "github:nix-community/dream2nix"; nixpkgs.follows = "dream2nix/nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; - src.url = "github:yusdacra/linemd/v0.4.0"; + src.url = "github:prettier/prettier/2.4.1"; src.flake = false; }; @@ -20,9 +20,15 @@ perSystem = {config, ...}: { # define an input for dream2nix to generate outputs for - dream2nix.inputs."linemd" = { + dream2nix.inputs."prettier" = { source = src; - settings = [{builder = "crane";}]; + projects = { + prettier = { + name = "prettier"; + subsystem = "nodejs"; + translator = "yarn-lock"; + }; + }; }; }; };