diff --git a/dev-flake/flake-compat.nix b/dev-flake/flake-compat.nix index 9f46d45d..269ae9e8 100644 --- a/dev-flake/flake-compat.nix +++ b/dev-flake/flake-compat.nix @@ -200,7 +200,13 @@ rootOverrides = mapAttrs' - (input: lockKey: nameValuePair lockKey (impureOverrides.${input} or null)) + (input: lockKey': let + lockKey = + if builtins.isList lockKey' + then builtins.concatStringsSep "/" lockKey' + else lockKey'; + in + nameValuePair lockKey (impureOverrides.${input} or null)) lockFile.nodes.${lockFile.root}.inputs; allNodes = diff --git a/dev-flake/flake.lock b/dev-flake/flake.lock index 3a00f56a..20b6371e 100644 --- a/dev-flake/flake.lock +++ b/dev-flake/flake.lock @@ -71,6 +71,39 @@ "type": "github" } }, + "mdbook-nixdoc": { + "inputs": { + "flake-parts": [ + "nix-unit", + "flake-parts" + ], + "nix-github-actions": [ + "nix-unit", + "nix-github-actions" + ], + "nixpkgs": [ + "nix-unit", + "nixpkgs" + ], + "treefmt-nix": [ + "nix-unit", + "treefmt-nix" + ] + }, + "locked": { + "lastModified": 1708342507, + "narHash": "sha256-KoaAog/peCdkrtapUPq7F9aneJLXZMwo6CCcQ3+OtOA=", + "owner": "adisbladis", + "repo": "mdbook-nixdoc", + "rev": "ce2d327032fb3d6f20144a54e7dc1195c108a1b1", + "type": "github" + }, + "original": { + "owner": "adisbladis", + "repo": "mdbook-nixdoc", + "type": "github" + } + }, "nix-github-actions": { "inputs": { "nixpkgs": [ @@ -97,16 +130,19 @@ "flake-parts": [ "flake-parts" ], + "mdbook-nixdoc": "mdbook-nixdoc", "nix-github-actions": "nix-github-actions", - "nixpkgs": "nixpkgs", + "nixpkgs": [ + "nixpkgs" + ], "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1701194455, - "narHash": "sha256-TlCq5DnIlfMFS+b/kqi0vR/8gRgVpqisOr+gjpo1VPc=", + "lastModified": 1708394818, + "narHash": "sha256-75YUg5yCnWVYrrrevYpa0r8axH5c0Uub2n8tQCBAPT4=", "owner": "nix-community", "repo": "nix-unit", - "rev": "2a061b6981c12b1a995740799a4465c3124f60ac", + "rev": "1fda41fe253ec495a2884f50e1b2ab836a7a94b4", "type": "github" }, "original": { @@ -117,32 +153,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1701045352, - "narHash": "sha256-iWsDbWzBP4gotkRfg/lH2A3O9wFoJc+yVO8CDuHLRe8=", + "lastModified": 1708751719, + "narHash": "sha256-0uWOKSpXJXmXswOvDM5Vk3blB74apFB6rNGWV5IjoN0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5171694860f185961daff3b1b413dabcab421300", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "master", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1696604326, - "narHash": "sha256-YXUNI0kLEcI5g8lqGMb0nh67fY9f2YoJsILafh6zlMo=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "87828a0e03d1418e848d3dd3f3014a632e4a4f64", + "rev": "f63ce824cd2f036216eb5f637dfef31e1a03ee89", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-unstable", + "ref": "nixpkgs-unstable", "type": "indirect" } }, @@ -173,7 +193,7 @@ "flake-compat": "flake-compat", "flake-parts": "flake-parts", "nix-unit": "nix-unit", - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs", "pre-commit-hooks": "pre-commit-hooks" } }, diff --git a/dev-flake/flake.nix b/dev-flake/flake.nix index 8e08e5fb..0f6b5e34 100644 --- a/dev-flake/flake.nix +++ b/dev-flake/flake.nix @@ -7,7 +7,7 @@ }; inputs = { - nixpkgs.url = "nixpkgs/nixos-unstable"; + nixpkgs.url = "nixpkgs/nixpkgs-unstable"; flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; @@ -19,7 +19,7 @@ pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; nix-unit.url = "github:nix-community/nix-unit"; - # nix-unit.inputs.nixpkgs.follows = "nixpkgs"; + nix-unit.inputs.nixpkgs.follows = "nixpkgs"; nix-unit.inputs.flake-parts.follows = "flake-parts"; devshell = { diff --git a/examples/packages/single-language/python-package-apache-airflow/default.nix b/examples/packages/single-language/python-package-apache-airflow/default.nix index 79b151a4..7e67b1d8 100644 --- a/examples/packages/single-language/python-package-apache-airflow/default.nix +++ b/examples/packages/single-language/python-package-apache-airflow/default.nix @@ -61,7 +61,7 @@ in { dream2nix.modules.dream2nix.nixpkgs-overrides ]; nixpkgs-overrides = { - exclude = ["propagatedBuildInputs"]; + exclude = ["propagatedBuildInputs" "dependencies"]; }; }; lazy-object-proxy = { diff --git a/flake.lock b/flake.lock index 18b00c69..9aa26a21 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1702272962, - "narHash": "sha256-D+zHwkwPc6oYQ4G3A1HuadopqRwUY/JkMwHz1YF7j4Q=", + "lastModified": 1708751719, + "narHash": "sha256-0uWOKSpXJXmXswOvDM5Vk3blB74apFB6rNGWV5IjoN0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e97b3e4186bcadf0ef1b6be22b8558eab1cdeb5d", + "rev": "f63ce824cd2f036216eb5f637dfef31e1a03ee89", "type": "github" }, "original": { diff --git a/modules/dream2nix/core/eval-cache/interface.nix b/modules/dream2nix/core/eval-cache/interface.nix index 6a1ec437..0bea4efa 100644 --- a/modules/dream2nix/core/eval-cache/interface.nix +++ b/modules/dream2nix/core/eval-cache/interface.nix @@ -49,7 +49,7 @@ in { refresh = l.mkOption { type = t.path; - description = "Script to refresh the cache file of this package"; + description = "Script to refresh the eval cache file"; readOnly = true; }; }; diff --git a/modules/dream2nix/core/lock/default.nix b/modules/dream2nix/core/lock/default.nix index 9d4c128e..5cb7f151 100644 --- a/modules/dream2nix/core/lock/default.nix +++ b/modules/dream2nix/core/lock/default.nix @@ -47,7 +47,7 @@ refresh_scripts = json.loads('${l.toJSON cfg.fields}') # noqa: E501 repo_path = Path(subprocess.run( ['${config.paths.findRoot}'], # noqa: E501 - check=True, text=True, capture_output=True) + check=True, text=True, stdout=subprocess.PIPE) .stdout.strip()) lock_path_rel = Path('${config.paths.package}/${config.paths.lockFile}') # noqa: E501 lock_path = repo_path / lock_path_rel.relative_to(lock_path_rel.anchor) @@ -200,7 +200,7 @@ in { config = { lock.refresh = config.deps.writeScriptBin "refresh" '' - #!/usr/bin/env bash + #!${config.deps.bash}/bin/bash set -Eeuo pipefail ### Executing auto generated refresh script @@ -221,7 +221,7 @@ in { deps = {nixpkgs, ...}: l.mapAttrs (_: l.mkDefault) { - inherit (nixpkgs) nix writeScriptBin; + inherit (nixpkgs) bash nix writeScriptBin; inherit (nixpkgs.writers) writePython3 writePython3Bin; }; }; diff --git a/modules/dream2nix/core/lock/interface.nix b/modules/dream2nix/core/lock/interface.nix index 9593a65f..02423f8a 100644 --- a/modules/dream2nix/core/lock/interface.nix +++ b/modules/dream2nix/core/lock/interface.nix @@ -79,7 +79,7 @@ in { refresh = l.mkOption { type = t.package; - description = "Script to refresh the cache file of this package"; + description = "Script to refresh the lock file"; readOnly = true; }; diff --git a/modules/dream2nix/pip/default.nix b/modules/dream2nix/pip/default.nix index 524bb8ad..894e6bfa 100644 --- a/modules/dream2nix/pip/default.nix +++ b/modules/dream2nix/pip/default.nix @@ -135,7 +135,7 @@ in { inherit (cfg) env pypiSnapshotDate pipFlags pipVersion requirementsList requirementsFiles nativeBuildInputs; inherit (config.deps) writePureShellScript nix; inherit (config.paths) findRoot; - inherit (nixpkgs) gitMinimal nix-prefetch-scripts python3 writeText openssh; + inherit (nixpkgs) fetchFromGitHub fetchurl gitMinimal nix-prefetch-scripts openssh python3 rustPlatform writeText; pythonInterpreter = "${python}/bin/python"; }; setuptools = config.deps.python.pkgs.setuptools; diff --git a/modules/dream2nix/pip/tests/packages/lock-script-works/default.nix b/modules/dream2nix/pip/tests/packages/lock-script-works/default.nix new file mode 100644 index 00000000..07d74848 --- /dev/null +++ b/modules/dream2nix/pip/tests/packages/lock-script-works/default.nix @@ -0,0 +1,56 @@ +# An example package with dependencies defined via pyproject.toml +{ + config, + lib, + dream2nix, + packageSets, + ... +}: let + pyproject = lib.importTOML (config.mkDerivation.src + /pyproject.toml); + pkgs = packageSets.nixpkgs; +in { + imports = [ + dream2nix.modules.dream2nix.pip + ]; + + deps = {nixpkgs, ...}: { + python = nixpkgs.python310; + }; + + inherit (pyproject.project) name version; + + mkDerivation = { + src = ./.; + }; + + pip = { + pypiSnapshotDate = "2023-08-27"; + requirementsList = + pyproject.build-system.requires + or [] + ++ pyproject.project.dependencies; + flattenDependencies = true; + }; + + paths.projectRootFile = "pyproject.toml"; + + public = lib.mkForce (pkgs.runCommand "pip-lock-script-works" { + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + # invalidate by including into the name a hash over: + # - the path to the nixpkgs + # - TODO: the implementation of the fetch script + name = let + hash = builtins.hashString "sha256" (builtins.unsafeDiscardStringContext '' + ${pkgs.path} + ''); + in "pip-lock-script-works-${lib.substring 0 16 hash}"; + } '' + cp -r ${config.mkDerivation.src}/* . + chmod +w -R . + ls -lah + ${config.lock.refresh}/bin/refresh + touch $out + ''); +} diff --git a/modules/dream2nix/pip/tests/packages/lock-script-works/my_tool/__init__.py b/modules/dream2nix/pip/tests/packages/lock-script-works/my_tool/__init__.py new file mode 100644 index 00000000..c1ce29af --- /dev/null +++ b/modules/dream2nix/pip/tests/packages/lock-script-works/my_tool/__init__.py @@ -0,0 +1,9 @@ +from setuptools import setup + + +def main(): + print("Hello World!") + + +if __name__ == "__main__": + main() diff --git a/modules/dream2nix/pip/tests/packages/lock-script-works/pyproject.toml b/modules/dream2nix/pip/tests/packages/lock-script-works/pyproject.toml new file mode 100644 index 00000000..2c6b1929 --- /dev/null +++ b/modules/dream2nix/pip/tests/packages/lock-script-works/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = [ "setuptools" ] +build-backend = "setuptools.build_meta" + +[project] +name = "my-tool" +description = "my tool" +version = "1.0.0" +dependencies = [ + "setuptools" +] + +[project.scripts] +my-tool = "my_tool:main" diff --git a/pkgs/fetchPipMetadata/script.nix b/pkgs/fetchPipMetadata/script.nix index 6aea6f9f..8dc3594b 100644 --- a/pkgs/fetchPipMetadata/script.nix +++ b/pkgs/fetchPipMetadata/script.nix @@ -43,6 +43,9 @@ writePureShellScript, nix-prefetch-scripts, openssh, + fetchFromGitHub, + fetchurl, + rustPlatform, }: let package = import ./package.nix { inherit @@ -53,9 +56,63 @@ ; }; + pythonFixed = python3.override { + packageOverrides = curr: prev: { + /* + downgrading to version 10.1.*, as 10.2.0 introduces a breakage triggering: + [17:42:11.824][[::1]:56958] client connect + [17:42:11.909][[::1]:56958] server connect pypi.org:443 (151.101.64.223:443) + [17:42:11.958] Deferring layer decision, not enough data: [...] + */ + mitmproxy = prev.mitmproxy.overridePythonAttrs (old: rec { + version = "10.1.6"; + src = fetchFromGitHub { + owner = "mitmproxy"; + repo = "mitmproxy"; + rev = "refs/tags/${version}"; + hash = "sha256-W+gxK5bNCit1jK9ojwE/HVjUz6OJcNw6Ac1lN5FxGgw="; + }; + doCheck = false; + pyproject = true; + # format = "pyproject"; + }); + mitmproxy-rs = prev.mitmproxy-rs.overrideAttrs (old: rec { + version = "0.4.1"; + src = fetchFromGitHub { + owner = "mitmproxy"; + repo = "mitmproxy_rs"; + rev = version; + hash = "sha256-Vc7ez/W40CefO2ZLAHot14p478pDPtQor865675vCtI="; + }; + cargoDeps = rustPlatform.importCargoLock { + lockFile = "${src}/Cargo.lock"; + outputHashes = { + "internet-packet-0.1.0" = "sha256-VtEuCE1sulBIFVymh7YW7VHCuIBjtb6tHoPz2tjxX+Q="; + }; + }; + }); + + mitmproxy-macos = prev.buildPythonPackage rec { + pname = "mitmproxy-macos"; + version = "0.4.1"; + format = "wheel"; + + src = fetchurl { + url = "https://files.pythonhosted.org/packages/85/79/f11ba4cf6e89408ed52d9317c00d3ae4ad18c51cf710821c9342fc95cd0f/mitmproxy_macos-0.5.1-py3-none-any.whl"; + hash = "sha256-P7T8mTCzMQEphnWuumZF3ucb4XYgyMsHyBC6i+1sKkI="; + }; + + pythonImportsCheck = ["mitmproxy_macos"]; + nativeBuildInputs = [ + prev.hatchling + ]; + }; + }; + }; + # We use nixpkgs python3 to run mitmproxy, see function parameters pythonWithMitmproxy = - python3.withPackages + pythonFixed.withPackages (ps: [ps.mitmproxy ps.python-dateutil]); path = [nix gitMinimal openssh] ++ nativeBuildInputs; diff --git a/tests/nix-unit/test_python-pdm-lib/default.nix b/tests/nix-unit/test_python-pdm-lib/default.nix index 43088c0b..6681206d 100644 --- a/tests/nix-unit/test_python-pdm-lib/default.nix +++ b/tests/nix-unit/test_python-pdm-lib/default.nix @@ -6,7 +6,7 @@ }: let libpdm = (import ../../../modules/dream2nix/WIP-python-pdm/lib.nix) { inherit lib libpyproject; - python3 = pkgs.python3; + python3 = pkgs.python310; targetPlatform = lib.systems.elaborate "x86_64-linux"; }; pyproject-nix = inputs.pyproject-nix;