From b727a07775a47fb039c848e04d9891a645fd388f Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 13 Mar 2024 14:13:39 +0700 Subject: [PATCH] override: unify overriding across modules Add options `overrides.${name}` and `overrideAll` to all language modules that manage a dependency tree. This is done in a backward compatible fashion. Old options for overriding dependencies continue working, though those are not displayed in the manual anymore. For the following modules the manual now suggest using the new unified override options: - for pip: use `pip.overrides` and `pip.overrideAll` instead of `pip.drvs` - for nodejs-granular-v3: use `nodejs-granular-v3.overrides` and `nodejs-granular-v3.overrideAll` instead of `nodejs-granular-v3.deps` - `php-granular`: use `php-granular.override` and `php-granular.overrideAll` instead of `php-granular.deps` --- CHANGELOG.md | 13 + .../default.nix | 2 +- .../default.nix | 2 +- .../python-packaging-odoo/default.nix | 2 +- lib/internal/mkSubmodule.nix | 13 + .../dream2nix/nodejs-granular-v3/default.nix | 7 +- .../nodejs-granular-v3/interface.nix | 74 +++--- modules/dream2nix/overrides/default.nix | 5 + modules/dream2nix/overrides/interface.nix | 45 ++++ modules/dream2nix/php-granular/default.nix | 10 +- modules/dream2nix/php-granular/interface.nix | 43 ++-- modules/dream2nix/pip/default.nix | 2 + modules/dream2nix/pip/interface.nix | 243 ++++++++++-------- modules/flake-parts/website/default.nix | 2 + 14 files changed, 291 insertions(+), 172 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 lib/internal/mkSubmodule.nix create mode 100644 modules/dream2nix/overrides/default.nix create mode 100644 modules/dream2nix/overrides/interface.nix diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..b8e0c881 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) + +## [Unreleased] + +### Added + +#### Unified override interface across languages + +New options `overrides.${name}` and `overrideAll` for all language modules that manage a dependency tree. diff --git a/examples/packages/languages/python-local-development-multiple-packages/default.nix b/examples/packages/languages/python-local-development-multiple-packages/default.nix index 95867cfa..e81bf49b 100644 --- a/examples/packages/languages/python-local-development-multiple-packages/default.nix +++ b/examples/packages/languages/python-local-development-multiple-packages/default.nix @@ -38,6 +38,6 @@ in { "${config.paths.package}/subpkg1" "${config.paths.package}/subpkg2" ]; - drvs.subpkg2 = buildWithSetuptools; + overrides.subpkg2 = buildWithSetuptools; }; } diff --git a/examples/packages/languages/python-packaging-apache-airflow/default.nix b/examples/packages/languages/python-packaging-apache-airflow/default.nix index 7e67b1d8..feba5751 100644 --- a/examples/packages/languages/python-packaging-apache-airflow/default.nix +++ b/examples/packages/languages/python-packaging-apache-airflow/default.nix @@ -53,7 +53,7 @@ in { "setuptools-scm" ]; - drvs = { + overrides = { # We include fixes from nixpkgs for pendulum, but keep # our dependencies to avoid version conflicts pendulum = { diff --git a/examples/packages/languages/python-packaging-odoo/default.nix b/examples/packages/languages/python-packaging-odoo/default.nix index b0e3f5e4..857bd2c3 100644 --- a/examples/packages/languages/python-packaging-odoo/default.nix +++ b/examples/packages/languages/python-packaging-odoo/default.nix @@ -45,7 +45,7 @@ in { nativeBuildInputs = [config.deps.postgresql]; # fix some builds via package-specific overrides - drvs = { + overrides = { psycopg2 = { imports = [ dream2nix.modules.dream2nix.nixpkgs-overrides diff --git a/lib/internal/mkSubmodule.nix b/lib/internal/mkSubmodule.nix new file mode 100644 index 00000000..516e4769 --- /dev/null +++ b/lib/internal/mkSubmodule.nix @@ -0,0 +1,13 @@ +{ + lib, + specialArgs, + ... +}: module: +lib.mkOption { + type = lib.types.submoduleWith { + inherit specialArgs; + modules = [ + module + ]; + }; +} diff --git a/modules/dream2nix/nodejs-granular-v3/default.nix b/modules/dream2nix/nodejs-granular-v3/default.nix index ce6236c6..12d7c86b 100644 --- a/modules/dream2nix/nodejs-granular-v3/default.nix +++ b/modules/dream2nix/nodejs-granular-v3/default.nix @@ -49,11 +49,6 @@ source = pdefs.${name}.${version}.source; }; - nameVersionPair = name: version: { - name = name; - version = version; - }; - # name: version: -> [ {name=; version=; } ] getDependencies = pname: version: l.filter @@ -87,6 +82,8 @@ imports = [ (commonModule name version) (depsModule name version) + cfg.overrideAll + (cfg.overrides.${name} or {}) ]; }) versions diff --git a/modules/dream2nix/nodejs-granular-v3/interface.nix b/modules/dream2nix/nodejs-granular-v3/interface.nix index bbd27392..cb93984b 100644 --- a/modules/dream2nix/nodejs-granular-v3/interface.nix +++ b/modules/dream2nix/nodejs-granular-v3/interface.nix @@ -3,44 +3,58 @@ lib, dream2nix, packageSets, + specialArgs, ... }: let l = lib // builtins; t = l.types; + cfg = config.nodejs-granular-v3; + mkSubmodule = import ../../../lib/internal/mkSubmodule.nix {inherit lib specialArgs;}; in { - options.nodejs-granular-v3 = l.mapAttrs (_: l.mkOption) { - buildScript = { - type = t.nullOr (t.oneOf [t.str t.path t.package]); - description = '' - A command or script to execute instead of `npm run build`. - Is only executed if `runBuild = true`. - ''; - }; - installMethod = { - type = t.enum [ - "symlink" - "copy" + options.nodejs-granular-v3 = mkSubmodule { + imports = [ + ../overrides + ]; + config.overrideType = { + imports = [ + dream2nix.modules.dream2nix.mkDerivation ]; - description = '' - Strategy to use for populating ./node_modules. - Symlinking is quicker, but often introduces compatibility issues with bundlers like webpack and other build tools. - Copying is slow, but more reliable; - ''; }; - runBuild = { - type = t.bool; - description = '' - Whether to run a package's build script (aka. `npm run build`) - ''; - }; - deps = { - type = t.attrsOf (t.attrsOf (t.submodule { - imports = [ - dream2nix.modules.dream2nix.core - dream2nix.modules.dream2nix.mkDerivation + options = l.mapAttrs (_: l.mkOption) { + buildScript = { + type = t.nullOr (t.oneOf [t.str t.path t.package]); + description = '' + A command or script to execute instead of `npm run build`. + Is only executed if `runBuild = true`. + ''; + }; + installMethod = { + type = t.enum [ + "symlink" + "copy" ]; - _module.args = {inherit dream2nix packageSets;}; - })); + description = '' + Strategy to use for populating ./node_modules. + Symlinking is quicker, but often introduces compatibility issues with bundlers like webpack and other build tools. + Copying is slow, but more reliable; + ''; + }; + runBuild = { + type = t.bool; + description = '' + Whether to run a package's build script (aka. `npm run build`) + ''; + }; + deps = { + internal = true; + # hack because internal doesn't propagate to submodule options + visible = "shallow"; + type = t.attrsOf (t.attrsOf (t.submodule { + imports = [ + cfg.overrideType + ]; + })); + }; }; }; } diff --git a/modules/dream2nix/overrides/default.nix b/modules/dream2nix/overrides/default.nix new file mode 100644 index 00000000..bd4ca0b5 --- /dev/null +++ b/modules/dream2nix/overrides/default.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ./interface.nix + ]; +} diff --git a/modules/dream2nix/overrides/interface.nix b/modules/dream2nix/overrides/interface.nix new file mode 100644 index 00000000..3adfac73 --- /dev/null +++ b/modules/dream2nix/overrides/interface.nix @@ -0,0 +1,45 @@ +{ + lib, + specialArgs, + config, + ... +}: let + t = lib.types; + staticModules = [ + {_module.args = specialArgs;} + config.overrideType + ]; +in { + options = { + overrideAll = lib.mkOption { + type = t.deferredModuleWith {inherit staticModules;}; + description = '' + Overrides applied on all dependencies. + ''; + default = {}; + example = { + mkDerivation.doCheck = false; + }; + }; + + overrides = lib.mkOption { + type = t.attrsOf (t.deferredModuleWith {inherit staticModules;}); + description = '' + Overrides applied only on dependencies matching the specified name. + ''; + default = {}; + example = { + hello.mkDerivation.postPatch = '' + substituteInPlace Makefile --replace /usr/local /usr + ''; + }; + }; + + ## INTERNAL + overrideType = lib.mkOption { + type = t.deferredModule; + default = {}; + internal = true; + }; + }; +} diff --git a/modules/dream2nix/php-granular/default.nix b/modules/dream2nix/php-granular/default.nix index 5318fd73..f7b09274 100644 --- a/modules/dream2nix/php-granular/default.nix +++ b/modules/dream2nix/php-granular/default.nix @@ -165,8 +165,8 @@ if version == "unknown" then "0.0.0" else version; - - module = {config, ...}: { + in + {config, ...}: { imports = [ dream2nix.modules.dream2nix.mkDerivation ]; @@ -289,8 +289,6 @@ ''; }; }; - in - module; in { imports = [ ./interface.nix @@ -308,7 +306,9 @@ in { imports = [ ./interface.nix (commonModule name version) - dream2nix.modules.dream2nix.mkDerivation + cfg.overrideType + cfg.overrideAll + (cfg.overrides.${name} or {}) ]; inherit name version; } diff --git a/modules/dream2nix/php-granular/interface.nix b/modules/dream2nix/php-granular/interface.nix index a1d68516..df1b4283 100644 --- a/modules/dream2nix/php-granular/interface.nix +++ b/modules/dream2nix/php-granular/interface.nix @@ -1,29 +1,40 @@ { config, dream2nix, - packageSets, lib, + packageSets, + specialArgs, ... }: let l = lib // builtins; t = l.types; + cfg = config.php-granular; + mkSubmodule = import ../../../lib/internal/mkSubmodule.nix {inherit lib specialArgs;}; in { - options.php-granular = l.mapAttrs (_: l.mkOption) { - deps = { - type = t.lazyAttrsOf (t.lazyAttrsOf (t.submodule { - imports = [ - dream2nix.modules.dream2nix.core - # TODO: fix this - # putting mkDerivation here leads to an error when generating docs: - # The option `php-granular.deps...version' is used but not defined. - # dream2nix.modules.dream2nix.mkDerivation - ]; - _module.args = {inherit dream2nix packageSets;}; - })); + options.php-granular = mkSubmodule { + imports = [ + ../overrides + ]; + config.overrideType = { + imports = [ + dream2nix.modules.dream2nix.mkDerivation + ]; }; - composerInstallFlags = { - type = t.listOf t.str; - default = []; + options = l.mapAttrs (_: l.mkOption) { + deps = { + internal = true; + visible = "shallow"; + type = t.lazyAttrsOf (t.lazyAttrsOf (t.submodule { + imports = [ + dream2nix.modules.dream2nix.core + cfg.overrideType + ]; + })); + }; + composerInstallFlags = { + type = t.listOf t.str; + default = []; + }; }; }; } diff --git a/modules/dream2nix/pip/default.nix b/modules/dream2nix/pip/default.nix index 894e6bfa..ac4a0e3b 100644 --- a/modules/dream2nix/pip/default.nix +++ b/modules/dream2nix/pip/default.nix @@ -33,6 +33,8 @@ imports = [ commonModule dependencyModule + cfg.overrideAll + (cfg.overrides.${name} or {}) # include community overrides (dream2nix.overrides.python.${name} or {}) ]; diff --git a/modules/dream2nix/pip/interface.nix b/modules/dream2nix/pip/interface.nix index 35ef88ed..f5cfb5bb 100644 --- a/modules/dream2nix/pip/interface.nix +++ b/modules/dream2nix/pip/interface.nix @@ -3,130 +3,147 @@ lib, dream2nix, packageSets, + specialArgs, ... }: let l = lib // builtins; t = l.types; + mkSubmodule = import ../../../lib/internal/mkSubmodule.nix {inherit lib specialArgs;}; in { - options.pip = { - # internal options to pass data between pip-hotfixes and pip - targets = l.mkOption { - type = t.raw; - internal = true; - description = "the targets of the lock file to build"; - }; - rootDependencies = l.mkOption { - type = t.attrsOf t.bool; - internal = true; - description = "the names of the selected top-level dependencies"; + options.pip = mkSubmodule { + imports = [ + ../overrides + ]; + + config.overrideType = { + imports = [ + dream2nix.modules.dream2nix.buildPythonPackage + ]; }; - # user interface - env = l.mkOption { - type = t.attrsOf t.str; - default = {}; - description = '' - environment variables exported while locking - ''; - example = lib.literalExpression '' - { - PIP_FIND_LINKS = "''${config.deps.setuptools.dist}"; - } - ''; - }; - pypiSnapshotDate = l.mkOption { - type = t.nullOr t.str; - description = '' - maximum release date for packages - Choose any date from the past. - ''; - example = "2023-01-01"; - default = null; - }; - pipFlags = l.mkOption { - type = t.listOf t.str; - description = '' - list of flags for pip install - ''; - default = []; - }; - pipVersion = l.mkOption { - type = t.str; - description = '' - pip version to use to generate the report - ''; - default = "23.1"; - }; - requirementsList = l.mkOption { - type = t.listOf t.str; - default = []; - description = '' - list of strings of requirements.txt entries - ''; - }; - requirementsFiles = l.mkOption { - type = t.listOf t.str; - default = []; - description = '' - list of requirements.txt files - ''; - }; - buildDependencies = l.mkOption { - type = t.attrsOf t.bool; - default = { - cython = true; - flit-core = true; - flit-scm = true; - hatch-fancy-pypi-readme = true; - hatch-nodejs-version = true; - hatch-vcs = true; - hatchling = true; - pbr = true; - pdm-pep517 = true; - poetry-core = true; - poetry-dynamic-versioning = true; - setuptools = true; - setuptools-odoo = true; - setuptools-scm = true; - versioneer = true; - wheel = true; + options = { + # internal options to pass data between pip-hotfixes and pip + targets = l.mkOption { + type = t.raw; + internal = true; + description = "the targets of the lock file to build"; + }; + rootDependencies = l.mkOption { + type = t.attrsOf t.bool; + internal = true; + description = "the names of the selected top-level dependencies"; }; - description = '' - python packages to be added only as buildInputs. - These should be somehow installable from `requirementsList` or - `requirementsFiles` too; listing them here doesn't do that automatically. - ''; - example = lib.literalExpression '' - { - setuptools-scm = false; # To disable the default - easy_install = true; # To select easy_install as a buildInput - } - ''; - }; - buildExtras = l.mkOption { - type = t.listOf t.str; - default = []; - description = '' - list of python "extras" to build with. This can be a subset of the - extras in your lock file. - ''; - }; + # user interface + env = l.mkOption { + type = t.attrsOf t.str; + default = {}; + description = '' + environment variables exported while locking + ''; + example = lib.literalExpression '' + { + PIP_FIND_LINKS = "''${config.deps.setuptools.dist}"; + } + ''; + }; + pypiSnapshotDate = l.mkOption { + type = t.nullOr t.str; + description = '' + maximum release date for packages + Choose any date from the past. + ''; + example = "2023-01-01"; + default = null; + }; + pipFlags = l.mkOption { + type = t.listOf t.str; + description = '' + list of flags for pip install + ''; + default = []; + }; + pipVersion = l.mkOption { + type = t.str; + description = '' + pip version to use to generate the report + ''; + default = "23.1"; + }; + requirementsList = l.mkOption { + type = t.listOf t.str; + default = []; + description = '' + list of strings of requirements.txt entries + ''; + }; + requirementsFiles = l.mkOption { + type = t.listOf t.str; + default = []; + description = '' + list of requirements.txt files + ''; + }; + buildDependencies = l.mkOption { + type = t.attrsOf t.bool; + default = { + cython = true; + flit-core = true; + flit-scm = true; + hatch-fancy-pypi-readme = true; + hatch-nodejs-version = true; + hatch-vcs = true; + hatchling = true; + pbr = true; + pdm-pep517 = true; + poetry-core = true; + poetry-dynamic-versioning = true; + setuptools = true; + setuptools-odoo = true; + setuptools-scm = true; + versioneer = true; + wheel = true; + }; + description = '' + python packages to be added only as buildInputs. + These should be somehow installable from `requirementsList` or + `requirementsFiles` too; listing them here doesn't do that automatically. + ''; + example = lib.literalExpression '' + { + setuptools-scm = false; # To disable the default + easy_install = true; # To select easy_install as a buildInput + } + ''; + }; - nativeBuildInputs = l.mkOption { - type = t.listOf t.package; - default = []; - description = '' - list of native packages to include during metadata generation - ''; - }; + buildExtras = l.mkOption { + type = t.listOf t.str; + default = []; + description = '' + list of python "extras" to build with. This can be a subset of the + extras in your lock file. + ''; + }; - drvs = l.mkOption { - type = t.lazyAttrsOf (t.submoduleWith { - modules = [dream2nix.modules.dream2nix.core]; - specialArgs = {inherit packageSets dream2nix;}; - }); - description = "drv-parts modules that define python dependencies"; + nativeBuildInputs = l.mkOption { + type = t.listOf t.package; + default = []; + description = '' + list of native packages to include during metadata generation + ''; + }; + + drvs = l.mkOption { + internal = true; + # hack because internal=true doesn't propagate to the submodule options + visible = "shallow"; + type = t.lazyAttrsOf (t.submoduleWith { + modules = [dream2nix.modules.dream2nix.core]; + specialArgs = {inherit packageSets dream2nix;}; + }); + description = "drv-parts modules that define python dependencies"; + }; }; }; } diff --git a/modules/flake-parts/website/default.nix b/modules/flake-parts/website/default.nix index ffc79fe4..9720bf2f 100644 --- a/modules/flake-parts/website/default.nix +++ b/modules/flake-parts/website/default.nix @@ -27,8 +27,10 @@ "_template" ]; public = lib.genAttrs [ + "nodejs-granular-v3" "nodejs-package-lock-v3" "php-composer-lock" + "php-granular" "pip" "rust-cargo-lock" "rust-crane"