From 44d41411686bc798876bd6d9f36a4c1143138d85 Mon Sep 17 00:00:00 2001 From: dusk Date: Tue, 12 Nov 2024 18:09:27 +0300 Subject: [PATCH] refactor(rust): deduplicate vendor and utils code (#1054) * refactor(rust): consolidate the separated vendoring code into rust-cargo-vendor * refactor(rust): move the utility code in utils.nix to rust-cargo-lock and rust-cargo-vendor * fix(rust-crane): add the necessary dependencies to config.deps for devshell * refactor(rust-cargo-vendor): remove unnecessary mkMerge * fix(rust-crane): override cargo in config.deps with our toolchain so that rust-cargo-vendor uses it too --- .../default.nix | 1 + .../default.nix | 1 + .../languages/rust-packaging/default.nix | 1 + .../dream2nix/buildRustPackage/default.nix | 124 ++---------- modules/dream2nix/buildRustPackage/utils.nix | 191 ------------------ .../dream2nix/rust-cargo-lock/cargoLock.nix | 152 ++++++++++++++ modules/dream2nix/rust-cargo-lock/default.nix | 30 ++- .../dream2nix/rust-cargo-lock/interface.nix | 10 + modules/dream2nix/rust-cargo-vendor/README.md | 6 + .../dream2nix/rust-cargo-vendor/default.nix | 115 +++++++++++ .../dream2nix/rust-cargo-vendor/interface.nix | 35 ++++ .../patch-workspace-deps.jq | 0 .../vendor.nix | 32 +++ modules/dream2nix/rust-crane/default.nix | 120 ++--------- .../rust-crane/patch-workspace-deps.jq | 48 ----- modules/dream2nix/rust-crane/utils.nix | 191 ------------------ modules/dream2nix/rust-crane/vendor.nix | 148 -------------- 17 files changed, 417 insertions(+), 788 deletions(-) delete mode 100644 modules/dream2nix/buildRustPackage/utils.nix create mode 100644 modules/dream2nix/rust-cargo-lock/cargoLock.nix create mode 100644 modules/dream2nix/rust-cargo-vendor/README.md create mode 100644 modules/dream2nix/rust-cargo-vendor/default.nix create mode 100644 modules/dream2nix/rust-cargo-vendor/interface.nix rename modules/dream2nix/{buildRustPackage => rust-cargo-vendor}/patch-workspace-deps.jq (100%) rename modules/dream2nix/{buildRustPackage => rust-cargo-vendor}/vendor.nix (81%) delete mode 100644 modules/dream2nix/rust-crane/patch-workspace-deps.jq delete mode 100644 modules/dream2nix/rust-crane/utils.nix delete mode 100644 modules/dream2nix/rust-crane/vendor.nix diff --git a/examples/packages/languages/rust-local-development-workspace/default.nix b/examples/packages/languages/rust-local-development-workspace/default.nix index c3ab54ba..57894d16 100644 --- a/examples/packages/languages/rust-local-development-workspace/default.nix +++ b/examples/packages/languages/rust-local-development-workspace/default.nix @@ -6,6 +6,7 @@ }: { imports = [ dream2nix.modules.dream2nix.rust-cargo-lock + dream2nix.modules.dream2nix.rust-cargo-vendor dream2nix.modules.dream2nix.rust-crane ]; diff --git a/examples/packages/languages/rust-packaging-buildRustPackage/default.nix b/examples/packages/languages/rust-packaging-buildRustPackage/default.nix index 60b54140..7b56e3de 100644 --- a/examples/packages/languages/rust-packaging-buildRustPackage/default.nix +++ b/examples/packages/languages/rust-packaging-buildRustPackage/default.nix @@ -6,6 +6,7 @@ }: { imports = [ dream2nix.modules.dream2nix.rust-cargo-lock + dream2nix.modules.dream2nix.rust-cargo-vendor dream2nix.modules.dream2nix.buildRustPackage ]; diff --git a/examples/packages/languages/rust-packaging/default.nix b/examples/packages/languages/rust-packaging/default.nix index fc2d9a1e..22b131d0 100644 --- a/examples/packages/languages/rust-packaging/default.nix +++ b/examples/packages/languages/rust-packaging/default.nix @@ -6,6 +6,7 @@ }: { imports = [ dream2nix.modules.dream2nix.rust-cargo-lock + dream2nix.modules.dream2nix.rust-cargo-vendor dream2nix.modules.dream2nix.rust-crane ]; diff --git a/modules/dream2nix/buildRustPackage/default.nix b/modules/dream2nix/buildRustPackage/default.nix index 302e3da9..58928cc7 100644 --- a/modules/dream2nix/buildRustPackage/default.nix +++ b/modules/dream2nix/buildRustPackage/default.nix @@ -3,133 +3,52 @@ lib, dream2nix, ... -} @ topArgs: let +}: let l = lib // builtins; - dreamLock = config.rust-cargo-lock.dreamLock; - - sourceRoot = config.mkDerivation.src; - - fetchDreamLockSources = - import ../../../lib/internal/fetchDreamLockSources.nix - {inherit lib;}; - getDreamLockSource = import ../../../lib/internal/getDreamLockSource.nix {inherit lib;}; readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;}; - hashPath = import ../../../lib/internal/hashPath.nix { - inherit lib; - inherit (config.deps) runCommandLocal nix; - }; - hashFile = import ../../../lib/internal/hashFile.nix { - inherit lib; - inherit (config.deps) runCommandLocal nix; - }; - - # fetchers - fetchers = { - git = import ../../../lib/internal/fetchers/git { - inherit hashPath; - inherit (config.deps) fetchgit; - }; - http = import ../../../lib/internal/fetchers/http { - inherit hashFile lib; - inherit (config.deps.stdenv) mkDerivation; - inherit (config.deps) fetchurl; - }; - crates-io = import ../../../lib/internal/fetchers/crates-io { - inherit hashFile; - inherit (config.deps) fetchurl runCommandLocal; - }; - }; dreamLockLoaded = readDreamLock {inherit (config.rust-cargo-lock) dreamLock;}; dreamLockInterface = dreamLockLoaded.interface; - fetchedSources' = fetchDreamLockSources { - inherit (dreamLockInterface) defaultPackageName defaultPackageVersion; - inherit (dreamLockLoaded.lock) sources; - inherit fetchers; - }; - - fetchedSources = - fetchedSources' - // { - ${defaultPackageName}.${defaultPackageVersion} = sourceRoot; - }; - - # name: version: -> store-path - getSource = getDreamLockSource fetchedSources; - inherit (dreamLockInterface) - getDependencies # name: version: -> [ {name=; version=; } ] - # Attributes - subsystemAttrs # attrset - packageVersions - defaultPackageName - defaultPackageVersion ; - toTOML = import ../../../lib/internal/toTOML.nix {inherit lib;}; - - utils = import ./utils.nix { - inherit dreamLock getSource lib toTOML sourceRoot; - inherit - (dreamLockInterface) - getSourceSpec - getRoot - subsystemAttrs - packages - ; - inherit - (config.deps) - writeText - ; - }; - - vendoring = import ./vendor.nix { - inherit dreamLock getSource lib; - inherit - (dreamLockInterface) - getSourceSpec - subsystemAttrs - ; - inherit - (config.deps) - cargo - jq - moreutils - python3Packages - runCommandLocal - writePython3 - ; - }; + meta = let + meta = subsystemAttrs.meta.${pname}.${version}; + in + meta + // { + license = l.map (name: l.licenses.${name}) meta.license; + }; pname = config.name; version = config.version; - src = utils.getRootSource pname version; + src = config.rust-cargo-vendor.getRootSource pname version; replacePaths = - utils.replaceRelativePathsWithAbsolute + config.rust-cargo-vendor.replaceRelativePathsWithAbsolute subsystemAttrs.relPathReplacements.${pname}.${version}; - writeGitVendorEntries = vendoring.writeGitVendorEntries "vendored-sources"; + writeGitVendorEntries = config.rust-cargo-vendor.writeGitVendorEntries "vendored-sources"; cargoBuildFlags = "--package ${pname}"; buildArgs = { inherit pname version; src = lib.mkForce src; - meta = utils.getMeta pname version; + inherit meta; cargoBuildFlags = cargoBuildFlags; cargoTestFlags = cargoBuildFlags; cargoVendorDir = "../nix-vendor"; - dream2nixVendorDir = vendoring.vendoredDependencies; + dream2nixVendorDir = config.rust-cargo-vendor.vendoredSources; postUnpack = '' - ${vendoring.copyVendorDir "$dream2nixVendorDir" "./nix-vendor"} + ${config.rust-cargo-vendor.copyVendorDir "$dream2nixVendorDir" "./nix-vendor"} export CARGO_HOME=$(pwd)/.cargo_home ''; @@ -140,7 +59,7 @@ fi ${writeGitVendorEntries} ${replacePaths} - ${utils.writeCargoLock} + ${config.rust-cargo-lock.writeCargoLock} ''; }; in { @@ -154,24 +73,13 @@ in { package-func.args = buildArgs; public = { - meta = utils.getMeta pname version; + inherit meta; }; deps = {nixpkgs, ...}: { inherit (nixpkgs) - cargo - fetchurl - jq - moreutils - python3Packages - runCommandLocal rustPlatform - writeText - ; - inherit - (nixpkgs.writers) - writePython3 ; }; } diff --git a/modules/dream2nix/buildRustPackage/utils.nix b/modules/dream2nix/buildRustPackage/utils.nix deleted file mode 100644 index b9d7631d..00000000 --- a/modules/dream2nix/buildRustPackage/utils.nix +++ /dev/null @@ -1,191 +0,0 @@ -{ - dreamLock, - getSourceSpec, - getSource, - getRoot, - sourceRoot, - subsystemAttrs, - packages, - lib, - toTOML, - writeText, - ... -}: let - l = lib // builtins; - isInPackages = name: version: (packages.${name} or null) == version; -in rec { - getMeta = pname: version: let - meta = subsystemAttrs.meta.${pname}.${version}; - in - meta - // { - license = l.map (name: l.licenses.${name}) meta.license; - }; - - # Gets the root source for a package - getRootSource = pname: version: let - root = getRoot pname version; - in - getSource root.pname root.version; - - # Generates a script that replaces relative path dependency paths with absolute - # ones, if the path dependency isn't in the source dream2nix provides - replaceRelativePathsWithAbsolute = replacements: let - replace = - l.concatStringsSep - " \\\n" - ( - l.mapAttrsToList - ( - # TODO: this is not great, because it forces us to include the entire - # sourceRoot here, which could possibly cause more rebuilds than necessary - # when source is changed (although this mostly depends on how the project - # repository is structured). doing this properly is pretty complex, but - # it should still be done later. - from: relPath: ''--replace "\"${from}\"" "\"${sourceRoot}/${relPath}\""'' - ) - replacements - ); - in '' - echo "dream2nix: replacing relative dependency paths with absolute paths in Cargo.toml" - substituteInPlace ./Cargo.toml \ - ${replace} - ''; - - # Backup original Cargo.lock if it exists and write our own one - writeCargoLock = '' - echo "dream2nix: replacing Cargo.lock with ${cargoLock}" - mv -f Cargo.lock Cargo.lock.orig || echo "dream2nix: no Cargo.lock was found beforehand" - cat ${cargoLock} > Cargo.lock - ''; - - # The Cargo.lock for this dreamLock. - cargoLock = let - mkPkgEntry = { - name, - version, - ... - } @ args: let - # constructs source string for dependency - makeSource = sourceSpec: let - source = - if sourceSpec.type == "crates-io" - then "registry+https://github.com/rust-lang/crates.io-index" - else if sourceSpec.type == "git" - then let - gitSpec = - l.findFirst - (src: src.url == sourceSpec.url && src.sha == sourceSpec.rev) - (throw "no git source: ${sourceSpec.url}#${sourceSpec.rev}") - (subsystemAttrs.gitSources or {}); - refPart = - l.optionalString - (gitSpec ? type) - "?${gitSpec.type}=${gitSpec.value}"; - in "git+${sourceSpec.url}${refPart}#${sourceSpec.rev}" - else null; - in - source; - # constructs source string for dependency entry - makeDepSource = sourceSpec: - if sourceSpec.type == "crates-io" - then makeSource sourceSpec - else if sourceSpec.type == "git" - then l.concatStringsSep "#" (l.init (l.splitString "#" (makeSource sourceSpec))) - else null; - # removes source type information from the version - normalizeVersion = version: srcType: l.removeSuffix ("$" + srcType) version; - - sourceSpec = getSourceSpec name version; - - normalizedVersion = normalizeVersion version sourceSpec.type; - - source = let - src = makeSource sourceSpec; - in - if src == null - then throw "source type '${sourceSpec.type}' not supported" - else src; - dependencies = - l.map - ( - dep: let - depSourceSpec = getSourceSpec dep.name dep.version; - depSource = makeDepSource depSourceSpec; - - normalizedDepVersion = normalizeVersion dep.version depSourceSpec.type; - - hasMultipleVersions = - l.length (l.attrValues dreamLock.sources.${dep.name}) > 1; - hasDuplicateVersions = dep.version != normalizedDepVersion; - - # only put version if there are different versions of the dep - versionString = - l.optionalString hasMultipleVersions " ${normalizedDepVersion}"; - # only put source if there are duplicate versions of the dep - # cargo vendor does not support this anyway and so builds will fail - # until https://github.com/rust-lang/cargo/issues/10310 is resolved. - srcString = - l.optionalString hasDuplicateVersions " (${depSource})"; - in "${dep.name}${versionString}${srcString}" - ) - args.dependencies; - - isMainPackage = isInPackages name version; - in - { - name = sourceSpec.pname or name; - version = sourceSpec.version or normalizedVersion; - } - # put dependencies like how cargo expects them - // ( - l.optionalAttrs - (l.length dependencies > 0) - {inherit dependencies;} - ) - // ( - l.optionalAttrs - (sourceSpec.type != "path" && !isMainPackage) - {inherit source;} - ) - // ( - l.optionalAttrs - (sourceSpec.type == "crates-io" && !isMainPackage) - {checksum = sourceSpec.hash;} - ); - _package = l.flatten ( - l.mapAttrsToList - ( - name: versions: - l.mapAttrsToList - ( - version: dependencies: - mkPkgEntry {inherit name version dependencies;} - ) - versions - ) - dreamLock.dependencies - ); - package = - ( - # add packages as dependencies because Cargo expects them to be there aswell - l.filter - (pkg: ! l.any (opkg: pkg.name == opkg.name && pkg.version == opkg.version) _package) - ( - l.mapAttrsToList - (pname: version: { - name = pname; - inherit version; - }) - dreamLock._generic.packages - ) - ) - ++ _package; - lockTOML = toTOML { - # the lockfile we generate is of version 3 - version = 3; - inherit package; - }; - in - writeText "Cargo.lock" lockTOML; -} diff --git a/modules/dream2nix/rust-cargo-lock/cargoLock.nix b/modules/dream2nix/rust-cargo-lock/cargoLock.nix new file mode 100644 index 00000000..22a9b215 --- /dev/null +++ b/modules/dream2nix/rust-cargo-lock/cargoLock.nix @@ -0,0 +1,152 @@ +{ + lib, + dreamLock, + # nixpkgs + writeText, +}: +# The Cargo.lock for this dreamLock. +let + l = lib // builtins; + + readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;}; + + dreamLockLoaded = readDreamLock {inherit dreamLock;}; + dreamLockInterface = dreamLockLoaded.interface; + + inherit + (dreamLockInterface) + getSourceSpec + subsystemAttrs + packages + ; + + toTOML = import ../../../lib/internal/toTOML.nix {inherit lib;}; + + isInPackages = name: version: (packages.${name} or null) == version; + mkPkgEntry = { + name, + version, + ... + } @ args: let + # constructs source string for dependency + makeSource = sourceSpec: let + source = + if sourceSpec.type == "crates-io" + then "registry+https://github.com/rust-lang/crates.io-index" + else if sourceSpec.type == "git" + then let + gitSpec = + l.findFirst + (src: src.url == sourceSpec.url && src.sha == sourceSpec.rev) + (throw "no git source: ${sourceSpec.url}#${sourceSpec.rev}") + (subsystemAttrs.gitSources or {}); + refPart = + l.optionalString + (gitSpec ? type) + "?${gitSpec.type}=${gitSpec.value}"; + in "git+${sourceSpec.url}${refPart}#${sourceSpec.rev}" + else null; + in + source; + # constructs source string for dependency entry + makeDepSource = sourceSpec: + if sourceSpec.type == "crates-io" + then makeSource sourceSpec + else if sourceSpec.type == "git" + then l.concatStringsSep "#" (l.init (l.splitString "#" (makeSource sourceSpec))) + else null; + # removes source type information from the version + normalizeVersion = version: srcType: l.removeSuffix ("$" + srcType) version; + + sourceSpec = getSourceSpec name version; + + normalizedVersion = normalizeVersion version sourceSpec.type; + + source = let + src = makeSource sourceSpec; + in + if src == null + then throw "source type '${sourceSpec.type}' not supported" + else src; + dependencies = + l.map + ( + dep: let + depSourceSpec = getSourceSpec dep.name dep.version; + depSource = makeDepSource depSourceSpec; + + normalizedDepVersion = normalizeVersion dep.version depSourceSpec.type; + + hasMultipleVersions = + l.length (l.attrValues dreamLock.sources.${dep.name}) > 1; + hasDuplicateVersions = dep.version != normalizedDepVersion; + + # only put version if there are different versions of the dep + versionString = + l.optionalString hasMultipleVersions " ${normalizedDepVersion}"; + # only put source if there are duplicate versions of the dep + # cargo vendor does not support this anyway and so builds will fail + # until https://github.com/rust-lang/cargo/issues/10310 is resolved. + srcString = + l.optionalString hasDuplicateVersions " (${depSource})"; + in "${dep.name}${versionString}${srcString}" + ) + args.dependencies; + + isMainPackage = isInPackages name version; + in + { + name = sourceSpec.pname or name; + version = sourceSpec.version or normalizedVersion; + } + # put dependencies like how cargo expects them + // ( + l.optionalAttrs + (l.length dependencies > 0) + {inherit dependencies;} + ) + // ( + l.optionalAttrs + (sourceSpec.type != "path" && !isMainPackage) + {inherit source;} + ) + // ( + l.optionalAttrs + (sourceSpec.type == "crates-io" && !isMainPackage) + {checksum = sourceSpec.hash;} + ); + _package = l.flatten ( + l.mapAttrsToList + ( + name: versions: + l.mapAttrsToList + ( + version: dependencies: + mkPkgEntry {inherit name version dependencies;} + ) + versions + ) + dreamLock.dependencies + ); + package = + ( + # add packages as dependencies because Cargo expects them to be there aswell + l.filter + (pkg: ! l.any (opkg: pkg.name == opkg.name && pkg.version == opkg.version) _package) + ( + l.mapAttrsToList + (pname: version: { + name = pname; + inherit version; + }) + dreamLock._generic.packages + ) + ) + ++ _package; + lockTOML = toTOML { + # the lockfile we generate is of version 3 + version = 3; + inherit package; + }; +in + writeText "Cargo.lock" lockTOML diff --git a/modules/dream2nix/rust-cargo-lock/default.nix b/modules/dream2nix/rust-cargo-lock/default.nix index fe65fda0..88c278ac 100644 --- a/modules/dream2nix/rust-cargo-lock/default.nix +++ b/modules/dream2nix/rust-cargo-lock/default.nix @@ -20,12 +20,38 @@ projectRelPath = ""; tree = prepareSourceTree {source = cfg.source;}; }; + + cargoLock = import ./cargoLock.nix { + inherit lib; + inherit (cfg) dreamLock; + inherit (config.deps) writeText; + }; + + # Backup original Cargo.lock if it exists and write our own one + writeCargoLock = '' + echo "dream2nix: replacing Cargo.lock with ${cfg.cargoLock}" + mv -f Cargo.lock Cargo.lock.orig || echo "dream2nix: no Cargo.lock was found beforehand" + cat ${cfg.cargoLock} > Cargo.lock + ''; in { imports = [ ./interface.nix - dream2nix.modules.dream2nix.mkDerivation + dream2nix.modules.dream2nix.core ]; + rust-cargo-lock = { - inherit dreamLock; + inherit + cargoLock + dreamLock + writeCargoLock + ; }; + + deps = {nixpkgs, ...}: + l.mapAttrs (_: l.mkOverride 997) { + inherit + (nixpkgs) + writeText + ; + }; } diff --git a/modules/dream2nix/rust-cargo-lock/interface.nix b/modules/dream2nix/rust-cargo-lock/interface.nix index 3bcd1b3e..7f0f2e38 100644 --- a/modules/dream2nix/rust-cargo-lock/interface.nix +++ b/modules/dream2nix/rust-cargo-lock/interface.nix @@ -7,11 +7,21 @@ t = l.types; in { options.rust-cargo-lock = l.mapAttrs (_: l.mkOption) { + cargoLock = { + type = t.path; + internal = true; + description = "The dreamlock that was generated as a Cargo.lock file"; + }; dreamLock = { type = t.attrs; internal = true; description = "The content of the dream2nix generated lock file"; }; + writeCargoLock = { + type = t.str; + internal = true; + description = "Shell commands to backup original Cargo.lock and use dream2nix one in a rust derivation"; + }; source = { type = t.either t.path t.package; description = "Source of the package"; diff --git a/modules/dream2nix/rust-cargo-vendor/README.md b/modules/dream2nix/rust-cargo-vendor/README.md new file mode 100644 index 00000000..f1e8bda1 --- /dev/null +++ b/modules/dream2nix/rust-cargo-vendor/README.md @@ -0,0 +1,6 @@ +--- +title: "rust-cargo-vendor" +state: "experimental" +maintainers: + - DavHau +--- diff --git a/modules/dream2nix/rust-cargo-vendor/default.nix b/modules/dream2nix/rust-cargo-vendor/default.nix new file mode 100644 index 00000000..9fe6f1ac --- /dev/null +++ b/modules/dream2nix/rust-cargo-vendor/default.nix @@ -0,0 +1,115 @@ +{ + config, + lib, + dream2nix, + ... +}: let + l = lib // builtins; + cfg = config.rust-cargo-vendor; + + dreamLock = config.rust-cargo-lock.dreamLock; + + sourceRoot = config.mkDerivation.src; + + fetchDreamLockSources = + import ../../../lib/internal/fetchDreamLockSources.nix + {inherit lib;}; + getDreamLockSource = import ../../../lib/internal/getDreamLockSource.nix {inherit lib;}; + readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;}; + hashPath = import ../../../lib/internal/hashPath.nix { + inherit lib; + inherit (config.deps) runCommandLocal nix; + }; + hashFile = import ../../../lib/internal/hashFile.nix { + inherit lib; + inherit (config.deps) runCommandLocal nix; + }; + + # fetchers + fetchers = { + git = import ../../../lib/internal/fetchers/git { + inherit hashPath; + inherit (config.deps) fetchgit; + }; + crates-io = import ../../../lib/internal/fetchers/crates-io { + inherit hashFile; + inherit (config.deps) fetchurl runCommandLocal; + }; + path = import ../../../lib/internal/fetchers/path { + inherit hashPath; + }; + }; + + dreamLockLoaded = readDreamLock {inherit dreamLock;}; + dreamLockInterface = dreamLockLoaded.interface; + + inherit (dreamLockInterface) defaultPackageName defaultPackageVersion; + + fetchedSources' = fetchDreamLockSources { + inherit defaultPackageName defaultPackageVersion; + inherit (dreamLockLoaded.lock) sources; + inherit fetchers; + }; + + fetchedSources = + fetchedSources' + // { + ${defaultPackageName}.${defaultPackageVersion} = sourceRoot; + }; + + getSource = getDreamLockSource fetchedSources; + + vendoring = import ./vendor.nix { + inherit dreamLock getSource lib sourceRoot; + inherit + (dreamLockInterface) + getSourceSpec + getRoot + subsystemAttrs + ; + inherit + (config.deps) + cargo + jq + moreutils + python3Packages + runCommandLocal + writePython3 + ; + }; +in { + imports = [ + ./interface.nix + dream2nix.modules.dream2nix.core + ]; + + rust-cargo-vendor = { + vendoredSources = vendoring.vendoredDependencies; + inherit + (vendoring) + copyVendorDir + getRootSource + writeGitVendorEntries + replaceRelativePathsWithAbsolute + ; + }; + + deps = {nixpkgs, ...}: + l.mapAttrs (_: l.mkOverride 998) { + inherit + (nixpkgs) + cargo + jq + moreutils + python3Packages + runCommandLocal + fetchurl + fetchgit + nix + ; + inherit + (nixpkgs.writers) + writePython3 + ; + }; +} diff --git a/modules/dream2nix/rust-cargo-vendor/interface.nix b/modules/dream2nix/rust-cargo-vendor/interface.nix new file mode 100644 index 00000000..fbdff70d --- /dev/null +++ b/modules/dream2nix/rust-cargo-vendor/interface.nix @@ -0,0 +1,35 @@ +{ + config, + lib, + ... +}: let + l = lib // builtins; + t = l.types; +in { + options.rust-cargo-vendor = l.mapAttrs (_: l.mkOption) { + vendoredSources = { + type = t.package; + description = "Path to vendored sources"; + }; + copyVendorDir = { + type = t.functionTo (t.functionTo t.str); + internal = true; + description = "Makes shell command(s) that copies the vendored sources correctly in a rust derivation"; + }; + getRootSource = { + type = t.functionTo (t.functionTo t.path); + internal = true; + description = "Gets root source for a given package"; + }; + writeGitVendorEntries = { + type = t.functionTo t.str; + internal = true; + description = "Makes shell command(s) that writes vendored git sources to .cargo/config so cargo uses the sources we vendored"; + }; + replaceRelativePathsWithAbsolute = { + type = t.functionTo t.str; + internal = true; + description = "Makes shell commands that will replace relative dependency paths with absolute paths in Cargo.toml"; + }; + }; +} diff --git a/modules/dream2nix/buildRustPackage/patch-workspace-deps.jq b/modules/dream2nix/rust-cargo-vendor/patch-workspace-deps.jq similarity index 100% rename from modules/dream2nix/buildRustPackage/patch-workspace-deps.jq rename to modules/dream2nix/rust-cargo-vendor/patch-workspace-deps.jq diff --git a/modules/dream2nix/buildRustPackage/vendor.nix b/modules/dream2nix/rust-cargo-vendor/vendor.nix similarity index 81% rename from modules/dream2nix/buildRustPackage/vendor.nix rename to modules/dream2nix/rust-cargo-vendor/vendor.nix index a9f43f92..e7bfff87 100644 --- a/modules/dream2nix/buildRustPackage/vendor.nix +++ b/modules/dream2nix/rust-cargo-vendor/vendor.nix @@ -1,7 +1,9 @@ { lib, + getRoot, getSource, getSourceSpec, + sourceRoot, subsystemAttrs, dreamLock, moreutils, @@ -145,4 +147,34 @@ in rec { echo "dream2nix: installing cargo vendor directory from ${from} to ${to}" cp -rs --no-preserve=mode,ownership ${from} ${to} ''; + + # Gets the root source for a package + getRootSource = pname: version: let + root = getRoot pname version; + in + getSource root.pname root.version; + + # Generates a script that replaces relative path dependency paths with absolute + # ones, if the path dependency isn't in the source dream2nix provides + replaceRelativePathsWithAbsolute = replacements: let + replace = + l.concatStringsSep + " \\\n" + ( + l.mapAttrsToList + ( + # TODO: this is not great, because it forces us to include the entire + # sourceRoot here, which could possibly cause more rebuilds than necessary + # when source is changed (although this mostly depends on how the project + # repository is structured). doing this properly is pretty complex, but + # it should still be done later. + from: relPath: ''--replace "\"${from}\"" "\"${sourceRoot}/${relPath}\""'' + ) + replacements + ); + in '' + echo "dream2nix: replacing relative dependency paths with absolute paths in Cargo.toml" + substituteInPlace ./Cargo.toml \ + ${replace} + ''; } diff --git a/modules/dream2nix/rust-crane/default.nix b/modules/dream2nix/rust-crane/default.nix index 128df4da..70e8b446 100644 --- a/modules/dream2nix/rust-crane/default.nix +++ b/modules/dream2nix/rust-crane/default.nix @@ -10,108 +10,36 @@ dreamLock = config.rust-cargo-lock.dreamLock; - sourceRoot = config.mkDerivation.src; - - fetchDreamLockSources = - import ../../../lib/internal/fetchDreamLockSources.nix - {inherit lib;}; - getDreamLockSource = import ../../../lib/internal/getDreamLockSource.nix {inherit lib;}; readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;}; - hashPath = import ../../../lib/internal/hashPath.nix { - inherit lib; - inherit (config.deps) runCommandLocal nix; - }; - hashFile = import ../../../lib/internal/hashFile.nix { - inherit lib; - inherit (config.deps) runCommandLocal nix; - }; - - # fetchers - fetchers = { - git = import ../../../lib/internal/fetchers/git { - inherit hashPath; - inherit (config.deps) fetchgit; - }; - crates-io = import ../../../lib/internal/fetchers/crates-io { - inherit hashFile; - inherit (config.deps) fetchurl runCommandLocal; - }; - path = import ../../../lib/internal/fetchers/path { - inherit hashPath; - }; - }; dreamLockLoaded = readDreamLock {inherit dreamLock;}; dreamLockInterface = dreamLockLoaded.interface; - inherit (dreamLockInterface) defaultPackageName defaultPackageVersion; - - fetchedSources' = fetchDreamLockSources { - inherit defaultPackageName defaultPackageVersion; - inherit (dreamLockLoaded.lock) sources; - inherit fetchers; - }; - - fetchedSources = - fetchedSources' + meta = let + meta = dreamLockInterface.subsystemAttrs.meta.${pname}.${version}; + in + meta // { - ${defaultPackageName}.${defaultPackageVersion} = sourceRoot; + license = l.map (name: l.licenses.${name}) meta.license; }; - getSource = getDreamLockSource fetchedSources; - - toTOML = import ../../../lib/internal/toTOML.nix {inherit lib;}; - - utils = import ./utils.nix { - inherit dreamLock getSource lib toTOML sourceRoot; - inherit - (dreamLockInterface) - getSourceSpec - getRoot - subsystemAttrs - packages - ; - inherit - (config.deps) - writeText - ; - }; - _crane = import config.deps.craneSource { pkgs = config.deps.cranePkgs; }; crane = _crane.overrideToolchain config.deps.mkRustToolchain; rustToolchain = config.deps.mkRustToolchain config.deps.cranePkgs; - vendoring = import ./vendor.nix { - inherit dreamLock getSource lib; - inherit - (dreamLockInterface) - getSourceSpec - subsystemAttrs - ; - inherit - (config.deps) - jq - moreutils - python3Packages - runCommandLocal - writePython3 - ; - cargo = rustToolchain; - }; - pname = config.name; version = config.version; replacePaths = - utils.replaceRelativePathsWithAbsolute + config.rust-cargo-vendor.replaceRelativePathsWithAbsolute dreamLockInterface.subsystemAttrs.relPathReplacements.${pname}.${version}; - writeGitVendorEntries = vendoring.writeGitVendorEntries "nix-sources"; + writeGitVendorEntries = config.rust-cargo-vendor.writeGitVendorEntries "nix-sources"; # common args we use for both buildDepsOnly and buildPackage common = { - src = lib.mkForce (utils.getRootSource pname version); + src = lib.mkForce (config.rust-cargo-vendor.getRootSource pname version); postUnpack = '' export CARGO_HOME=$(pwd)/.cargo_home export cargoVendorDir="$TMPDIR/nix-vendor" @@ -140,7 +68,7 @@ depsNameSuffix = "-deps"; depsArgs = { preUnpack = '' - ${vendoring.copyVendorDir "$dream2nixVendorDir" common.cargoVendorDir} + ${config.rust-cargo-vendor.copyVendorDir "$dream2nixVendorDir" common.cargoVendorDir} ''; # move the vendored dependencies folder to $out for main derivation to use postInstall = '' @@ -148,25 +76,25 @@ ''; # we pass cargoLock path to buildDepsOnly # so that crane's mkDummySrc adds it to the dummy source - inherit (utils) cargoLock; + inherit (config.rust-cargo-lock) cargoLock; pname = l.mkOverride 99 pname; pnameSuffix = depsNameSuffix; # Make sure cargo only checks the package we want cargoCheckCommand = "cargo check \${cargoBuildFlags:-} --profile \${cargoBuildProfile} --package ${pname}"; - dream2nixVendorDir = vendoring.vendoredDependencies; + dream2nixVendorDir = config.rust-cargo-vendor.vendoredSources; }; buildArgs = { # link the vendor dir we used earlier to the correct place preUnpack = '' - ${vendoring.copyVendorDir "$cargoArtifacts/nix-vendor" common.cargoVendorDir} + ${config.rust-cargo-vendor.copyVendorDir "$cargoArtifacts/nix-vendor" common.cargoVendorDir} ''; # write our cargo lock # note: we don't do this in buildDepsOnly since # that uses a cargoLock argument instead preConfigure = l.mkForce '' ${common.preConfigure} - ${utils.writeCargoLock} + ${config.rust-cargo-lock.writeCargoLock} ''; cargoArtifacts = cfg.depsDrv.public; }; @@ -200,11 +128,15 @@ in { cargo = rustToolchain; }; dependencies = cfg.depsDrv.public; - meta = (utils.getMeta pname version) // config.mkDerivation.meta; + meta = meta // config.mkDerivation.meta; }; deps = {nixpkgs, ...}: l.mkMerge [ + { + # override cargo package to be the rust toolchain so that rust-cargo-vendor uses the custom provided toolchain if any + cargo = l.mkOverride 1001 rustToolchain; + } (l.mapAttrs (_: l.mkDefault) { craneSource = config.deps.fetchFromGitHub { owner = "ipetkov"; @@ -215,24 +147,12 @@ in { cranePkgs = nixpkgs.pkgs; mkRustToolchain = pkgs: pkgs.cargo; }) - # maybe it would be better to put these under `options.rust-crane.deps` instead of this `deps` - # since it conflicts with a lot of stuff? (l.mapAttrs (_: l.mkOverride 999) { inherit (nixpkgs) - fetchurl - jq - moreutils - python3Packages - runCommandLocal - writeText - fetchFromGitHub - libiconv mkShell - ; - inherit - (nixpkgs.writers) - writePython3 + libiconv + fetchFromGitHub ; }) ]; diff --git a/modules/dream2nix/rust-crane/patch-workspace-deps.jq b/modules/dream2nix/rust-crane/patch-workspace-deps.jq deleted file mode 100644 index 0eb57d74..00000000 --- a/modules/dream2nix/rust-crane/patch-workspace-deps.jq +++ /dev/null @@ -1,48 +0,0 @@ -def normalizeWorkspaceDep: - if ($workspaceDependencies."\(.key)" | type) == "object" - then [.value, $workspaceDependencies."\(.key)"] | add - else [.value, {"version":$workspaceDependencies."\(.key)"}] | add - end - # remove workspace option from the dependency - | del(.workspace) -; - -# normalizes workspace inherited dependencies for one list -def mapWorkspaceDepsFor(name): - if has(name) - then - ."\(name)" = ( - ."\(name)" - | to_entries - | map( - if (.value | type) == "object" and .value.workspace == true - then .value = (. | normalizeWorkspaceDep) - else . - end - ) - | from_entries - ) - else . - end -; - -# shorthand for normalizing all the dependencies list -def mapWorkspaceDeps: - mapWorkspaceDepsFor("dependencies") - | mapWorkspaceDepsFor("dev-dependencies") - | mapWorkspaceDepsFor("build-dependencies") -; - -# normalize workspace inherited deps -mapWorkspaceDeps -| if has("target") - then - # normalize workspace inherited deps in target specific deps - .target = ( - .target - | to_entries - | map(.value = (.value | mapWorkspaceDeps)) - | from_entries - ) - else . - end diff --git a/modules/dream2nix/rust-crane/utils.nix b/modules/dream2nix/rust-crane/utils.nix deleted file mode 100644 index b9d7631d..00000000 --- a/modules/dream2nix/rust-crane/utils.nix +++ /dev/null @@ -1,191 +0,0 @@ -{ - dreamLock, - getSourceSpec, - getSource, - getRoot, - sourceRoot, - subsystemAttrs, - packages, - lib, - toTOML, - writeText, - ... -}: let - l = lib // builtins; - isInPackages = name: version: (packages.${name} or null) == version; -in rec { - getMeta = pname: version: let - meta = subsystemAttrs.meta.${pname}.${version}; - in - meta - // { - license = l.map (name: l.licenses.${name}) meta.license; - }; - - # Gets the root source for a package - getRootSource = pname: version: let - root = getRoot pname version; - in - getSource root.pname root.version; - - # Generates a script that replaces relative path dependency paths with absolute - # ones, if the path dependency isn't in the source dream2nix provides - replaceRelativePathsWithAbsolute = replacements: let - replace = - l.concatStringsSep - " \\\n" - ( - l.mapAttrsToList - ( - # TODO: this is not great, because it forces us to include the entire - # sourceRoot here, which could possibly cause more rebuilds than necessary - # when source is changed (although this mostly depends on how the project - # repository is structured). doing this properly is pretty complex, but - # it should still be done later. - from: relPath: ''--replace "\"${from}\"" "\"${sourceRoot}/${relPath}\""'' - ) - replacements - ); - in '' - echo "dream2nix: replacing relative dependency paths with absolute paths in Cargo.toml" - substituteInPlace ./Cargo.toml \ - ${replace} - ''; - - # Backup original Cargo.lock if it exists and write our own one - writeCargoLock = '' - echo "dream2nix: replacing Cargo.lock with ${cargoLock}" - mv -f Cargo.lock Cargo.lock.orig || echo "dream2nix: no Cargo.lock was found beforehand" - cat ${cargoLock} > Cargo.lock - ''; - - # The Cargo.lock for this dreamLock. - cargoLock = let - mkPkgEntry = { - name, - version, - ... - } @ args: let - # constructs source string for dependency - makeSource = sourceSpec: let - source = - if sourceSpec.type == "crates-io" - then "registry+https://github.com/rust-lang/crates.io-index" - else if sourceSpec.type == "git" - then let - gitSpec = - l.findFirst - (src: src.url == sourceSpec.url && src.sha == sourceSpec.rev) - (throw "no git source: ${sourceSpec.url}#${sourceSpec.rev}") - (subsystemAttrs.gitSources or {}); - refPart = - l.optionalString - (gitSpec ? type) - "?${gitSpec.type}=${gitSpec.value}"; - in "git+${sourceSpec.url}${refPart}#${sourceSpec.rev}" - else null; - in - source; - # constructs source string for dependency entry - makeDepSource = sourceSpec: - if sourceSpec.type == "crates-io" - then makeSource sourceSpec - else if sourceSpec.type == "git" - then l.concatStringsSep "#" (l.init (l.splitString "#" (makeSource sourceSpec))) - else null; - # removes source type information from the version - normalizeVersion = version: srcType: l.removeSuffix ("$" + srcType) version; - - sourceSpec = getSourceSpec name version; - - normalizedVersion = normalizeVersion version sourceSpec.type; - - source = let - src = makeSource sourceSpec; - in - if src == null - then throw "source type '${sourceSpec.type}' not supported" - else src; - dependencies = - l.map - ( - dep: let - depSourceSpec = getSourceSpec dep.name dep.version; - depSource = makeDepSource depSourceSpec; - - normalizedDepVersion = normalizeVersion dep.version depSourceSpec.type; - - hasMultipleVersions = - l.length (l.attrValues dreamLock.sources.${dep.name}) > 1; - hasDuplicateVersions = dep.version != normalizedDepVersion; - - # only put version if there are different versions of the dep - versionString = - l.optionalString hasMultipleVersions " ${normalizedDepVersion}"; - # only put source if there are duplicate versions of the dep - # cargo vendor does not support this anyway and so builds will fail - # until https://github.com/rust-lang/cargo/issues/10310 is resolved. - srcString = - l.optionalString hasDuplicateVersions " (${depSource})"; - in "${dep.name}${versionString}${srcString}" - ) - args.dependencies; - - isMainPackage = isInPackages name version; - in - { - name = sourceSpec.pname or name; - version = sourceSpec.version or normalizedVersion; - } - # put dependencies like how cargo expects them - // ( - l.optionalAttrs - (l.length dependencies > 0) - {inherit dependencies;} - ) - // ( - l.optionalAttrs - (sourceSpec.type != "path" && !isMainPackage) - {inherit source;} - ) - // ( - l.optionalAttrs - (sourceSpec.type == "crates-io" && !isMainPackage) - {checksum = sourceSpec.hash;} - ); - _package = l.flatten ( - l.mapAttrsToList - ( - name: versions: - l.mapAttrsToList - ( - version: dependencies: - mkPkgEntry {inherit name version dependencies;} - ) - versions - ) - dreamLock.dependencies - ); - package = - ( - # add packages as dependencies because Cargo expects them to be there aswell - l.filter - (pkg: ! l.any (opkg: pkg.name == opkg.name && pkg.version == opkg.version) _package) - ( - l.mapAttrsToList - (pname: version: { - name = pname; - inherit version; - }) - dreamLock._generic.packages - ) - ) - ++ _package; - lockTOML = toTOML { - # the lockfile we generate is of version 3 - version = 3; - inherit package; - }; - in - writeText "Cargo.lock" lockTOML; -} diff --git a/modules/dream2nix/rust-crane/vendor.nix b/modules/dream2nix/rust-crane/vendor.nix deleted file mode 100644 index a9f43f92..00000000 --- a/modules/dream2nix/rust-crane/vendor.nix +++ /dev/null @@ -1,148 +0,0 @@ -{ - lib, - getSource, - getSourceSpec, - subsystemAttrs, - dreamLock, - moreutils, - writePython3, - python3Packages, - runCommandLocal, - ... -} @ args: let - l = lib // builtins; - - allDependencies = - l.flatten - ( - l.mapAttrsToList - ( - name: versions: - l.map (version: {inherit name version;}) (l.attrNames versions) - ) - dreamLock.dependencies - ); -in rec { - # Generates a shell script that writes git vendor entries to .cargo/config. - # `replaceWith` is the name of the vendored source(s) to use. - writeGitVendorEntries = replaceWith: let - makeEntry = source: '' - [source."${source.url}${l.optionalString (source ? type) "?${source.type}=${source.value}"}"] - replace-with = "${replaceWith}" - git = "${source.url}" - ${l.optionalString (source ? type) "${source.type} = \"${source.value}\""} - ''; - entries = l.map makeEntry subsystemAttrs.gitSources; - in '' - echo "dream2nix: Writing git vendor entries to $CARGO_HOME/config.toml" - mkdir -p $CARGO_HOME && touch $CARGO_HOME/config.toml - cat >> $CARGO_HOME/config.toml <&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree" - exit 1 - fi - else - # we need to patch dependencies with `workspace = true` (workspace inheritance) - workspaceDependencies="$(cat "$tree/Cargo.toml" | ${tomlToJson} | ${jq} -cr '.workspace.dependencies')" - if [[ "$workspaceDependencies" != "null" ]]; then - tree="$(pwd)/${pkg.name}-${pkg.version}" - cp -prd --no-preserve=mode,ownership "$(dirname $crateCargoTOML)" "$tree" - crateCargoTOML="$tree/Cargo.toml" - cat "$crateCargoTOML" \ - | ${tomlToJson} \ - | ${jq} -cr --argjson workspaceDependencies "$workspaceDependencies" \ - --from-file ${./patch-workspace-deps.jq} \ - | ${jsonToToml} \ - | ${sponge} "$crateCargoTOML" - fi - fi - echo Found crate ${pkg.name} at $crateCargoTOML - tree="$(dirname $crateCargoTOML)" - ''; - makeScript = source: let - isGit = source.spec.type == "git"; - isPath = source.spec.type == "path"; - in - l.optionalString (!isPath) '' - tree="${source.path}" - ${l.optionalString isGit (findCrateSource source)} - echo Vendoring crate ${source.name} - if [ -d $out/${source.name} ]; then - echo Crate is already vendored - echo Crates with duplicate versions cannot be vendored as Cargo does not support this behaviour - exit 1 - else - cp -prd "$tree" $out/${source.name} - chmod u+w $out/${source.name} - ${l.optionalString isGit "printf '{\"files\":{},\"package\":null}' > $out/${source.name}/.cargo-checksum.json"} - fi - ''; - in - runCommandLocal "vendor" {} '' - mkdir -p $out - - ${ - l.concatMapStringsSep "\n" - makeScript - sources - } - ''; - - # All dependencies in the Cargo.lock file, vendored - vendoredDependencies = vendorDependencies allDependencies; - - copyVendorDir = from: to: '' - echo "dream2nix: installing cargo vendor directory from ${from} to ${to}" - cp -rs --no-preserve=mode,ownership ${from} ${to} - ''; -}