refactor(rust): deduplicate vendor and utils code (#1054)

* refactor(rust): consolidate the separated vendoring code into rust-cargo-vendor

* refactor(rust): move the utility code in utils.nix to rust-cargo-lock and rust-cargo-vendor

* fix(rust-crane): add the necessary dependencies to config.deps for devshell

* refactor(rust-cargo-vendor): remove unnecessary mkMerge

* fix(rust-crane): override cargo in config.deps with our toolchain so that rust-cargo-vendor uses it too

examples/packages/languages/rust-local-development-workspace/default.nix

@@ -6,6 +6,7 @@
 }: {
   imports = [
     dream2nix.modules.dream2nix.rust-cargo-lock
+    dream2nix.modules.dream2nix.rust-cargo-vendor
     dream2nix.modules.dream2nix.rust-crane
   ];
 

examples/packages/languages/rust-packaging-buildRustPackage/default.nix

@@ -6,6 +6,7 @@
 }: {
   imports = [
     dream2nix.modules.dream2nix.rust-cargo-lock
+    dream2nix.modules.dream2nix.rust-cargo-vendor
     dream2nix.modules.dream2nix.buildRustPackage
   ];
 

examples/packages/languages/rust-packaging/default.nix

@@ -6,6 +6,7 @@
 }: {
   imports = [
     dream2nix.modules.dream2nix.rust-cargo-lock
+    dream2nix.modules.dream2nix.rust-cargo-vendor
     dream2nix.modules.dream2nix.rust-crane
   ];
 

modules/dream2nix/buildRustPackage/default.nix

@@ -3,133 +3,52 @@
   lib,
   dream2nix,
   ...
-} @ topArgs: let
+}: let
   l = lib // builtins;
 
-  dreamLock = config.rust-cargo-lock.dreamLock;
-
-  sourceRoot = config.mkDerivation.src;
-
-  fetchDreamLockSources =
-    import ../../../lib/internal/fetchDreamLockSources.nix
-    {inherit lib;};
-  getDreamLockSource = import ../../../lib/internal/getDreamLockSource.nix {inherit lib;};
   readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;};
-  hashPath = import ../../../lib/internal/hashPath.nix {
-    inherit lib;
-    inherit (config.deps) runCommandLocal nix;
-  };
-  hashFile = import ../../../lib/internal/hashFile.nix {
-    inherit lib;
-    inherit (config.deps) runCommandLocal nix;
-  };
-
-  # fetchers
-  fetchers = {
-    git = import ../../../lib/internal/fetchers/git {
-      inherit hashPath;
-      inherit (config.deps) fetchgit;
-    };
-    http = import ../../../lib/internal/fetchers/http {
-      inherit hashFile lib;
-      inherit (config.deps.stdenv) mkDerivation;
-      inherit (config.deps) fetchurl;
-    };
-    crates-io = import ../../../lib/internal/fetchers/crates-io {
-      inherit hashFile;
-      inherit (config.deps) fetchurl runCommandLocal;
-    };
-  };
 
   dreamLockLoaded =
     readDreamLock {inherit (config.rust-cargo-lock) dreamLock;};
   dreamLockInterface = dreamLockLoaded.interface;
 
-  fetchedSources' = fetchDreamLockSources {
-    inherit (dreamLockInterface) defaultPackageName defaultPackageVersion;
-    inherit (dreamLockLoaded.lock) sources;
-    inherit fetchers;
-  };
-
-  fetchedSources =
-    fetchedSources'
-    // {
-      ${defaultPackageName}.${defaultPackageVersion} = sourceRoot;
-    };
-
-  # name: version: -> store-path
-  getSource = getDreamLockSource fetchedSources;
-
   inherit
     (dreamLockInterface)
-    getDependencies # name: version: -> [ {name=; version=; } ]
-    # Attributes
-    
     subsystemAttrs # attrset
-    packageVersions
-    defaultPackageName
-    defaultPackageVersion
     ;
 
-  toTOML = import ../../../lib/internal/toTOML.nix {inherit lib;};
+  meta = let
-
+    meta = subsystemAttrs.meta.${pname}.${version};
-  utils = import ./utils.nix {
+  in
-    inherit dreamLock getSource lib toTOML sourceRoot;
+    meta
-    inherit
+    // {
-      (dreamLockInterface)
+      license = l.map (name: l.licenses.${name}) meta.license;
-      getSourceSpec
-      getRoot
-      subsystemAttrs
-      packages
-      ;
-    inherit
-      (config.deps)
-      writeText
-      ;
-  };
-
-  vendoring = import ./vendor.nix {
-    inherit dreamLock getSource lib;
-    inherit
-      (dreamLockInterface)
-      getSourceSpec
-      subsystemAttrs
-      ;
-    inherit
-      (config.deps)
-      cargo
-      jq
-      moreutils
-      python3Packages
-      runCommandLocal
-      writePython3
-      ;
     };
 
   pname = config.name;
   version = config.version;
 
-  src = utils.getRootSource pname version;
+  src = config.rust-cargo-vendor.getRootSource pname version;
   replacePaths =
-    utils.replaceRelativePathsWithAbsolute
+    config.rust-cargo-vendor.replaceRelativePathsWithAbsolute
     subsystemAttrs.relPathReplacements.${pname}.${version};
-  writeGitVendorEntries = vendoring.writeGitVendorEntries "vendored-sources";
+  writeGitVendorEntries = config.rust-cargo-vendor.writeGitVendorEntries "vendored-sources";
 
   cargoBuildFlags = "--package ${pname}";
   buildArgs = {
     inherit pname version;
     src = lib.mkForce src;
 
-    meta = utils.getMeta pname version;
+    inherit meta;
 
     cargoBuildFlags = cargoBuildFlags;
     cargoTestFlags = cargoBuildFlags;
 
     cargoVendorDir = "../nix-vendor";
-    dream2nixVendorDir = vendoring.vendoredDependencies;
+    dream2nixVendorDir = config.rust-cargo-vendor.vendoredSources;
 
     postUnpack = ''
-      ${vendoring.copyVendorDir "$dream2nixVendorDir" "./nix-vendor"}
+      ${config.rust-cargo-vendor.copyVendorDir "$dream2nixVendorDir" "./nix-vendor"}
       export CARGO_HOME=$(pwd)/.cargo_home
     '';
 
@@ -140,7 +59,7 @@
       fi
       ${writeGitVendorEntries}
       ${replacePaths}
-      ${utils.writeCargoLock}
+      ${config.rust-cargo-lock.writeCargoLock}
     '';
   };
 in {
@@ -154,24 +73,13 @@ in {
   package-func.args = buildArgs;
 
   public = {
-    meta = utils.getMeta pname version;
+    inherit meta;
   };
 
   deps = {nixpkgs, ...}: {
     inherit
       (nixpkgs)
-      cargo
-      fetchurl
-      jq
-      moreutils
-      python3Packages
-      runCommandLocal
       rustPlatform
-      writeText
-      ;
-    inherit
-      (nixpkgs.writers)
-      writePython3
       ;
   };
 }

modules/dream2nix/buildRustPackage/utils.nix

@@ -1,191 +0,0 @@
-{
-  dreamLock,
-  getSourceSpec,
-  getSource,
-  getRoot,
-  sourceRoot,
-  subsystemAttrs,
-  packages,
-  lib,
-  toTOML,
-  writeText,
-  ...
-}: let
-  l = lib // builtins;
-  isInPackages = name: version: (packages.${name} or null) == version;
-in rec {
-  getMeta = pname: version: let
-    meta = subsystemAttrs.meta.${pname}.${version};
-  in
-    meta
-    // {
-      license = l.map (name: l.licenses.${name}) meta.license;
-    };
-
-  # Gets the root source for a package
-  getRootSource = pname: version: let
-    root = getRoot pname version;
-  in
-    getSource root.pname root.version;
-
-  # Generates a script that replaces relative path dependency paths with absolute
-  # ones, if the path dependency isn't in the source dream2nix provides
-  replaceRelativePathsWithAbsolute = replacements: let
-    replace =
-      l.concatStringsSep
-      " \\\n"
-      (
-        l.mapAttrsToList
-        (
-          # TODO: this is not great, because it forces us to include the entire
-          # sourceRoot here, which could possibly cause more rebuilds than necessary
-          # when source is changed (although this mostly depends on how the project
-          # repository is structured). doing this properly is pretty complex, but
-          # it should still be done later.
-          from: relPath: ''--replace "\"${from}\"" "\"${sourceRoot}/${relPath}\""''
-        )
-        replacements
-      );
-  in ''
-    echo "dream2nix: replacing relative dependency paths with absolute paths in Cargo.toml"
-    substituteInPlace ./Cargo.toml \
-      ${replace}
-  '';
-
-  # Backup original Cargo.lock if it exists and write our own one
-  writeCargoLock = ''
-    echo "dream2nix: replacing Cargo.lock with ${cargoLock}"
-    mv -f Cargo.lock Cargo.lock.orig || echo "dream2nix: no Cargo.lock was found beforehand"
-    cat ${cargoLock} > Cargo.lock
-  '';
-
-  # The Cargo.lock for this dreamLock.
-  cargoLock = let
-    mkPkgEntry = {
-      name,
-      version,
-      ...
-    } @ args: let
-      # constructs source string for dependency
-      makeSource = sourceSpec: let
-        source =
-          if sourceSpec.type == "crates-io"
-          then "registry+https://github.com/rust-lang/crates.io-index"
-          else if sourceSpec.type == "git"
-          then let
-            gitSpec =
-              l.findFirst
-              (src: src.url == sourceSpec.url && src.sha == sourceSpec.rev)
-              (throw "no git source: ${sourceSpec.url}#${sourceSpec.rev}")
-              (subsystemAttrs.gitSources or {});
-            refPart =
-              l.optionalString
-              (gitSpec ? type)
-              "?${gitSpec.type}=${gitSpec.value}";
-          in "git+${sourceSpec.url}${refPart}#${sourceSpec.rev}"
-          else null;
-      in
-        source;
-      # constructs source string for dependency entry
-      makeDepSource = sourceSpec:
-        if sourceSpec.type == "crates-io"
-        then makeSource sourceSpec
-        else if sourceSpec.type == "git"
-        then l.concatStringsSep "#" (l.init (l.splitString "#" (makeSource sourceSpec)))
-        else null;
-      # removes source type information from the version
-      normalizeVersion = version: srcType: l.removeSuffix ("$" + srcType) version;
-
-      sourceSpec = getSourceSpec name version;
-
-      normalizedVersion = normalizeVersion version sourceSpec.type;
-
-      source = let
-        src = makeSource sourceSpec;
-      in
-        if src == null
-        then throw "source type '${sourceSpec.type}' not supported"
-        else src;
-      dependencies =
-        l.map
-        (
-          dep: let
-            depSourceSpec = getSourceSpec dep.name dep.version;
-            depSource = makeDepSource depSourceSpec;
-
-            normalizedDepVersion = normalizeVersion dep.version depSourceSpec.type;
-
-            hasMultipleVersions =
-              l.length (l.attrValues dreamLock.sources.${dep.name}) > 1;
-            hasDuplicateVersions = dep.version != normalizedDepVersion;
-
-            # only put version if there are different versions of the dep
-            versionString =
-              l.optionalString hasMultipleVersions " ${normalizedDepVersion}";
-            # only put source if there are duplicate versions of the dep
-            # cargo vendor does not support this anyway and so builds will fail
-            # until https://github.com/rust-lang/cargo/issues/10310 is resolved.
-            srcString =
-              l.optionalString hasDuplicateVersions " (${depSource})";
-          in "${dep.name}${versionString}${srcString}"
-        )
-        args.dependencies;
-
-      isMainPackage = isInPackages name version;
-    in
-      {
-        name = sourceSpec.pname or name;
-        version = sourceSpec.version or normalizedVersion;
-      }
-      # put dependencies like how cargo expects them
-      // (
-        l.optionalAttrs
-        (l.length dependencies > 0)
-        {inherit dependencies;}
-      )
-      // (
-        l.optionalAttrs
-        (sourceSpec.type != "path" && !isMainPackage)
-        {inherit source;}
-      )
-      // (
-        l.optionalAttrs
-        (sourceSpec.type == "crates-io" && !isMainPackage)
-        {checksum = sourceSpec.hash;}
-      );
-    _package = l.flatten (
-      l.mapAttrsToList
-      (
-        name: versions:
-          l.mapAttrsToList
-          (
-            version: dependencies:
-              mkPkgEntry {inherit name version dependencies;}
-          )
-          versions
-      )
-      dreamLock.dependencies
-    );
-    package =
-      (
-        # add packages as dependencies because Cargo expects them to be there aswell
-        l.filter
-        (pkg: ! l.any (opkg: pkg.name == opkg.name && pkg.version == opkg.version) _package)
-        (
-          l.mapAttrsToList
-          (pname: version: {
-            name = pname;
-            inherit version;
-          })
-          dreamLock._generic.packages
-        )
-      )
-      ++ _package;
-    lockTOML = toTOML {
-      # the lockfile we generate is of version 3
-      version = 3;
-      inherit package;
-    };
-  in
-    writeText "Cargo.lock" lockTOML;
-}

modules/dream2nix/rust-cargo-lock/cargoLock.nix

@@ -0,0 +1,152 @@
+{
+  lib,
+  dreamLock,
+  # nixpkgs
+  writeText,
+}:
+# The Cargo.lock for this dreamLock.
+let
+  l = lib // builtins;
+
+  readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;};
+
+  dreamLockLoaded = readDreamLock {inherit dreamLock;};
+  dreamLockInterface = dreamLockLoaded.interface;
+
+  inherit
+    (dreamLockInterface)
+    getSourceSpec
+    subsystemAttrs
+    packages
+    ;
+
+  toTOML = import ../../../lib/internal/toTOML.nix {inherit lib;};
+
+  isInPackages = name: version: (packages.${name} or null) == version;
+  mkPkgEntry = {
+    name,
+    version,
+    ...
+  } @ args: let
+    # constructs source string for dependency
+    makeSource = sourceSpec: let
+      source =
+        if sourceSpec.type == "crates-io"
+        then "registry+https://github.com/rust-lang/crates.io-index"
+        else if sourceSpec.type == "git"
+        then let
+          gitSpec =
+            l.findFirst
+            (src: src.url == sourceSpec.url && src.sha == sourceSpec.rev)
+            (throw "no git source: ${sourceSpec.url}#${sourceSpec.rev}")
+            (subsystemAttrs.gitSources or {});
+          refPart =
+            l.optionalString
+            (gitSpec ? type)
+            "?${gitSpec.type}=${gitSpec.value}";
+        in "git+${sourceSpec.url}${refPart}#${sourceSpec.rev}"
+        else null;
+    in
+      source;
+    # constructs source string for dependency entry
+    makeDepSource = sourceSpec:
+      if sourceSpec.type == "crates-io"
+      then makeSource sourceSpec
+      else if sourceSpec.type == "git"
+      then l.concatStringsSep "#" (l.init (l.splitString "#" (makeSource sourceSpec)))
+      else null;
+    # removes source type information from the version
+    normalizeVersion = version: srcType: l.removeSuffix ("$" + srcType) version;
+
+    sourceSpec = getSourceSpec name version;
+
+    normalizedVersion = normalizeVersion version sourceSpec.type;
+
+    source = let
+      src = makeSource sourceSpec;
+    in
+      if src == null
+      then throw "source type '${sourceSpec.type}' not supported"
+      else src;
+    dependencies =
+      l.map
+      (
+        dep: let
+          depSourceSpec = getSourceSpec dep.name dep.version;
+          depSource = makeDepSource depSourceSpec;
+
+          normalizedDepVersion = normalizeVersion dep.version depSourceSpec.type;
+
+          hasMultipleVersions =
+            l.length (l.attrValues dreamLock.sources.${dep.name}) > 1;
+          hasDuplicateVersions = dep.version != normalizedDepVersion;
+
+          # only put version if there are different versions of the dep
+          versionString =
+            l.optionalString hasMultipleVersions " ${normalizedDepVersion}";
+          # only put source if there are duplicate versions of the dep
+          # cargo vendor does not support this anyway and so builds will fail
+          # until https://github.com/rust-lang/cargo/issues/10310 is resolved.
+          srcString =
+            l.optionalString hasDuplicateVersions " (${depSource})";
+        in "${dep.name}${versionString}${srcString}"
+      )
+      args.dependencies;
+
+    isMainPackage = isInPackages name version;
+  in
+    {
+      name = sourceSpec.pname or name;
+      version = sourceSpec.version or normalizedVersion;
+    }
+    # put dependencies like how cargo expects them
+    // (
+      l.optionalAttrs
+      (l.length dependencies > 0)
+      {inherit dependencies;}
+    )
+    // (
+      l.optionalAttrs
+      (sourceSpec.type != "path" && !isMainPackage)
+      {inherit source;}
+    )
+    // (
+      l.optionalAttrs
+      (sourceSpec.type == "crates-io" && !isMainPackage)
+      {checksum = sourceSpec.hash;}
+    );
+  _package = l.flatten (
+    l.mapAttrsToList
+    (
+      name: versions:
+        l.mapAttrsToList
+        (
+          version: dependencies:
+            mkPkgEntry {inherit name version dependencies;}
+        )
+        versions
+    )
+    dreamLock.dependencies
+  );
+  package =
+    (
+      # add packages as dependencies because Cargo expects them to be there aswell
+      l.filter
+      (pkg: ! l.any (opkg: pkg.name == opkg.name && pkg.version == opkg.version) _package)
+      (
+        l.mapAttrsToList
+        (pname: version: {
+          name = pname;
+          inherit version;
+        })
+        dreamLock._generic.packages
+      )
+    )
+    ++ _package;
+  lockTOML = toTOML {
+    # the lockfile we generate is of version 3
+    version = 3;
+    inherit package;
+  };
+in
+  writeText "Cargo.lock" lockTOML

modules/dream2nix/rust-cargo-lock/default.nix

@@ -20,12 +20,38 @@
     projectRelPath = "";
     tree = prepareSourceTree {source = cfg.source;};
   };
+
+  cargoLock = import ./cargoLock.nix {
+    inherit lib;
+    inherit (cfg) dreamLock;
+    inherit (config.deps) writeText;
+  };
+
+  # Backup original Cargo.lock if it exists and write our own one
+  writeCargoLock = ''
+    echo "dream2nix: replacing Cargo.lock with ${cfg.cargoLock}"
+    mv -f Cargo.lock Cargo.lock.orig || echo "dream2nix: no Cargo.lock was found beforehand"
+    cat ${cfg.cargoLock} > Cargo.lock
+  '';
 in {
   imports = [
     ./interface.nix
-    dream2nix.modules.dream2nix.mkDerivation
+    dream2nix.modules.dream2nix.core
   ];
+
   rust-cargo-lock = {
-    inherit dreamLock;
+    inherit
+      cargoLock
+      dreamLock
+      writeCargoLock
+      ;
+  };
+
+  deps = {nixpkgs, ...}:
+    l.mapAttrs (_: l.mkOverride 997) {
+      inherit
+        (nixpkgs)
+        writeText
+        ;
     };
 }

modules/dream2nix/rust-cargo-lock/interface.nix

@@ -7,11 +7,21 @@
   t = l.types;
 in {
   options.rust-cargo-lock = l.mapAttrs (_: l.mkOption) {
+    cargoLock = {
+      type = t.path;
+      internal = true;
+      description = "The dreamlock that was generated as a Cargo.lock file";
+    };
     dreamLock = {
       type = t.attrs;
       internal = true;
       description = "The content of the dream2nix generated lock file";
     };
+    writeCargoLock = {
+      type = t.str;
+      internal = true;
+      description = "Shell commands to backup original Cargo.lock and use dream2nix one in a rust derivation";
+    };
     source = {
       type = t.either t.path t.package;
       description = "Source of the package";

modules/dream2nix/rust-cargo-vendor/README.md

@@ -0,0 +1,6 @@
+---
+title: "rust-cargo-vendor"
+state: "experimental"
+maintainers:
+  - DavHau
+---

modules/dream2nix/rust-cargo-vendor/default.nix

@@ -0,0 +1,115 @@
+{
+  config,
+  lib,
+  dream2nix,
+  ...
+}: let
+  l = lib // builtins;
+  cfg = config.rust-cargo-vendor;
+
+  dreamLock = config.rust-cargo-lock.dreamLock;
+
+  sourceRoot = config.mkDerivation.src;
+
+  fetchDreamLockSources =
+    import ../../../lib/internal/fetchDreamLockSources.nix
+    {inherit lib;};
+  getDreamLockSource = import ../../../lib/internal/getDreamLockSource.nix {inherit lib;};
+  readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;};
+  hashPath = import ../../../lib/internal/hashPath.nix {
+    inherit lib;
+    inherit (config.deps) runCommandLocal nix;
+  };
+  hashFile = import ../../../lib/internal/hashFile.nix {
+    inherit lib;
+    inherit (config.deps) runCommandLocal nix;
+  };
+
+  # fetchers
+  fetchers = {
+    git = import ../../../lib/internal/fetchers/git {
+      inherit hashPath;
+      inherit (config.deps) fetchgit;
+    };
+    crates-io = import ../../../lib/internal/fetchers/crates-io {
+      inherit hashFile;
+      inherit (config.deps) fetchurl runCommandLocal;
+    };
+    path = import ../../../lib/internal/fetchers/path {
+      inherit hashPath;
+    };
+  };
+
+  dreamLockLoaded = readDreamLock {inherit dreamLock;};
+  dreamLockInterface = dreamLockLoaded.interface;
+
+  inherit (dreamLockInterface) defaultPackageName defaultPackageVersion;
+
+  fetchedSources' = fetchDreamLockSources {
+    inherit defaultPackageName defaultPackageVersion;
+    inherit (dreamLockLoaded.lock) sources;
+    inherit fetchers;
+  };
+
+  fetchedSources =
+    fetchedSources'
+    // {
+      ${defaultPackageName}.${defaultPackageVersion} = sourceRoot;
+    };
+
+  getSource = getDreamLockSource fetchedSources;
+
+  vendoring = import ./vendor.nix {
+    inherit dreamLock getSource lib sourceRoot;
+    inherit
+      (dreamLockInterface)
+      getSourceSpec
+      getRoot
+      subsystemAttrs
+      ;
+    inherit
+      (config.deps)
+      cargo
+      jq
+      moreutils
+      python3Packages
+      runCommandLocal
+      writePython3
+      ;
+  };
+in {
+  imports = [
+    ./interface.nix
+    dream2nix.modules.dream2nix.core
+  ];
+
+  rust-cargo-vendor = {
+    vendoredSources = vendoring.vendoredDependencies;
+    inherit
+      (vendoring)
+      copyVendorDir
+      getRootSource
+      writeGitVendorEntries
+      replaceRelativePathsWithAbsolute
+      ;
+  };
+
+  deps = {nixpkgs, ...}:
+    l.mapAttrs (_: l.mkOverride 998) {
+      inherit
+        (nixpkgs)
+        cargo
+        jq
+        moreutils
+        python3Packages
+        runCommandLocal
+        fetchurl
+        fetchgit
+        nix
+        ;
+      inherit
+        (nixpkgs.writers)
+        writePython3
+        ;
+    };
+}

modules/dream2nix/rust-cargo-vendor/interface.nix

@@ -0,0 +1,35 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+  t = l.types;
+in {
+  options.rust-cargo-vendor = l.mapAttrs (_: l.mkOption) {
+    vendoredSources = {
+      type = t.package;
+      description = "Path to vendored sources";
+    };
+    copyVendorDir = {
+      type = t.functionTo (t.functionTo t.str);
+      internal = true;
+      description = "Makes shell command(s) that copies the vendored sources correctly in a rust derivation";
+    };
+    getRootSource = {
+      type = t.functionTo (t.functionTo t.path);
+      internal = true;
+      description = "Gets root source for a given package";
+    };
+    writeGitVendorEntries = {
+      type = t.functionTo t.str;
+      internal = true;
+      description = "Makes shell command(s) that writes vendored git sources to .cargo/config so cargo uses the sources we vendored";
+    };
+    replaceRelativePathsWithAbsolute = {
+      type = t.functionTo t.str;
+      internal = true;
+      description = "Makes shell commands that will replace relative dependency paths with absolute paths in Cargo.toml";
+    };
+  };
+}

modules/dream2nix/rust-cargo-vendor/vendor.nix

@@ -1,7 +1,9 @@
 {
   lib,
+  getRoot,
   getSource,
   getSourceSpec,
+  sourceRoot,
   subsystemAttrs,
   dreamLock,
   moreutils,
@@ -145,4 +147,34 @@ in rec {
     echo "dream2nix: installing cargo vendor directory from ${from} to ${to}"
     cp -rs --no-preserve=mode,ownership ${from} ${to}
   '';
+
+  # Gets the root source for a package
+  getRootSource = pname: version: let
+    root = getRoot pname version;
+  in
+    getSource root.pname root.version;
+
+  # Generates a script that replaces relative path dependency paths with absolute
+  # ones, if the path dependency isn't in the source dream2nix provides
+  replaceRelativePathsWithAbsolute = replacements: let
+    replace =
+      l.concatStringsSep
+      " \\\n"
+      (
+        l.mapAttrsToList
+        (
+          # TODO: this is not great, because it forces us to include the entire
+          # sourceRoot here, which could possibly cause more rebuilds than necessary
+          # when source is changed (although this mostly depends on how the project
+          # repository is structured). doing this properly is pretty complex, but
+          # it should still be done later.
+          from: relPath: ''--replace "\"${from}\"" "\"${sourceRoot}/${relPath}\""''
+        )
+        replacements
+      );
+  in ''
+    echo "dream2nix: replacing relative dependency paths with absolute paths in Cargo.toml"
+    substituteInPlace ./Cargo.toml \
+      ${replace}
+  '';
 }

modules/dream2nix/rust-crane/default.nix

@@ -10,71 +10,17 @@
 
   dreamLock = config.rust-cargo-lock.dreamLock;
 
-  sourceRoot = config.mkDerivation.src;
-
-  fetchDreamLockSources =
-    import ../../../lib/internal/fetchDreamLockSources.nix
-    {inherit lib;};
-  getDreamLockSource = import ../../../lib/internal/getDreamLockSource.nix {inherit lib;};
   readDreamLock = import ../../../lib/internal/readDreamLock.nix {inherit lib;};
-  hashPath = import ../../../lib/internal/hashPath.nix {
-    inherit lib;
-    inherit (config.deps) runCommandLocal nix;
-  };
-  hashFile = import ../../../lib/internal/hashFile.nix {
-    inherit lib;
-    inherit (config.deps) runCommandLocal nix;
-  };
-
-  # fetchers
-  fetchers = {
-    git = import ../../../lib/internal/fetchers/git {
-      inherit hashPath;
-      inherit (config.deps) fetchgit;
-    };
-    crates-io = import ../../../lib/internal/fetchers/crates-io {
-      inherit hashFile;
-      inherit (config.deps) fetchurl runCommandLocal;
-    };
-    path = import ../../../lib/internal/fetchers/path {
-      inherit hashPath;
-    };
-  };
 
   dreamLockLoaded = readDreamLock {inherit dreamLock;};
   dreamLockInterface = dreamLockLoaded.interface;
 
-  inherit (dreamLockInterface) defaultPackageName defaultPackageVersion;
+  meta = let
-
+    meta = dreamLockInterface.subsystemAttrs.meta.${pname}.${version};
-  fetchedSources' = fetchDreamLockSources {
+  in
-    inherit defaultPackageName defaultPackageVersion;
+    meta
-    inherit (dreamLockLoaded.lock) sources;
-    inherit fetchers;
-  };
-
-  fetchedSources =
-    fetchedSources'
     // {
-      ${defaultPackageName}.${defaultPackageVersion} = sourceRoot;
+      license = l.map (name: l.licenses.${name}) meta.license;
-    };
-
-  getSource = getDreamLockSource fetchedSources;
-
-  toTOML = import ../../../lib/internal/toTOML.nix {inherit lib;};
-
-  utils = import ./utils.nix {
-    inherit dreamLock getSource lib toTOML sourceRoot;
-    inherit
-      (dreamLockInterface)
-      getSourceSpec
-      getRoot
-      subsystemAttrs
-      packages
-      ;
-    inherit
-      (config.deps)
-      writeText
-      ;
     };
 
   _crane = import config.deps.craneSource {
@@ -83,35 +29,17 @@
   crane = _crane.overrideToolchain config.deps.mkRustToolchain;
   rustToolchain = config.deps.mkRustToolchain config.deps.cranePkgs;
 
-  vendoring = import ./vendor.nix {
-    inherit dreamLock getSource lib;
-    inherit
-      (dreamLockInterface)
-      getSourceSpec
-      subsystemAttrs
-      ;
-    inherit
-      (config.deps)
-      jq
-      moreutils
-      python3Packages
-      runCommandLocal
-      writePython3
-      ;
-    cargo = rustToolchain;
-  };
-
   pname = config.name;
   version = config.version;
 
   replacePaths =
-    utils.replaceRelativePathsWithAbsolute
+    config.rust-cargo-vendor.replaceRelativePathsWithAbsolute
     dreamLockInterface.subsystemAttrs.relPathReplacements.${pname}.${version};
-  writeGitVendorEntries = vendoring.writeGitVendorEntries "nix-sources";
+  writeGitVendorEntries = config.rust-cargo-vendor.writeGitVendorEntries "nix-sources";
 
   # common args we use for both buildDepsOnly and buildPackage
   common = {
-    src = lib.mkForce (utils.getRootSource pname version);
+    src = lib.mkForce (config.rust-cargo-vendor.getRootSource pname version);
     postUnpack = ''
       export CARGO_HOME=$(pwd)/.cargo_home
       export cargoVendorDir="$TMPDIR/nix-vendor"
@@ -140,7 +68,7 @@
   depsNameSuffix = "-deps";
   depsArgs = {
     preUnpack = ''
-      ${vendoring.copyVendorDir "$dream2nixVendorDir" common.cargoVendorDir}
+      ${config.rust-cargo-vendor.copyVendorDir "$dream2nixVendorDir" common.cargoVendorDir}
     '';
     # move the vendored dependencies folder to $out for main derivation to use
     postInstall = ''
@@ -148,25 +76,25 @@
     '';
     # we pass cargoLock path to buildDepsOnly
     # so that crane's mkDummySrc adds it to the dummy source
-    inherit (utils) cargoLock;
+    inherit (config.rust-cargo-lock) cargoLock;
     pname = l.mkOverride 99 pname;
     pnameSuffix = depsNameSuffix;
     # Make sure cargo only checks the package we want
     cargoCheckCommand = "cargo check \${cargoBuildFlags:-} --profile \${cargoBuildProfile} --package ${pname}";
-    dream2nixVendorDir = vendoring.vendoredDependencies;
+    dream2nixVendorDir = config.rust-cargo-vendor.vendoredSources;
   };
 
   buildArgs = {
     # link the vendor dir we used earlier to the correct place
     preUnpack = ''
-      ${vendoring.copyVendorDir "$cargoArtifacts/nix-vendor" common.cargoVendorDir}
+      ${config.rust-cargo-vendor.copyVendorDir "$cargoArtifacts/nix-vendor" common.cargoVendorDir}
     '';
     # write our cargo lock
     # note: we don't do this in buildDepsOnly since
     # that uses a cargoLock argument instead
     preConfigure = l.mkForce ''
       ${common.preConfigure}
-      ${utils.writeCargoLock}
+      ${config.rust-cargo-lock.writeCargoLock}
     '';
     cargoArtifacts = cfg.depsDrv.public;
   };
@@ -200,11 +128,15 @@ in {
       cargo = rustToolchain;
     };
     dependencies = cfg.depsDrv.public;
-    meta = (utils.getMeta pname version) // config.mkDerivation.meta;
+    meta = meta // config.mkDerivation.meta;
   };
 
   deps = {nixpkgs, ...}:
     l.mkMerge [
+      {
+        # override cargo package to be the rust toolchain so that rust-cargo-vendor uses the custom provided toolchain if any
+        cargo = l.mkOverride 1001 rustToolchain;
+      }
       (l.mapAttrs (_: l.mkDefault) {
         craneSource = config.deps.fetchFromGitHub {
           owner = "ipetkov";
@@ -215,24 +147,12 @@ in {
         cranePkgs = nixpkgs.pkgs;
         mkRustToolchain = pkgs: pkgs.cargo;
       })
-      # maybe it would be better to put these under `options.rust-crane.deps` instead of this `deps`
-      # since it conflicts with a lot of stuff?
       (l.mapAttrs (_: l.mkOverride 999) {
         inherit
           (nixpkgs)
-          fetchurl
-          jq
-          moreutils
-          python3Packages
-          runCommandLocal
-          writeText
-          fetchFromGitHub
-          libiconv
           mkShell
-          ;
+          libiconv
-        inherit
+          fetchFromGitHub
-          (nixpkgs.writers)
-          writePython3
           ;
       })
     ];

modules/dream2nix/rust-crane/patch-workspace-deps.jq

@@ -1,48 +0,0 @@
-def normalizeWorkspaceDep:
-  if ($workspaceDependencies."\(.key)" | type) == "object"
-  then [.value, $workspaceDependencies."\(.key)"] | add
-  else [.value, {"version":$workspaceDependencies."\(.key)"}] | add
-  end
-  # remove workspace option from the dependency
-  | del(.workspace)
-;
-
-# normalizes workspace inherited dependencies for one list
-def mapWorkspaceDepsFor(name):
-  if has(name)
-  then
-    ."\(name)" = (
-      ."\(name)"
-      | to_entries
-      | map(
-        if (.value | type) == "object" and .value.workspace == true
-        then .value = (. | normalizeWorkspaceDep)
-        else .
-        end
-      )
-      | from_entries
-    )
-  else .
-  end
-;
-
-# shorthand for normalizing all the dependencies list
-def mapWorkspaceDeps:
-  mapWorkspaceDepsFor("dependencies")
-  | mapWorkspaceDepsFor("dev-dependencies")
-  | mapWorkspaceDepsFor("build-dependencies")
-;
-
-# normalize workspace inherited deps
-mapWorkspaceDeps
-| if has("target")
-  then
-    # normalize workspace inherited deps in target specific deps
-    .target = (
-      .target
-      | to_entries
-      | map(.value = (.value | mapWorkspaceDeps))
-      | from_entries
-    )
-  else .
-  end

modules/dream2nix/rust-crane/utils.nix

@@ -1,191 +0,0 @@
-{
-  dreamLock,
-  getSourceSpec,
-  getSource,
-  getRoot,
-  sourceRoot,
-  subsystemAttrs,
-  packages,
-  lib,
-  toTOML,
-  writeText,
-  ...
-}: let
-  l = lib // builtins;
-  isInPackages = name: version: (packages.${name} or null) == version;
-in rec {
-  getMeta = pname: version: let
-    meta = subsystemAttrs.meta.${pname}.${version};
-  in
-    meta
-    // {
-      license = l.map (name: l.licenses.${name}) meta.license;
-    };
-
-  # Gets the root source for a package
-  getRootSource = pname: version: let
-    root = getRoot pname version;
-  in
-    getSource root.pname root.version;
-
-  # Generates a script that replaces relative path dependency paths with absolute
-  # ones, if the path dependency isn't in the source dream2nix provides
-  replaceRelativePathsWithAbsolute = replacements: let
-    replace =
-      l.concatStringsSep
-      " \\\n"
-      (
-        l.mapAttrsToList
-        (
-          # TODO: this is not great, because it forces us to include the entire
-          # sourceRoot here, which could possibly cause more rebuilds than necessary
-          # when source is changed (although this mostly depends on how the project
-          # repository is structured). doing this properly is pretty complex, but
-          # it should still be done later.
-          from: relPath: ''--replace "\"${from}\"" "\"${sourceRoot}/${relPath}\""''
-        )
-        replacements
-      );
-  in ''
-    echo "dream2nix: replacing relative dependency paths with absolute paths in Cargo.toml"
-    substituteInPlace ./Cargo.toml \
-      ${replace}
-  '';
-
-  # Backup original Cargo.lock if it exists and write our own one
-  writeCargoLock = ''
-    echo "dream2nix: replacing Cargo.lock with ${cargoLock}"
-    mv -f Cargo.lock Cargo.lock.orig || echo "dream2nix: no Cargo.lock was found beforehand"
-    cat ${cargoLock} > Cargo.lock
-  '';
-
-  # The Cargo.lock for this dreamLock.
-  cargoLock = let
-    mkPkgEntry = {
-      name,
-      version,
-      ...
-    } @ args: let
-      # constructs source string for dependency
-      makeSource = sourceSpec: let
-        source =
-          if sourceSpec.type == "crates-io"
-          then "registry+https://github.com/rust-lang/crates.io-index"
-          else if sourceSpec.type == "git"
-          then let
-            gitSpec =
-              l.findFirst
-              (src: src.url == sourceSpec.url && src.sha == sourceSpec.rev)
-              (throw "no git source: ${sourceSpec.url}#${sourceSpec.rev}")
-              (subsystemAttrs.gitSources or {});
-            refPart =
-              l.optionalString
-              (gitSpec ? type)
-              "?${gitSpec.type}=${gitSpec.value}";
-          in "git+${sourceSpec.url}${refPart}#${sourceSpec.rev}"
-          else null;
-      in
-        source;
-      # constructs source string for dependency entry
-      makeDepSource = sourceSpec:
-        if sourceSpec.type == "crates-io"
-        then makeSource sourceSpec
-        else if sourceSpec.type == "git"
-        then l.concatStringsSep "#" (l.init (l.splitString "#" (makeSource sourceSpec)))
-        else null;
-      # removes source type information from the version
-      normalizeVersion = version: srcType: l.removeSuffix ("$" + srcType) version;
-
-      sourceSpec = getSourceSpec name version;
-
-      normalizedVersion = normalizeVersion version sourceSpec.type;
-
-      source = let
-        src = makeSource sourceSpec;
-      in
-        if src == null
-        then throw "source type '${sourceSpec.type}' not supported"
-        else src;
-      dependencies =
-        l.map
-        (
-          dep: let
-            depSourceSpec = getSourceSpec dep.name dep.version;
-            depSource = makeDepSource depSourceSpec;
-
-            normalizedDepVersion = normalizeVersion dep.version depSourceSpec.type;
-
-            hasMultipleVersions =
-              l.length (l.attrValues dreamLock.sources.${dep.name}) > 1;
-            hasDuplicateVersions = dep.version != normalizedDepVersion;
-
-            # only put version if there are different versions of the dep
-            versionString =
-              l.optionalString hasMultipleVersions " ${normalizedDepVersion}";
-            # only put source if there are duplicate versions of the dep
-            # cargo vendor does not support this anyway and so builds will fail
-            # until https://github.com/rust-lang/cargo/issues/10310 is resolved.
-            srcString =
-              l.optionalString hasDuplicateVersions " (${depSource})";
-          in "${dep.name}${versionString}${srcString}"
-        )
-        args.dependencies;
-
-      isMainPackage = isInPackages name version;
-    in
-      {
-        name = sourceSpec.pname or name;
-        version = sourceSpec.version or normalizedVersion;
-      }
-      # put dependencies like how cargo expects them
-      // (
-        l.optionalAttrs
-        (l.length dependencies > 0)
-        {inherit dependencies;}
-      )
-      // (
-        l.optionalAttrs
-        (sourceSpec.type != "path" && !isMainPackage)
-        {inherit source;}
-      )
-      // (
-        l.optionalAttrs
-        (sourceSpec.type == "crates-io" && !isMainPackage)
-        {checksum = sourceSpec.hash;}
-      );
-    _package = l.flatten (
-      l.mapAttrsToList
-      (
-        name: versions:
-          l.mapAttrsToList
-          (
-            version: dependencies:
-              mkPkgEntry {inherit name version dependencies;}
-          )
-          versions
-      )
-      dreamLock.dependencies
-    );
-    package =
-      (
-        # add packages as dependencies because Cargo expects them to be there aswell
-        l.filter
-        (pkg: ! l.any (opkg: pkg.name == opkg.name && pkg.version == opkg.version) _package)
-        (
-          l.mapAttrsToList
-          (pname: version: {
-            name = pname;
-            inherit version;
-          })
-          dreamLock._generic.packages
-        )
-      )
-      ++ _package;
-    lockTOML = toTOML {
-      # the lockfile we generate is of version 3
-      version = 3;
-      inherit package;
-    };
-  in
-    writeText "Cargo.lock" lockTOML;
-}

modules/dream2nix/rust-crane/vendor.nix

@@ -1,148 +0,0 @@
-{
-  lib,
-  getSource,
-  getSourceSpec,
-  subsystemAttrs,
-  dreamLock,
-  moreutils,
-  writePython3,
-  python3Packages,
-  runCommandLocal,
-  ...
-} @ args: let
-  l = lib // builtins;
-
-  allDependencies =
-    l.flatten
-    (
-      l.mapAttrsToList
-      (
-        name: versions:
-          l.map (version: {inherit name version;}) (l.attrNames versions)
-      )
-      dreamLock.dependencies
-    );
-in rec {
-  # Generates a shell script that writes git vendor entries to .cargo/config.
-  # `replaceWith` is the name of the vendored source(s) to use.
-  writeGitVendorEntries = replaceWith: let
-    makeEntry = source: ''
-      [source."${source.url}${l.optionalString (source ? type) "?${source.type}=${source.value}"}"]
-      replace-with = "${replaceWith}"
-      git = "${source.url}"
-      ${l.optionalString (source ? type) "${source.type} = \"${source.value}\""}
-    '';
-    entries = l.map makeEntry subsystemAttrs.gitSources;
-  in ''
-    echo "dream2nix: Writing git vendor entries to $CARGO_HOME/config.toml"
-    mkdir -p $CARGO_HOME && touch $CARGO_HOME/config.toml
-    cat >> $CARGO_HOME/config.toml <<EOF
-    ${l.concatStringsSep "\n" entries}
-    EOF
-  '';
-
-  # Vendors the dependencies passed as Cargo expects them
-  vendorDependencies = deps: let
-    makeSource = dep: let
-      path = getSource dep.name dep.version;
-      spec = getSourceSpec dep.name dep.version;
-      normalizeVersion = version: l.removeSuffix ("$" + spec.type) version;
-    in {
-      inherit path spec dep;
-      name = "${dep.name}-${normalizeVersion dep.version}";
-    };
-    sources = l.map makeSource deps;
-
-    findCrateSource = source: let
-      cargo = "${args.cargo}/bin/cargo";
-      jq = "${args.jq}/bin/jq";
-      sponge = "${moreutils}/bin/sponge";
-
-      writeConvertScript = from: to:
-        writePython3
-        "${from}-to-${to}.py"
-        {libraries = [python3Packages.toml];}
-        ''
-          import toml
-          import json
-          import sys
-          t = ${from}.loads(sys.stdin.read())
-          sys.stdout.write(${to}.dumps(t))
-        '';
-      tomlToJson = writeConvertScript "toml" "json";
-      jsonToToml = writeConvertScript "json" "toml";
-
-      pkg = source.dep;
-    in ''
-      # If the target package is in a workspace, or if it's the top-level
-      # crate, we should find the crate path using `cargo metadata`.
-      crateCargoTOML=$(${cargo} metadata --format-version 1 --no-deps --manifest-path $tree/Cargo.toml | \
-        ${jq} -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path')
-      # If the repository is not a workspace the package might be in a subdirectory.
-      if [[ -z $crateCargoTOML ]]; then
-        for manifest in $(find $tree -name "Cargo.toml"); do
-          echo Looking at $manifest
-          crateCargoTOML=$(${cargo} metadata --format-version 1 --no-deps --manifest-path "$manifest" | ${jq} -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path' || :)
-          if [[ ! -z $crateCargoTOML ]]; then
-            break
-          fi
-        done
-        if [[ -z $crateCargoTOML ]]; then
-          >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree"
-          exit 1
-        fi
-      else
-        # we need to patch dependencies with `workspace = true` (workspace inheritance)
-        workspaceDependencies="$(cat "$tree/Cargo.toml" | ${tomlToJson} | ${jq} -cr '.workspace.dependencies')"
-        if [[ "$workspaceDependencies" != "null" ]]; then
-          tree="$(pwd)/${pkg.name}-${pkg.version}"
-          cp -prd --no-preserve=mode,ownership "$(dirname $crateCargoTOML)" "$tree"
-          crateCargoTOML="$tree/Cargo.toml"
-          cat "$crateCargoTOML" \
-          | ${tomlToJson} \
-          | ${jq} -cr --argjson workspaceDependencies "$workspaceDependencies" \
-            --from-file ${./patch-workspace-deps.jq} \
-          | ${jsonToToml} \
-          | ${sponge} "$crateCargoTOML"
-        fi
-      fi
-      echo Found crate ${pkg.name} at $crateCargoTOML
-      tree="$(dirname $crateCargoTOML)"
-    '';
-    makeScript = source: let
-      isGit = source.spec.type == "git";
-      isPath = source.spec.type == "path";
-    in
-      l.optionalString (!isPath) ''
-        tree="${source.path}"
-        ${l.optionalString isGit (findCrateSource source)}
-        echo Vendoring crate ${source.name}
-        if [ -d $out/${source.name} ]; then
-          echo Crate is already vendored
-          echo Crates with duplicate versions cannot be vendored as Cargo does not support this behaviour
-          exit 1
-        else
-          cp -prd "$tree" $out/${source.name}
-          chmod u+w $out/${source.name}
-          ${l.optionalString isGit "printf '{\"files\":{},\"package\":null}' > $out/${source.name}/.cargo-checksum.json"}
-        fi
-      '';
-  in
-    runCommandLocal "vendor" {} ''
-      mkdir -p $out
-
-      ${
-        l.concatMapStringsSep "\n"
-        makeScript
-        sources
-      }
-    '';
-
-  # All dependencies in the Cargo.lock file, vendored
-  vendoredDependencies = vendorDependencies allDependencies;
-
-  copyVendorDir = from: to: ''
-    echo "dream2nix: installing cargo vendor directory from ${from} to ${to}"
-    cp -rs --no-preserve=mode,ownership ${from} ${to}
-  '';
-}