feat: implement overrideRustToolchain for crane and build-rust-package (#175)

docs/src/subsystems/rust.md

@@ -16,7 +16,7 @@ generated lockfile.
 
 ## Builders
 
-### build-rust-package (pure)
+### build-rust-package (pure) (default)
 
 Builds a package using `buildRustPackage` from `nixpkgs`.
 
@@ -42,4 +42,87 @@ where `<crate-name>` is the name of the crate you are building.
   };
   # ...
 }
+```
+
+#### On the IFD marking
+
+The `crane` builder utilizes IFD to clean the source your crates reside in.
+This is needed to not rebuild the dependency only derivation everytime the
+source for your crates is changed.
+
+However this does not mean that the IFD will always be triggered. If you
+are passing dream2nix a path source or a flake source, then IFD won't be
+triggered as these sources are already realized. But if you are passing
+the result of a `pkgs.fetchFromGitHub` for example, this will trigger IFD
+since it is not already realized.
+
+### Specifying the Rust toolchain
+
+Specify an override for all packages that override the Rust toolchain used.
+This can be done like so:
+
+```nix
+{
+  # ...
+  packageOverrides = {
+    # ...
+    "^.*".set-toolchain.overrideRustToolchain = old: {
+      inherit (pkgs) cargo rustc;
+    };
+    # ...
+  };
+  # ...
+}
+```
+
+You can also of course override the toolchain for only certain crates:
+
+```nix
+{
+  # ...
+  packageOverrides = {
+    # ...
+    crate-name.set-toolchain.overrideRustToolchain = old: {
+      inherit (pkgs) cargo rustc;
+    };
+    # ...
+  };
+  # ...
+}
+```
+
+#### crane notes
+
+The crane builder does not require a `rustc` package in the toolchain specified,
+only a `cargo` package is needed. If cross-compiling, keep in mind that it also
+takes `cargo` packages like so:
+
+```nix
+{
+  cargoHostTarget = cargo-package;
+  cargoBuildBuild = other-cargo-package;
+}
+```
+
+where `cargoHostTarget` has the same meaning as coming from a `pkgsHostTarget`
+and `cargoBuildBuild` has the same meaning as coming from a `pkgsBuildBuild`.
+
+Also keep in mind that if you want to override the toolchain for a package
+and you are using the `crane` builder, you would need to set an override
+for both the dependencies and the main package derivation:
+
+```nix
+let
+  toolchainOverride = old: { /* ... */ };
+in
+{
+  # ...
+  packageOverrides = {
+    # ...
+    crate-name.set-toolchain.overrideRustToolchain = toolchainOverride;
+    crate-name-deps.set-toolchain.overrideRustToolchain = toolchainOverride;
+    # ...
+  };
+  # ...
+}
 ```

examples/set-rust-toolchain/flake.nix

@@ -0,0 +1,53 @@
+{
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    fenix.url = "github:nix-community/fenix/720b54260dee864d2a21745bd2bb55223f58e297";
+    fenix.inputs.nixpkgs.follows = "nixpkgs";
+    dream2nix.url = "path:../..";
+    dream2nix.inputs.nixpkgs.follows = "nixpkgs";
+    src.url = "github:BurntSushi/ripgrep/13.0.0";
+    src.flake = false;
+  };
+
+  outputs = {
+    self,
+    dream2nix,
+    fenix,
+    src,
+    ...
+  } @ inp: let
+    system = "x86_64-linux";
+    toolchain = fenix.packages.${system}.minimal.toolchain;
+  in
+    (dream2nix.lib.makeFlakeOutputs {
+      systems = [system];
+      config.projectRoot = ./.;
+      source = src;
+      settings = [
+        {
+          builder = "crane";
+          translator = "cargo-lock";
+        }
+      ];
+      packageOverrides = {
+        # override all packages and set a toolchain
+        "^.*" = {
+          set-toolchain.overrideRustToolchain = old: {cargo = toolchain;};
+          check-toolchain-version.overrideAttrs = old: {
+            buildPhase = ''
+              currentCargoVersion="$(cargo --version)"
+              customCargoVersion="$(${toolchain}/bin/cargo --version)"
+              if [[ "$currentCargoVersion" != "$customCargoVersion" ]]; then
+                echo "cargo version is $currentCargoVersion but it needs to be $customCargoVersion"
+                exit 1
+              fi
+              ${old.buildPhase or ""}
+            '';
+          };
+        };
+      };
+    })
+    // {
+      checks.x86_64-linux.ripgrep = self.packages.x86_64-linux.ripgrep;
+    };
+}

src/default.nix

@@ -87,71 +87,76 @@
         ({inherit name;} // attrs)
         "${externalSources.crane}/pkgs/${name}.sh";
       genHooks = names: attrs: lib.genAttrs names (makeHook attrs);
-
-      otherHooks = genHooks [
-        "configureCargoCommonVarsHook"
-        "configureCargoVendoredDepsHook"
-        "remapSourcePathPrefixHook"
-      ] {};
-      installHooks =
-        genHooks [
-          "inheritCargoArtifactsHook"
-          "installCargoArtifactsHook"
-        ] {
+    in
+      {
+        cargoHostTarget,
+        cargoBuildBuild,
+      }: rec {
+        otherHooks = genHooks [
+          "configureCargoCommonVarsHook"
+          "configureCargoVendoredDepsHook"
+          "remapSourcePathPrefixHook"
+        ] {};
+        installHooks =
+          genHooks [
+            "inheritCargoArtifactsHook"
+            "installCargoArtifactsHook"
+          ] {
+            substitutions = {
+              zstd = "${pkgs.pkgsBuildBuild.zstd}/bin/zstd";
+            };
+          };
+        installLogHook = genHooks ["installFromCargoBuildLogHook"] {
           substitutions = {
-            zstd = "${pkgs.pkgsBuildBuild.zstd}/bin/zstd";
+            cargo = "${cargoBuildBuild}/bin/cargo";
+            jq = "${pkgs.pkgsBuildBuild.jq}/bin/jq";
           };
         };
-      installLogHook = genHooks ["installFromCargoBuildLogHook"] {
-        substitutions = {
-          cargo = "${pkgs.pkgsBuildBuild.cargo}/bin/cargo";
-          jq = "${pkgs.pkgsBuildBuild.jq}/bin/jq";
+
+        # These aren't used by dream2nix
+        crateNameFromCargoToml = null;
+        vendorCargoDeps = null;
+
+        writeTOML = importLibFile "writeTOML" {
+          inherit (pkgs) runCommand pkgsBuildBuild;
+        };
+        cleanCargoToml = importLibFile "cleanCargoToml" {};
+        findCargoFiles = importLibFile "findCargoFiles" {
+          inherit (pkgs) lib;
+        };
+        mkDummySrc = importLibFile "mkDummySrc" {
+          inherit (pkgs) writeText runCommandLocal lib;
+          inherit writeTOML cleanCargoToml findCargoFiles;
+        };
+
+        mkCargoDerivation = importLibFile "mkCargoDerivation" ({
+            cargo = cargoHostTarget;
+            inherit (pkgs) stdenv lib;
+          }
+          // installHooks
+          // otherHooks);
+        buildDepsOnly = importLibFile "buildDepsOnly" {
+          inherit
+            mkCargoDerivation
+            crateNameFromCargoToml
+            vendorCargoDeps
+            mkDummySrc
+            ;
+        };
+        cargoBuild = importLibFile "cargoBuild" {
+          inherit
+            mkCargoDerivation
+            buildDepsOnly
+            crateNameFromCargoToml
+            vendorCargoDeps
+            ;
+        };
+        buildPackage = importLibFile "buildPackage" {
+          inherit (pkgs) lib;
+          inherit (installLogHook) installFromCargoBuildLogHook;
+          inherit cargoBuild;
         };
       };
-    in rec {
-      # These aren't used by dream2nix
-      crateNameFromCargoToml = null;
-      vendorCargoDeps = null;
-
-      writeTOML = importLibFile "writeTOML" {
-        inherit (pkgs) runCommand pkgsBuildBuild;
-      };
-      cleanCargoToml = importLibFile "cleanCargoToml" {};
-      findCargoFiles = importLibFile "findCargoFiles" {
-        inherit (pkgs) lib;
-      };
-      mkDummySrc = importLibFile "mkDummySrc" {
-        inherit (pkgs) writeText runCommandLocal lib;
-        inherit writeTOML cleanCargoToml findCargoFiles;
-      };
-
-      mkCargoDerivation = importLibFile "mkCargoDerivation" ({
-          inherit (pkgs) cargo stdenv lib;
-        }
-        // installHooks
-        // otherHooks);
-      buildDepsOnly = importLibFile "buildDepsOnly" {
-        inherit
-          mkCargoDerivation
-          crateNameFromCargoToml
-          vendorCargoDeps
-          mkDummySrc
-          ;
-      };
-      cargoBuild = importLibFile "cargoBuild" {
-        inherit
-          mkCargoDerivation
-          buildDepsOnly
-          crateNameFromCargoToml
-          vendorCargoDeps
-          ;
-      };
-      buildPackage = importLibFile "buildPackage" {
-        inherit (pkgs) lib;
-        inherit (installLogHook) installFromCargoBuildLogHook;
-        inherit cargoBuild;
-      };
-    };
   };
 
   dreamOverrides = let
@@ -313,14 +318,12 @@
   makeOutputsForDreamLock = {
     dreamLock,
     sourceRoot ? null,
-    builder ? null,
     fetcher ? null,
+    builder ? null,
+    builderArgs ? {},
     inject ? {},
     sourceOverrides ? oldSources: {},
     packageOverrides ? {},
-    builderArgs ? {},
-    translator ? null,
-    translatorArgs ? {},
   } @ args: let
     # parse dreamLock
     dreamLockLoaded = utils.readDreamLock {inherit (args) dreamLock;};

src/lib.nix

@@ -76,8 +76,6 @@
     packageOverrides ? {},
     settings ? [],
     sourceOverrides ? oldSources: {},
-    translator ? null,
-    translatorArgs ? {},
   } @ args: let
     allPkgs = makeNixpkgs pkgs systems;
 

src/subsystems/rust/builders/build-rust-package/default.nix

@@ -22,6 +22,13 @@
     utils = import ../utils.nix (args // topArgs);
     vendoring = import ../vendor.nix (args // topArgs);
 
+    buildWithToolchain =
+      utils.mkBuildWithToolchain
+      (toolchain: (pkgs.makeRustPlatform toolchain).buildRustPackage);
+    defaultToolchain = {
+      inherit (pkgs) cargo rustc;
+    };
+
     buildPackage = pname: version: let
       src = utils.getRootSource pname version;
       vendorDir = vendoring.vendoredDependencies;
@@ -32,7 +39,7 @@
 
       cargoBuildFlags = "--package ${pname}";
     in
-      produceDerivation pname (pkgs.rustPlatform.buildRustPackage {
+      produceDerivation pname (buildWithToolchain defaultToolchain {
         inherit pname version src;
 
         cargoBuildFlags = cargoBuildFlags;

src/subsystems/rust/builders/crane/default.nix

@@ -22,7 +22,26 @@
     utils = import ../utils.nix (args // topArgs);
     vendoring = import ../vendor.nix (args // topArgs);
 
-    crane = externals.crane;
+    mkCrane = toolchain:
+      if toolchain ? cargoHostTarget && toolchain ? cargoBuildBuild
+      then externals.crane toolchain
+      else if toolchain ? cargo
+      then
+        externals.crane {
+          cargoHostTarget = toolchain.cargo;
+          cargoBuildBuild = toolchain.cargo;
+        }
+      else throw "crane toolchain must include either a 'cargo' or both of 'cargoHostTarget' and 'cargoBuildBuild'";
+
+    buildDepsWithToolchain =
+      utils.mkBuildWithToolchain
+      (toolchain: (mkCrane toolchain).buildDepsOnly);
+    buildPackageWithToolchain =
+      utils.mkBuildWithToolchain
+      (toolchain: (mkCrane toolchain).buildPackage);
+    defaultToolchain = {
+      inherit (pkgs) cargo;
+    };
 
     buildPackage = pname: version: let
       replacePaths = utils.replaceRelativePathsWithAbsolute {
@@ -54,7 +73,10 @@
           inherit (utils) cargoLock;
           pnameSuffix = depsNameSuffix;
         };
-      deps = produceDerivation "${pname}${depsNameSuffix}" (crane.buildDepsOnly depsArgs);
+      deps =
+        produceDerivation
+        "${pname}${depsNameSuffix}"
+        (buildDepsWithToolchain defaultToolchain depsArgs);
 
       buildArgs =
         common
@@ -72,7 +94,9 @@
           '';
         };
     in
-      produceDerivation pname (crane.buildPackage buildArgs);
+      produceDerivation
+      pname
+      (buildPackageWithToolchain defaultToolchain buildArgs);
   in {
     packages =
       l.mapAttrs

src/subsystems/rust/builders/utils.nix

@@ -36,6 +36,26 @@ in rec {
       ${replacements}
   '';
 
+  mkBuildWithToolchain = mkBuildFunc: let
+    buildWithToolchain = toolchain: args: let
+      drv = (mkBuildFunc toolchain) args;
+    in
+      drv
+      // {
+        overrideRustToolchain = f: let
+          newToolchain = toolchain // (f toolchain);
+          maybePassthru =
+            l.optionalAttrs
+            (newToolchain ? passthru)
+            {inherit (newToolchain) passthru;};
+        in
+          buildWithToolchain newToolchain (args // maybePassthru);
+        overrideAttrs = f:
+          buildWithToolchain toolchain (args // (f drv));
+      };
+  in
+    buildWithToolchain;
+
   # Script to write the Cargo.lock if it doesn't already exist.
   writeCargoLock = ''
     rm -f "$PWD/Cargo.lock"

src/utils/override.nix

@@ -75,14 +75,12 @@
 
     # filter the overrides by the package name and conditions
     overridesToApply = let
-      # TODO: figure out if regex names will be useful
-      regexOverrides = {};
-      # lib.filterAttrs
-      #   (name: data:
-      #     lib.hasPrefix "^" name
-      #     &&
-      #     b.match name pname != null)
-      #   conditionalOverrides;
+      regexOverrides =
+        lib.filterAttrs
+        (name: data:
+          lib.hasPrefix "^" name
+          && b.match name pname != null)
+        conditionalOverrides;
 
       overridesForPackage =
         b.foldl'
@@ -182,10 +180,10 @@
       (overrideFuncs ++ singleArgOverrideFuncs);
   in
     # apply the overrides to the given pkg
-    (lib.foldl
-      (pkg: condOverride: applyOneOverride pkg condOverride)
-      pkg
-      overridesToApply);
+    lib.foldl
+    (pkg: condOverride: applyOneOverride pkg condOverride)
+    pkg
+    overridesToApply;
 in {
   inherit applyOverridesToPackage loadOverridesDirs;
 }