Splice overrideToolchain (#652)

---------

Co-authored-by: Ivan Petkov <ivanppetkov@gmail.com>

CHANGELOG.md

@@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   `craneLib.cleanCargoSource (craneLib.path ./.)`): it is recommended to either
   use `craneLib.cleanCargoSource ./.` directly (if the default source cleaning
   is desired) or `craneLib.path ./.` (if not).
+* `overrideToolchain` has been updated to better handle cross-compilation
+  splicing for a customized toolchain. This means that `overrideToolchain`
+  should now be called with a function which constructs said toolchain for any
+  given `pkgs` instantiation. For example: `craneLib.overrideToolchain (p:
+  p.rust-bin.stable.latest.default)`
 
 ### Fixed
 * The cross compilation example also hows how to set the `TARGET_CC` environment
@@ -23,6 +28,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   `cleanCargoSource`)
 * `removeReferencesToVendoredSources` handles the edge case where
   `cargoVendorDir` does not point to a path within the Nix store
+* It is now possible to use `.overrideScope` to change what instance of
+  `craneUtils` will be used during vendoring.
 
 ## [0.17.3] - 2024-06-02
 

checks/default.nix

@@ -8,7 +8,7 @@ in
 onlyDrvs (lib.makeScope myLib.newScope (self:
 let
   callPackage = self.newScope { };
-  myLibLlvmTools = myLib.overrideToolchain (pkgs.rust-bin.stable.latest.minimal.override {
+  myLibLlvmTools = myLib.overrideToolchain (p: p.rust-bin.stable.latest.minimal.override {
     extensions = [ "llvm-tools" ];
   });
   x64Linux = pkgs.hostPlatform.system == "x86_64-linux";
@@ -407,7 +407,7 @@ in
 
   noStd =
     let
-      noStdLib = myLib.overrideToolchain (pkgs.rust-bin.stable.latest.minimal.override {
+      noStdLib = myLib.overrideToolchain (p: p.rust-bin.stable.latest.minimal.override {
         targets = [
           "thumbv6m-none-eabi"
           "x86_64-unknown-none"
@@ -423,7 +423,7 @@ in
 
   bindeps =
     let
-      bindepsLib = myLib.overrideToolchain (pkgs.rust-bin.nightly.latest.minimal.override {
+      bindepsLib = myLib.overrideToolchain (p: p.rust-bin.nightly.latest.minimal.override {
         targets = [
           "wasm32-unknown-unknown"
           "x86_64-unknown-none"

checks/trunk.nix

@@ -5,7 +5,7 @@
 }:
 
 let
-  wasmToolchain = pkgs.rust-bin.stable.latest.minimal.override {
+  wasmToolchainFor = p: p.rust-bin.stable.latest.minimal.override {
     targets = [ "wasm32-unknown-unknown" ];
   };
 
@@ -13,7 +13,7 @@ let
     url = "https://github.com/NixOS/nixpkgs/archive/4e6868b1aa3766ab1de169922bb3826143941973.tar.gz";
     sha256 = "sha256:1q6bj2jjlwb10sfrhqmjpzsc3yc4x76cvky16wh0z52p7d2lhdpv";
   };
-  myLibWasm = (myLib.overrideToolchain wasmToolchain).overrideScope (_final: _prev: {
+  myLibWasm = (myLib.overrideToolchain wasmToolchainFor).overrideScope (_final: _prev: {
     inherit (import tarball { inherit (stdenv) system; }) wasm-bindgen-cli;
   });
 

default.nix

@@ -1,13 +1,3 @@
 { pkgs ? import <nixpkgs> { } }:
 
-import ./lib {
-  inherit (pkgs) lib makeScopeWithSplicing';
-  otherSplices = {
-    selfBuildBuild = pkgs.pkgsBuildBuild;
-    selfBuildHost = pkgs.pkgsBuildHost;
-    selfBuildTarget = pkgs.pkgsBuildTarget;
-    selfHostHost = pkgs.pkgsHostHost;
-    selfHostTarget = pkgs.pkgsHostTarget;
-    selfTargetTarget = pkgs.pkgsTargetTarget;
-  };
-}
+pkgs.callPackage ./lib { }

docs/API.md

@@ -1193,15 +1193,20 @@ build caches. More specifically:
 
 ### `craneLib.overrideToolchain`
 
-`overrideToolchain :: drv -> set`
+`overrideToolchain :: (set -> drv) -> set`
+`overrideToolchain :: drv -> set` (legacy)
 
 A convenience method to override and use tools (like `cargo`, `clippy`,
 `rustfmt`, `rustc`, etc.) from one specific toolchain. The input should be a
 single derivation which contains all the tools as binaries. For example, this
 can be the output of `oxalica/rust-overlay`.
 
+Note that in order to best support cross compilation, `overrideToolchain` should
+be provided a function (whose argument is a cross-compilation aware version of
+`pkgs`) which constructs the toolchain:
+
 ```nix
-craneLib.overrideToolchain myCustomToolchain
+craneLib.overrideToolchain (p: myCustomToolchainForPkgs p)
 ```
 
 ### `craneLib.path`

docs/faq/custom-nixpkgs.md

@@ -26,12 +26,10 @@ let
     inherit system;
     overlays = [ (import rust-overlay) ];
   };
-
-  rustToolchain = pkgs.rust-bin.stable.latest.default.override {
-    targets = [ "wasm32-wasi" ];
-  };
 in
-(crane.mkLib pkgs).overrideToolchain rustToolchain
+(crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default.override {
+    targets = [ "wasm32-wasi" ];
+})
 ```
 
 Finally, specific inputs can be overridden for the entire library via the

docs/faq/invalid-metadata-files-for-crate.md

@@ -7,7 +7,8 @@ used in the build:
 
 ```nix
 let
-  rustToolchain = ...;
+  rustToolchainForPkgs = p: ...;
+  rustToolchain = rustToolchainForPkgs pkgs;
 in
 # Incorrect usage, missing `clippy` override!
 #(crane.mkLib pkgs).overrideScope (final: prev: {
@@ -17,5 +18,5 @@ in
 #});
 
 # Correct usage (`overrideToolchain` handles the details for us)
-(crane.mkLib pkgs).overrideToolchain rustToolchain
+(crane.mkLib pkgs).overrideToolchain rustToolchainForPkgs
 ```

docs/overriding_derivations.md

@@ -83,7 +83,7 @@ craneLib.buildPackage {
 
           # Use a different `craneLib` instantiation: one with a nightly compiler
           my-crate-nightly = my-crate.override {
-            craneLib = craneLib.overrideToolchain pkgs.rust-bin.nightly.latest.default;
+            craneLib = craneLib.overrideToolchain (p: p.rust-bin.nightly.latest.default);
           };
         };
       });

examples/build-std/flake.nix

@@ -28,16 +28,17 @@
           overlays = [ (import rust-overlay) ];
         };
 
-        rustToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default.override {
+        rustToolchainFor = p: p.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default.override {
           extensions = [ "rust-src" ];
           targets = [ "x86_64-unknown-linux-gnu" ];
         });
+        rustToolchain = rustToolchainFor pkgs;
 
         # NB: we don't need to overlay our custom toolchain for the *entire*
         # pkgs (which would require rebuidling anything else which uses rust).
         # Instead, we just want to update the scope that crane will use by appending
         # our specific toolchain there.
-        craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
+        craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchainFor;
 
         src = craneLib.cleanCargoSource ./.;
 
@@ -83,7 +84,6 @@
           # Extra inputs can be added here; cargo and rustc are provided by default
           # from the toolchain that was specified earlier.
           packages = [
-            # rustToolchain
           ];
         };
       });

examples/cross-musl/flake.nix

@@ -28,11 +28,9 @@
           overlays = [ (import rust-overlay) ];
         };
 
-        rustToolchain = pkgs.rust-bin.stable.latest.default.override {
+        craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default.override {
           targets = [ "x86_64-unknown-linux-musl" ];
-        };
-
-        craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
+        });
 
         my-crate = craneLib.buildPackage {
           src = craneLib.cleanCargoSource ./.;

examples/cross-rust-overlay/flake.nix

@@ -31,11 +31,7 @@
           overlays = [ (import rust-overlay) ];
         };
 
-        rustToolchain = pkgs.pkgsBuildHost.rust-bin.stable.latest.default.override {
-          targets = [ "aarch64-unknown-linux-gnu" ];
-        };
-
-        craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
+        craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default);
 
         # Note: we have to use the `callPackage` approach here so that Nix
         # can "splice" the packages in such a way that dependencies are

examples/custom-toolchain/flake.nix

@@ -28,15 +28,13 @@
           overlays = [ (import rust-overlay) ];
         };
 
-        rustWithWasiTarget = pkgs.rust-bin.stable.latest.default.override {
-          targets = [ "wasm32-wasi" ];
-        };
-
         # NB: we don't need to overlay our custom toolchain for the *entire*
         # pkgs (which would require rebuidling anything else which uses rust).
         # Instead, we just want to update the scope that crane will use by appending
         # our specific toolchain there.
-        craneLib = (crane.mkLib pkgs).overrideToolchain rustWithWasiTarget;
+        craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default.override {
+          targets = [ "wasm32-wasi" ];
+        });
 
         my-crate = craneLib.buildPackage {
           src = craneLib.cleanCargoSource ./.;
@@ -76,7 +74,6 @@
           # Extra inputs can be added here; cargo and rustc are provided by default
           # from the toolchain that was specified earlier.
           packages = [
-            # rustWithWasiTarget
           ];
         };
       });

examples/end-to-end-testing/flake.nix

@@ -27,8 +27,7 @@
         };
         inherit (pkgs) lib;
 
-        rustToolchain = pkgs.rust-bin.stable.latest.default;
-        craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
+        craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default);
         src = craneLib.cleanCargoSource ./.;
 
         workspace = craneLib.buildPackage {

examples/trunk-workspace/flake.nix

@@ -34,12 +34,12 @@
 
         inherit (pkgs) lib;
 
-        rustToolchain = pkgs.rust-bin.stable.latest.default.override {
+        rustToolchainFor = p: p.rust-bin.stable.latest.default.override {
           # Set the build targets supported by the toolchain,
           # wasm32-unknown-unknown is required for trunk.
           targets = [ "wasm32-unknown-unknown" ];
         };
-        craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchain).overrideScope (_final: _prev: {
+        craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchainFor).overrideScope (_final: _prev: {
           # The version of wasm-bindgen-cli needs to match the version in Cargo.lock. You
           # can unpin this if your nixpkgs commit contains the appropriate wasm-bindgen-cli version
           inherit (import nixpkgs-for-wasm-bindgen { inherit system; }) wasm-bindgen-cli;

examples/trunk/flake.nix

@@ -34,12 +34,12 @@
 
         inherit (pkgs) lib;
 
-        rustToolchain = pkgs.rust-bin.stable.latest.default.override {
+        rustToolchainFor = p: p.rust-bin.stable.latest.default.override {
           # Set the build targets supported by the toolchain,
           # wasm32-unknown-unknown is required for trunk
           targets = [ "wasm32-unknown-unknown" ];
         };
-        craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchain).overrideScope (_final: _prev: {
+        craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchainFor).overrideScope (_final: _prev: {
           # The version of wasm-bindgen-cli needs to match the version in Cargo.lock. You
           # can unpin this if your nixpkgs commit contains the appropriate wasm-bindgen-cli version
           inherit (import nixpkgs-for-wasm-bindgen { inherit system; }) wasm-bindgen-cli;

lib/default.nix

@@ -1,6 +1,13 @@
 { lib
+, stdenv
 , makeScopeWithSplicing'
-, otherSplices
+, splicePackages
+, pkgsBuildBuild
+, pkgsBuildHost
+, pkgsBuildTarget
+, pkgsHostHost
+, pkgsHostTarget
+, pkgsTargetTarget
 }:
 
 let
@@ -9,20 +16,37 @@ let
   isUnsupported = lib.versionOlder current minSupported;
   msg = "crane requires at least nixpkgs-${minSupported}, supplied nixpkgs-${current}";
 
-  mySplice = f: makeScopeWithSplicing' {
-    inherit otherSplices f;
+  # Helps keep things in sync between `overrideToolchain` and `keep`
+  attrsForToolchainOverride = [
+    "cargo"
+    "clippy"
+    "rustc"
+    "rustfmt"
+  ];
+
+  spliceToolchain = toolchainFn:
+    let
+      splices = {
+        pkgsBuildBuild = { toolchain = toolchainFn pkgsBuildBuild; };
+        pkgsBuildHost = { toolchain = toolchainFn pkgsBuildHost; };
+        pkgsBuildTarget = { toolchain = toolchainFn pkgsBuildTarget; };
+        pkgsHostHost = { toolchain = toolchainFn pkgsHostHost; };
+        pkgsHostTarget = { toolchain = toolchainFn pkgsHostTarget; };
+        pkgsTargetTarget = lib.optionalAttrs (pkgsTargetTarget?newScope) { toolchain = toolchainFn pkgsTargetTarget; };
       };
-in
-lib.warnIf isUnsupported msg (mySplice (self:
-let
+    in
+    (splicePackages splices).toolchain;
+
+  scopeFn = self:
+    let
       inherit (self) callPackage;
 
       internalCrateNameFromCargoToml = callPackage ./internalCrateNameFromCargoToml.nix { };
       internalCrateNameForCleanSource = callPackage ./internalCrateNameForCleanSource.nix {
         inherit internalCrateNameFromCargoToml;
       };
-in
-{
+    in
+    {
       appendCrateRegistries = input: self.overrideScope (_final: prev: {
         crateRegistries = prev.crateRegistries // (lib.foldl (a: b: a // b) { } input);
       });
@@ -47,7 +71,7 @@ in
       cleanCargoToml = callPackage ./cleanCargoToml.nix { };
       configureCargoCommonVarsHook = callPackage ./setupHooks/configureCargoCommonVars.nix { };
       configureCargoVendoredDepsHook = callPackage ./setupHooks/configureCargoVendoredDeps.nix { };
-  devShell = callPackage ./devShell.nix { };
+      craneUtils = callPackage ../pkgs/crane-utils { };
 
       crateNameFromCargoToml = callPackage ./crateNameFromCargoToml.nix {
         inherit internalCrateNameFromCargoToml;
@@ -58,6 +82,7 @@ in
         indexUrl = "https://github.com/rust-lang/crates.io-index";
       };
 
+      devShell = callPackage ./devShell.nix { };
       downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
       downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
       filterCargoSources = callPackage ./filterCargoSources.nix { };
@@ -68,12 +93,22 @@ in
       mkCargoDerivation = callPackage ./mkCargoDerivation.nix { };
       mkDummySrc = callPackage ./mkDummySrc.nix { };
 
-  overrideToolchain = toolchain: self.overrideScope (_final: _prev: {
-    cargo = toolchain;
-    clippy = toolchain;
-    rustc = toolchain;
-    rustfmt = toolchain;
-  });
+      overrideToolchain = toolchainArg: self.overrideScope (_final: _prev:
+        let
+          toolchain =
+            if lib.isFunction toolchainArg
+            then spliceToolchain toolchainArg
+            else toolchainArg;
+          needsSplicing = stdenv.buildPlatform != stdenv.hostPlatform && toolchain?__spliced == false;
+          warningMsg = ''
+            craneLib.overrideToolchain requires a spliced toolchain when cross-compiling. Consider specifying
+            a function which constructs a toolchain for any given `pkgs` instantiation:
+
+            (crane.mkLib pkgs).overrideToolchain (p: ...)
+          '';
+        in
+        lib.warnIf needsSplicing warningMsg (lib.genAttrs attrsForToolchainOverride (_: toolchain))
+      );
 
       path = callPackage ./path.nix {
         inherit internalCrateNameForCleanSource;
@@ -90,4 +125,20 @@ in
       vendorCargoRegistries = callPackage ./vendorCargoRegistries.nix { };
       vendorGitDeps = callPackage ./vendorGitDeps.nix { };
       writeTOML = callPackage ./writeTOML.nix { };
-}))
+    };
+
+  craneSpliced = makeScopeWithSplicing' {
+    f = scopeFn;
+    otherSplices = {
+      selfBuildBuild = lib.makeScope pkgsBuildBuild.newScope scopeFn;
+      selfBuildHost = lib.makeScope pkgsBuildHost.newScope scopeFn;
+      selfBuildTarget = lib.makeScope pkgsBuildTarget.newScope scopeFn;
+      selfHostHost = lib.makeScope pkgsHostHost.newScope scopeFn;
+      selfHostTarget = lib.makeScope pkgsHostTarget.newScope scopeFn;
+      selfTargetTarget = lib.optionalAttrs (pkgsTargetTarget?newScope) (lib.makeScope pkgsTargetTarget.newScope scopeFn);
+    };
+    keep = self: lib.optionalAttrs (self?cargo)
+      (lib.genAttrs attrsForToolchainOverride (n: self.${n}));
+  };
+in
+lib.warnIf isUnsupported msg (craneSpliced)

lib/downloadCargoPackageFromGit.nix

@@ -1,5 +1,6 @@
 { lib
 , cargo
+, craneUtils
 , jq
 , pkgsBuildBuild
 }:
@@ -8,8 +9,6 @@ let
   inherit (pkgsBuildBuild)
     fetchgit
     stdenv;
-
-  craneUtils = pkgsBuildBuild.callPackage ../pkgs/crane-utils { };
 in
 { git
 , rev