Splice overrideToolchain (#652)

---------

Co-authored-by: Ivan Petkov <ivanppetkov@gmail.com>
This commit is contained in:
Simon Žlender 2024-06-29 20:29:52 +02:00 committed by GitHub
parent 9791c77eb7
commit aa5dcd0518
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 174 additions and 133 deletions

View File

@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
`craneLib.cleanCargoSource (craneLib.path ./.)`): it is recommended to either
use `craneLib.cleanCargoSource ./.` directly (if the default source cleaning
is desired) or `craneLib.path ./.` (if not).
* `overrideToolchain` has been updated to better handle cross-compilation
splicing for a customized toolchain. This means that `overrideToolchain`
should now be called with a function which constructs said toolchain for any
given `pkgs` instantiation. For example: `craneLib.overrideToolchain (p:
p.rust-bin.stable.latest.default)`
### Fixed
* The cross compilation example also hows how to set the `TARGET_CC` environment
@ -23,6 +28,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
`cleanCargoSource`)
* `removeReferencesToVendoredSources` handles the edge case where
`cargoVendorDir` does not point to a path within the Nix store
* It is now possible to use `.overrideScope` to change what instance of
`craneUtils` will be used during vendoring.
## [0.17.3] - 2024-06-02

View File

@ -8,7 +8,7 @@ in
onlyDrvs (lib.makeScope myLib.newScope (self:
let
callPackage = self.newScope { };
myLibLlvmTools = myLib.overrideToolchain (pkgs.rust-bin.stable.latest.minimal.override {
myLibLlvmTools = myLib.overrideToolchain (p: p.rust-bin.stable.latest.minimal.override {
extensions = [ "llvm-tools" ];
});
x64Linux = pkgs.hostPlatform.system == "x86_64-linux";
@ -407,7 +407,7 @@ in
noStd =
let
noStdLib = myLib.overrideToolchain (pkgs.rust-bin.stable.latest.minimal.override {
noStdLib = myLib.overrideToolchain (p: p.rust-bin.stable.latest.minimal.override {
targets = [
"thumbv6m-none-eabi"
"x86_64-unknown-none"
@ -423,7 +423,7 @@ in
bindeps =
let
bindepsLib = myLib.overrideToolchain (pkgs.rust-bin.nightly.latest.minimal.override {
bindepsLib = myLib.overrideToolchain (p: p.rust-bin.nightly.latest.minimal.override {
targets = [
"wasm32-unknown-unknown"
"x86_64-unknown-none"

View File

@ -5,7 +5,7 @@
}:
let
wasmToolchain = pkgs.rust-bin.stable.latest.minimal.override {
wasmToolchainFor = p: p.rust-bin.stable.latest.minimal.override {
targets = [ "wasm32-unknown-unknown" ];
};
@ -13,7 +13,7 @@ let
url = "https://github.com/NixOS/nixpkgs/archive/4e6868b1aa3766ab1de169922bb3826143941973.tar.gz";
sha256 = "sha256:1q6bj2jjlwb10sfrhqmjpzsc3yc4x76cvky16wh0z52p7d2lhdpv";
};
myLibWasm = (myLib.overrideToolchain wasmToolchain).overrideScope (_final: _prev: {
myLibWasm = (myLib.overrideToolchain wasmToolchainFor).overrideScope (_final: _prev: {
inherit (import tarball { inherit (stdenv) system; }) wasm-bindgen-cli;
});

View File

@ -1,13 +1,3 @@
{ pkgs ? import <nixpkgs> { } }:
import ./lib {
inherit (pkgs) lib makeScopeWithSplicing';
otherSplices = {
selfBuildBuild = pkgs.pkgsBuildBuild;
selfBuildHost = pkgs.pkgsBuildHost;
selfBuildTarget = pkgs.pkgsBuildTarget;
selfHostHost = pkgs.pkgsHostHost;
selfHostTarget = pkgs.pkgsHostTarget;
selfTargetTarget = pkgs.pkgsTargetTarget;
};
}
pkgs.callPackage ./lib { }

View File

@ -1193,15 +1193,20 @@ build caches. More specifically:
### `craneLib.overrideToolchain`
`overrideToolchain :: drv -> set`
`overrideToolchain :: (set -> drv) -> set`
`overrideToolchain :: drv -> set` (legacy)
A convenience method to override and use tools (like `cargo`, `clippy`,
`rustfmt`, `rustc`, etc.) from one specific toolchain. The input should be a
single derivation which contains all the tools as binaries. For example, this
can be the output of `oxalica/rust-overlay`.
Note that in order to best support cross compilation, `overrideToolchain` should
be provided a function (whose argument is a cross-compilation aware version of
`pkgs`) which constructs the toolchain:
```nix
craneLib.overrideToolchain myCustomToolchain
craneLib.overrideToolchain (p: myCustomToolchainForPkgs p)
```
### `craneLib.path`

View File

@ -26,12 +26,10 @@ let
inherit system;
overlays = [ (import rust-overlay) ];
};
rustToolchain = pkgs.rust-bin.stable.latest.default.override {
targets = [ "wasm32-wasi" ];
};
in
(crane.mkLib pkgs).overrideToolchain rustToolchain
(crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default.override {
targets = [ "wasm32-wasi" ];
})
```
Finally, specific inputs can be overridden for the entire library via the

View File

@ -7,7 +7,8 @@ used in the build:
```nix
let
rustToolchain = ...;
rustToolchainForPkgs = p: ...;
rustToolchain = rustToolchainForPkgs pkgs;
in
# Incorrect usage, missing `clippy` override!
#(crane.mkLib pkgs).overrideScope (final: prev: {
@ -17,5 +18,5 @@ in
#});
# Correct usage (`overrideToolchain` handles the details for us)
(crane.mkLib pkgs).overrideToolchain rustToolchain
(crane.mkLib pkgs).overrideToolchain rustToolchainForPkgs
```

View File

@ -83,7 +83,7 @@ craneLib.buildPackage {
# Use a different `craneLib` instantiation: one with a nightly compiler
my-crate-nightly = my-crate.override {
craneLib = craneLib.overrideToolchain pkgs.rust-bin.nightly.latest.default;
craneLib = craneLib.overrideToolchain (p: p.rust-bin.nightly.latest.default);
};
};
});

View File

@ -28,16 +28,17 @@
overlays = [ (import rust-overlay) ];
};
rustToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default.override {
rustToolchainFor = p: p.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default.override {
extensions = [ "rust-src" ];
targets = [ "x86_64-unknown-linux-gnu" ];
});
rustToolchain = rustToolchainFor pkgs;
# NB: we don't need to overlay our custom toolchain for the *entire*
# pkgs (which would require rebuidling anything else which uses rust).
# Instead, we just want to update the scope that crane will use by appending
# our specific toolchain there.
craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchainFor;
src = craneLib.cleanCargoSource ./.;
@ -83,7 +84,6 @@
# Extra inputs can be added here; cargo and rustc are provided by default
# from the toolchain that was specified earlier.
packages = [
# rustToolchain
];
};
});

View File

@ -28,11 +28,9 @@
overlays = [ (import rust-overlay) ];
};
rustToolchain = pkgs.rust-bin.stable.latest.default.override {
craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default.override {
targets = [ "x86_64-unknown-linux-musl" ];
};
craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
});
my-crate = craneLib.buildPackage {
src = craneLib.cleanCargoSource ./.;

View File

@ -31,11 +31,7 @@
overlays = [ (import rust-overlay) ];
};
rustToolchain = pkgs.pkgsBuildHost.rust-bin.stable.latest.default.override {
targets = [ "aarch64-unknown-linux-gnu" ];
};
craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default);
# Note: we have to use the `callPackage` approach here so that Nix
# can "splice" the packages in such a way that dependencies are

View File

@ -28,15 +28,13 @@
overlays = [ (import rust-overlay) ];
};
rustWithWasiTarget = pkgs.rust-bin.stable.latest.default.override {
targets = [ "wasm32-wasi" ];
};
# NB: we don't need to overlay our custom toolchain for the *entire*
# pkgs (which would require rebuidling anything else which uses rust).
# Instead, we just want to update the scope that crane will use by appending
# our specific toolchain there.
craneLib = (crane.mkLib pkgs).overrideToolchain rustWithWasiTarget;
craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default.override {
targets = [ "wasm32-wasi" ];
});
my-crate = craneLib.buildPackage {
src = craneLib.cleanCargoSource ./.;
@ -76,7 +74,6 @@
# Extra inputs can be added here; cargo and rustc are provided by default
# from the toolchain that was specified earlier.
packages = [
# rustWithWasiTarget
];
};
});

View File

@ -27,8 +27,7 @@
};
inherit (pkgs) lib;
rustToolchain = pkgs.rust-bin.stable.latest.default;
craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;
craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default);
src = craneLib.cleanCargoSource ./.;
workspace = craneLib.buildPackage {

View File

@ -34,12 +34,12 @@
inherit (pkgs) lib;
rustToolchain = pkgs.rust-bin.stable.latest.default.override {
rustToolchainFor = p: p.rust-bin.stable.latest.default.override {
# Set the build targets supported by the toolchain,
# wasm32-unknown-unknown is required for trunk.
targets = [ "wasm32-unknown-unknown" ];
};
craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchain).overrideScope (_final: _prev: {
craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchainFor).overrideScope (_final: _prev: {
# The version of wasm-bindgen-cli needs to match the version in Cargo.lock. You
# can unpin this if your nixpkgs commit contains the appropriate wasm-bindgen-cli version
inherit (import nixpkgs-for-wasm-bindgen { inherit system; }) wasm-bindgen-cli;

View File

@ -34,12 +34,12 @@
inherit (pkgs) lib;
rustToolchain = pkgs.rust-bin.stable.latest.default.override {
rustToolchainFor = p: p.rust-bin.stable.latest.default.override {
# Set the build targets supported by the toolchain,
# wasm32-unknown-unknown is required for trunk
targets = [ "wasm32-unknown-unknown" ];
};
craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchain).overrideScope (_final: _prev: {
craneLib = ((crane.mkLib pkgs).overrideToolchain rustToolchainFor).overrideScope (_final: _prev: {
# The version of wasm-bindgen-cli needs to match the version in Cargo.lock. You
# can unpin this if your nixpkgs commit contains the appropriate wasm-bindgen-cli version
inherit (import nixpkgs-for-wasm-bindgen { inherit system; }) wasm-bindgen-cli;

View File

@ -1,6 +1,13 @@
{ lib
, stdenv
, makeScopeWithSplicing'
, otherSplices
, splicePackages
, pkgsBuildBuild
, pkgsBuildHost
, pkgsBuildTarget
, pkgsHostHost
, pkgsHostTarget
, pkgsTargetTarget
}:
let
@ -9,85 +16,129 @@ let
isUnsupported = lib.versionOlder current minSupported;
msg = "crane requires at least nixpkgs-${minSupported}, supplied nixpkgs-${current}";
mySplice = f: makeScopeWithSplicing' {
inherit otherSplices f;
# Helps keep things in sync between `overrideToolchain` and `keep`
attrsForToolchainOverride = [
"cargo"
"clippy"
"rustc"
"rustfmt"
];
spliceToolchain = toolchainFn:
let
splices = {
pkgsBuildBuild = { toolchain = toolchainFn pkgsBuildBuild; };
pkgsBuildHost = { toolchain = toolchainFn pkgsBuildHost; };
pkgsBuildTarget = { toolchain = toolchainFn pkgsBuildTarget; };
pkgsHostHost = { toolchain = toolchainFn pkgsHostHost; };
pkgsHostTarget = { toolchain = toolchainFn pkgsHostTarget; };
pkgsTargetTarget = lib.optionalAttrs (pkgsTargetTarget?newScope) { toolchain = toolchainFn pkgsTargetTarget; };
};
in
(splicePackages splices).toolchain;
scopeFn = self:
let
inherit (self) callPackage;
internalCrateNameFromCargoToml = callPackage ./internalCrateNameFromCargoToml.nix { };
internalCrateNameForCleanSource = callPackage ./internalCrateNameForCleanSource.nix {
inherit internalCrateNameFromCargoToml;
};
in
{
appendCrateRegistries = input: self.overrideScope (_final: prev: {
crateRegistries = prev.crateRegistries // (lib.foldl (a: b: a // b) { } input);
});
buildDepsOnly = callPackage ./buildDepsOnly.nix { };
buildPackage = callPackage ./buildPackage.nix { };
buildTrunkPackage = callPackage ./buildTrunkPackage.nix { };
cargoAudit = callPackage ./cargoAudit.nix { };
cargoBuild = callPackage ./cargoBuild.nix { };
cargoClippy = callPackage ./cargoClippy.nix { };
cargoDeny = callPackage ./cargoDeny.nix { };
cargoDoc = callPackage ./cargoDoc.nix { };
cargoFmt = callPackage ./cargoFmt.nix { };
cargoHelperFunctionsHook = callPackage ./setupHooks/cargoHelperFunctions.nix { };
cargoLlvmCov = callPackage ./cargoLlvmCov.nix { };
cargoNextest = callPackage ./cargoNextest.nix { };
cargoTarpaulin = callPackage ./cargoTarpaulin.nix { };
cargoTest = callPackage ./cargoTest.nix { };
cleanCargoSource = callPackage ./cleanCargoSource.nix {
inherit internalCrateNameForCleanSource;
};
cleanCargoToml = callPackage ./cleanCargoToml.nix { };
configureCargoCommonVarsHook = callPackage ./setupHooks/configureCargoCommonVars.nix { };
configureCargoVendoredDepsHook = callPackage ./setupHooks/configureCargoVendoredDeps.nix { };
craneUtils = callPackage ../pkgs/crane-utils { };
crateNameFromCargoToml = callPackage ./crateNameFromCargoToml.nix {
inherit internalCrateNameFromCargoToml;
};
crateRegistries = self.registryFromDownloadUrl {
dl = "https://static.crates.io/crates";
indexUrl = "https://github.com/rust-lang/crates.io-index";
};
devShell = callPackage ./devShell.nix { };
downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
filterCargoSources = callPackage ./filterCargoSources.nix { };
findCargoFiles = callPackage ./findCargoFiles.nix { };
inheritCargoArtifactsHook = callPackage ./setupHooks/inheritCargoArtifacts.nix { };
installCargoArtifactsHook = callPackage ./setupHooks/installCargoArtifacts.nix { };
installFromCargoBuildLogHook = callPackage ./setupHooks/installFromCargoBuildLog.nix { };
mkCargoDerivation = callPackage ./mkCargoDerivation.nix { };
mkDummySrc = callPackage ./mkDummySrc.nix { };
overrideToolchain = toolchainArg: self.overrideScope (_final: _prev:
let
toolchain =
if lib.isFunction toolchainArg
then spliceToolchain toolchainArg
else toolchainArg;
needsSplicing = stdenv.buildPlatform != stdenv.hostPlatform && toolchain?__spliced == false;
warningMsg = ''
craneLib.overrideToolchain requires a spliced toolchain when cross-compiling. Consider specifying
a function which constructs a toolchain for any given `pkgs` instantiation:
(crane.mkLib pkgs).overrideToolchain (p: ...)
'';
in
lib.warnIf needsSplicing warningMsg (lib.genAttrs attrsForToolchainOverride (_: toolchain))
);
path = callPackage ./path.nix {
inherit internalCrateNameForCleanSource;
};
registryFromDownloadUrl = callPackage ./registryFromDownloadUrl.nix { };
registryFromGitIndex = callPackage ./registryFromGitIndex.nix { };
registryFromSparse = callPackage ./registryFromSparse.nix { };
removeReferencesToVendoredSourcesHook = callPackage ./setupHooks/removeReferencesToVendoredSources.nix { };
replaceCargoLockHook = callPackage ./setupHooks/replaceCargoLockHook.nix { };
urlForCargoPackage = callPackage ./urlForCargoPackage.nix { };
vendorCargoDeps = callPackage ./vendorCargoDeps.nix { };
vendorMultipleCargoDeps = callPackage ./vendorMultipleCargoDeps.nix { };
vendorCargoRegistries = callPackage ./vendorCargoRegistries.nix { };
vendorGitDeps = callPackage ./vendorGitDeps.nix { };
writeTOML = callPackage ./writeTOML.nix { };
};
craneSpliced = makeScopeWithSplicing' {
f = scopeFn;
otherSplices = {
selfBuildBuild = lib.makeScope pkgsBuildBuild.newScope scopeFn;
selfBuildHost = lib.makeScope pkgsBuildHost.newScope scopeFn;
selfBuildTarget = lib.makeScope pkgsBuildTarget.newScope scopeFn;
selfHostHost = lib.makeScope pkgsHostHost.newScope scopeFn;
selfHostTarget = lib.makeScope pkgsHostTarget.newScope scopeFn;
selfTargetTarget = lib.optionalAttrs (pkgsTargetTarget?newScope) (lib.makeScope pkgsTargetTarget.newScope scopeFn);
};
keep = self: lib.optionalAttrs (self?cargo)
(lib.genAttrs attrsForToolchainOverride (n: self.${n}));
};
in
lib.warnIf isUnsupported msg (mySplice (self:
let
inherit (self) callPackage;
internalCrateNameFromCargoToml = callPackage ./internalCrateNameFromCargoToml.nix { };
internalCrateNameForCleanSource = callPackage ./internalCrateNameForCleanSource.nix {
inherit internalCrateNameFromCargoToml;
};
in
{
appendCrateRegistries = input: self.overrideScope (_final: prev: {
crateRegistries = prev.crateRegistries // (lib.foldl (a: b: a // b) { } input);
});
buildDepsOnly = callPackage ./buildDepsOnly.nix { };
buildPackage = callPackage ./buildPackage.nix { };
buildTrunkPackage = callPackage ./buildTrunkPackage.nix { };
cargoAudit = callPackage ./cargoAudit.nix { };
cargoBuild = callPackage ./cargoBuild.nix { };
cargoClippy = callPackage ./cargoClippy.nix { };
cargoDeny = callPackage ./cargoDeny.nix { };
cargoDoc = callPackage ./cargoDoc.nix { };
cargoFmt = callPackage ./cargoFmt.nix { };
cargoHelperFunctionsHook = callPackage ./setupHooks/cargoHelperFunctions.nix { };
cargoLlvmCov = callPackage ./cargoLlvmCov.nix { };
cargoNextest = callPackage ./cargoNextest.nix { };
cargoTarpaulin = callPackage ./cargoTarpaulin.nix { };
cargoTest = callPackage ./cargoTest.nix { };
cleanCargoSource = callPackage ./cleanCargoSource.nix {
inherit internalCrateNameForCleanSource;
};
cleanCargoToml = callPackage ./cleanCargoToml.nix { };
configureCargoCommonVarsHook = callPackage ./setupHooks/configureCargoCommonVars.nix { };
configureCargoVendoredDepsHook = callPackage ./setupHooks/configureCargoVendoredDeps.nix { };
devShell = callPackage ./devShell.nix { };
crateNameFromCargoToml = callPackage ./crateNameFromCargoToml.nix {
inherit internalCrateNameFromCargoToml;
};
crateRegistries = self.registryFromDownloadUrl {
dl = "https://static.crates.io/crates";
indexUrl = "https://github.com/rust-lang/crates.io-index";
};
downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
filterCargoSources = callPackage ./filterCargoSources.nix { };
findCargoFiles = callPackage ./findCargoFiles.nix { };
inheritCargoArtifactsHook = callPackage ./setupHooks/inheritCargoArtifacts.nix { };
installCargoArtifactsHook = callPackage ./setupHooks/installCargoArtifacts.nix { };
installFromCargoBuildLogHook = callPackage ./setupHooks/installFromCargoBuildLog.nix { };
mkCargoDerivation = callPackage ./mkCargoDerivation.nix { };
mkDummySrc = callPackage ./mkDummySrc.nix { };
overrideToolchain = toolchain: self.overrideScope (_final: _prev: {
cargo = toolchain;
clippy = toolchain;
rustc = toolchain;
rustfmt = toolchain;
});
path = callPackage ./path.nix {
inherit internalCrateNameForCleanSource;
};
registryFromDownloadUrl = callPackage ./registryFromDownloadUrl.nix { };
registryFromGitIndex = callPackage ./registryFromGitIndex.nix { };
registryFromSparse = callPackage ./registryFromSparse.nix { };
removeReferencesToVendoredSourcesHook = callPackage ./setupHooks/removeReferencesToVendoredSources.nix { };
replaceCargoLockHook = callPackage ./setupHooks/replaceCargoLockHook.nix { };
urlForCargoPackage = callPackage ./urlForCargoPackage.nix { };
vendorCargoDeps = callPackage ./vendorCargoDeps.nix { };
vendorMultipleCargoDeps = callPackage ./vendorMultipleCargoDeps.nix { };
vendorCargoRegistries = callPackage ./vendorCargoRegistries.nix { };
vendorGitDeps = callPackage ./vendorGitDeps.nix { };
writeTOML = callPackage ./writeTOML.nix { };
}))
lib.warnIf isUnsupported msg (craneSpliced)

View File

@ -1,5 +1,6 @@
{ lib
, cargo
, craneUtils
, jq
, pkgsBuildBuild
}:
@ -8,8 +9,6 @@ let
inherit (pkgsBuildBuild)
fetchgit
stdenv;
craneUtils = pkgsBuildBuild.callPackage ../pkgs/crane-utils { };
in
{ git
, rev