git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #95542 from aaronjanse/aj-rust-custom-target

buildRustPackage: support custom targets

commit304913841bb2be5f6bd9634c7d7f8c1e38a4b7ea authorJohn Ericson <git@JohnEricson.me> dateSat 28 Nov 2020, 17:20 (6 years ago) parentaed3b7cb7650aa
14 files changed+309−21
doc/languages-frameworks/rust.section.md+55−3 pkgs/build-support/rust/build-rust-crate/build-crate.nix+1−1 pkgs/build-support/rust/build-rust-crate/configure-crate.nix+2−2 pkgs/build-support/rust/default.nix+33−7 pkgs/build-support/rust/sysroot/Cargo.lock+29 pkgs/build-support/rust/sysroot/cargo.py+45 pkgs/build-support/rust/sysroot/default.nix+41 pkgs/build-support/rust/sysroot/update-lockfile.sh+21 pkgs/development/compilers/rust/default.nix+10−2 pkgs/development/compilers/rust/rust-src.nix+5−2 pkgs/development/compilers/rust/rustc.nix+3−3 pkgs/development/libraries/relibc/default.nix+2−1 pkgs/test/default.nix+2 pkgs/test/rust-sysroot/default.nix+60
doc/languages-frameworks/rust.section.md+55−3
@@ -63,9 +63,52 @@ The fetcher will verify that the `Cargo.lock` file is in sync with the `src`
6363attribute, and fail the build if not. It will also will compress the vendor
6464directory into a tar.gz archive.
6565
66−### Building a crate for a different target
67−
68−To build your crate with a different cargo `--target` simply specify the `target` attribute:
66+### Cross compilation
67+
68+By default, Rust packages are compiled for the host platform, just like any
69+other package is. The `--target` passed to rust tools is computed from this.
70+By default, it takes the `stdenv.hostPlatform.config` and replaces components
71+where they are known to differ. But there are ways to customize the argument:
72+
73+ - To choose a different target by name, define
74+ `stdenv.hostPlatform.rustc.config` as that name (a string), and that
75+ name will be used instead.
76+
77+ For example:
78+ ```nix
79+ import <nixpkgs> {
80+ crossSystem = (import <nixpkgs/lib>).systems.examples.armhf-embedded // {
81+ rustc.config = "thumbv7em-none-eabi";
82+ };
83+ }
84+ ```
85+ will result in:
86+ ```shell
87+ --target thumbv7em-none-eabi
88+ ```
89+
90+ - To pass a completely custom target, define
91+ `stdenv.hostPlatform.rustc.config` with its name, and
92+ `stdenv.hostPlatform.rustc.platform` with the value. The value will be
93+ serialized to JSON in a file called
94+ `${stdenv.hostPlatform.rustc.config}.json`, and the path of that file
95+ will be used instead.
96+
97+ For example:
98+ ```nix
99+ import <nixpkgs> {
100+ crossSystem = (import <nixpkgs/lib>).systems.examples.armhf-embedded // {
101+ rustc.config = "thumb-crazy";
102+ rustc.platform = { foo = ""; bar = ""; };
103+ };
104+ }
105+ will result in:
106+ ```shell
107+ --target /nix/store/asdfasdfsadf-thumb-crazy.json # contains {"foo":"","bar":""}
108+ ```
109+
110+Finally, as an ad-hoc escape hatch, a computed target (string or JSON file
111+path) can be passed directly to `buildRustPackage`:
69112
70113```nix
71114pkgs.rustPlatform.buildRustPackage {
@@ -74,6 +117,15 @@ pkgs.rustPlatform.buildRustPackage {
74117}
75118```
76119
120+This is useful to avoid rebuilding Rust tools, since they are actually target
121+agnostic and don't need to be rebuilt. But in the future, we should always
122+build the Rust tools and standard library crates separately so there is no
123+reason not to take the `stdenv.hostPlatform.rustc`-modifying approach, and the
124+ad-hoc escape hatch to `buildRustPackage` can be removed.
125+
126+Note that currently custom targets aren't compiled with `std`, so `cargo test`
127+will fail. This can be ignored by adding `doCheck = false;` to your derivation.
128+
77129### Running package tests
78130
79131When using `buildRustPackage`, the `checkPhase` is enabled by default and runs
pkgs/build-support/rust/build-rust-crate/build-crate.nix+1−1
@@ -15,7 +15,7 @@
1515 ++ [(mkRustcDepArgs dependencies crateRenames)]
1616 ++ [(mkRustcFeatureArgs crateFeatures)]
1717 ++ extraRustcOpts
18− ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTarget stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc"
18+ ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTargetSpec stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc"
1919 # since rustc 1.42 the "proc_macro" crate is part of the default crate prelude
2020 # https://github.com/rust-lang/cargo/commit/4d64eb99a4#diff-7f98585dbf9d30aa100c8318e2c77e79R1021-R1022
2121 ++ lib.optional (lib.elem "proc-macro" crateType) "--extern proc_macro"
pkgs/build-support/rust/build-rust-crate/configure-crate.nix+2−2
@@ -135,8 +135,8 @@ in ''
135135 export CARGO_MANIFEST_DIR=$(pwd)
136136 export DEBUG="${toString (!release)}"
137137 export OPT_LEVEL="${toString optLevel}"
138− export TARGET="${rust.toRustTarget stdenv.hostPlatform}"
139− export HOST="${rust.toRustTarget stdenv.buildPlatform}"
138+ export TARGET="${rust.toRustTargetSpec stdenv.hostPlatform}"
139+ export HOST="${rust.toRustTargetSpec stdenv.buildPlatform}"
140140 export PROFILE=${if release then "release" else "debug"}
141141 export OUT_DIR=$(pwd)/target/build/${crateName}.out
142142 export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0}
pkgs/build-support/rust/default.nix+33−7
@@ -4,6 +4,10 @@
44, cargo
55, diffutils
66, fetchCargoTarball
7+, runCommandNoCC
8+, rustPlatform
9+, callPackage
10+, remarshal
711, git
812, rust
913, rustc
@@ -26,12 +30,15 @@
2630, cargoBuildFlags ? []
2731, buildType ? "release"
2832, meta ? {}
29−, target ? null
33+, target ? rust.toRustTargetSpec stdenv.hostPlatform
3034, cargoVendorDir ? null
3135, checkType ? buildType
3236, depsExtraArgs ? {}
3337, cargoParallelTestThreads ? true
3438
39+# Toggles whether a custom sysroot is created when the target is a .json file.
40+, __internal_dontAddSysroot ? false
41+
3542# Needed to `pushd`/`popd` into a subdir of a tarball if this subdir
3643# contains a Cargo.toml, but isn't part of a workspace (which is e.g. the
3744# case for `rustfmt`/etc from the `rust-sources).
@@ -69,13 +76,26 @@ let
6976 cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
7077 '';
7178
72− rustTarget = if target == null then rust.toRustTarget stdenv.hostPlatform else target;
79+ targetIsJSON = stdenv.lib.hasSuffix ".json" target;
80+ useSysroot = targetIsJSON && !__internal_dontAddSysroot;
81+
82+ # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168
83+ # the "${}" is needed to transform the path into a /nix/store path before baseNameOf
84+ shortTarget = if targetIsJSON then
85+ (stdenv.lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
86+ else target;
87+
88+ sysroot = (callPackage ./sysroot {}) {
89+ inherit target shortTarget;
90+ RUSTFLAGS = args.RUSTFLAGS or "";
91+ originalCargoToml = src + /Cargo.toml; # profile info is later extracted
92+ };
7393
7494 ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
7595 cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
7696 ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
7797 cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
78− releaseDir = "target/${rustTarget}/${buildType}";
98+ releaseDir = "target/${shortTarget}/${buildType}";
7999 tmpDir = "${releaseDir}-tmp";
80100
81101 # Specify the stdenv's `diff` by abspath to ensure that the user's build
@@ -85,7 +105,13 @@ let
85105
86106in
87107
88−stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
108+# Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`.
109+# See https://os.phil-opp.com/testing/ for more information.
110+assert useSysroot -> !(args.doCheck or true);
111+
112+stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // stdenv.lib.optionalAttrs useSysroot {
113+ RUSTFLAGS = "--sysroot ${sysroot} " + (args.RUSTFLAGS or "");
114+} // {
89115 inherit cargoDeps;
90116
91117 patchRegistryDeps = ./patch-registry-deps;
@@ -115,7 +141,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
115141 [target."${rust.toRustTarget stdenv.buildPlatform}"]
116142 "linker" = "${ccForBuild}"
117143 ${stdenv.lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
118− [target."${rustTarget}"]
144+ [target."${shortTarget}"]
119145 "linker" = "${ccForHost}"
120146 ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
121147 stdenv.lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
@@ -185,7 +211,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
185211 "CXX_${rust.toRustTarget stdenv.hostPlatform}"="${cxxForHost}" \
186212 cargo build -j $NIX_BUILD_CORES \
187213 ${stdenv.lib.optionalString (buildType == "release") "--release"} \
188− --target ${rustTarget} \
214+ --target ${target} \
189215 --frozen ${concatStringsSep " " cargoBuildFlags}
190216 )
191217
@@ -205,7 +231,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
205231 '';
206232
207233 checkPhase = args.checkPhase or (let
208− argstr = "${stdenv.lib.optionalString (checkType == "release") "--release"} --target ${rustTarget} --frozen";
234+ argstr = "${stdenv.lib.optionalString (checkType == "release") "--release"} --target ${target} --frozen";
209235 threads = if cargoParallelTestThreads then "$NIX_BUILD_CORES" else "1";
210236 in ''
211237 ${stdenv.lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"}
pkgs/build-support/rust/sysroot/Cargo.locknew file+29
@@ -0,0 +1,29 @@
1+# This file is automatically @generated by Cargo.
2+# It is not intended for manual editing.
3+[[package]]
4+name = "alloc"
5+version = "0.0.0"
6+dependencies = [
7+ "compiler_builtins",
8+ "core",
9+]
10+
11+[[package]]
12+name = "compiler_builtins"
13+version = "0.1.36"
14+source = "registry+https://github.com/rust-lang/crates.io-index"
15+checksum = "7cd0782e0a7da7598164153173e5a5d4d9b1da094473c98dce0ff91406112369"
16+dependencies = [
17+ "rustc-std-workspace-core",
18+]
19+
20+[[package]]
21+name = "core"
22+version = "0.0.0"
23+
24+[[package]]
25+name = "rustc-std-workspace-core"
26+version = "1.99.0"
27+dependencies = [
28+ "core",
29+]
pkgs/build-support/rust/sysroot/cargo.pynew file+45
@@ -0,0 +1,45 @@
1+import os
2+import toml
3+
4+rust_src = os.environ['RUSTC_SRC']
5+orig_cargo = os.environ['ORIG_CARGO'] if 'ORIG_CARGO' in os.environ else None
6+
7+base = {
8+ 'package': {
9+ 'name': 'alloc',
10+ 'version': '0.0.0',
11+ 'authors': ['The Rust Project Developers'],
12+ 'edition': '2018',
13+ },
14+ 'dependencies': {
15+ 'compiler_builtins': {
16+ 'version': '0.1.0',
17+ 'features': ['rustc-dep-of-std', 'mem'],
18+ },
19+ 'core': {
20+ 'path': os.path.join(rust_src, 'libcore'),
21+ },
22+ },
23+ 'lib': {
24+ 'name': 'alloc',
25+ 'path': os.path.join(rust_src, 'liballoc/lib.rs'),
26+ },
27+ 'patch': {
28+ 'crates-io': {
29+ 'rustc-std-workspace-core': {
30+ 'path': os.path.join(rust_src, 'tools/rustc-std-workspace-core'),
31+ },
32+ },
33+ },
34+}
35+
36+if orig_cargo is not None:
37+ with open(orig_cargo, 'r') as f:
38+ src = toml.loads(f.read())
39+ if 'profile' in src:
40+ base['profile'] = src['profile']
41+
42+out = toml.dumps(base)
43+
44+with open('Cargo.toml', 'x') as f:
45+ f.write(out)
pkgs/build-support/rust/sysroot/default.nixnew file+41
@@ -0,0 +1,41 @@
1+{ stdenv, rust, rustPlatform, buildPackages }:
2+
3+{ shortTarget, originalCargoToml, target, RUSTFLAGS }:
4+
5+let
6+ cargoSrc = stdenv.mkDerivation {
7+ name = "cargo-src";
8+ preferLocalBuild = true;
9+ phases = [ "installPhase" ];
10+ installPhase = ''
11+ RUSTC_SRC=${rustPlatform.rustcSrc.override { minimalContent = false; }} ORIG_CARGO=${originalCargoToml} \
12+ ${buildPackages.python3.withPackages (ps: with ps; [ toml ])}/bin/python3 ${./cargo.py}
13+ mkdir -p $out
14+ cp Cargo.toml $out/Cargo.toml
15+ cp ${./Cargo.lock} $out/Cargo.lock
16+ '';
17+ };
18+in rustPlatform.buildRustPackage {
19+ inherit target RUSTFLAGS;
20+
21+ name = "custom-sysroot";
22+ src = cargoSrc;
23+
24+ RUSTC_BOOTSTRAP = 1;
25+ __internal_dontAddSysroot = true;
26+ cargoSha256 = "0y6dqfhsgk00y3fv5bnjzk0s7i30nwqc1rp0xlrk83hkh80x81mw";
27+
28+ doCheck = false;
29+
30+ installPhase = ''
31+ export LIBS_DIR=$out/lib/rustlib/${shortTarget}/lib
32+ mkdir -p $LIBS_DIR
33+ for f in target/${shortTarget}/release/deps/*.{rlib,rmeta}; do
34+ cp $f $LIBS_DIR
35+ done
36+
37+ export RUST_SYSROOT=$(rustc --print=sysroot)
38+ host=${rust.toRustTarget stdenv.buildPlatform}
39+ cp -r $RUST_SYSROOT/lib/rustlib/$host $out
40+ '';
41+}
pkgs/build-support/rust/sysroot/update-lockfile.shnew file+21
@@ -0,0 +1,21 @@
1+#!/usr/bin/env nix-shell
2+#!nix-shell -i bash -p python3 python3.pkgs.toml cargo
3+
4+set -e
5+
6+HERE=$(dirname "${BASH_SOURCE[0]}")
7+NIXPKGS_ROOT="$HERE/../../../.."
8+
9+# https://unix.stackexchange.com/a/84980/390173
10+tempdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-lockfile')
11+
12+cd "$tempdir"
13+nix-build -E "with import (/. + \"${NIXPKGS_ROOT}\") {}; pkgs.rustPlatform.rustcSrc.override { minimalContent = false; }"
14+RUSTC_SRC="$(pwd)/result" python3 "$HERE/cargo.py"
15+RUSTC_BOOTSTRAP=1 cargo build || echo "Build failure is expected. All that's needed is the lockfile."
16+
17+cp Cargo.lock "$HERE"
18+
19+rm -rf "$tempdir"
20+
21+
pkgs/development/compilers/rust/default.nix+10−2
@@ -24,9 +24,10 @@
2424 if platform.isDarwin then "macos"
2525 else platform.parsed.kernel.name;
2626
27− # Target triple. Rust has slightly different naming conventions than we use.
27+ # Returns the name of the rust target, even if it is custom. Adjustments are
28+ # because rust has slightly different naming conventions than we do.
2829 toRustTarget = platform: with platform.parsed; let
29− cpu_ = platform.rustc.arch or {
30+ cpu_ = platform.rustc.platform.arch or {
3031 "armv7a" = "armv7";
3132 "armv7l" = "armv7";
3233 "armv6l" = "arm";
@@ -34,6 +35,13 @@
3435 in platform.rustc.config
3536 or "${cpu_}-${vendor.name}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}";
3637
38+ # Returns the name of the rust target if it is standard, or the json file
39+ # containing the custom target spec.
40+ toRustTargetSpec = platform:
41+ if (platform.rustc or {}) ? platform
42+ then builtins.toFile (toRustTarget platform + ".json") (builtins.toJSON platform.rustc.platform)
43+ else toRustTarget platform;
44+
3745 # This just contains tools for now. But it would conceivably contain
3846 # libraries too, say if we picked some default/recommended versions from
3947 # `cratesIO` to build by Hydra and/or try to prefer/bias in Cargo.lock for
pkgs/development/compilers/rust/rust-src.nix+5−2
@@ -1,4 +1,4 @@
1−{ stdenv, rustc }:
1+{ stdenv, rustc, minimalContent ? true }:
22
33stdenv.mkDerivation {
44 name = "rust-src";
@@ -6,6 +6,9 @@ stdenv.mkDerivation {
66 phases = [ "unpackPhase" "installPhase" ];
77 installPhase = ''
88 mv src $out
9− rm -rf $out/{ci,doc,etc,grammar,llvm-project,llvm-emscripten,rtstartup,rustllvm,test,tools,vendor,stdarch}
9+ rm -rf $out/{${if minimalContent
10+ then "ci,doc,etc,grammar,llvm-project,llvm-emscripten,rtstartup,rustllvm,test,tools,vendor,stdarch"
11+ else "ci,doc,etc,grammar,llvm-project,llvm-emscripten,rtstartup,rustllvm,test,vendor"
12+ }}
1013 '';
1114}
pkgs/development/compilers/rust/rustc.nix+3−3
@@ -70,9 +70,9 @@ in stdenv.mkDerivation rec {
7070 "--set=build.cargo=${rustPlatform.rust.cargo}/bin/cargo"
7171 "--enable-rpath"
7272 "--enable-vendor"
73− "--build=${rust.toRustTarget stdenv.buildPlatform}"
74− "--host=${rust.toRustTarget stdenv.hostPlatform}"
75− "--target=${rust.toRustTarget stdenv.targetPlatform}"
73+ "--build=${rust.toRustTargetSpec stdenv.buildPlatform}"
74+ "--host=${rust.toRustTargetSpec stdenv.hostPlatform}"
75+ "--target=${rust.toRustTargetSpec stdenv.targetPlatform}"
7676
7777 "${setBuild}.cc=${ccForBuild}"
7878 "${setHost}.cc=${ccForHost}"
pkgs/development/libraries/relibc/default.nix+2−1
@@ -63,7 +63,8 @@ redoxRustPlatform.buildRustPackage rec {
6363 DESTDIR=$out make install
6464 '';
6565
66− TARGET = buildPackages.rust.toRustTarget stdenvNoCC.targetPlatform;
66+ # TODO: should be hostPlatform
67+ TARGET = buildPackages.rust.toRustTargetSpec stdenvNoCC.targetPlatform;
6768
6869 cargoSha256 = "1fzz7ba3ga57x1cbdrcfrdwwjr70nh4skrpxp4j2gak2c3scj6rz";
6970
pkgs/test/default.nix+2
@@ -38,6 +38,8 @@ with pkgs;
3838
3939 cross = callPackage ./cross {};
4040
41+ rustCustomSysroot = callPackage ./rust-sysroot {};
42+
4143 nixos-functions = callPackage ./nixos-functions {};
4244
4345 patch-shebangs = callPackage ./patch-shebangs {};
pkgs/test/rust-sysroot/default.nixnew file+60
@@ -0,0 +1,60 @@
1+{ lib, rust, rustPlatform, fetchFromGitHub }:
2+
3+let
4+ mkBlogOsTest = target: rustPlatform.buildRustPackage rec {
5+ name = "blog_os-sysroot-test";
6+
7+ src = fetchFromGitHub {
8+ owner = "phil-opp";
9+ repo = "blog_os";
10+ rev = "4e38e7ddf8dd021c3cd7e4609dfa01afb827797b";
11+ sha256 = "0k9ipm9ddm1bad7bs7368wzzp6xwrhyfzfpckdax54l4ffqwljcg";
12+ };
13+
14+ cargoSha256 = "1cbcplgz28yxshyrp2krp1jphbrcqdw6wxx3rry91p7hiqyibd30";
15+
16+ inherit target;
17+
18+ RUSTFLAGS = "-C link-arg=-nostartfiles";
19+
20+ # Tests don't work for `no_std`. See https://os.phil-opp.com/testing/
21+ doCheck = false;
22+
23+ meta = with lib; {
24+ description = "Test for using custom sysroots with buildRustPackage";
25+ maintainers = with maintainers; [ aaronjanse ];
26+ platforms = lib.platforms.x86_64;
27+ };
28+ };
29+
30+ # The book uses rust-lld for linking, but rust-lld is not currently packaged for NixOS.
31+ # The justification in the book for using rust-lld suggests that gcc can still be used for testing:
32+ # > Instead of using the platform's default linker (which might not support Linux targets),
33+ # > we use the cross platform LLD linker that is shipped with Rust for linking our kernel.
34+ # https://github.com/phil-opp/blog_os/blame/7212ffaa8383122b1eb07fe1854814f99d2e1af4/blog/content/second-edition/posts/02-minimal-rust-kernel/index.md#L157
35+ targetContents = {
36+ "llvm-target" = "x86_64-unknown-none";
37+ "data-layout" = "e-m:e-i64:64-f80:128-n8:16:32:64-S128";
38+ "arch" = "x86_64";
39+ "target-endian" = "little";
40+ "target-pointer-width" = "64";
41+ "target-c-int-width" = "32";
42+ "os" = "none";
43+ "executables" = true;
44+ "linker-flavor" = "gcc";
45+ "panic-strategy" = "abort";
46+ "disable-redzone" = true;
47+ "features" = "-mmx,-sse,+soft-float";
48+ };
49+
50+in {
51+ blogOS-targetByFile = mkBlogOsTest (builtins.toFile "x86_64-blog_os.json" (builtins.toJSON targetContents));
52+ blogOS-targetByNix = let
53+ plat = lib.systems.elaborate { config = "x86_64-none"; } // {
54+ rustc = {
55+ config = "x86_64-blog_os";
56+ platform = targetContents;
57+ };
58+ };
59+ in mkBlogOsTest (rust.toRustTargetSpec plat);
60+}