rust-overlay/rust-overlay.nix
Albert Cervin 70fc350a3f
Fix cross-compilation
Dependencies were specified in `depsTargetTarget` which would make them end up in the
dependencies for the target platform when they are actually dependencies needed on the
buildPlatform using the rust package. I.e. they are target dependencies of the compiler
package but will be used as build platform dependencies when propagated.
2021-05-18 09:59:38 +02:00

628 lines
26 KiB
Nix

# Modified from: https://github.com/mozilla/nixpkgs-mozilla/blob/8c007b60731c07dd7a052cce508de3bb1ae849b4/rust-overlay.nix
# This file provide a Rust overlay, which provides pre-packaged bleeding edge versions of rustc
# and cargo.
self: super:
let
# Manifest selector.
selectManifest = { channel, date ? null }: let
inherit (self.rust-bin) manifests;
inherit (builtins) match elemAt;
assertWith = cond: msg: body: if cond then body else throw msg;
asVersion = match "[0-9]+\\.[0-9]+\\.[0-9]+" channel;
asNightlyDate = let m = match "nightly-([0-9]+-[0-9]+-[0-9]+)" channel; in
if m == null then null else elemAt m 0;
asBetaDate = let m = match "beta-([0-9]+-[0-9]+-[0-9]+)" channel; in
if m == null then null else elemAt m 0;
in
# "stable"
if channel == "stable" then
assertWith (date == null) "Stable version with specific date is not supported"
manifests.stable.latest
# "nightly"
else if channel == "nightly" then
manifests.nightly.${if date != null then date else "latest"} or (throw "Nightly ${date} is not available")
# "beta"
else if channel == "beta" then
manifests.beta.${if date != null then date else "latest"} or (throw "Beta ${date} is not available")
# "1.49.0"
else if asVersion != null then
assertWith (date == null) "Stable version with specific date is not supported"
manifests.stable.${channel} or (throw "Stable ${channel} is not available")
# "beta-2021-01-01"
else if asBetaDate != null then
assertWith (date == null) "Cannot specify date in both `channel` and `date`"
manifests.beta.${asBetaDate} or (throw "Beta ${asBetaDate} is not available")
# "nightly-2021-01-01"
else if asNightlyDate != null then
assertWith (date == null) "Cannot specify date in both `channel` and `date`"
manifests.nightly.${asNightlyDate} or (throw "Nightly ${asNightlyDate} is not available")
# Otherwise
else throw "Unknown channel: ${channel}";
# Select a toolchain and aggregate components by rustup's `rust-toolchain` file format.
# See: https://rust-lang.github.io/rustup/concepts/profiles.html
# Or see source: https://github.com/rust-lang/rustup/blob/84974df1387812269c7b29fa5f3bb1c6480a6500/doc/src/overrides.md#the-toolchain-file
fromRustupToolchain = { path ? null, channel ? null, profile ? null, components ? [], targets ? [] }:
if path != null then throw "`path` is not supported, please directly add it to your PATH instead"
else if channel == null then throw "`channel` is required"
else
let
toolchain = toolchainFromManifest (selectManifest { inherit channel; });
profile' = if profile == null then "default" else profile;
pkg =
if toolchain._profiles != {} then
toolchain._profiles.${profile'} or (throw ''
Rust ${toolchain._version} doesn't have profile `${profile'}`.
Available profiles are: ${self.lib.concatStringsSep ", " (builtins.attrNames toolchain._profiles)}
'')
# Fallback to package `rust` when profiles are not supported and not specified.
else if profile == null then
toolchain.rust
else
throw "Cannot select profile `${profile'}` since rust ${toolchain._version} is too early to support profiles";
in pkg.override {
extensions = components;
inherit targets;
};
# Same as `fromRustupToolchain` but read from a `rust-toolchain` file (legacy one-line string or in TOML).
fromRustupToolchainFile = path: let
inherit (builtins) readFile match fromTOML head;
content = readFile path;
legacy = match "([^\r\n]+)\r?\n?" content;
in if legacy != null
then fromRustupToolchain { channel = head legacy; }
else fromRustupToolchain (fromTOML content).toolchain;
getComponentsWithFixedPlatform = pkgs: pkgname: stdenv:
let
pkg = pkgs.${pkgname};
srcInfo = pkg.target.${super.rust.toRustTarget stdenv.targetPlatform} or pkg.target."*";
components = srcInfo.components or [];
componentNamesList =
builtins.map (pkg: pkg.pkg) (builtins.filter (pkg: (pkg.target != "*")) components);
in
componentNamesList;
getExtensions = pkgs: pkgname: stdenv:
let
inherit (super.lib) unique;
pkg = pkgs.${pkgname};
rustTarget = super.rust.toRustTarget stdenv.targetPlatform;
srcInfo = pkg.target.${rustTarget} or pkg.target."*" or (throw "${pkgname} is no available");
extensions = srcInfo.extensions or [];
extensionNamesList = unique (builtins.map (pkg: pkg.pkg) extensions);
in
extensionNamesList;
hasTarget = pkgs: pkgname: target:
pkgs ? ${pkgname}.target.${target};
getTuples = pkgs: name: targets:
builtins.map (target: { inherit name target; }) (builtins.filter (target: hasTarget pkgs name target) targets);
# In the manifest, a package might have different components which are bundled with it, as opposed as the extensions which can be added.
# By default, a package will include the components for the same architecture, and offers them as extensions for other architectures.
#
# This functions returns a list of { name, target } attribute sets, which includes the current system package, and all its components for the selected targets.
# The list contains the package for the pkgTargets as well as the packages for components for all compTargets
getTargetPkgTuples = pkgs: pkgname: pkgTargets: compTargets: stdenv:
let
inherit (builtins) elem;
inherit (super.lib) intersectLists;
components = getComponentsWithFixedPlatform pkgs pkgname stdenv;
extensions = getExtensions pkgs pkgname stdenv;
compExtIntersect = intersectLists components extensions;
tuples = (getTuples pkgs pkgname pkgTargets) ++ (builtins.map (name: getTuples pkgs name compTargets) compExtIntersect);
in
tuples;
getFetchUrl = pkgs: pkgname: target: stdenv: fetchurl:
let
srcInfo = pkgs.${pkgname}.target.${target};
in
mkComponentSrc {
url = srcInfo.xz_url;
sha256 = srcInfo.xz_hash;
inherit fetchurl;
};
mkComponentSrc = { url, sha256, fetchurl }:
let
inherit (builtins) match elemAt;
url' = builtins.replaceStrings [" "] ["%20"] url; # This is required or download will fail.
# Filter names like `llvm-tools-1.34.2 (6c2484dc3 2019-05-13)-aarch64-unknown-linux-gnu.tar.xz`
matchParenPart = match ".*/([^ /]*) [(][^)]*[)](.*)" url;
name = if matchParenPart == null then "" else (elemAt matchParenPart 0) + (elemAt matchParenPart 1);
in
fetchurl { inherit name sha256; url = url'; };
checkMissingExtensions = pkgs: pkgname: stdenv: extensions:
let
inherit (builtins) head;
inherit (super.lib) concatStringsSep subtractLists;
availableExtensions = getExtensions pkgs pkgname stdenv;
missingExtensions = subtractLists availableExtensions extensions;
extensionsToInstall =
if missingExtensions == [] then extensions else throw ''
While compiling ${pkgname}: the extension ${head missingExtensions} is not available.
Select extensions from the following list:
${concatStringsSep "\n" availableExtensions}'';
in
extensionsToInstall;
getComponents = pkgs: pkgname: targets: extensions: targetExtensions: stdenv: fetchurl:
let
inherit (builtins) head map;
inherit (super.lib) flatten remove subtractLists unique;
targetExtensionsToInstall = checkMissingExtensions pkgs pkgname stdenv targetExtensions;
extensionsToInstall = checkMissingExtensions pkgs pkgname stdenv extensions;
hostTargets = [ "*" (super.rust.toRustTarget stdenv.hostPlatform) (super.rust.toRustTarget stdenv.targetPlatform) ];
pkgTuples = flatten (getTargetPkgTuples pkgs pkgname hostTargets targets stdenv);
extensionTuples = flatten (map (name: getTargetPkgTuples pkgs name hostTargets targets stdenv) extensionsToInstall);
targetExtensionTuples = flatten (map (name: getTargetPkgTuples pkgs name targets targets stdenv) targetExtensionsToInstall);
pkgsTuples = pkgTuples ++ extensionTuples ++ targetExtensionTuples;
missingTargets = subtractLists (map (tuple: tuple.target) pkgsTuples) (remove "*" targets);
pkgsTuplesToInstall =
if missingTargets == [] then pkgsTuples else throw ''
While compiling ${pkgname}: the target ${head missingTargets} is not available for any package.'';
in
map (tuple: { name = tuple.name; src = (getFetchUrl pkgs tuple.name tuple.target stdenv fetchurl); }) pkgsTuplesToInstall;
mkComponent = { pname, version, src, rustc /* some components depend on rustc */ }:
self.stdenv.mkDerivation {
inherit pname version src;
# No point copying src to a build server, then copying back the
# entire unpacked contents after just a little twiddling.
preferLocalBuild = true;
nativeBuildInputs = [ self.cpio ];
installPhase = ''
runHook preInstall
installerVersion=$(< ./rust-installer-version)
if [[ "$installerVersion" != 3 ]]; then
echo "Unknown installer version: $installerVersion"
fi
while read -r comp; do
echo "Installing component $comp"
# Use cpio with file list instead of forking tons of cp.
cut -d: -f2 <"$comp/manifest.in" | cpio --quiet -pdD "$comp" "$out"
done <./components
runHook postInstall
'';
# This code is inspired by patchelf/setup-hook.sh to iterate over all binaries.
preFixup =
let
inherit (super.lib) optionalString;
inherit (self.stdenv) hostPlatform;
in
optionalString hostPlatform.isLinux ''
setInterpreter() {
local dir="$1"
[ -e "$dir" ] || return 0
header "Patching interpreter of ELF executables and libraries in $dir"
local i
while IFS= read -r -d ''$'\0' i; do
if [[ "$i" =~ .build-id ]]; then continue; fi
if ! isELF "$i"; then continue; fi
echo "setting interpreter of $i"
if [[ -x "$i" ]]; then
# Handle executables
patchelf \
--set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
--set-rpath "${super.lib.makeLibraryPath [ self.zlib ]}:$out/lib" \
"$i" || true
else
# Handle libraries
patchelf \
--set-rpath "${super.lib.makeLibraryPath [ self.zlib ]}:$out/lib" \
"$i" || true
fi
done < <(find "$dir" -type f -print0)
}
setInterpreter $out
'' + optionalString (pname == "clippy-preview" || pname == "rls-preview") ''
for f in $out/bin/*; do
${optionalString hostPlatform.isLinux ''
patchelf \
--set-rpath "${rustc}/lib:${super.lib.makeLibraryPath [ self.zlib ]}:$out/lib" \
"$f" || true
''}
${optionalString hostPlatform.isDarwin ''
install_name_tool \
-add_rpath "${rustc}/lib" \
"$f" || true
''}
done
'' + optionalString (pname == "llvm-tools-preview" && hostPlatform.isLinux) ''
dir="$out/lib/rustlib/${super.rust.toRustTarget hostPlatform}"
for f in "$dir"/bin/*; do
patchelf --set-rpath "$dir/lib" "$f" || true
done
'';
postFixup = ''
# Function moves well-known files from etc/
handleEtc() {
if [[ -d "$1" ]]; then
mkdir -p "$(dirname "$2")"
mv -T "$1" "$2"
fi
}
if [[ -e "$out/etc" ]]; then
handleEtc "$out/etc/bash_completion.d" "$out/share/bash-completion/completions"
rmdir $out/etc || { echo "Installer tries to install to /etc: $(ls $out/etc)"; exit 1; }
fi
'';
dontStrip = true;
};
aggregateComponents = { pname, version, components }:
self.pkgs.symlinkJoin {
name = pname + "-" + version;
inherit pname version;
paths = components;
postBuild = ''
# If rustc or rustdoc is in the derivation, we need to copy their
# executable into the final derivation. This is required
# for making them find the correct SYSROOT.
for target in $out/bin/{rustc,rustdoc,miri}; do
if [ -e $target ]; then
cp --remove-destination "$(realpath -e $target)" $target
fi
done
if [ -e $out/bin/cargo-miri ]; then
cargo_miri=$(readlink $out/bin/cargo-miri)
cp -f ${./cargo-miri-wrapper.sh} $out/bin/cargo-miri
chmod +w $out/bin/cargo-miri
substituteInPlace $out/bin/cargo-miri \
--replace "@bash@" "${self.pkgs.bash}/bin/bash" \
--replace "@miri@" "$cargo_miri" \
--replace "@out@" "$out"
fi
# `symlinkJoin` (`runCommand`) doesn't handle propagated dependencies.
# Need to do it manually.
mkdir -p "$out/nix-support"
echo "$propagatedBuildInputs" > "$out/nix-support/propagated-build-inputs"
if [[ -n "$propagatedNativeBuildInputs" ]]; then
echo "$propagatedNativeBuildInputs" > "$out/nix-support/propagated-native-build-inputs"
fi
'';
# FIXME: If these propagated dependencies go components, darwin build will fail with "`-liconv` not found".
# use propagatedNativebuildinputs here because we want these dependencies to end up
# as buildPlatform dependencies on the derivation using the compiler even though
# they are targetPlatform dependencies from the compiler's perspective
propagatedNativeBuildInputs = [ self.stdenv.cc self.targetPackages.stdenv.cc ] ++
self.lib.optional (self.stdenv.targetPlatform.isDarwin) self.targetPackages.libiconv;
meta.platforms = self.lib.platforms.all;
};
# Resolve final components to install from mozilla-overlay style `extensions`, `targets` and `targetExtensions`.
#
# `componentSet` has a layout of `componentSet.<name>.<rust-target> : Derivation`.
# `targetComponentsList` is a list of all component names for target platforms.
# `name` is only used for error message.
#
# Returns a list of component derivations, or throw if failed.
resolveComponents = { name, componentSet, targetComponentsList, extensions, targets, targetExtensions }:
let
inherit (self.lib) flatten elem isString filter any remove concatStringsSep concatMapStrings attrNames;
rustHostPlatform = self.rust.toRustTarget self.stdenv.hostPlatform;
collectComponentTargets = compName: comp:
# Platform irrelevent components like `rust-src`.
if comp ? "*" then
comp."*"
# Components for target platform like `rust-std`.
else if elem compName targetComponentsList then
collectTargetComponentTargets compName comp
# Components for host platform like `rustc`.
else
comp.${rustHostPlatform} or "Host component `${compName}` doesn't support target `${rustHostPlatform}`";
collectTargetComponentTargets = compName: comp:
let selected = remove null (map (tgt: comp.${tgt} or null) targets); in
if selected == []
then throw "Extension `${compName}` doesn't support any of targets: ${concatStringsSep ", " targets}"
else selected;
collectComponents = name: collectComponentTargets name (componentSet.${name} or "Missing extension `${name}`");
collectTargetComponents = name: collectTargetComponentTargets name (componentSet.${name} or "Missing target extension `${name}`");
result =
flatten (map collectComponents extensions) ++
flatten (map collectTargetComponents targetExtensions);
isTargetUnused = target:
!any (name: componentSet ? ${name}.${target})
(filter (name: elem name targetComponentsList) extensions ++ targetExtensions);
errors = filter isString result ++
map (tgt: "Target `${tgt}` is not supported by any components or extensions")
(filter isTargetUnused targets);
in
if errors == [] then result
else throw ''
Component resolution failed for ${name}
- note: available extensions are ${concatStringsSep ", " (attrNames componentSet)}
${concatMapStrings (msg: "- ${msg}\n") errors}
'';
# Genereate the toolchain set from a parsed manifest.
#
# Manifest files are organized as follow:
# { date = "2017-03-03";
# pkg.cargo.version= "0.18.0-nightly (5db6d64 2017-03-03)";
# pkg.cargo.target.x86_64-unknown-linux-gnu = {
# available = true;
# hash = "abce..."; # sha256
# url = "https://static.rust-lang.org/dist/....tar.gz";
# xz_hash = "abce..."; # sha256
# xz_url = "https://static.rust-lang.org/dist/....tar.xz";
# };
# }
#
# The packages available usually are:
# cargo, rust-analysis, rust-docs, rust-src, rust-std, rustc, and
# rust, which aggregates them in one package.
#
# For each package the following options are available:
# extensions - The extensions that should be installed for the package.
# For example, install the package rust and add the extension rust-src.
# targets - The package will always be installed for the host system, but with this option
# extra targets can be specified, e.g. "mips-unknown-linux-musl". The target
# will only apply to components of the package that support being installed for
# a different architecture. For example, the rust package will install rust-std
# for the host system and the targets.
# targetExtensions - If you want to force extensions to be installed for the given targets, this is your option.
# All extensions in this list will be installed for the target architectures.
# *Attention* If you want to install an extension like rust-src, that has no fixed architecture (arch *),
# you will need to specify this extension in the extensions options or it will not be installed!
toolchainFromManifest = manifest: let
inherit (builtins) elemAt;
inherit (super) makeOverridable;
inherit (super.lib) flip mapAttrs;
inherit (super.rust) toRustTarget;
maybeRename = name: manifest.renames.${name}.to or name;
# For legacy pre-aggregated package `rust`.
mkPackage = name: pkg:
makeOverridable ({ extensions, targets, targetExtensions, stdenv, fetchurl, patchelf }:
let
extensions' = map maybeRename extensions;
targetExtensions' = map maybeRename targetExtensions;
namesAndSrcs = getComponents manifest.pkg name targets extensions' targetExtensions' stdenv fetchurl;
in
aggregateComponents {
pname = name;
version = manifest.version;
components = map ({ name, src }: (mkComponent {
pname = name;
inherit (manifest) version;
inherit src;
# The component name is `rust`.
# clippy-driver will be patched to $out/lib without touching this.
rustc = null;
})) namesAndSrcs;
}
) {
extensions = [];
targets = [];
targetExtensions = [];
inherit (self) stdenv fetchurl patchelf;
};
# componentSet.cargo.x86_64-unknown-linux-gnu = <derivation>;
componentSet = mapAttrs (name: pkg:
mapAttrs (target: { xz_hash, xz_url }:
mkComponent {
pname = name;
inherit (manifest) version;
src = mkComponentSrc {
url = xz_url;
sha256 = xz_hash;
fetchurl = self.fetchurl;
};
rustc = componentSet.rustc.${target} or
(throw "clippy depends on rustc, which is not available");
}
) pkg.target
) (removeAttrs manifest.pkg ["rust"]) //
mapAttrs (name: { to }: componentSet.${to}) manifest.renames;
mkProfile = name: componentNames:
makeOverridable ({ extensions, targets, targetExtensions }:
aggregateComponents {
pname = "rust-${name}";
version = manifest.version;
components = resolveComponents {
name = "rust-${name}-${manifest.version}";
inherit componentSet;
inherit (manifest) targetComponentsList;
extensions = componentNames ++ extensions;
targets = [
(toRustTarget self.stdenv.hostPlatform) # Build script requires host std.
(toRustTarget self.stdenv.targetPlatform)
] ++ targets;
inherit targetExtensions;
};
}
) {
extensions = [];
targets = [];
targetExtensions = [];
};
profiles = mapAttrs mkProfile manifest.profiles;
in
# Components.
mapAttrs (name: targets: targets."*" or targets.${toRustTarget self.stdenv.hostPlatform} or null) componentSet //
# Profiles.
profiles //
{
# Legacy support for special pre-aggregated package.
# It has more components than `default` profile but less than `complete` profile.
rust =
let pkg = mkPackage "rust" manifest.pkg.rust; in
if builtins.match ".*[.].*[.].*" != null && profiles != {}
then builtins.trace ''
Rust ${manifest.version}:
Pre-aggregated package `rust` is not encouraged for stable channel since it contains almost all and uncertain components.
Consider use `default` profile like `rust-bin.stable.latest.default` and override it with extensions you need.
See README for more information.
'' pkg
else pkg;
# Internal use.
_components = componentSet;
_profiles = profiles;
_version = manifest.version;
};
# Same as `toolchainFromManifest` but read from a manifest file.
toolchainFromManifestFile = path: toolchainFromManifest (builtins.fromTOML (builtins.readFile path));
# Override all pkgs of a toolchain set.
overrideToolchain = attrs: super.lib.mapAttrs (name: pkg: pkg.override attrs);
# From a git revision of rustc.
# This does the same thing as crate `rustup-toolchain-install-master`.
# But you need to manually provide component hashes.
fromRustcRev = {
# Package name of the derivation.
pname ? "rust-custom",
# Git revision of rustc.
rev,
# Attrset with component name as key and its SRI hash as value.
components,
# Rust target to download.
target ? super.rust.toRustTarget self.stdenv.targetPlatform
}: let
shortRev = builtins.substring 0 7 rev;
components' = super.lib.mapAttrs (compName: hash: mkComponent {
pname = compName;
version = shortRev;
src = self.fetchurl {
url = if compName == "rust-src"
then "https://ci-artifacts.rust-lang.org/rustc-builds/${rev}/${compName}-nightly.tar.xz"
else "https://ci-artifacts.rust-lang.org/rustc-builds/${rev}/${compName}-nightly-${target}.tar.xz";
inherit hash;
};
rustc = components'.rustc or (throw "rustc is required for clippy");
}) components;
in
aggregateComponents {
inherit pname;
version = shortRev;
components = builtins.attrValues components';
};
# Select latest nightly toolchain which makes selected profile builds.
# Some components are missing in some nightly releases.
# Usage:
# `selectLatestNightlyWith (toolchain: toolchain.default.override { extensions = "llvm-tools-preview"; })`
selectLatestNightlyWith = selector:
let
inherit (builtins) attrNames removeAttrs elemAt length trace tryEval;
nightlyDates = attrNames (removeAttrs self.rust-bin.nightly [ "latest" ]);
dateLength = length nightlyDates;
go = idx:
let ret = selector (self.rust-bin.nightly.${elemAt nightlyDates idx}); in
if idx == 0 then
ret
else if dateLength - idx >= 256 then
trace "Failed to select nightly version after 100 tries" ret
else if ret != null && (tryEval ret.drvPath).success then
ret
else
go (idx - 1);
in
go (length nightlyDates - 1);
in {
# For each channel:
# rust-bin.stable.latest.{minimal,default,complete} # Profiles.
# rust-bin.stable.latest.rust # Pre-aggregate from upstream.
# rust-bin.stable.latest.cargo # Components...
# rust-bin.stable.latest.rustc
# rust-bin.stable.latest.rust-docs
# ...
#
# For a specific version of stable:
# rust-bin.stable."1.47.0".default
#
# For a specific date of beta:
# rust-bin.beta."2021-01-01".default
#
# For a specific date of nightly:
# rust-bin.nightly."2020-01-01".default
rust-bin = with builtins;
(super.rust-bin or {}) //
mapAttrs (channel: mapAttrs (version: toolchainFromManifest)) super.rust-bin.manifests //
{
inherit fromRustupToolchain fromRustupToolchainFile;
inherit selectLatestNightlyWith;
# Experimental feature.
inherit fromRustcRev;
};
# All attributes below are for compatiblity with mozilla overlay.
lib = (super.lib or {}) // {
rustLib = (super.lib.rustLib or {}) // {
manifest_v2_url = throw ''
`manifest_v2_url` is not supported.
Select a toolchain from `rust-bin` or using `rustChannelOf` instead.
See also README at https://github.com/oxalica/rust-overlay
'';
fromManifest = throw ''
`fromManifest` is not supported due to network access during evaluation.
Select a toolchain from `rust-bin` or using `rustChannelOf` instead.
See also README at https://github.com/oxalica/rust-overlay
'';
fromManifestFile = manifestFilePath: { stdenv, fetchurl, patchelf }@deps: builtins.trace ''
`fromManifestFile` is deprecated.
Select a toolchain from `rust-bin` or using `rustChannelOf` instead.
See also README at https://github.com/oxalica/rust-overlay
'' (overrideToolchain deps (toolchainFromManifestFile manifestFilePath));
};
};
rustChannelOf = manifestArgs: toolchainFromManifest (selectManifest manifestArgs);
latest = (super.latest or {}) // {
rustChannels = {
stable = self.rust-bin.stable.latest;
beta = self.rust-bin.beta.latest;
nightly = self.rust-bin.nightly.latest;
};
};
rustChannelOfTargets = channel: date: targets:
(self.rustChannelOf { inherit channel date; })
.rust.override { inherit targets; };
rustChannels = self.latest.rustChannels;
}