From f69c8a4c4458aefda58a9cf39553023acb5844e0 Mon Sep 17 00:00:00 2001 From: Ivan Petkov Date: Mon, 3 Jan 2022 14:22:08 -0800 Subject: [PATCH] Add `buildPackage` for building and installing cargo packages --- lib/buildPackage.nix | 53 +++++++++++++++++++++++ lib/buildWithCargo.nix | 3 -- lib/default.nix | 1 + pkgs/default.nix | 13 +++++- pkgs/installFromCargoBuildLogHook.sh | 42 ++++++++++++++++++ tests/default.nix | 25 +++++++++-- tests/with-libs/Cargo.lock | 28 ++++++++++++ tests/with-libs/Cargo.toml | 7 +++ tests/with-libs/all-types/Cargo.toml | 15 +++++++ tests/with-libs/all-types/src/lib.rs | 4 ++ tests/with-libs/only-cdylib/Cargo.toml | 13 ++++++ tests/with-libs/only-cdylib/src/lib.rs | 4 ++ tests/with-libs/only-staticlib/Cargo.toml | 13 ++++++ tests/with-libs/only-staticlib/src/lib.rs | 4 ++ tests/with-libs/some-dep/Cargo.lock | 7 +++ tests/with-libs/some-dep/Cargo.toml | 11 +++++ tests/with-libs/some-dep/src/lib.rs | 4 ++ 17 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 lib/buildPackage.nix create mode 100644 pkgs/installFromCargoBuildLogHook.sh create mode 100644 tests/with-libs/Cargo.lock create mode 100644 tests/with-libs/Cargo.toml create mode 100644 tests/with-libs/all-types/Cargo.toml create mode 100644 tests/with-libs/all-types/src/lib.rs create mode 100644 tests/with-libs/only-cdylib/Cargo.toml create mode 100644 tests/with-libs/only-cdylib/src/lib.rs create mode 100644 tests/with-libs/only-staticlib/Cargo.toml create mode 100644 tests/with-libs/only-staticlib/src/lib.rs create mode 100644 tests/with-libs/some-dep/Cargo.lock create mode 100644 tests/with-libs/some-dep/Cargo.toml create mode 100644 tests/with-libs/some-dep/src/lib.rs diff --git a/lib/buildPackage.nix b/lib/buildPackage.nix new file mode 100644 index 0000000..0728db7 --- /dev/null +++ b/lib/buildPackage.nix @@ -0,0 +1,53 @@ +{ buildWithCargo +, installFromCargoBuildLogHook +}: + +{ cargoBuildCommand ? "cargo build --workspace --release" +, cargoExtraArgs ? "" +, ... +}@args: +# NB: we use overrideAttrs here so that our extra additions here do not end up +# invalidating any deps builds by being inherited. For example, we probably don't +# care about installing bins/libs from the deps only build, so there's no point to +# trying to build it with the install scripts in its build environment. +(buildWithCargo args).overrideAttrs (old: { + # Don't copy target dir by default since we are going to be installing bins/libs + doCopyTargetToOutput = args.doCopyTargetToOutput or false; + + buildPhase = args.buildPhase or '' + runHook preBuild + ${args.buildPhaseCargoCommand or '' + cargoBuildLog=$(mktemp cargoBuildLogXXXX.json) + ${cargoBuildCommand} --message-format json-render-diagnostics ${cargoExtraArgs} >"$cargoBuildLog" + ''} + runHook postBuild + ''; + + installPhase = args.installPhase or '' + runHook preInstall + ${args.installPhaseCargoCommand or '' + if [ -n "$cargoBuildLog" -a -f "$cargoBuildLog" ]; then + installFromCargoBuildLog "$out" "$cargoBuildLog" + else + echo ' + $cargoBuildLog is either undefined or does not point to a valid file location! + By default `buildPackage` will capture cargo'"'"'s output and use it to determine which binaries + should be installed (instead of just guessing based on what is present in cargo'"'"'s target directory). + If you are overriding the derivation with a custom build step, you have two options: + 1. override `installPhaseCargoCommand` with the appropriate installation steps + 2. ensure that cargo'"'"'s build log is captured in a file and point $cargoBuildLog at it + At a minimum, the latter option can be achieved with running: + cargoBuildLog=$(mktemp cargoBuildLogXXXX.json) + cargo build --message-format json-render-diagnostics >"$cargoBuildLog" + ' + + false + fi + ''} + runHook postInstall + ''; + + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ + installFromCargoBuildLogHook + ]; +}) diff --git a/lib/buildWithCargo.nix b/lib/buildWithCargo.nix index ac96ee1..b3bfdf7 100644 --- a/lib/buildWithCargo.nix +++ b/lib/buildWithCargo.nix @@ -48,9 +48,6 @@ mkCargoDerivation (args // { # This can be inferred automatically if the `src` root has a Cargo.lock file. cargoVendorDir = args.cargoVendorDir or (vendorCargoDepsFromArgs args); - # Don't copy target dir by default since we are going to be installing bins/libs - doCopyTargetToOutput = args.doCopyTargetToOutput or false; - nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ installFromCargoArtifactsHook ]; diff --git a/lib/default.nix b/lib/default.nix index bf31dc4..196bd75 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -13,6 +13,7 @@ lib.makeScope newScope (self: inherit fromTOML toTOML; buildDepsOnly = callPackage ./buildDepsOnly.nix { }; + buildPackage = callPackage ./buildPackage.nix { }; buildWithCargo = callPackage ./buildWithCargo.nix { }; cleanCargoToml = callPackage ./cleanCargoToml.nix { }; crateNameFromCargoToml = callPackage ./crateNameFromCargoToml.nix { }; diff --git a/pkgs/default.nix b/pkgs/default.nix index 8b00bde..9f56fb1 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -1,4 +1,6 @@ -{ makeSetupHook +{ cargo +, makeSetupHook +, jq , rsync , zstd }: @@ -35,6 +37,15 @@ name = "installFromCargoArtifactsHook"; } ./installFromCargoArtifactsHook.sh; + installFromCargoBuildLogHook = makeSetupHook + { + name = "installFromCargoBuildLogHook"; + substitutions = { + cargo = "${cargo}/bin/cargo"; + jq = "${jq}/bin/jq"; + }; + } ./installFromCargoBuildLogHook.sh; + remapSourcePathPrefixHook = makeSetupHook { name = "remapSourcePathPrefixHook"; diff --git a/pkgs/installFromCargoBuildLogHook.sh b/pkgs/installFromCargoBuildLogHook.sh new file mode 100644 index 0000000..a212ad5 --- /dev/null +++ b/pkgs/installFromCargoBuildLogHook.sh @@ -0,0 +1,42 @@ +function installFromCargoBuildLog() ( + local dest=${1:-${out}} + local log=${2:-${cargoBuildLog:?not defined}} + + if ! [ -f "${log}" ]; then + echo unable to install, cargo build log does not exist at: ${log} + false + fi + + echo searching for bins/libs to install from cargo build log at ${log} + + local select_non_test='select(.reason == "compiler-artifact" and .profile.test == false)' + local select_bins="${select_non_test} | .executable | select(.!= null)" + local select_lib_files="${select_non_test}"' + | select(.target.kind | contains(["staticlib"]) or contains(["cdylib"])) + | .filenames[] + | select(endswith(".rlib") | not) + ' + + function installArtifacts() { + local loc=${1?:missing} + mkdir -p "${loc}" + + while IFS= read -r to_install; do + echo installing ${to_install} + cp "${to_install}" "${loc}" + done + + rmdir --ignore-fail-on-non-empty "${loc}" + } + + @jq@ -r <"${log}" "${select_bins}" | installArtifacts "${dest}/bin" + + @cargo@ metadata --format-version 1 | @jq@ '.workspace_members[]' | ( + while IFS= read -r ws_member; do + local select_member_libs="select(.package_id == ${ws_member}) | ${select_lib_files}" + @jq@ -r <"${log}" "${select_member_libs}" | installArtifacts "${dest}/lib" + done + ) + + echo searching for bins/libs complete! +) diff --git a/tests/default.nix b/tests/default.nix index 44476f7..feb5605 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -45,8 +45,26 @@ pkgs.lib.makeScope myLib.newScope (self: src = ./various-targets; }; - simple = myLib.buildWithCargo { - doCopyTargetToOutput = false; + manyLibs = myLib.buildPackage { + src = ./with-libs; + pname = "my-libs"; + version = "0.0.1"; + cargoArtifacts = null; + }; + + manyLibsInstalledAsExpected = pkgs.runCommand "manyLibsInstalledAsExpected" { } '' + cat >expected < i32 { + some_dep::foo(a) * a +} diff --git a/tests/with-libs/only-cdylib/Cargo.toml b/tests/with-libs/only-cdylib/Cargo.toml new file mode 100644 index 0000000..0564c64 --- /dev/null +++ b/tests/with-libs/only-cdylib/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "only-cdylib" +version = "0.1.0" +edition = "2021" + +[dependencies] +some-dep = { path = "../some-dep" } + +[lib] +crate-type = [ + "cdylib", +] + diff --git a/tests/with-libs/only-cdylib/src/lib.rs b/tests/with-libs/only-cdylib/src/lib.rs new file mode 100644 index 0000000..5f28c20 --- /dev/null +++ b/tests/with-libs/only-cdylib/src/lib.rs @@ -0,0 +1,4 @@ +#[no_mangle] +pub fn bar(a: i32) -> i32 { + some_dep::foo(a) * a +} diff --git a/tests/with-libs/only-staticlib/Cargo.toml b/tests/with-libs/only-staticlib/Cargo.toml new file mode 100644 index 0000000..81482f7 --- /dev/null +++ b/tests/with-libs/only-staticlib/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "only-staticlib" +version = "0.1.0" +edition = "2021" + +[dependencies] +some-dep = { path = "../some-dep" } + +[lib] +crate-type = [ + "staticlib", +] + diff --git a/tests/with-libs/only-staticlib/src/lib.rs b/tests/with-libs/only-staticlib/src/lib.rs new file mode 100644 index 0000000..5f28c20 --- /dev/null +++ b/tests/with-libs/only-staticlib/src/lib.rs @@ -0,0 +1,4 @@ +#[no_mangle] +pub fn bar(a: i32) -> i32 { + some_dep::foo(a) * a +} diff --git a/tests/with-libs/some-dep/Cargo.lock b/tests/with-libs/some-dep/Cargo.lock new file mode 100644 index 0000000..3950a1b --- /dev/null +++ b/tests/with-libs/some-dep/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "some-dep" +version = "0.1.0" diff --git a/tests/with-libs/some-dep/Cargo.toml b/tests/with-libs/some-dep/Cargo.toml new file mode 100644 index 0000000..9cd06ee --- /dev/null +++ b/tests/with-libs/some-dep/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "some-dep" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = [ + "cdylib", + "rlib", + "staticlib", +] diff --git a/tests/with-libs/some-dep/src/lib.rs b/tests/with-libs/some-dep/src/lib.rs new file mode 100644 index 0000000..e43f870 --- /dev/null +++ b/tests/with-libs/some-dep/src/lib.rs @@ -0,0 +1,4 @@ +#[no_mangle] +pub fn foo(a: i32) -> i32 { + a + 42 +}