Add support for git dependencies

checks/default.nix

@@ -91,6 +91,9 @@ onlyDrvs (lib.makeScope myLib.newScope (self:
     simple = myLib.buildPackage {
       src = ./simple;
     };
+    simpleGit = myLib.buildPackage {
+      src = ./simple-git;
+    };
 
     # Test building a real world example
     ripgrep = myLib.buildPackage {
@@ -99,6 +102,7 @@ onlyDrvs (lib.makeScope myLib.newScope (self:
 
     smoke = callPackage ./smoke.nix { };
     smokeSimple = self.smoke [ "simple" ] self.simple;
+    smokeSimpleGit = self.smoke [ "simple-git" ] self.simpleGit;
     smokeAltRegistry = self.smoke [ "alt-registry" ] (
       let
         myLibWithRegistry = myLib.appendCrateRegistries [

checks/simple-git/Cargo.lock

@@ -0,0 +1,52 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "git+https://github.com/BurntSushi/byteorder.git#663358f9d29bddadc1a8e84290ec96925f2cb851"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc 0.2.117 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c"
+
+[[package]]
+name = "libc"
+version = "0.2.117"
+source = "git+https://github.com/rust-lang/libc.git?branch=master#c8ca99b26b335d34aad4fdae1e19b687252e1bdd"
+
+[[package]]
+name = "num_cpus"
+version = "1.13.1"
+source = "git+https://github.com/seanmonstar/num_cpus.git?tag=v1.13.1#5f1b03332000b4c4274b5bd35fac516049ff1c6b"
+dependencies = [
+ "hermit-abi",
+ "libc 0.2.117 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.6"
+source = "git+https://github.com/dtolnay/rustversion.git?rev=2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb#2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb"
+
+[[package]]
+name = "simple-git"
+version = "0.1.0"
+dependencies = [
+ "byteorder",
+ "libc 0.2.117 (git+https://github.com/rust-lang/libc.git?branch=master)",
+ "num_cpus",
+ "rustversion",
+]

checks/simple-git/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "simple-git"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies.byteorder]
+version = "*"
+git = "https://github.com/BurntSushi/byteorder.git"
+
+[dependencies.libc]
+version = "*"
+git = "https://github.com/rust-lang/libc.git"
+branch = "master"
+
+[dependencies.num_cpus]
+version = "*"
+git = "https://github.com/seanmonstar/num_cpus.git"
+tag = "v1.13.1"
+
+[dependencies.rustversion]
+version = "*"
+git = "https://github.com/dtolnay/rustversion.git"
+rev = "2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb"

checks/simple-git/src/main.rs

@@ -0,0 +1,10 @@
+#[rustversion::stable]
+const CHANNEL: &str = "stable";
+#[rustversion::nightly]
+const CHANNEL: &str = "nightly";
+
+fn main() {
+    println!("{:?}", std::any::TypeId::of::<byteorder::LittleEndian>());
+    println!("{:?}", std::any::TypeId::of::<libc::c_int>());
+    println!("{}: {}", CHANNEL, num_cpus::get());
+}

docs/API.md

@@ -420,6 +420,18 @@ vendoring.
 * `source`: the source key recorded in the Cargo.lock file
 * `version`: the version of the crate
 
+### `lib.downloadCargoPackageFromGit`
+
+`downloadCargoPackageFromGit :: set -> drv`
+
+Download a cargo crate from a git repository and prepare it for vendoring.
+
+#### Required input attributes
+* `git`: the URL to the repository
+* `name`: the name of the crate
+* `rev`: the exact revision to check out
+* `version`: the version of the crate
+
 ### `lib.findCargoFiles`
 
 `findCargoFiles :: path -> set of lists`
@@ -681,6 +693,25 @@ cargo can use for subsequent builds without needing network access.
 * `sources`: an attribute set of all the newly created cargo sources' names to
   their location in the Nix store
 
+### `lib.vendorGitDeps`
+
+`vendorGitDeps :: set -> set`
+
+Creates the derivations necessary to download all crates from all git
+dependencies referenced by a `Cargo.lock` file, and prepare the vendored
+directories which cargo can use for subsequent builds without needing network
+access.
+
+#### Input attributes
+* `lockPackages`: a list of all `[[package]]` entries found in the project's
+  `Cargo.lock` file (parsed via `fromTOML`)
+
+#### Output attributes
+* `config`: the configuration entires needed to point cargo to the vendored
+  sources. This is intended to be appended to `$CARGO_HOME/config.toml` verbatim
+* `sources`: an attribute set of all the newly created cargo sources' names to
+  their location in the Nix store
+
 ### `lib.writeTOML`
 
 `writeTOML :: String -> String -> drv`

lib/default.nix

@@ -31,6 +31,7 @@ lib.makeScope newScope (self:
     };
 
     downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
+    downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
     findCargoFiles = callPackage ./findCargoFiles.nix { };
     mkCargoDerivation = callPackage ./mkCargoDerivation.nix { };
     mkDummySrc = callPackage ./mkDummySrc.nix { };
@@ -39,5 +40,6 @@ lib.makeScope newScope (self:
     urlForCargoPackage = callPackage ./urlForCargoPackage.nix { };
     vendorCargoDeps = callPackage ./vendorCargoDeps.nix { };
     vendorCargoRegistries = callPackage ./vendorCargoRegistries.nix { };
+    vendorGitDeps = callPackage ./vendorGitDeps.nix { };
     writeTOML = callPackage ./writeTOML.nix { };
   })

lib/downloadCargoPackageFromGit.nix

@@ -0,0 +1,21 @@
+{ runCommandLocal
+}:
+
+{ name
+, version
+, git
+, rev
+, ...
+}@args:
+let
+  repo = builtins.fetchGit {
+    inherit rev;
+    url = git;
+    submodules = true;
+  };
+in
+runCommandLocal "cargo-git-${name}-${version}" { } ''
+  cp -r ${repo} $out
+  chmod +w $out
+  echo '{"files":{}, "package":null}' > $out/.cargo-checksum.json
+''

lib/vendorCargoDeps.nix

@@ -3,6 +3,7 @@
 , lib
 , runCommandLocal
 , vendorCargoRegistries
+, vendorGitDeps
 }:
 
 args:
@@ -37,21 +38,30 @@ let
       ''
   );
 
-  lock = fromTOML cargoLockContents;
-  vendoredRegistries = vendorCargoRegistries {
-    cargoConfigs = (findCargoFiles src).cargoConfigs;
-    lockPackages = lock.package or (throw "Cargo.lock missing [[package]] definitions");
-  };
+  linkSources = sources: concatMapStrings
+    (name: ''
+      ln -s ${escapeShellArg sources.${name}} $out/${escapeShellArg name}
+    '')
+    (attrNames sources);
 
-  inherit (vendoredRegistries) sources;
+  lock = fromTOML cargoLockContents;
+  lockPackages = lock.package or (throw "Cargo.lock missing [[package]] definitions");
+
+  vendoredRegistries = vendorCargoRegistries {
+    inherit lockPackages;
+    cargoConfigs = (findCargoFiles src).cargoConfigs;
+  };
+  vendoredGit = vendorGitDeps {
+    inherit lockPackages;
+  };
 in
 runCommandLocal "vendor-cargo-deps" { } ''
   mkdir -p $out
   cat >>$out/config.toml <<EOF
   ${vendoredRegistries.config}
+  ${vendoredGit.config}
   EOF
 
-  ${concatMapStrings (name: ''
-    ln -s ${escapeShellArg sources.${name}} $out/${escapeShellArg name}
-  '') (attrNames sources)}
+  ${linkSources vendoredRegistries.sources}
+  ${linkSources vendoredGit.sources}
 ''

lib/vendorGitDeps.nix

@@ -0,0 +1,123 @@
+{ downloadCargoPackageFromGit
+, lib
+, runCommandLocal
+, toTOML
+}:
+
+{ lockPackages
+}:
+let
+  inherit (builtins)
+    any
+    attrNames
+    filter
+    groupBy
+    hashString
+    head
+    isString
+    listToAttrs
+    split;
+
+  inherit (lib)
+    concatMapStrings
+    concatStrings
+    escapeShellArg
+    hasPrefix
+    last
+    mapAttrs'
+    mapAttrsToList
+    nameValuePair
+    removePrefix;
+
+  knownGitParams = [ "branch" "rev" "tag" ];
+  parseGitUrl = lockUrl:
+    let
+      revSplit = split "#" (removePrefix "git+" lockUrl);
+      # uniquely identifies the repo in terms of what cargo can address via
+      # source replacement (e.g. the url along with any branch/tag/rev).
+      id = head revSplit;
+      # NB: this is distict from the `rev` query param which may show up
+      # if the dependency is explicitly listed with a `rev` value.
+      lockedRev = last revSplit;
+
+      querySplit = split "\\?" (head revSplit);
+      git = head querySplit;
+
+      queryParamSplit = filter
+        (qp: (isString qp) && any (p: hasPrefix p qp) knownGitParams)
+        (split "&" (last querySplit));
+      extractedParams = listToAttrs (map
+        (qp:
+          let
+            kvSplit = (split "=" qp);
+          in
+          nameValuePair (head kvSplit) (last kvSplit)
+        )
+        queryParamSplit
+      );
+    in
+    extractedParams // {
+      inherit git id lockedRev;
+    };
+
+  hash = hashString "sha256";
+
+  # Local crates will show up in the lock file with no checksum/source
+  lockedPackagesFromGit = filter
+    (p: hasPrefix "git" (p.source or ""))
+    lockPackages;
+  lockedGitGroups = groupBy (p: p.id) (map
+    (p: (parseGitUrl p.source) // { package = p; })
+    lockedPackagesFromGit
+  );
+
+  linkGitSource = p:
+    let
+      inherit (p.package) name version;
+      source = downloadCargoPackageFromGit {
+        inherit name version;
+        inherit (p) git;
+        rev = p.lockedRev;
+      };
+    in
+    ''
+      ln -s ${escapeShellArg source} $out/${escapeShellArg "${name}-${version}"}
+    '';
+  vendorSingleGitSource = ps: runCommandLocal "vendor-git" { } ''
+    mkdir -p $out
+    ${concatMapStrings linkGitSource ps}
+  '';
+
+  sources = mapAttrs'
+    (id: ps: nameValuePair (hash id) (vendorSingleGitSource ps))
+    lockedGitGroups;
+
+  configLocalSources = concatMapStrings
+    (hashedId: ''
+      [source.nix-sources-${hashedId}]
+      directory = "${sources.${hashedId}}"
+    '')
+    (attrNames sources);
+
+  configReplaceGitSources = mapAttrsToList
+    (hashedId: ps:
+      let
+        p = head ps;
+        extractAttr = attr:
+          if p ? ${attr} then ''
+            ${attr} = "${p.${attr}}"
+          '' else "";
+        sourceValues = concatMapStrings extractAttr ([ "git" ] ++ knownGitParams);
+      in
+      ''
+        [source."${p.git}"]
+        replace-with = "nix-sources-${hash p.id}"
+        ${sourceValues}
+      ''
+    )
+    lockedGitGroups;
+in
+{
+  inherit sources;
+  config = configLocalSources + (concatStrings configReplaceGitSources);
+}