Allow passing arbitrary fetchurl args per registry (#168)

Co-authored-by: Amos Wenger <amoswenger@gmail.com>

CHANGELOG.md

@@ -21,7 +21,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   - Note that `buildPackage` will continue to use zstd compressed tarballs while
     building dependencies (unless either of `cargoArtifacts` or
     `installCargoArtifactsMode` is defined, in which case they will be honored)
+* **Breaking**: the format for defining crate registries has been changed: each
+  registry URL should map to a set containing a `downloadUrl` attribute. This
+  set may also define `fetchurlExtraArgs` (another set) which will be forwarded
+  to the
+  `fetchurl` invocations for crates for that registry.
 * `registryFromGitIndex` now uses shallow checkouts for better performance
+* `registryFromDownloadUrl` and `registryFromGitIndex` now allow specifying
+  `fetchurlExtraArgs` which will be forwarded to the `fetchurl` invocations for
+  crates for that registry
 
 ### Fixed
 * Unpacking a git repository now ignores duplicate crates to match cargo's

docs/API.md

@@ -44,12 +44,14 @@ newLib = lib.appendCrateRegistries [
   (lib.registryFromDownloadUrl {
     indexUrl = "https://github.com/rust-lang/crates.io-index";
     dl = "https://crates.io/api/v1/crates";
+    fetchurlExtraArgs = {};
   })
 
   # Or, alternatively
   (lib.registryFromGitIndex {
     indexUrl = "https://github.com/Hirevo/alexandrie-index";
     rev = "90df25daf291d402d1ded8c32c23d5e1498c6725";
+    fetchurlExtraArgs = {};
   })
 ];
 ```
@@ -648,6 +650,10 @@ raised during evaluation.
 Download a packaged cargo crate (e.g. from crates.io) and prepare it for
 vendoring.
 
+The registry's `fetchurlExtraArgs` will be passed through to `fetchurl` when
+downloading the crate, making it possible to influence interacting with the
+registry's API if necessary.
+
 #### Required input attributes
 * `checksum`: the (sha256) checksum recorded in the Cargo.lock file
 * `name`: the name of the crate
@@ -904,16 +910,29 @@ If the registry's download endpoint changes more frequently and you would like
 to infer the configuration directly from a git revision, consider using
 `registryFromGitIndex` as an alternative.
 
+If the registry needs a special way of accessing crate sources the
+`fetchurlExtraArgs` set can be used to influence the behavior of fetching the
+crate sources (e.g. by setting `curlOptsList`)
+
 #### Required attributes
 * `dl`: the value of the `dl` entry in the registry's `config.json` file
 * `indexUrl`: an HTTP URL to the index
 
+#### Optional attributes
+* `fetchurlExtraArgs`: a set of arguments which will be passed on to the
+  `fetchurl` for each crate being sourced from this registry
+
 ```nix
 lib.registryFromDownloadUrl {
   dl = "https://crates.io/api/v1/crates";
   indexUrl = "https://github.com/rust-lang/crates.io-index";
 }
-# { "registry+https://github.com/rust-lang/crates.io-index" = "https://crates.io/api/v1/crates/{crate}/{version}/download"; }
+# {
+#   "registry+https://github.com/rust-lang/crates.io-index" = {
+#     downloadUrl = "https://crates.io/api/v1/crates/{crate}/{version}/download";
+#     fetchurlExtraArgs = {};
+#   };
+# }
 ```
 
 ### `lib.registryFromGitIndex`
@@ -934,24 +953,42 @@ specified revision will be added to the Nix store during evaluation time, and
 that IFD will need to be enabled. If this is unsatisfactory, consider using
 `registryFromDownloadUrl` as a simpler alternative.
 
+If the registry needs a special way of accessing crate sources the
+`fetchurlExtraArgs` set can be used to influence the behavior of fetching the
+crate sources (e.g. by setting `curlOptsList`)
+
 #### Required attributes
 * `indexUrl`: an HTTP URL to the index
 * `rev`: any git revision which contains the latest `config.json` definition
 
+#### Optional attributes
+* `fetchurlExtraArgs`: a set of arguments which will be passed on to the
+  `fetchurl` for each crate being sourced from this registry
+
 ```nix
 lib.registryFromGitIndex {
   url = "https://github.com/Hirevo/alexandrie-index";
   rev = "90df25daf291d402d1ded8c32c23d5e1498c6725";
 }
-# { "registry+https://github.com/Hirevo/alexandrie-index" = "https://crates.polomack.eu/api/v1/crates/{crate}/{version}/download"; }
+# {
+#   "registry+https://github.com/Hirevo/alexandrie-index" = {
+#     downloadUrl = "https://crates.polomack.eu/api/v1/crates/{crate}/{version}/download";
+#     fetchurlExtraArgs = {};
+#   };
+# }
 ```
 
 ### `lib.urlForCargoPackage`
 
-`urlForCargoPackage :: set -> String`
+`urlForCargoPackage :: set -> set`
 
-Returns the URL for downloading a particular crate. For now, only crates.io is
+Returns info pertaining to the URL for downloading a particular crate if the
-supported.
+crate's registry is configured (an error will be thrown if it is not).
+
+The result will contain two attributes:
+- `url`: A string representing the URL at which the crate can be fetched
+- `fetchurlExtraArgs`: A set of attributes specific to this registry which will
+  be passed on to the `fetchurl` invocation.
 
 #### Required input attributes
 * `name`: the name of the crate

examples/alt-registry/flake.nix

@@ -35,6 +35,9 @@
           (craneLibOrig.registryFromGitIndex {
             indexUrl = "https://github.com/Hirevo/alexandrie-index";
             rev = "90df25daf291d402d1ded8c32c23d5e1498c6725";
+            fetchurlExtraArgs = {
+              # Extra parameters which will be passed to the fetchurl invocation for each crate
+            };
           })
 
           # As a more lightweight alternative, the `dl` endpoint of the registry's `config.json`
@@ -42,6 +45,9 @@
           # (craneLibOrig.registryFromDownloadUrl {
           #   indexUrl = "https://github.com/Hirevo/alexandrie-index";
           #   dl = "https://crates.polomack.eu/api/v1/crates/{crate}/{version}/download";
+          #   fetchurlExtraArgs = {
+          #     # Extra parameters which will be passed to the fetchurl invocation for each crate
+          #   };
           # })
         ];
 

lib/downloadCargoPackage.nix

@@ -9,11 +9,12 @@
 , ...
 }@args:
 let
-  tarball = fetchurl {
+  pkgInfo = urlForCargoPackage args;
+  tarball = fetchurl (pkgInfo.fetchurlExtraArgs // {
+    inherit (pkgInfo) url;
     name = "${name}-${version}";
     sha256 = checksum;
-    url = urlForCargoPackage args;
+  });
-  };
 in
 runCommandLocal "cargo-package-${name}-${version}" { } ''
   mkdir -p $out

lib/registryFromDownloadUrl.nix

@@ -4,6 +4,7 @@
 # https://doc.rust-lang.org/cargo/reference/registries.html
 { dl
 , indexUrl
+, fetchurlExtraArgs ? {}
 }:
 let
   matches = m: builtins.match ".*${lib.escapeRegex m}.*" dl;
@@ -24,5 +25,8 @@ let
     else "${registryPrefix}${indexUrl}";
 in
 {
-  "${registryIndexUrl}" = fullDownloadUrl;
+  "${registryIndexUrl}" = {
+    inherit fetchurlExtraArgs;
+    downloadUrl = fullDownloadUrl;
+  };
 }

lib/registryFromGitIndex.nix

@@ -3,6 +3,7 @@
 
 { indexUrl
 , rev
+, fetchurlExtraArgs ? {}
 }:
 
 let
@@ -26,5 +27,5 @@ let
   '');
 in
 registryFromDownloadUrl {
-  inherit dl indexUrl;
+  inherit dl indexUrl fetchurlExtraArgs;
 }

lib/urlForCargoPackage.nix

@@ -19,7 +19,7 @@ let
     else if nameLen == 3 then "3/${substr 0 1 name}"
     else "${substr 0 2 name}/${substr 2 4 name}";
 
-  dl = crateRegistries.${source} or (throw ''
+  registry = crateRegistries.${source} or (throw ''
     not sure how to download crates from ${source}.
     for example, this can be resolved with:
 
@@ -42,19 +42,22 @@ let
     }
   '');
 in
-builtins.replaceStrings
+  {
-  [
+    fetchurlExtraArgs = registry.fetchurlExtraArgs or {};
-    "{crate}"
+    url = builtins.replaceStrings
-    "{version}"
+      [
-    "{prefix}"
+        "{crate}"
-    "{lowerprefix}"
+        "{version}"
-    "{sha256-checksum}"
+        "{prefix}"
-  ]
+        "{lowerprefix}"
-  [
+        "{sha256-checksum}"
-    name
+      ]
-    version
+      [
-    prefix
+        name
-    (lib.toLower prefix)
+        version
-    checksum
+        prefix
-  ]
+        (lib.toLower prefix)
-  dl
+        checksum
+      ]
+      registry.downloadUrl;
+  }