on iko/ghcjs-patch-master help sign in
Avoid filtering repo to just 01-index.tar.gz file (#1432)

When downloading a `repository` block we get a number of files (not just the index). For some repositories these files will be necessary for `cabal configure` and the vanilla ones created by haskell.nix will not work. This change keeps these extra files and combines them with the main hackage index using `lndir`, so that changes to the repo will not require a new copy of the hackage index in the store. This change also introduces `inputMap` that allows flake or niv inputs to be used to pin repository. It also works with `source-repository-package` blocks.

commite81f3bb011233b25f459579e4f771d1e3659219d authorHamish Mackenzie <Hamish.K.Mackenzie@gmail.com> dateThu 14 Apr 2022, 02:35 (5 years ago) parentecc6219
6 files changed+85−53
lib/cabal-project-parser.nix+34−27 lib/call-cabal-project-to-nix.nix+23−8 modules/cabal-project.nix+8 overlays/haskell.nix+14−4 test/ghcjs-overlay/cabal.project+1−1 test/unit.nix+5−13
lib/cabal-project-parser.nix+34−27
@@ -122,7 +122,7 @@ let
122122 # This works because `cabal configure` does not include any of the `/nix/sore/`
123123 # paths in the `plan.json` (so materialized plan-nix will still work as expeced).
124124 # See tests/unit.nix for examples of input and output.
125− parseRepositoryBlock = cabalProjectFileName: sha256map: cabal-install: nix-tools: block:
125+ parseRepositoryBlock = cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: block:
126126 let
127127 lines = pkgs.lib.splitString "\n" block;
128128 # The first line will contain the repository name.
@@ -135,9 +135,27 @@ let
135135 in rec {
136136 # This is `some-name` from the `repository some-name` line in the `cabal.project` file.
137137 name = __head lines;
138− # The $HOME dir with `.cabal` sub directory after running `cabal new-update` to download the repository
139− home =
140− pkgs.evalPackages.runCommand name ({
138+ # The $HOME/.cabal/packages/${name} after running `cabal v2-update` to download the repository
139+ repoContents = if inputMap ? ${attrs.url}
140+ # If there is an input use it to make `file:` url and create a suitable `.cabal/packages/${name}` directory
141+ then pkgs.evalPackages.runCommand name ({
142+ nativeBuildInputs = [ cabal-install ];
143+ preferLocalBuild = true;
144+ }) ''
145+ HOME=$(mktemp -d)
146+ mkdir -p $HOME/.cabal/packages/${name}
147+ cat <<EOF > $HOME/.cabal/config
148+ repository ${name}
149+ url: file:${inputMap.${attrs.url}}
150+ ${pkgs.lib.optionalString (attrs ? secure) "secure: ${attrs.secure}"}
151+ ${pkgs.lib.optionalString (attrs ? root-keys) "root-keys: ${attrs.root-keys}"}
152+ ${pkgs.lib.optionalString (attrs ? key-threshold) "key-threshold: ${attrs.key-threshold}"}
153+ EOF
154+
155+ cabal v2-update ${name}
156+ cp -r $HOME/.cabal/packages/${name} $out
157+ ''
158+ else pkgs.evalPackages.runCommand name ({
141159 nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
142160 LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
143161 LANG = "en_US.UTF-8";
@@ -147,8 +165,9 @@ let
147165 outputHashAlgo = "sha256";
148166 outputHash = sha256;
149167 }) ''
150− mkdir -p $out/.cabal
151− cat <<EOF > $out/.cabal/config
168+ HOME=$(mktemp -d)
169+ mkdir -p $HOME/.cabal/packages/${name}
170+ cat <<EOF > $HOME/.cabal/config
152171 repository ${name}
153172 url: ${attrs.url}
154173 ${pkgs.lib.optionalString (attrs ? secure) "secure: ${attrs.secure}"}
@@ -157,8 +176,8 @@ let
157176 EOF
158177
159178 export SSL_CERT_FILE=${pkgs.evalPackages.cacert}/etc/ssl/certs/ca-bundle.crt
160− mkdir -p $out/.cabal/packages/${name}
161− HOME=$out cabal new-update ${name}
179+ cabal v2-update ${name}
180+ cp -r $HOME/.cabal/packages/${name} $out
162181 '';
163182 # Output of hackage-to-nix
164183 hackage = import (
@@ -169,34 +188,22 @@ let
169188 preferLocalBuild = true;
170189 } ''
171190 mkdir -p $out
172− hackage-to-nix $out ${home}/.cabal/packages/${name}/01-index.tar ${attrs.url}
191+ hackage-to-nix $out ${repoContents}/01-index.tar ${attrs.url}
173192 '');
174− # Tarball info in the same format as `extra-hackage-tarballs` passed in to cabalProject
175− tarball = {
176− ${name} = home + "/.cabal/packages/${name}/01-index.tar.gz";
193+ # Directory to `lndir` when constructing a suitable $HOME/.cabal dir
194+ repo = {
195+ ${name} = repoContents;
177196 };
178− # Replacement `repository` block using `file:` uri and the remaining text (text that was not part of the attribute block).
179− updatedText = ''
180− repository ${name}
181− url: file:${home + "/.cabal/packages/${name}"}
182− secure: True
183− root-keys:
184− key-threshold: 0
185− '' + pkgs.lib.strings.concatStringsSep "\n" x.snd;
186197 };
187198
188− parseRepositories = cabalProjectFileName: sha256map: cabal-install: nix-tools: projectFile:
199+ parseRepositories = cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: projectFile:
189200 let
190201 # This will leave the name of repository in the first line of each block
191202 blocks = pkgs.lib.splitString "\nrepository " ("\n" + projectFile);
192− initialText = pkgs.lib.lists.take 1 blocks;
193− repoBlocks = builtins.map (parseRepositoryBlock cabalProjectFileName sha256map cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
203+ repoBlocks = builtins.map (parseRepositoryBlock cabalProjectFileName sha256map inputMap cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
194204 in {
195205 extra-hackages = pkgs.lib.lists.map (block: block.hackage) repoBlocks;
196− tarballs = pkgs.lib.lists.foldl' (x: block: x // block.tarball) {} repoBlocks;
197− updatedText = pkgs.lib.strings.concatStringsSep "\n" (
198− initialText
199− ++ (builtins.map (x: x.updatedText) repoBlocks));
206+ repos = pkgs.lib.lists.foldl' (x: block: x // block.repo) {} repoBlocks;
200207 };
201208
202209in {
lib/call-cabal-project-to-nix.nix+23−8
@@ -37,6 +37,14 @@ in
3737 # "https://raw.githubusercontent.com/input-output-hk/hackage-overlay-ghcjs/bfc363b9f879c360e0a0460ec0c18ec87222ec32"
3838 # = "sha256-g9xGgJqYmiczjxjQ5JOiK5KUUps+9+nlNGI/0SpSOpg=";
3939 # };
40+, inputMap ? {}
41+ # An alternative to providing a `sha256` handy for flakes
42+ # cabal.project file:
43+ # inputs.pandoc-citeproc.url = "github:jgm/pandoc-citeproc/0.17";
44+ # inputs.pandoc-citeproc.flake = false;
45+ # outputs = inputs:
46+ # ...
47+ # inputMap."https://github.com/jgm/pandoc-citeproc" = inputs.inputs.pandoc-citeproc;
4048, extra-hackage-tarballs ? {}
4149, source-repo-override ? {} # Cabal seems to behave incoherently when
4250 # two source-repository-package entries
@@ -194,8 +202,15 @@ let
194202 fetchPackageRepo = fetchgit: repoData:
195203 let
196204 fetched =
197− if repoData.sha256 != null
198− then fetchgit { inherit (repoData) url sha256; rev = repoData.ref; }
205+ if inputMap ? "${repoData.url}/${repoData.ref}"
206+ then inputMap."${repoData.url}/${repoData.ref}"
207+ else if inputMap ? ${repoData.url}
208+ then
209+ (if inputMap.${repoData.url}.rev != repoData.ref
210+ then throw "${inputMap.${repoData.url}.rev} may not match ${repoData.ref} for ${repoData.url} use \"${repoData.url}/${repoData.ref}\" as the inputMap key if ${repoData.ref} is a branch or tag that points to ${inputMap.${repoData.url}.rev}."
211+ else inputMap.${repoData.url})
212+ else if repoData.sha256 != null
213+ then fetchgit { inherit (repoData) url sha256; rev = repoData.ref; }
199214 else
200215 let drv = builtins.fetchGit { inherit (repoData) url ref; };
201216 in __trace "WARNING: No sha256 found for source-repository-package ${repoData.url} ${repoData.ref} download may fail in restricted mode (hydra)"
@@ -226,7 +241,7 @@ let
226241
227242 # Parse the `repository` blocks
228243 repoResult = pkgs.haskell-nix.haskellLib.parseRepositories
229− cabalProjectFileName sha256map cabal-install nix-tools sourceRepoPackageResult.otherText;
244+ cabalProjectFileName sha256map inputMap cabal-install nix-tools sourceRepoPackageResult.otherText;
230245
231246 # we need the repository content twice:
232247 # * at eval time (below to build the fixed project file)
@@ -241,9 +256,9 @@ let
241256 sourceReposBuild = builtins.map (x: (fetchPackageRepo pkgs.fetchgit x).fetched) sourceRepoPackageResult.sourceRepos;
242257 in {
243258 sourceRepos = sourceReposBuild;
244− inherit (repoResult) tarballs extra-hackages;
259+ inherit (repoResult) repos extra-hackages;
245260 makeFixedProjectFile = ''
246− cp -f ${pkgs.evalPackages.writeText "cabal.project" repoResult.updatedText} ./cabal.project
261+ cp -f ${pkgs.evalPackages.writeText "cabal.project" sourceRepoPackageResult.otherText} ./cabal.project
247262 '' +
248263 pkgs.lib.optionalString (builtins.length sourceReposEval != 0) (''
249264 chmod +w -R ./cabal.project
@@ -278,7 +293,7 @@ let
278293
279294 fixedProject =
280295 if rawCabalProject == null
281− then { sourceRepos = []; tarballs = {}; extra-hackages = []; makeFixedProjectFile = ""; replaceLocations = ""; }
296+ then { sourceRepos = []; repos = {}; extra-hackages = []; makeFixedProjectFile = ""; replaceLocations = ""; }
282297 else replaceSourceRepos rawCabalProject;
283298
284299 # The use of the actual GHC can cause significant problems:
@@ -523,8 +538,8 @@ let
523538 # some packages that will be excluded by `index-state-found`
524539 # which is used by cabal (cached-index-state >= index-state-found).
525540 dotCabal {
526− inherit cabal-install nix-tools;
527− extra-hackage-tarballs = fixedProject.tarballs // extra-hackage-tarballs;
541+ inherit cabal-install nix-tools extra-hackage-tarballs;
542+ extra-hackage-repos = fixedProject.repos;
528543 index-state = cached-index-state;
529544 sha256 = index-sha256-found;
530545 }
modules/cabal-project.nix+8
@@ -105,6 +105,14 @@ in {
105105 = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; };
106106 '';
107107 };
108+ inputMap = mkOption {
109+ type = nullOr attrs;
110+ default = {};
111+ description = ''
112+ Specifies the contents of urls in the cabal.project file.
113+ The `.rev` attribute is checked against the `tag` for `source-repository-packages`.
114+ '';
115+ };
108116 extra-hackage-tarballs = mkOption {
109117 type = nullOr attrs;
110118 default = {};
overlays/haskell.nix+14−4
@@ -203,16 +203,15 @@ final: prev: {
203203 # Creates Cabal local repository from { name, index } set.
204204 mkLocalHackageRepo = import ../mk-local-hackage-repo final;
205205
206− dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? {}, ... }@args:
206+ dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? {}, extra-hackage-repos ? {}, ... }@args:
207207 let
208208 allTarballs = hackageTarball args // extra-hackage-tarballs;
209209 allNames = final.lib.concatStringsSep "-" (builtins.attrNames allTarballs);
210210 # Main Hackage index-state is embedded in its name and thus will propagate to
211211 # dotCabalName anyway.
212212 dotCabalName = "dot-cabal-" + allNames;
213− in
214− # This is very big, and cheap to build: prefer building it locally
215− final.evalPackages.runCommand dotCabalName { nativeBuildInputs = [ cabal-install ]; preferLocalBuild = true; } ''
213+ # This is very big, and cheap to build: prefer building it locally
214+ tarballRepos = final.evalPackages.runCommand dotCabalName { nativeBuildInputs = [ cabal-install ]; preferLocalBuild = true; } ''
216215 mkdir -p $out/.cabal
217216 cat <<EOF > $out/.cabal/config
218217 ${final.lib.concatStrings (
@@ -237,6 +236,17 @@ final: prev: {
237236 ${final.lib.concatStrings (map (name: ''
238237 HOME=$out cabal new-update ${name}
239238 '') (builtins.attrNames allTarballs))}
239+ '';
240+ in
241+ # Add the extra-hackage-repos where we have all the files needed.
242+ final.evalPackages.runCommand dotCabalName { nativeBuildInputs = [ final.evalPackages.xorg.lndir ]; } ''
243+ mkdir $out
244+ lndir ${tarballRepos} $out
245+
246+ ${final.lib.concatStrings (final.lib.mapAttrsToList (name: repo: ''
247+ mkdir -p $out/.cabal/packages/${name}
248+ lndir ${repo} $out/.cabal/packages/${name}
249+ '') extra-hackage-repos)}
240250 '';
241251
242252 # Some of features of haskell.nix rely on using a hackage index
test/ghcjs-overlay/cabal.project+1−1
@@ -5,5 +5,5 @@ repository ghcjs-overlay
55 secure: True
66 root-keys:
77 key-threshold: 0
8− --sha256: sha256-g9xGgJqYmiczjxjQ5JOiK5KUUps+9+nlNGI/0SpSOpg=
8+ --sha256: sha256-y1vQnXI1XzkjnC4h66tVDmu2TZjZPcMrZEnE3m0XOfg=
99
test/unit.nix+5−13
@@ -119,7 +119,7 @@ lib.runTests {
119119 };
120120
121121 testParseRepositoryBlock = {
122− expr = __toJSON (haskellLib.parseRepositoryBlock "cabal.project" {}
122+ expr = __toJSON (haskellLib.parseRepositoryBlock "cabal.project" {} {}
123123 pkgs.evalPackages.haskell-nix.cabal-install.${compiler-nix-name}
124124 pkgs.evalPackages.haskell-nix.nix-tools.${compiler-nix-name} ''
125125 ghcjs-overlay
@@ -127,22 +127,14 @@ lib.runTests {
127127 secure: True
128128 root-keys:
129129 key-threshold: 0
130− --sha256: sha256-g9xGgJqYmiczjxjQ5JOiK5KUUps+9+nlNGI/0SpSOpg=
130+ --sha256: sha256-y1vQnXI1XzkjnC4h66tVDmu2TZjZPcMrZEnE3m0XOfg=
131131 -- end of block
132132 '');
133133 expected = __toJSON {
134134 name = "ghcjs-overlay";
135− updatedText = ''
136− repository ghcjs-overlay
137− url: file:/nix/store/0lx8536b53bqssqzzbyzi22j117a7q2g-ghcjs-overlay/.cabal/packages/ghcjs-overlay
138− secure: True
139− root-keys:
140− key-threshold: 0
141− -- end of block
142− '';
143− home = "/nix/store/0lx8536b53bqssqzzbyzi22j117a7q2g-ghcjs-overlay";
144− tarball = {
145− ghcjs-overlay = "/nix/store/0lx8536b53bqssqzzbyzi22j117a7q2g-ghcjs-overlay/.cabal/packages/ghcjs-overlay/01-index.tar.gz";
135+ repoContents = "/nix/store/gzjj6rjjgvkm5midldy292ghbq7hszna-ghcjs-overlay";
136+ repo = {
137+ ghcjs-overlay = "/nix/store/gzjj6rjjgvkm5midldy292ghbq7hszna-ghcjs-overlay";
146138 };
147139 hackage = {
148140 Cabal = {