on iko/ghcjs-patch-master help sign in
Add sha256 to source-repository-packages for restricted eval mode (#170)

* Add sha256 to source-repository-packages for hydra This allows source-repository-package to be downloaded and replaced in with `package: /nix/store/...` by callCabalPackageToNix so that `cabal` and `plan-to-nix` do not need to download them.

commitdedb91e725009b1c735ee4e5f9c4c3a05e24dbfc authorHamish Mackenzie <Hamish.Mackenzie@iohk.io> dateTue 16 Jul 2019, 19:13 (7 years ago) parent3d02d75
7 files changed+178−10
lib/call-cabal-project-to-nix.nix+105−8 test/cabal-source-repo/Main.hs+4 test/cabal-source-repo/cabal.project+9 test/cabal-source-repo/default.nix+33 test/cabal-source-repo/use-cabal-simple.cabal+26 test/call-cabal-project-to-nix/default.nix−2 test/default.nix+1
lib/call-cabal-project-to-nix.nix+105−8
@@ -12,7 +12,7 @@ let
1212 maybeCleanedSource =
1313 if pkgs.lib.canCleanSource src
1414 then pkgs.lib.cleanSourceWith {
15− src = builtins.trace "src = ${src};" src;
15+ inherit src;
1616 filter = path: type:
1717 type == "directory" ||
1818 pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }
@@ -21,9 +21,14 @@ let
2121 # Using origSrc bypasses any cleanSourceWith so that it will work when
2222 # access to the store is restricted. If origSrc was already in the store
2323 # you can pass the project in as a string.
24− rawCabalProject = if cabalProject != null
24+ rawCabalProject =
25+ let origSrcDir = maybeCleanedSource.origSrc or maybeCleanedSource;
26+ in if cabalProject != null
2527 then cabalProject
26− else builtins.readFile ((maybeCleanedSource.origSrc or maybeCleanedSource) + "/cabal.project");
28+ else
29+ if ((builtins.readDir origSrcDir)."cabal.project" or "") == "regular"
30+ then builtins.readFile (origSrcDir + "/cabal.project")
31+ else null;
2732
2833 # Look for a index-state: field in the cabal.project file
2934 parseIndexState = rawCabalProject:
@@ -37,7 +42,9 @@ let
3742
3843 index-state-found = if index-state != null
3944 then index-state
40− else parseIndexState rawCabalProject;
45+ else if rawCabalProject != null
46+ then parseIndexState rawCabalProject
47+ else null;
4148
4249 # Lookup hash for the index state we found
4350 index-sha256-found = if index-sha256 != null
@@ -51,13 +58,93 @@ in
5158 then throw "provided sha256 for index-state ${index-state-found} is null!" else true);
5259
5360let
61+ span = pred: list:
62+ let n = pkgs.lib.lists.foldr (x: acc: if pred x then acc + 1 else 0) 0 list;
63+ in { fst = pkgs.lib.lists.take n list; snd = pkgs.lib.lists.drop n list; };
64+
65+ # Parse lines of a source-repository-package block
66+ parseBlockLines = blockLines: builtins.listToAttrs (builtins.concatMap (s:
67+ let pair = builtins.match " *([^:]*): *(.*)" s;
68+ in pkgs.lib.optional (pair != null) (pkgs.lib.attrsets.nameValuePair
69+ (builtins.head pair)
70+ (builtins.elemAt pair 1))) blockLines);
71+
72+ fetchRepo = repo: (pkgs.fetchgit {
73+ url = repo.location;
74+ rev = repo.tag;
75+ sha256 = repo."--sha256";
76+ }) + (if repo.subdir or "" == "" then "" else "/" + repo.subdir);
77+
78+ # Parse a source-repository-package and fetch it if it containts
79+ # a line of the form
80+ # --shar256: <<SHA256>>
81+ parseBlock = block:
82+ let
83+ x = span (pkgs.lib.strings.hasPrefix " ") (pkgs.lib.splitString "\n" block);
84+ attrs = parseBlockLines x.fst;
85+ in
86+ if attrs."--sha256" or "" == ""
87+ then {
88+ sourceRepo = [];
89+ otherText = "\nsource-repository-package\n" + block;
90+ }
91+ else {
92+ sourceRepo = [ (fetchRepo attrs) ];
93+ otherText = pkgs.lib.strings.concatStringsSep "\n" x.snd;
94+ };
95+
96+ # Deal with source-repository-packages in a way that will work in
97+ # restricted-eval mode (as long as a sha256 is included).
98+ # Replace source-repository-package blocks that have a sha256 with
99+ # packages: block containing nix sotre paths of the fetched repos.
100+ replaceSoureRepos = projectFile:
101+ let
102+ blocks = pkgs.lib.splitString "\nsource-repository-package\n" projectFile;
103+ initialText = pkgs.lib.lists.take 1 blocks;
104+ repoBlocks = builtins.map parseBlock (pkgs.lib.lists.drop 1 blocks);
105+ sourceRepos = pkgs.lib.lists.concatMap (x: x.sourceRepo) repoBlocks;
106+ otherText = pkgs.writeText "cabal.project" (pkgs.lib.strings.concatStringsSep "\n" (
107+ initialText
108+ ++ (builtins.map (x: x.otherText) repoBlocks)));
109+ in {
110+ inherit sourceRepos;
111+ makeFixedProjectFile = ''
112+ cp -f ${otherText} ./cabal.project
113+ chmod +w -R ./cabal.project
114+ echo "packages:" >> ./cabal.project
115+ mkdir -p ./.source-repository-packages
116+ '' +
117+ ( pkgs.lib.strings.concatStrings (
118+ pkgs.lib.lists.zipListsWith (n: f: ''
119+ mkdir -p ./.source-repository-packages/${builtins.toString n}
120+ rsync -a --prune-empty-dirs \
121+ --include '*/' --include '*.cabal' --include 'package.yaml' \
122+ --exclude '*' \
123+ "${f}/" "./.source-repository-packages/${builtins.toString n}/"
124+ echo " ./.source-repository-packages/${builtins.toString n}" >> ./cabal.project
125+ '')
126+ (pkgs.lib.lists.range 0 ((builtins.length fixedProject.sourceRepos) - 1))
127+ sourceRepos
128+ )
129+ );
130+ };
131+
132+ fixedProject =
133+ if rawCabalProject == null
134+ then {
135+ sourceRepos = [];
136+ makeFixedProjectFile = "";
137+ }
138+ else replaceSoureRepos rawCabalProject;
139+
54140 plan = runCommand "plan-to-nix-pkgs" {
55141 nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ];
56− } ''
142+ } (''
57143 tmp=$(mktemp -d)
58144 cd $tmp
59145 cp -r ${maybeCleanedSource}/* .
60146 chmod +w -R .
147+ ${fixedProject.makeFixedProjectFile}
61148 # warning: this may not generate the proper cabal file.
62149 # hpack allows globbing, and turns that into module lists
63150 # without the source available (we cleaneSourceWith'd it),
@@ -96,7 +183,7 @@ let
96183
97184 # move pkgs.nix to default.nix ensure we can just nix `import` the result.
98185 mv $out/pkgs.nix $out/default.nix
99− '';
186+ '');
100187in
101188 # TODO: We really want this (symlinks) instead of copying the source over each and
102189 # every time. However this will not work with sandboxed builds. They won't
@@ -114,13 +201,23 @@ in
114201 # # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
115202 # buildInputs = [ plan src ];
116203 # }
117− runCommand "plan-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
204+ runCommand "plan-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } (''
118205 mkdir $out
119206 # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
120207 rsync -a "${src}/" "$out/"
121208 rsync -a ${plan}/ $out/
209+ '' +
210+ ( pkgs.lib.strings.concatStrings (
211+ pkgs.lib.lists.zipListsWith (n: f: ''
212+ mkdir -p $out/.source-repository-packages/${builtins.toString n}
213+ rsync -a "${f}/" "$out/.source-repository-packages/${builtins.toString n}/"
214+ '')
215+ (pkgs.lib.lists.range 0 ((builtins.length fixedProject.sourceRepos) - 1))
216+ fixedProject.sourceRepos
217+ )
218+ ) + ''
122219 # Rsync will have made $out read only and that can cause problems when
123220 # nix sandboxing is enabled (since it can prevent nix from moving the directory
124221 # out of the chroot sandbox).
125222 chmod +w $out
126− ''
223+ '')
test/cabal-source-repo/Main.hsnew file+4
@@ -0,0 +1,4 @@
1+module Main where
2+
3+main :: IO ()
4+main = putStrLn "Hello, Haskell!"
test/cabal-source-repo/cabal.projectnew file+9
@@ -0,0 +1,9 @@
1+packages:
2+ ./
3+
4+source-repository-package
5+ type: git
6+ location: https://github.com/input-output-hk/haskell.nix.git
7+ tag: bc01ebc05a8105035c9449943046b46c8364b932
8+ subdir: test/cabal-simple
9+ --sha256: 003lm3pm024vhbfmii7xcdd9v2rczpflxf7gdl2pyxia7p014i8z
test/cabal-source-repo/default.nixnew file+33
@@ -0,0 +1,33 @@
1+{ stdenv, mkCabalProjectPkgSet, callCabalProjectToNix }:
2+
3+with stdenv.lib;
4+
5+let
6+ pkgSet = mkCabalProjectPkgSet {
7+ plan-pkgs = import (callCabalProjectToNix {
8+ index-state = "2019-04-30T00:00:00Z";
9+ # reuse the cabal-simple test project
10+ src = ./.;
11+ });
12+ };
13+ packages = pkgSet.config.hsPkgs;
14+in
15+ stdenv.mkDerivation {
16+ name = "call-cabal-project-to-nix-test";
17+
18+ buildCommand = ''
19+ exe="${packages.use-cabal-simple.components.exes.use-cabal-simple}/bin/use-cabal-simple"
20+
21+ printf "checking whether executable runs... " >& 2
22+ $exe
23+
24+ touch $out
25+ '';
26+
27+ meta.platforms = platforms.all;
28+
29+ passthru = {
30+ # Attributes used for debugging with nix repl
31+ inherit pkgSet packages;
32+ };
33+ }
test/cabal-source-repo/use-cabal-simple.cabalnew file+26
@@ -0,0 +1,26 @@
1+cabal-version: >=1.10
2+-- Initial package description 'cabal-simple.cabal' generated by 'cabal
3+-- init'. For further documentation, see
4+-- http://haskell.org/cabal/users-guide/
5+
6+name: use-cabal-simple
7+version: 0.1.0.0
8+-- synopsis:
9+-- description:
10+-- bug-reports:
11+license: PublicDomain
12+author: Rodney Lorrimar
13+maintainer: rodney.lorrimar@iohk.io
14+-- category:
15+build-type: Simple
16+
17+executable use-cabal-simple
18+ main-is: Main.hs
19+ -- other-modules:
20+ -- other-extensions:
21+ build-depends: base
22+ , cabal-simple
23+ , extra
24+ , optparse-applicative
25+ -- hs-source-dirs:
26+ default-language: Haskell2010
test/call-cabal-project-to-nix/default.nix−2
@@ -8,8 +8,6 @@ let
88 index-state = "2019-04-30T00:00:00Z";
99 # reuse the cabal-simple test project
1010 src = ../cabal-simple;
11− # Hydra currently has issues reading from files in the store
12− cabalProject = builtins.readFile ../cabal-simple/cabal.project;
1311 });
1412 };
1513 packages = pkgSet.config.hsPkgs;
test/default.nix+1
@@ -20,6 +20,7 @@ in pkgs.recurseIntoAttrs {
2020 shell-for = haskell.callPackage ./shell-for {};
2121 callStackToNix = haskell.callPackage ./call-stack-to-nix {};
2222 callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
23+ cabal-source-repo = haskell.callPackage ./cabal-source-repo {};
2324
2425 # Run unit tests with: nix-instantiate --eval --strict -A unit.tests
2526 # An empty list means success.