git.iko.soy/obelisk
on decrease-android-validity help sign in
obelisk/lib/asset/assets.nix
252 lines 8.7 kB history raw
1# Asset generation pipeline using Nix, which generates a directory structure to be served via obelisk-asset-serve-*
2{ nixpkgs }:
4with nixpkgs.lib;
6rec {
8# Default encoding generation function for this platform; usually zopfliEncodings, but gzipEncodings on darwin due to zopfli not building on darwin.
9#
10# defaultEncodings :: String -> Derivation
11defaultEncodings =
12 # zopfli isn't supported on macOS as of nixpkgs eafd703a63
13 if nixpkgs.stdenv.isDarwin then gzipEncodings else zopfliEncodings;
15# Encoding generation function which uses zopfli to encode the asset with very high compression efficiency, at the cost of CPU time compressing.
16# Generates gzip, compress/zlib, and deflate outputs all using zopfli with 5 iterations.
18# zopfliEncodings :: String -> Derivation
19zopfliEncodings = file:
20 nixpkgs.stdenv.mkDerivation {
21 name = "encodings";
23 input = file;
25 builder = builtins.toFile "builder.sh" ''
26 source "$stdenv/setup"
28 mkdir -p "$out"
30 ln -s "$input" "$out/identity"
32 zopfli -c --i5 --gzip "$input" >"$out/gzip"
33 zopfli -c --i5 --zlib "$input" >"$out/compress"
34 zopfli -c --i5 --deflate "$input" >"$out/deflate"
35 '';
37 buildInputs = [
38 nixpkgs.zopfli
39 ];
40 };
42# Encoding generation function which uses gzip to encode the asset with decent compression efficiency and a small CPU cost. Only generates a gzip output.
44# gzipEncodings :: String -> Derivation
45gzipEncodings = file:
46 nixpkgs.stdenv.mkDerivation {
47 name = "encodings";
49 input = file;
51 builder = builtins.toFile "builder.sh" ''
52 source "$stdenv/setup"
54 mkdir -p "$out"
56 ln -s "$input" "$out/identity"
58 gzip -c7 "$input" > "$out/gzip"
59 '';
61 buildInputs = [
62 nixpkgs.gzip
63 ];
64 };
66# Encoding generation function which doesn't do any compression.
68# noEncodings :: String -> Derivation
69noEncodings = file:
70 nixpkgs.stdenv.mkDerivation {
71 name = "encodings";
73 input = file;
75 builder = builtins.toFile "builder.sh" ''
76 source "$stdenv/setup"
78 mkdir -p "$out"
80 ln -s "$input" "$out/identity"
81 '';
83 buildInputs = [
84 ];
85 };
87# Convert an attrset into a list of name/value pairs.
89# setToList :: AttrSet a -> [{ name :: String, value :: a }]
90setToList = mapAttrsToList (name: value: { inherit name value; });
92# Apply some function (chunkDerivations, for example) to the values in a list of name/value pairs then reattach the names.
94# mapSnds :: ([b] -> [b']) -> [{ fst :: a, snd :: b }] -> [{ fst :: a, snd :: b' }]
95mapSnds = f: l: zipLists (map (x: x.fst) l) (f (map (x: x.snd) l));
97# Chunk derivations in groups of up to a given size to avoid https://github.com/NixOS/nix/issues/875
99# It works by creating a dependent chain of derivations that must be built to finish evaluation of the overall nix expression, without evaluating too many.
100# Based on the current operation of nix, n should not be greater than 1012, and should probably be substantially less.
102# chunkDerivations :: Int -> [a] -> [a]
103chunkDerivations = n: l:
104 let a = take n l;
105 b = drop n l;
106 go = x: if x == [] then builtins.toFile "emptyList.nix" "[]" else makeChunkDerivation (take n x) (go (drop n x));
108 # makeChunkDerivation :: [a] -> ExprFile [a] -> ExprFile [a]
109 makeChunkDerivation = chunk': next:
110 let chunk = map (path: "(import ${builtins.unsafeDiscardOutputDependency path.drvPath}).${path.outputName}") chunk';
111 in builtins.seq (builtins.toPath next) (nixpkgs.stdenv.mkDerivation {
112 name = "chunkDerivations";
113 inherit chunk next;
114 preferLocalBuild = true;
115 builder = builtins.toFile "chunkDerivations.sh" ''
116 source "$stdenv/setup"
118 echo $chunk
120 echo "[ $chunk ] ++ (import $next)" >"$out"
121 '';
122 });
124 in a ++ import (go b);
126# Recursively map some function over a @AttrSet DirEntry@, unioning the results of the application at each level.
128# unionMapFilesWithName :: ({name :: String, value :: DirEntry} -> [{name :: String, value :: a}]) -> AttrSet DirEntry -> AttrSet DirEntry
129unionMapFilesWithName = f: d:
130 let go = {name, value}:
131 if value.type == "directory" then [{
132 inherit name;
133 value = {
134 inherit (value) type;
135 contents = unionMapFilesWithName f value.contents;
136 };
137 }] else setToList (f {
138 inherit name value;
139 });
140 in builtins.listToAttrs (builtins.concatLists (map go (setToList d)));
142# Read a directory structure recursively into an @AttrSet DirEntry@.
144# readDirRecursive :: String -> AttrSet DirEntry
145readDirRecursive = dir:
146 let d = filterAttrs (n: d: !(hasPrefix "." n)) (builtins.readDir dir);
147 go = name:
148 let path = dir + "/${name}";
149 type = d.${name};
150 in if type == "directory" then {
151 inherit name;
152 value = {
153 inherit type;
154 contents = readDirRecursive path;
155 };
156 } else {
157 inherit name;
158 value = {
159 inherit type path;
160 };
161 };
162 in builtins.listToAttrs (map go (builtins.attrNames d));
164# Render a string as a double quoted string with any internal double quotes or backslashes escaped using backslash, appropriate for a nix expression.
166# doubleQuoteString :: String -> String
167doubleQuoteString = s: "\"" + builtins.replaceStrings ["\\" "\""] ["\\\\" "\\\""] s + "\"";
169# Given an @AttrSet DirEntry@ generate a derivation with symlink entries for each @DirEntry@ given.
171# dirToPath :: AttrSet DirEntry -> Derivation
172dirToPath = contents:
173 let pairsToBashArray = xs: builtins.concatStringsSep " " (map (np: "[" + doubleQuoteString np.fst + "]=" + doubleQuoteString (toString np.snd)) xs);
174 files = pairsToBashArray (mapSnds (chunkDerivations 128) (mapAttrsToList (fst: entry: { inherit fst; snd = toPath entry; }) contents));
175 filesIsLarge = builtins.stringLength files > 65536;
176 in
177 nixpkgs.stdenv.mkDerivation {
178 name = "toPath";
179 inherit files;
180 passAsFile = optional filesIsLarge "files";
181 preferLocalBuild = true;
182 builder = builtins.toFile "builder.sh" ''
183 set -e
184 source "$stdenv/setup"
186 eval "declare -A files=(${if filesIsLarge then ''$(cat "$filesPath")'' else ''$files''})"
188 mkdir "$out"
190 for filename in "''${!files[@]}" ; do
191 ln -s "''${files[$filename]}" "$out/$filename"
192 done
193 '';
194 };
196# Given a @DirEntry@ either use 'dirToPath' to make a derivation with its contents or return the path directly.
197toPath = x: if x.type == "directory" then dirToPath x.contents else x.path;
199# Given a file path, hash its contents as nix does and return the hash string.
200hashFile = path:
201 builtins.readFile (nixpkgs.runCommand "hashFile" {
202 buildInputs = [
203 nixpkgs.nix
204 ];
205 preferLocalBuild = true;
206 inherit path;
207 } ''
208 nix-hash --flat --base32 --type sha256 "$path" | tr -d '\n' >"$out"
209 '');
211# Build a DirEntry of the shape { type :: String, contents :: {DirEntry} } with type set to directory. Used to create a data model to pass to toPath.
213# dir :: String -> DirEntry
214dir = contents: {
215 type = "directory";
216 inherit contents;
217};
219# Build a DirEntry of the shape { type :: String, path :: String } with type set to symlink. Used to create a data model to pass to toPath.
221# symlink :: String -> DirEntry
222symlink = path: {
223 type = "symlink";
224 inherit path;
225};
227# Given a encoding generation function and a file entry resulting from readDirRecursive in the form { name :: String, value: { path :: String } },
228# build a DirEntry for dirToPath with the various encodings of the asset for dirToPath to build into a final directory tree.
229mkAsset = encodings: {name, value}:
230 let nameWithHash = "${hashFile value.path}-${name}";
231 in {
232 ${nameWithHash} = dir {
233 type = symlink (builtins.toFile "type" "immutable");
234 encodings = symlink (encodings value.path);
235 };
236 ${name} = dir {
237 type = symlink (builtins.toFile "type" "redirect");
238 target = symlink (builtins.toFile "target" "${nameWithHash}");
239 };
240 };
242# Given an encoding generation function to use and a directory containing assets, recursively walk the directory and encode each asset.
244# mkAssetsWith :: (String -> Derivation) -> String -> Derivation
245mkAssetsWith = encodings: d: dirToPath (unionMapFilesWithName (mkAsset encodings) (readDirRecursive d));
247# Given an input directory containing assets, recursively walk the directory and encode each asset with the default encodings.
249# mkAssets :: String -> Derivation
250mkAssets = mkAssetsWith defaultEncodings;