@@ -1,65 +1,63 @@
1−{
2− bashInteractive,
3− buildPackages,
4− cacert,
5− callPackage,
6− closureInfo,
7− coreutils,
8− e2fsprogs,
9− fakeroot,
10− findutils,
11− go,
12− jq,
13− jshon,
14− lib,
15− makeWrapper,
16− moreutils,
17− nix,
18− pigz,
19− pkgs,
20− rsync,
21− runCommand,
22− runtimeShell,
23− shadow,
24− skopeo,
25− storeDir ? builtins.storeDir,
26− substituteAll,
27− symlinkJoin,
28− util-linux,
29− vmTools,
30− writeReferencesToFile,
31− writeScript,
32− writeText,
33− writeTextDir,
34− writePython3,
35− system, # Note: This is the cross system we're compiling for
1+{ bashInteractive
2+, buildPackages
3+, cacert
4+, callPackage
5+, closureInfo
6+, coreutils
7+, e2fsprogs
8+, fakeroot
9+, findutils
10+, go
11+, jq
12+, jshon
13+, lib
14+, makeWrapper
15+, moreutils
16+, nix
17+, pigz
18+, pkgs
19+, rsync
20+, runCommand
21+, runtimeShell
22+, shadow
23+, skopeo
24+, storeDir ? builtins.storeDir
25+, substituteAll
26+, symlinkJoin
27+, util-linux
28+, vmTools
29+, writeReferencesToFile
30+, writeScript
31+, writeText
32+, writeTextDir
33+, writePython3
34+, system
35+, # Note: This is the cross system we're compiling for
3636}:
3737
3838let
3939
40− inherit (lib)
41− optionals
42− ;
43−
44− mkDbExtraCommand = contents: let
45− contentsList = if builtins.isList contents then contents else [ contents ];
46− in ''
47− echo "Generating the nix database..."
48− echo "Warning: only the database of the deepest Nix layer is loaded."
49− echo " If you want to use nix commands in the container, it would"
50− echo " be better to only have one layer that contains a nix store."
51−
52− export NIX_REMOTE=local?root=$PWD
53− # A user is required by nix
54− # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478
55− export USER=nobody
56− ${buildPackages.nix}/bin/nix-store --load-db < ${closureInfo {rootPaths = contentsList;}}/registration
57−
58− mkdir -p nix/var/nix/gcroots/docker/
59− for i in ${lib.concatStringsSep " " contentsList}; do
60− ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
61− done;
62− '';
40+ mkDbExtraCommand = contents:
41+ let
42+ contentsList = if builtins.isList contents then contents else [ contents ];
43+ in
44+ ''
45+ echo "Generating the nix database..."
46+ echo "Warning: only the database of the deepest Nix layer is loaded."
47+ echo " If you want to use nix commands in the container, it would"
48+ echo " be better to only have one layer that contains a nix store."
49+
50+ export NIX_REMOTE=local?root=$PWD
51+ # A user is required by nix
52+ # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478
53+ export USER=nobody
54+ ${buildPackages.nix}/bin/nix-store --load-db < ${closureInfo {rootPaths = contentsList;}}/registration
55+
56+ mkdir -p nix/var/nix/gcroots/docker/
57+ for i in ${lib.concatStringsSep " " contentsList}; do
58+ ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
59+ done;
60+ '';
6361
6462 # The OCI Image specification recommends that configurations use values listed
6563 # in the Go Language document for GOARCH.
@@ -70,14 +68,14 @@ let
7068
7169in
7270rec {
73−
7471 examples = callPackage ./examples.nix {
7572 inherit buildImage buildLayeredImage fakeNss pullImage shadowSetup buildImageWithNixDb;
7673 };
7774
78− pullImage = let
79− fixName = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
80− in
75+ pullImage =
76+ let
77+ fixName = name: builtins.replaceStrings [ "/" ":" ] [ "-" "-" ] name;
78+ in
8179 { imageName
8280 # To find the digest of an image, you can use skopeo:
8381 # see doc/functions.xml
@@ -96,21 +94,22 @@ rec {
9694 , name ? fixName "docker-image-${finalImageName}-${finalImageTag}.tar"
9795 }:
9896
99− runCommand name {
100− inherit imageDigest;
101− imageName = finalImageName;
102− imageTag = finalImageTag;
103− impureEnvVars = lib.fetchers.proxyImpureEnvVars;
104− outputHashMode = "flat";
105− outputHashAlgo = "sha256";
106− outputHash = sha256;
107−
108− nativeBuildInputs = lib.singleton skopeo;
109− SSL_CERT_FILE = "${cacert.out}/etc/ssl/certs/ca-bundle.crt";
110−
111− sourceURL = "docker://${imageName}@${imageDigest}";
112− destNameTag = "${finalImageName}:${finalImageTag}";
113− } ''
97+ runCommand name
98+ {
99+ inherit imageDigest;
100+ imageName = finalImageName;
101+ imageTag = finalImageTag;
102+ impureEnvVars = lib.fetchers.proxyImpureEnvVars;
103+ outputHashMode = "flat";
104+ outputHashAlgo = "sha256";
105+ outputHash = sha256;
106+
107+ nativeBuildInputs = lib.singleton skopeo;
108+ SSL_CERT_FILE = "${cacert.out}/etc/ssl/certs/ca-bundle.crt";
109+
110+ sourceURL = "docker://${imageName}@${imageDigest}";
111+ destNameTag = "${finalImageName}:${finalImageTag}";
112+ } ''
114113 skopeo \
115114 --src-tls-verify=${lib.boolToString tlsVerify} \
116115 --insecure-policy \
@@ -126,13 +125,14 @@ rec {
126125 tarsum = pkgs.tarsum;
127126
128127 # buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
129− mergeDrvs = {
130− derivations,
131− onlyDeps ? false
132− }:
133− runCommand "merge-drvs" {
134− inherit derivations onlyDeps;
135− } ''
128+ mergeDrvs =
129+ { derivations
130+ , onlyDeps ? false
131+ }:
132+ runCommand "merge-drvs"
133+ {
134+ inherit derivations onlyDeps;
135+ } ''
136136 if [[ -n "$onlyDeps" ]]; then
137137 echo $derivations > $out
138138 exit 0
@@ -180,104 +180,105 @@ rec {
180180 '';
181181
182182 # Run commands in a virtual machine.
183− runWithOverlay = {
184− name,
185− fromImage ? null,
186− fromImageName ? null,
187− fromImageTag ? null,
188− diskSize ? 1024,
189− preMount ? "",
190− postMount ? "",
191− postUmount ? ""
192− }:
193− let
194− result = vmTools.runInLinuxVM (
195− runCommand name {
196− preVM = vmTools.createEmptyImage {
197− size = diskSize;
198− fullName = "docker-run-disk";
199− };
200− inherit fromImage fromImageName fromImageTag;
201−
202− nativeBuildInputs = [ util-linux e2fsprogs jshon rsync jq ];
203− } ''
204− mkdir disk
205− mkfs /dev/${vmTools.hd}
206− mount /dev/${vmTools.hd} disk
207− cd disk
208−
209− if [[ -n "$fromImage" ]]; then
210− echo "Unpacking base image..."
211− mkdir image
212− tar -C image -xpf "$fromImage"
213−
214− if [[ -n "$fromImageName" ]] && [[ -n "$fromImageTag" ]]; then
215− parentID="$(
216− cat "image/manifest.json" |
217− jq -r '.[] | select(.RepoTags | contains([$desiredTag])) | rtrimstr(".json")' \
218− --arg desiredTag "$fromImageName:$fromImageTag"
219− )"
220− else
221− echo "From-image name or tag wasn't set. Reading the first ID."
222− parentID="$(cat "image/manifest.json" | jq -r '.[0].Config | rtrimstr(".json")')"
223− fi
183+ runWithOverlay =
184+ { name
185+ , fromImage ? null
186+ , fromImageName ? null
187+ , fromImageTag ? null
188+ , diskSize ? 1024
189+ , preMount ? ""
190+ , postMount ? ""
191+ , postUmount ? ""
192+ }:
193+ let
194+ result = vmTools.runInLinuxVM (
195+ runCommand name
196+ {
197+ preVM = vmTools.createEmptyImage {
198+ size = diskSize;
199+ fullName = "docker-run-disk";
200+ };
201+ inherit fromImage fromImageName fromImageTag;
202+
203+ nativeBuildInputs = [ util-linux e2fsprogs jshon rsync jq ];
204+ } ''
205+ mkdir disk
206+ mkfs /dev/${vmTools.hd}
207+ mount /dev/${vmTools.hd} disk
208+ cd disk
209+
210+ if [[ -n "$fromImage" ]]; then
211+ echo "Unpacking base image..."
212+ mkdir image
213+ tar -C image -xpf "$fromImage"
214+
215+ if [[ -n "$fromImageName" ]] && [[ -n "$fromImageTag" ]]; then
216+ parentID="$(
217+ cat "image/manifest.json" |
218+ jq -r '.[] | select(.RepoTags | contains([$desiredTag])) | rtrimstr(".json")' \
219+ --arg desiredTag "$fromImageName:$fromImageTag"
220+ )"
221+ else
222+ echo "From-image name or tag wasn't set. Reading the first ID."
223+ parentID="$(cat "image/manifest.json" | jq -r '.[0].Config | rtrimstr(".json")')"
224+ fi
224225
225− cat ./image/manifest.json | jq -r '.[0].Layers | .[]' > layer-list
226− else
227− touch layer-list
228− fi
226+ cat ./image/manifest.json | jq -r '.[0].Layers | .[]' > layer-list
227+ else
228+ touch layer-list
229+ fi
229230
230− # Unpack all of the parent layers into the image.
231− lowerdir=""
232− extractionID=0
233− for layerTar in $(tac layer-list); do
234− echo "Unpacking layer $layerTar"
235− extractionID=$((extractionID + 1))
231+ # Unpack all of the parent layers into the image.
232+ lowerdir=""
233+ extractionID=0
234+ for layerTar in $(tac layer-list); do
235+ echo "Unpacking layer $layerTar"
236+ extractionID=$((extractionID + 1))
236237
237− mkdir -p image/$extractionID/layer
238− tar -C image/$extractionID/layer -xpf image/$layerTar
239− rm image/$layerTar
238+ mkdir -p image/$extractionID/layer
239+ tar -C image/$extractionID/layer -xpf image/$layerTar
240+ rm image/$layerTar
240241
241− find image/$extractionID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
242+ find image/$extractionID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
242243
243− # Get the next lower directory and continue the loop.
244− lowerdir=image/$extractionID/layer''${lowerdir:+:}$lowerdir
245− done
244+ # Get the next lower directory and continue the loop.
245+ lowerdir=image/$extractionID/layer''${lowerdir:+:}$lowerdir
246+ done
246247
247− mkdir work
248− mkdir layer
249− mkdir mnt
248+ mkdir work
249+ mkdir layer
250+ mkdir mnt
250251
251− ${lib.optionalString (preMount != "") ''
252− # Execute pre-mount steps
253− echo "Executing pre-mount steps..."
254− ${preMount}
255− ''}
252+ ${lib.optionalString (preMount != "") ''
253+ # Execute pre-mount steps
254+ echo "Executing pre-mount steps..."
255+ ${preMount}
256+ ''}
256257
257− if [ -n "$lowerdir" ]; then
258− mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
259− else
260− mount --bind layer mnt
261− fi
258+ if [ -n "$lowerdir" ]; then
259+ mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
260+ else
261+ mount --bind layer mnt
262+ fi
262263
263− ${lib.optionalString (postMount != "") ''
264− # Execute post-mount steps
265− echo "Executing post-mount steps..."
266− ${postMount}
267− ''}
264+ ${lib.optionalString (postMount != "") ''
265+ # Execute post-mount steps
266+ echo "Executing post-mount steps..."
267+ ${postMount}
268+ ''}
268269
269− umount mnt
270+ umount mnt
270271
271− (
272− cd layer
273− cmd='name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"'
274− find . -type c -exec bash -c "$cmd" \;
275− )
272+ (
273+ cd layer
274+ cmd='name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"'
275+ find . -type c -exec bash -c "$cmd" \;
276+ )
276277
277− ${postUmount}
278− '');
278+ ${postUmount}
279+ '');
279280 in
280− runCommand name {} ''
281+ runCommand name { } ''
281282 mkdir -p $out
282283 cd ${result}
283284 cp layer.tar json VERSION $out
@@ -293,7 +294,6 @@ rec {
293294 '';
294295 };
295296
296−
297297 # Create an executable shell script which has the coreutils in its
298298 # PATH. Since root scripts are executed in a blank environment, even
299299 # things like `ls` or `echo` will be missing.
@@ -306,94 +306,100 @@ rec {
306306 '';
307307
308308 # Create a "layer" (set of files).
309− mkPureLayer = {
310− # Name of the layer
311− name,
312− # JSON containing configuration and metadata for this layer.
313− baseJson,
314− # Files to add to the layer.
315− contents ? null,
316− # When copying the contents into the image, preserve symlinks to
317− # directories (see `rsync -K`). Otherwise, transform those symlinks
318− # into directories.
319− keepContentsDirlinks ? false,
320− # Additional commands to run on the layer before it is tar'd up.
321− extraCommands ? "", uid ? 0, gid ? 0
322− }:
323− runCommand "docker-layer-${name}" {
324− inherit baseJson contents extraCommands;
325− nativeBuildInputs = [ jshon rsync tarsum ];
326− }
327− ''
328− mkdir layer
329− if [[ -n "$contents" ]]; then
330− echo "Adding contents..."
331− for item in $contents; do
332− echo "Adding $item"
333− rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
334− done
335− else
336− echo "No contents to add to layer."
337− fi
309+ mkPureLayer =
310+ {
311+ # Name of the layer
312+ name
313+ , # JSON containing configuration and metadata for this layer.
314+ baseJson
315+ , # Files to add to the layer.
316+ contents ? null
317+ , # When copying the contents into the image, preserve symlinks to
318+ # directories (see `rsync -K`). Otherwise, transform those symlinks
319+ # into directories.
320+ keepContentsDirlinks ? false
321+ , # Additional commands to run on the layer before it is tar'd up.
322+ extraCommands ? ""
323+ , uid ? 0
324+ , gid ? 0
325+ }:
326+ runCommand "docker-layer-${name}"
327+ {
328+ inherit baseJson contents extraCommands;
329+ nativeBuildInputs = [ jshon rsync tarsum ];
330+ }
331+ ''
332+ mkdir layer
333+ if [[ -n "$contents" ]]; then
334+ echo "Adding contents..."
335+ for item in $contents; do
336+ echo "Adding $item"
337+ rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
338+ done
339+ else
340+ echo "No contents to add to layer."
341+ fi
338342
339− chmod ug+w layer
343+ chmod ug+w layer
340344
341− if [[ -n "$extraCommands" ]]; then
342− (cd layer; eval "$extraCommands")
343− fi
345+ if [[ -n "$extraCommands" ]]; then
346+ (cd layer; eval "$extraCommands")
347+ fi
344348
345− # Tar up the layer and throw it into 'layer.tar'.
346− echo "Packing layer..."
347− mkdir $out
348− tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee -p $out/layer.tar | tarsum)
349+ # Tar up the layer and throw it into 'layer.tar'.
350+ echo "Packing layer..."
351+ mkdir $out
352+ tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee -p $out/layer.tar | tarsum)
349353
350− # Add a 'checksum' field to the JSON, with the value set to the
351− # checksum of the tarball.
352− cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json
354+ # Add a 'checksum' field to the JSON, with the value set to the
355+ # checksum of the tarball.
356+ cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json
353357
354− # Indicate to docker that we're using schema version 1.0.
355− echo -n "1.0" > $out/VERSION
358+ # Indicate to docker that we're using schema version 1.0.
359+ echo -n "1.0" > $out/VERSION
356360
357− echo "Finished building layer '${name}'"
358− '';
361+ echo "Finished building layer '${name}'"
362+ '';
359363
360364 # Make a "root" layer; required if we need to execute commands as a
361365 # privileged user on the image. The commands themselves will be
362366 # performed in a virtual machine sandbox.
363− mkRootLayer = {
364− # Name of the image.
365− name,
366− # Script to run as root. Bash.
367− runAsRoot,
368− # Files to add to the layer. If null, an empty layer will be created.
369− contents ? null,
370− # When copying the contents into the image, preserve symlinks to
371− # directories (see `rsync -K`). Otherwise, transform those symlinks
372− # into directories.
373− keepContentsDirlinks ? false,
374− # JSON containing configuration and metadata for this layer.
375− baseJson,
376− # Existing image onto which to append the new layer.
377− fromImage ? null,
378− # Name of the image we're appending onto.
379− fromImageName ? null,
380− # Tag of the image we're appending onto.
381− fromImageTag ? null,
382− # How much disk to allocate for the temporary virtual machine.
383− diskSize ? 1024,
384− # Commands (bash) to run on the layer; these do not require sudo.
385− extraCommands ? ""
386− }:
367+ mkRootLayer =
368+ {
369+ # Name of the image.
370+ name
371+ , # Script to run as root. Bash.
372+ runAsRoot
373+ , # Files to add to the layer. If null, an empty layer will be created.
374+ contents ? null
375+ , # When copying the contents into the image, preserve symlinks to
376+ # directories (see `rsync -K`). Otherwise, transform those symlinks
377+ # into directories.
378+ keepContentsDirlinks ? false
379+ , # JSON containing configuration and metadata for this layer.
380+ baseJson
381+ , # Existing image onto which to append the new layer.
382+ fromImage ? null
383+ , # Name of the image we're appending onto.
384+ fromImageName ? null
385+ , # Tag of the image we're appending onto.
386+ fromImageTag ? null
387+ , # How much disk to allocate for the temporary virtual machine.
388+ diskSize ? 1024
389+ , # Commands (bash) to run on the layer; these do not require sudo.
390+ extraCommands ? ""
391+ }:
387392 # Generate an executable script from the `runAsRoot` text.
388393 let
389394 runAsRootScript = shellScript "run-as-root.sh" runAsRoot;
390395 extraCommandsScript = shellScript "extra-commands.sh" extraCommands;
391− in runWithOverlay {
396+ in
397+ runWithOverlay {
392398 name = "docker-layer-${name}";
393399
394400 inherit fromImage fromImageName fromImageTag diskSize;
395401
396− preMount = lib.optionalString (contents != null && contents != []) ''
402+ preMount = lib.optionalString (contents != null && contents != [ ]) ''
397403 echo "Adding contents..."
398404 for item in ${toString contents}; do
399405 echo "Adding $item..."
@@ -445,11 +451,12 @@ rec {
445451 '';
446452 };
447453
448− buildLayeredImage = {name, ...}@args:
454+ buildLayeredImage = { name, ... }@args:
449455 let
450456 stream = streamLayeredImage args;
451457 in
452− runCommand "${baseNameOf name}.tar.gz" {
458+ runCommand "${baseNameOf name}.tar.gz"
459+ {
453460 inherit (stream) imageName;
454461 passthru = { inherit (stream) imageTag; };
455462 nativeBuildInputs = [ pigz ];
@@ -461,40 +468,45 @@ rec {
461468 # 4. compute the layer id
462469 # 5. put the layer in the image
463470 # 6. repack the image
464− buildImage = args@{
465− # Image name.
466− name,
467− # Image tag, when null then the nix output hash will be used.
468− tag ? null,
469− # Parent image, to append to.
470− fromImage ? null,
471− # Name of the parent image; will be read from the image otherwise.
472− fromImageName ? null,
473− # Tag of the parent image; will be read from the image otherwise.
474− fromImageTag ? null,
475− # Files to put on the image (a nix store path or list of paths).
476− contents ? null,
477− # When copying the contents into the image, preserve symlinks to
478− # directories (see `rsync -K`). Otherwise, transform those symlinks
479− # into directories.
480− keepContentsDirlinks ? false,
481− # Docker config; e.g. what command to run on the container.
482− config ? null,
483− # Optional bash script to run on the files prior to fixturizing the layer.
484− extraCommands ? "", uid ? 0, gid ? 0,
485− # Optional bash script to run as root on the image when provisioning.
486− runAsRoot ? null,
487− # Size of the virtual machine disk to provision when building the image.
488− diskSize ? 1024,
489− # Time of creation of the image.
490− created ? "1970-01-01T00:00:01Z",
491− }:
471+ buildImage =
472+ args@{
473+ # Image name.
474+ name
475+ , # Image tag, when null then the nix output hash will be used.
476+ tag ? null
477+ , # Parent image, to append to.
478+ fromImage ? null
479+ , # Name of the parent image; will be read from the image otherwise.
480+ fromImageName ? null
481+ , # Tag of the parent image; will be read from the image otherwise.
482+ fromImageTag ? null
483+ , # Files to put on the image (a nix store path or list of paths).
484+ contents ? null
485+ , # When copying the contents into the image, preserve symlinks to
486+ # directories (see `rsync -K`). Otherwise, transform those symlinks
487+ # into directories.
488+ keepContentsDirlinks ? false
489+ , # Docker config; e.g. what command to run on the container.
490+ config ? null
491+ , # Optional bash script to run on the files prior to fixturizing the layer.
492+ extraCommands ? ""
493+ , uid ? 0
494+ , gid ? 0
495+ , # Optional bash script to run as root on the image when provisioning.
496+ runAsRoot ? null
497+ , # Size of the virtual machine disk to provision when building the image.
498+ diskSize ? 1024
499+ , # Time of creation of the image.
500+ created ? "1970-01-01T00:00:01Z"
501+ ,
502+ }:
492503
493504 let
494505 baseName = baseNameOf name;
495506
496507 # Create a JSON blob of the configuration. Set the date to unix zero.
497− baseJson = let
508+ baseJson =
509+ let
498510 pure = writeText "${baseName}-config.json" (builtins.toJSON {
499511 inherit created config;
500512 architecture = defaultArch;
@@ -503,36 +515,41 @@ rec {
503515 impure = runCommand "${baseName}-config.json"
504516 { nativeBuildInputs = [ jq ]; }
505517 ''
506− jq ".created = \"$(TZ=utc date --iso-8601="seconds")\"" ${pure} > $out
518+ jq ".created = \"$(TZ=utc date --iso-8601="seconds")\"" ${pure} > $out
507519 '';
508− in if created == "now" then impure else pure;
520+ in
521+ if created == "now" then impure else pure;
509522
510523 layer =
511524 if runAsRoot == null
512− then mkPureLayer {
513− name = baseName;
514− inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
515− } else mkRootLayer {
516− name = baseName;
517− inherit baseJson fromImage fromImageName fromImageTag
518− contents keepContentsDirlinks runAsRoot diskSize
519− extraCommands;
520− };
521− result = runCommand "docker-image-${baseName}.tar.gz" {
522− nativeBuildInputs = [ jshon pigz coreutils findutils jq moreutils ];
523− # Image name must be lowercase
524− imageName = lib.toLower name;
525− imageTag = if tag == null then "" else tag;
526− inherit fromImage baseJson;
527− layerClosure = writeReferencesToFile layer;
528− passthru.buildArgs = args;
529− passthru.layer = layer;
530− passthru.imageTag =
531− if tag != null
525+ then
526+ mkPureLayer
527+ {
528+ name = baseName;
529+ inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
530+ } else
531+ mkRootLayer {
532+ name = baseName;
533+ inherit baseJson fromImage fromImageName fromImageTag
534+ contents keepContentsDirlinks runAsRoot diskSize
535+ extraCommands;
536+ };
537+ result = runCommand "docker-image-${baseName}.tar.gz"
538+ {
539+ nativeBuildInputs = [ jshon pigz coreutils findutils jq moreutils ];
540+ # Image name must be lowercase
541+ imageName = lib.toLower name;
542+ imageTag = if tag == null then "" else tag;
543+ inherit fromImage baseJson;
544+ layerClosure = writeReferencesToFile layer;
545+ passthru.buildArgs = args;
546+ passthru.layer = layer;
547+ passthru.imageTag =
548+ if tag != null
532549 then tag
533550 else
534551 lib.head (lib.strings.splitString "-" (baseNameOf result.outPath));
535− } ''
552+ } ''
536553 ${lib.optionalString (tag == null) ''
537554 outName="$(basename "$out")"
538555 outHash=$(echo "$outName" | cut -d - -f 1)
@@ -732,14 +749,14 @@ rec {
732749 root:x:0:
733750 nobody:x:65534:
734751 '')
735− (runCommand "var-empty" {} ''
752+ (runCommand "var-empty" { } ''
736753 mkdir -p $out/var/empty
737754 '')
738755 ];
739756 };
740757
741758 # This provides /bin/sh, pointing to bashInteractive.
742− binSh = runCommand "bin-sh" {} ''
759+ binSh = runCommand "bin-sh" { } ''
743760 mkdir -p $out/bin
744761 ln -s ${bashInteractive}/bin/bash $out/bin/sh
745762 '';
@@ -760,189 +777,194 @@ rec {
760777 })
761778 );
762779
763− streamLayeredImage = {
764− # Image Name
765− name,
766− # Image tag, the Nix's output hash will be used if null
767− tag ? null,
768− # Parent image, to append to.
769− fromImage ? null,
770− # Files to put on the image (a nix store path or list of paths).
771− contents ? [],
772− # Docker config; e.g. what command to run on the container.
773− config ? {},
774− # Time of creation of the image. Passing "now" will make the
775− # created date be the time of building.
776− created ? "1970-01-01T00:00:01Z",
777− # Optional bash script to run on the files prior to fixturizing the layer.
778− extraCommands ? "",
779− # Optional bash script to run inside fakeroot environment.
780− # Could be used for changing ownership of files in customisation layer.
781− fakeRootCommands ? "",
782− # We pick 100 to ensure there is plenty of room for extension. I
783− # believe the actual maximum is 128.
784− maxLayers ? 100,
785− # Whether to include store paths in the image. You generally want to leave
786− # this on, but tooling may disable this to insert the store paths more
787− # efficiently via other means, such as bind mounting the host store.
788− includeStorePaths ? true,
789− }:
790− assert
780+ streamLayeredImage =
781+ {
782+ # Image Name
783+ name
784+ , # Image tag, the Nix's output hash will be used if null
785+ tag ? null
786+ , # Parent image, to append to.
787+ fromImage ? null
788+ , # Files to put on the image (a nix store path or list of paths).
789+ contents ? [ ]
790+ , # Docker config; e.g. what command to run on the container.
791+ config ? { }
792+ , # Time of creation of the image. Passing "now" will make the
793+ # created date be the time of building.
794+ created ? "1970-01-01T00:00:01Z"
795+ , # Optional bash script to run on the files prior to fixturizing the layer.
796+ extraCommands ? ""
797+ , # Optional bash script to run inside fakeroot environment.
798+ # Could be used for changing ownership of files in customisation layer.
799+ fakeRootCommands ? ""
800+ , # We pick 100 to ensure there is plenty of room for extension. I
801+ # believe the actual maximum is 128.
802+ maxLayers ? 100
803+ , # Whether to include store paths in the image. You generally want to leave
804+ # this on, but tooling may disable this to insert the store paths more
805+ # efficiently via other means, such as bind mounting the host store.
806+ includeStorePaths ? true
807+ ,
808+ }:
809+ assert
791810 (lib.assertMsg (maxLayers > 1)
792− "the maxLayers argument of dockerTools.buildLayeredImage function must be greather than 1 (current value: ${toString maxLayers})");
793− let
794− baseName = baseNameOf name;
795−
796− streamScript = writePython3 "stream" {} ./stream_layered_image.py;
797− baseJson = writeText "${baseName}-base.json" (builtins.toJSON {
798− inherit config;
799− architecture = defaultArch;
800− os = "linux";
801− });
802−
803− contentsList = if builtins.isList contents then contents else [ contents ];
804−
805− # We store the customisation layer as a tarball, to make sure that
806− # things like permissions set on 'extraCommands' are not overriden
807− # by Nix. Then we precompute the sha256 for performance.
808− customisationLayer = symlinkJoin {
809− name = "${baseName}-customisation-layer";
810− paths = contentsList;
811− inherit extraCommands fakeRootCommands;
812− nativeBuildInputs = [ fakeroot ];
813− postBuild = ''
814− mv $out old_out
815− (cd old_out; eval "$extraCommands" )
816−
817− mkdir $out
818−
819− fakeroot bash -c '
820− source $stdenv/setup
821− cd old_out
822− eval "$fakeRootCommands"
823− tar \
824− --sort name \
825− --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \
826− --hard-dereference \
827− -cf $out/layer.tar .
828− '
829−
830− sha256sum $out/layer.tar \
831− | cut -f 1 -d ' ' \
832− > $out/checksum
833− '';
834− };
835−
836− closureRoots = optionals includeStorePaths /* normally true */ (
837− [ baseJson ] ++ contentsList
838− );
839− overallClosure = writeText "closure" (lib.concatStringsSep " " closureRoots);
840−
841− # These derivations are only created as implementation details of docker-tools,
842− # so they'll be excluded from the created images.
843− unnecessaryDrvs = [ baseJson overallClosure ];
844−
845− conf = runCommand "${baseName}-conf.json" {
846− inherit fromImage maxLayers created;
847− imageName = lib.toLower name;
848− passthru.imageTag =
849− if tag != null
850− then tag
851− else
852− lib.head (lib.strings.splitString "-" (baseNameOf conf.outPath));
853− paths = buildPackages.referencesByPopularity overallClosure;
854− nativeBuildInputs = [ jq ];
855− } ''
856− ${if (tag == null) then ''
857− outName="$(basename "$out")"
858− outHash=$(echo "$outName" | cut -d - -f 1)
859−
860− imageTag=$outHash
861− '' else ''
862− imageTag="${tag}"
863− ''}
811+ "the maxLayers argument of dockerTools.buildLayeredImage function must be greather than 1 (current value: ${toString maxLayers})");
812+ let
813+ baseName = baseNameOf name;
814+
815+ streamScript = writePython3 "stream" { } ./stream_layered_image.py;
816+ baseJson = writeText "${baseName}-base.json" (builtins.toJSON {
817+ inherit config;
818+ architecture = defaultArch;
819+ os = "linux";
820+ });
821+
822+ contentsList = if builtins.isList contents then contents else [ contents ];
823+
824+ # We store the customisation layer as a tarball, to make sure that
825+ # things like permissions set on 'extraCommands' are not overriden
826+ # by Nix. Then we precompute the sha256 for performance.
827+ customisationLayer = symlinkJoin {
828+ name = "${baseName}-customisation-layer";
829+ paths = contentsList;
830+ inherit extraCommands fakeRootCommands;
831+ nativeBuildInputs = [ fakeroot ];
832+ postBuild = ''
833+ mv $out old_out
834+ (cd old_out; eval "$extraCommands" )
835+
836+ mkdir $out
837+
838+ fakeroot bash -c '
839+ source $stdenv/setup
840+ cd old_out
841+ eval "$fakeRootCommands"
842+ tar \
843+ --sort name \
844+ --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \
845+ --hard-dereference \
846+ -cf $out/layer.tar .
847+ '
848+
849+ sha256sum $out/layer.tar \
850+ | cut -f 1 -d ' ' \
851+ > $out/checksum
852+ '';
853+ };
864854
865− # convert "created" to iso format
866− if [[ "$created" != "now" ]]; then
867− created="$(date -Iseconds -d "$created")"
868− fi
855+ closureRoots = lib.optionals includeStorePaths /* normally true */ (
856+ [ baseJson ] ++ contentsList
857+ );
858+ overallClosure = writeText "closure" (lib.concatStringsSep " " closureRoots);
859+
860+ # These derivations are only created as implementation details of docker-tools,
861+ # so they'll be excluded from the created images.
862+ unnecessaryDrvs = [ baseJson overallClosure ];
863+
864+ conf = runCommand "${baseName}-conf.json"
865+ {
866+ inherit fromImage maxLayers created;
867+ imageName = lib.toLower name;
868+ passthru.imageTag =
869+ if tag != null
870+ then tag
871+ else
872+ lib.head (lib.strings.splitString "-" (baseNameOf conf.outPath));
873+ paths = buildPackages.referencesByPopularity overallClosure;
874+ nativeBuildInputs = [ jq ];
875+ } ''
876+ ${if (tag == null) then ''
877+ outName="$(basename "$out")"
878+ outHash=$(echo "$outName" | cut -d - -f 1)
879+
880+ imageTag=$outHash
881+ '' else ''
882+ imageTag="${tag}"
883+ ''}
884+
885+ # convert "created" to iso format
886+ if [[ "$created" != "now" ]]; then
887+ created="$(date -Iseconds -d "$created")"
888+ fi
869889
870− paths() {
871− cat $paths ${lib.concatMapStringsSep " "
872− (path: "| (grep -v ${path} || true)")
873− unnecessaryDrvs}
874− }
890+ paths() {
891+ cat $paths ${lib.concatMapStringsSep " "
892+ (path: "| (grep -v ${path} || true)")
893+ unnecessaryDrvs}
894+ }
875895
876− # Compute the number of layers that are already used by a potential
877− # 'fromImage' as well as the customization layer. Ensure that there is
878− # still at least one layer available to store the image contents.
879− usedLayers=0
896+ # Compute the number of layers that are already used by a potential
897+ # 'fromImage' as well as the customization layer. Ensure that there is
898+ # still at least one layer available to store the image contents.
899+ usedLayers=0
880900
881− # subtract number of base image layers
882− if [[ -n "$fromImage" ]]; then
883− (( usedLayers += $(tar -xOf "$fromImage" manifest.json | jq '.[0].Layers | length') ))
884− fi
901+ # subtract number of base image layers
902+ if [[ -n "$fromImage" ]]; then
903+ (( usedLayers += $(tar -xOf "$fromImage" manifest.json | jq '.[0].Layers | length') ))
904+ fi
885905
886− # one layer will be taken up by the customisation layer
887− (( usedLayers += 1 ))
906+ # one layer will be taken up by the customisation layer
907+ (( usedLayers += 1 ))
888908
889− if ! (( $usedLayers < $maxLayers )); then
890− echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \
891− "'extraCommands', but only maxLayers=$maxLayers were" \
892− "allowed. At least 1 layer is required to store contents."
893− exit 1
894− fi
895− availableLayers=$(( maxLayers - usedLayers ))
909+ if ! (( $usedLayers < $maxLayers )); then
910+ echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \
911+ "'extraCommands', but only maxLayers=$maxLayers were" \
912+ "allowed. At least 1 layer is required to store contents."
913+ exit 1
914+ fi
915+ availableLayers=$(( maxLayers - usedLayers ))
916+
917+ # Create $maxLayers worth of Docker Layers, one layer per store path
918+ # unless there are more paths than $maxLayers. In that case, create
919+ # $maxLayers-1 for the most popular layers, and smush the remainaing
920+ # store paths in to one final layer.
921+ #
922+ # The following code is fiddly w.r.t. ensuring every layer is
923+ # created, and that no paths are missed. If you change the
924+ # following lines, double-check that your code behaves properly
925+ # when the number of layers equals:
926+ # maxLayers-1, maxLayers, and maxLayers+1, 0
927+ store_layers="$(
928+ paths |
929+ jq -sR '
930+ rtrimstr("\n") | split("\n")
931+ | (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ]
932+ | map(select(length > 0))
933+ ' \
934+ --argjson maxLayers "$availableLayers"
935+ )"
896936
897− # Create $maxLayers worth of Docker Layers, one layer per store path
898− # unless there are more paths than $maxLayers. In that case, create
899− # $maxLayers-1 for the most popular layers, and smush the remainaing
900− # store paths in to one final layer.
901− #
902− # The following code is fiddly w.r.t. ensuring every layer is
903− # created, and that no paths are missed. If you change the
904− # following lines, double-check that your code behaves properly
905− # when the number of layers equals:
906− # maxLayers-1, maxLayers, and maxLayers+1, 0
907− store_layers="$(
908− paths |
909− jq -sR '
910− rtrimstr("\n") | split("\n")
911− | (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ]
912− | map(select(length > 0))
913− ' \
914− --argjson maxLayers "$availableLayers"
915− )"
916−
917− cat ${baseJson} | jq '
918− . + {
919− "store_dir": $store_dir,
920− "from_image": $from_image,
921− "store_layers": $store_layers,
922− "customisation_layer", $customisation_layer,
923− "repo_tag": $repo_tag,
924− "created": $created
925− }
926− ' --arg store_dir "${storeDir}" \
927− --argjson from_image ${if fromImage == null then "null" else "'\"${fromImage}\"'"} \
928− --argjson store_layers "$store_layers" \
929− --arg customisation_layer ${customisationLayer} \
930− --arg repo_tag "$imageName:$imageTag" \
931− --arg created "$created" |
932− tee $out
933− '';
934− result = runCommand "stream-${baseName}" {
935− inherit (conf) imageName;
936− passthru = {
937− inherit (conf) imageTag;
938−
939− # Distinguish tarballs and exes at the Nix level so functions that
940− # take images can know in advance how the image is supposed to be used.
941− isExe = true;
942− };
943− nativeBuildInputs = [ makeWrapper ];
944− } ''
945− makeWrapper ${streamScript} $out --add-flags ${conf}
946− '';
947− in result;
937+ cat ${baseJson} | jq '
938+ . + {
939+ "store_dir": $store_dir,
940+ "from_image": $from_image,
941+ "store_layers": $store_layers,
942+ "customisation_layer", $customisation_layer,
943+ "repo_tag": $repo_tag,
944+ "created": $created
945+ }
946+ ' --arg store_dir "${storeDir}" \
947+ --argjson from_image ${if fromImage == null then "null" else "'\"${fromImage}\"'"} \
948+ --argjson store_layers "$store_layers" \
949+ --arg customisation_layer ${customisationLayer} \
950+ --arg repo_tag "$imageName:$imageTag" \
951+ --arg created "$created" |
952+ tee $out
953+ '';
954+ result = runCommand "stream-${baseName}"
955+ {
956+ inherit (conf) imageName;
957+ passthru = {
958+ inherit (conf) imageTag;
959+
960+ # Distinguish tarballs and exes at the Nix level so functions that
961+ # take images can know in advance how the image is supposed to be used.
962+ isExe = true;
963+ };
964+ nativeBuildInputs = [ makeWrapper ];
965+ } ''
966+ makeWrapper ${streamScript} $out --add-flags ${conf}
967+ '';
968+ in
969+ result;
948970}