Merge pull request #179801 from hercules-ci/fix-footgun-dockerTools-buildImage-contents

dockerTools.buildImage: Add copyToRoot to replace contents, explain usage

doc/builders/images/dockertools.section.md

@@ -20,7 +20,12 @@ buildImage {
   fromImageName = null;
   fromImageTag = "latest";
 
-  contents = pkgs.redis;
+  copyToRoot = pkgs.buildEnv {
+    name = "image-root";
+    paths = [ pkgs.redis ];
+    pathsToLink = [ "/bin" ];
+  };
+
   runAsRoot = ''
     #!${pkgs.runtimeShell}
     mkdir -p /data
@@ -46,7 +51,7 @@ The above example will build a Docker image `redis/latest` from the given base i
 
 - `fromImageTag` can be used to further specify the tag of the base image within the repository, in case an image contains multiple tags. By default it's `null`, in which case `buildImage` will peek the first tag available for the base image.
 
-- `contents` is a derivation that will be copied in the new layer of the resulting image. This can be similarly seen as `ADD contents/ /` in a `Dockerfile`. By default it's `null`.
+- `copyToRoot` is a derivation that will be copied in the new layer of the resulting image. This can be similarly seen as `ADD contents/ /` in a `Dockerfile`. By default it's `null`.
 
 - `runAsRoot` is a bash script that will run as root in an environment that overlays the existing layers of the base image with the new resulting layer, including the previously copied `contents` derivation. This can be similarly seen as `RUN ...` in a `Dockerfile`.
 
@@ -81,7 +86,11 @@ pkgs.dockerTools.buildImage {
   name = "hello";
   tag = "latest";
   created = "now";
-  contents = pkgs.hello;
+  copyToRoot = pkgs.buildEnv {
+    name = "image-root";
+    paths = [ pkgs.hello ];
+    pathsToLink = [ "/bin" ];
+  };
 
   config.Cmd = [ "/bin/hello" ];
 }

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -303,6 +303,15 @@
           and require manual remediation.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <literal>dockerTools.buildImage</literal> deprecates the
+          misunderstood <literal>contents</literal> parameter, in favor
+          of <literal>copyToRoot</literal>. Use
+          <literal>copyToRoot = buildEnv { ... };</literal> or similar
+          if you intend to add packages to <literal>/bin</literal>.
+        </para>
+      </listitem>
       <listitem>
         <para>
           memtest86+ was updated from 5.00-coreboot-002 to 6.00-beta2.

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -114,6 +114,9 @@ Use `configure.packages` instead.
 
 - Matrix Synapse now requires entries in the `state_group_edges` table to be unique, in order to prevent accidentally introducing duplicate information (for example, because a database backup was restored multiple times). If your Synapse database already has duplicate rows in this table, this could fail with an error and require manual remediation.
 
+- `dockerTools.buildImage` deprecates the misunderstood `contents` parameter, in favor of `copyToRoot`.
+  Use `copyToRoot = buildEnv { ... };` or similar if you intend to add packages to `/bin`.
+
 - memtest86+ was updated from 5.00-coreboot-002 to 6.00-beta2. It is now the upstream version from https://www.memtest.org/, as coreboot's fork is no longer available.
 
 - Add udev rules for the Teensy family of microcontrollers.

nixos/tests/docker-tools-cross.nix

@@ -24,7 +24,11 @@ let
   hello1 = remoteCrossPkgs.dockerTools.buildImage {
     name = "hello1";
     tag = "latest";
-    contents = remoteCrossPkgs.hello;
+    copyToRoot = remoteCrossPkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ remoteCrossPkgs.hello ];
+    };
   };
 
   hello2 = remoteCrossPkgs.dockerTools.buildLayeredImage {

pkgs/build-support/docker/default.nix

@@ -332,7 +332,7 @@ rec {
     , # JSON containing configuration and metadata for this layer.
       baseJson
     , # Files to add to the layer.
-      contents ? null
+      copyToRoot ? null
     , # When copying the contents into the image, preserve symlinks to
       # directories (see `rsync -K`).  Otherwise, transform those symlinks
       # into directories.
@@ -344,7 +344,8 @@ rec {
     }:
     runCommand "docker-layer-${name}"
       {
-        inherit baseJson contents extraCommands;
+        inherit baseJson extraCommands;
+        contents = copyToRoot;
         nativeBuildInputs = [ jshon rsync tarsum ];
       }
       ''
@@ -390,7 +391,8 @@ rec {
     , # Script to run as root. Bash.
       runAsRoot
     , # Files to add to the layer. If null, an empty layer will be created.
-      contents ? null
+      # To add packages to /bin, use `buildEnv` or similar.
+      copyToRoot ? null
     , # When copying the contents into the image, preserve symlinks to
       # directories (see `rsync -K`).  Otherwise, transform those symlinks
       # into directories.
@@ -418,9 +420,9 @@ rec {
 
       inherit fromImage fromImageName fromImageTag diskSize;
 
-      preMount = lib.optionalString (contents != null && contents != [ ]) ''
+      preMount = lib.optionalString (copyToRoot != null && copyToRoot != [ ]) ''
         echo "Adding contents..."
-        for item in ${escapeShellArgs (map (c: "${c}") (toList contents))}; do
+        for item in ${escapeShellArgs (map (c: "${c}") (toList copyToRoot))}; do
           echo "Adding $item..."
           rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
         done
@@ -500,7 +502,7 @@ rec {
     , # Tag of the parent image; will be read from the image otherwise.
       fromImageTag ? null
     , # Files to put on the image (a nix store path or list of paths).
-      contents ? null
+      copyToRoot ? null
     , # When copying the contents into the image, preserve symlinks to
       # directories (see `rsync -K`).  Otherwise, transform those symlinks
       # into directories.
@@ -517,10 +519,20 @@ rec {
       diskSize ? 1024
     , # Time of creation of the image.
       created ? "1970-01-01T00:00:01Z"
+    , # Deprecated.
+      contents ? null
     ,
     }:
 
     let
+      checked =
+        lib.warnIf (contents != null)
+          "in docker image ${name}: The contents parameter is deprecated. Change to copyToRoot if the contents are designed to be copied to the root filesystem, such as when you use `buildEnv` or similar between contents and your packages. Use copyToRoot = buildEnv { ... }; or similar if you intend to add packages to /bin."
+        lib.throwIf (contents != null && copyToRoot != null) "in docker image ${name}: You can not specify both contents and copyToRoot."
+        ;
+
+      rootContents = if copyToRoot == null then contents else copyToRoot;
+
       baseName = baseNameOf name;
 
       # Create a JSON blob of the configuration. Set the date to unix zero.
@@ -545,13 +557,15 @@ rec {
           mkPureLayer
             {
               name = baseName;
-              inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
+              inherit baseJson keepContentsDirlinks extraCommands uid gid;
+              copyToRoot = rootContents;
             } else
           mkRootLayer {
             name = baseName;
             inherit baseJson fromImage fromImageName fromImageTag
-              contents keepContentsDirlinks runAsRoot diskSize
+              keepContentsDirlinks runAsRoot diskSize
               extraCommands;
+            copyToRoot = rootContents;
           };
       result = runCommand "docker-image-${baseName}.tar.gz"
         {
@@ -715,7 +729,7 @@ rec {
       '';
 
     in
-    result;
+    checked result;
 
   # Merge the tarballs of images built with buildImage into a single
   # tarball that contains all images. Running `docker load` on the resulting
@@ -776,12 +790,14 @@ rec {
   # contents. The main purpose is to be able to use nix commands in
   # the container.
   # Be careful since this doesn't work well with multilayer.
-  buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }: (
+  # TODO: add the dependencies of the config json.
+  buildImageWithNixDb = args@{ copyToRoot ? contents, contents ? null, extraCommands ? "", ... }: (
     buildImage (args // {
-      extraCommands = (mkDbExtraCommand contents) + extraCommands;
+      extraCommands = (mkDbExtraCommand copyToRoot) + extraCommands;
     })
   );
 
+  # TODO: add the dependencies of the config json.
   buildLayeredImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }: (
     buildLayeredImage (args // {
       extraCommands = (mkDbExtraCommand contents) + extraCommands;

pkgs/build-support/docker/examples.nix

@@ -24,7 +24,11 @@ rec {
   bash = buildImage {
     name = "bash";
     tag = "latest";
-    contents = pkgs.bashInteractive;
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      paths = [ pkgs.bashInteractive ];
+      pathsToLink = [ "/bin" ];
+    };
   };
 
   # 2. service example, layered on another image
@@ -36,7 +40,12 @@ rec {
     fromImage = bash;
     # fromImage = debian;
 
-    contents = pkgs.redis;
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      paths = [ pkgs.redis ];
+      pathsToLink = [ "/bin" ];
+    };
+
     runAsRoot = ''
       mkdir -p /data
     '';
@@ -118,13 +127,17 @@ rec {
   # 5. example of multiple contents, emacs and vi happily coexisting
   editors = buildImage {
     name = "editors";
-    contents = [
-      pkgs.coreutils
-      pkgs.bash
-      pkgs.emacs
-      pkgs.vim
-      pkgs.nano
-    ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [
+        pkgs.coreutils
+        pkgs.bash
+        pkgs.emacs
+        pkgs.vim
+        pkgs.nano
+      ];
+    };
   };
 
   # 6. nix example to play with the container nix store
@@ -132,13 +145,17 @@ rec {
   nix = buildImageWithNixDb {
     name = "nix";
     tag = "latest";
-    contents = [
-      # nix-store uses cat program to display results as specified by
-      # the image env variable NIX_PAGER.
-      pkgs.coreutils
-      pkgs.nix
-      pkgs.bash
-    ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [
+        # nix-store uses cat program to display results as specified by
+        # the image env variable NIX_PAGER.
+        pkgs.coreutils
+        pkgs.nix
+        pkgs.bash
+      ];
+    };
     config = {
       Env = [
         "NIX_PAGER=cat"
@@ -155,7 +172,11 @@ rec {
     name = "onTopOfPulledImage";
     tag = "latest";
     fromImage = nixFromDockerHub;
-    contents = [ pkgs.hello ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ pkgs.hello ];
+    };
   };
 
   # 8. regression test for erroneous use of eval and string expansion.
@@ -163,7 +184,11 @@ rec {
   runAsRootExtraCommands = pkgs.dockerTools.buildImage {
     name = "runAsRootExtraCommands";
     tag = "latest";
-    contents = [ pkgs.coreutils ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ pkgs.coreutils ];
+    };
     # The parens here are to create problematic bash to embed and eval. In case
     # this is *embedded* into the script (with nix expansion) the initial quotes
     # will close the string and the following parens are unexpected
@@ -176,7 +201,11 @@ rec {
   unstableDate = pkgs.dockerTools.buildImage {
     name = "unstable-date";
     tag = "latest";
-    contents = [ pkgs.coreutils ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ pkgs.coreutils ];
+    };
     created = "now";
   };
 
@@ -265,7 +294,11 @@ rec {
     name = "l3";
     fromImage = l2;
     tag = "latest";
-    contents = [ pkgs.coreutils ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ pkgs.coreutils ];
+    };
     extraCommands = ''
       mkdir -p tmp
       echo layer3 > tmp/layer3
@@ -290,7 +323,11 @@ rec {
     name = "child";
     fromImage = environmentVariablesParent;
     tag = "latest";
-    contents = [ pkgs.coreutils ];
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ pkgs.coreutils ];
+    };
     config = {
       Env = [
         "FROM_CHILD=true"
@@ -424,7 +461,11 @@ rec {
         name = "layers-unpack-order-${layerName}";
         tag = "latest";
         fromImage = parent;
-        contents = [ pkgs.coreutils ];
+        copyToRoot = pkgs.buildEnv {
+          name = "image-root";
+          pathsToLink = [ "/bin" ];
+          paths = [ pkgs.coreutils ];
+        };
         runAsRoot = ''
           #!${pkgs.runtimeShell}
           echo -n "${layerName}" >> /layer-order
@@ -441,7 +482,8 @@ rec {
   # buildImage without explicit tag
   bashNoTag = pkgs.dockerTools.buildImage {
     name = "bash-no-tag";
-    contents = pkgs.bashInteractive;
+    # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
+    copyToRoot = pkgs.bashInteractive;
   };
 
   # buildLayeredImage without explicit tag
@@ -501,7 +543,11 @@ rec {
   in crossPkgs.dockerTools.buildImage {
     name = "hello-cross";
     tag = "latest";
-    contents = crossPkgs.hello;
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ crossPkgs.hello ];
+    };
   };
 
   # layered image where a store path is itself a symlink
@@ -643,7 +689,8 @@ rec {
   build-image-with-path = buildImage {
     name = "build-image-with-path";
     tag = "latest";
-    contents = [ pkgs.bashInteractive ./test-dummy ];
+    # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
+    copyToRoot = [ pkgs.bashInteractive ./test-dummy ];
   };
 
   layered-image-with-path = pkgs.dockerTools.streamLayeredImage {