Pull image layers from a supplied manifest.

cmd/image.go

@@ -58,6 +58,36 @@ func imageFromDir(outputFilename, directory string) error {
 	return nil
 }
 
+var imageFromManifestCmd = &cobra.Command{
+	Use:   "image-from-manifest OUTPUT-FILENAME MANIFEST.JSON BLOBS.JSON",
+	Short: "Write an image.json file to OUTPUT-FILENAME from a skopeo raw manifest and blobs JSON.",
+	Args:  cobra.MinimumNArgs(3),
+	Run: func(cmd *cobra.Command, args []string) {
+		err := imageFromManifest(args[0], args[1], args[2])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func imageFromManifest(outputFilename, manifestFilename string, blobsFilename string) error {
+	image, err := nix.NewImageFromManifest(manifestFilename, blobsFilename)
+	if err != nil {
+		return err
+	}
+	res, err := json.MarshalIndent(image, "", "\t")
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile(outputFilename, []byte(res), 0666)
+	if err != nil {
+		return err
+	}
+	logrus.Infof("Image has been written to %s", outputFilename)
+	return nil
+}
+
 func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string) error {
 	var imageConfig v1.ImageConfig
 	var image types.Image
@@ -116,4 +146,5 @@ func init() {
 	rootCmd.AddCommand(imageCmd)
 	imageCmd.Flags().StringVarP(&fromImageFilename, "from-image", "", "", "A JSON file describing the base image")
 	rootCmd.AddCommand(imageFromDirCmd)
+	rootCmd.AddCommand(imageFromManifestCmd)
 }

default.nix

@@ -129,6 +129,49 @@ let
       ${nix2container-bin}/bin/nix2container image-from-dir $out ${dir}
     '';
 
+  pullImageByManifest =
+    { imagePath
+    , imageManifest
+    , tlsVerify ? true
+    , registryApiUrl ? "registry.hub.docker.com/v2"
+    }: let
+      manifest = l.fromJSON (l.readFile imageManifest);
+
+      buildImageBlob = digest:
+        let
+          blobUrl = "https://${registryApiUrl}/${imagePath}/blobs/${digest}";
+          plainDigest = l.replaceStrings ["sha256:"] [""] digest;
+          insecureFlag = l.strings.optionalString (!tlsVerify) "--insecure";
+        in (pkgs.runCommand plainDigest {} ''
+          SSL_CERT_FILE="${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt";
+
+          # This initial access is expected to fail as we don't have a token.
+          ${pkgs.curl}/bin/curl ${insecureFlag} "${blobUrl}" --head --silent --write-out '%header{www-authenticate}' --output /dev/null > bearer.txt
+          tokenUrl=$(sed -n 's/Bearer realm="\(.*\)",service="\(.*\)",scope="\(.*\)"/\1?service=\2\&scope=\3/p' bearer.txt)
+
+          echo "Token URL: $tokenUrl"
+          ${pkgs.curl}/bin/curl ${insecureFlag} --fail --silent "$tokenUrl" --output token.json
+          token="$(${pkgs.jq}/bin/jq --raw-output .token token.json)"
+
+          echo "Blob URL: ${blobUrl}"
+          ${pkgs.curl}/bin/curl ${insecureFlag} --fail -H "Authorization: Bearer $token" "${blobUrl}" --location --output $out
+        '').overrideAttrs(_: {
+          outputHash = plainDigest;
+          outputHashMode = "flat";
+          outputHashAlgo = "sha256";
+        });
+
+      # Pull the blobs (archives) for all layers, as well as the one for the image's config JSON.
+      layerBlobs = map (layerManifest: buildImageBlob layerManifest.digest) manifest.layers;
+      configBlob = buildImageBlob manifest.config.digest;
+
+      # Write the blob map out to a JSON file for the GO executable to consume.
+      blobMap = l.listToAttrs(map (drv: { name = drv.name; value = drv; }) (layerBlobs ++ [configBlob]));
+      blobMapFile = pkgs.writeText "${imagePath}-blobs.json" (l.toJSON blobMap);
+    in pkgs.runCommand "nix2container-${imagePath}.json" { } ''
+      ${nix2container-bin}/bin/nix2container image-from-manifest $out ${imageManifest} ${blobMapFile}
+    '';
+
   buildLayer = {
     # A list of store paths to include in the layer.
     deps ? [],
@@ -359,5 +402,5 @@ let
 in
 {
   inherit nix2container-bin skopeo-nix2container;
-  nix2container = { inherit buildImage buildLayer pullImage; };
+  nix2container = { inherit buildImage buildLayer pullImage pullImageByManifest; };
 }

examples/alpine-manifest.json

@@ -0,0 +1,16 @@
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+  "config": {
+    "mediaType": "application/vnd.docker.container.image.v1+json",
+    "size": 1472,
+    "digest": "sha256:5e2b554c1c45d22c9d1aa836828828e320a26011b76c08631ac896cbc3625e3e"
+  },
+  "layers": [
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 3397490,
+      "digest": "sha256:8a49fdb3b6a5ff2bd8ec6a86c05b2922a0f7454579ecc07637e94dfd1d0639b6"
+    }
+  ]
+}

examples/default.nix

@@ -1,10 +1,11 @@
-{ pkgs, nix2container }: {
+{ pkgs, nix2container, skopeo-nix2container }: {
   hello = pkgs.callPackage ./hello.nix { inherit nix2container; };
   nginx = pkgs.callPackage ./nginx.nix { inherit nix2container; };
   bash = pkgs.callPackage ./bash.nix { inherit nix2container; };
   basic = pkgs.callPackage ./basic.nix { inherit nix2container; };
   nonReproducible = pkgs.callPackage ./non-reproducible.nix { inherit nix2container; };
   fromImage = pkgs.callPackage ./from-image.nix { inherit nix2container; };
+  fromImageManifest = pkgs.callPackage ./from-image-manifest.nix { inherit nix2container; };
   uwsgi = pkgs.callPackage ./uwsgi { inherit nix2container; };
   openbar = pkgs.callPackage ./openbar.nix { inherit nix2container; };
   layered = pkgs.callPackage ./layered.nix { inherit nix2container; };
@@ -12,4 +13,15 @@
   nix = pkgs.callPackage ./nix.nix { inherit nix2container; };
   nix-user = pkgs.callPackage ./nix-user.nix { inherit nix2container; };
   ownership = pkgs.callPackage ./ownership.nix { inherit nix2container; };
+
+  update-manifests = let
+    image = "library/alpine";
+    skopeo = "${skopeo-nix2container}/bin/skopeo";
+    jq = "${pkgs.jq}/bin/jq";
+    filter = ''.manifests[] | select((.platform.os=="linux") and (.platform.architecture=="amd64")) | .digest'';
+  in pkgs.writeShellScriptBin "update-manifests" ''
+    set -e
+    hash=$(${skopeo} inspect docker://${image} --raw | ${jq} -r '${filter}')
+    ${skopeo} inspect docker://${image}@$hash --raw | ${jq} > examples/alpine-manifest.json
+  '';
 }

examples/from-image-manifest.nix

@@ -0,0 +1,14 @@
+{ pkgs, nix2container }: let
+  alpine = nix2container.pullImageByManifest {
+    imagePath = "library/alpine";
+    # nix run .#examples.update-manifests to update this to the latest.
+    imageManifest = ./alpine-manifest.json;
+  };
+in
+nix2container.buildImage {
+  name = "from-image-manifest";
+  fromImage = alpine;
+  config = {
+    entrypoint = [ "${pkgs.coreutils}/bin/ls" "-l" "/etc/alpine-release"];
+  };
+}

flake.nix

@@ -13,7 +13,7 @@
         };
         examples = import ./examples {
           inherit pkgs;
-          inherit (nix2container) nix2container;
+          inherit (nix2container) nix2container skopeo-nix2container;
         };
         tests = import ./tests {
           inherit pkgs examples;

nix/image.go

@@ -96,7 +96,7 @@ func getV1Image(image types.Image) (imageV1 v1.Image, err error) {
 	return
 }
 
-// NewImageFromDir creates an Image from a JSON file describing an
+// NewImageFromFile creates an Image from a JSON file describing an
 // image. This file has usually been created by Nix through the
 // nix2container binary.
 func NewImageFromFile(filename string) (image types.Image, err error) {
@@ -177,6 +177,70 @@ func NewImageFromDir(directory string) (image types.Image, err error) {
 	return image, nil
 }
 
+// NewImageFromManifest builds an Image based on a registry manifest
+// and a separate JSON mapping pointing to the locations of the
+// associated blobs (layer archives).
+func NewImageFromManifest(manifestFilename string, blobMapFilename string) (image types.Image, err error) {
+	image.Version = types.ImageVersion
+
+	content, err := os.ReadFile(manifestFilename)
+	if err != nil {
+		return image, err
+	}
+	var v1Manifest v1.Manifest
+	err = json.Unmarshal(content, &v1Manifest)
+	if err != nil {
+		return image, err
+	}
+
+	var blobMap map[string]string
+	content, err = os.ReadFile(blobMapFilename)
+	if err != nil {
+		return image, err
+	}
+	err = json.Unmarshal(content, &blobMap)
+	if err != nil {
+		return image, err
+	}
+
+	var configFilename = blobMap[v1Manifest.Config.Digest.Encoded()]
+	content, err = os.ReadFile(configFilename)
+	if err != nil {
+		return image, err
+	}
+	var v1ImageConfig manifest.Schema2Image
+	err = json.Unmarshal(content, &v1ImageConfig)
+	if err != nil {
+		return image, err
+	}
+
+	for i, l := range v1Manifest.Layers {
+		layerFilename := blobMap[l.Digest.Encoded()]
+		logrus.Infof("Adding tar file '%s' as image layer", layerFilename)
+		layer := types.Layer{
+			LayerPath: layerFilename,
+			Digest:    l.Digest.String(),
+			DiffIDs:   v1ImageConfig.RootFS.DiffIDs[i].String(),
+		}
+		switch l.MediaType {
+		case "application/vnd.docker.image.rootfs.diff.tar":
+			layer.MediaType = v1.MediaTypeImageLayer
+		case "application/vnd.docker.image.rootfs.diff.tar.gzip":
+			layer.MediaType = v1.MediaTypeImageLayerGzip
+		case "application/vnd.oci.image.layer.v1.tar":
+			layer.MediaType = l.MediaType
+		case "application/vnd.oci.image.layer.v1.tar+gzip":
+			layer.MediaType = l.MediaType
+		case "application/vnd.oci.image.layer.v1.tar+zstd":
+			layer.MediaType = l.MediaType
+		default:
+			return image, fmt.Errorf("Unsupported media type: %q", l.MediaType)
+		}
+		image.Layers = append(image.Layers, layer)
+	}
+	return image, nil
+}
+
 type nopCloser struct {
 	io.Reader
 }

tests/default.nix

@@ -54,6 +54,10 @@ let
       image = examples.fromImage;
       pattern = "/etc/alpine-release$";
     };
+    fromImageManifest = testScript {
+      image = examples.fromImageManifest;
+      pattern = "/etc/alpine-release$";
+    };
     layered = testScript {
       image = examples.layered;
       pattern = "Hello, world";