Revert "dockerTools.buildImage: Switch to the format image generated by Skopeo"

This reverts commit 35f205a4b6.

This does not use a standard format and by that breaks mesos

pkgs/build-support/docker/default.nix

@@ -137,7 +137,7 @@ rec {
         };
         inherit fromImage fromImageName fromImageTag;
 
-        buildInputs = [ utillinux e2fsprogs jshon rsync jq ];
+        buildInputs = [ utillinux e2fsprogs jshon rsync ];
       } ''
       rm -rf $out
 
@@ -146,29 +146,44 @@ rec {
       mount /dev/${vmTools.hd} disk
       cd disk
 
-      layers=""
       if [[ -n "$fromImage" ]]; then
         echo "Unpacking base image..."
         mkdir image
         tar -C image -xpf "$fromImage"
-        layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json)
+
+        # If the image name isn't set, read it from the image repository json.
+        if [[ -z "$fromImageName" ]]; then
+          fromImageName=$(jshon -k < image/repositories | head -n 1)
+          echo "From-image name wasn't set. Read $fromImageName."
         fi
 
-      # Unpack all of the layers into the image.
+        # If the tag isn't set, use the name as an index into the json
-      # Layer list is ordered starting from the base image
+        # and read the first key found.
-      lowerdir=""
+        if [[ -z "$fromImageTag" ]]; then
-      for layer in $layers; do
+          fromImageTag=$(jshon -e $fromImageName -k < image/repositories \
-        echo "Unpacking layer $layer"
+                         | head -n1)
-        layerDir=image/$(echo $layer | cut -d':' -f2)"_unpacked"
+          echo "From-image tag wasn't set. Read $fromImageTag."
-        mkdir -p $layerDir
+        fi
-        tar -C $layerDir -xpf image/$layer
-        chmod a+w image/$layer
-        rm image/$layer
 
-        find $layerDir -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
+        # Use the name and tag to get the parent ID field.
+        parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
+                   < image/repositories)
+      fi
+
+      # Unpack all of the parent layers into the image.
+      lowerdir=""
+      while [[ -n "$parentID" ]]; do
+        echo "Unpacking layer $parentID"
+        mkdir -p image/$parentID/layer
+        tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
+        rm image/$parentID/layer.tar
+
+        find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
 
         # Get the next lower directory and continue the loop.
-        lowerdir=$lowerdir''${lowerdir:+:}$layerDir
+        lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
+        parentID=$(cat image/$parentID/json \
+                  | (jshon -e parent -u 2>/dev/null || true))
       done
 
       mkdir work
@@ -446,17 +461,26 @@ rec {
 
         mkdir image
         touch baseFiles
-        layers=""
         if [[ -n "$fromImage" ]]; then
           echo "Unpacking base image..."
           tar -C image -xpf "$fromImage"
-          config=$(jq -r '.[0].Config' image/manifest.json)
+          # Do not import the base image configuration and manifest
-          layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json)
+          chmod a+w image image/*.json
-          for l in $layers; do
+          rm -f image/*.json
-            ls_tar image/$l >> baseFiles
+
+          if [[ -z "$fromImageName" ]]; then
+            fromImageName=$(jshon -k < image/repositories|head -n1)
+          fi
+          if [[ -z "$fromImageTag" ]]; then
+            fromImageTag=$(jshon -e $fromImageName -k \
+                           < image/repositories|head -n1)
+          fi
+          parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
+                     < image/repositories)
+
+          for l in image/*/layer.tar; do
+            ls_tar $l >> baseFiles
           done
-          chmod u+w image image/$config
-          rm image/$config
         fi
 
         chmod -R ug+rw image
@@ -483,28 +507,47 @@ rec {
         tar -rpf temp/layer.tar --mtime="@$SOURCE_DATE_EPOCH" \
           --owner=0 --group=0 --no-recursion --files-from newFiles
 
-        gzip temp/layer.tar
+        echo "Adding meta..."
-        layerID="sha256:$(sha256sum temp/layer.tar.gz | cut -d ' ' -f 1)"
-        mv temp/layer.tar.gz image/$layerID
 
-        echo "Generating image configuration and manifest..."
+        # If we have a parentID, add it to the json metadata.
+        if [[ -n "$parentID" ]]; then
+          cat temp/json | jshon -s "$parentID" -i parent > tmpjson
+          mv tmpjson temp/json
+        fi
+
+        # Take the sha256 sum of the generated json and use it as the layer ID.
+        # Compute the size and add it to the json under the 'Size' field.
+        layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
+        size=$(stat --printf="%s" temp/layer.tar)
+        cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
+        mv tmpjson temp/json
+
+        # Use the temp folder we've been working on to create a new image.
+        mv temp image/$layerID
+
+        # Create image json and image manifest
         imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
         manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
-
+        currentID=$layerID
-        # The layer list is ordered starting from the base image
+        while [[ -n "$currentID" ]]; do
-        layers=$(echo $layers $layerID)
+          layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
-        for i in $(echo $layers); do
           imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
-          diffId=$(gzip -dc image/$i | sha256sum | cut -d" " -f1)
+          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
-          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$diffId\"] + .")
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
-          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$i\"] + .")
+
+          currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
         done
 
         imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
-        echo "$imageJson" > "image/sha256:$imageJsonChecksum"
+        echo "$imageJson" > "image/$imageJsonChecksum.json"
-        manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"sha256:$imageJsonChecksum\"")
+        manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
         echo "$manifestJson" > image/manifest.json
 
+        # Store the json under the name image/repositories.
+        jshon -n object \
+          -n object -s "$layerID" -i "$imageTag" \
+          -i "$imageName" > image/repositories
+
         # Make the image read-only.
         chmod -R a-w image