docker tools: fix pull image function

This commit is contained in:
Matej Cotman 2017-05-26 18:06:24 +02:00
parent e284740dd5
commit 8d3cb0af9e
3 changed files with 59 additions and 122 deletions

View File

@ -83,16 +83,12 @@ rec {
};
# 4. example of pulling an image. could be used as a base for other images
#
# ***** Currently broken, getting 404s. Perhaps the docker API has changed?
#
#
# debian = pullImage {
# imageName = "debian";
# imageTag = "jessie";
# # this hash will need change if the tag is updated at docker hub
# sha256 = "18kd495lc2k35h03bpcbdjnix17nlqbwf6nmq3sb161blf0dk14q";
# };
nix = pullImage {
imageName = "nixos/nix";
imageTag = "1.11";
# this hash will need change if the tag is updated at docker hub
sha256 = "1gk4bq05vl3rj3mh4mlbl4iicgndmimlv8jvkhdk4hrv0r44bwr3";
};
# 5. example of multiple contents, emacs and vi happily coexisting
editors = buildImage {

View File

@ -1,41 +1,32 @@
{ stdenv, lib, curl, jshon, python, runCommand }:
# Inspired and simplified version of fetchurl.
{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
let
nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
in
# For simplicity we only support sha256.
{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
let
pullImage = vmTools.runInLinuxVM (
stdenv.mkDerivation {
inherit name imageId;
# Currently only registry v1 is supported, compatible with Docker Hub.
certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
{ imageName, imageTag ? "latest", imageId ? null
, sha256, name ? "${imageName}-${imageTag}"
, indexUrl ? "https://index.docker.io"
, registryVersion ? "v1"
, curlOpts ? "" }:
builder = ./pull.sh;
assert registryVersion == "v1";
buildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
let layer = stdenv.mkDerivation {
inherit name imageName imageTag imageId
indexUrl registryVersion curlOpts;
outputHashAlgo = "sha256";
outputHash = sha256;
builder = ./pull.sh;
detjson = ./detjson.py;
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
buildInputs = [ curl jshon python ];
preVM = vmTools.createEmptyImage {
size = 2048;
fullName = "${name}-disk";
};
outputHashAlgo = "sha256";
outputHash = sha256;
outputHashMode = "recursive";
impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
# This variable allows the user to pass additional options to curl
"NIX_CURL_FLAGS"
];
# Doing the download on a remote machine just duplicates network
# traffic, so don't do that.
preferLocalBuild = true;
};
in runCommand "${name}.tar.gz" {} ''
tar -C ${layer} -czf $out .
''
QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
});
in
pullImage

View File

@ -1,86 +1,36 @@
# Reference: docker src contrib/download-frozen-image.sh
source $stdenv/setup
# Curl flags to handle redirects, not use EPSV, handle cookies for
# servers to need them during redirects, and work on SSL without a
# certificate (this isn't a security problem because we check the
# cryptographic hash of the output anyway).
curl=$(command -v curl)
curl() {
[[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@"
$curl \
--location --max-redirs 20 \
--retry 3 \
--fail \
--disable-epsv \
--cookie-jar cookies \
--insecure \
$curlOpts \
$NIX_CURL_FLAGS \
"$@"
}
mkdir -p /var/lib/docker
mkfs.ext4 /dev/vda
mount -t ext4 /dev/vda /var/lib/docker
fetchLayer() {
local url="$1"
local dest="$2"
local curlexit=18;
modprobe virtio_net
dhclient eth0
# if we get error code 18, resume partial download
while [ $curlexit -eq 18 ]; do
# keep this inside an if statement, since on failure it doesn't abort the script
if curl -C - "$url" --output "$dest"; then
return 0
else
curlexit=$?;
fi
done
mkdir -p /etc/ssl/certs/
cp "$certs" "/etc/ssl/certs/"
return $curlexit
}
headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \
"$indexUrl/$registryVersion/repositories/$imageName/images")
header() {
grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2
}
# this only takes the first endpoint, more may be provided
# https://docs.docker.com/v1.6/reference/api/docker-io_api/
if ! registryUrl=$(header X-Docker-Endpoints); then
echo "error: index returned no endpoint"
exit 1
fi
baseUrl="https://$registryUrl/$registryVersion"
token="$(header X-Docker-Token || true)";
if [ -z "$imageId" ]; then
imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")"
imageId="${imageId//\"/}"
if [ -z "$imageId" ]; then
echo "error: no image ID found for ${imageName}:${imageTag}"
exit 1
# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
cd /sys/fs/cgroup
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
mkdir -p $sys
if ! mountpoint -q $sys; then
if ! mount -n -t cgroup -o $sys cgroup $sys; then
rmdir $sys || true
fi
echo "found image ${imageName}:${imageTag}@$imageId"
fi
mkdir -p $out
jshon -n object \
-n object -s "$imageId" -i "$imageTag" \
-i "$imageName" > $out/repositories
curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json
layerIds=$(jshon -a -u < ancestry.json)
for layerId in $layerIds; do
echo "fetching layer $layerId"
mkdir "$out/$layerId"
echo '1.0' > "$out/$layerId/VERSION"
curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
fi
done
# run docker daemon
dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
until $(curl --output /dev/null --silent --connect-timeout 2 http://127.0.0.1:5555); do
printf '.'
sleep 1
done
rm -r $out
docker pull ${imageId}
docker save ${imageId} > $out