mirror of
https://github.com/nlewo/nix2container.git
synced 2024-09-11 16:36:39 +03:00
490 lines
20 KiB
Nix
490 lines
20 KiB
Nix
{ pkgs ? import <nixpkgs> { }, system }:
|
|
let
|
|
l = pkgs.lib // builtins;
|
|
|
|
nix2container-bin = pkgs.buildGoModule rec {
|
|
pname = "nix2container";
|
|
version = "1.0.0";
|
|
src = l.cleanSourceWith {
|
|
src = ./.;
|
|
filter = path: type:
|
|
let
|
|
p = baseNameOf path;
|
|
in !(
|
|
p == "flake.nix" ||
|
|
p == "flake.lock" ||
|
|
p == "examples" ||
|
|
p == "tests" ||
|
|
p == "README.md" ||
|
|
p == "default.nix"
|
|
);
|
|
};
|
|
vendorHash = "sha256-/j4ZHOwU5Xi8CE/fHha+2iZhsLd/y2ovzVhvg8HDV78=";
|
|
ldflags = pkgs.lib.optionals pkgs.stdenv.isDarwin [
|
|
"-X github.com/nlewo/nix2container/nix.useNixCaseHack=true"
|
|
];
|
|
};
|
|
|
|
skopeo-nix2container = pkgs.skopeo.overrideAttrs (old: {
|
|
EXTRA_LDFLAGS = pkgs.lib.optionalString pkgs.stdenv.isDarwin "-X github.com/nlewo/nix2container/nix.useNixCaseHack=true";
|
|
nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.patchutils ];
|
|
preBuild = let
|
|
patch = pkgs.fetchurl {
|
|
url = "https://github.com/nlewo/image/commit/c2254c998433cf02af60bf0292042bd80b96a77e.patch";
|
|
sha256 = "sha256-1Tj9D+ePjGL5u04aT7zr5rJw4vHAVrXAsr4owdooC/Y=";
|
|
|
|
};
|
|
in ''
|
|
mkdir -p vendor/github.com/nlewo/nix2container/
|
|
cp -r ${nix2container-bin.src}/* vendor/github.com/nlewo/nix2container/
|
|
cd vendor/github.com/containers/image/v5
|
|
mkdir nix/
|
|
touch nix/transport.go
|
|
# The patch for alltransports.go does not apply cleanly to skopeo > 1.14,
|
|
# filter the patch and insert the import manually here instead.
|
|
filterdiff -x '*/alltransports.go' ${patch} | patch -p1
|
|
sed -i '\#_ "github.com/containers/image/v5/tarball"#a _ "github.com/containers/image/v5/nix"' transports/alltransports/alltransports.go
|
|
cd -
|
|
'';
|
|
});
|
|
|
|
writeSkopeoApplication = name: text: pkgs.writeShellApplication {
|
|
inherit name text;
|
|
runtimeInputs = [ pkgs.jq skopeo-nix2container ];
|
|
excludeShellChecks = [ "SC2068" ];
|
|
};
|
|
|
|
copyToDockerDaemon = image: writeSkopeoApplication "copy-to-docker-daemon" ''
|
|
echo "Copy to Docker daemon image ${image.imageName}:${image.imageTag}"
|
|
skopeo --insecure-policy copy nix:${image} docker-daemon:${image.imageName}:${image.imageTag} $@
|
|
'';
|
|
|
|
copyToRegistry = image: writeSkopeoApplication "copy-to-registry" ''
|
|
echo "Copy to Docker registry image ${image.imageName}:${image.imageTag}"
|
|
skopeo --insecure-policy copy nix:${image} docker://${image.imageName}:${image.imageTag} $@
|
|
'';
|
|
|
|
copyTo = image: writeSkopeoApplication "copy-to" ''
|
|
echo Running skopeo --insecure-policy copy nix:${image} $@
|
|
skopeo --insecure-policy copy nix:${image} $@
|
|
'';
|
|
|
|
copyToPodman = image: writeSkopeoApplication "copy-to-podman" ''
|
|
echo "Copy to podman image ${image.imageName}:${image.imageTag}"
|
|
skopeo --insecure-policy copy nix:${image} containers-storage:${image.imageName}:${image.imageTag}
|
|
skopeo --insecure-policy inspect containers-storage:${image.imageName}:${image.imageTag}
|
|
'';
|
|
|
|
# Pull an image from a registry with Skopeo and translate it to a
|
|
# nix2container image.json file.
|
|
# This mainly comes from nixpkgs/build-support/docker/default.nix.
|
|
#
|
|
# Credentials:
|
|
# If you use the nix daemon for building, here is how you set up creds:
|
|
# docker login URL to whatever it is
|
|
# copy ~/.docker/config.json to /etc/nix/skopeo/auth.json
|
|
# Make the directory and all the files readable to the nixbld group
|
|
# sudo chmod -R g+rx /etc/nix/skopeo
|
|
# sudo chgrp -R nixbld /etc/nix/skopeo
|
|
# Now, bind mount the file into the nix build sandbox
|
|
# extra-sandbox-paths = /etc/skopeo/auth.json=/etc/nix/skopeo/auth.json
|
|
# update /etc/nix/skopeo/auth.json every time you add a new registry auth
|
|
pullImage =
|
|
let
|
|
fixName = name: l.replaceStrings [ "/" ":" ] [ "-" "-" ] name;
|
|
in
|
|
{ imageName
|
|
# To find the digest of an image, you can use skopeo:
|
|
# see doc/functions.xml
|
|
, imageDigest
|
|
, sha256
|
|
, os ? "linux"
|
|
, arch ? pkgs.go.GOARCH
|
|
, tlsVerify ? true
|
|
, name ? fixName "docker-image-${imageName}"
|
|
}: let
|
|
authFile = "/etc/skopeo/auth.json";
|
|
dir = pkgs.runCommand name
|
|
{
|
|
inherit imageDigest;
|
|
impureEnvVars = l.fetchers.proxyImpureEnvVars;
|
|
outputHashMode = "recursive";
|
|
outputHashAlgo = "sha256";
|
|
outputHash = sha256;
|
|
|
|
nativeBuildInputs = l.singleton pkgs.skopeo;
|
|
SSL_CERT_FILE = "${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt";
|
|
|
|
sourceURL = "docker://${imageName}@${imageDigest}";
|
|
} ''
|
|
skopeo \
|
|
--insecure-policy \
|
|
--tmpdir=$TMPDIR \
|
|
--override-os ${os} \
|
|
--override-arch ${arch} \
|
|
copy \
|
|
--src-tls-verify=${l.boolToString tlsVerify} \
|
|
$(
|
|
if test -f "${authFile}"
|
|
then
|
|
echo "--authfile=${authFile} $sourceURL"
|
|
else
|
|
echo "$sourceURL"
|
|
fi
|
|
) \
|
|
"dir://$out" \
|
|
| cat # pipe through cat to force-disable progress bar
|
|
'';
|
|
in pkgs.runCommand "nix2container-${imageName}.json" { } ''
|
|
${nix2container-bin}/bin/nix2container image-from-dir $out ${dir}
|
|
'';
|
|
|
|
pullImageFromManifest =
|
|
{ imageName
|
|
, imageManifest ? null
|
|
# The manifest dictates what is pulled; these three are only used for
|
|
# the supplied manifest-pulling script.
|
|
, imageTag ? "latest"
|
|
, os ? "linux"
|
|
, arch ? pkgs.go.GOARCH
|
|
, tlsVerify ? true
|
|
, registryUrl ? "registry.hub.docker.com"
|
|
, meta ? {}
|
|
}: let
|
|
manifest = l.fromJSON (l.readFile imageManifest);
|
|
|
|
buildImageBlob = digest:
|
|
let
|
|
blobUrl = "https://${registryUrl}/v2/${imageName}/blobs/${digest}";
|
|
plainDigest = l.replaceStrings ["sha256:"] [""] digest;
|
|
insecureFlag = l.strings.optionalString (!tlsVerify) "--insecure";
|
|
in pkgs.runCommand plainDigest {
|
|
outputHash = plainDigest;
|
|
outputHashMode = "flat";
|
|
outputHashAlgo = "sha256";
|
|
} ''
|
|
SSL_CERT_FILE="${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt";
|
|
|
|
# This initial access is expected to fail as we don't have a token.
|
|
${pkgs.curl}/bin/curl --location ${insecureFlag} "${blobUrl}" --head --silent --write-out '%header{www-authenticate}' --output /dev/null > bearer.txt
|
|
tokenUrl=$(sed -n 's/Bearer realm="\(.*\)",service="\(.*\)",scope="\(.*\)"/\1?service=\2\&scope=\3/p' bearer.txt)
|
|
|
|
declare -a auth_args
|
|
if [ -n "$tokenUrl" ]; then
|
|
echo "Token URL: $tokenUrl"
|
|
${pkgs.curl}/bin/curl --location ${insecureFlag} --fail --silent "$tokenUrl" --output token.json
|
|
token="$(${pkgs.jq}/bin/jq --raw-output .token token.json)"
|
|
auth_args=(-H "Authorization: Bearer $token")
|
|
else
|
|
echo "No token URL found, trying without authentication"
|
|
auth_args=()
|
|
fi
|
|
|
|
echo "Blob URL: ${blobUrl}"
|
|
${pkgs.curl}/bin/curl ${insecureFlag} --fail "''${auth_args[@]}" "${blobUrl}" --location --output $out
|
|
'';
|
|
|
|
# Pull the blobs (archives) for all layers, as well as the one for the image's config JSON.
|
|
layerBlobs = map (layerManifest: buildImageBlob layerManifest.digest) manifest.layers;
|
|
configBlob = buildImageBlob manifest.config.digest;
|
|
|
|
# Write the blob map out to a JSON file for the GO executable to consume.
|
|
blobMap = l.listToAttrs(map (drv: { name = drv.name; value = drv; }) (layerBlobs ++ [configBlob]));
|
|
blobMapFile = pkgs.writeText "${imageName}-blobs.json" (l.toJSON blobMap);
|
|
|
|
# Convenience scripts for manifest-updating.
|
|
filter = ''.manifests[] | select((.platform.os=="${os}") and (.platform.architecture=="${arch}")) | .digest'';
|
|
getManifest = writeSkopeoApplication "get-manifest" ''
|
|
set -e
|
|
manifest=$(skopeo inspect docker://${registryUrl}/${imageName}:${imageTag} --raw | jq)
|
|
if echo "$manifest" | jq -e .manifests >/dev/null; then
|
|
# Multi-arch image, pick the one that matches the supplied platform details.
|
|
hash=$(echo "$manifest" | jq -r '${filter}')
|
|
skopeo inspect "docker://${registryUrl}/${imageName}@$hash" --raw | jq
|
|
else
|
|
# Single-arch image, return the initial response.
|
|
echo "$manifest"
|
|
fi
|
|
'';
|
|
|
|
in pkgs.runCommand "nix2container-${imageName}.json" { passthru = { inherit getManifest; }; } ''
|
|
${nix2container-bin}/bin/nix2container image-from-manifest $out ${imageManifest} ${blobMapFile}
|
|
'';
|
|
|
|
buildLayer = {
|
|
# A list of store paths to include in the layer.
|
|
deps ? [],
|
|
# A derivation (or list of derivations) to include in the layer
|
|
# root directory. The store path prefix /nix/store/hash-path is
|
|
# removed. The store path content is then located at the layer /.
|
|
copyToRoot ? null,
|
|
# A store path to ignore. This is mainly useful to ignore the
|
|
# configuration file from the container layer.
|
|
ignore ? null,
|
|
# A list of layers built with the buildLayer function: if a store
|
|
# path in deps or copyToRoot belongs to one of these layers, this
|
|
# store path is skipped. This is pretty useful to
|
|
# isolate store paths that are often updated from more stable
|
|
# store paths, to speed up build and push time.
|
|
layers ? [],
|
|
# Store the layer tar in the derivation. This is useful when the
|
|
# layer dependencies are not bit reproducible.
|
|
reproducible ? true,
|
|
# A list of file permisssions which are set when the tar layer is
|
|
# created: these permissions are not written to the Nix store.
|
|
#
|
|
# Each element of this permission list is a dict such as
|
|
# { path = "a store path";
|
|
# regex = ".*";
|
|
# mode = "0664";
|
|
# }
|
|
# The mode is applied on a specific path. In this path subtree,
|
|
# the mode is then applied on all files matching the regex.
|
|
perms ? [],
|
|
# The maximun number of layer to create. This is based on the
|
|
# store path "popularity" as described in
|
|
# https://grahamc.com/blog/nix-and-layered-docker-images
|
|
maxLayers ? 1,
|
|
# Deprecated: will be removed on v1
|
|
contents ? null,
|
|
}: let
|
|
subcommand = if reproducible
|
|
then "layers-from-reproducible-storepaths"
|
|
else "layers-from-non-reproducible-storepaths";
|
|
copyToRootList =
|
|
let derivations = if !isNull contents then contents else copyToRoot;
|
|
in if isNull derivations
|
|
then []
|
|
else if !builtins.isList derivations
|
|
then [derivations]
|
|
else derivations;
|
|
# This is to move all storepaths in the copyToRoot attribute to the
|
|
# image root.
|
|
rewrites = l.map (p: {
|
|
path = p;
|
|
regex = "^${p}";
|
|
repl = "";
|
|
}) copyToRootList;
|
|
rewritesFile = pkgs.writeText "rewrites.json" (l.toJSON rewrites);
|
|
rewritesFlag = "--rewrites ${rewritesFile}";
|
|
permsFile = pkgs.writeText "perms.json" (l.toJSON perms);
|
|
permsFlag = l.optionalString (perms != []) "--perms ${permsFile}";
|
|
allDeps = deps ++ copyToRootList;
|
|
tarDirectory = l.optionalString (! reproducible) "--tar-directory $out";
|
|
layersJSON = pkgs.runCommand "layers.json" {} ''
|
|
mkdir $out
|
|
${nix2container-bin}/bin/nix2container ${subcommand} \
|
|
$out/layers.json \
|
|
${closureGraph allDeps ignore} \
|
|
--max-layers ${toString maxLayers} \
|
|
${rewritesFlag} \
|
|
${permsFlag} \
|
|
${tarDirectory} \
|
|
${l.concatMapStringsSep " " (l: l + "/layers.json") layers} \
|
|
'';
|
|
in checked { inherit copyToRoot contents; } layersJSON;
|
|
|
|
# Create a nix database from all paths contained in the given closureGraphJson.
|
|
# Also makes all these paths store roots to prevent them from being garbage collected.
|
|
makeNixDatabase = closureGraphJson:
|
|
assert l.isDerivation closureGraphJson;
|
|
pkgs.runCommand "nix-database" {}''
|
|
mkdir $out
|
|
echo "Generating the nix database from ${closureGraphJson}..."
|
|
export NIX_REMOTE=local?root=$PWD
|
|
# A user is required by nix
|
|
# https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478
|
|
export USER=nobody
|
|
export PATH=${pkgs.jq.bin}/bin:${pkgs.sqlite}/bin:"$PATH"
|
|
# Avoid including the closureGraph derivation itself.
|
|
# Transformation taken from https://github.com/NixOS/nixpkgs/blob/e7f49215422317c96445e0263f21e26e0180517e/pkgs/build-support/closure-info.nix#L33
|
|
jq -r 'map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' ${closureGraphJson} \
|
|
| head -n -1 \
|
|
| ${pkgs.nix}/bin/nix-store --load-db -j 1
|
|
|
|
# Sanitize time stamps
|
|
sqlite3 $PWD/nix/var/nix/db/db.sqlite \
|
|
'UPDATE ValidPaths SET registrationTime = 0;';
|
|
|
|
# Dump and reimport to ensure that the update order doesn't somehow change the DB.
|
|
sqlite3 $PWD/nix/var/nix/db/db.sqlite '.dump' > db.dump
|
|
mkdir -p $out/nix/var/nix/db/
|
|
sqlite3 $out/nix/var/nix/db/db.sqlite '.read db.dump'
|
|
mkdir -p $out/nix/store/.links
|
|
|
|
mkdir -p $out/nix/var/nix/gcroots/docker/
|
|
for i in $(jq -r 'map("\(.path)\n") | add' ${closureGraphJson}); do
|
|
ln -s $i $out/nix/var/nix/gcroots/docker/$(basename $i)
|
|
done;
|
|
'';
|
|
|
|
# Write the references of `path' to a file but do not include `ignore' itself if non-null.
|
|
closureGraph = paths: ignore:
|
|
let ignoreList =
|
|
if ignore == null
|
|
then []
|
|
else if !(builtins.isList ignore)
|
|
then [ignore]
|
|
else ignore;
|
|
in pkgs.runCommand "closure-graph.json"
|
|
{
|
|
exportReferencesGraph.graph = paths;
|
|
__structuredAttrs = true;
|
|
PATH = "${pkgs.jq}/bin";
|
|
ignoreListJson = builtins.toJSON (builtins.map builtins.toString ignoreList);
|
|
outputChecks.out = {
|
|
disallowedReferences = ignoreList;
|
|
};
|
|
builder = l.toFile "builder"
|
|
''
|
|
. .attrs.sh
|
|
jq --argjson ignore "$ignoreListJson" \
|
|
'.graph|map(select(.path as $p | $ignore | index($p) | not))|map(.references|=sort_by(.))|sort_by(.path)' \
|
|
.attrs.json \
|
|
> ''${outputs[out]}
|
|
'';
|
|
}
|
|
"";
|
|
|
|
buildImage = {
|
|
name,
|
|
# Image tag, when null then the nix output hash will be used.
|
|
tag ? null,
|
|
# An attribute set describing an image configuration as defined in
|
|
# https://github.com/opencontainers/image-spec/blob/8b9d41f48198a7d6d0a5c1a12dc2d1f7f47fc97f/specs-go/v1/config.go#L23
|
|
config ? {},
|
|
# A list of layers built with the buildLayer function: if a store
|
|
# path in deps or copyToRoot belongs to one of these layers, this
|
|
# store path is skipped. This is pretty useful to
|
|
# isolate store paths that are often updated from more stable
|
|
# store paths, to speed up build and push time.
|
|
layers ? [],
|
|
# A derivation (or list of derivation) to include in the layer
|
|
# root. The store path prefix /nix/store/hash-path is removed. The
|
|
# store path content is then located at the image /.
|
|
copyToRoot ? null,
|
|
# An image that is used as base image of this image.
|
|
fromImage ? "",
|
|
# A list of file permisssions which are set when the tar layer is
|
|
# created: these permissions are not written to the Nix store.
|
|
#
|
|
# Each element of this permission list is a dict such as
|
|
# { path = "a store path";
|
|
# regex = ".*";
|
|
# mode = "0664";
|
|
# }
|
|
# The mode is applied on a specific path. In this path subtree,
|
|
# the mode is then applied on all files matching the regex.
|
|
perms ? [],
|
|
# The maximun number of layer to create. This is based on the
|
|
# store path "popularity" as described in
|
|
# https://grahamc.com/blog/nix-and-layered-docker-images
|
|
# Note this is applied on the image layers and not on layers added
|
|
# with the buildImage.layers attribute
|
|
maxLayers ? 1,
|
|
# If set to true, the Nix database is initialized with all store
|
|
# paths added into the image. Note this is only useful to run nix
|
|
# commands from the image, for instance to build an image used by
|
|
# a CI to run Nix builds.
|
|
initializeNixDatabase ? false,
|
|
# If initializeNixDatabase is set to true, the uid/gid of /nix can be
|
|
# controlled using nixUid/nixGid.
|
|
nixUid ? 0,
|
|
nixGid ? 0,
|
|
# Time of creation of the image.
|
|
created ? "0001-01-01T00:00:00Z",
|
|
# Deprecated: will be removed
|
|
contents ? null,
|
|
meta ? {},
|
|
}:
|
|
let
|
|
configFile = pkgs.writeText "config.json" (l.toJSON config);
|
|
copyToRootList =
|
|
let derivations = if !isNull contents then contents else copyToRoot;
|
|
in if isNull derivations
|
|
then []
|
|
else if !builtins.isList derivations
|
|
then [derivations]
|
|
else derivations;
|
|
|
|
# Expand the given list of layers to include all their transitive layer dependencies.
|
|
layersWithNested = layers:
|
|
let layerWithNested = layer: [layer] ++ (builtins.concatMap layerWithNested (layer.layers or []));
|
|
in builtins.concatMap layerWithNested layers;
|
|
explodedLayers = layersWithNested layers;
|
|
ignore = [configFile]++explodedLayers;
|
|
|
|
closureGraphForAllLayers = closureGraph ([configFile] ++ copyToRootList ++ layers) ignore;
|
|
nixDatabase = makeNixDatabase closureGraphForAllLayers;
|
|
# This layer contains all config dependencies. We ignore the
|
|
# configFile because it is already part of the image, as a
|
|
# specific blob.
|
|
|
|
perms' = perms ++ l.optionals initializeNixDatabase
|
|
[
|
|
{
|
|
path = nixDatabase;
|
|
regex = ".*";
|
|
mode = "0755";
|
|
uid = nixUid;
|
|
gid = nixGid;
|
|
}
|
|
];
|
|
|
|
customizationLayer = buildLayer {
|
|
inherit maxLayers;
|
|
perms = perms';
|
|
copyToRoot = if initializeNixDatabase
|
|
then copyToRootList ++ [nixDatabase]
|
|
else copyToRootList;
|
|
deps = [configFile];
|
|
ignore = configFile;
|
|
layers = layers;
|
|
};
|
|
fromImageFlag = l.optionalString (fromImage != "") "--from-image ${fromImage}";
|
|
createdFlag = "--created ${created}";
|
|
layerPaths = l.concatMapStringsSep " " (l: l + "/layers.json") (layers ++ [customizationLayer]);
|
|
image = let
|
|
imageName = l.toLower name;
|
|
imageTag =
|
|
if tag != null
|
|
then tag
|
|
else
|
|
l.head (l.strings.splitString "-" (baseNameOf image.outPath));
|
|
in pkgs.runCommand "image-${baseNameOf name}.json"
|
|
{
|
|
inherit imageName meta;
|
|
passthru = {
|
|
inherit fromImage imageTag;
|
|
# provide a cheap to evaluate image reference for use with external tools like docker
|
|
# DO NOT use as an input to other derivations, as there is no guarantee that the image
|
|
# reference will exist in the store.
|
|
imageRefUnsafe = builtins.unsafeDiscardStringContext "${imageName}:${imageTag}";
|
|
copyToDockerDaemon = copyToDockerDaemon image;
|
|
copyToRegistry = copyToRegistry image;
|
|
copyToPodman = copyToPodman image;
|
|
copyTo = copyTo image;
|
|
};
|
|
}
|
|
''
|
|
${nix2container-bin}/bin/nix2container image \
|
|
$out \
|
|
${fromImageFlag} \
|
|
${createdFlag} \
|
|
${configFile} \
|
|
${layerPaths}
|
|
'';
|
|
in checked { inherit copyToRoot contents; } image;
|
|
|
|
checked = { copyToRoot, contents }:
|
|
pkgs.lib.warnIf (contents != null)
|
|
"The contents parameter is deprecated. Change to copyToRoot if the contents are designed to be copied to the root filesystem, such as when you use `buildEnv` or similar between contents and your packages. Use copyToRoot = buildEnv { ... }; or similar if you intend to add packages to /bin."
|
|
pkgs.lib.throwIf (contents != null && copyToRoot != null)
|
|
"You can not specify both contents and copyToRoot."
|
|
;
|
|
in
|
|
{
|
|
inherit nix2container-bin skopeo-nix2container;
|
|
nix2container = { inherit buildImage buildLayer pullImage pullImageFromManifest; };
|
|
}
|