Merge pull request #97145 from lheckemann/initrd-improvements

Initrd improvements

lib/systems/default.nix

@@ -83,7 +83,7 @@ rec {
         if final.isAarch32 then "arm"
         else if final.isAarch64 then "arm64"
         else if final.isx86_32 then "x86"
-        else if final.isx86_64 then "ia64"
+        else if final.isx86_64 then "x86"
         else if final.isMips then "mips"
         else final.parsed.cpu.name;
 

nixos/modules/system/boot/stage-1.nix

@@ -308,7 +308,7 @@ let
   # the initial RAM disk.
   initialRamdisk = pkgs.makeInitrd {
     name = "initrd-${kernel-name}";
-    inherit (config.boot.initrd) compressor prepend;
+    inherit (config.boot.initrd) compressor compressorArgs prepend;
 
     contents =
       [ { object = bootStage1;
@@ -334,7 +334,9 @@ let
 
   # Script to add secret files to the initrd at bootloader update time
   initialRamdiskSecretAppender =
-    pkgs.writeScriptBin "append-initrd-secrets"
+    let
+      compressorExe = initialRamdisk.compressorExecutableFunction pkgs;
+    in pkgs.writeScriptBin "append-initrd-secrets"
       ''
         #!${pkgs.bash}/bin/bash -e
         function usage {
@@ -376,7 +378,7 @@ let
          }
 
         (cd "$tmp" && find . -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null) | \
-          ${config.boot.initrd.compressor} >> "$1"
+          ${compressorExe} ${lib.escapeShellArgs initialRamdisk.compressorArgs} >> "$1"
       '';
 
 in
@@ -511,13 +513,28 @@ in
     };
 
     boot.initrd.compressor = mkOption {
-      internal = true;
-      default = "gzip -9n";
-      type = types.str;
-      description = "The compressor to use on the initrd image.";
+      default = "gzip";
+      type = types.unspecified; # We don't have a function type...
+      description = ''
+        The compressor to use on the initrd image. May be any of:
+
+        <itemizedlist>
+         <listitem><para>The name of one of the predefined compressors, see <filename>pkgs/build-support/kernel/initrd-compressor-meta.nix</filename> for the definitions.</para></listitem>
+         <listitem><para>A function which, given the nixpkgs package set, returns the path to a compressor tool, e.g. <literal>pkgs: "''${pkgs.pigz}/bin/pigz"</literal></para></listitem>
+         <listitem><para>(not recommended, because it does not work when cross-compiling) the full path to a compressor tool, e.g. <literal>"''${pkgs.pigz}/bin/pigz"</literal></para></listitem>
+        </itemizedlist>
+
+        The given program should read data from stdin and write it to stdout compressed.
+      '';
       example = "xz";
     };
 
+    boot.initrd.compressorArgs = mkOption {
+      default = null;
+      type = types.nullOr (types.listOf types.str);
+      description = "Arguments to pass to the compressor for the initrd image, or null to use the compressor's defaults.";
+    };
+
     boot.initrd.secrets = mkOption
       { default = {};
         type = types.attrsOf (types.nullOr types.path);

nixos/tests/all-tests.nix

@@ -168,6 +168,7 @@ in
   initrd-network-openvpn = handleTest ./initrd-network-openvpn {};
   initrd-network-ssh = handleTest ./initrd-network-ssh {};
   initrdNetwork = handleTest ./initrd-network.nix {};
+  initrd-secrets = handleTest ./initrd-secrets.nix {};
   installer = handleTest ./installer.nix {};
   iodine = handleTest ./iodine.nix {};
   ipfs = handleTest ./ipfs.nix {};

nixos/tests/initrd-secrets.nix

@@ -0,0 +1,35 @@
+{ system ? builtins.currentSystem
+, config ? {}
+, pkgs ? import ../.. { inherit system config; }
+, lib ? pkgs.lib
+, testing ? import ../lib/testing-python.nix { inherit system pkgs; }
+}:
+let
+  secretInStore = pkgs.writeText "topsecret" "iamasecret";
+  testWithCompressor = compressor: testing.makeTest {
+    name = "initrd-secrets-${compressor}";
+
+    meta.maintainers = [ lib.maintainers.lheckemann ];
+
+    machine = { ... }: {
+      virtualisation.useBootLoader = true;
+      boot.initrd.secrets."/test" = secretInStore;
+      boot.initrd.postMountCommands = ''
+        cp /test /mnt-root/secret-from-initramfs
+      '';
+      boot.initrd.compressor = compressor;
+      # zstd compression is only supported from 5.9 onwards. Remove when 5.10 becomes default.
+      boot.kernelPackages = pkgs.linuxPackages_latest;
+    };
+
+    testScript = ''
+      start_all()
+      machine.wait_for_unit("multi-user.target")
+      machine.succeed(
+          "cmp ${secretInStore} /secret-from-initramfs"
+      )
+    '';
+  };
+in lib.flip lib.genAttrs testWithCompressor [
+  "cat" "gzip" "bzip2" "xz" "lzma" "lzop" "pigz" "pixz" "zstd"
+]

pkgs/build-support/kernel/initrd-compressor-meta.nix

@@ -0,0 +1,53 @@
+rec {
+  cat = {
+    executable = pkgs: "cat";
+    ubootName = "none";
+    extension = ".cpio";
+  };
+  gzip = {
+    executable = pkgs: "${pkgs.gzip}/bin/gzip";
+    defaultArgs = ["-9n"];
+    ubootName = "gzip";
+    extension = ".gz";
+  };
+  bzip2 = {
+    executable = pkgs: "${pkgs.bzip2}/bin/bzip2";
+    ubootName = "bzip2";
+    extension = ".bz2";
+  };
+  xz = {
+    executable = pkgs: "${pkgs.xz}/bin/xz";
+    defaultArgs = ["--check=crc32" "--lzma2=dict=512KiB"];
+    extension = ".xz";
+  };
+  lzma = {
+    executable = pkgs: "${pkgs.xz}/bin/lzma";
+    defaultArgs = ["--check=crc32" "--lzma1=dict=512KiB"];
+    ubootName = "lzma";
+    extension = ".lzma";
+  };
+  lz4 = {
+    executable = pkgs: "${pkgs.lz4}/bin/lz4";
+    defaultArgs = ["-l"];
+    ubootName = "lz4";
+    extension = ".lz4";
+  };
+  lzop = {
+    executable = pkgs: "${pkgs.lzop}/bin/lzop";
+    ubootName = "lzo";
+    extension = ".lzo";
+  };
+  zstd = {
+    executable = pkgs: "${pkgs.zstd}/bin/zstd";
+    defaultArgs = ["-10"];
+    ubootName = "zstd";
+    extension = ".zst";
+  };
+  pigz = gzip // {
+    executable = pkgs: "${pkgs.pigz}/bin/pigz";
+  };
+  pixz = xz // {
+    executable = pkgs: "${pkgs.pixz}/bin/pixz";
+    defaultArgs = [];
+  };
+}

pkgs/build-support/kernel/make-initrd.nix

@@ -1,22 +1,74 @@
-# Create an initial ramdisk containing the closure of the specified
-# file system objects.  An initial ramdisk is used during the initial
+# Create an initramfs containing the closure of the specified
+# file system objects.  An initramfs is used during the initial
 # stages of booting a Linux system.  It is loaded by the boot loader
 # along with the kernel image.  It's supposed to contain everything
 # (such as kernel modules) necessary to allow us to mount the root
 # file system.  Once the root file system is mounted, the `real' boot
 # script can be called.
 #
-# An initrd is really just a gzipped cpio archive.
-#
-# Symlinks are created for each top-level file system object.  E.g.,
-# `contents = {object = ...; symlink = /init;}' is a typical
-# argument.
-
-{ stdenvNoCC, perl, cpio, contents, ubootTools
+# An initramfs is a cpio archive, and may be compressed with a number
+# of algorithms.
+let
+  # Some metadata on various compression programs, relevant to naming
+  # the initramfs file and, if applicable, generating a u-boot image
+  # from it.
+  compressors = import ./initrd-compressor-meta.nix;
+  # Get the basename of the actual compression program from the whole
+  # compression command, for the purpose of guessing the u-boot
+  # compression type and filename extension.
+  compressorName = fullCommand: builtins.elemAt (builtins.match "([^ ]*/)?([^ ]+).*" fullCommand) 1;
+in
+{ stdenvNoCC, perl, cpio, ubootTools, lib, pkgsBuildHost
+# Name of the derivation (not of the resulting file!)
 , name ? "initrd"
-, compressor ? "gzip -9n"
+
+# Program used to compress the cpio archive; use "cat" for no compression.
+# This can also be a function which takes a package set and returns the path to the compressor,
+# such as `pkgs: "${pkgs.lzop}/bin/lzop"`.
+, compressor ? "gzip"
+, _compressorFunction ?
+  if lib.isFunction compressor then compressor
+  else if ! builtins.hasContext compressor && builtins.hasAttr compressor compressors then compressors.${compressor}.executable
+  else _: compressor
+, _compressorExecutable ? _compressorFunction pkgsBuildHost
+, _compressorName ? compressorName _compressorExecutable
+, _compressorMeta ? compressors.${_compressorName} or {}
+
+# List of arguments to pass to the compressor program, or null to use its defaults
+, compressorArgs ? null
+, _compressorArgsReal ? if compressorArgs == null then _compressorMeta.defaultArgs or [] else compressorArgs
+
+# Filename extension to use for the compressed initramfs. This is
+# included for clarity, but $out/initrd will always be a symlink to
+# the final image.
+# If this isn't guessed, you may want to complete the metadata above and send a PR :)
+, extension ? _compressorMeta.extension or
+    (throw "Unrecognised compressor ${_compressorName}, please specify filename extension")
+
+# List of { object = path_or_derivation; symlink = "/path"; }
+# The paths are copied into the initramfs in their nix store path
+# form, then linked at the root according to `symlink`.
+, contents
+
+# List of uncompressed cpio files to prepend to the initramfs. This
+# can be used to add files in specified paths without them becoming
+# symlinks to store paths.
 , prepend ? []
-, lib
+
+# Whether to wrap the initramfs in a u-boot image.
+, makeUInitrd ? stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage"
+
+# If generating a u-boot image, the architecture to use. The default
+# guess may not align with u-boot's nomenclature correctly, so it can
+# be overridden.
+# See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L81-106 for a list.
+, uInitrdArch ? stdenvNoCC.hostPlatform.kernelArch
+
+# The name of the compression, as recognised by u-boot.
+# See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L195-204 for a list.
+# If this isn't guessed, you may want to complete the metadata above and send a PR :)
+, uInitrdCompression ? _compressorMeta.ubootName or
+    (throw "Unrecognised compressor ${_compressorName}, please specify uInitrdCompression")
 }:
 let
   # !!! Move this into a public lib function, it is probably useful for others
@@ -24,15 +76,26 @@ let
     lib.concatStringsSep "-" (filter (x: !(isList x)) (split "[^a-zA-Z0-9_=.?-]+" x));
 
 in stdenvNoCC.mkDerivation rec {
-  inherit name;
+  inherit name makeUInitrd extension uInitrdArch prepend;
+
+  ${if makeUInitrd then "uinitrdCompression" else null} = uInitrdCompression;
 
   builder = ./make-initrd.sh;
 
-  makeUInitrd = stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage";
-
   nativeBuildInputs = [ perl cpio ]
     ++ stdenvNoCC.lib.optional makeUInitrd ubootTools;
 
+  compress = "${_compressorExecutable} ${lib.escapeShellArgs _compressorArgsReal}";
+
+  # Pass the function through, for reuse in append-initrd-secrets. The
+  # function is used instead of the string, in order to support
+  # cross-compilation (append-initrd-secrets running on a different
+  # architecture than what the main initramfs is built on).
+  passthru = {
+    compressorExecutableFunction = _compressorFunction;
+    compressorArgs = _compressorArgsReal;
+  };
+
   # !!! should use XML.
   objects = map (x: x.object) contents;
   symlinks = map (x: x.symlink) contents;
@@ -47,6 +110,4 @@ in stdenvNoCC.mkDerivation rec {
       contents
       (lib.range 0 (lib.length contents - 1));
   pathsFromGraph = ./paths-from-graph.pl;
-
-  inherit compressor prepend;
 }

pkgs/build-support/kernel/make-initrd.sh

@@ -39,10 +39,13 @@ mkdir -p $out
 for PREP in $prepend; do
   cat $PREP >> $out/initrd
 done
-(cd root && find * -print0 | xargs -0r touch -h -d '@1')
-(cd root && find * -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | $compressor >> $out/initrd)
+(cd root && find * .[^.*] -exec touch -h -d '@1' '{}' +)
+(cd root && find * .[^.*] -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | eval -- $compress >> "$out/initrd")
 
 if [ -n "$makeUInitrd" ]; then
-    mv $out/initrd $out/initrd.gz
-    mkimage -A arm -O linux -T ramdisk -C gzip -d $out/initrd.gz $out/initrd
+    mkimage -A $uInitrdArch -O linux -T ramdisk -C "$uInitrdCompression" -d $out/initrd"$extension" $out/initrd.img
+    # Compatibility symlink
+    ln -s "initrd.img" "$out/initrd"
+else
+    ln -s "initrd" "$out/initrd$extension"
 fi