ilyakooo0/nixpkgs
1
0
Fork 0
mirror of https://github.com/ilyakooo0/nixpkgs.git synced 2024-11-11 15:27:20 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #75592 from lovesegfault/ext4-fs-compression

Browse Source 
nixos: compress make-ext4-fs with zstd
This commit is contained in:
Samuel Dionne-Riel 2019-12-13 21:42:34 -05:00 committed by GitHub
commit f8ab1a9c17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 35 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
nixos/lib/make-ext4-fs.nix
View File

@ -4,8 +4,11 @@
# generated image is sized to only fit its contents, with the expectation
# that a script resizes the filesystem at boot time.
{ pkgs
, lib
# List of derivations to be included
, storePaths
# Whether or not to compress the resulting image with zstd
, compressImage ? false, zstd
# Shell commands to populate the ./files directory.
# All files in that directory are copied to the root of the FS.
, populateImageCommands ? ""
@ -20,18 +23,20 @@
let
sdClosureInfo = pkgs.buildPackages.closureInfo { rootPaths = storePaths; };
in
pkgs.stdenv.mkDerivation {
name = "ext4-fs.img";
name = "ext4-fs.img${lib.optionalString compressImage ".zst"}";
nativeBuildInputs = [e2fsprogs.bin libfaketime perl lkl];
nativeBuildInputs = [ e2fsprogs.bin libfaketime perl lkl ]
++ lib.optional compressImage zstd;
buildCommand =
''
${if compressImage then "img=temp.img" else "img=$out"}
(
mkdir -p ./files
${populateImageCommands}
)
# Add the closures of the top-level store objects.
storePaths=$(cat ${sdClosureInfo}/store-paths)
@ -42,28 +47,26 @@ pkgs.stdenv.mkDerivation {
bytes=$((2 * 4096 * $numInodes + 4096 * $numDataBlocks))
echo "Creating an EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks)"
truncate -s $bytes $out
faketime -f "1970-01-01 00:00:01" mkfs.ext4 -L ${volumeLabel} -U ${uuid} $out
truncate -s $bytes $img
faketime -f "1970-01-01 00:00:01" mkfs.ext4 -L ${volumeLabel} -U ${uuid} $img
# Also include a manifest of the closures in a format suitable for nix-store --load-db.
cp ${sdClosureInfo}/registration nix-path-registration
cptofs -t ext4 -i $out nix-path-registration /
cptofs -t ext4 -i $img nix-path-registration /
# Create nix/store before copying paths
faketime -f "1970-01-01 00:00:01" mkdir -p nix/store
cptofs -t ext4 -i $out nix /
cptofs -t ext4 -i $img nix /
echo "copying store paths to image..."
cptofs -t ext4 -i $out $storePaths /nix/store/
cptofs -t ext4 -i $img $storePaths /nix/store/
(
echo "copying files to image..."
cd ./files
cptofs -t ext4 -i $out ./* /
)
cptofs -t ext4 -i $img ./files/* /
# I have ended up with corrupted images sometimes, I suspect that happens when the build machine's disk gets full during the build.
if ! fsck.ext4 -n -f $out; then
if ! fsck.ext4 -n -f $img; then
echo "--- Fsck failed for EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks) ---"
cat errorlog
return 1
@ -71,9 +74,9 @@ pkgs.stdenv.mkDerivation {
(
# Resizes **snugly** to its actual limits (or closer to)
free=$(dumpe2fs $out | grep '^Free blocks:')
blocksize=$(dumpe2fs $out | grep '^Block size:')
blocks=$(dumpe2fs $out | grep '^Block count:')
free=$(dumpe2fs $img | grep '^Free blocks:')
blocksize=$(dumpe2fs $img | grep '^Block size:')
blocks=$(dumpe2fs $img | grep '^Block count:')
blocks=$((''${blocks##*:})) # format the number.
blocksize=$((''${blocksize##*:})) # format the number.
# System can't boot with 0 blocks free.
@ -82,10 +85,15 @@ pkgs.stdenv.mkDerivation {
size=$(( blocks - ''${free##*:} + fudge ))
echo "Resizing from $blocks blocks to $size blocks. (~ $((size*blocksize/1024/1024))MiB)"
EXT2FS_NO_MTAB_OK=yes resize2fs $out -f $size
EXT2FS_NO_MTAB_OK=yes resize2fs $img -f $size
)
# And a final fsck, because of the previous truncating.
fsck.ext4 -n -f $out
fsck.ext4 -n -f $img
if [ ${builtins.toString compressImage} ]; then
echo "Compressing image"
zstd -v --no-progress ./$img -o $out
fi
'';
}

13
nixos/modules/installer/cd-dvd/sd-image.nix
View File

@ -18,6 +18,7 @@ with lib;
let
rootfsImage = pkgs.callPackage ../../../lib/make-ext4-fs.nix ({
inherit (config.sdImage) storePaths;
compressImage = true;
populateImageCommands = config.sdImage.populateRootCommands;
volumeLabel = "NIXOS_SD";
} // optionalAttrs (config.sdImage.rootPartitionUUID != null) {
@ -128,10 +129,11 @@ in
sdImage.storePaths = [ config.system.build.toplevel ];
system.build.sdImage = pkgs.callPackage ({ stdenv, dosfstools, e2fsprogs, mtools, libfaketime, utillinux, bzip2 }: stdenv.mkDerivation {
system.build.sdImage = pkgs.callPackage ({ stdenv, dosfstools, e2fsprogs,
mtools, libfaketime, utillinux, bzip2, zstd }: stdenv.mkDerivation {
name = config.sdImage.imageName;
nativeBuildInputs = [ dosfstools e2fsprogs mtools libfaketime utillinux bzip2 ];
nativeBuildInputs = [ dosfstools e2fsprogs mtools libfaketime utillinux bzip2 zstd ];
inherit (config.sdImage) compressImage;
@ -146,11 +148,14 @@ in
echo "file sd-image $img" >> $out/nix-support/hydra-build-products
fi
echo "Decompressing rootfs image"
zstd -d --no-progress "${rootfsImage}" -o ./root-fs.img
# Gap in front of the first partition, in MiB
gap=8
# Create the image file sized to fit /boot/firmware and /, plus slack for the gap.
rootSizeBlocks=$(du -B 512 --apparent-size ${rootfsImage} | awk '{ print $1 }')
rootSizeBlocks=$(du -B 512 --apparent-size ./root-fs.img | awk '{ print $1 }')
firmwareSizeBlocks=$((${toString config.sdImage.firmwareSize} * 1024 * 1024 / 512))
imageSize=$((rootSizeBlocks * 512 + firmwareSizeBlocks * 512 + gap * 1024 * 1024))
truncate -s $imageSize $img
@ -168,7 +173,7 @@ in
# Copy the rootfs into the SD image
eval $(partx $img -o START,SECTORS --nr 2 --pairs)
dd conv=notrunc if=${rootfsImage} of=$img seek=$START count=$SECTORS
dd conv=notrunc if=./root-fs.img of=$img seek=$START count=$SECTORS
# Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
eval $(partx $img -o START,SECTORS --nr 1 --pairs)