NixOS/mobile-nixos
1
1
Fork 0
mirror of https://github.com/NixOS/mobile-nixos.git synced 2024-12-11 09:04:01 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #622 from samueldr-wip/feature/update-image-builder

Browse Source 
Update image builder with a modules system based approach
This commit is contained in:
Samuel Dionne-Riel 2023-09-15 14:48:59 -04:00 committed by GitHub
commit f0c5329991
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 2024 additions and 1913 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/hello/configuration.nix
View File

@ -52,7 +52,7 @@ in
mobile.generatedFilesystems = {
rootfs = lib.mkDefault {
label = lib.mkForce "MOBILE_HELLO";
id = lib.mkForce "12345678-1324-1234-0000-D00D00000001";
ext4.partitionID = lib.mkForce "12345678-1324-1234-0000-D00D00000001";
};
};

2
examples/installer/modules/filesystems.nix
View File

@ -9,7 +9,7 @@ in
mobile.generatedFilesystems = {
rootfs = mkDefault {
label = mkForce "MOBILE_INSTALLER";
id = mkForce "12345678-9000-0001-0000-D00D00000001";
ext4.partitionID = mkForce "12345678-9000-0001-0000-D00D00000001";
};
};

47
lib/image-builder/default.nix
View File

@ -1,47 +0,0 @@
{ lib, newScope }:
let
inherit (lib) makeScope;
in
makeScope newScope (self:
let
inherit (self) callPackage;
in
# Note: Prefer using `self.something.deep` rather than making `something` a
# recursive set. Otherwise it won't override as expected.
{
makeFilesystem = callPackage ./makeFilesystem.nix {};
# All known supported filesystems for image generation.
# Use stand-alone (outside of a disk image) is supported.
fileSystem = {
makeExt4 = callPackage ./makeExt4.nix {};
makeBtrfs = callPackage ./makeBtrfs.nix {};
makeFAT32 = callPackage ./makeFAT32.nix {};
# Specialization of `makeFAT32` with (1) filesystemType showing as ESP,
# and (2) the name defaults to ESP.
makeESP = args: self.fileSystem.makeFAT32 ({ name = "ESP"; filesystemType = "ESP"; } // args);
};
gap = length: {
inherit length;
isGap = true;
};
# All supported disk formats for image generation.
diskImage = {
makeMBR = callPackage ./makeMBR.nix {};
makeGPT = callPackage ./makeGPT.nix {};
};
# Don't do maths yourselves, just use the helpers.
# Yes, this is the bibytes family of units.
# (This is fine as rec; it won't be overriden.)
size = rec {
TiB = x: 1024 * (GiB x);
GiB = x: 1024 * (MiB x);
MiB = x: 1024 * (KiB x);
KiB = x: 1024 * x;
};
}
)

19
lib/image-builder/in-depth-tests/README.md
View File

@ -1,19 +0,0 @@
In-depth tests
==============
Those are rather bulky integration-type tests. They are not ran by default, but
should be used to better test the image builder infrastructure.
Changes to the infra should be passed through this test suite in addition to the
slimmer usual tests suite.
> **Tip:**
>
> From the image-builder infra directory, run the following.
>
> ```
> nix-build in-depth-tests/[...].nix -I nixpkgs-overlays=$PWD/lib/tests/test-overlay.nix
> ```
>
> This allows you to track the bigger builds more easily.

96
lib/image-builder/in-depth-tests/raspi-like.nix
View File

@ -1,96 +0,0 @@
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder ubootTools;
configTxt = pkgs.writeText "config.txt" ''
kernel=u-boot-rpi3.bin
# Boot in 64-bit mode.
arm_control=0x200
# Prevent the firmware from smashing the framebuffer setup done by the mainline kernel
# when attempting to show low-voltage or overtemperature warnings.
avoid_warnings=1
'';
scrTxt = pkgs.writeText "uboot.scr.txt" ''
echo
echo
echo
echo
echo " **"
echo " ** Image Builder sanity checks!"
echo " ** This will appear to freeze since the kernel is not built with the VC4 kernel built-in."
echo " ** Stay assured, the kernel should be panicking anyway since there is no initrd, no init, and no useful FS."
echo " **"
echo
load $devtype $devnum:$distro_bootpart $kernel_addr_r boot/kernel
booti $kernel_addr_r
'';
scr = pkgs.runCommand "uboot-script" {} ''
mkdir -p $out
${ubootTools}/bin/mkimage \
-A arm64 \
-O linux \
-T script \
-C none \
-n ${scrTxt} -d ${scrTxt} \
$out/boot.scr
'';
# Here, we built a fictitious system cloning the AArch64 sd-image setup.
# The chosen derivations are known to build fully when cross-compiled.
pkgsAArch64 = (if pkgs.stdenv.isAarch64 then pkgs else pkgs.pkgsCross.aarch64-multiplatform);
# The kernel for the device.
kernel = pkgsAArch64.linux_rpi;
# TODO: for completeness' sake an initrd with the vc4 driver should be built
# to show that this works as a self-contained demo.
in
with imageBuilder;
/**
* This disk image is built to be functionally compatible with the usual `sd_image`
* from NixOS, but *it is not* an actual `sd_image` compatible system.
*
* The main thing it aims to do is *minimally* create a bootable system.
*/
diskImage.makeMBR {
name = "diskimage";
diskID = "01234567";
partitions = [
(gap (size.MiB 10))
(fileSystem.makeFAT32 {
# Size-less
name = "FIRMWARE";
partitionID = "ABADF00D";
extraPadding = size.MiB 10;
populateCommands = ''
(
src=${pkgsAArch64.raspberrypifw}/share/raspberrypi/boot
cp $src/bootcode.bin $src/fixup*.dat $src/start*.elf ./
cp ${pkgsAArch64.ubootRaspberryPi3_64bit}/u-boot.bin ./u-boot-rpi3.bin
cp ${configTxt} ./config.txt
)
'';
})
(fileSystem.makeExt4 {
bootable = true;
name = "NIXOS";
partitionID = "44444444-4444-4444-8888-888888888888";
populateCommands = ''
mkdir -p ./boot
cp ${kernel}/Image ./boot/kernel
cp ${scr}/boot.scr ./boot/boot.scr
'';
})
];
}

87
lib/image-builder/in-depth-tests/size/ext4.nix
View File

@ -1,87 +0,0 @@
# Ensures we can fit stuff in an ext4 image.
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
makeNull = size: let
filename = "null.img";
filesystemType = "FAT32"; # meh, good enough
in
''
mkdir -p $out
dd if=/dev/zero of=./${toString size}.img bs=${toString size} count=1
'';
in
with imageBuilder;
{
eight = fileSystem.makeExt4 {
name = "eight";
partitionID = "44444444-4444-4444-0000-000000000008";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 8)}
'';
};
eleven = fileSystem.makeExt4 {
name = "eleven";
partitionID = "44444444-4444-4444-0000-000000000011";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 11)}
'';
};
sixteen = fileSystem.makeExt4 {
name = "sixteen";
partitionID = "44444444-4444-4444-0000-000000000016";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 16)}
'';
};
one_twenty_eight = fileSystem.makeExt4 {
name = "one_twenty_eight";
partitionID = "44444444-4444-4444-0000-000000000128";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 128)}
'';
};
two_fifty_six = fileSystem.makeExt4 {
name = "two_fifty_six";
partitionID = "44444444-4444-4444-0000-000000000256";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 256)}
'';
};
five_twelve = fileSystem.makeExt4 {
name = "five_twelve";
partitionID = "44444444-4444-4444-0000-000000000512";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 512)}
'';
};
with_space = fileSystem.makeExt4 {
name = "with_space";
partitionID = "44444444-4444-4444-0000-000000000005";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 5)}
'';
extraPadding = size.MiB 10;
};
# Fills 512 MiB (the downard slump in the high fudge factor) with 512 1MiB
# files so we ensure the filesystem overhead is accounted for.
multiple-files = fileSystem.makeExt4 {
name = "multiple-files";
partitionID = "44444444-4444-4444-0000-000000000512";
populateCommands = ''
for i in {1..512}; do
dd if=/dev/zero of=./$i.img bs=${toString (imageBuilder.size.MiB 1)} count=1
done
'';
};
}

5
lib/image-builder/lib/overlay.nix
View File

@ -1,5 +0,0 @@
[
(self: super: { imageBuilder = self.callPackage ../. {}; })
# All the software will be upstreamed with NixOS when upstreaming the library.
(import ../../../overlay/overlay.nix)
]

18
lib/image-builder/lib/tests/test-overlay.nix
View File

@ -1,18 +0,0 @@
# Adds the imageBuilder overlay.
(import ../overlay.nix) ++
[
# Makes the imageBuilder build impure to force rebuilds to more easily test
# reproducibility of outputs.
(self: super:
let
inherit (self.lib.attrsets) mapAttrs;
in
{
imageBuilder = super.imageBuilder.overrideScope'(self: super: {
makeFilesystem = args: super.makeFilesystem (args // {
REBUILD = "# ${toString builtins.currentTime}";
});
});
}
)
]

61
lib/image-builder/lib/tests/verify.rb
View File

@ -1,61 +0,0 @@
#!/usr/bin/env ruby
require "shellwords"
require "open3"
# Test harness to validate results of a `nix-build`.
# This script will `#load()` the given script.
if ARGV.length < 2 then
abort "Usage: verify.rb <script> <result>"
end
$failures = []
$script = ARGV.shift
$result = ARGV.shift
module Helpers
def compare_output(cmd, expected, message: nil)
expected ||= ""
message ||= "Command `#{cmd.shelljoin}` has unexpected output:"
expected = expected.strip
out = `#{cmd.shelljoin}`.strip
unless out == expected then
$failures << "#{message}:\n\tGot: #{out}\n\tExpected #{expected}"
end
end
def sha256sum(filename, expected)
filename = File.join($result, filename)
compare_output(
["nix-hash", "--flat", "--type", "sha256", filename], expected,
message: "File #{filename.shellescape} has unexpected hash",
)
end
def file(filename, expected)
filename = File.join($result, filename)
compare_output(
["file", "--dereference", "--brief", filename], expected,
message: "File #{filename.shellescape} has unexpected file type",
)
end
end
include Helpers
# Executes the script
load($script)
at_exit do
if $failures.length > 0 then
puts "Verification failed:"
$failures.each do |failure|
puts " → #{failure}"
end
exit 1
end
exit 0
end

20
lib/image-builder/makeBtrfs.nix
View File

@ -1,20 +0,0 @@
{ lib, imageBuilder, libfaketime, btrfs-progs }:
/* */ let scope = { "fileSystem.makeBtrfs" =
{ partitionID, ... }@args:
imageBuilder.makeFilesystem (args // {
filesystemType = "btrfs";
blockSize = 4096; # dummy
nativeBuildInputs = [btrfs-progs];
copyPhase = ''
mkfs.btrfs \
-r . \
-L "$partName" \
-U "$partitionID" \
--shrink \
"$img"
'';
})
/* */ ;}; in scope."fileSystem.makeBtrfs"

122
lib/image-builder/makeExt4.nix
View File

@ -1,122 +0,0 @@
{ lib, imageBuilder, libfaketime, e2fsprogs, make_ext4fs }:
/* */ let scope = { "fileSystem.makeExt4" =
let
inherit (lib.strings) splitString;
inherit (imageBuilder) makeFilesystem;
# Bash doesn't do floating point representations. Multiplications and divisions
# are handled with enough precision that we can multiply and divide to get a precision.
precision = 1000;
first = list: lib.lists.last (lib.lists.reverseList list);
chopDecimal = f: first (splitString "." (toString f));
makeFudge = f: toString (chopDecimal (f * precision));
# This applies only to 256MiB and greater.
# For smaller than 256MiB images the overhead from the FS is much greater.
# This will also let *some* slack space at the end at greater sizes.
# This is the value at 512MiB where it goes slightly down compared to 256MiB.
fudgeFactor = makeFudge 0.05208587646484375;
# This table was built using a script that built an image with `make_ext4fs`
# for the given size in MiB, and recorded the available size according to `df`.
smallFudgeLookup = lib.strings.concatStringsSep "\n" (lib.lists.reverseList(
lib.attrsets.mapAttrsToList (size: factor: ''
elif (( size > ${toString size} )); then
fudgeFactor=${toString factor}
'') {
"${toString (imageBuilder.size.MiB 5)}" = makeFudge 0.84609375;
"${toString (imageBuilder.size.MiB 8)}" = makeFudge 0.5419921875;
"${toString (imageBuilder.size.MiB 16)}" = makeFudge 0.288818359375;
"${toString (imageBuilder.size.MiB 32)}" = makeFudge 0.1622314453125;
"${toString (imageBuilder.size.MiB 64)}" = makeFudge 0.09893798828125;
"${toString (imageBuilder.size.MiB 128)}" = makeFudge 0.067291259765625;
"${toString (imageBuilder.size.MiB 256)}" = makeFudge 0.0518646240234375;
}
));
minimumSize = imageBuilder.size.MiB 5;
in
{ partitionID
, blockSize ? imageBuilder.size.KiB 4
, ... } @ args:
makeFilesystem (args // {
filesystemType = "ext4";
inherit blockSize minimumSize;
nativeBuildInputs = [
e2fsprogs
make_ext4fs
libfaketime
];
filesystemPhase = ''
:
'';
computeMinimalSize = ''
# `local size` is in bytes.
# We don't have a static reserved factor figured out. It is rather hard with
# ext4fs as there are multiple factors increasing the overhead.
local reservedSize=0
local fudgeFactor=${toString fudgeFactor}
# Instead we rely on a lookup table. See how it is built in the derivation file.
if (( size < ${toString (imageBuilder.size.MiB 256)} )); then
echo "$size is smaller than 256MiB; using the lookup table." 1>&2
# A bit of a hack, though allows us to build the lookup table using only
# elifs.
if false; then
:
${smallFudgeLookup}
else
# The data is smaller than 5MiB... The filesystem image size will likely
# not be able to accomodate... here we handle it in another way.
fudgeFactor=0
echo "Fudge factor skipped for extra small partition. Instead increasing by a fixed amount." 1>&2
size=$(( size + ${toString minimumSize}))
fi
fi
local reservedSize=$(( size * $fudgeFactor / ${toString precision} ))
echo "Fudge factor: $fudgeFactor / ${toString precision}" 1>&2
echo -n "Adding reservedSize: $size + $reservedSize = " 1>&2
size=$((size + reservedSize))
echo "$size" 1>&2
'';
copyPhase = ''
echo "Computing inode count..."
inodes=$(find . ! -type d -print0 | du --files0-from=- --inodes | cut -f1 | sum-lines)
echo " Min inodes: $inodes" 1>&2
inodes=$(( inodes * 2 ))
echo " Inodes reserved: $inodes" 1>&2
echo ""
faketime -f "1970-01-01 00:00:01" \
make_ext4fs \
-i $inodes \
-b $blockSize \
-L $partName \
-l $size \
-U $partitionID \
"$img" \
.
'';
checkPhase = ''
'';
# FIXME:
# Padding at end of inode bitmap is not set. Fix? no
# exit code
#EXT2FS_NO_MTAB_OK=yes fsck.ext4 -n -f $img
})
/* */ ;}; in scope."fileSystem.makeExt4"

107
lib/image-builder/makeFAT32.nix
View File

@ -1,107 +0,0 @@
{ lib, imageBuilder, dosfstools, mtools, libfaketime}:
/* */ let scope = { "fileSystem.makeFAT32" =
let
inherit (lib.strings) splitString;
inherit (imageBuilder) makeFilesystem;
# The default from `mkfs.fat`.
reservedSectors = 32;
# The default from `mkfs.fat`.
hiddenSectors = 0;
# The default from `mkfs.fat`.
numberOfFats = 2;
# Extra padding per FAT, a constant in code
fatPadding = 4;
# I have not been able to validate that it could be different from 1 for FAT32.
# It seems the different values (e.g. 4) are for FAT12 and FAT16.
# This is the only "bad" assumption here.
clusterSize = 1;
# Bash doesn't do floating point representations. Multiplications and divisions
# are handled with enough precision that we can multiply and divide to get a precision.
precision = 1000;
first = list: lib.lists.last (lib.lists.reverseList list);
chopDecimal = f: first (splitString "." (toString f));
in
{ partitionID
# These defaults are assuming small~ish FAT32 filesystems are generated.
, blockSize ? 512
, sectorSize ? 512
, ... } @ args:
makeFilesystem (args // {
# FAT32 can be used for ESP. Let's make this obvious.
filesystemType = if args ? filesystemType then args.filesystemType else "FAT32";
inherit blockSize sectorSize;
minimumSize = imageBuilder.size.KiB 500;
nativeBuildInputs = [
libfaketime
dosfstools
mtools
];
computeMinimalSize = ''
# `local size` is in bytes.
# This amount is a static amount of reserved space.
local static_reserved=${toString ( (reservedSectors + hiddenSectors) * sectorSize )}
# This is a constant representing the relative reserved space ratio.
local relative_reserved=${
chopDecimal (
precision - (
1.0 * sectorSize / ((clusterSize * sectorSize) + (numberOfFats * fatPadding))
# ^ forces floating point
) * precision
)
}
# Rounds up the likely truncated result. At worst it's a bit more space.
(( relative_reserved++ ))
echo "static_reserved=$static_reserved" 1>&2
echo "relative_reserved=$relative_reserved" 1>&2
local reservedSize=$(( (static_reserved + size) * relative_reserved / ${toString precision} + static_reserved ))
echo -n "Adding reservedSize: $size + $reservedSize = " 1>&2
size=$((size + reservedSize))
echo "$size" 1>&2
'';
filesystemPhase = ''
fatSize=16
if (( size > 1024*1024*32 )); then
fatSize=32
fi
faketime -f "1970-01-01 00:00:01" mkfs.vfat \
-F $fatSize \
-R ${toString reservedSectors} \
-h ${toString hiddenSectors} \
-s ${toString (blockSize / sectorSize)} \
-S ${toString sectorSize} \
-i $partitionID \
-n $partName \
"$img"
'';
copyPhase = ''
for f in ./* ./.*; do
if [[ "$f" != "./." && "$f" != "./.." ]]; then
faketime -f "1970-01-01 00:00:01" \
mcopy -psv -i "$img" "$f" ::
fi
done
'';
checkPhase = ''
# Always verify FS
fsck.vfat -vn "$img"
'';
})
/* */ ;}; in scope."fileSystem.makeFAT32"

156
lib/image-builder/makeFilesystem.nix
View File

@ -1,156 +0,0 @@
{ stdenvNoCC, lib, writeText }:
/* */ let scope = { "fileSystem.makeFilesystem" =
let
inherit (lib) optionals optionalString assertMsg;
in
{
name
# Size (in bytes) the filesystem image will be given.
# When size is not given, it is assumed that `populateCommands` will populate
# the filesystem, and the size will be derived (see computeMinimalSize).
, size ? null
# The populate commands are executed in a subshell. The CWD at the star is the
# public API to know where to add files that will be added to the image.
, populateCommands ? null
# Used with the assumption that files are rounded up to blockSize increments.
, blockSize
# Additional commands to compute a required increase in size to fit files.
, computeMinimalSize ? null
# When automatic sizing is used, additional amount of bytes to pad the image by.
, extraPadding ? 0
, ...
} @ args:
assert lib.asserts.assertMsg
(size !=null || populateCommands != null)
"Either a size or populateCommands needs to be given to build a filesystem.";
let
partName = name;
in
stdenvNoCC.mkDerivation (args // rec {
# Do not inherit `size`; we don't want to accidentally use it. The `size` can
# be dynamic depending on the contents.
inherit partName blockSize;
name = "partition-${partName}";
filename = "${partName}.img";
img = "${placeholder "out"}/${filename}";
nativeBuildInputs = [
] ++ optionals (args ? nativeBuildInputs) args.nativeBuildInputs;
buildCommand = ''
adjust-minimal-size() {
size="$1"
echo "$size"
}
compute-minimal-size() {
local size=0
(
cd files
# Size rounded in blocks. This assumes all files are to be rounded to a
# multiple of blockSize.
# Use of `--apparent-size` is to ensure we don't get the block size of the underlying FS.
# Use of `--block-size` is to get *our* block size.
size=$(find . ! -type d -print0 | du --files0-from=- --apparent-size --block-size "$blockSize" | cut -f1 | sum-lines)
echo "Reserving $size sectors for files..." 1>&2
# Adds one blockSize per directory, they do take some place, in the end.
# FIXME: write test to confirm this assumption
local directories=$(find . -type d | wc -l)
echo "Reserving $directories sectors for directories..." 1>&2
size=$(( directories + size ))
size=$((size * blockSize))
${if computeMinimalSize == null then "" else computeMinimalSize}
size=$(( size + ${toString extraPadding} ))
echo "$size"
)
}
sum-lines() {
local acc=0
while read -r number; do
acc=$((acc+number))
done
echo "$acc"
}
# The default stdenv/generic clashes with `runHook`.
# It doesn't override as expected.
unset -f checkPhase
mkdir -p $out
mkdir -p files
${optionalString (populateCommands != null) ''
echo
echo "Populating disk image"
echo
(
cd files
${populateCommands}
)
''}
${optionalString (size == null) ''
size=$(compute-minimal-size)
''}
if (( size < minimumSize )); then
size=$minimumSize
echo "WARNING: the '$partName' partition was too small, size increased to $minimumSize bytes."
fi
echo
echo "Building partition ${partName}"
echo "With ${if size == null
then "automatic size ($size bytes)"
else "$size bytes"
}"
echo
echo " -> Allocating space"
truncate -s $size "$img"
echo " -> Making filesystem"
runHook filesystemPhase
echo " -> Copying files"
(
cd files
runHook copyPhase
)
echo " -> Checking filesystem"
echo "$checkPhase"
runHook checkPhase
if [ -n "$postProcess" ]; then
echo "-> Running post-processing"
runHook postProcess
fi
'';
})
# mkdir -p $out/nix-support
# cat ${writeText "${name}-metadata" (builtins.toJSON {
# inherit size;
# })} > $out/nix-support/partition-metadata.json
/* */ ;}; in scope."fileSystem.makeFilesystem"

190
lib/image-builder/makeGPT.nix
View File

@ -1,190 +0,0 @@
{ stdenvNoCC, lib
, imageBuilder
, vboot_reference
}:
/* */ let scope = { "diskImage.makeGPT" =
let
inherit (lib) concatMapStringsSep optionalString;
# List of known mappings of GPT partition types to filesystems.
# This is not exhaustive, only used as a default.
# See also: https://sourceforge.net/p/gptfdisk/code/ci/master/tree/parttypes.cc
types = {
"FAT32" = "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7";
"ESP" = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B";
"LUKS" = "CA7D7CCB-63ED-4C53-861C-1742536059CC";
"ext2" = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
"ext3" = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
"ext4" = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
"btrfs" = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
};
in
{
name
, partitions
, diskID
, headerHole ? 0 # in bytes
, postProcess ? null
}:
let
_name = name;
eachPart = partitions: fn: (
concatMapStringsSep "\n" (partition:
fn partition
) partitions);
# Default alignment.
alignment = toString (imageBuilder.size.MiB 1);
image = partition:
if lib.isDerivation partition then
"${partition}/${partition.filename}"
else
partition.filename
;
in
stdenvNoCC.mkDerivation rec {
name = "disk-image-${_name}";
filename = "${_name}.img";
img = "${placeholder "out"}/${filename}";
inherit
postProcess
;
nativeBuildInputs = [
vboot_reference
];
buildCommand = let
# This fragment is used to compute the (aligned) size of the partition.
# It is used *only* to track the tally of the space used, thus the starting
# offset of the next partition. The filesystem sizes are untouched.
sizeFragment = partition: ''
start=$totalSize
${
if partition ? length then
''size=$((${toString partition.length}))''
else
''size=$(($(du --apparent-size -B 512 "$input_img" | awk '{ print $1 }') * 512))''
}
size=$(( $(if (($size % ${alignment})); then echo 1; else echo 0; fi ) + size / ${alignment} ))
size=$(( size * ${alignment} ))
totalSize=$(( totalSize + size ))
echo "Partition: start $start | size $size | totalSize $totalSize"
'';
# This fragment is used to add the desired gap to `totalSize`.
# We're setting `start` and `size` only to mirror the information shown
# for partitions.
# Do note that gaps are always aligned, so two gaps sized half the alignment
# would create 2× the space expected.
# What may *instead* be done at one point is always align `start` for partitions.
gapFragment = partition: ''
start=$totalSize
size=${toString partition.length}
size=$(( $(if (($size % ${alignment})); then echo 1; else echo 0; fi ) + size / ${alignment} ))
totalSize=$(( totalSize + size ))
echo "Gap: start $start | size $size | totalSize $totalSize"
'';
in ''
mkdir -p $out
# 34 is the base GPT header size, as added to -p by cgpt.
gptSize=$((${toString headerHole} + 34*512))
touch commands.sh
cat <<EOF > commands.sh
# Zeroes the GPT
cgpt create -z $img
# Create the GPT with space if desired
cgpt create -p ${toString (headerHole / 512)} $img
# Add the PMBR
cgpt boot -p $img
EOF
totalSize=$((gptSize))
echo
echo "Gathering information about partitions."
${eachPart partitions (partition:
if partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${image partition}"
${sizeFragment partition}
echo " -> ${partition.name}: $size / ${if partition ? filesystemType then partition.filesystemType else ""}"
(
printf "cgpt add"
printf " -b %s" "$((start/512))"
printf " -s %s" "$((size/512))"
printf " -t %s" '${
if partition ? partitionType then
partition.partitionType
else
types.${partition.filesystemType}
}'
${optionalString (partition ? partitionUUID)
"printf ' -u %s' '${partition.partitionUUID}'"}
${optionalString (partition ? bootable && partition.bootable)
"printf ' -B 1'"}
${optionalString (partition ? partitionLabel)
"printf ' -l \"%s\"' '${partition.partitionLabel}'"}
printf " $img\n"
) >> commands.sh
''
)}
# Allow space for secondary partition table / header.
totalSize=$(( totalSize + 34*512 ))
echo "--- script ----"
cat commands.sh
echo "--- script ----"
echo
echo "Making image, $totalSize bytes..."
truncate -s $((totalSize)) $img
PS4=" > " sh -x commands.sh
totalSize=$((gptSize))
echo
echo "Writing partitions into image"
${eachPart partitions (partition:
if partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${image partition}"
${sizeFragment partition}
echo " -> ${partition.name}: $size / ${if partition ? filesystemType then partition.filesystemType else ""}"
echo "$start / $size"
dd conv=notrunc if=$input_img of=$img seek=$((start/512)) count=$((size/512)) bs=512
''
)}
echo
echo "Information about the image:"
ls -lh $img
cgpt show $img
if [ -n "$postProcess" ]; then
echo "-> Running post-processing"
runHook postProcess
fi
'';
}
/* */ ;}; in scope."diskImage.makeGPT"

140
lib/image-builder/makeMBR.nix
View File

@ -1,140 +0,0 @@
{ stdenvNoCC, lib
, imageBuilder
, util-linux
}:
/* */ let scope = { "diskImage.makeMBR" =
let
inherit (lib) concatMapStringsSep optionalString;
# List of known mappings of MBR partition types to filesystems.
types = {
"FAT32" = "b";
"ESP" = "ef";
"ext2" = "83";
"ext3" = "83";
"ext4" = "83";
};
in
{
name
, partitions
# Without the prefixed `0x`
, diskID
}:
let
_name = name;
eachPart = partitions: fn: (
concatMapStringsSep "\n" (partition:
fn partition
) partitions);
# Default alignment.
alignment = toString (imageBuilder.size.MiB 1);
in
stdenvNoCC.mkDerivation rec {
name = "disk-image-${_name}";
filename = "${_name}.img";
img = "${placeholder "out"}/${filename}";
nativeBuildInputs = [
util-linux
];
buildCommand = let
# This fragment is used to compute the (aligned) size of the partition.
# It is used *only* to track the tally of the space used, thus the starting
# offset of the next partition. The filesystem sizes are untouched.
sizeFragment = ''
start=$totalSize
size=$(($(du --apparent-size -B 512 "$input_img" | awk '{ print $1 }') * 512))
size=$(( $(if (($size % ${alignment})); then echo 1; else echo 0; fi ) + size / ${alignment} ))
size=$(( size * ${alignment} ))
totalSize=$(( totalSize + size ))
echo "Partition: start $start | size $size | totalSize $totalSize"
'';
# This fragment is used to add the desired gap to `totalSize`.
# We're setting `start` and `size` only to mirror the information shown
# for partitions.
# Do note that gaps are always aligned, so two gaps sized half the alignment
# would create 2× the space expected.
# What may *instead* be done at one point is always align `start` for partitions.
gapFragment = partition: ''
start=$totalSize
size=${toString partition.length}
size=$(( $(if (($size % ${alignment})); then echo 1; else echo 0; fi ) + size / ${alignment} ))
size=$(( size * ${alignment} ))
totalSize=$(( totalSize + size ))
echo "Gap: start $start | size $size | totalSize $totalSize"
'';
in ''
mkdir -p $out
cat <<EOF > script.sfdisk
label: dos
grain: 1024
label-id: 0x${diskID}
EOF
totalSize=${alignment}
echo
echo "Gathering information about partitions."
${eachPart partitions (partition:
if partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${partition}/${partition.filename}"
${sizeFragment}
echo " -> ${partition.name}: $size / ${partition.filesystemType}"
(
# The size is /1024; otherwise it's in sectors.
echo -n 'start='"$((start/1024))"'KiB'
echo -n ', size='"$((size/1024))"'KiB'
echo -n ', type=${types."${partition.filesystemType}"}'
${optionalString (partition ? bootable && partition.bootable)
"echo -n ', bootable'"}
echo "" # Finishes the command
) >> script.sfdisk
''
)}
echo "--- script ----"
cat script.sfdisk
echo "--- script ----"
echo
echo "Making image, $totalSize bytes..."
truncate -s $((totalSize)) $img
sfdisk $img < script.sfdisk
totalSize=${alignment}
echo
echo "Writing partitions into image"
${eachPart partitions (partition:
if partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${partition}/${partition.filename}"
${sizeFragment}
echo " -> ${partition.name}: $size / ${partition.filesystemType}"
echo "$start / $size"
dd conv=notrunc if=$input_img of=$img seek=$((start/512)) count=$((size/512)) bs=512
''
)}
echo
echo "Information about the image:"
ls -lh $img
sfdisk -V --list $img
'';
}
/* */ ;}; in scope."diskImage.makeMBR"

144
lib/image-builder/test.rb
View File

@ -1,144 +0,0 @@
#!/usr/bin/env nix-shell
# All dependencies `verify` will need too.
#!nix-shell --pure -p nix -p ruby -p file -i ruby
require "tmpdir"
require "open3"
require "json"
# require "fileutils"
prefix = File.join(__dir__, "tests")
NIX_PATH = "nixpkgs-overlays=#{__dir__}/lib/tests/test-overlay.nix:nixpkgs=channel:nixos-19.03"
# Default directives for the test.
DEFAULT_DIRECTIVES = {
# Default is to succeed.
status: 0,
# Nothing to grep for in particular.
# (Caution! Successes are likely not to have logs!)
grep: nil,
}
Env = {
"NIX_PATH" => NIX_PATH,
}
tests =
if ARGV.count > 0 then
ARGV
else
# Assumes all nix files in `./tests` are tests to `nix-build`.
Dir.glob(File.join(prefix, "**/*.nix"))
end
tests.sort!
$exit = 0
$failed_tests = []
puts ""
puts "Tests"
puts "====="
puts ""
tests.each_with_index do |file, index|
short_name = file.sub("#{prefix}/", "")
print "Running test #{(index+1).to_s.rjust(tests.length.to_s.length)}/#{tests.length} '#{short_name}' "
failures = []
# Reads the first line of the file. It may hold a json snippet
# configuring the expected test results.
directives = File.read(file).split("\n").first || ""
directives = DEFAULT_DIRECTIVES.merge(
if directives.match(/^\s*#\s*expect:/i) then
# Parse...
JSON.parse(
# Everything after the first colon as json
directives.split(":", 2).last,
symbolize_names: true,
)
else
{}
end
)
# The result symlink is in a temp directory...
Dir.mktmpdir("image-builder-test") do |dir|
result = File.join(dir, "result")
# TODO : figure out how to keep stdout/stderr synced but logged separately.
log, status = Open3.capture2e(Env, "nix-build", "--show-trace", "--out-link", result, file)
unless status.exitstatus == directives[:status]
failures << "Build exited with status #{status.exitstatus} expected #{directives[:status]}."
end
if directives[:grep] then
unless log.match(directives[:grep])
failures << "Output log did not match `#{directives[:grep]}`."
end
end
# Do we test further with the test scripts?
if failures.length == 0 and status.success? then
script = file.sub(/\.nix$/, ".rb")
if File.exists?(script) then
log, status = Open3.capture2e(Env, File.join(__dir__, "lib/tests/verify.rb"), script, result)
end
unless status.exitstatus == 0
failures << "Verification exited with status #{status.exitstatus} expected 0."
end
store_path = File.readlink(result)
end
if failures.length == 0 then
puts "[Success] (#{store_path || "no output"}) "
else
$exit = 1
$failed_tests << file
puts "[Failed] (#{store_path || "no output"}) "
puts ""
puts "Failures:"
failures.each do |failure|
puts " - #{failure}"
end
puts ""
puts "Directives:"
puts ""
puts "```"
puts "#{directives.inspect}"
puts "```"
puts ""
puts "Output from the test:"
puts ""
puts "````"
puts "#{log}"
puts "````"
end
end
end
puts ""
puts "* * *"
puts ""
puts "Test summary"
puts "============"
puts ""
puts "#{tests.length - $failed_tests.length}/#{tests.length} tests successful."
if $failed_tests.length > 0 then
puts ""
puts "Failed tests:\n"
$failed_tests.each do |filename|
puts " - #{filename.sub(/^#{__dir__}/, ".")}"
end
puts ""
end
exit $exit

50
lib/image-builder/tests/alignment/aligned.nix
View File

@ -1,50 +0,0 @@
# Verifies that filesystems sized to be aligned works.
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
makeNull = size: pkgs.runCommand "filesystems-test" {
filename = "null.img";
filesystemType = "FAT32"; # meh, good enough
} ''
mkdir -p $out
dd if=/dev/zero of=$out/$filename bs=${toString size} count=1
'';
in
with imageBuilder;
{
one = diskImage.makeMBR {
name = "diskimage";
diskID = "012345678";
partitions = [
(makeNull (size.MiB 1))
(makeNull (size.MiB 1))
];
};
nine = diskImage.makeMBR {
name = "diskimage";
diskID = "012345678";
partitions = [
(makeNull (size.MiB 9))
(makeNull (size.MiB 9))
];
};
ten = diskImage.makeMBR {
name = "diskimage";
diskID = "012345678";
partitions = [
(makeNull (size.MiB 10))
(makeNull (size.MiB 10))
];
};
eleven = diskImage.makeMBR {
name = "diskimage";
diskID = "012345678";
partitions = [
(makeNull (size.MiB 11))
(makeNull (size.MiB 11))
];
};
}

42
lib/image-builder/tests/alignment/unaligned.nix
View File

@ -1,42 +0,0 @@
# Verifies that filesystems sized to be unaligned will work.
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
makeNull = size: pkgs.runCommand "filesystems-test" {
filename = "null.img";
filesystemType = "FAT32"; # meh, good enough
} ''
mkdir -p $out
dd if=/dev/zero of=$out/$filename bs=${toString size} count=1
'';
in
with imageBuilder;
# Through empirical testing, it was found out that the defaults from `sfdisk`
# are not as documented.
# First of all, on small disks no alignment will be made.
# Starting with 3MiB (empirically derived) alignment will be made.
# The alignment is documented as being based on the I/O limits. It seems like
# for files it ends up causing alignments at the 2MiB boundaries.
# Such, `grain: 1024` has to be set to configure sfdisk for the sane default
# documented in its manpage
#
# > grain Specify minimal size in bytes used to calculate partitions alignment.
# > The default is 1MiB and it's strongly recommended to use the default.
# > Do not modify this variable if you're not sure.
#
# The default is *not* 1MiB and will break the generation of images if it tries
# to align a small partition at the very end of the disk, when the disk is sized
# just right to fit.
#
# This is what this test validates.
diskImage.makeMBR {
name = "diskimage";
diskID = "012345678";
partitions = [
(makeNull (size.MiB 3))
(makeNull (size.MiB 1))
];
}

13
lib/image-builder/tests/failures/filesystem-without_size_without_commands.nix
View File

@ -1,13 +0,0 @@
# Expect: { "status": 1, "grep": "Either a size or populateCommands needs to be given to build a filesystem." }
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
in
with imageBuilder;
fileSystem.makeFAT32 {
name = "whatever";
partitionID = "0123456789ABCDEF";
}

35
lib/image-builder/tests/filesystems/default.nix
View File

@ -1,35 +0,0 @@
# Tests all known filesystems as empty, with defaults.
# This test helps ensure the basic interface stays stable, and works.
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
inherit (pkgs.lib.attrsets) mapAttrsToList;
inherit (pkgs.lib.strings) concatStringsSep removePrefix;
IDs = {
FAT32 = "0123456789ABCDEF";
ESP = "0123456789ABCDEF";
Ext4 = "44444444-4444-4444-1324-123456789098";
};
in
with imageBuilder;
let
cmds =
mapAttrsToList (fn_name: fn:
let
fs = fn rec {
name = removePrefix "make" fn_name;
size = imageBuilder.size.MiB 10;
partitionID = IDs."${name}";
};
in
''
ln -s ${fs}/${fs.filename} $out/
'') fileSystem;
in
pkgs.runCommand "filesystems-test" {} ''
mkdir -p $out/
${concatStringsSep "\n" cmds}
''

19
lib/image-builder/tests/filesystems/default.rb
View File

@ -1,19 +0,0 @@
hashes = {
"ESP.img" => "f9b39d98bfb797b050467ca5214671b5b427b7896cae44d27d2fc8dbceaccd88",
"FAT32.img" => "79028d8af97ae4400ea2ab36d34e2a80684c9f8d31ea75e3f54d908c75adc3a4",
"Ext4.img" => "8182df3038f43cdf2aba6f3980d9fa794affab0f040f9c07450ebbf0d3d8c2ad",
}
filetypes = {
"ESP.img" => 'DOS/MBR boot sector, code offset 0x3c+2, OEM-ID "mkfs.fat", reserved sectors 32, root entries 512, sectors 20480 (volumes <=32 MB), Media descriptor 0xf8, sectors/FAT 80, sectors/track 32, heads 64, serial number 0x89abcdef, label: "ESP ", FAT (16 bit)',
"FAT32.img" => 'DOS/MBR boot sector, code offset 0x3c+2, OEM-ID "mkfs.fat", reserved sectors 32, root entries 512, sectors 20480 (volumes <=32 MB), Media descriptor 0xf8, sectors/FAT 80, sectors/track 32, heads 64, serial number 0x89abcdef, label: "FAT32 ", FAT (16 bit)',
"Ext4.img" => 'Linux rev 1.0 ext4 filesystem data, UUID=44444444-4444-4444-1324-123456789098, volume name "Ext4" (extents) (large files)'
}
# By globbing on the output, we can validate all built images are verified.
# The builder should have built everything under `fileSystems`.
Dir.glob(File.join($result, "**/*")) do |file|
name = File.basename(file)
sha256sum(name, hashes[name])
file(name, filetypes[name])
end

39
lib/image-builder/tests/filesystems/default_with_files.nix
View File

@ -1,39 +0,0 @@
# Tests all known filesystems with files.
# This test helps ensure the basic interface stays stable, and works.
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
inherit (pkgs.lib.attrsets) mapAttrsToList;
inherit (pkgs.lib.strings) concatStringsSep removePrefix;
IDs = {
FAT32 = "0123456789ABCDEF";
ESP = "0123456789ABCDEF";
Ext4 = "44444444-4444-4444-1324-123456789098";
};
in
with imageBuilder;
let
cmds =
mapAttrsToList (fn_name: fn:
let
fs = fn rec {
name = removePrefix "make" fn_name;
partitionID = IDs."${name}";
populateCommands = ''
echo "I am ${name}." > file
ls -lA
'';
};
in
''
ln -s ${fs}/${fs.filename} $out/
'') fileSystem;
in
pkgs.runCommand "filesystems-test" {} ''
mkdir -p $out/
${concatStringsSep "\n" cmds}
''

19
lib/image-builder/tests/filesystems/default_with_files.rb
View File

@ -1,19 +0,0 @@
hashes = {
"ESP.img" => "d70b24594f0615b83fddb8608b4819432380359bc5c6763d7c318e0c3233ea64",
"FAT32.img" => "73ab1acd845fd0f7f5ddd30edea074f0b14bf4f2b4137f904a90939f1cb2ac8d",
"Ext4.img" => "56181a43406c4ba731ed14b3ed32ed6c169b1c2abfe385eef989faef55e3cd07",
}
filetypes = {
"ESP.img" => 'DOS/MBR boot sector, code offset 0x3c+2, OEM-ID "mkfs.fat", reserved sectors 32, root entries 512, sectors 1000 (volumes <=32 MB), Media descriptor 0xf8, sectors/FAT 3, sectors/track 32, heads 64, serial number 0x89abcdef, label: "ESP ", FAT (12 bit)',
"FAT32.img" => 'DOS/MBR boot sector, code offset 0x3c+2, OEM-ID "mkfs.fat", reserved sectors 32, root entries 512, sectors 1000 (volumes <=32 MB), Media descriptor 0xf8, sectors/FAT 3, sectors/track 32, heads 64, serial number 0x89abcdef, label: "FAT32 ", FAT (12 bit)',
"Ext4.img" => 'Linux rev 1.0 ext4 filesystem data, UUID=44444444-4444-4444-1324-123456789098, volume name "Ext4" (extents) (large files)'
}
# By globbing on the output, we can validate all built images are verified.
# The builder should have built everything under `fileSystems`.
Dir.glob(File.join($result, "**/*")) do |file|
name = File.basename(file)
sha256sum(name, hashes[name])
file(name, filetypes[name])
end

70
lib/image-builder/tests/size/ext4.nix
View File

@ -1,70 +0,0 @@
# Ensures we can fit stuff in an ext4 image.
{ pkgs ? import <nixpkgs> {} }:
let
inherit (pkgs) imageBuilder;
makeNull = size: let
filename = "null.img";
filesystemType = "FAT32"; # meh, good enough
in
''
mkdir -p $out
dd if=/dev/zero of=./${toString size}.img bs=${toString size} count=1
'';
in
with imageBuilder;
{
one = fileSystem.makeExt4 {
name = "one";
partitionID = "44444444-4444-4444-0000-000000000001";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 1)}
'';
};
two = fileSystem.makeExt4 {
name = "two";
partitionID = "44444444-4444-4444-0000-000000000002";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 2)}
'';
};
three = fileSystem.makeExt4 {
name = "three";
partitionID = "44444444-4444-4444-0000-000000000003";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 3)}
'';
};
four = fileSystem.makeExt4 {
name = "four";
partitionID = "44444444-4444-4444-0000-000000000004";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 4)}
'';
};
five = fileSystem.makeExt4 {
name = "five";
partitionID = "44444444-4444-4444-0000-000000000005";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 5)}
'';
};
# This is the boundary where otherwise it would begin to fail.
five_plus_one = fileSystem.makeExt4 {
name = "five_plus_one";
partitionID = "44444444-4444-4444-0001-000000000005";
populateCommands = ''
${makeNull ((imageBuilder.size.MiB 5) + 1)}
'';
};
six = fileSystem.makeExt4 {
name = "six";
partitionID = "44444444-4444-4444-0000-000000000006";
populateCommands = ''
${makeNull (imageBuilder.size.MiB 6)}
'';
};
# For bigger tests, see in-depth-tests
}

45
modules/disk-image.nix Normal file
View File

@ -0,0 +1,45 @@
{ lib, config, ... }:
# Common defaults for the generated disk image.
let
inherit (lib)
mkAfter
mkDefault
;
inherit (config.mobile.generatedFilesystems) rootfs;
deviceName = config.mobile.device.name;
# Name used for some image file output.
name = "${config.mobile.configurationName}-${deviceName}";
in
{
config = {
mobile.generatedDiskImages.disk-image = {
inherit name;
location = "/${name}.img";
partitioningScheme = mkDefault "gpt";
mbr = {
diskID = "12345678";
};
gpt = {
diskID = "b0486952-db96-4ebd-8c61-bef753fd69db";
};
partitions = mkAfter [
{
name = "mn-rootfs";
partitionLabel = rootfs.label;
partitionUUID = "CFB21B5C-A580-DE40-940F-B9644B4466E3";
raw = rootfs.imagePath;
}
];
additionalCommands = ''
echo ":: Adding hydra-build-products"
(PS4=" $ "; set -x
mkdir -p $out_path/nix-support
cat <<EOF > $out_path/nix-support/hydra-build-products
file disk-image $img
EOF
)
'';
};
};
}

32
modules/generated-disk-images.nix Normal file
View File

@ -0,0 +1,32 @@
{ config, lib, pkgs, ...}:
let
inherit (lib)
mapAttrs
mkOption
types
;
in
{
options = {
mobile.generatedDiskImages = mkOption {
type = types.attrsOf (pkgs.image-builder.types.disk-image);
description = lib.mdDoc ''
Disk image definitions that will be created at build.
'';
};
mobile.outputs.generatedDiskImages = mkOption {
type = with types; attrsOf package;
internal = true;
description = lib.mdDoc ''
All generated disk images from the build.
'';
};
};
config = {
mobile.outputs.generatedDiskImages =
mapAttrs (name: config: config.output) config.mobile.generatedDiskImages
;
};
}

104
modules/generated-filesystems.nix
View File

@ -4,118 +4,32 @@
{ config, lib, pkgs, ...}:
let
inherit (lib) types;
filesystemFunctions = {
"ext4" = pkgs.imageBuilder.fileSystem.makeExt4;
"btrfs" = pkgs.imageBuilder.fileSystem.makeBtrfs;
};
filesystemSubmodule =
{ name, config, ... }: {
options = {
type = lib.mkOption {
type = types.enum [ "ext4" "btrfs" ];
description = lib.mdDoc ''
Type of the generated filesystem.
'';
};
label = lib.mkOption {
type = types.str;
description = lib.mdDoc ''
The label used by the generated rootfs, when generating a rootfs, and
the filesystem label a Mobile NixOS system will look for by default.
'';
};
id = lib.mkOption {
type = types.str;
description = lib.mdDoc ''
The UUID used by the generated rootfs, when generating a rootfs.
'';
};
populateCommands = lib.mkOption {
type = types.lines;
description = lib.mdDoc ''
Commands used to fill the filesystem.
`$PWD` is the root of the filesystem.
'';
};
postProcess = lib.mkOption {
type = types.lines;
internal = true;
description = lib.mdDoc ''
Commands used to manipulate the filesystem after it has been
created.
'';
};
extraPadding = lib.mkOption {
type = types.int;
description = lib.mdDoc ''
Extra padding to add to the filesystem image.
'';
};
zstd = lib.mkOption {
internal = true;
type = types.bool;
description = lib.mdDoc ''
Whether to compress this artifact; used to work around size
limitations in CI situations.
'';
};
raw = lib.mkOption {
internal = true;
type = types.nullOr types.package;
default = null;
description = lib.mdDoc ''
Use an output directly rather than creating it from the options.
'';
};
};
config = {
};
}
inherit (lib)
mapAttrs
mkOption
types
;
in
{
options = {
mobile.generatedFilesystems = lib.mkOption {
type = types.attrsOf (types.submodule filesystemSubmodule);
mobile.generatedFilesystems = mkOption {
type = types.attrsOf (pkgs.image-builder.types.filesystem-image);
description = lib.mdDoc ''
Filesystem definitions that will be created at build.
'';
};
mobile.outputs.generatedFilesystems = lib.mkOption {
mobile.outputs.generatedFilesystems = mkOption {
type = with types; attrsOf package;
internal = true;
description = lib.mdDoc ''
All generated filesystems from the build.
'';
};
mobile.outputs.rootfs = lib.mkOption {
type = types.package;
visible = false;
description = lib.mdDoc ''
The rootfs image for the build.
'';
};
};
config = {
mobile.outputs.generatedFilesystems = lib.attrsets.mapAttrs (name: {raw, type, id, label, ...} @ attrs:
if raw != null then raw else
filesystemFunctions."${type}" (attrs // {
name = label;
partitionID = id;
})
) config.mobile.generatedFilesystems;
mobile.outputs.rootfs = config.mobile.outputs.generatedFilesystems.rootfs;
# Compatibility alias with the previous path.
system.build.rootfs =
builtins.trace "`system.build.rootfs` is being deprecated. Use `mobile.outputs.rootfs` instead. It will be removed after 2022-05"
config.mobile.outputs.generatedFilesystems.rootfs
mobile.outputs.generatedFilesystems =
mapAttrs (name: config: config.output) config.mobile.generatedFilesystems
;
};
}

2
modules/module-list.nix
View File

@ -9,8 +9,10 @@
./bootloader.nix
./cross-workarounds.nix
./devices-metadata.nix
./disk-image.nix
./documentation.nix
./hardware-eink.nix
./generated-disk-images.nix
./generated-filesystems.nix
./hardware-allwinner.nix
./hardware-exynos.nix

30
modules/rootfs.nix
View File

@ -38,9 +38,9 @@ in
boot.growPartition = lib.mkDefault true;
mobile.generatedFilesystems.rootfs = lib.mkDefault {
type = "ext4";
filesystem = "ext4";
label = "NIXOS_SYSTEM";
id = "44444444-4444-4444-8888-888888888888";
ext4.partitionID = "44444444-4444-4444-8888-888888888888";
populateCommands =
let
@ -58,29 +58,31 @@ in
'';
# Give some headroom for initial mounting.
extraPadding = pkgs.imageBuilder.size.MiB 20;
extraPadding = pkgs.image-builder.helpers.size.MiB 20;
location = "/rootfs.img${optionalString compressLargeArtifacts ".zst"}";
# FIXME: See #117, move compression into the image builder.
# Zstd can take a long time to complete successfully at high compression
# levels. Increasing the compression level could lead to timeouts.
postProcess = optionalString compressLargeArtifacts ''
additionalCommands = optionalString compressLargeArtifacts ''
echo ":: Compressing rootfs image"
(PS4=" $ "; set -x
PATH="$PATH:${buildPackages.zstd}/bin"
cd $out
ls -lh
time zstd -10 --rm "$filename"
ls -lh
cd $out_path
# Hacky, but the img path here already has .zst appended.
# Let's rename it (we assume rootfs.img) and do the compression here.
mv "$img" "rootfs.img"
time ${buildPackages.zstd}/bin/zstd -10 --rm "rootfs.img"
)
'' + ''
echo ":: Adding hydra-build-products"
(PS4=" $ "; set -x
mkdir $out/nix-support
cat <<EOF > $out/nix-support/hydra-build-products
file rootfs${optionalString compressLargeArtifacts "-zstd"} $out/$filename${optionalString compressLargeArtifacts ".zst"}
mkdir -p $out_path/nix-support
cat <<EOF > $out_path/nix-support/hydra-build-products
file rootfs${optionalString compressLargeArtifacts "-zstd"} $img
EOF
)
'';
zstd = compressLargeArtifacts;
};
boot.postBootCommands = mkIf (config.mobile.rootfs.rehydrateStore) ''

4
modules/system-types/android/default.nix
View File

@ -24,7 +24,7 @@ let
android-recovery = recovery.mobile.outputs.android.android-bootimg;
inherit (config.mobile.outputs.generatedFilesystems) rootfs;
inherit (config.mobile.generatedFilesystems) rootfs;
# Note:
# The flash scripts, by design, are not using nix-provided paths for
@ -33,7 +33,7 @@ let
# output should be usable even on systems without Nix.
android-fastboot-images = pkgs.runCommand "android-fastboot-images-${device.name}" {} ''
mkdir -p $out
cp -v ${rootfs}/${rootfs.filename} $out/system.img
cp -v ${rootfs.imagePath} $out/system.img
cp -v ${android-bootimg} $out/boot.img
${optionalString has_recovery_partition ''
cp -v ${android-recovery} $out/recovery.img

135
modules/system-types/depthcharge/default.nix
View File

@ -3,25 +3,95 @@
let
enabled = config.mobile.system.type == "depthcharge";
inherit (lib) types;
inherit (lib)
concatStringsSep
mkBefore
mkIf
mkMerge
mkOption
removeSuffix
types
;
inherit (pkgs) image-builder;
inherit (config.mobile.outputs) stage-0;
inherit (stage-0.mobile.boot.stage-1) kernel;
inherit (config.mobile.system.depthcharge.kpart) dtbs;
deviceName = config.mobile.device.name;
kernel = stage-0.mobile.boot.stage-1.kernel.package;
kernel_file = "${kernel}/${if kernel ? file then kernel.file else pkgs.stdenv.hostPlatform.linux-kernel.target}";
inherit (config.mobile.generatedFilesystems) rootfs;
build = pkgs.callPackage ./depthcharge-build.nix {
inherit (config.mobile.system.depthcharge.kpart) dtbs;
device_name = config.mobile.device.name;
inherit (config.mobile.outputs) initrd;
system = config.mobile.outputs.generatedFilesystems.rootfs;
cmdline = lib.concatStringsSep " " config.boot.kernelParams;
kernel = kernel.package;
arch = lib.strings.removeSuffix "-linux" config.mobile.system.system;
# Name used for some image file output.
name = "${config.mobile.configurationName}-${deviceName}";
# https://www.chromium.org/chromium-os/chromiumos-design-docs/disk-format
# This doesn't fit into the generic makeGPT, some of those are really specific
# to depthcharge.
GPT_ENTRY_TYPES = {
UNUSED = "00000000-0000-0000-0000-000000000000";
EFI = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B";
CHROMEOS_FIRMWARE = "CAB6E88E-ABF3-4102-A07A-D4BB9BE3C1D3";
CHROMEOS_KERNEL = "FE3A2A5D-4F32-41A7-B725-ACCC3285A309";
CHROMEOS_ROOTFS = "3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC";
CHROMEOS_RESERVED = "2E0A753D-9E48-43B0-8337-B15192CB1B5E";
LINUX_DATA = "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7";
LINUX_FS = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
};
arch = removeSuffix "-linux" config.mobile.system.system;
# https://github.com/thefloweringash/kevin-nix/issues/3
make-kernel-its = pkgs.fetchurl {
url = "https://raw.githubusercontent.com/thefloweringash/kevin-nix/a14a3bad3be7757575040b31e4c8e1bb801a8ed3/modules/make-kernel-its.sh";
sha256 = "1c0zbk69lyd3n8a636njc6in174zccg3hpjmafhxvfmyf45vxjis";
};
# Kernel command line for vbutil_kernel.
kpart_config = pkgs.writeTextFile {
name = "kpart-config-${deviceName}";
text = concatStringsSep " " config.boot.kernelParams;
};
# The image file containing the kernel and initrd.
kpart = pkgs.runCommand "kpart-${name}" {
nativeBuildInputs = with pkgs; [
dtc
ubootTools
vboot_reference
xz
];
} ''
# Bootloader
dd if=/dev/zero of=bootloader.bin bs=512 count=1
# Kernel
lzma --threads 0 < ${kernel_file} > kernel.lzma
ln -s ${dtbs} dtbs
ln -s ${stage-0.mobile.outputs.initrd} initrd
bash ${make-kernel-its} $PWD > kernel.its
mkimage \
-D "-I dts -O dtb -p 2048" \
-f kernel.its \
vmlinux.uimg
futility vbutil_kernel \
--version 1 \
--bootloader bootloader.bin \
--vmlinuz vmlinux.uimg \
--arch ${arch} \
--keyblock ${pkgs.vboot_reference}/share/vboot/devkeys/kernel.keyblock \
--signprivate ${pkgs.vboot_reference}/share/vboot/devkeys/kernel_data_key.vbprivk \
--config ${kpart_config} \
--pack $out
'';
in
{
options = {
mobile.system.depthcharge = {
kpart = {
dtbs = lib.mkOption {
dtbs = mkOption {
type = types.path;
default = null;
description = "Path to a directory with device trees, to be put in the kpart image";
@ -32,14 +102,14 @@ in
mobile = {
outputs = {
depthcharge = {
disk-image = lib.mkOption {
disk-image = mkOption {
type = types.package;
description = lib.mdDoc ''
Full Mobile NixOS disk image for a depthcharge-based system.
'';
visible = false;
};
kpart = lib.mkOption {
kpart = mkOption {
type = types.package;
description = lib.mdDoc ''
Kernel partition for a depthcharge-based system.
@ -51,14 +121,45 @@ in
};
};
config = lib.mkMerge [
config = mkMerge [
{ mobile.system.types = [ "depthcharge" ]; }
(lib.mkIf enabled {
(mkIf enabled {
mobile.generatedDiskImages.disk-image = {
partitions = mkBefore [
{
name = "kernel";
raw = kpart;
partitionLabel = "KERNEL-A";
partitionType = GPT_ENTRY_TYPES.CHROMEOS_KERNEL;
length = pkgs.image-builder.helpers.size.MiB 128;
}
];
# Add the missing bits to the kernel partition for depthcharge.
additionalCommands = ''
echo ":: Making image bootable by depthcharge"
(PS4=" $ "; set -x
${pkgs.buildPackages.vboot_reference}/bin/cgpt ${concatStringsSep " " [
"add"
"-i 1" # Work on the first partition (instead of adding)
"-S 1" # Mark as successful (so it'll be booted from)
"-T 5" # Tries remaining
"-P 10" # Priority
"$img"
]}
${pkgs.buildPackages.vboot_reference}/bin/cgpt ${concatStringsSep " " [
"show"
"$img"
]}
)
'';
};
mobile.outputs = {
default = build.disk-image;
default = config.mobile.outputs.depthcharge.disk-image;
depthcharge = {
inherit (build) disk-image kpart;
inherit kpart;
disk-image = config.mobile.generatedDiskImages.disk-image.output;
};
};
})

136
modules/system-types/depthcharge/depthcharge-build.nix
View File

@ -1,136 +0,0 @@
{ lib
, stdenv
, buildPackages
, fetchurl
, runCommand
, initrd
, system
, imageBuilder
, cmdline
, arch
, dtbs
, kernel
, device_name
, dtc
, ubootTools
, vboot_reference
, xz
, writeTextFile
}:
let
inherit (imageBuilder) size;
inherit (imageBuilder.diskImage) makeGPT;
# https://www.chromium.org/chromium-os/chromiumos-design-docs/disk-format
# This doesn't fit into the generic makeGPT, some of those are really specific
# to depthcharge.
GPT_ENTRY_TYPES = {
UNUSED = "00000000-0000-0000-0000-000000000000";
EFI = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B";
CHROMEOS_FIRMWARE = "CAB6E88E-ABF3-4102-A07A-D4BB9BE3C1D3";
CHROMEOS_KERNEL = "FE3A2A5D-4F32-41A7-B725-ACCC3285A309";
CHROMEOS_ROOTFS = "3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC";
CHROMEOS_RESERVED = "2E0A753D-9E48-43B0-8337-B15192CB1B5E";
LINUX_DATA = "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7";
LINUX_FS = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
};
# Kernel used in kpart.
kernel_file = "${kernel}/${if kernel ? file then kernel.file else stdenv.hostPlatform.linux-kernel.target}";
# Kernel command line for vbutil_kernel.
kpart_config = writeTextFile {
name = "kpart-config-${device_name}";
text = cmdline;
};
# Name used for some image file output.
name = "mobile-nixos-${device_name}";
# https://github.com/thefloweringash/kevin-nix/issues/3
make-kernel-its = fetchurl {
url = "https://raw.githubusercontent.com/thefloweringash/kevin-nix/a14a3bad3be7757575040b31e4c8e1bb801a8ed3/modules/make-kernel-its.sh";
sha256 = "1c0zbk69lyd3n8a636njc6in174zccg3hpjmafhxvfmyf45vxjis";
};
# The image file containing the kernel and initrd.
kpart = runCommand "kpart-${device_name}" {
nativeBuildInputs = [
dtc
ubootTools
vboot_reference
xz
];
} ''
# Bootloader
dd if=/dev/zero of=bootloader.bin bs=512 count=1
# Kernel
lzma --threads 0 < ${kernel_file} > kernel.lzma
ln -s ${dtbs} dtbs
ln -s ${initrd} initrd
bash ${make-kernel-its} $PWD > kernel.its
mkimage \
-D "-I dts -O dtb -p 2048" \
-f kernel.its \
vmlinux.uimg
mkdir -p $out/
futility vbutil_kernel \
--version 1 \
--bootloader bootloader.bin \
--vmlinuz vmlinux.uimg \
--arch ${arch} \
--keyblock ${buildPackages.vboot_reference}/share/vboot/devkeys/kernel.keyblock \
--signprivate ${buildPackages.vboot_reference}/share/vboot/devkeys/kernel_data_key.vbprivk \
--config ${kpart_config} \
--pack $out/kpart
'';
# An "unfinished" disk image.
# It's missing some minor cgpt magic.
# FIXME : make(MBR|GPT) should have a postBuild hook to manipulate the image.
image = makeGPT {
inherit name;
diskID = "44444444-4444-4444-8888-888888888888";
partitions = [
{
name = "kernel";
filename = "${kpart}/kpart";
partitionType = GPT_ENTRY_TYPES.CHROMEOS_KERNEL;
length = size.MiB 64;
}
system
];
};
in
{
inherit kpart;
# Takes the built image, and do some light editing using `cgpt`.
# This uses some depthcharge-specific fields to make the image bootable.
# FIXME : integrate into the makeGPT call with postBuild or something
disk-image = runCommand "depthcharge-${device_name}" { nativeBuildInputs = [ vboot_reference ]; } ''
# Copy the generated image...
# Note that while it's GPT, it's lacking some depthcharge magic attributes
cp ${image}/${name}.img ./
chmod +w ${name}.img
# Which is what we're adding back with cgpt!
cgpt add ${lib.concatStringsSep " " [
"-i 1" # Work on the first partition (instead of adding)
"-S 1" # Mark as successful (so it'll be booted from)
"-T 5" # Tries remaining
"-P 10" # Priority
"${name}.img"
]}
mkdir -p $out
cp ${name}.img $out/
'';
}

115
modules/system-types/u-boot/default.nix
View File

@ -4,13 +4,15 @@ let
enabled = config.mobile.system.type == "u-boot";
inherit (config.mobile.outputs) recovery stage-0;
inherit (pkgs) buildPackages imageBuilder runCommand;
inherit (lib) mkIf mkOption types;
inherit (pkgs) buildPackages image-builder runCommand;
inherit (lib) mkBefore mkIf mkOption types;
cfg = config.mobile.quirks.u-boot;
inherit (cfg) soc;
deviceName = config.mobile.device.name;
kernel = stage-0.mobile.boot.stage-1.kernel.package;
kernel_file = "${kernel}/${if kernel ? file then kernel.file else pkgs.stdenv.hostPlatform.linux-kernel.target}";
inherit (config.mobile.generatedFilesystems) rootfs;
boot-partition = config.mobile.generatedFilesystems.boot.output;
# Look-up table to translate from targetPlatform to U-Boot names.
ubootPlatforms = {
@ -103,77 +105,6 @@ let
} ''
mkimage -C none -A ${ubootPlatforms.${pkgs.stdenv.targetPlatform.system}} -T script -d ${bootcmd} $out
'';
# TODO: use generatedFilesystems
boot-partition =
imageBuilder.fileSystem.makeExt4 {
name = "mobile-nixos-boot";
partitionLabel = "boot";
partitionID = "ED3902B6-920A-4971-BC07-966D4E021683";
partitionUUID = "CFB21B5C-A580-DE40-940F-B9644B4466E1";
# Let's give us a *bunch* of space to play around.
# And let's not forget we have the kernel and stage-1 twice.
size = imageBuilder.size.MiB 128;
bootable = true;
populateCommands = ''
mkdir -vp mobile-nixos/{boot,recovery}
(
cd mobile-nixos/boot
cp -v ${stage-0.mobile.outputs.initrd} stage-1
cp -v ${kernel_file} kernel
cp -vr ${kernel}/dtbs dtbs
)
(
cd mobile-nixos/recovery
cp -v ${recovery.mobile.outputs.initrd} stage-1
cp -v ${kernel_file} kernel
cp -vr ${kernel}/dtbs dtbs
)
cp -v ${bootscr} ./boot.scr
'';
}
;
miscPartition = {
# Used as a BCB.
name = "misc";
partitionLabel = "misc";
partitionUUID = "5A7FA69C-9394-8144-A74C-6726048B129D";
length = imageBuilder.size.MiB 1;
partitionType = "EF32A33B-A409-486C-9141-9FFB711F6266";
filename = "/dev/null";
};
persistPartition = imageBuilder.fileSystem.makeExt4 {
# To work more like Android-based systems.
name = "persist";
partitionLabel = "persist";
partitionID = "5553F4AD-53E1-2645-94BA-2AFC60C12D38";
partitionUUID = "5553F4AD-53E1-2645-94BA-2AFC60C12D39";
size = imageBuilder.size.MiB 16;
partitionType = "EBC597D0-2053-4B15-8B64-E0AAC75F4DB1";
};
disk-image = imageBuilder.diskImage.makeGPT {
name = config.mobile.configurationName;
diskID = "01234567";
partitions = [
miscPartition
persistPartition
boot-partition
config.mobile.outputs.generatedFilesystems.rootfs
];
postProcess = ''
(PS4=" $ "; set -x
mkdir $out/nix-support
cat <<EOF > $out/nix-support/hydra-build-products
file disk-image $out/$filename
EOF
)
'';
};
in
{
options.mobile = {
@ -217,11 +148,47 @@ in
config = lib.mkMerge [
{ mobile.system.types = [ "u-boot" ]; }
(mkIf enabled {
mobile.generatedDiskImages.disk-image = {
partitions = mkBefore [
{
name = "mn-boot";
partitionLabel = "boot";
partitionUUID = "CFB21B5C-A580-DE40-940F-B9644B4466E1";
bootable = true;
raw = boot-partition;
}
];
};
mobile.generatedFilesystems.boot = {
filesystem = "ext4";
# Let's give us a *bunch* of space to play around.
# And let's not forget we have the kernel and stage-1 twice.
size = pkgs.image-builder.helpers.size.MiB 128;
ext4.partitionID = "ED3902B6-920A-4971-BC07-966D4E021683";
populateCommands = ''
mkdir -vp mobile-nixos/{boot,recovery}
(
cd mobile-nixos/boot
cp -v ${stage-0.mobile.outputs.initrd} stage-1
cp -v ${kernel_file} kernel
cp -vr ${kernel}/dtbs dtbs
)
(
cd mobile-nixos/recovery
cp -v ${recovery.mobile.outputs.initrd} stage-1
cp -v ${kernel_file} kernel
cp -vr ${kernel}/dtbs dtbs
)
cp -v ${bootscr} ./boot.scr
'';
};
mobile.outputs = {
default = config.mobile.outputs.u-boot.disk-image;
u-boot = {
inherit boot-partition;
disk-image = disk-image;
disk-image = config.mobile.generatedDiskImages.disk-image.output;
};
};
})

99
modules/system-types/uefi/default.nix
View File

@ -3,14 +3,15 @@
let
enabled = config.mobile.system.type == "uefi";
inherit (lib) mkEnableOption mkIf mkOption types;
inherit (lib) mkBefore mkEnableOption mkIf mkOption types;
inherit (pkgs.stdenv) hostPlatform;
inherit (pkgs) imageBuilder runCommand;
inherit (pkgs) image-builder runCommand;
inherit (config.mobile.outputs) recovery stage-0;
cfg = config.mobile.quirks.uefi;
deviceName = config.mobile.device.name;
kernel = stage-0.mobile.boot.stage-1.kernel.package;
kernelFile = "${kernel}/${if kernel ? file then kernel.file else pkgs.stdenv.hostPlatform.linux-kernel.target}";
inherit (config.mobile.generatedFilesystems) rootfs;
boot-partition = config.mobile.generatedFilesystems.boot.output;
# Look-up table to translate from targetPlatform to U-Boot names.
uefiPlatforms = {
@ -20,74 +21,21 @@ let
};
uefiPlatform = uefiPlatforms.${pkgs.stdenv.targetPlatform.system};
kernelParamsFile = pkgs.writeText "${deviceName}-boot.cmd" config.boot.kernelParams;
efiKernel = pkgs.runCommand "${deviceName}-efiKernel" {
kernelParamsFile = pkgs.writeText "${deviceName}-boot.cmd" config.boot.kernelParams;
nativeBuildInputs = [
pkgs.stdenv.cc.bintools.bintools_bin
];
} ''
(PS4=" $ "; set -x
${pkgs.stdenv.cc.bintools.targetPrefix}objcopy \
--add-section .cmdline="${kernelParamsFile}" --change-section-vma .cmdline=0x30000 \
--add-section .cmdline="$kernelParamsFile" --change-section-vma .cmdline=0x30000 \
--add-section .linux="${kernelFile}" --change-section-vma .linux=0x2000000 \
--add-section .initrd="${config.mobile.outputs.initrd}" --change-section-vma .initrd=0x3000000 \
"${pkgs.udev}/lib/systemd/boot/efi/linux${uefiPlatform}.efi.stub" \
"$out"
)
'';
# TODO: use generatedFilesystems
boot-partition =
imageBuilder.fileSystem.makeESP {
name = "mn-ESP"; # volume name (up to 11 characters long)
partitionLabel = "mn-ESP";
partitionID = "4E021684"; # FIXME: forwarded to filesystem volume ID, it shouldn't be
partitionUUID = "CFB21B5C-A580-DE40-940F-B9644B4466E2";
# Let's give us a *bunch* of space to play around.
# And let's not forget we have the kernel and stage-1 twice.
size = imageBuilder.size.MiB 128;
populateCommands = ''
mkdir -p EFI/boot
cp ${stage-0.mobile.outputs.uefi.efiKernel} EFI/boot/boot${uefiPlatform}.efi
cp ${recovery.mobile.outputs.uefi.efiKernel} EFI/boot/recovery${uefiPlatform}.efi
'';
}
;
miscPartition = {
# Used as a BCB.
name = "misc";
partitionLabel = "misc";
partitionUUID = "5A7FA69C-9394-8144-A74C-6726048B129D";
length = imageBuilder.size.MiB 1;
partitionType = "EF32A33B-A409-486C-9141-9FFB711F6266";
filename = "/dev/null";
};
persistPartition = imageBuilder.fileSystem.makeExt4 {
# To work more like Android-based systems.
name = "persist";
partitionLabel = "persist";
partitionID = "5553F4AD-53E1-2645-94BA-2AFC60C12D38";
partitionUUID = "5553F4AD-53E1-2645-94BA-2AFC60C12D39";
size = imageBuilder.size.MiB 16;
partitionType = "EBC597D0-2053-4B15-8B64-E0AAC75F4DB1";
};
disk-image = imageBuilder.diskImage.makeGPT {
name = "mobile-nixos";
diskID = "01234567";
headerHole = cfg.initialGapSize;
partitions = [
boot-partition
miscPartition
persistPartition
config.mobile.outputs.generatedFilesystems.rootfs
];
};
in
{
imports = [
@ -95,16 +43,6 @@ in
];
options.mobile = {
quirks.uefi = {
initialGapSize = mkOption {
type = types.int;
default = 0;
description = lib.mdDoc ''
Size (in bytes) to keep reserved in front of the first partition.
'';
};
};
outputs = {
uefi = {
boot-partition = mkOption {
@ -135,12 +73,35 @@ in
config = lib.mkMerge [
{ mobile.system.types = [ "uefi" ]; }
(mkIf enabled {
mobile.generatedDiskImages.disk-image = {
partitions = mkBefore [
(pkgs.image-builder.helpers.makeESP {
name = "mn-ESP"; # volume name (up to 11 characters long)
partitionLabel = "mn-ESP";
partitionUUID = "CFB21B5C-A580-DE40-940F-B9644B4466E2";
raw = boot-partition;
})
];
};
mobile.generatedFilesystems.boot = {
filesystem = "fat32";
# Let's give us a *bunch* of space to play around.
# And let's not forget we have the kernel and stage-1 twice.
size = pkgs.image-builder.helpers.size.MiB 128;
fat32.partitionID = "4E021684";
populateCommands = ''
mkdir -p EFI/boot
cp ${stage-0.mobile.outputs.uefi.efiKernel} EFI/boot/boot${uefiPlatform}.efi
cp ${recovery.mobile.outputs.uefi.efiKernel} EFI/boot/recovery${uefiPlatform}.efi
'';
};
mobile.outputs = {
default = config.mobile.outputs.uefi.disk-image;
uefi = {
inherit efiKernel;
inherit boot-partition;
inherit disk-image;
disk-image = config.mobile.generatedDiskImages.disk-image.output;
};
};
})

6
modules/system-types/uefi/vm.nix
View File

@ -7,7 +7,7 @@ let
inherit (lib) mkAfter mkIf mkMerge mkOption types;
inherit (config.mobile) device hardware;
inherit (config.mobile.boot) stage-1;
inherit (config.mobile.outputs.uefi) disk-image;
inherit (config.mobile.generatedDiskImages) disk-image;
ram = toString hardware.ram;
xres = toString hardware.screen.width;
@ -61,7 +61,7 @@ in
-bios "${pkgs.OVMF.fd}/FV/OVMF.fd"
-m "${ram}M"
-serial "mon:stdio"
-drive "file=${disk-image}/${disk-image.filename},format=raw,snapshot=on"
-drive "file=${disk-image.imagePath},format=raw,snapshot=on"
-device "VGA,edid=on,xres=${xres},yres=${yres}"
-device "usb-ehci"
@ -81,7 +81,7 @@ in
mobile.generatedFilesystems.rootfs = lib.mkDefault {
# Give some headroom in the VM, as it won't be actually resized.
extraPadding = lib.mkForce (pkgs.imageBuilder.size.MiB 512);
extraPadding = lib.mkForce (pkgs.image-builder.helpers.size.MiB 512);
};
})
(mkIf (!config.mobile.quirks.uefi.enableVM) {

23
overlay/image-builder/default.nix Normal file
View File

@ -0,0 +1,23 @@
{ pkgs, lib }:
{
evaluateFilesystemImage = { config ? {}, modules ? [] }: import ./filesystem-image/eval-config.nix {
inherit pkgs config modules;
};
evaluateDiskImage = { config ? {}, modules ? [] }: import ./disk-image/eval-config.nix {
inherit pkgs config modules;
};
types = {
disk-image = lib.types.submodule ({
imports = [ ./disk-image ];
_module.args.pkgs = pkgs;
});
filesystem-image = lib.types.submodule ({
imports = [ ./filesystem-image ];
_module.args.pkgs = pkgs;
});
};
helpers = (import ./helpers.nix { inherit lib; }).config.helpers;
}

99
overlay/image-builder/disk-image/basic.nix Normal file
View File

@ -0,0 +1,99 @@
{ config, lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
options = {
name = mkOption {
type = types.str;
default = "disk-image";
description = "Base name of the output";
};
alignment = mkOption {
type = types.int;
default = config.helpers.size.MiB 1;
description = lib.mdDoc ''
Partitions alignment.
Automatically computed partition start position will be aligned to
multiples of this value.
The default value is most likely appropriate.
'';
};
sectorSize = mkOption {
type = types.int;
default = 512;
internal = true;
description = lib.mdDoc ''
Sector size. This is used mainly internally. Changing this should have
no effects on the actual disk image produced.
The default value is most likely appropriate.
'';
};
location = mkOption {
type = types.str;
default = "";
description = lib.mdDoc ''
Location of the image in the `$out` path.
The default value means that `$img == $out`, which means that the
image is bare at the out path.
Other values should start with the directory separator (`/`), and
refer to the desired name.
The `$img` variable in the build script refers to `$out$location`.
'';
};
output = mkOption {
type = types.package;
internal = true;
description = lib.mdDoc ''
The build output for the disk image.
'';
};
imagePath = mkOption {
type = types.path;
default = "${config.output}${config.location}";
defaultText = lib.literalExpression "\"\${config.output}\${config.location}\"";
readOnly = true;
description = lib.mdDoc ''
Output path for the image file.
'';
};
additionalCommands = mkOption {
type = types.lines;
default = "";
description = lib.mdDoc ''
Additional commands to run during the disk image build.
'';
};
# TODO: implement this:
# mergeDerivationScripts = mkOption {
# type = types.bool;
# default = false;
# internal = true;
# description = lib.mdDoc ''
# Whether to produce discrete derivations for each steps, or to produce
# a single derivation that builds the image from A to Z.
#
# Setting this to true may be helpful with some CI/CD environments where
# limitations in output sizes makes it impossible to produce discrete
# derivations for every steps along the way.
# '';
# };
};
}

10
overlay/image-builder/disk-image/default.nix Normal file
View File

@ -0,0 +1,10 @@
{
# Note: No `filesystem` modules are to be added here. Filesystems are used as
# submodules in the partitions options.
imports = [
../helpers.nix
./basic.nix
./partitioning-scheme
./partitions.nix
];
}

18
overlay/image-builder/disk-image/eval-config.nix Normal file
View File

@ -0,0 +1,18 @@
/**
* Evaluates the configuration for a disk image build.
*/
{ pkgs
, modules ? []
, config ? {}
}:
let config' = config; in
rec {
module = { imports = [ ./. ]; };
config = (pkgs.lib.evalModules {
modules = [
{ _module.args.pkgs = pkgs; }
module
config'
] ++ modules;
}).config;
}

27
overlay/image-builder/disk-image/partitioning-scheme/default.nix Normal file
View File

@ -0,0 +1,27 @@
{ config, lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
imports = [
./gpt
./mbr
];
options = {
availablePartitioningSchemes = mkOption {
type = with types; listOf str;
internal = true;
};
partitioningScheme = mkOption {
type = types.enum config.availablePartitioningSchemes;
description = lib.mdDoc ''
Partitioning scheme for the disk image output.
'';
};
};
}

203
overlay/image-builder/disk-image/partitioning-scheme/gpt/builder.nix Normal file
View File

@ -0,0 +1,203 @@
{ stdenvNoCC
, lib
, fetchpatch
, gptfdisk
, buildPackages
, utillinux
, config
}:
let
inherit (lib)
concatMapStringsSep
concatStringsSep
optionalString
;
inherit (config.helpers)
each
;
inherit (config)
partitions
;
inherit (config.gpt)
hybridMBR
;
in
stdenvNoCC.mkDerivation rec {
inherit (config)
name
alignment
sectorSize
location
additionalCommands
;
inherit (config.gpt)
diskID
partitionEntriesCount
;
nativeBuildInputs = [
gptfdisk
utillinux
];
buildCommand = let
# This fragment is used to compute the (aligned) size of the partition.
# It is used *only* to track the tally of the space used, thus the starting
# offset of the next partition. The filesystem sizes are untouched.
sizeFragment = partition: ''
${if (partition ? offset && partition.offset != null) then ''
# If a partition asks to start at a specific offset, restart tally at
# that location.
offset=$((${toString partition.offset}))
if (( offset < totalSize )); then
echo "Partition '${partition.name}' wanted to start at $offset while we were already at $totalSize"
echo "As of right now, partitions need to be in order."
exit 1
else
totalSize=$offset
fi
start=$totalSize
# *by design* we're not aligning the start of the partition here if an
# offset was given.
'' else ''
# Assume we start where we left off...
start=$totalSize
# Align to the nearest alignment
if (( start % alignment )); then
start=$(( start + (alignment - start % alignment) ))
fi
''}
${
if (partition ? length && partition.length != null) then
''size=$((${toString partition.length}))''
else
''size=$(($(du --apparent-size -B 512 "$input_img" | awk '{ print $1 }') * 512))''
}
size=$(( $(if ((size % alignment)); then echo 1; else echo 0; fi ) + size / alignment ))
size=$(( size * alignment ))
totalSize=$(( totalSize + size ))
# Align the end too
if (( totalSize % alignment )); then
totalSize=$(( totalSize + (alignment - totalSize % alignment) ))
fi
echo "Partition: start $start | size $size | totalSize $totalSize"
'';
# This fragment is used to add the desired gap to `totalSize`.
# We're setting `start` and `size` only to mirror the information shown
# for partitions.
gapFragment = partition: ''
start=$totalSize
size=${toString partition.length}
totalSize=$(( totalSize + size ))
echo "Gap: start $start | size $size | totalSize $totalSize"
'';
in ''
set -u
# Referring to `$out` is forbidden, use `$img`.
# This is because the image path may or may not be at the root.
img="$out$location"
out_path="$out"
unset out
mkdir -p "$(dirname "$img")"
# LBA0 and LBA1 contains the PMBR and GPT.
#
# 2 is LBA2, where the header hole starts.
# One partition entry is 128 bytes long.
gptSize=$((2*512 + partitionEntriesCount*128))
cat <<EOF > script.sfdisk
label: gpt
unit: sectors
first-lba: $(( gptSize % sectorSize ? gptSize / sectorSize + 1 : gptSize / sectorSize ))
sector-size: $sectorSize
table-length: $partitionEntriesCount
grain: $alignment
${optionalString (diskID != null) ''
label-id: ${diskID}
''}
EOF
totalSize=$((gptSize))
echo
echo "Gathering information about partitions."
${each partitions (partition:
if partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${if partition.raw != null then partition.raw else ""}"
${sizeFragment partition}
echo ' -> '${lib.escapeShellArg partition.name}": $size / ${if partition ? filesystemType then partition.filesystemType else ""}"
(
echo -n 'start='"$((start/sectorSize))"
echo -n ', size='"$((size/sectorSize))"
echo -n ', type=${partition.partitionType}'
${optionalString (partition.partitionUUID != null)
"echo -n ', uuid=${partition.partitionUUID}'"}
${optionalString (partition ? bootable && partition.bootable)
''echo -n ', attrs="LegacyBIOSBootable"' ''}
${optionalString (partition ? requiredPartition && partition.requiredPartition)
''echo -n ', attrs="RequiredPartition"' ''}
${optionalString (partition ? partitionLabel)
''echo -n ', name="${partition.partitionLabel}"' ''}
echo "" # Finishes the command
) >> script.sfdisk
''
)}
# Allow space for secondary partition table / header.
totalSize=$(( totalSize + gptSize ))
echo "--- script ----"
cat script.sfdisk
echo "--- script ----"
echo
echo "Making image, $totalSize bytes..."
truncate -s $((totalSize)) $img
sfdisk $img < script.sfdisk
totalSize=$((gptSize))
echo
echo "Writing partitions into image"
${each partitions (partition:
if !(partition ? raw && partition.raw != null) && partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${if partition.raw != null then partition.raw else ""}"
if [[ "$input_img" == "" ]]; then
input_img="/dev/zero"
fi
${sizeFragment partition}
echo ' -> '${lib.escapeShellArg partition.name}": $size / ${if partition ? filesystemType then partition.filesystemType else ""}"
echo "$start / $size"
dd conv=notrunc if=$input_img of=$img seek=$((start/512)) count=$((size/512)) bs=512
''
)}
${optionalString (hybridMBR != []) ''
echo
echo "Making Hybrid MBR"
echo
sgdisk --hybrid=${concatStringsSep ":" hybridMBR} "$img"
''}
echo
echo "Information about the image:"
ls -lh $img
sfdisk -V --list $img
runHook additionalCommands
set +u
'';
}

56
overlay/image-builder/disk-image/partitioning-scheme/gpt/default.nix Normal file
View File

@ -0,0 +1,56 @@
{ config, lib, pkgs, ... }:
let
enabled = config.partitioningScheme == "gpt";
inherit (lib)
mkIf
mkMerge
mkOption
types
;
in
{
options.gpt = {
diskID = mkOption {
type = types.nullOr config.helpers.types.uuid;
default = null;
description = lib.mdDoc ''
Identifier for the disk.
'';
};
partitionEntriesCount = mkOption {
type = types.int;
default = 128;
description = lib.mdDoc ''
Number of partitions in the partition table.
The default value is likely appropriate.
'';
};
hybridMBR = mkOption {
type = with types; listOf str;
default = [];
example = [ "1" "3" "6" "EE" ];
description = lib.mdDoc ''
Creates an hybrid MBR with the given (string) partition numbers.
Up to three partitions can be present in the hybrid MBR, an additional
special partition can be placed last, named `EE`. When `EE` is present
last the protective partition for the GPT will be placed last.
See `man sgdisk`
'';
};
};
config = mkMerge [
{ availablePartitioningSchemes = [ "gpt" ]; }
(mkIf enabled {
output = pkgs.callPackage ./builder.nix {
inherit config;
};
})
];
}

180
overlay/image-builder/disk-image/partitioning-scheme/mbr/builder.nix Normal file
View File

@ -0,0 +1,180 @@
{ stdenvNoCC
, lib
, fetchpatch
, buildPackages
, utillinux
, config
}:
let
inherit (lib)
concatMapStringsSep
concatStringsSep
optionalString
;
inherit (config.helpers)
each
;
inherit (config)
partitions
;
in
stdenvNoCC.mkDerivation rec {
inherit (config)
name
alignment
sectorSize
location
additionalCommands
;
inherit (config.mbr)
diskID
;
nativeBuildInputs = [
utillinux
];
buildCommand = let
# This fragment is used to compute the (aligned) size of the partition.
# It is used *only* to track the tally of the space used, thus the starting
# offset of the next partition. The filesystem sizes are untouched.
sizeFragment = partition: ''
${if (partition ? offset && partition.offset != null) then ''
# If a partition asks to start at a specific offset, restart tally at
# that location.
offset=$((${toString partition.offset}))
if (( offset < totalSize )); then
echo "Partition '${partition.name}' wanted to start at $offset while we were already at $totalSize"
echo "As of right now, partitions need to be in order."
exit 1
else
totalSize=$offset
fi
start=$totalSize
# *by design* we're not aligning the start of the partition here if an
# offset was given.
'' else ''
# Assume we start where we left off...
start=$totalSize
# Align to the nearest alignment
if (( start % alignment )); then
start=$(( start + (alignment - start % alignment) ))
fi
''}
${
if (partition ? length && partition.length != null) then
''size=$((${toString partition.length}))''
else
''size=$(($(du --apparent-size -B 512 "$input_img" | awk '{ print $1 }') * 512))''
}
size=$(( $(if ((size % alignment)); then echo 1; else echo 0; fi ) + size / alignment ))
size=$(( size * alignment ))
totalSize=$(( totalSize + size ))
# Align the end too
if (( totalSize % alignment )); then
totalSize=$(( totalSize + (alignment - totalSize % alignment) ))
fi
echo "Partition: start $start | size $size | totalSize $totalSize"
'';
# This fragment is used to add the desired gap to `totalSize`.
# We're setting `start` and `size` only to mirror the information shown
# for partitions.
gapFragment = partition: ''
start=$totalSize
size=${toString partition.length}
totalSize=$(( totalSize + size ))
echo "Gap: start $start | size $size | totalSize $totalSize"
'';
in ''
set -u
# Referring to `$out` is forbidden, use `$img`.
# This is because the image path may or may not be at the root.
img="$out$location"
out_path="$out"
unset out
mkdir -p "$(dirname "$img")"
# LBA0 contains the MBR.
mbrSize=$((1*512))
cat <<EOF > script.sfdisk
label: dos
unit: sectors
first-lba: $(( mbrSize % sectorSize ? mbrSize / sectorSize + 1 : mbrSize / sectorSize ))
sector-size: $sectorSize
grain: $alignment
${optionalString (diskID != null) ''
label-id: ${diskID}
''}
EOF
totalSize=$((mbrSize))
echo
echo "Gathering information about partitions."
${each partitions (partition:
if partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${if partition.raw != null then partition.raw else ""}"
${sizeFragment partition}
echo ' -> '${lib.escapeShellArg partition.name}": $size / ${if partition ? filesystemType then partition.filesystemType else ""}"
(
echo -n 'start='"$((start/sectorSize))"
echo -n ', size='"$((size/sectorSize))"
echo -n ', type=${partition.partitionType}'
${optionalString (partition ? bootable && partition.bootable)
''echo -n ', bootable' ''}
echo "" # Finishes the command
) >> script.sfdisk
''
)}
# Allow space for secondary partition table / header.
totalSize=$(( totalSize + mbrSize ))
echo "--- script ----"
cat script.sfdisk
echo "--- script ----"
echo
echo "Making image, $totalSize bytes..."
truncate -s $((totalSize)) $img
sfdisk $img < script.sfdisk
totalSize=$((mbrSize))
echo
echo "Writing partitions into image"
${each partitions (partition:
if !(partition ? raw && partition.raw != null) && partition ? isGap && partition.isGap then
(gapFragment partition)
else
''
input_img="${if partition.raw != null then partition.raw else ""}"
if [[ "$input_img" == "" ]]; then
input_img="/dev/zero"
fi
${sizeFragment partition}
echo ' -> '${lib.escapeShellArg partition.name}": $size / ${if partition ? filesystemType then partition.filesystemType else ""}"
echo "$start / $size"
dd conv=notrunc if=$input_img of=$img seek=$((start/512)) count=$((size/512)) bs=512
''
)}
echo
echo "Information about the image:"
ls -lh $img
sfdisk -V --list $img
runHook additionalCommands
set +u
'';
}

34
overlay/image-builder/disk-image/partitioning-scheme/mbr/default.nix Normal file
View File

@ -0,0 +1,34 @@
{ config, lib, pkgs, ... }:
let
enabled = config.partitioningScheme == "mbr";
inherit (lib)
mkIf
mkMerge
mkOption
types
;
in
{
options.mbr = {
diskID = mkOption {
type = types.strMatching (
let hex = "[0-9a-fA-F]"; in
"${hex}{8}"
);
default = null;
description = lib.mdDoc ''
Identifier for the disk.
'';
};
};
config = mkMerge [
{ availablePartitioningSchemes = [ "mbr" ]; }
(mkIf enabled {
output = pkgs.callPackage ./builder.nix {
inherit config;
};
})
];
}

153
overlay/image-builder/disk-image/partitions.nix Normal file
View File

@ -0,0 +1,153 @@
{ config, lib, pkgs, ... }:
let
inherit (lib)
mkIf
mkMerge
mkOption
types
;
listEntrySubmodule = {
options = {
};
};
inherit (config) helpers;
partitionSubmodule = { name, config, ... }: {
options = {
name = mkOption {
type = types.str;
description = lib.mdDoc ''
Identifier for the partition.
'';
};
partitionLabel = mkOption {
type = types.str;
default = config.name;
defaultText = lib.literalExpression "config.name";
description = lib.mdDoc ''
Partition label on supported partition schemes. Defaults to ''${name}.
Not to be confused with _filesystem_ label.
'';
};
partitionUUID = mkOption {
type = types.nullOr helpers.types.uuid;
default = null;
description = lib.mdDoc ''
Partition UUID, for supported partition schemes.
Not to be confused with _filesystem_ UUID.
Not to be confused with _partitionType_.
'';
};
length = mkOption {
type = types.nullOr types.int;
default = null;
description = lib.mdDoc ''
Size in bytes for the partition.
Defaults to the filesystem image length (computed at runtime).
'';
};
offset = mkOption {
type = types.nullOr types.int;
default = null;
description = lib.mdDoc ''
Offset (in bytes) the partition starts at.
Defaults to the next aligned location on disk.
'';
};
partitionType = mkOption {
type = types.oneOf [
helpers.types.uuid
(types.strMatching "[0-9a-fA-F]{2}")
];
default = "0FC63DAF-8483-4772-8E79-3D69D8477DE4";
defaultText = "Linux filesystem data (0FC63DAF-8483-4772-8E79-3D69D8477DE4)";
description = lib.mdDoc ''
Partition type UUID.
Not to be confused with _partitionUUID_.
See: https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs
'';
};
bootable = mkOption {
type = types.bool;
default = false;
description = lib.mdDoc ''
Sets the "legacy bios bootable flag" on the partition.
'';
};
requiredPartition = mkOption {
type = types.bool;
default = false;
description = lib.mdDoc ''
For GPT, sets the Required Partition attribute on the partition.
'';
};
filesystem = mkOption {
type = types.nullOr (types.submodule ({
imports = [ ../filesystem-image ];
_module.args.pkgs = pkgs;
}));
default = null;
description = lib.mdDoc ''
A filesystem image configuration.
The filesystem image produced by this configuration is the default
value for the `raw` submodule option, unless overriden.
'';
};
isGap = mkOption {
type = types.bool;
default = false;
description = lib.mdDoc ''
When set to true, only the length attribute is used, and describes
an unpartitioned span in the disk image.
'';
};
raw = mkOption {
type = with types; nullOr (oneOf [ package path ]);
defaultText = "[contents of the filesystem attribute]";
default = null;
description = lib.mdDoc ''
Raw image to be used as the partition content.
By default uses the output of the `filesystem` submodule.
'';
};
};
config = mkMerge [
(mkIf (!config.isGap && config.filesystem != null) {
raw = lib.mkDefault config.filesystem.output;
})
];
};
in
{
options = {
partitions = mkOption {
type = with types; listOf (submodule partitionSubmodule);
description = lib.mdDoc ''
List of partitions to include in the disk image.
'';
};
};
}

308
overlay/image-builder/filesystem-image/basic.nix Normal file
View File

@ -0,0 +1,308 @@
{ config, lib, pkgs, ... }:
let
inherit (lib)
mkOption
optionalString
types
;
inherit (config)
computeMinimalSize
extraPadding
size
;
in
{
options = {
name = mkOption {
type = types.str;
default = "filesystem-image";
description = "Base name of the output";
};
label = mkOption {
type = with types; nullOr str;
default = null;
description = lib.mdDoc ''
Filesystem label
Not to be confused with either the output name, or the partition label.
'';
};
sectorSize = mkOption {
type = types.int;
internal = true;
description = lib.mdDoc ''
Default value should probably not be changed. Used internally for some
automatic size maths.
'';
};
blockSize = mkOption {
type = types.int;
internal = true;
description = lib.mdDoc ''
Used with the assumption that files are rounded up to blockSize increments.
'';
};
size = mkOption {
type = with types; nullOr int;
default = null;
defaultText = "[automatically computed]";
description = lib.mdDoc ''
When null, size is computed automatically.
Otherwise sets the size of the filesystem image.
Note that the usable space in the disk image will most likely be
smaller than the size given here!
'';
};
computeMinimalSize = mkOption {
internal = true;
type = types.str;
description = lib.mdDoc ''
Filesystem-specific snippet to adapt the size of the filesystem image
so the content can fit.
'';
};
extraPadding = mkOption {
type = types.int;
default = 0;
description = lib.mdDoc ''
When size is computed automatically, how many bytes to add to the
filesystem total size.
'';
};
minimumSize = mkOption {
type = types.int;
internal = true;
default = 0;
description = lib.mdDoc ''
Minimum usable size the filesystem must have. "Usable" here may not
actually be useful.
'';
};
builderFunctions = mkOption {
type = types.lines;
internal = true;
default = "";
description = lib.mdDoc ''
Bash functions required by the builder.
'';
};
buildPhases = mkOption {
type = with types; lazyAttrsOf str;
internal = true;
description = lib.mdDoc ''
Implementation of build phases for the filesystem image.
'';
};
buildPhasesOrder = mkOption {
type = with types; listOf str;
internal = true;
description = lib.mdDoc ''
Order of the filesystem image build phase. Adding to this is likely to
cause issues. Use this sparingly, and as a last resort.
'';
};
populateCommands = lib.mkOption {
type = types.lines;
default = "";
description = lib.mdDoc ''
Commands used to fill the filesystem.
`$PWD` is the root of the filesystem.
'';
};
buildInputs = mkOption {
type = with types; listOf package;
internal = true;
description = lib.mdDoc ''
Allows adding to the builder buildInputs.
'';
};
nativeBuildInputs = mkOption {
type = with types; listOf package;
internal = true;
description = lib.mdDoc ''
Allows adding to the builder nativeBuildInputs.
As this list is built without *splicing*, use `pkgs.buildPackages` as
a source for packages.
'';
};
location = mkOption {
type = types.str;
default = "";
description = lib.mdDoc ''
Location of the image in the `$out` path.
The default value means that `$img == $out`, which means that the
image is bare at the out path.
Other values should start with the directory separator (`/`), and
refer to the desired name.
The `$img` variable in the build script refers to `$out$location`.
'';
};
output = mkOption {
type = types.package;
internal = true;
description = lib.mdDoc ''
The build output for the filesystem image.
'';
};
imagePath = mkOption {
type = types.path;
default = "${config.output}${config.location}";
defaultText = lib.literalExpression "\"\${config.output}\${config.location}\"";
readOnly = true;
description = lib.mdDoc ''
Output path for the image file.
'';
};
additionalCommands = mkOption {
type = types.lines;
default = "";
description = lib.mdDoc ''
Additional commands to run during the filesystem image build.
'';
};
};
config = {
buildInputs = [
];
nativeBuildInputs = with pkgs.buildPackages; [
libfaketime
];
buildPhasesOrder = [
# Copy the files to be copied into the target filesystem image first in
# the `pwd` during this phase.
"populatePhase"
# Default phase should be fine, pre-allocate the disk image file.
"allocationPhase"
# Phase in which the `mkfs` command is called on the disk image file.
"filesystemPhase"
# Commands to copy the files into the disk image (if the creation command
# does not intrinsically do it).
# The prep/unprep phases change the CWD to the directory containing the
# files to copy into the filesystem image, and back into the build dir.
"_prepCopyPhase"
"copyPhase"
"_unprepCopyPhase"
# Commands where a filesystem check should be ran.
"checkPhase"
# Any other extra business to run, normally left to the consumer.
"additionalCommandsPhase"
];
buildPhases = {
"allocationPhase" = lib.mkDefault ''
${optionalString (size == null) ''
size=$(compute-minimal-size)
''}
if (( size < minimumSize )); then
size=$minimumSize
echo "WARNING: the '${"\${label:-(unlabeled)}"}' partition was too small, size increased to $minimumSize bytes."
fi
truncate -s $size "$img"
'';
"populatePhase" = lib.mkDefault ''
(
cd "$files"
${config.populateCommands}
# This also activates dotglob automatically.
# Using this means hidden files will be added too.
GLOBIGNORE=".:.."
if (( $(find -maxdepth 1 | wc -l) == 1 )); then
(set -x; ls -l)
echo ""
echo "ERROR: populatePhase produced no files."
echo " tip: using mkForce or mkDefault at different places may unexpected overwrite values."
exit 2
fi
)
'';
"_prepCopyPhase" = ''
cd "$files"
'';
"_unprepCopyPhase" = ''
cd "$NIX_BUILD_TOP"
'';
"additionalCommandsPhase" = config.additionalCommands;
};
builderFunctions = ''
compute-minimal-size() {
local size=0
(
cd files
# Size rounded in blocks. This assumes all files are to be rounded to a
# multiple of blockSize.
# Use of `--apparent-size` is to ensure we don't get the block size of the underlying FS.
# Use of `--block-size` is to get *our* block size.
size=$(find . ! -type d -print0 | du --files0-from=- --apparent-size --block-size "$blockSize" | cut -f1 | sum-lines)
echo "Reserving $size sectors for files..." 1>&2
# Adds one blockSize per directory, they do take some place, in the end.
local directories=$(find . -type d | wc -l)
echo "Reserving $directories sectors for directories..." 1>&2
size=$(( directories + size ))
size=$((size * blockSize))
${computeMinimalSize}
size=$(( size + ${toString extraPadding} ))
echo "$size"
)
}
sum-lines() {
local acc=0
while read -r number; do
acc=$((acc+number))
done
echo "$acc"
}
'';
output = pkgs.callPackage ./builder.nix { inherit config; };
};
}

75
overlay/image-builder/filesystem-image/builder.nix Normal file
View File

@ -0,0 +1,75 @@
{ stdenvNoCC, config, ncdu, tree }:
# This builder is heavily influenced by the configuration.
stdenvNoCC.mkDerivation (config.buildPhases // {
inherit (config)
name
buildInputs
filesystem
label
size
minimumSize
blockSize
sectorSize
location
additionalCommands
;
nativeBuildInputs = config.nativeBuildInputs ++ [
ncdu
tree
];
phases = config.buildPhasesOrder;
outputs = [ "out" "metadata" ];
buildCommand = ''
PS4=" $ "
set -u
# Referring to `$out` is forbidden, use `$img`.
# This is because the image path may or may not be at the root.
img="$out$location"
out_path="$out"
unset out
mkdir -p "$(dirname "$img")"
header() {
printf "\n:: %s\n\n" "$1"
}
${config.builderFunctions}
# The default stdenv/generic clashes with `runHook`.
# It doesn't override as expected.
unset -f checkPhase
# Location where extra metadata about the filesystem can be stored.
# Use for extra useful debugging data.
mkdir -p $metadata
# Location where the filesystem content will be copied to.
mkdir -p files
files="$(cd files; pwd)"
for phase in ''${phases[@]}; do
if [[ "$phase" != _* ]]; then
header "Running $phase"
fi
runHook "$phase"
done
(
cd "$files"
faketime -f "1970-01-01 00:00:01" tree -a | xz > $metadata/tree.xz
faketime -f "1970-01-01 00:00:01" ncdu -0x -o - | xz > $metadata/ncdu.xz
)
set +u
'';
passthru = {
inherit (config) filesystem;
};
})

7
overlay/image-builder/filesystem-image/default.nix Normal file
View File

@ -0,0 +1,7 @@
{
imports = [
../helpers.nix
./basic.nix
./filesystem
];
}

18
overlay/image-builder/filesystem-image/eval-config.nix Normal file
View File

@ -0,0 +1,18 @@
/**
* Evaluates the configuration for a single filesystem image build.
*/
{ pkgs
, modules ? []
, config ? {}
}:
let config' = config; in
rec {
module = { imports = [ ./. ]; };
config = (pkgs.lib.evalModules {
modules = [
{ _module.args.pkgs = pkgs; }
module
config'
] ++ modules;
}).config;
}

62
overlay/image-builder/filesystem-image/filesystem/btrfs.nix Normal file
View File

@ -0,0 +1,62 @@
{ config, lib, pkgs, ... }:
let
enabled = config.filesystem == "btrfs";
inherit (lib)
escapeShellArg
mkIf
mkMerge
mkOption
optionalString
types
;
inherit (config) label sectorSize blockSize;
inherit (config.btrfs) partitionID;
in
{
options.btrfs = {
partitionID = mkOption {
type = types.nullOr config.helpers.types.uuid;
example = "45454545-4545-4545-4545-454545454545";
default = null;
description = lib.mdDoc ''
Volume ID of the filesystem.
'';
};
};
config = mkMerge [
{ availableFilesystems = [ "btrfs" ]; }
(mkIf enabled {
nativeBuildInputs = with pkgs.buildPackages; [
btrfs-progs
];
blockSize = config.helpers.size.KiB 4;
sectorSize = lib.mkDefault 512;
# Generated an empty filesystem, it was 114294784 bytes long.
# Rounded up to 110MiB.
minimumSize = config.helpers.size.MiB 110;
computeMinimalSize = ''
'';
buildPhases = {
copyPhase = ''
mkfs.btrfs \
-r . \
${optionalString (partitionID != null) "-U ${partitionID}"} \
${optionalString (label != null) "-L ${escapeShellArg label}"} \
${optionalString (config.size == null) "--shrink"} \
"$img"
'';
checkPhase = ''
faketime -f "1970-01-01 00:00:01" btrfs check "$img"
'';
};
})
];
}

29
overlay/image-builder/filesystem-image/filesystem/default.nix Normal file
View File

@ -0,0 +1,29 @@
{ config, lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
imports = [
./btrfs.nix
./ext4.nix
./fat32.nix
./squashfs.nix
];
options = {
availableFilesystems = mkOption {
type = with types; listOf str;
internal = true;
};
filesystem = mkOption {
type = types.enum config.availableFilesystems;
description = lib.mdDoc ''
Filesystem used in this filesystem image.
'';
};
};
}

128
overlay/image-builder/filesystem-image/filesystem/ext4.nix Normal file
View File

@ -0,0 +1,128 @@
{ config, lib, pkgs, ... }:
let
enabled = config.filesystem == "ext4";
inherit (lib)
escapeShellArg
mkIf
mkMerge
mkOption
optionalString
types
;
inherit (config.helpers)
chopDecimal
;
inherit (config) label sectorSize blockSize;
inherit (config.ext4) partitionID;
# Bash doesn't do floating point representations. Multiplications and divisions
# are handled with enough precision that we can multiply and divide to get a precision.
precision = 1000;
makeFudge = f: toString (chopDecimal (f * precision));
# This applies only to 256MiB and greater.
# For smaller than 256MiB images the overhead from the FS is much greater.
# This will also let *some* slack space at the end at greater sizes.
# This is the value at 512MiB where it goes slightly down compared to 256MiB.
fudgeFactor = makeFudge 0.05208587646484375;
# This table was built using a script that built an image with `make_ext4fs`
# for the given size in MiB, and recorded the available size according to `df`.
smallFudgeLookup = lib.strings.concatStringsSep "\n" (lib.lists.reverseList(
lib.attrsets.mapAttrsToList (size: factor: ''
elif (( size > ${toString size} )); then
fudgeFactor=${toString factor}
'') {
"${toString (config.helpers.size.MiB 5)}" = makeFudge 0.84609375;
"${toString (config.helpers.size.MiB 8)}" = makeFudge 0.5419921875;
"${toString (config.helpers.size.MiB 16)}" = makeFudge 0.288818359375;
"${toString (config.helpers.size.MiB 32)}" = makeFudge 0.1622314453125;
"${toString (config.helpers.size.MiB 64)}" = makeFudge 0.09893798828125;
"${toString (config.helpers.size.MiB 128)}" = makeFudge 0.067291259765625;
"${toString (config.helpers.size.MiB 256)}" = makeFudge 0.0518646240234375;
}
));
minimumSize = config.helpers.size.MiB 5;
in
{
options.ext4 = {
partitionID = mkOption {
type = types.nullOr config.helpers.types.uuid;
example = "45454545-4545-4545-4545-454545454545";
default = null;
description = lib.mdDoc ''
Volume ID of the filesystem.
'';
};
};
config = mkMerge [
{ availableFilesystems = [ "ext4" ]; }
(mkIf enabled {
nativeBuildInputs = with pkgs.buildPackages; [
e2fsprogs
make_ext4fs
];
blockSize = config.helpers.size.KiB 4;
sectorSize = lib.mkDefault 512;
inherit minimumSize;
computeMinimalSize = ''
# `local size` is in bytes.
# We don't have a static reserved factor figured out. It is rather hard with
# ext4fs as there are multiple factors increasing the overhead.
local reservedSize=0
local fudgeFactor=${toString fudgeFactor}
# Instead we rely on a lookup table. See how it is built in the derivation file.
if (( size < ${toString (config.helpers.size.MiB 256)} )); then
echo "$size is smaller than 256MiB; using the lookup table." 1>&2
# A bit of a hack, though allows us to build the lookup table using only elifs.
if false; then
:
${smallFudgeLookup}
else
# The data is smaller than 5MiB... The filesystem image size will likely
# not be able to accomodate... here we handle it in another way.
fudgeFactor=0
echo "Fudge factor skipped for extra small partition. Instead increasing by a fixed amount." 1>&2
size=$(( size + ${toString minimumSize}))
fi
fi
local reservedSize=$(( size * $fudgeFactor / ${toString precision} ))
echo "Fudge factor: $fudgeFactor / ${toString precision}" 1>&2
echo -n "Adding reservedSize: $size + $reservedSize = " 1>&2
size=$((size + reservedSize))
echo "$size" 1>&2
'';
buildPhases = {
copyPhase = ''
faketime -f "1970-01-01 00:00:01" \
make_ext4fs \
-b $blockSize \
-l $size \
${optionalString (partitionID != null) "-U ${partitionID}"} \
${optionalString (label != null) "-L ${escapeShellArg label}"} \
"$img" \
.
'';
checkPhase = ''
EXT2FS_NO_MTAB_OK=yes faketime -f "1970-01-01 00:00:01" fsck.ext4 -y -f "$img" || :
EXT2FS_NO_MTAB_OK=yes faketime -f "1970-01-01 00:00:01" fsck.ext4 -n -f "$img"
'';
};
})
];
}

136
overlay/image-builder/filesystem-image/filesystem/fat32.nix Normal file
View File

@ -0,0 +1,136 @@
{ config, lib, pkgs, ... }:
let
enabled = config.filesystem == "fat32";
inherit (lib)
escapeShellArg
mkIf
mkMerge
mkOption
optionalString
types
;
inherit (config.helpers)
chopDecimal
;
type32bitHex = types.strMatching "[0-9a-fA-F]{1,8}";
inherit (config) label sectorSize blockSize;
inherit (config.fat32) partitionID;
# Should these be configurable?
# The default from `mkfs.fat`.
reservedSectors = 32;
# The default from `mkfs.fat`.
hiddenSectors = 0;
# The default from `mkfs.fat`.
numberOfFats = 2;
# Extra padding per FAT, a constant in code
fatPadding = 4;
# I have not been able to validate that it could be different from 1 for FAT32.
# It seems the different values (e.g. 4) are for FAT12 and FAT16.
# This is the only "bad" assumption here.
clusterSize = 1;
# Bash doesn't do floating point representations. Multiplications and divisions
# are handled with enough precision that we can multiply and divide to get a precision.
precision = 1000;
in
{
options.fat32 = {
partitionID = mkOption {
type = types.nullOr type32bitHex;
example = "2e24ec82";
default = null;
defaultText = "[Depends on the file system creation time]";
description = lib.mdDoc ''
Volume ID of the filesystem.
The default is a number which depends on the file system creation time.
'';
};
};
config = mkMerge [
{ availableFilesystems = [ "fat32" ]; }
(mkIf enabled {
nativeBuildInputs = with pkgs.buildPackages; [
dosfstools
mtools
];
blockSize = lib.mkDefault 512;
sectorSize = lib.mkDefault 512;
minimumSize = config.helpers.size.KiB 500;
buildPhases = {
checkPhase = ''
# Always check and verify FS
fsck.vfat -a "$img" || :
fsck.vfat -vn "$img"
'';
filesystemPhase = ''
fatSize=16
if (( size > 1024*1024*32 )); then
fatSize=32
fi
faketime -f "1970-01-01 00:00:01" mkfs.vfat \
-F $fatSize \
-R ${toString reservedSectors} \
-h ${toString hiddenSectors} \
-s ${toString (blockSize / sectorSize)} \
-S ${toString sectorSize} \
${optionalString (partitionID != null) "-i ${partitionID}"} \
${optionalString (label != null) "-n ${escapeShellArg label}"} \
"$img"
'';
copyPhase = ''
(
for f in ./* ./.*; do
if [[ "$f" != "./." && "$f" != "./.." ]]; then
faketime -f "1970-01-01 00:00:01" \
mcopy -psv -i "$img" "$f" ::
fi
done
)
'';
};
computeMinimalSize = ''
# `local size` is in bytes.
# This amount is a static amount of reserved space.
local static_reserved=${toString ( (reservedSectors + hiddenSectors) * sectorSize )}
# This is a constant representing the relative reserved space ratio.
local relative_reserved=${
chopDecimal (
precision - (
1.0 * sectorSize / ((clusterSize * sectorSize) + (numberOfFats * fatPadding))
# ^ forces floating point
) * precision
)
}
# Rounds up the likely truncated result. At worst it's a bit more space.
(( relative_reserved++ ))
echo "static_reserved=$static_reserved" 1>&2
echo "relative_reserved=$relative_reserved" 1>&2
local reservedSize=$(( (static_reserved + size) * relative_reserved / ${toString precision} + static_reserved ))
echo -n "Adding reservedSize: $size + $reservedSize = " 1>&2
size=$((size + reservedSize))
echo "$size" 1>&2
'';
})
];
}

86
overlay/image-builder/filesystem-image/filesystem/squashfs.nix Normal file
View File

@ -0,0 +1,86 @@
{ config, lib, pkgs, ... }:
let
enabled = config.filesystem == "squashfs";
inherit (lib)
escapeShellArg
mkIf
mkMerge
mkOption
optionalString
types
;
inherit (config) label sectorSize blockSize;
inherit (config.squashfs)
compression
compressionParams
;
in
{
options.squashfs = {
compression = mkOption {
type = types.enum [
# Uses the name used in the -comp param
"gzip"
"xz"
"zstd"
];
default = "xz";
description = lib.mdDoc ''
Volume ID of the filesystem.
'';
};
compressionParams = mkOption {
type = types.str;
internal = true;
};
};
config = mkMerge [
{ availableFilesystems = [ "squashfs" ]; }
(mkIf enabled {
squashfs.compressionParams = mkMerge [
(mkIf (compression == "xz") "-Xdict-size 100%")
(mkIf (compression == "zstd") "-Xcompression-level 6")
];
nativeBuildInputs = with pkgs.buildPackages; [
squashfsTools
];
# NixOS's make-squashfs uses 1MiB
# mksquashfs's help says its default is 128KiB
blockSize = config.helpers.size.MiB 1;
# This is actually unused (and irrelevant)
sectorSize = lib.mkDefault 512;
minimumSize = 0;
computeMinimalSize = "";
buildPhases = {
copyPhase = ''
# The empty pre-allocated file will confuse mksquashfs
rm "$img"
(
# This also activates dotglob automatically.
# Using this means hidden files will be added too.
GLOBIGNORE=".:.."
# Using `.` as the input will put the PWD, including its name
# in the root of the filesystem.
mksquashfs \
* \
"$img" \
-info \
-b "$blockSize" \
-no-hardlinks -keep-as-directory -all-root \
-comp "${compression}" ${compressionParams} \
-processors $NIX_BUILD_CORES
)
'';
};
})
];
}

70
overlay/image-builder/helpers.nix Normal file
View File

@ -0,0 +1,70 @@
{ lib, ... }:
let
inherit (lib)
concatMapStringsSep
mkOption
splitString
types
;
in
{
options.helpers = mkOption {
# Unspecified on purpose
type = types.attrs;
internal = true;
};
config.helpers = rec {
/**
* Silly alias to concat/map script segments for a given list.
*/
each = els: fn: (
concatMapStringsSep "\n" (el:
fn el
) els);
/**
* Provides user-friendly aliases for defining sizes.
*/
size = rec {
TiB = x: 1024 * (GiB x);
GiB = x: 1024 * (MiB x);
MiB = x: 1024 * (KiB x);
KiB = x: 1024 * x;
};
/**
* Drops the decimal portion of a floating point number.
*/
chopDecimal = f: first (splitString "." (toString f));
/**
* Like `last`, but for the first element of a list.
*/
first = list: lib.lists.last (lib.lists.reverseList list);
types = {
uuid = lib.types.strMatching (
let hex = "[0-9a-fA-F]"; in
"${hex}{8}-${hex}{4}-${hex}{4}-${hex}{4}-${hex}{12}"
);
};
makeGap = length: {
isGap = true;
inherit length;
};
makeESP = args: lib.recursiveUpdate {
name = "ESP-filesystem";
partitionLabel = "$ESP";
partitionType = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B";
partitionUUID = "63E19453-EF00-4BD9-9AAF-000000000000";
filesystem = {
filesystem = "fat32";
label = "$ESP";
fat32.partitionID = "ef00ef00";
};
} args;
};
}

2
overlay/overlay.nix
View File

@ -162,5 +162,5 @@ in
cross-canary-test-static = self.pkgsStatic.callPackage ./mobile-nixos/cross-canary/test.nix {};
};
imageBuilder = callPackage ../lib/image-builder {};
image-builder = callPackage ./image-builder {};
}

2
release.nix
View File

@ -82,7 +82,7 @@ let
# lib-like attributes...
# How should we handle these?
imageBuilder = null;
image-builder = null;
mobile-nixos = (onlyDerivationsAndAttrsets overlay.mobile-nixos) // {
# The cross canaries attrsets will be used as constituents.
# Filter out `override` and `overrideAttrs` early.