Merge pull request #27 from nix-community/zfs

zfs, lvm raid, btrfs subvolumes support & some fixups
This commit is contained in:
Jörg Thalheim 2022-08-26 14:59:28 +01:00 committed by GitHub
commit adf901d581
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 610 additions and 117 deletions

2
ci.nix
View File

@ -3,6 +3,6 @@ let
in {
test = pkgs.writeScript "test" ''
#!/bin/sh
nix-build "${toString ./tests/test.nix}";
nix-build "${toString ./tests}";
'';
}

View File

@ -4,7 +4,8 @@ with builtins;
let
helper.find-device = device: let
helper.find-device = device:
let
environment = helper.device-id device;
in
# DEVICE points already to /dev/disk, so we don't handle it via /dev/disk/by-path
@ -38,6 +39,13 @@ let
};
};
config.zfs_filesystem = q: x: {
fileSystems.${x.mountpoint} = {
device = q.device;
fsType = "zfs";
};
};
config.devices = q: x:
foldl' recursiveUpdate { } (mapAttrsToList (name: config-f { device = "/dev/${name}"; }) x.content);
@ -45,10 +53,10 @@ let
boot.initrd.luks.devices.${x.name}.device = q.device;
} // config-f { device = "/dev/mapper/${x.name}"; } x.content;
config.lv = q: x:
config-f { device = "/dev/mapper/${q.vgname}-${q.name}"; } x.content;
config.lvm_lv = q: x:
config-f { device = "/dev/${q.vgname}/${q.name}"; } x.content;
config.lvm = q: x:
config.lvm_vg = q: x:
foldl' recursiveUpdate { } (mapAttrsToList (name: config-f { inherit name; vgname = x.name; }) x.lvs);
config.noop = q: x: { };
@ -62,14 +70,30 @@ let
create-f = q: x: create.${x.type} q x;
create.filesystem = q: x: ''
mkfs.${x.format} ${q.device}
create.btrfs = q: x: ''
mkfs.btrfs ${q.device}
${lib.optionalString (!isNull x.subvolumes or null) ''
MNTPOINT=$(mktemp -d)
(
mount ${q.device} "$MNTPOINT"
trap 'umount $MNTPOINT; rm -rf $MNTPOINT' EXIT
${concatMapStringsSep "\n" (subvolume: "btrfs subvolume create \"$MNTPOINT\"/${subvolume}") x.subvolumes}
)
''}
'';
create.devices = q: x: let
raid-devices = lib.filterAttrs (_: dev: dev.type == "mdadm") x.content;
other-devices = lib.filterAttrs (_: dev: dev.type != "mdadm") x.content;
in ''
create.filesystem = q: x: ''
mkfs.${x.format} \
${lib.optionalString (!isNull x.extraArgs or null) x.extraArgs} \
${q.device}
'';
create.devices = q: x:
let
raid-devices = lib.filterAttrs (_: dev: dev.type == "mdadm" || dev.type == "zpool" || dev.type == "lvm_vg") x.content;
other-devices = lib.filterAttrs (_: dev: dev.type != "mdadm" && dev.type != "zpool" && dev.type != "lvm_vg") x.content;
in
''
${concatStrings (mapAttrsToList (name: create-f { device = "/dev/${name}"; }) other-devices)}
${concatStrings (mapAttrsToList (name: create-f { device = "/dev/${name}"; name = name; }) raid-devices)}
'';
@ -91,22 +115,33 @@ let
${create-f { device = "/dev/mapper/${x.name}"; } x.content}
'';
create.lv = q: x: ''
lvcreate ${if hasInfix "%" x.size then "-l" else "-L"} ${x.size} -n ${q.name} ${q.vgname}
${create-f { device = "/dev/mapper/${q.vgname}-${q.name}"; } x.content}
create.lvm_pv = q: x: ''
pvcreate ${q.device}
LVMDEVICES_${x.vg}="''${LVMDEVICES_${x.vg}:-}${q.device} "
'';
create.lvm = q: x: ''
pvcreate ${q.device}
vgcreate ${x.name} ${q.device}
${concatStrings (mapAttrsToList (name: create-f { inherit name; vgname = x.name; }) x.lvs)}
create.lvm_lv = q: x: ''
lvcreate \
${if hasInfix "%" x.size then "-l" else "-L"} ${x.size} \
-n ${q.name} \
${lib.optionalString (!isNull x.lvm_type or null) "--type=${x.lvm_type}"} \
${lib.optionalString (!isNull x.extraArgs or null) x.extraArgs} \
${q.vgname}
${create-f { device = "/dev/${q.vgname}/${q.name}"; } x.content}
'';
create.lvm_vg = q: x: ''
vgcreate ${q.name} $LVMDEVICES_${q.name}
${concatStrings (mapAttrsToList (name: create-f { inherit name; vgname = q.name; }) x.lvs)}
'';
create.noop = q: x: "";
create.partition = q: x: let
create.partition = q: x:
let
env = helper.device-id q.device;
in ''
in
''
parted -s "''${${env}}" mkpart ${x.part-type} ${x.fs-type or ""} ${x.start} ${x.end}
# ensure /dev/disk/by-path/..-partN exists before continuing
udevadm trigger --subsystem-match=block; udevadm settle
@ -125,51 +160,98 @@ let
${concatStrings (imap (index: create-f (q // { inherit index; })) x.partitions)}
'';
create.zfs = q: x: ''
ZFSDEVICES_${x.pool}="''${ZFSDEVICES_${x.pool}:-}${q.device} "
'';
create.zfs_filesystem = q: x: ''
zfs create ${q.pool}/${x.name} \
${lib.optionalString (isAttrs x.options or null) (concatStringsSep " " (mapAttrsToList (n: v: "-o ${n}=${v}") x.options))}
'';
create.zfs_volume = q: x: ''
zfs create ${q.pool}/${x.name} \
-V ${x.size} \
${lib.optionalString (isAttrs x.options or null) (concatStringsSep " " (mapAttrsToList (n: v: "-o ${n}=${v}") x.options))}
udevadm trigger --subsystem-match=block; udevadm settle
${create-f { device = "/dev/zvol/${q.pool}/${x.name}"; } x.content}
'';
create.zpool = q: x: ''
zpool create ${q.name} \
${lib.optionalString (!isNull (x.mode or null) && x.mode != "stripe") x.mode} \
${lib.optionalString (isAttrs x.options or null) (concatStringsSep " " (mapAttrsToList (n: v: "-o ${n}=${v}") x.options))} \
${lib.optionalString (isAttrs x.rootFsOptions or null) (concatStringsSep " " (mapAttrsToList (n: v: "-O ${n}=${v}") x.rootFsOptions))} \
''${ZFSDEVICES_${q.name}}
${concatMapStrings (create-f (q // { pool = q.name; })) x.datasets}
'';
mount-f = q: x: mount.${x.type} q x;
mount.filesystem = q: x: {
fs.${x.mountpoint} = ''
if ! findmnt ${q.device} "/mnt${x.mountpoint}" > /dev/null 2>&1; then
mount ${q.device} "/mnt${x.mountpoint}" -o X-mount.mkdir
mount ${q.device} "/mnt${x.mountpoint}" \
-o X-mount.mkdir \
${lib.optionalString (isList x.mountOptions or null) (concatStringsSep " " x.mountOptions)}
fi
'';
};
mount.devices = q: x: let
z = foldl' recursiveUpdate {} (mapAttrsToList (name: mount-f { device = "/dev/${name}"; }) x.content);
mount.zfs_filesystem = q: x:
optionalAttrs ((x.options.mountpoint or "") != "none")
(mount.filesystem (q // { device = q.dataset; }) (x // { mountOptions = [
(lib.optionalString ((x.options.mountpoint or "") != "legacy") "-o zfsutil")
"-t zfs"
]; }));
mount.btrfs = mount.filesystem;
mount.devices = q: x:
let
z = foldl' recursiveUpdate { } (mapAttrsToList (name: mount-f { device = "/dev/${name}"; inherit name; }) x.content);
# attrValues returns values sorted by name. This is important, because it
# ensures that "/" is processed before "/foo" etc.
in ''
in
''
${optionalString (hasAttr "table" z) (concatStringsSep "\n" (attrValues z.table))}
${optionalString (hasAttr "luks" z) (concatStringsSep "\n" (attrValues z.luks))}
${optionalString (hasAttr "lvm" z) (concatStringsSep "\n" (attrValues z.lvm))}
${optionalString (hasAttr "zpool" z) (concatStringsSep "\n" (attrValues z.zpool))}
${optionalString (hasAttr "zfs" z) (concatStringsSep "\n" (attrValues z.zfs))}
${optionalString (hasAttr "fs" z) (concatStringsSep "\n" (attrValues z.fs))}
'';
mount.luks = q: x: (
recursiveUpdate
(mount-f { device = "/dev/mapper/${x.name}"; } x.content)
{luks.${q.device} = ''
cryptsetup luksOpen ${q.device} ${x.name} ${if builtins.hasAttr "keyfile" x then "--key-file " + x.keyfile else ""}
'';}
{
luks.${q.device} = ''
cryptsetup status ${x.name} >/dev/null 2>/dev/null || cryptsetup luksOpen ${q.device} ${x.name} ${if builtins.hasAttr "keyfile" x then "--key-file " + x.keyfile else ""}
'';
}
);
mount.lv = q: x:
mount-f { device = "/dev/mapper/${q.vgname}-${q.name}"; } x.content;
mount.lvm_lv = q: x:
mount-f { device = "/dev/${q.vgname}/${q.name}"; } x.content;
mount.lvm = q: x: (
mount.lvm_vg = q: x: (
recursiveUpdate
(foldl' recursiveUpdate {} (mapAttrsToList (name: mount-f { inherit name; vgname = x.name; }) x.lvs))
{lvm.${q.device} = ''
(foldl' recursiveUpdate { } (mapAttrsToList (name: mount-f { inherit name; vgname = q.name; }) x.lvs))
{
lvm.${q.device} = ''
vgchange -a y
'';}
'';
}
);
mount.lvm_pv = mount.noop;
mount.noop = q: x: { };
# TODO maybe we need to do something here?
mount.mdadm = mount.noop;
mount.mdadm = q: x:
mount-f { device = "/dev/md/${q.name}"; } x.content;
mount.mdraid = mount.noop;
mount.partition = q: x:
@ -180,7 +262,43 @@ let
(foldl' recursiveUpdate { } (imap (index: mount-f (q // { inherit index; device = helper.device-id q.device; })) x.partitions))
{ table.${q.device} = helper.find-device q.device; }
);
in {
mount.zfs = mount.noop;
mount.zpool = q: x:
let
datasets = [{
inherit (q) name;
type = "zfs_filesystem";
dataset = q.name;
mountpoint = x.mountpoint or "/${q.name}";
options = q.rootFsOptions or { };
}] ++ x.datasets;
in
recursiveUpdate
(foldl' recursiveUpdate { }
(
(map
(x: mount-f
({
dataset = x.dataset or "${q.name}/${x.name}";
mountpoint = x.mountpoint or "/${q.name}/${x.name}";
} // q)
x)
datasets)
)
)
{
zpool.${q.device} = ''
zpool list '${q.name}' >/dev/null 2>/dev/null || zpool import '${q.name}'
'';
};
mount.zfs_volume = q: x:
mount-f { device = "/dev/zvol/${q.dataset}"; } x.content;
in
{
config = config-f { };
create = cfg: ''
set -efux

View File

@ -0,0 +1,26 @@
{
type = "devices";
content = {
vdb = {
type = "table";
format = "gpt";
partitions = [
{
type = "partition";
part-type = "primary";
start = "0%";
end = "100%";
content = {
type = "btrfs";
mountpoint = "/";
subvolumes = [
"/home"
"/test"
];
};
}
];
};
};
}

81
example/luks-lvm.nix Normal file
View File

@ -0,0 +1,81 @@
{
type = "devices";
content = {
vdb = {
type = "table";
format = "gpt";
partitions = [
{
type = "partition";
part-type = "ESP";
start = "1MiB";
end = "100MiB";
fs-type = "FAT32";
bootable = true;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
options = [
"defaults"
];
};
}
{
type = "partition";
part-type = "primary";
start = "100MiB";
end = "100%";
content = {
type = "luks";
algo = "aes-xts...";
name = "crypted";
keyfile = "/tmp/secret.key";
extraArgs = [
"--hash sha512"
"--iter-time 5000"
];
content = {
type = "lvm_pv";
vg = "pool";
};
};
}
];
};
pool = {
type = "lvm_vg";
lvs = {
root = {
type = "lvm_lv";
size = "100M";
mountpoint = "/";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
options = [
"defaults"
];
};
};
home = {
type = "lvm_lv";
size = "10M";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/home";
};
};
raw = {
type = "lvm_lv";
size = "10M";
content = {
type = "noop";
};
};
};
};
};
}

66
example/lvm-raid.nix Normal file
View File

@ -0,0 +1,66 @@
{
type = "devices";
content = {
vdb = {
type = "table";
format = "gpt";
partitions = [
{
type = "partition";
part-type = "primary";
start = "0%";
end = "100%";
content = {
type = "lvm_pv";
vg = "pool";
};
}
];
};
vdc = {
type = "table";
format = "gpt";
partitions = [
{
type = "partition";
part-type = "primary";
start = "0%";
end = "100%";
content = {
type = "lvm_pv";
vg = "pool";
};
}
];
};
pool = {
type = "lvm_vg";
lvs = {
root = {
type = "lvm_lv";
size = "100M";
mountpoint = "/";
lvm_type = "mirror";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
options = [
"defaults"
];
};
};
home = {
type = "lvm_lv";
size = "10M";
lvm_type = "raid0";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/home";
};
};
};
};
};
}

View File

@ -53,7 +53,6 @@
};
}
];
};
};
};

View File

@ -0,0 +1,42 @@
{
type = "devices";
content = {
vdb = {
type = "table";
format = "gpt";
partitions = [
{
type = "partition";
# leave space for the grub aka BIOS boot
start = "0%";
end = "100%";
part-type = "primary";
bootable = true;
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
}
];
};
vdc = {
type = "zfs";
pool = "zroot";
};
zroot = {
type = "zpool";
mountpoint = "/";
datasets = [
{
type = "zfs_filesystem";
name = "zfs_fs";
mountpoint = "/zfs_fs";
options."com.sun:auto-snapshot" = "true";
}
];
};
};
}

53
example/zfs.nix Normal file
View File

@ -0,0 +1,53 @@
{
type = "devices";
content = {
vdb = {
type = "zfs";
pool = "zroot";
};
vdc = {
type = "zfs";
pool = "zroot";
};
zroot = {
type = "zpool";
mode = "mirror";
rootFsOptions = {
compression = "lz4";
"com.sun:auto-snapshot" = "false";
};
mountpoint = "/";
datasets = [
{
type = "zfs_filesystem";
name = "zfs_fs";
mountpoint = "/zfs_fs";
options."com.sun:auto-snapshot" = "true";
}
{
type = "zfs_filesystem";
name = "zfs_unmounted_fs";
options.mountpoint = "none";
}
{
type = "zfs_filesystem";
name = "zfs_legacy_fs";
options.mountpoint = "legacy";
mountpoint = "/zfs_legacy_fs";
}
{
type = "zfs_volume";
name = "zfs_testvolume";
size = "10M";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/ext4onzfs";
};
}
];
};
};
}

View File

@ -9,11 +9,11 @@
};
checks.x86_64-linux = let
pkgs = nixpkgs.legacyPackages.x86_64-linux;
in {
in
# Run tests: nix flake check -L
nixos-test = pkgs.callPackage ./tests/test.nix {
import ./tests {
inherit pkgs;
makeTest = import (pkgs.path + "/nixos/tests/make-test-python.nix");
};
};
};
}

View File

@ -0,0 +1,11 @@
{ pkgs ? (import <nixpkgs> { })
, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
}:
makeDiskoTest {
disko-config = import ../example/btrfs-subvolumes.nix;
extraTestScript = ''
machine.succeed("test -e /mnt/test");
machine.succeed("btrfs subvolume list /mnt | grep -qs 'path test$'");
'';
}

16
tests/default.nix Normal file
View File

@ -0,0 +1,16 @@
{ makeTest ? import <nixpkgs/nixos/tests/make-test-python.nix>
, pkgs ? (import <nixpkgs> { })
}@args:
let
lib = pkgs.lib;
makeDiskoTest = (pkgs.callPackage ./lib.nix { inherit makeTest; }).makeDiskoTest;
allTestFilenames =
builtins.map (lib.removeSuffix ".nix") (
builtins.filter
(x: lib.hasSuffix ".nix" x && x != "default.nix" && x != "lib.nix")
(lib.attrNames (builtins.readDir ./.))
);
allTests = lib.genAttrs (allTestFilenames) (test: import (./. + "/${test}.nix") { inherit makeDiskoTest; });
in
allTests

48
tests/lib.nix Normal file
View File

@ -0,0 +1,48 @@
{ pkgs ? (import <nixpkgs> { })
, makeTest ? import <nixpkgs/nixos/tests/make-test-python.nix>
, ...
}:
{
makeDiskoTest =
{ disko-config
, extraTestScript
, extraConfig ? { }
}:
let
lib = pkgs.lib;
makeTest' = args:
makeTest args {
inherit pkgs;
inherit (pkgs) system;
};
tsp-create = pkgs.writeScript "create" ((pkgs.callPackage ../. { }).create disko-config);
tsp-mount = pkgs.writeScript "mount" ((pkgs.callPackage ../. { }).mount disko-config);
num-disks = builtins.length (builtins.filter (x: builtins.match "vd." x == [ ]) (lib.attrNames disko-config.content));
in
makeTest' {
name = "disko";
nodes.machine =
{ config, pkgs, modulesPath, ... }:
{
imports = [
(modulesPath + "/profiles/installation-device.nix")
(modulesPath + "/profiles/base.nix")
];
# speed-up eval
documentation.enable = false;
virtualisation.emptyDiskImages = builtins.genList (_: 512) num-disks;
} // extraConfig;
testScript = ''
machine.succeed("echo 'secret' > /tmp/secret.key");
machine.succeed("${tsp-create}");
machine.succeed("${tsp-mount}");
machine.succeed("${tsp-mount}"); # verify that the command is idempotent
${extraTestScript}
'';
};
}

10
tests/luks-lvm.nix Normal file
View File

@ -0,0 +1,10 @@
{ pkgs ? (import <nixpkgs> { })
, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
}:
makeDiskoTest {
disko-config = import ../example/luks-lvm.nix;
extraTestScript = ''
machine.succeed("cryptsetup isLuks /dev/vdb2");
machine.succeed("mountpoint /mnt/home");
'';
}

12
tests/lvm-raid.nix Normal file
View File

@ -0,0 +1,12 @@
{ pkgs ? (import <nixpkgs> { })
, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
}:
makeDiskoTest {
disko-config = import ../example/lvm-raid.nix;
extraTestScript = ''
machine.succeed("mountpoint /mnt/home");
'';
extraConfig = {
boot.kernelModules = [ "dm-raid" "dm-mirror" ];
};
}

10
tests/mdadm.nix Normal file
View File

@ -0,0 +1,10 @@
{ pkgs ? (import <nixpkgs> { })
, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
}:
makeDiskoTest {
disko-config = import ../example/mdadm.nix;
extraTestScript = ''
machine.succeed("test -b /dev/md/raid1");
machine.succeed("mountpoint /mnt/raid");
'';
}

View File

@ -1,38 +0,0 @@
{ makeTest ? import <nixpkgs/nixos/tests/make-test-python.nix>
, pkgs ? (import <nixpkgs> {})
}:
let
makeTest' = args:
makeTest args {
inherit pkgs;
inherit (pkgs) system;
};
disko-config = import ../example/raid.nix;
tsp-create = pkgs.writeScript "create" ((pkgs.callPackage ../. {}).create disko-config);
tsp-mount = pkgs.writeScript "mount" ((pkgs.callPackage ../. {}).mount disko-config);
in makeTest' {
name = "disko";
nodes.machine =
{ config, pkgs, modulesPath, ... }:
{
imports = [
(modulesPath + "/profiles/installation-device.nix")
(modulesPath + "/profiles/base.nix")
];
# speed-up eval
documentation.enable = false;
virtualisation.emptyDiskImages = [ 512 512 ];
};
testScript = ''
machine.succeed("echo 'secret' > /tmp/secret.key");
machine.succeed("${tsp-create}");
machine.succeed("${tsp-mount}");
machine.succeed("${tsp-mount}"); # verify that the command is idempotent
machine.succeed("test -b /dev/md/raid1");
'';
}

12
tests/zfs-over-legacy.nix Normal file
View File

@ -0,0 +1,12 @@
{ pkgs ? (import <nixpkgs> { })
, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
}:
makeDiskoTest {
disko-config = import ../example/zfs-over-legacy.nix;
extraTestScript = ''
machine.succeed("test -e /mnt/zfs_fs");
machine.succeed("mountpoint /mnt");
machine.succeed("mountpoint /mnt/zfs_fs");
'';
}

27
tests/zfs.nix Normal file
View File

@ -0,0 +1,27 @@
{ pkgs ? (import <nixpkgs> { })
, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
}:
makeDiskoTest {
disko-config = import ../example/zfs.nix;
extraTestScript = ''
machine.succeed("test -b /dev/zvol/zroot/zfs_testvolume");
def assert_property(ds, property, expected_value):
out = machine.succeed(f"zfs get -H {property} {ds} -o value").rstrip()
assert (
out == expected_value
), f"Expected {property}={expected_value} on {ds}, got: {out}"
assert_property("zroot", "compression", "lz4")
assert_property("zroot/zfs_fs", "compression", "lz4")
assert_property("zroot", "com.sun:auto-snapshot", "false")
assert_property("zroot/zfs_fs", "com.sun:auto-snapshot", "true")
assert_property("zroot/zfs_testvolume", "volsize", "10M")
assert_property("zroot/zfs_unmounted_fs", "mountpoint", "none")
machine.succeed("mountpoint /mnt");
machine.succeed("mountpoint /mnt/zfs_fs");
machine.succeed("mountpoint /mnt/zfs_legacy_fs");
machine.succeed("mountpoint /mnt/ext4onzfs");
'';
}