Merge pull request #48 from nix-community/module

README.md

@@ -1,80 +1,182 @@
-disko
-=====
+# disko - declarative disk partitioning
 
-nix-powered automatic disk partitioning
+Disko takes the NixOS module system and makes it work for disk partitioning
+as well.
 
-Usage
-=====
+I wanted to write a curses NixOS installer, and that was the first step that I
+hit; the disk formatting is a manual process. Once that's done, the NixOS
+system itself is declarative, but the actual formatting of disks is manual.
 
-Master Boot Record
-------------------
-This is how your iso configuation may look like
+## Features
 
-/etc/nixos/configuration.nix
-```nix
-{ pkgs, modulesPath, ... }:
-let
-  disko = pkgs.callPackage (builtins.fetchGit {
-    url = "https://github.com/nix-community/disko";
-    ref = "master";
-  }) {};
-  cfg = {
-    disk = {
-      sda = {
-        device = "/dev/sda";
-        type = "device";
-        content = {
-          type = "table";
-          format = "msdos";
-          partitions = [
-            {
-              name = "root";
-              type = "partition";
-              part-type = "primary";
-              start = "1M";
-              end = "100%";
-              bootable = true;
-              content = {
-                type = "filesystem";
-                format = "ext4";
-                mountpoint = "/";
-              };
-            }
-          ];
-        };
-      };
+* supports LVM, ZFS, btrfs, GPT, mdadm, ext4, ...
+* supports recursive layouts
+* outputs a NixOS-compatible module
+* CLI
+
+## How-to guides
+
+### NixOS installation
+
+During the NixOS installation process, replace the [Partitioning and
+formatting](https://nixos.org/manual/nixos/stable/index.html#sec-installation-partitioning)
+steps with the following:
+
+1. Find a disk layout in ./examples that you like.
+2. Write the config based on the example and your disk layout.
+4. Run the CLI (`nix run github:nix-community/disko`) to apply the changes.
+5. FIXME: Copy the disko module and disk layout around.
+6. Continue the NixOS installation.
+
+### Using without NixOS
+
+## Reference
+
+### Module options
+
+TODO: link to generated module options
+
+### Examples
+
+./examples
+
+### CLI
+
+TODO: output of the cli --help
+
+## Installing NixOS module
+
+You can use the NixOS module in one of the following ways:
+
+### Flakes
+
+If you use nix flakes support:
+
+``` nix
+{
+  inputs.disko.url = "github:nix-community/disko";
+  inputs.disko.inputs.nixpkgs.follows = "nixpkgs";
+
+  outputs = { self, nixpkgs, disko }: {
+    # change `yourhostname` to your actual hostname
+    nixosConfigurations.yourhostname = nixpkgs.lib.nixosSystem {
+      # change to your system:
+      system = "x86_64-linux";
+      modules = [
+        ./configuration.nix
+        disko.nixosModules.disko
+      ];
     };
   };
-in {
-  imports = [
-    (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix")
-  ];
-  environment.systemPackages = with pkgs;[
-    (pkgs.writeScriptBin "tsp-create" (disko.create cfg))
-    (pkgs.writeScriptBin "tsp-mount" (disko.mount cfg))
-  ];
-  ## Optional: Automatically creates a service which runs at startup to perform the partitioning
-  #systemd.services.install-to-hd = {
-  #  enable = true;
-  #  wantedBy = ["multi-user.target"];
-  #  after = ["getty@tty1.service" ];
-  #  serviceConfig = {
-  #    Type = "oneshot";
-  #    ExecStart = [ (disko.create cfg) (disk.mount cfg) ];
-  #    StandardInput = "null";
-  #    StandardOutput = "journal+console";
-  #    StandardError = "inherit";
-  #  };
-  #};
 }
 ```
 
-After `nixos-rebuild switch` this will add a `tsp-create` and a `tsp-mount`
-command:
+### [niv](https://github.com/nmattia/niv) (Current recommendation)
+  First add it to niv:
 
-- **tsp-create**: creates & formats the partitions according to `tsp-disk.json`
-- **tsp-mount**: mounts the partitions to `/mnt`
+```console
+$ niv add nix-community/disko
+```
 
-GUID Partition Table, LVM and dm-crypt
---------------------------------------
-See `examples/`
+  Then add the following to your configuration.nix in the `imports` list:
+
+```nix
+{
+  imports = [ "${(import ./nix/sources.nix).disko}/modules/disko.nix" ];
+}
+```
+
+### nix-channel
+
+  As root run:
+
+```console
+$ nix-channel --add https://github.com/nix-community/disko/archive/main.tar.gz disko
+$ nix-channel --update
+```
+
+  Then add the following to your configuration.nix in the `imports` list:
+
+```nix
+{
+  imports = [ <disko/modules/disko.nix> ];
+}
+```
+
+### fetchTarball
+
+  Add the following to your configuration.nix:
+
+``` nix
+{
+  imports = [ "${builtins.fetchTarball "https://github.com/nix-community/disko/archive/main.tar.gz"}/modules/disko.nix" ];
+}
+```
+
+  or with pinning:
+
+```nix
+{
+  imports = let
+    # replace this with an actual commit id or tag
+    commit = "f2783a8ef91624b375a3cf665c3af4ac60b7c278";
+  in [ 
+    "${builtins.fetchTarball {
+      url = "https://github.com/nix-community/disko/archive/${commit}.tar.gz";
+      # replace this with an actual hash
+      sha256 = "0000000000000000000000000000000000000000000000000000";
+    }}/module.nix"
+  ];
+}
+```
+
+## Using the NixOS module
+
+```nix
+{
+  # checkout the example folder for how to configure different diska layouts
+  disko.devices = {
+    disk.sda = {
+      device = "/dev/sda";
+      type = "disk";
+      content = {
+        type = "table";
+        format = "gpt";
+        partitions = [
+          {
+            type = "partition";
+            name = "ESP";
+            start = "1MiB";
+            end = "100MiB";
+            bootable = true;
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+            };
+          }
+          {
+            name = "root";
+            type = "partition";
+            start = "100MiB";
+            end = "100%";
+            part-type = "primary";
+            bootable = true;
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+            };
+          }
+        ];
+      };
+    };
+  };
+}
+```
+
+this will configure `fileSystems` and other required NixOS options to boot the specified configuration.
+
+If you are on an installer, you probably want to disable `enableConfig`.
+
+disko will create the scripts `disko-create` and `disko-mount` which can be used to create/mount the configured disk layout.

cli.nix

@@ -0,0 +1,19 @@
+{ pkgs ? import <nixpkgs> {}
+, mode ? "mount"
+, fromFlake ? null
+, diskoFile
+, ... }@args:
+let
+  disko = import ./. { };
+  diskFormat =
+    if fromFlake != null
+    then (builtins.getFlake fromFlake) + "/${diskoFile}"
+    else import diskoFile;
+  diskoEval = if (mode == "create") then
+    disko.createScript diskFormat pkgs
+  else if (mode == "mount") then
+    disko.mountScript diskFormat pkgs
+  else
+    builtins.abort "invalid mode"
+  ;
+in diskoEval

default.nix

@@ -4,17 +4,26 @@ let
   eval = cfg: lib.evalModules {
     modules = lib.singleton {
       # _file = toString input;
-      imports = lib.singleton { topLevel.devices = cfg; };
+      imports = lib.singleton { devices = cfg; };
       options = {
-        topLevel = lib.mkOption {
-          type = types.topLevel;
+        devices = lib.mkOption {
+          type = types.devices;
         };
       };
     };
   };
 in {
   types = types;
-  create = cfg: (eval cfg).config.topLevel.create;
-  mount = cfg: (eval cfg).config.topLevel.mount;
-  config = cfg: (eval cfg).config.topLevel.config;
+  create = cfg: types.diskoLib.create (eval cfg).config.devices;
+  createScript = cfg: pkgs: pkgs.writeScript "disko-create" ''
+    export PATH=${lib.makeBinPath (types.diskoLib.packages (eval cfg).config.devices pkgs)}
+    ${types.diskoLib.create (eval cfg).config.devices}
+  '';
+  mount = cfg: types.diskoLib.mount (eval cfg).config.devices;
+  mountScript = cfg: pkgs: pkgs.writeScript "disko-mount" ''
+    export PATH=${lib.makeBinPath (types.diskoLib.packages (eval cfg).config.devices pkgs)}
+    ${types.diskoLib.mount (eval cfg).config.devices}
+  '';
+  config = cfg: { imports = types.diskoLib.config (eval cfg).config.devices; };
+  packages = cfg: types.diskoLib.packages (eval cfg).config.devices;
 }

disko

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+readonly libexec_dir="${0%/*}"
+
+# a file with the disko config
+declare disko_config
+
+# a flake uri, if present disko config is relative to the flake root
+declare from_flake
+
+# mount was chosen as the default mode because it's less destructive
+mode=mount
+nix_args=()
+
+showUsage() {
+  cat <<USAGE
+Usage: $0 [options] disk-config.nix
+
+Options:
+
+* -m, --mode mode
+  set the mode, either create or mount
+* -f, --flake uri
+  fetch the disko config relative to this flake's root
+* --arg name value
+  pass value to nix-build. can be used to set disk-names for example
+* --argstr name value
+  pass value to nix-build as string
+USAGE
+}
+
+abort() {
+  echo "aborted: $*" >&2
+  exit 1
+}
+
+## Main ##
+
+[[ $# -eq 0 ]] && {
+  showUsage
+  exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -m | --mode)
+      mode=$2
+      shift
+      ;;
+    -f | --flake)
+      from_flake="$2"
+      nix_args+=("--argstr" "fromFlake" "$2")
+      shift
+      ;;
+    --argstr | --arg)
+      nix_args+=("$1" "$2" "$3")
+      shift
+      shift
+      ;;
+    --help)
+      showUsage
+      exit 0
+      ;;
+    *)
+      if [ -z ${disko_config+x} ]; then
+        disko_config=$1
+      else
+        showUsage
+        exit 1
+      fi
+      ;;
+  esac
+  shift
+done
+
+if ! ([[ $mode = "create" ]] || [[ $mode = "mount" ]]); then
+  abort "mode must be either create or mount"
+fi
+
+if [[ -e "${disko_config}" ]]; then
+  nix_args+=("--arg" "diskoFile" "$disko_config")
+elif [[ -n "${from_flake+x}" ]]; then
+  nix_args+=("--argstr" "diskoFile" "$disko_config")
+else
+  abort "disko config must be an exising file of flake must be set"
+fi
+
+script=$(nix-build "${libexec_dir}"/cli.nix \
+  --argstr diskoFile "$disko_config" \
+  --argstr mode "$mode" \
+  "${nix_args[@]}"
+)
+exec "$script"

example/boot-raid1.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" "/dev/vdc" ] }: {
+{ disks ? [ "/dev/vdb" "/dev/vdc" ], ... }: {
   disk = {
     one = {
       type = "disk";

example/btrfs-subvolumes.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" ] }: {
+{ disks ? [ "/dev/vdb" ], ... }: {
   disk = {
     vdb = {
       type = "disk";

example/complex.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" "/dev/vdc" ] }: {
+{ disks ? [ "/dev/vdb" "/dev/vdc" "/dev/vdd" ], ... }: {
   disk = {
     disk0 = {
       type = "disk";

example/gpt-bios-compat.nix

@@ -1,5 +1,5 @@
 # Example to create a bios compatible gpt partition
-{ disks ? [ "/dev/vdb" ] }: {
+{ disks ? [ "/dev/vdb" ], ... }: {
   disk = {
     vdb = {
       device = builtins.elemAt disks 0;

example/luks-lvm.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" ] }: {
+{ disks ? [ "/dev/vdb" ], ... }: {
   disk = {
     vdb = {
       type = "disk";

example/lvm-raid.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" "/dev/vdc" ] }: {
+{ disks ? [ "/dev/vdb" "/dev/vdc" ], ... }: {
   disk = {
     one = {
       type = "disk";

example/mdadm.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" "/dev/vdc" ] }: {
+{ disks ? [ "/dev/vdb" "/dev/vdc" ], ... }: {
   disk = {
     vdb = {
       type = "disk";

example/simple-efi.nix

@@ -0,0 +1,40 @@
+{ disks ? [ "/dev/vdb" ], ... }: {
+  disk = {
+    vdb = {
+      device = builtins.elemAt disks 0;
+      type = "disk";
+      content = {
+        type = "table";
+        format = "gpt";
+        partitions = [
+          {
+            type = "partition";
+            name = "ESP";
+            start = "1MiB";
+            end = "100MiB";
+            bootable = true;
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+            };
+          }
+          {
+            name = "root";
+            type = "partition";
+            start = "100MiB";
+            end = "100%";
+            part-type = "primary";
+            bootable = true;
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+            };
+          }
+        ];
+      };
+    };
+  };
+}
+

example/with-lib.nix

@@ -0,0 +1,35 @@
+# Example to create a bios compatible gpt partition
+{ disks ? [ "/dev/vdb" ], lib, ... }: {
+  disk = lib.traceValSeq (lib.genAttrs [ (lib.head disks) ] (device: {
+    device = device;
+    type = "disk";
+    content = {
+      type = "table";
+      format = "gpt";
+      partitions = [
+        {
+          name = "boot";
+          type = "partition";
+          start = "0";
+          end = "1M";
+          part-type = "primary";
+          flags = ["bios_grub"];
+        }
+        {
+          name = "root";
+          type = "partition";
+          # leave space for the grub aka BIOS boot
+          start = "1M";
+          end = "100%";
+          part-type = "primary";
+          bootable = true;
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+          };
+        }
+      ];
+    };
+  }));
+}

example/zfs-over-legacy.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" "/dev/vdc" ] }: {
+{ disks ? [ "/dev/vdb" "/dev/vdc" ], ... }: {
   disk = {
     vdb = {
       type = "disk";

example/zfs.nix

@@ -1,4 +1,4 @@
-{ disks ? [ "/dev/vdb" "/dev/vdc" ] }: {
+{ disks ? [ "/dev/vdb" "/dev/vdc" ], ... }: {
   disk = {
     x = {
       type = "disk";

flake.lock

@@ -1,27 +0,0 @@
-{
-  "nodes": {
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1660646295,
-        "narHash": "sha256-V4G+egGRc3elXPTr7QLJ7r7yrYed0areIKDiIAlMLC8=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "762b003329510ea855b4097a37511eb19c7077f0",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "root": {
-      "inputs": {
-        "nixpkgs": "nixpkgs"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

flake.nix

@@ -1,12 +1,49 @@
 {
   description = "Description for the project";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  # don't lock to give precedence to a USB live-installer's registry
+  inputs.nixpkgs.url = "nixpkgs";
 
   outputs = { self, nixpkgs, ... }: {
+    nixosModules.disko = import ./module.nix;
     lib = import ./. {
       inherit (nixpkgs) lib;
     };
+    packages.x86_64-linux.disko = let
+      pkgs = nixpkgs.legacyPackages.x86_64-linux;
+      inherit (pkgs) lib;
+      inclFiles = {src, name}: files: lib.cleanSourceWith {
+        inherit src name;
+        filter = _path: _type: _type == "regular"
+          && lib.any (file: builtins.baseNameOf _path == file) files;
+      };
+    in derivation rec{
+      system = "x86_64-linux";
+      name = "disko";
+      builder = "/bin/sh";
+      PATH    = "${pkgs.coreutils}/bin:${pkgs.gnused}/bin";
+      passAsFile = ["buildPhase"];
+      buildPhase = ''
+        mkdir -p $out/bin $out/share/disko
+	cp -r $src/* $out/share/disko
+	sed \
+	  -e "s|libexec_dir=\".*\"|libexec_dir=\"$out/share/disko\"|" \
+	  -e "s|#!/usr/bin/env.*|#!/usr/bin/env bash|" \
+	  $src/disko > $out/bin/disko
+	chmod 755 $out/bin/disko
+      '';
+      args = ["-c" ". $buildPhasePath"];
+      src = inclFiles { inherit name; src = ./.; } [
+        "disko"
+        "cli.nix"
+        "default.nix"
+        "types.nix"
+        "options.nix"
+      ];
+    } // {
+      meta.description = "Format disks with nix-config";
+    };
+    packages.x86_64-linux.default = self.packages.x86_64-linux.disko;
     checks.x86_64-linux = let
       pkgs = nixpkgs.legacyPackages.x86_64-linux;
     in

module.nix

@@ -0,0 +1,43 @@
+{ config, lib, pkgs, ... }:
+let
+  types = import ./types.nix { inherit lib; };
+  cfg = config.disko;
+in {
+  options.disko = {
+    devices = lib.mkOption {
+      type = types.devices;
+    };
+    enableConfig = lib.mkOption {
+      description = ''
+        configure nixos with the specified devices
+        should be true if the system is booted with those devices
+        should be false on an installer image etc.
+      '';
+      type = lib.types.bool;
+      default = true;
+    };
+    addScripts = lib.mkOption {
+      description = ''
+        add disko-create and disko-mount scripts to systemPackages.
+      '';
+      type = lib.types.bool;
+      default = true;
+    };
+  };
+  config = {
+    environment.systemPackages = (lib.optionals cfg.addScripts [
+      (pkgs.writers.writeDashBin "disko-create" ''
+        export PATH=${lib.makeBinPath (types.diskoLib.packages cfg.devices pkgs)}
+        ${types.diskoLib.create cfg.devices}
+      '')
+      (pkgs.writers.writeDashBin "disko-mount" ''
+        export PATH=${lib.makeBinPath (types.diskoLib.packages cfg.devices pkgs)}
+        ${types.diskoLib.mount cfg.devices}
+      '')
+    ]) ++ lib.optionals cfg.enableConfig (types.diskoLib.packages cfg.devices pkgs);
+
+    # Remember to add config keys here if they are added to types
+    fileSystems = lib.mkIf cfg.enableConfig (lib.mkMerge (lib.catAttrs "fileSystems" (types.diskoLib.config cfg.devices)));
+    boot = lib.mkIf cfg.enableConfig (lib.mkMerge (lib.catAttrs "boot" (types.diskoLib.config cfg.devices)));
+  };
+}

tests/boot-raid1.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/boot-raid1.nix;
+  disko-config = ../example/boot-raid1.nix;
   extraTestScript = ''
     machine.succeed("test -b /dev/md/boot");
     machine.succeed("mountpoint /boot");

tests/btrfs-subvolumes.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/btrfs-subvolumes.nix;
+  disko-config = ../example/btrfs-subvolumes.nix;
   extraTestScript = ''
     machine.succeed("test -e /test");
     machine.succeed("btrfs subvolume list / | grep -qs 'path test$'");

tests/cli.nix

@@ -0,0 +1,28 @@
+{ pkgs ? (import <nixpkgs> { })
+, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
+}:
+makeDiskoTest {
+  disko-config = ../example/complex.nix;
+  extraConfig = {
+    fileSystems."/zfs_legacy_fs".options = [ "nofail" ]; # TODO find out why we need this!
+  };
+  testMode = "cli";
+  extraTestScript = ''
+    machine.succeed("test -b /dev/zroot/zfs_testvolume");
+    machine.succeed("test -b /dev/md/raid1p1");
+
+
+    machine.succeed("mountpoint /zfs_fs");
+    machine.succeed("mountpoint /zfs_legacy_fs");
+    machine.succeed("mountpoint /ext4onzfs");
+    machine.succeed("mountpoint /ext4_on_lvm");
+  '';
+  enableOCR = true;
+  bootCommands = ''
+    machine.wait_for_text("Passphrase for")
+    machine.send_chars("secret\n")
+  '';
+  extraConfig = {
+    boot.kernelModules = [ "dm-raid" "dm-mirror" ];
+  };
+}

tests/complex.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/complex.nix;
+  disko-config = ../example/complex.nix;
   extraConfig = {
     fileSystems."/zfs_legacy_fs".options = [ "nofail" ]; # TODO find out why we need this!
   };

tests/gpt-bios-compat.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/gpt-bios-compat.nix;
+  disko-config = ../example/gpt-bios-compat.nix;
   extraTestScript = ''
     machine.succeed("mountpoint /");
   '';

tests/lib.nix

@@ -12,6 +12,7 @@
     , grub-devices ? [ "nodev" ]
     , efi ? true
     , enableOCR ? false
+    , testMode ? "direct" # can be one of direct module cli
     }:
     let
       lib = pkgs.lib;
@@ -21,13 +22,21 @@
           inherit (pkgs) system;
         };
       disks = [ "/dev/vda" "/dev/vdb" "/dev/vdc" "/dev/vdd" "/dev/vde" "/dev/vdf" ];
-      tsp-create = pkgs.writeScript "create" ((pkgs.callPackage ../. { }).create (disko-config { disks = builtins.tail disks; }));
-      tsp-mount = pkgs.writeScript "mount" ((pkgs.callPackage ../. { }).mount (disko-config { disks = builtins.tail disks; }));
-      tsp-config = (pkgs.callPackage ../. { }).config (disko-config { inherit disks; });
-      num-disks = builtins.length (lib.attrNames (disko-config {}).disk);
+      tsp-create = pkgs.writeScript "create" ((pkgs.callPackage ../. { }).create (import disko-config { disks = builtins.tail disks; inherit lib; }));
+      tsp-mount = pkgs.writeScript "mount" ((pkgs.callPackage ../. { }).mount (import disko-config { disks = builtins.tail disks; inherit lib; }));
+      tsp-config = (pkgs.callPackage ../. { }).config (import disko-config { inherit disks; inherit lib; });
+      num-disks = builtins.length (lib.attrNames (import disko-config { inherit lib; }).disk);
       installed-system = { modulesPath, ... }: {
         imports = [
-          tsp-config
+          (lib.optionalAttrs (testMode == "direct" || testMode == "cli") tsp-config)
+          (lib.optionalAttrs (testMode == "module") {
+            imports = [ ../module.nix ];
+            disko = {
+              addScripts = false;
+              enableConfig = true;
+              devices = import disko-config { inherit disks lib; };
+            };
+          })
           (modulesPath + "/testing/test-instrumentation.nix")
           (modulesPath + "/profiles/qemu-guest.nix")
           (modulesPath + "/profiles/minimal.nix")
@@ -63,6 +72,21 @@
       inherit enableOCR;
       nodes.machine = { config, pkgs, modulesPath, ... }: {
         imports = [
+          (lib.optionalAttrs (testMode == "module") {
+            imports = [ ../module.nix ];
+            disko = {
+              addScripts = true;
+              enableConfig = false;
+              devices = import disko-config { disks = builtins.tail disks; inherit lib; };
+            };
+          })
+          (lib.optionalAttrs (testMode == "cli") {
+            imports = [ (modulesPath + "/installer/cd-dvd/channel.nix") ];
+            system.extraDependencies = [
+              ((pkgs.callPackage ../. { }).createScript (import disko-config { disks = builtins.tail disks; inherit lib; }) pkgs)
+              ((pkgs.callPackage ../. { }).mountScript (import disko-config { disks = builtins.tail disks; inherit lib; }) pkgs)
+            ];
+          })
           (modulesPath + "/profiles/base.nix")
           (modulesPath + "/profiles/minimal.nix")
           extraConfig
@@ -97,9 +121,25 @@
 
         machine.start()
         machine.succeed("echo -n 'secret' > /tmp/secret.key")
-        machine.succeed("${tsp-create}")
-        machine.succeed("${tsp-mount}")
-        machine.succeed("${tsp-mount}") # verify that the command is idempotent
+        ${lib.optionalString (testMode == "direct") ''
+          machine.succeed("${tsp-create}")
+          machine.succeed("${tsp-mount}")
+          machine.succeed("${tsp-mount}") # verify that the command is idempotent
+        ''}
+        ${lib.optionalString (testMode == "module") ''
+          machine.succeed("disko-create")
+          machine.succeed("disko-mount")
+          machine.succeed("disko-mount") # verify that the command is idempotent
+        ''}
+        ${lib.optionalString (testMode == "cli") ''
+          # TODO use the disko cli here
+          # machine.succeed("${../.}/disko --no-pkgs --mode create ${disko-config}")
+          # machine.succeed("${../.}/disko --no-pkgs --mode mount ${disko-config}")
+          # machine.succeed("${../.}/disko --no-pkgs --mode mount ${disko-config}") # verify that the command is idempotent
+          machine.succeed("${tsp-create}")
+          machine.succeed("${tsp-mount}")
+          machine.succeed("${tsp-mount}") # verify that the command is idempotent
+        ''}
 
         # mount nix-store in /mnt
         machine.succeed("mkdir -p /mnt/nix/store")

tests/luks-lvm.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/luks-lvm.nix;
+  disko-config = ../example/luks-lvm.nix;
   extraTestScript = ''
     machine.succeed("cryptsetup isLuks /dev/vda2");
     machine.succeed("mountpoint /home");

tests/lvm-raid.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/lvm-raid.nix;
+  disko-config = ../example/lvm-raid.nix;
   extraTestScript = ''
     machine.succeed("mountpoint /home");
   '';

tests/mdadm.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/mdadm.nix;
+  disko-config = ../example/mdadm.nix;
   extraTestScript = ''
     machine.succeed("test -b /dev/md/raid1");
     machine.succeed("mountpoint /");

tests/module.nix

@@ -0,0 +1,28 @@
+{ pkgs ? (import <nixpkgs> { })
+, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
+}:
+makeDiskoTest {
+  disko-config = ../example/complex.nix;
+  extraConfig = {
+    fileSystems."/zfs_legacy_fs".options = [ "nofail" ]; # TODO find out why we need this!
+  };
+  testMode = "module";
+  extraTestScript = ''
+    machine.succeed("test -b /dev/zroot/zfs_testvolume");
+    machine.succeed("test -b /dev/md/raid1p1");
+
+
+    machine.succeed("mountpoint /zfs_fs");
+    machine.succeed("mountpoint /zfs_legacy_fs");
+    machine.succeed("mountpoint /ext4onzfs");
+    machine.succeed("mountpoint /ext4_on_lvm");
+  '';
+  enableOCR = true;
+  bootCommands = ''
+    machine.wait_for_text("Passphrase for")
+    machine.send_chars("secret\n")
+  '';
+  extraConfig = {
+    boot.kernelModules = [ "dm-raid" "dm-mirror" ];
+  };
+}

tests/simple-efi.nix

@@ -0,0 +1,9 @@
+{ pkgs ? (import <nixpkgs> { })
+, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
+}:
+makeDiskoTest {
+  disko-config = ../example/simple-efi.nix;
+  extraTestScript = ''
+    machine.succeed("mountpoint /");
+  '';
+}

tests/with-lib.nix

@@ -0,0 +1,11 @@
+{ pkgs ? (import <nixpkgs> { })
+, makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
+}:
+makeDiskoTest {
+  disko-config = ../example/with-lib.nix;
+  extraTestScript = ''
+    machine.succeed("mountpoint /");
+  '';
+  efi = false;
+  grub-devices = [ "/dev/vdb" ];
+}

tests/zfs-over-legacy.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/zfs-over-legacy.nix;
+  disko-config = ../example/zfs-over-legacy.nix;
   extraTestScript = ''
     machine.succeed("test -e /zfs_fs");
     machine.succeed("mountpoint /zfs_fs");

tests/zfs.nix

@@ -2,7 +2,7 @@
 , makeDiskoTest ? (pkgs.callPackage ./lib.nix { }).makeDiskoTest
 }:
 makeDiskoTest {
-  disko-config = import ../example/zfs.nix;
+  disko-config = ../example/zfs.nix;
   extraConfig = {
     fileSystems."/zfs_legacy_fs".options = [ "nofail" ]; # TODO find out why we need this!
   };

types.nix

@@ -81,15 +81,15 @@ rec {
       };
     in valueType;
 
-    /* Given a attrset of dependencies and a devices attrset
-       returns a sorted list by dependencies. aborts if a loop is found
+    /* Given a attrset of deviceDependencies and a devices attrset
+       returns a sorted list by deviceDependencies. aborts if a loop is found
 
        sortDevicesByDependencies :: AttrSet -> AttrSet -> [ [ str str ] ]
     */
-    sortDevicesByDependencies = dependencies: devices:
+    sortDevicesByDependencies = deviceDependencies: devices:
       let
         dependsOn = a: b:
-          elem a (attrByPath b [] dependencies);
+          elem a (attrByPath b [] deviceDependencies);
         maybeSortedDevices = toposort dependsOn (diskoLib.deviceList devices);
       in
         if (hasAttr "cycle" maybeSortedDevices) then
@@ -119,6 +119,49 @@ rec {
          => "hello world"
     */
     maybeStr = x: optionalString (!isNull x) x;
+
+    /* Takes a disko device specification, returns an attrset with metadata
+
+       meta :: types.devices -> AttrSet
+    */
+    meta = devices: diskoLib.deepMergeMap (dev: dev._meta) (flatten (map attrValues (attrValues devices)));
+    /* Takes a disko device specification and returns a string which formats the disks
+
+       create :: types.devices -> str
+    */
+    create = devices: let
+      sortedDeviceList = diskoLib.sortDevicesByDependencies (diskoLib.meta devices).deviceDependencies devices;
+    in ''
+      set -efux
+      ${concatStrings (map (dev: attrByPath (dev ++ [ "_create" ]) "" devices) sortedDeviceList)}
+    '';
+    /* Takes a disko device specification and returns a string which mounts the disks
+
+       mount :: types.devices -> str
+    */
+    mount = devices: let
+      fsMounts = diskoLib.deepMergeMap (dev: dev._mount.fs or {}) (flatten (map attrValues (attrValues devices)));
+      sortedDeviceList = diskoLib.sortDevicesByDependencies (diskoLib.meta devices).deviceDependencies devices;
+    in ''
+      set -efux
+      # first create the neccessary devices
+      ${concatStrings (map (dev: attrByPath (dev ++ [ "_mount" "dev" ]) "" devices) sortedDeviceList)}
+
+      # and then mount the filesystems in alphabetical order
+      # attrValues returns values sorted by name.  This is important, because it
+      # ensures that "/" is processed before "/foo" etc.
+      ${concatStrings (attrValues fsMounts)}
+    '';
+    /* Takes a disko device specification and returns a nixos configuration
+
+       config :: types.devices -> nixosConfig
+    */
+    config = devices: flatten (map (dev: dev._config) (flatten (map attrValues (attrValues devices))));
+    /* Takes a disko device specification and returns a function to get the needed packages to format/mount the disks
+
+       packages :: types.devices -> pkgs -> [ derivation ]
+    */
+    packages = devices: pkgs: unique (flatten (map (dev: dev._pkgs pkgs) (flatten (map attrValues (attrValues devices)))));
   };
 
   optionTypes = rec {
@@ -152,79 +195,27 @@ rec {
   };
 
   /* topLevel type of the disko config, takes attrsets of disks mdadms zpools and lvm vgs.
-     exports create, mount, meta and config
   */
-  topLevel = types.submodule ({ config, ... }: {
+  devices = types.submodule {
     options = {
-      devices = {
-        disk = mkOption {
-          type = types.attrsOf disk;
-          default = {};
-        };
-        mdadm = mkOption {
-          type = types.attrsOf mdadm;
-          default = {};
-        };
-        zpool = mkOption {
-          type = types.attrsOf zpool;
-          default = {};
-        };
-        lvm_vg = mkOption {
-          type = types.attrsOf lvm_vg;
-          default = {};
-        };
+      disk = mkOption {
+        type = types.attrsOf disk;
+        default = {};
       };
-      meta = mkOption {
-        readOnly = true;
-        default = diskoLib.deepMergeMap (dev: dev._meta) (flatten (map attrValues [
-          config.devices.disk
-          config.devices.lvm_vg
-          config.devices.mdadm
-          config.devices.zpool
-        ])) // {
-          sortedDeviceList = diskoLib.sortDevicesByDependencies config.meta.dependencies config.devices;
-        };
+      mdadm = mkOption {
+        type = types.attrsOf mdadm;
+        default = {};
       };
-      create = mkOption {
-        readOnly = true;
-        type = types.str;
-        default = ''
-          set -efux
-          ${concatStrings (map (dev: attrByPath (dev ++ [ "_create" ]) "" config.devices) config.meta.sortedDeviceList)}
-        '';
+      zpool = mkOption {
+        type = types.attrsOf zpool;
+        default = {};
       };
-      mount = mkOption {
-        readOnly = true;
-        type = types.str;
-        default = let
-          fsMounts = diskoLib.deepMergeMap (dev: dev._mount.fs or {}) (flatten (map attrValues [
-            config.devices.disk
-            config.devices.lvm_vg
-            config.devices.mdadm
-            config.devices.zpool
-          ]));
-        in ''
-          set -efux
-          # first create the neccessary devices
-          ${concatStrings (map (dev: attrByPath (dev ++ [ "_mount" "dev" ]) "" config.devices) config.meta.sortedDeviceList)}
-
-          # and then mount the filesystems in alphabetical order
-          # attrValues returns values sorted by name.  This is important, because it
-          # ensures that "/" is processed before "/foo" etc.
-          ${concatStrings (attrValues fsMounts)}
-        '';
-      };
-      config = mkOption {
-        readOnly = true;
-        default = { imports = flatten (map (dev: dev._config) (flatten (map attrValues [
-          config.devices.disk
-          config.devices.lvm_vg
-          config.devices.mdadm
-          config.devices.zpool
-        ])));};
+      lvm_vg = mkOption {
+        type = types.attrsOf lvm_vg;
+        default = {};
       };
     };
-  });
+  };
 
   btrfs = types.submodule ({ config, ... }: {
     options = {
@@ -290,6 +281,12 @@ rec {
           };
         }];
       };
+      _pkgs= mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [];
+      };
     };
   });
 
@@ -358,6 +355,22 @@ rec {
           };
         }];
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        # type = types.functionTo (types.listOf types.package);
+        default = pkgs:
+          [ pkgs.util-linux ] ++ (
+            # TODO add many more
+            if (config.format == "xfs") then [ pkgs.xfsprogs ]
+            else if (config.format == "btrfs") then [ pkgs.btrfs-progs ]
+            else if (config.format == "vfat") then [ pkgs.dosfstools ]
+            else if (config.format == "ext2") then [ pkgs.e2fsprogs ]
+            else if (config.format == "ext3") then [ pkgs.e2fsprogs ]
+            else if (config.format == "ext4") then [ pkgs.e2fsprogs ]
+            else []
+          );
+      };
     };
   });
 
@@ -411,6 +424,13 @@ rec {
         default = dev:
           map (partition: partition._config dev) config.partitions;
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs:
+          [ pkgs.parted pkgs.systemdMinimal ] ++ flatten (map (partition: partition._pkgs pkgs) config.partitions);
+      };
     };
   });
 
@@ -497,6 +517,12 @@ rec {
         default = dev:
           optional (!isNull config.content) (config.content._config (diskoLib.deviceNumbering dev config.index));
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: optionals (!isNull config.content) (config.content._pkgs pkgs);
+      };
     };
   });
 
@@ -514,7 +540,7 @@ rec {
         readOnly = true;
         type = types.functionTo diskoLib.jsonType;
         default = dev: {
-          dependencies.lvm_vg.${config.vg} = [ dev ];
+          deviceDependencies.lvm_vg.${config.vg} = [ dev ];
         };
       };
       _create = mkOption {
@@ -538,6 +564,12 @@ rec {
         readOnly = true;
         default = dev: [];
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [ pkgs.lvm2 ];
+      };
     };
   });
 
@@ -591,6 +623,12 @@ rec {
         default =
           map (lv: lv._config config.name) (attrValues config.lvs);
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: flatten (map (lv: lv._pkgs pkgs) (attrValues config.lvs));
+      };
     };
   });
 
@@ -656,6 +694,12 @@ rec {
             })
           ];
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: lib.optionals (!isNull config.content) (config.content._pkgs pkgs);
+      };
     };
   });
 
@@ -673,7 +717,7 @@ rec {
         readOnly = true;
         type = types.functionTo diskoLib.jsonType;
         default = dev: {
-          dependencies.zpool.${config.pool} = [ dev ];
+          deviceDependencies.zpool.${config.pool} = [ dev ];
         };
       };
       _create = mkOption {
@@ -696,6 +740,12 @@ rec {
         readOnly = true;
         default = dev: [];
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [ pkgs.zfs ];
+      };
     };
   });
 
@@ -779,17 +829,22 @@ rec {
       _config = mkOption {
         internal = true;
         readOnly = true;
-        default =
-          [
-            (map (dataset: dataset._config config.name) (attrValues config.datasets))
-            (optional (!isNull config.mountpoint) {
-              fileSystems.${config.mountpoint} = {
-                device = config.name;
-                fsType = "zfs";
-                options = lib.optional ((config.options.mountpoint or "") != "legacy") "zfsutil";
-              };
-            })
-          ];
+        default = [
+          (map (dataset: dataset._config config.name) (attrValues config.datasets))
+          (optional (!isNull config.mountpoint) {
+            fileSystems.${config.mountpoint} = {
+              device = config.name;
+              fsType = "zfs";
+              options = lib.optional ((config.options.mountpoint or "") != "legacy") "zfsutil";
+            };
+          })
+        ];
+      };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [ pkgs.util-linux ] ++ flatten (map (dataset: dataset._pkgs pkgs) (attrValues config.datasets));
       };
     };
   });
@@ -880,6 +935,12 @@ rec {
             };
           });
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [ pkgs.util-linux ] ++ lib.optionals (!isNull config.content) (config.content._pkgs pkgs);
+      };
     };
   });
 
@@ -939,6 +1000,12 @@ rec {
         default =
           optional (!isNull config.content) (config.content._config "/dev/md/${config.name}");
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: (lib.optionals (!isNull config.content) (config.content._pkgs pkgs));
+      };
     };
   });
 
@@ -957,7 +1024,7 @@ rec {
         readOnly = true;
         type = types.functionTo diskoLib.jsonType;
         default = dev: {
-          dependencies.mdadm.${config.name} = [ dev ];
+          deviceDependencies.mdadm.${config.name} = [ dev ];
         };
       };
       _create = mkOption {
@@ -981,6 +1048,12 @@ rec {
         readOnly = true;
         default = dev: [];
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [ pkgs.mdadm ];
+      };
     };
   });
 
@@ -1045,6 +1118,12 @@ rec {
             { boot.initrd.luks.devices.${config.name}.device = dev; }
           ] ++ (optional (!isNull config.content) (config.content._config "/dev/mapper/${config.name}"));
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: [ pkgs.cryptsetup ] ++ (lib.optionals (!isNull config.content) (config.content._pkgs pkgs));
+      };
     };
   });
 
@@ -1087,6 +1166,12 @@ rec {
         default =
           optional (!isNull config.content) (config.content._config config.device);
       };
+      _pkgs = mkOption {
+        internal = true;
+        readOnly = true;
+        type = types.functionTo (types.listOf types.package);
+        default = pkgs: lib.optionals (!isNull config.content) (config.content._pkgs pkgs);
+      };
     };
   });
 }