git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #72060 from lopsided98/sanoid-init

sanoid: add package, NixOS module and test

commit637bb9fa984238f2f7e6235ca11a9f887601aa52 authorSilvan Mosberger <contact@infinisil.com> dateMon 10 Feb 2020, 01:28 (7 years ago) parent6169eef7684537
7 files changed+551
nixos/modules/module-list.nix+2 nixos/modules/services/backup/sanoid.nix+213 nixos/modules/services/backup/syncoid.nix+168 nixos/tests/all-tests.nix+1 nixos/tests/sanoid.nix+90 pkgs/tools/backup/sanoid/default.nix+75 pkgs/top-level/all-packages.nix+2
nixos/modules/module-list.nix+2
@@ -229,6 +229,8 @@
229229 ./services/backup/restic.nix
230230 ./services/backup/restic-rest-server.nix
231231 ./services/backup/rsnapshot.nix
232+ ./services/backup/sanoid.nix
233+ ./services/backup/syncoid.nix
232234 ./services/backup/tarsnap.nix
233235 ./services/backup/tsm.nix
234236 ./services/backup/zfs-replication.nix
nixos/modules/services/backup/sanoid.nixnew file+213
@@ -0,0 +1,213 @@
1+{ config, lib, pkgs, ... }:
2+
3+with lib;
4+
5+let
6+ cfg = config.services.sanoid;
7+
8+ datasetSettingsType = with types;
9+ (attrsOf (nullOr (oneOf [ str int bool (listOf str) ]))) // {
10+ description = "dataset/template options";
11+ };
12+
13+ # Default values from https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf
14+
15+ commonOptions = {
16+ hourly = mkOption {
17+ description = "Number of hourly snapshots.";
18+ type = types.ints.unsigned;
19+ default = 48;
20+ };
21+
22+ daily = mkOption {
23+ description = "Number of daily snapshots.";
24+ type = types.ints.unsigned;
25+ default = 90;
26+ };
27+
28+ monthly = mkOption {
29+ description = "Number of monthly snapshots.";
30+ type = types.ints.unsigned;
31+ default = 6;
32+ };
33+
34+ yearly = mkOption {
35+ description = "Number of yearly snapshots.";
36+ type = types.ints.unsigned;
37+ default = 0;
38+ };
39+
40+ autoprune = mkOption {
41+ description = "Whether to automatically prune old snapshots.";
42+ type = types.bool;
43+ default = true;
44+ };
45+
46+ autosnap = mkOption {
47+ description = "Whether to automatically take snapshots.";
48+ type = types.bool;
49+ default = true;
50+ };
51+
52+ settings = mkOption {
53+ description = ''
54+ Free-form settings for this template/dataset. See
55+ <link xlink:href="https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf"/>
56+ for allowed values.
57+ '';
58+ type = datasetSettingsType;
59+ };
60+ };
61+
62+ commonConfig = config: {
63+ settings = {
64+ hourly = mkDefault config.hourly;
65+ daily = mkDefault config.daily;
66+ monthly = mkDefault config.monthly;
67+ yearly = mkDefault config.yearly;
68+ autoprune = mkDefault config.autoprune;
69+ autosnap = mkDefault config.autosnap;
70+ };
71+ };
72+
73+ datasetOptions = {
74+ useTemplate = mkOption {
75+ description = "Names of the templates to use for this dataset.";
76+ type = (types.listOf (types.enum (attrNames cfg.templates))) // {
77+ description = "list of template names";
78+ };
79+ default = [];
80+ };
81+
82+ recursive = mkOption {
83+ description = "Whether to recursively snapshot dataset children.";
84+ type = types.bool;
85+ default = false;
86+ };
87+
88+ processChildrenOnly = mkOption {
89+ description = "Whether to only snapshot child datasets if recursing.";
90+ type = types.bool;
91+ default = false;
92+ };
93+ };
94+
95+ datasetConfig = config: {
96+ settings = {
97+ use_template = mkDefault config.useTemplate;
98+ recursive = mkDefault config.recursive;
99+ process_children_only = mkDefault config.processChildrenOnly;
100+ };
101+ };
102+
103+ # Extract pool names from configured datasets
104+ pools = unique (map (d: head (builtins.match "([^/]+).*" d)) (attrNames cfg.datasets));
105+
106+ configFile = let
107+ mkValueString = v:
108+ if builtins.isList v then concatStringsSep "," v
109+ else generators.mkValueStringDefault {} v;
110+
111+ mkKeyValue = k: v: if v == null then ""
112+ else generators.mkKeyValueDefault { inherit mkValueString; } "=" k v;
113+ in generators.toINI { inherit mkKeyValue; } cfg.settings;
114+
115+ configDir = pkgs.writeTextDir "sanoid.conf" configFile;
116+
117+in {
118+
119+ # Interface
120+
121+ options.services.sanoid = {
122+ enable = mkEnableOption "Sanoid ZFS snapshotting service";
123+
124+ interval = mkOption {
125+ type = types.str;
126+ default = "hourly";
127+ example = "daily";
128+ description = ''
129+ Run sanoid at this interval. The default is to run hourly.
130+
131+ The format is described in
132+ <citerefentry><refentrytitle>systemd.time</refentrytitle>
133+ <manvolnum>7</manvolnum></citerefentry>.
134+ '';
135+ };
136+
137+ datasets = mkOption {
138+ type = types.attrsOf (types.submodule ({ config, ... }: {
139+ options = commonOptions // datasetOptions;
140+ config = mkMerge [ (commonConfig config) (datasetConfig config) ];
141+ }));
142+ default = {};
143+ description = "Datasets to snapshot.";
144+ };
145+
146+ templates = mkOption {
147+ type = types.attrsOf (types.submodule ({ config, ... }: {
148+ options = commonOptions;
149+ config = commonConfig config;
150+ }));
151+ default = {};
152+ description = "Templates for datasets.";
153+ };
154+
155+ settings = mkOption {
156+ type = types.attrsOf datasetSettingsType;
157+ description = ''
158+ Free-form settings written directly to the config file. See
159+ <link xlink:href="https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf"/>
160+ for allowed values.
161+ '';
162+ };
163+
164+ extraArgs = mkOption {
165+ type = types.listOf types.str;
166+ default = [];
167+ example = [ "--verbose" "--readonly" "--debug" ];
168+ description = ''
169+ Extra arguments to pass to sanoid. See
170+ <link xlink:href="https://github.com/jimsalterjrs/sanoid/#sanoid-command-line-options"/>
171+ for allowed options.
172+ '';
173+ };
174+ };
175+
176+ # Implementation
177+
178+ config = mkIf cfg.enable {
179+ services.sanoid.settings = mkMerge [
180+ (mapAttrs' (d: v: nameValuePair ("template_" + d) v.settings) cfg.templates)
181+ (mapAttrs (d: v: v.settings) cfg.datasets)
182+ ];
183+
184+ systemd.services.sanoid = {
185+ description = "Sanoid snapshot service";
186+ serviceConfig = {
187+ ExecStartPre = map (pool: lib.escapeShellArgs [
188+ "+/run/booted-system/sw/bin/zfs" "allow"
189+ "sanoid" "snapshot,mount,destroy" pool
190+ ]) pools;
191+ ExecStart = lib.escapeShellArgs ([
192+ "${pkgs.sanoid}/bin/sanoid"
193+ "--cron"
194+ "--configdir" configDir
195+ ] ++ cfg.extraArgs);
196+ ExecStopPost = map (pool: lib.escapeShellArgs [
197+ "+/run/booted-system/sw/bin/zfs" "unallow" "sanoid" pool
198+ ]) pools;
199+ User = "sanoid";
200+ Group = "sanoid";
201+ DynamicUser = true;
202+ RuntimeDirectory = "sanoid";
203+ CacheDirectory = "sanoid";
204+ };
205+ # Prevents missing snapshots during DST changes
206+ environment.TZ = "UTC";
207+ after = [ "zfs.target" ];
208+ startAt = cfg.interval;
209+ };
210+ };
211+
212+ meta.maintainers = with maintainers; [ lopsided98 ];
213+ }
nixos/modules/services/backup/syncoid.nixnew file+168
@@ -0,0 +1,168 @@
1+{ config, lib, pkgs, ... }:
2+
3+with lib;
4+
5+let
6+ cfg = config.services.syncoid;
7+in {
8+
9+ # Interface
10+
11+ options.services.syncoid = {
12+ enable = mkEnableOption "Syncoid ZFS synchronization service";
13+
14+ interval = mkOption {
15+ type = types.str;
16+ default = "hourly";
17+ example = "*-*-* *:15:00";
18+ description = ''
19+ Run syncoid at this interval. The default is to run hourly.
20+
21+ The format is described in
22+ <citerefentry><refentrytitle>systemd.time</refentrytitle>
23+ <manvolnum>7</manvolnum></citerefentry>.
24+ '';
25+ };
26+
27+ user = mkOption {
28+ type = types.str;
29+ default = "root";
30+ example = "backup";
31+ description = ''
32+ The user for the service. Sudo or ZFS privilege delegation must be
33+ configured to use a user other than root.
34+ '';
35+ };
36+
37+ sshKey = mkOption {
38+ type = types.nullOr types.path;
39+ # Prevent key from being copied to store
40+ apply = mapNullable toString;
41+ default = null;
42+ description = ''
43+ SSH private key file to use to login to the remote system. Can be
44+ overridden in individual commands.
45+ '';
46+ };
47+
48+ commonArgs = mkOption {
49+ type = types.listOf types.str;
50+ default = [];
51+ example = [ "--no-sync-snap" ];
52+ description = ''
53+ Arguments to add to every syncoid command, unless disabled for that
54+ command. See
55+ <link xlink:href="https://github.com/jimsalterjrs/sanoid/#syncoid-command-line-options"/>
56+ for available options.
57+ '';
58+ };
59+
60+ commands = mkOption {
61+ type = types.attrsOf (types.submodule ({ name, ... }: {
62+ options = {
63+ source = mkOption {
64+ type = types.str;
65+ example = "pool/dataset";
66+ description = ''
67+ Source ZFS dataset. Can be either local or remote. Defaults to
68+ the attribute name.
69+ '';
70+ };
71+
72+ target = mkOption {
73+ type = types.str;
74+ example = "user@server:pool/dataset";
75+ description = ''
76+ Target ZFS dataset. Can be either local
77+ (<replaceable>pool/dataset</replaceable>) or remote
78+ (<replaceable>user@server:pool/dataset</replaceable>).
79+ '';
80+ };
81+
82+ recursive = mkOption {
83+ type = types.bool;
84+ default = false;
85+ description = ''
86+ Whether to also transfer child datasets.
87+ '';
88+ };
89+
90+ sshKey = mkOption {
91+ type = types.nullOr types.path;
92+ # Prevent key from being copied to store
93+ apply = mapNullable toString;
94+ description = ''
95+ SSH private key file to use to login to the remote system.
96+ Defaults to <option>services.syncoid.sshKey</option> option.
97+ '';
98+ };
99+
100+ sendOptions = mkOption {
101+ type = types.separatedString " ";
102+ default = "";
103+ example = "Lc e";
104+ description = ''
105+ Advanced options to pass to zfs send. Options are specified
106+ without their leading dashes and separated by spaces.
107+ '';
108+ };
109+
110+ recvOptions = mkOption {
111+ type = types.separatedString " ";
112+ default = "";
113+ example = "ux recordsize o compression=lz4";
114+ description = ''
115+ Advanced options to pass to zfs recv. Options are specified
116+ without their leading dashes and separated by spaces.
117+ '';
118+ };
119+
120+ useCommonArgs = mkOption {
121+ type = types.bool;
122+ default = true;
123+ description = ''
124+ Whether to add the configured common arguments to this command.
125+ '';
126+ };
127+
128+ extraArgs = mkOption {
129+ type = types.listOf types.str;
130+ default = [];
131+ example = [ "--sshport 2222" ];
132+ description = "Extra syncoid arguments for this command.";
133+ };
134+ };
135+ config = {
136+ source = mkDefault name;
137+ sshKey = mkDefault cfg.sshKey;
138+ };
139+ }));
140+ default = {};
141+ example."pool/test".target = "root@target:pool/test";
142+ description = "Syncoid commands to run.";
143+ };
144+ };
145+
146+ # Implementation
147+
148+ config = mkIf cfg.enable {
149+ systemd.services.syncoid = {
150+ description = "Syncoid ZFS synchronization service";
151+ script = concatMapStringsSep "\n" (c: lib.escapeShellArgs
152+ ([ "${pkgs.sanoid}/bin/syncoid" ]
153+ ++ (optionals c.useCommonArgs cfg.commonArgs)
154+ ++ (optional c.recursive "-r")
155+ ++ (optionals (c.sshKey != null) [ "--sshkey" c.sshKey ])
156+ ++ c.extraArgs
157+ ++ [ "--sendoptions" c.sendOptions
158+ "--recvoptions" c.recvOptions
159+ c.source c.target
160+ ])) (attrValues cfg.commands);
161+ after = [ "zfs.target" ];
162+ serviceConfig.User = cfg.user;
163+ startAt = cfg.interval;
164+ };
165+ };
166+
167+ meta.maintainers = with maintainers; [ lopsided98 ];
168+ }
nixos/tests/all-tests.nix+1
@@ -257,6 +257,7 @@ in
257257 runInMachine = handleTest ./run-in-machine.nix {};
258258 rxe = handleTest ./rxe.nix {};
259259 samba = handleTest ./samba.nix {};
260+ sanoid = handleTest ./sanoid.nix {};
260261 sddm = handleTest ./sddm.nix {};
261262 shiori = handleTest ./shiori.nix {};
262263 signal-desktop = handleTest ./signal-desktop.nix {};
nixos/tests/sanoid.nixnew file+90
@@ -0,0 +1,90 @@
1+import ./make-test-python.nix ({ pkgs, ... }: let
2+ inherit (import ./ssh-keys.nix pkgs)
3+ snakeOilPrivateKey snakeOilPublicKey;
4+
5+ commonConfig = { pkgs, ... }: {
6+ virtualisation.emptyDiskImages = [ 2048 ];
7+ boot.supportedFilesystems = [ "zfs" ];
8+ environment.systemPackages = [ pkgs.parted ];
9+ };
10+in {
11+ name = "sanoid";
12+ meta = with pkgs.stdenv.lib.maintainers; {
13+ maintainers = [ lopsided98 ];
14+ };
15+
16+ nodes = {
17+ source = { ... }: {
18+ imports = [ commonConfig ];
19+ networking.hostId = "daa82e91";
20+
21+ programs.ssh.extraConfig = ''
22+ UserKnownHostsFile=/dev/null
23+ StrictHostKeyChecking=no
24+ '';
25+
26+ services.sanoid = {
27+ enable = true;
28+ templates.test = {
29+ hourly = 12;
30+ daily = 1;
31+ monthly = 1;
32+ yearly = 1;
33+
34+ autosnap = true;
35+ };
36+ datasets."pool/test".useTemplate = [ "test" ];
37+ };
38+
39+ services.syncoid = {
40+ enable = true;
41+ sshKey = "/root/.ssh/id_ecdsa";
42+ commonArgs = [ "--no-sync-snap" ];
43+ commands."pool/test".target = "root@target:pool/test";
44+ };
45+ };
46+ target = { ... }: {
47+ imports = [ commonConfig ];
48+ networking.hostId = "dcf39d36";
49+
50+ services.openssh.enable = true;
51+ users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
52+ };
53+ };
54+
55+ testScript = ''
56+ source.succeed(
57+ "mkdir /tmp/mnt",
58+ "parted --script /dev/vdb -- mklabel msdos mkpart primary 1024M -1s",
59+ "udevadm settle",
60+ "zpool create pool /dev/vdb1",
61+ "zfs create -o mountpoint=legacy pool/test",
62+ "mount -t zfs pool/test /tmp/mnt",
63+ "udevadm settle",
64+ )
65+ target.succeed(
66+ "parted --script /dev/vdb -- mklabel msdos mkpart primary 1024M -1s",
67+ "udevadm settle",
68+ "zpool create pool /dev/vdb1",
69+ "udevadm settle",
70+ )
71+
72+ source.succeed("mkdir -m 700 /root/.ssh")
73+ source.succeed(
74+ "cat '${snakeOilPrivateKey}' > /root/.ssh/id_ecdsa"
75+ )
76+ source.succeed("chmod 600 /root/.ssh/id_ecdsa")
77+
78+ source.succeed("touch /tmp/mnt/test.txt")
79+ source.systemctl("start --wait sanoid.service")
80+
81+ target.wait_for_open_port(22)
82+ source.systemctl("start --wait syncoid.service")
83+ target.succeed(
84+ "mkdir /tmp/mnt",
85+ "zfs set mountpoint=legacy pool/test",
86+ "mount -t zfs pool/test /tmp/mnt",
87+ )
88+ target.succeed("cat /tmp/mnt/test.txt")
89+ '';
90+})
pkgs/tools/backup/sanoid/default.nixnew file+75
@@ -0,0 +1,75 @@
1+{ lib, stdenv, fetchFromGitHub, fetchpatch, makeWrapper, coreutils, zfs
2+, perlPackages, procps, which, openssh, sudo, mbuffer, pv, lzop, gzip, pigz }:
3+
4+with lib;
5+
6+stdenv.mkDerivation rec {
7+ pname = "sanoid";
8+ version = "2.0.3";
9+
10+ src = fetchFromGitHub {
11+ owner = "jimsalterjrs";
12+ repo = pname;
13+ rev = "v${version}";
14+ sha256 = "1wmymzqg503nmhw8hrblfs67is1l3ljbk2fjvrqwyb01b7mbn80x";
15+ };
16+
17+ patches = [
18+ # Make sanoid look for programs in PATH
19+ (fetchpatch {
20+ url = "https://github.com/jimsalterjrs/sanoid/commit/dc2371775afe08af799d3097d47b48182d1716eb.patch";
21+ sha256 = "16hlwcbcb8h3ar1ywd2bzr3h3whgbcfk6walmp8z6j74wbx81aav";
22+ })
23+ # Make findoid look for programs in PATH
24+ (fetchpatch {
25+ url = "https://github.com/jimsalterjrs/sanoid/commit/44bcd21f269e17765acd1ad0d45161902a205c7b.patch";
26+ sha256 = "0zqyl8q5sfscqcc07acw68ysnlnh3nb57cigjfwbccsm0zwlwham";
27+ })
28+ # Add --cache-dir option
29+ (fetchpatch {
30+ url = "https://github.com/jimsalterjrs/sanoid/commit/a1f5e4c0c006e16a5047a16fc65c9b3663adb81e.patch";
31+ sha256 = "1bb4g2zxrbvf7fvcgzzxsr1cvxzrxg5dzh89sx3h7qlrd6grqhdy";
32+ })
33+ # Add --run-dir option
34+ (fetchpatch {
35+ url = "https://github.com/jimsalterjrs/sanoid/commit/59a07f92b4920952cc9137b03c1533656f48b121.patch";
36+ sha256 = "11v4jhc36v839gppzvhvzp5jd22904k8xqdhhpx6ghl75yyh4f4s";
37+ })
38+ ];
39+
40+ nativeBuildInputs = [ makeWrapper ];
41+ buildInputs = with perlPackages; [ perl ConfigIniFiles CaptureTiny ];
42+
43+ installPhase = ''
44+ mkdir -p "$out/bin"
45+ mkdir -p "$out/etc/sanoid"
46+ cp sanoid.defaults.conf "$out/etc/sanoid/sanoid.defaults.conf"
47+ # Hardcode path to default config
48+ substitute sanoid "$out/bin/sanoid" \
49+ --replace "\$args{'configdir'}/sanoid.defaults.conf" "$out/etc/sanoid/sanoid.defaults.conf"
50+ chmod +x "$out/bin/sanoid"
51+ # Prefer ZFS userspace tools from /run/booted-system/sw/bin to avoid
52+ # incompatibilities with the ZFS kernel module.
53+ wrapProgram "$out/bin/sanoid" \
54+ --prefix PERL5LIB : "$PERL5LIB" \
55+ --prefix PATH : "${makeBinPath [ procps "/run/booted-system/sw" zfs ]}"
56+
57+ install -m755 syncoid "$out/bin/syncoid"
58+ wrapProgram "$out/bin/syncoid" \
59+ --prefix PERL5LIB : "$PERL5LIB" \
60+ --prefix PATH : "${makeBinPath [ openssh procps which pv mbuffer lzop gzip pigz "/run/booted-system/sw" zfs ]}"
61+
62+ install -m755 findoid "$out/bin/findoid"
63+ wrapProgram "$out/bin/findoid" \
64+ --prefix PERL5LIB : "$PERL5LIB" \
65+ --prefix PATH : "${makeBinPath [ "/run/booted-system/sw" zfs ]}"
66+ '';
67+
68+ meta = {
69+ description = "A policy-driven snapshot management tool for ZFS filesystems";
70+ homepage = "https://github.com/jimsalterjrs/sanoid";
71+ license = licenses.gpl3;
72+ maintainers = with maintainers; [ lopsided98 ];
73+ platforms = platforms.all;
74+ };
75+}
pkgs/top-level/all-packages.nix+2
@@ -25344,6 +25344,8 @@ in
2534425344
2534525345 sane-frontends = callPackage ../applications/graphics/sane/frontends.nix { };
2534625346
25347+ sanoid = callPackage ../tools/backup/sanoid { };
25348+
2534725349 satysfi = callPackage ../tools/typesetting/satysfi { };
2534825350
2534925351 sc-controller = pythonPackages.callPackage ../misc/drivers/sc-controller {