git.iko.soy/typeable/nixpkgs
on master help sign in
nixos/cockroachdb: create new service

This also includes a full end-to-end CockroachDB clustering test to ensure everything basically works. However, this test is not currently enabled by default, though it can be run manually. See the included comments in the test for more information. Closes #51306. Closes #38665. Co-authored-by: Austin Seipp <aseipp@pobox.com> Signed-off-by: Austin Seipp <aseipp@pobox.com>

commit4226ddc0340cd492ee46f840c21b407bc229cadb authorJohn Boehr <john@johnboehr.is> dateMon 9 Apr 2018, 18:16 (9 years ago) parent4920f0c
4 files changed+350
nixos/modules/misc/ids.nix+2 nixos/modules/module-list.nix+1 nixos/modules/services/databases/cockroachdb.nix+221 nixos/tests/cockroachdb.nix+126
nixos/modules/misc/ids.nix+2
@@ -337,6 +337,7 @@
337337 alerta = 310;
338338 minetest = 311;
339339 rss2email = 312;
340+ cockroachdb = 313;
340341
341342 # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
342343
@@ -634,6 +635,7 @@
634635 alerta = 310;
635636 minetest = 311;
636637 rss2email = 312;
638+ cockroachdb = 313;
637639
638640 # When adding a gid, make sure it doesn't match an existing
639641 # uid. Users and groups with the same name should have equal
nixos/modules/module-list.nix+1
@@ -212,6 +212,7 @@
212212 ./services/databases/aerospike.nix
213213 ./services/databases/cassandra.nix
214214 ./services/databases/clickhouse.nix
215+ ./services/databases/cockroachdb.nix
215216 ./services/databases/couchdb.nix
216217 ./services/databases/firebird.nix
217218 ./services/databases/foundationdb.nix
nixos/modules/services/databases/cockroachdb.nixnew file+221
@@ -0,0 +1,221 @@
1+{ config, lib, pkgs, ... }:
2+
3+with lib;
4+
5+let
6+ cfg = config.services.cockroachdb;
7+ crdb = cfg.package;
8+
9+ escape = builtins.replaceStrings ["%"] ["%%"];
10+ ifNotNull = v: s: optionalString (!isNull v) s;
11+
12+ startupCommand = lib.concatStringsSep " "
13+ [ # Basic startup
14+ "${crdb}/bin/cockroach start"
15+ "--logtostderr"
16+ "--store=${cfg.dataDir}"
17+ (ifNotNull cfg.locality "--locality='${cfg.locality}'")
18+
19+ # WebUI settings
20+ "--http-addr='${cfg.http.address}:${toString cfg.http.port}'"
21+
22+ # Cluster listen address
23+ "--listen-addr='${cfg.listen.address}:${toString cfg.listen.port}'"
24+
25+ # Cluster configuration
26+ (ifNotNull cfg.join "--join=${cfg.join}")
27+
28+ # Cache and memory settings. Must be escaped.
29+ "--cache='${escape cfg.cache}'"
30+ "--max-sql-memory='${escape cfg.maxSqlMemory}'"
31+
32+ # Certificate/security settings.
33+ (if cfg.insecure then "--insecure" else "--certs-dir=${cfg.certsDir}")
34+ ];
35+
36+ addressOption = descr: defaultPort: {
37+ address = mkOption {
38+ type = types.str;
39+ default = "localhost";
40+ description = "Address to bind to for ${descr}";
41+ };
42+
43+ port = mkOption {
44+ type = types.int;
45+ default = defaultPort;
46+ description = "Port to bind to for ${descr}";
47+ };
48+ };
49+in
50+
51+{
52+ options = {
53+ services.cockroachdb = {
54+ enable = mkEnableOption "CockroachDB Server";
55+
56+ listen = addressOption "intra-cluster communication" 26257;
57+
58+ http = addressOption "http-based Admin UI" 8080;
59+
60+ locality = mkOption {
61+ type = types.nullOr types.str;
62+ default = null;
63+ description = ''
64+ An ordered, comma-separated list of key-value pairs that describe the
65+ topography of the machine. Topography might include country,
66+ datacenter or rack designations. Data is automatically replicated to
67+ maximize diversities of each tier. The order of tiers is used to
68+ determine the priority of the diversity, so the more inclusive
69+ localities like country should come before less inclusive localities
70+ like datacenter. The tiers and order must be the same on all nodes.
71+ Including more tiers is better than including fewer. For example:
72+
73+ country=us,region=us-west,datacenter=us-west-1b,rack=12
74+ country=ca,region=ca-east,datacenter=ca-east-2,rack=4
75+
76+ planet=earth,province=manitoba,colo=secondary,power=3
77+ '';
78+ };
79+
80+ join = mkOption {
81+ type = types.nullOr types.str;
82+ default = null;
83+ description = "The addresses for connecting the node to a cluster.";
84+ };
85+
86+ dataDir = mkOption {
87+ type = types.path;
88+ default = "/var/lib/cockroachdb";
89+ description = "Location where CockroachDB stores its table files";
90+ };
91+
92+ insecure = mkOption {
93+ type = types.bool;
94+ default = false;
95+ description = "Run in insecure mode.";
96+ };
97+
98+ certsDir = mkOption {
99+ type = types.nullOr types.path;
100+ default = null;
101+ description = "The path to the certificate directory.";
102+ };
103+
104+ user = mkOption {
105+ type = types.str;
106+ default = "cockroachdb";
107+ description = "User account under which CockroachDB runs";
108+ };
109+
110+ group = mkOption {
111+ type = types.str;
112+ default = "cockroachdb";
113+ description = "User account under which CockroachDB runs";
114+ };
115+
116+ openPorts = mkOption {
117+ type = types.bool;
118+ default = false;
119+ description = "Open firewall ports for cluster communication by default";
120+ };
121+
122+ cache = mkOption {
123+ type = types.str;
124+ default = "25%";
125+ description = ''
126+ The total size for caches.
127+
128+ This can be a percentage, expressed with a fraction sign or as a
129+ decimal-point number, or any bytes-based unit. For example, "25%",
130+ "0.25" both represent 25% of the available system memory. The values
131+ "1000000000" and "1GB" both represent 1 gigabyte of memory.
132+ '';
133+ };
134+
135+ maxSqlMemory = mkOption {
136+ type = types.str;
137+ default = "25%";
138+ description = ''
139+ The maximum in-memory storage capacity available to store temporary
140+ data for SQL queries.
141+
142+ This can be a percentage, expressed with a fraction sign or as a
143+ decimal-point number, or any bytes-based unit. For example, "25%",
144+ "0.25" both represent 25% of the available system memory. The values
145+ "1000000000" and "1GB" both represent 1 gigabyte of memory.
146+ '';
147+ };
148+
149+ package = mkOption {
150+ type = types.package;
151+ default = pkgs.cockroachdb;
152+ description = ''
153+ The CockroachDB derivation to use for running the service.
154+
155+ This would primarily be useful to enable Enterprise Edition features
156+ in your own custom CockroachDB build (Nixpkgs CockroachDB binaries
157+ only contain open source features and open source code).
158+ '';
159+ };
160+ };
161+ };
162+
163+ config = mkIf config.services.cockroachdb.enable {
164+ assertions = [
165+ { assertion = !cfg.insecure -> !(isNull cfg.certsDir);
166+ message = "CockroachDB must have a set of SSL certificates (.certsDir), or run in Insecure Mode (.insecure = true)";
167+ }
168+ ];
169+
170+ environment.systemPackages = [ crdb ];
171+
172+ users.users = optionalAttrs (cfg.user == "cockroachdb") (singleton
173+ { name = "cockroachdb";
174+ description = "CockroachDB Server User";
175+ uid = config.ids.uids.cockroachdb;
176+ group = cfg.group;
177+ });
178+
179+ users.groups = optionalAttrs (cfg.group == "cockroachdb") (singleton
180+ { name = "cockroachdb";
181+ gid = config.ids.gids.cockroachdb;
182+ });
183+
184+ networking.firewall.allowedTCPPorts = lib.optionals cfg.openPorts
185+ [ cfg.http.port cfg.listen.port ];
186+
187+ systemd.services.cockroachdb =
188+ { description = "CockroachDB Server";
189+ documentation = [ "man:cockroach(1)" "https://www.cockroachlabs.com" ];
190+
191+ after = [ "network.target" "time-sync.target" ];
192+ requires = [ "time-sync.target" ];
193+ wantedBy = [ "multi-user.target" ];
194+
195+ unitConfig.RequiresMountsFor = "${cfg.dataDir}";
196+
197+ preStart = ''
198+ if ! test -e ${cfg.dataDir}; then
199+ mkdir -m 0700 -p ${cfg.dataDir}
200+ chown -R ${cfg.user} ${cfg.dataDir}
201+ fi
202+ '';
203+
204+ serviceConfig =
205+ { ExecStart = startupCommand;
206+ Type = "notify";
207+ User = cfg.user;
208+ PermissionsStartOnly = true;
209+
210+ Restart = "always";
211+ TimeoutStopSec="60";
212+ RestartSec="10";
213+ StandardOutput="syslog";
214+ StandardError="syslog";
215+ SyslogIdentifier="cockroach";
216+ };
217+ };
218+ };
219+
220+ meta.maintainers = with lib.maintainers; [ thoughtpolice ];
221+}
nixos/tests/cockroachdb.nixnew file+126
@@ -0,0 +1,126 @@
1+# This performs a full 'end-to-end' test of a multi-node CockroachDB cluster
2+# using the built-in 'cockroach workload' command, to simulate a semi-realistic
3+# test load. It generally takes anywhere from 3-5 minutes to run and 1-2GB of
4+# RAM (though each of 3 workers gets 1GB allocated)
5+#
6+# CockroachDB requires synchronized system clocks within a small error window
7+# (~500ms by default) on each node in order to maintain a multi-node cluster.
8+# Cluster joins that are outside this window will fail, and nodes that skew
9+# outside the window after joining will promptly get kicked out.
10+#
11+# To accomodate this, we use QEMU/virtio infrastructure and load the 'ptp_kvm'
12+# driver inside a guest. This driver allows the host machine to pass its clock
13+# through to the guest as a hardware clock that appears as a Precision Time
14+# Protocol (PTP) Clock device, generally /dev/ptp0. PTP devices can be measured
15+# and used as hardware reference clocks (similar to an on-board GPS clock) by
16+# NTP software. In our case, we use Chrony to synchronize to the reference
17+# clock.
18+#
19+# This test is currently NOT enabled as a continuously-checked NixOS test.
20+# Ideally, this test would be run by Hydra and Borg on all relevant changes,
21+# except:
22+#
23+# - Not every build machine is compatible with the ptp_kvm driver.
24+# Virtualized EC2 instances, for example, do not support loading the ptp_kvm
25+# driver into guests. However, bare metal builders (e.g. Packet) do seem to
26+# work just fine. In practice, this means x86_64-linux builds would fail
27+# randomly, depending on which build machine got the job. (This is probably
28+# worth some investigation; I imagine it's based on ptp_kvm's usage of paravirt
29+# support which may not be available in 'nested' environments.)
30+#
31+# - ptp_kvm is not supported on aarch64, otherwise it seems likely Cockroach
32+# could be tested there, as well. This seems to be due to the usage of
33+# the TSC in ptp_kvm, which isn't supported (easily) on AArch64. (And:
34+# testing stuff, not just making sure it builds, is important to ensure
35+# aarch64 support remains viable.)
36+#
37+# For future developers who are reading this message, are daring and would want
38+# to fix this, some options are:
39+#
40+# - Just test a single node cluster instead (boring and less thorough).
41+# - Move all CI to bare metal packet builders, and we can at least do x86_64-linux.
42+# - Get virtualized clocking working in aarch64, somehow.
43+# - Add a 4th node that acts as an NTP service and uses no PTP clocks for
44+# references, at the client level. This bloats the node and memory
45+# requirements, but would probably allow both aarch64/x86_64 to work.
46+#
47+
48+let
49+
50+ # Creates a node. If 'joinNode' parameter, a string containing an IP address,
51+ # is non-null, then the CockroachDB server will attempt to join/connect to
52+ # the cluster node specified at that address.
53+ makeNode = locality: myAddr: joinNode:
54+ { nodes, pkgs, lib, config, ... }:
55+
56+ {
57+ # Bank/TPC-C benchmarks take some memory to complete
58+ virtualisation.memorySize = 1024;
59+
60+ # Install the KVM PTP "Virtualized Clock" driver. This allows a /dev/ptp0
61+ # device to appear as a reference clock, synchronized to the host clock.
62+ # Because CockroachDB *requires* a time-synchronization mechanism for
63+ # the system time in a cluster scenario, this is necessary to work.
64+ boot.kernelModules = [ "ptp_kvm" ];
65+
66+ # Enable and configure Chrony, using the given virtualized clock passed
67+ # through by KVM.
68+ services.chrony.enable = true;
69+ services.chrony.servers = lib.mkForce [ ];
70+ services.chrony.extraConfig = ''
71+ refclock PHC /dev/ptp0 poll 2 prefer require refid KVM
72+ makestep 0.1 3
73+ '';
74+
75+ # Enable CockroachDB. In order to ensure that Chrony has performed its
76+ # first synchronization at boot-time (which may take ~10 seconds) before
77+ # starting CockroachDB, we block the ExecStartPre directive using the
78+ # 'waitsync' command. This ensures Cockroach doesn't have its system time
79+ # leap forward out of nowhere during startup/execution.
80+ #
81+ # Note that the default threshold for NTP-based skew in CockroachDB is
82+ # ~500ms by default, so making sure it's started *after* accurate time
83+ # synchronization is extremely important.
84+ services.cockroachdb.enable = true;
85+ services.cockroachdb.insecure = true;
86+ services.cockroachdb.openPorts = true;
87+ services.cockroachdb.locality = locality;
88+ services.cockroachdb.listen.address = myAddr;
89+ services.cockroachdb.join = lib.mkIf (joinNode != null) joinNode;
90+
91+ # Hold startup until Chrony has performed its first measurement (which
92+ # will probably result in a full timeskip, thanks to makestep)
93+ systemd.services.cockroachdb.preStart = ''
94+ ${pkgs.chrony}/bin/chronyc waitsync
95+ '';
96+ };
97+
98+in import ./make-test.nix ({ pkgs, ...} : {
99+ name = "cockroachdb";
100+ meta.maintainers = with pkgs.stdenv.lib.maintainers;
101+ [ thoughtpolice ];
102+
103+ nodes = rec {
104+ node1 = makeNode "country=us,region=east,dc=1" "192.168.1.1" null;
105+ node2 = makeNode "country=us,region=west,dc=2b" "192.168.1.2" "192.168.1.1";
106+ node3 = makeNode "country=eu,region=west,dc=2" "192.168.1.3" "192.168.1.1";
107+ };
108+
109+ # NOTE: All the nodes must start in order and you must NOT use startAll, because
110+ # there's otherwise no way to guarantee that node1 will start before the others try
111+ # to join it.
112+ testScript = ''
113+ $node1->start;
114+ $node1->waitForUnit("cockroachdb");
115+
116+ $node2->start;
117+ $node2->waitForUnit("cockroachdb");
118+
119+ $node3->start;
120+ $node3->waitForUnit("cockroachdb");
121+
122+ $node1->mustSucceed("cockroach sql --host=192.168.1.1 --insecure -e 'SHOW ALL CLUSTER SETTINGS' 2>&1");
123+ $node1->mustSucceed("cockroach workload init bank 'postgresql://root\@192.168.1.1:26257?sslmode=disable'");
124+ $node1->mustSucceed("cockroach workload run bank --duration=1m 'postgresql://root\@192.168.1.1:26257?sslmode=disable'");
125+ '';
126+})