git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #59179 from JohnAZoidberg/cassandra-module

Fix Cassandra, improve config and tests

commitfadceeb07589c901e96cd565a87a7bc68fa9e891 authorAaron Andersen <aaron@fosslib.net> dateThu 13 Jun 2019, 20:37 (7 years ago) parent7815c860350312
4 files changed+275−53
nixos/modules/services/databases/cassandra.nix+191−19 nixos/tests/all-tests.nix+1 nixos/tests/cassandra.nix+69−27 pkgs/servers/nosql/cassandra/generic.nix+14−7
nixos/modules/services/databases/cassandra.nix+191−19
@@ -8,18 +8,21 @@ let
88 cassandraConfig = flip recursiveUpdate cfg.extraConfig
99 ({ commitlog_sync = "batch";
1010 commitlog_sync_batch_window_in_ms = 2;
11+ start_native_transport = cfg.allowClients;
12+ cluster_name = cfg.clusterName;
1113 partitioner = "org.apache.cassandra.dht.Murmur3Partitioner";
1214 endpoint_snitch = "SimpleSnitch";
13− seed_provider =
14− [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
15− parameters = [ { seeds = "127.0.0.1"; } ];
16− }];
1715 data_file_directories = [ "${cfg.homeDir}/data" ];
1816 commitlog_directory = "${cfg.homeDir}/commitlog";
1917 saved_caches_directory = "${cfg.homeDir}/saved_caches";
20− } // (if builtins.compareVersions cfg.package.version "3" >= 0
21− then { hints_directory = "${cfg.homeDir}/hints"; }
22− else {})
18+ } // (lib.optionalAttrs (cfg.seedAddresses != []) {
19+ seed_provider = [{
20+ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
21+ parameters = [ { seeds = concatStringsSep "," cfg.seedAddresses; } ];
22+ }];
23+ }) // (lib.optionalAttrs (lib.versionAtLeast cfg.package.version "3") {
24+ hints_directory = "${cfg.homeDir}/hints";
25+ })
2326 );
2427 cassandraConfigWithAddresses = cassandraConfig //
2528 ( if cfg.listenAddress == null
@@ -39,15 +42,42 @@ let
3942 mkdir -p "$out"
4043
4144 echo "$cassandraYaml" > "$out/cassandra.yaml"
42− ln -s "$cassandraEnvPkg" "$out/cassandra-env.sh"
4345 ln -s "$cassandraLogbackConfig" "$out/logback.xml"
46+
47+ cp "$cassandraEnvPkg" "$out/cassandra-env.sh"
48+
49+ # Delete default JMX Port, otherwise we can't set it using env variable
50+ sed -i '/JMX_PORT="7199"/d' "$out/cassandra-env.sh"
51+
52+ # Delete default password file
53+ sed -i '/-Dcom.sun.management.jmxremote.password.file=\/etc\/cassandra\/jmxremote.password/d' "$out/cassandra-env.sh"
4454 '';
4555 };
56+ defaultJmxRolesFile = builtins.foldl'
57+ (left: right: left + right) ""
58+ (map (role: "${role.username} ${role.password}") cfg.jmxRoles);
59+ fullJvmOptions = cfg.jvmOpts
60+ ++ lib.optionals (cfg.jmxRoles != []) [
61+ "-Dcom.sun.management.jmxremote.authenticate=true"
62+ "-Dcom.sun.management.jmxremote.password.file=${cfg.jmxRolesFile}"
63+ ]
64+ ++ lib.optionals cfg.remoteJmx [
65+ "-Djava.rmi.server.hostname=${cfg.rpcAddress}"
66+ ];
4667in {
4768 options.services.cassandra = {
4869 enable = mkEnableOption ''
4970 Apache Cassandra – Scalable and highly available database.
5071 '';
72+ clusterName = mkOption {
73+ type = types.str;
74+ default = "NixOS Test Cluster";
75+ description = ''
76+ The name of the cluster.
77+ This setting prevents nodes in one logical cluster from joining
78+ another. All nodes in a cluster must have the same value.
79+ '';
80+ };
5181 user = mkOption {
5282 type = types.str;
5383 default = defaultUser;
@@ -162,6 +192,28 @@ in {
162192 XML logback configuration for cassandra
163193 '';
164194 };
195+ seedAddresses = mkOption {
196+ type = types.listOf types.str;
197+ default = [ "127.0.0.1" ];
198+ description = ''
199+ The addresses of hosts designated as contact points in the cluster. A
200+ joining node contacts one of the nodes in the seeds list to learn the
201+ topology of the ring.
202+ Set to 127.0.0.1 for a single node cluster.
203+ '';
204+ };
205+ allowClients = mkOption {
206+ type = types.bool;
207+ default = true;
208+ description = ''
209+ Enables or disables the native transport server (CQL binary protocol).
210+ This server uses the same address as the <literal>rpcAddress</literal>,
211+ but the port it uses is not <literal>rpc_port</literal> but
212+ <literal>native_transport_port</literal>. See the official Cassandra
213+ docs for more information on these variables and set them using
214+ <literal>extraConfig</literal>.
215+ '';
216+ };
165217 extraConfig = mkOption {
166218 type = types.attrs;
167219 default = {};
@@ -178,11 +230,11 @@ in {
178230 example = literalExample "null";
179231 description = ''
180232 Set the interval how often full repairs are run, i.e.
181− `nodetool repair --full` is executed. See
233+ <literal>nodetool repair --full</literal> is executed. See
182234 https://cassandra.apache.org/doc/latest/operating/repair.html
183235 for more information.
184236
185− Set to `null` to disable full repairs.
237+ Set to <literal>null</literal> to disable full repairs.
186238 '';
187239 };
188240 fullRepairOptions = mkOption {
@@ -199,11 +251,11 @@ in {
199251 example = literalExample "null";
200252 description = ''
201253 Set the interval how often incremental repairs are run, i.e.
202− `nodetool repair` is executed. See
254+ <literal>nodetool repair<literal> is executed. See
203255 https://cassandra.apache.org/doc/latest/operating/repair.html
204256 for more information.
205257
206− Set to `null` to disable incremental repairs.
258+ Set to <literal>null</literal> to disable incremental repairs.
207259 '';
208260 };
209261 incrementalRepairOptions = mkOption {
@@ -214,20 +266,135 @@ in {
214266 Options passed through to the incremental repair command.
215267 '';
216268 };
269+ maxHeapSize = mkOption {
270+ type = types.nullOr types.string;
271+ default = null;
272+ example = "4G";
273+ description = ''
274+ Must be left blank or set together with heapNewSize.
275+ If left blank a sensible value for the available amount of RAM and CPU
276+ cores is calculated.
277+
278+ Override to set the amount of memory to allocate to the JVM at
279+ start-up. For production use you may wish to adjust this for your
280+ environment. MAX_HEAP_SIZE is the total amount of memory dedicated
281+ to the Java heap. HEAP_NEWSIZE refers to the size of the young
282+ generation.
283+
284+ The main trade-off for the young generation is that the larger it
285+ is, the longer GC pause times will be. The shorter it is, the more
286+ expensive GC will be (usually).
287+ '';
288+ };
289+ heapNewSize = mkOption {
290+ type = types.nullOr types.string;
291+ default = null;
292+ example = "800M";
293+ description = ''
294+ Must be left blank or set together with heapNewSize.
295+ If left blank a sensible value for the available amount of RAM and CPU
296+ cores is calculated.
297+
298+ Override to set the amount of memory to allocate to the JVM at
299+ start-up. For production use you may wish to adjust this for your
300+ environment. HEAP_NEWSIZE refers to the size of the young
301+ generation.
302+
303+ The main trade-off for the young generation is that the larger it
304+ is, the longer GC pause times will be. The shorter it is, the more
305+ expensive GC will be (usually).
306+
307+ The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent pause
308+ times. If in doubt, and if you do not particularly want to tweak, go with
309+ 100 MB per physical CPU core.
310+ '';
311+ };
312+ mallocArenaMax = mkOption {
313+ type = types.nullOr types.int;
314+ default = null;
315+ example = 4;
316+ description = ''
317+ Set this to control the amount of arenas per-thread in glibc.
318+ '';
319+ };
320+ remoteJmx = mkOption {
321+ type = types.bool;
322+ default = false;
323+ description = ''
324+ Cassandra ships with JMX accessible *only* from localhost.
325+ To enable remote JMX connections set to true.
326+
327+ Be sure to also enable authentication and/or TLS.
328+ See: https://wiki.apache.org/cassandra/JmxSecurity
329+ '';
330+ };
331+ jmxPort = mkOption {
332+ type = types.int;
333+ default = 7199;
334+ description = ''
335+ Specifies the default port over which Cassandra will be available for
336+ JMX connections.
337+ For security reasons, you should not expose this port to the internet.
338+ Firewall it if needed.
339+ '';
340+ };
341+ jmxRoles = mkOption {
342+ default = [];
343+ description = ''
344+ Roles that are allowed to access the JMX (e.g. nodetool)
345+ BEWARE: The passwords will be stored world readable in the nix-store.
346+ It's recommended to use your own protected file using
347+ <literal>jmxRolesFile</literal>
348+
349+ Doesn't work in versions older than 3.11 because they don't like that
350+ it's world readable.
351+ '';
352+ type = types.listOf (types.submodule {
353+ options = {
354+ username = mkOption {
355+ type = types.string;
356+ description = "Username for JMX";
357+ };
358+ password = mkOption {
359+ type = types.string;
360+ description = "Password for JMX";
361+ };
362+ };
363+ });
364+ };
365+ jmxRolesFile = mkOption {
366+ type = types.nullOr types.path;
367+ default = if (lib.versionAtLeast cfg.package.version "3.11")
368+ then pkgs.writeText "jmx-roles-file" defaultJmxRolesFile
369+ else null;
370+ example = "/var/lib/cassandra/jmx.password";
371+ description = ''
372+ Specify your own jmx roles file.
373+
374+ Make sure the permissions forbid "others" from reading the file if
375+ you're using Cassandra below version 3.11.
376+ '';
377+ };
217378 };
218379
219380 config = mkIf cfg.enable {
220381 assertions =
221− [ { assertion =
222− (cfg.listenAddress == null || cfg.listenInterface == null)
223− && !(cfg.listenAddress == null && cfg.listenInterface == null);
382+ [ { assertion = (cfg.listenAddress == null) != (cfg.listenInterface == null);
224383 message = "You have to set either listenAddress or listenInterface";
225384 }
226− { assertion =
227− (cfg.rpcAddress == null || cfg.rpcInterface == null)
228− && !(cfg.rpcAddress == null && cfg.rpcInterface == null);
385+ { assertion = (cfg.rpcAddress == null) != (cfg.rpcInterface == null);
229386 message = "You have to set either rpcAddress or rpcInterface";
230387 }
388+ { assertion = (cfg.maxHeapSize == null) == (cfg.heapNewSize == null);
389+ message = "If you set either of maxHeapSize or heapNewSize you have to set both";
390+ }
391+ { assertion = cfg.remoteJmx -> cfg.jmxRolesFile != null;
392+ message = ''
393+ If you want JMX available remotely you need to set a password using
394+ <literal>jmxRoles</literal> or <literal>jmxRolesFile</literal> if
395+ using Cassandra older than v3.11.
396+ '';
397+ }
231398 ];
232399 users = mkIf (cfg.user == defaultUser) {
233400 extraUsers."${defaultUser}" =
@@ -245,7 +412,12 @@ in {
245412 after = [ "network.target" ];
246413 environment =
247414 { CASSANDRA_CONF = "${cassandraEtc}";
248− JVM_OPTS = builtins.concatStringsSep " " cfg.jvmOpts;
415+ JVM_OPTS = builtins.concatStringsSep " " fullJvmOptions;
416+ MAX_HEAP_SIZE = toString cfg.maxHeapSize;
417+ HEAP_NEWSIZE = toString cfg.heapNewSize;
418+ MALLOC_ARENA_MAX = toString cfg.mallocArenaMax;
419+ LOCAL_JMX = if cfg.remoteJmx then "no" else "yes";
420+ JMX_PORT = toString cfg.jmxPort;
249421 };
250422 wantedBy = [ "multi-user.target" ];
251423 serviceConfig =
nixos/tests/all-tests.nix+1
@@ -36,6 +36,7 @@ in
3636 borgbackup = handleTest ./borgbackup.nix {};
3737 buildbot = handleTest ./buildbot.nix {};
3838 cadvisor = handleTestOn ["x86_64-linux"] ./cadvisor.nix {};
39+ cassandra = handleTest ./cassandra.nix {};
3940 ceph = handleTestOn ["x86_64-linux"] ./ceph.nix {};
4041 certmgr = handleTest ./certmgr.nix {};
4142 cfssl = handleTestOn ["x86_64-linux"] ./cfssl.nix {};
nixos/tests/cassandra.nix+69−27
@@ -1,26 +1,43 @@
1−import ./make-test.nix ({ pkgs, ...}:
1+import ./make-test.nix ({ pkgs, lib, ... }:
22let
33 # Change this to test a different version of Cassandra:
44 testPackage = pkgs.cassandra;
5− cassandraCfg =
5+ clusterName = "NixOS Automated-Test Cluster";
6+
7+ testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
8+ jmxRoles = [{ username = "me"; password = "password"; }];
9+ jmxRolesFile = ./cassandra-jmx-roles;
10+ jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
11+
12+ # Would usually be assigned to 512M
13+ numMaxHeapSize = "400";
14+ getHeapLimitCommand = ''
15+ nodetool info | grep "^Heap Memory" | awk \'{print $NF}\'
16+ '';
17+ checkHeapLimitCommand = ''
18+ [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
19+ '';
20+
21+ cassandraCfg = ipAddress:
622 { enable = true;
7− listenAddress = null;
8− listenInterface = "eth1";
9− rpcAddress = null;
10− rpcInterface = "eth1";
11− extraConfig =
12− { start_native_transport = true;
13− seed_provider =
14− [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
15− parameters = [ { seeds = "cass0"; } ];
16− }];
17− };
23+ inherit clusterName;
24+ listenAddress = ipAddress;
25+ rpcAddress = ipAddress;
26+ seedAddresses = [ "192.168.1.1" ];
1827 package = testPackage;
28+ maxHeapSize = "${numMaxHeapSize}M";
29+ heapNewSize = "100M";
1930 };
20− nodeCfg = extra: {pkgs, config, ...}:
31+ nodeCfg = ipAddress: extra: {pkgs, config, ...}:
2132 { environment.systemPackages = [ testPackage ];
22− networking.firewall.enable = false;
23− services.cassandra = cassandraCfg // extra;
33+ networking = {
34+ firewall.allowedTCPPorts = [ 7000 7199 9042 ];
35+ useDHCP = false;
36+ interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
37+ { address = ipAddress; prefixLength = 24; }
38+ ];
39+ };
40+ services.cassandra = cassandraCfg ipAddress // extra;
2441 virtualisation.memorySize = 1024;
2542 };
2643in
@@ -28,40 +45,65 @@ in
2845 name = "cassandra-ci";
2946
3047 nodes = {
31− cass0 = nodeCfg {};
32− cass1 = nodeCfg {};
33− cass2 = nodeCfg { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
48+ cass0 = nodeCfg "192.168.1.1" {};
49+ cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; });
50+ cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
3451 };
3552
3653 testScript = ''
37− subtest "timers exist", sub {
54+ # Check configuration
55+ subtest "Timers exist", sub {
3856 $cass0->succeed("systemctl list-timers | grep cassandra-full-repair.timer");
3957 $cass0->succeed("systemctl list-timers | grep cassandra-incremental-repair.timer");
4058 };
41− subtest "can connect via cqlsh", sub {
59+ subtest "Can connect via cqlsh", sub {
4260 $cass0->waitForUnit("cassandra.service");
4361 $cass0->waitUntilSucceeds("nc -z cass0 9042");
4462 $cass0->succeed("echo 'show version;' | cqlsh cass0");
4563 };
46− subtest "nodetool is operational", sub {
64+ subtest "Nodetool is operational", sub {
4765 $cass0->waitForUnit("cassandra.service");
4866 $cass0->waitUntilSucceeds("nc -z localhost 7199");
4967 $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass0'");
5068 };
51− subtest "bring up cluster", sub {
69+ subtest "Cluster name was set", sub {
70+ $cass0->waitForUnit("cassandra.service");
71+ $cass0->waitUntilSucceeds("nc -z localhost 7199");
72+ $cass0->waitUntilSucceeds("nodetool describecluster | grep 'Name: ${clusterName}'");
73+ };
74+ subtest "Heap limit set correctly", sub {
75+ # Nodetool takes a while until it can display info
76+ $cass0->waitUntilSucceeds('nodetool info');
77+ $cass0->succeed('${checkHeapLimitCommand}');
78+ };
79+
80+ # Check cluster interaction
81+ subtest "Bring up cluster", sub {
5282 $cass1->waitForUnit("cassandra.service");
53− $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2");
83+ $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN' | grep 2");
5484 $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
5585 };
56− subtest "break and fix node", sub {
86+ '' + lib.optionalString testRemoteAuth ''
87+ subtest "Remote authenticated jmx", sub {
88+ # Doesn't work if not enabled
89+ $cass0->waitUntilSucceeds("nc -z localhost 7199");
90+ $cass1->fail("nc -z 192.168.1.1 7199");
91+ $cass1->fail("nodetool -h 192.168.1.1 status");
92+
93+ # Works if enabled
94+ $cass1->waitUntilSucceeds("nc -z localhost 7199");
95+ $cass0->succeed("nodetool -h 192.168.1.2 ${jmxAuthArgs} status");
96+ };
97+ '' + ''
98+ subtest "Break and fix node", sub {
5799 $cass1->block;
58100 $cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep -c '^DN[[:space:]]+cass1'");
59101 $cass0->succeed("nodetool status | egrep -c '^UN' | grep 1");
60102 $cass1->unblock;
61− $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2");
103+ $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN' | grep 2");
62104 $cass0->succeed("nodetool status | egrep -c '^UN' | grep 2");
63105 };
64− subtest "replace crashed node", sub {
106+ subtest "Replace crashed node", sub {
65107 $cass1->crash;
66108 $cass2->waitForUnit("cassandra.service");
67109 $cass0->waitUntilFails("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
pkgs/servers/nosql/cassandra/generic.nix+14−7
@@ -23,7 +23,7 @@ stdenv.mkDerivation rec {
2323 url = "mirror://apache/cassandra/${version}/apache-${name}-bin.tar.gz";
2424 };
2525
26− nativeBuildInputs = [ makeWrapper ];
26+ nativeBuildInputs = [ makeWrapper coreutils ];
2727
2828 installPhase = ''
2929 mkdir $out
@@ -51,8 +51,17 @@ stdenv.mkDerivation rec {
5151 bin/sstablescrub \
5252 bin/sstableupgrade \
5353 bin/sstableutil \
54− bin/sstableverify \
55− tools/bin/cassandra-stress \
54+ bin/sstableverify; do
55+ # Check if file exists because some don't exist across all versions
56+ if [ -f $out/$cmd ]; then
57+ wrapProgram $out/bin/$(basename "$cmd") \
58+ --suffix-each LD_LIBRARY_PATH : ${libPath} \
59+ --prefix PATH : ${binPath} \
60+ --set JAVA_HOME ${jre}
61+ fi
62+ done
63+
64+ for cmd in tools/bin/cassandra-stress \
5665 tools/bin/cassandra-stressd \
5766 tools/bin/sstabledump \
5867 tools/bin/sstableexpiredblockers \
@@ -62,11 +71,9 @@ stdenv.mkDerivation rec {
6271 tools/bin/sstablerepairedset \
6372 tools/bin/sstablesplit \
6473 tools/bin/token-generator; do
65−
66− # check if file exists because some bin tools don't exist across all
67− # cassandra versions
74+ # Check if file exists because some don't exist across all versions
6875 if [ -f $out/$cmd ]; then
69− makeWrapper $out/$cmd $out/bin/$(${coreutils}/bin/basename "$cmd") \
76+ makeWrapper $out/$cmd $out/bin/$(basename "$cmd") \
7077 --suffix-each LD_LIBRARY_PATH : ${libPath} \
7178 --prefix PATH : ${binPath} \
7279 --set JAVA_HOME ${jre}