@@ -1,121 +1,148 @@
11# This test is very comprehensive. It tests whether all hadoop services work well with each other.
22# Run this when updating the Hadoop package or making significant changes to the hadoop module.
33# For a more basic test, see hdfs.nix and yarn.nix
4−import ../make-test-python.nix ({pkgs, ...}: {
5−
6− nodes = let
7− package = pkgs.hadoop;
8− coreSite = {
9− "fs.defaultFS" = "hdfs://ns1";
10− };
11− hdfsSite = {
12− "dfs.namenode.rpc-bind-host" = "0.0.0.0";
13− "dfs.namenode.http-bind-host" = "0.0.0.0";
14− "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
15−
16− # HA Quorum Journal Manager configuration
17− "dfs.nameservices" = "ns1";
18− "dfs.ha.namenodes.ns1" = "nn1,nn2";
19− "dfs.namenode.shared.edits.dir.ns1.nn1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
20− "dfs.namenode.shared.edits.dir.ns1.nn2" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
21− "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
22− "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
23− "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
24− "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
25− "dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
26− "dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
27−
28− # Automatic failover configuration
29− "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
30− "dfs.ha.automatic-failover.enabled.ns1" = "true";
31− "dfs.ha.fencing.methods" = "shell(true)";
32− "ha.zookeeper.quorum" = "zk1:2181";
33− };
34− yarnSiteHA = {
35− "yarn.resourcemanager.zk-address" = "zk1:2181";
36− "yarn.resourcemanager.ha.enabled" = "true";
37− "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
38− "yarn.resourcemanager.hostname.rm1" = "rm1";
39− "yarn.resourcemanager.hostname.rm2" = "rm2";
40− "yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
41− "yarn.resourcemanager.cluster-id" = "cluster1";
42− # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
43− # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
44− # that causes AM containers to fail otherwise.
45− "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
46− "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
47− };
48− in {
49− zk1 = { ... }: {
50− services.zookeeper.enable = true;
51− networking.firewall.allowedTCPPorts = [ 2181 ];
52− };
53−
54− # HDFS cluster
55− nn1 = {pkgs, options, ...}: {
56− services.hadoop = {
57− inherit package coreSite hdfsSite;
58− hdfs.namenode.enable = true;
59− hdfs.zkfc.enable = true;
4+import ../make-test-python.nix ({ package, ... }: {
5+ name = "hadoop-combined";
6+
7+ nodes =
8+ let
9+ coreSite = {
10+ "fs.defaultFS" = "hdfs://ns1";
11+ };
12+ hdfsSite = {
13+ # HA Quorum Journal Manager configuration
14+ "dfs.nameservices" = "ns1";
15+ "dfs.ha.namenodes.ns1" = "nn1,nn2";
16+ "dfs.namenode.shared.edits.dir.ns1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
17+ "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
18+ "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
19+ "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
20+ "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
21+ "dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
22+ "dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
23+
24+ # Automatic failover configuration
25+ "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
26+ "dfs.ha.automatic-failover.enabled.ns1" = "true";
27+ "dfs.ha.fencing.methods" = "shell(true)";
28+ "ha.zookeeper.quorum" = "zk1:2181";
29+ };
30+ yarnSite = {
31+ "yarn.resourcemanager.zk-address" = "zk1:2181";
32+ "yarn.resourcemanager.ha.enabled" = "true";
33+ "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
34+ "yarn.resourcemanager.hostname.rm1" = "rm1";
35+ "yarn.resourcemanager.hostname.rm2" = "rm2";
36+ "yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
37+ "yarn.resourcemanager.cluster-id" = "cluster1";
38+ # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
39+ # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
40+ # that causes AM containers to fail otherwise.
41+ "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
42+ "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
43+ };
44+ in
45+ {
46+ zk1 = { ... }: {
47+ services.zookeeper.enable = true;
48+ networking.firewall.allowedTCPPorts = [ 2181 ];
6049 };
61− };
62− nn2 = {pkgs, options, ...}: {
63− services.hadoop = {
64− inherit package coreSite hdfsSite;
65− hdfs.namenode.enable = true;
66− hdfs.zkfc.enable = true;
50+
51+ # HDFS cluster
52+ nn1 = { ... }: {
53+ services.hadoop = {
54+ inherit package coreSite hdfsSite;
55+ hdfs.namenode = {
56+ enable = true;
57+ openFirewall = true;
58+ };
59+ hdfs.zkfc.enable = true;
60+ };
61+ };
62+ nn2 = { ... }: {
63+ services.hadoop = {
64+ inherit package coreSite hdfsSite;
65+ hdfs.namenode = {
66+ enable = true;
67+ openFirewall = true;
68+ };
69+ hdfs.zkfc.enable = true;
70+ };
6771 };
68− };
6972
70− jn1 = {pkgs, options, ...}: {
71− services.hadoop = {
72− inherit package coreSite hdfsSite;
73− hdfs.journalnode.enable = true;
73+ jn1 = { ... }: {
74+ services.hadoop = {
75+ inherit package coreSite hdfsSite;
76+ hdfs.journalnode = {
77+ enable = true;
78+ openFirewall = true;
79+ };
80+ };
7481 };
75− };
76− jn2 = {pkgs, options, ...}: {
77− services.hadoop = {
78− inherit package coreSite hdfsSite;
79− hdfs.journalnode.enable = true;
82+ jn2 = { ... }: {
83+ services.hadoop = {
84+ inherit package coreSite hdfsSite;
85+ hdfs.journalnode = {
86+ enable = true;
87+ openFirewall = true;
88+ };
89+ };
8090 };
81− };
82− jn3 = {pkgs, options, ...}: {
83− services.hadoop = {
84− inherit package coreSite hdfsSite;
85− hdfs.journalnode.enable = true;
91+ jn3 = { ... }: {
92+ services.hadoop = {
93+ inherit package coreSite hdfsSite;
94+ hdfs.journalnode = {
95+ enable = true;
96+ openFirewall = true;
97+ };
98+ };
99+ };
100+
101+ dn1 = { ... }: {
102+ services.hadoop = {
103+ inherit package coreSite hdfsSite;
104+ hdfs.datanode = {
105+ enable = true;
106+ openFirewall = true;
107+ };
108+ };
86109 };
87− };
88110
89− dn1 = {pkgs, options, ...}: {
90− services.hadoop = {
91− inherit package coreSite hdfsSite;
92− hdfs.datanode.enable = true;
111+ # YARN cluster
112+ rm1 = { options, ... }: {
113+ services.hadoop = {
114+ inherit package coreSite hdfsSite yarnSite;
115+ yarn.resourcemanager = {
116+ enable = true;
117+ openFirewall = true;
118+ };
119+ };
93120 };
94− };
95−
96− # YARN cluster
97− rm1 = {pkgs, options, ...}: {
98− services.hadoop = {
99− inherit package coreSite hdfsSite;
100− yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
101− yarn.resourcemanager.enable = true;
121+ rm2 = { options, ... }: {
122+ services.hadoop = {
123+ inherit package coreSite hdfsSite yarnSite;
124+ yarn.resourcemanager = {
125+ enable = true;
126+ openFirewall = true;
127+ };
128+ };
102129 };
103− };
104− rm2 = {pkgs, options, ...}: {
105− services.hadoop = {
106− inherit package coreSite hdfsSite;
107− yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
108− yarn.resourcemanager.enable = true;
130+ nm1 = { options, ... }: {
131+ virtualisation.memorySize = 2048;
132+ services.hadoop = {
133+ inherit package coreSite hdfsSite yarnSite;
134+ yarn.nodemanager = {
135+ enable = true;
136+ openFirewall = true;
137+ };
138+ };
109139 };
110− };
111− nm1 = {pkgs, options, ...}: {
112− virtualisation.memorySize = 2048;
113− services.hadoop = {
114− inherit package coreSite hdfsSite;
115− yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
116− yarn.nodemanager.enable = true;
140+ client = { options, ... }: {
141+ services.hadoop = {
142+ gatewayRole.enable = true;
143+ inherit package coreSite hdfsSite yarnSite;
144+ };
117145 };
118− };
119146 };
120147
121148 testScript = ''
@@ -173,26 +200,26 @@ import ../make-test-python.nix ({pkgs, ...}: {
173200 # DN should have started by now, but confirm anyway
174201 dn1.wait_for_unit("hdfs-datanode")
175202 # Print states of namenodes
176− dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
203+ client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
177204 # Wait for cluster to exit safemode
178− dn1.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
179− dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
205+ client.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
206+ client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
180207 # test R/W
181− dn1.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
182− assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
208+ client.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
209+ assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
183210
184211 # Test NN failover
185212 nn1.succeed("systemctl stop hdfs-namenode")
186− assert "active" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
187− dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
188− assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
213+ assert "active" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
214+ client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
215+ assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
189216
190217 nn1.succeed("systemctl start hdfs-namenode")
191218 nn1.wait_for_open_port(9870)
192219 nn1.wait_for_open_port(8022)
193220 nn1.wait_for_open_port(8020)
194− assert "standby" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
195− dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
221+ assert "standby" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
222+ client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
196223
197224 #### YARN tests ####
198225
@@ -208,21 +235,21 @@ import ../make-test-python.nix ({pkgs, ...}: {
208235 nm1.wait_for_unit("yarn-nodemanager")
209236 nm1.wait_for_open_port(8042)
210237 nm1.wait_for_open_port(8040)
211− nm1.wait_until_succeeds("yarn node -list | grep Nodes:1")
212− nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
213− nm1.succeed("sudo -u yarn yarn node -list | systemd-cat")
238+ client.wait_until_succeeds("yarn node -list | grep Nodes:1")
239+ client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
240+ client.succeed("sudo -u yarn yarn node -list | systemd-cat")
214241
215242 # Test RM failover
216243 rm1.succeed("systemctl stop yarn-resourcemanager")
217− assert "standby" not in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
218− nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
244+ assert "standby" not in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
245+ client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
219246 rm1.succeed("systemctl start yarn-resourcemanager")
220247 rm1.wait_for_unit("yarn-resourcemanager")
221248 rm1.wait_for_open_port(8088)
222− assert "standby" in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
223− nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
249+ assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
250+ client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
224251
225− assert "Estimated value of Pi is" in nm1.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
226− assert "SUCCEEDED" in nm1.succeed("yarn application -list -appStates FINISHED")
252+ assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
253+ assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")
227254 '';
228255})