@@ -1,16 +1,52 @@
1−import ./make-test-python.nix ({ lib, ... }:
1+import ./make-test-python.nix ({ lib, pkgs, ... }:
22let
3− mungekey = "mungeverryweakkeybuteasytointegratoinatest";
4−
53 slurmconfig = {
6− controlMachine = "control";
7− nodeName = [ "node[1-3] CPUs=1 State=UNKNOWN" ];
8− partitionName = [ "debug Nodes=node[1-3] Default=YES MaxTime=INFINITE State=UP" ];
9− extraConfig = ''
10− AccountingStorageHost=dbd
11− AccountingStorageType=accounting_storage/slurmdbd
12− '';
4+ services.slurm = {
5+ controlMachine = "control";
6+ nodeName = [ "node[1-3] CPUs=1 State=UNKNOWN" ];
7+ partitionName = [ "debug Nodes=node[1-3] Default=YES MaxTime=INFINITE State=UP" ];
8+ extraConfig = ''
9+ AccountingStorageHost=dbd
10+ AccountingStorageType=accounting_storage/slurmdbd
11+ '';
12+ };
13+ environment.systemPackages = [ mpitest ];
14+ networking.firewall.enable = false;
15+ systemd.tmpfiles.rules = [
16+ "f /etc/munge/munge.key 0400 munge munge - mungeverryweakkeybuteasytointegratoinatest"
17+ ];
1318 };
19+
20+ mpitest = let
21+ mpitestC = pkgs.writeText "mpitest.c" ''
22+ #include <stdio.h>
23+ #include <stdlib.h>
24+ #include <mpi.h>
25+
26+ int
27+ main (int argc, char *argv[])
28+ {
29+ int rank, size, length;
30+ char name[512];
31+
32+ MPI_Init (&argc, &argv);
33+ MPI_Comm_rank (MPI_COMM_WORLD, &rank);
34+ MPI_Comm_size (MPI_COMM_WORLD, &size);
35+ MPI_Get_processor_name (name, &length);
36+
37+ if ( rank == 0 ) printf("size=%d\n", size);
38+
39+ printf ("%s: hello world from process %d of %d\n", name, rank, size);
40+
41+ MPI_Finalize ();
42+
43+ return EXIT_SUCCESS;
44+ }
45+ '';
46+ in pkgs.runCommandNoCC "mpitest" {} ''
47+ mkdir -p $out/bin
48+ ${pkgs.openmpi}/bin/mpicc ${mpitestC} -o $out/bin/mpitest
49+ '';
1450in {
1551 name = "slurm";
1652
@@ -21,37 +57,40 @@ in {
2157 computeNode =
2258 { ...}:
2359 {
60+ imports = [ slurmconfig ];
2461 # TODO slurmd port and slurmctld port should be configurations and
2562 # automatically allowed by the firewall.
26− networking.firewall.enable = false;
2763 services.slurm = {
2864 client.enable = true;
29− } // slurmconfig;
65+ };
3066 };
3167 in {
3268
3369 control =
3470 { ...}:
3571 {
36− networking.firewall.enable = false;
72+ imports = [ slurmconfig ];
3773 services.slurm = {
3874 server.enable = true;
39− } // slurmconfig;
75+ };
4076 };
4177
4278 submit =
4379 { ...}:
4480 {
45− networking.firewall.enable = false;
81+ imports = [ slurmconfig ];
4682 services.slurm = {
4783 enableStools = true;
48− } // slurmconfig;
84+ };
4985 };
5086
5187 dbd =
5288 { pkgs, ... } :
5389 {
5490 networking.firewall.enable = false;
91+ systemd.tmpfiles.rules = [
92+ "f /etc/munge/munge.key 0400 munge munge - mungeverryweakkeybuteasytointegratoinatest"
93+ ];
5594 services.slurm.dbdserver = {
5695 enable = true;
5796 storagePass = "password123";
@@ -87,24 +126,7 @@ in {
87126 ''
88127 start_all()
89128
90− # Set up authentification across the cluster
91− for node in [submit, control, dbd, node1, node2, node3]:
92−
93− node.wait_for_unit("default.target")
94−
95− node.succeed("mkdir /etc/munge")
96− node.succeed(
97− "echo '${mungekey}' > /etc/munge/munge.key"
98− )
99− node.succeed("chmod 0400 /etc/munge/munge.key")
100− node.succeed("chown munge:munge /etc/munge/munge.key")
101− node.succeed("systemctl restart munged")
102−
103− node.wait_for_unit("munged")
104−
105−
106− # Restart the services since they have probably failed due to the munge init
107− # failure
129+ # Make sure DBD is up after DB initialzation
108130 with subtest("can_start_slurmdbd"):
109131 dbd.succeed("systemctl restart slurmdbd")
110132 dbd.wait_for_unit("slurmdbd.service")
@@ -137,5 +159,8 @@ in {
137159 # find the srun job from above in the database
138160 control.succeed("sleep 5")
139161 control.succeed("sacct | grep hostname")
162+
163+ with subtest("run_PMIx_mpitest"):
164+ submit.succeed("srun -N 3 --mpi=pmix mpitest | grep size=3")
140165 '';
141166})