git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #92048 from markuskowa/add-pmix

pmix: init at 3.1.5, add support to slurm, openmpi

commit6ce044250b5338b320a29fd84ec2227a381fb8a0 authormarkuskowa <markus.kowalewski@gmail.com> dateThu 9 Jul 2020, 11:41 (6 years ago) parentb5cd6a3e6403d2
6 files changed+134−38
nixos/tests/slurm.nix+59−34 pkgs/development/libraries/openmpi/default.nix+7−4 pkgs/development/libraries/pmix/default.nix+48 pkgs/servers/computing/slurm/default.nix+5 pkgs/servers/computing/slurm/pmix-configure.patch+13 pkgs/top-level/all-packages.nix+2
nixos/tests/slurm.nix+59−34
@@ -1,16 +1,52 @@
1−import ./make-test-python.nix ({ lib, ... }:
1+import ./make-test-python.nix ({ lib, pkgs, ... }:
22let
3− mungekey = "mungeverryweakkeybuteasytointegratoinatest";
4−
53 slurmconfig = {
6− controlMachine = "control";
7− nodeName = [ "node[1-3] CPUs=1 State=UNKNOWN" ];
8− partitionName = [ "debug Nodes=node[1-3] Default=YES MaxTime=INFINITE State=UP" ];
9− extraConfig = ''
10− AccountingStorageHost=dbd
11− AccountingStorageType=accounting_storage/slurmdbd
12− '';
4+ services.slurm = {
5+ controlMachine = "control";
6+ nodeName = [ "node[1-3] CPUs=1 State=UNKNOWN" ];
7+ partitionName = [ "debug Nodes=node[1-3] Default=YES MaxTime=INFINITE State=UP" ];
8+ extraConfig = ''
9+ AccountingStorageHost=dbd
10+ AccountingStorageType=accounting_storage/slurmdbd
11+ '';
12+ };
13+ environment.systemPackages = [ mpitest ];
14+ networking.firewall.enable = false;
15+ systemd.tmpfiles.rules = [
16+ "f /etc/munge/munge.key 0400 munge munge - mungeverryweakkeybuteasytointegratoinatest"
17+ ];
1318 };
19+
20+ mpitest = let
21+ mpitestC = pkgs.writeText "mpitest.c" ''
22+ #include <stdio.h>
23+ #include <stdlib.h>
24+ #include <mpi.h>
25+
26+ int
27+ main (int argc, char *argv[])
28+ {
29+ int rank, size, length;
30+ char name[512];
31+
32+ MPI_Init (&argc, &argv);
33+ MPI_Comm_rank (MPI_COMM_WORLD, &rank);
34+ MPI_Comm_size (MPI_COMM_WORLD, &size);
35+ MPI_Get_processor_name (name, &length);
36+
37+ if ( rank == 0 ) printf("size=%d\n", size);
38+
39+ printf ("%s: hello world from process %d of %d\n", name, rank, size);
40+
41+ MPI_Finalize ();
42+
43+ return EXIT_SUCCESS;
44+ }
45+ '';
46+ in pkgs.runCommandNoCC "mpitest" {} ''
47+ mkdir -p $out/bin
48+ ${pkgs.openmpi}/bin/mpicc ${mpitestC} -o $out/bin/mpitest
49+ '';
1450in {
1551 name = "slurm";
1652
@@ -21,37 +57,40 @@ in {
2157 computeNode =
2258 { ...}:
2359 {
60+ imports = [ slurmconfig ];
2461 # TODO slurmd port and slurmctld port should be configurations and
2562 # automatically allowed by the firewall.
26− networking.firewall.enable = false;
2763 services.slurm = {
2864 client.enable = true;
29− } // slurmconfig;
65+ };
3066 };
3167 in {
3268
3369 control =
3470 { ...}:
3571 {
36− networking.firewall.enable = false;
72+ imports = [ slurmconfig ];
3773 services.slurm = {
3874 server.enable = true;
39− } // slurmconfig;
75+ };
4076 };
4177
4278 submit =
4379 { ...}:
4480 {
45− networking.firewall.enable = false;
81+ imports = [ slurmconfig ];
4682 services.slurm = {
4783 enableStools = true;
48− } // slurmconfig;
84+ };
4985 };
5086
5187 dbd =
5288 { pkgs, ... } :
5389 {
5490 networking.firewall.enable = false;
91+ systemd.tmpfiles.rules = [
92+ "f /etc/munge/munge.key 0400 munge munge - mungeverryweakkeybuteasytointegratoinatest"
93+ ];
5594 services.slurm.dbdserver = {
5695 enable = true;
5796 storagePass = "password123";
@@ -87,24 +126,7 @@ in {
87126 ''
88127 start_all()
89128
90− # Set up authentification across the cluster
91− for node in [submit, control, dbd, node1, node2, node3]:
92−
93− node.wait_for_unit("default.target")
94−
95− node.succeed("mkdir /etc/munge")
96− node.succeed(
97− "echo '${mungekey}' > /etc/munge/munge.key"
98− )
99− node.succeed("chmod 0400 /etc/munge/munge.key")
100− node.succeed("chown munge:munge /etc/munge/munge.key")
101− node.succeed("systemctl restart munged")
102−
103− node.wait_for_unit("munged")
104−
105−
106− # Restart the services since they have probably failed due to the munge init
107− # failure
129+ # Make sure DBD is up after DB initialzation
108130 with subtest("can_start_slurmdbd"):
109131 dbd.succeed("systemctl restart slurmdbd")
110132 dbd.wait_for_unit("slurmdbd.service")
@@ -137,5 +159,8 @@ in {
137159 # find the srun job from above in the database
138160 control.succeed("sleep 5")
139161 control.succeed("sacct | grep hostname")
162+
163+ with subtest("run_PMIx_mpitest"):
164+ submit.succeed("srun -N 3 --mpi=pmix mpitest | grep size=3")
140165 '';
141166})
pkgs/development/libraries/openmpi/default.nix+7−4
@@ -1,6 +1,6 @@
11{ stdenv, fetchurl, fetchpatch, gfortran, perl, libnl
22, rdma-core, zlib, numactl, libevent, hwloc, targetPackages, symlinkJoin
3−, libpsm2, libfabric
3+, libpsm2, libfabric, pmix
44
55# Enable CUDA support
66, cudaSupport ? false, cudatoolkit ? null
@@ -46,7 +46,7 @@ in stdenv.mkDerivation rec {
4646 '';
4747
4848 buildInputs = with stdenv; [ gfortran zlib ]
49− ++ lib.optionals isLinux [ libnl numactl ]
49+ ++ lib.optionals isLinux [ libnl numactl pmix ]
5050 ++ lib.optionals cudaSupport [ cudatoolkit ]
5151 ++ [ libevent hwloc ]
5252 ++ lib.optional (isLinux || isFreeBSD) rdma-core
@@ -55,8 +55,11 @@ in stdenv.mkDerivation rec {
5555 nativeBuildInputs = [ perl ];
5656
5757 configureFlags = with stdenv; lib.optional (!cudaSupport) "--disable-mca-dso"
58− ++ lib.optional isLinux "--with-libnl=${libnl.dev}"
59− ++ lib.optional enableSGE "--with-sge"
58+ ++ lib.optionals isLinux [
59+ "--with-libnl=${libnl.dev}"
60+ "--with-pmix=${pmix}"
61+ "--with-pmix-libdir=${pmix}/lib"
62+ ] ++ lib.optional enableSGE "--with-sge"
6063 ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default"
6164 # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build
6265 # https://github.com/openucx/ucx
pkgs/development/libraries/pmix/default.nixnew file+48
@@ -0,0 +1,48 @@
1+{ stdenv, fetchFromGitHub, perl, autoconf, automake
2+, libtool, flex, libevent, hwloc, munge, zlib
3+} :
4+
5+let
6+ version = "3.1.5";
7+
8+in stdenv.mkDerivation {
9+ pname = "pmix";
10+ inherit version;
11+
12+ src = fetchFromGitHub {
13+ repo = "openpmix";
14+ owner = "openpmix";
15+ rev = "v${version}";
16+ sha256 = "0fvfsig20amcigyn4v3gcdxc0jif44vqg37b8zzh0s8jqqj7jz5w";
17+ };
18+
19+ postPatch = ''
20+ patchShebangs ./autogen.pl
21+ patchShebangs ./config
22+ '';
23+
24+ nativeBuildInputs = [ perl autoconf automake libtool flex ];
25+
26+ buildInputs = [ libevent hwloc munge zlib ];
27+
28+ configureFlags = [
29+ "--with-libevent=${libevent.dev}"
30+ "--with-munge=${munge}"
31+ "--with-hwloc=${hwloc.dev}"
32+ ];
33+
34+ preConfigure = ''
35+ ./autogen.pl
36+ '';
37+
38+ enableParallelBuilding = true;
39+
40+ meta = with stdenv.lib; {
41+ description = "Process Management Interface for HPC environments";
42+ homepage = "https://openpmix.github.io/";
43+ license = licenses.bsd3;
44+ maintainers = [ maintainers.markuskowa ];
45+ platforms = platforms.linux;
46+ };
47+}
48+
pkgs/servers/computing/slurm/default.nix+5
@@ -2,6 +2,7 @@
22, python, munge, perl, pam, zlib, shadow, coreutils
33, ncurses, libmysqlclient, gtk2, lua, hwloc, numactl
44, readline, freeipmi, xorg, lz4, rdma-core, nixosTests
5+, pmix
56# enable internal X11 support via libssh2
67, enableX11 ? true
78}:
@@ -26,6 +27,8 @@ stdenv.mkDerivation rec {
2627 # increase string length to allow for full
2728 # path of 'echo' in nix store
2829 ./common-env-echo.patch
30+ # Required for configure to pick up the right dlopen path
31+ ./pmix-configure.patch
2932 ];
3033
3134 prePatch = ''
@@ -46,6 +49,7 @@ stdenv.mkDerivation rec {
4649 curl python munge perl pam zlib
4750 libmysqlclient ncurses gtk2 lz4 rdma-core
4851 lua hwloc numactl readline freeipmi shadow.su
52+ pmix
4953 ] ++ stdenv.lib.optionals enableX11 [ xorg.xauth ];
5054
5155 configureFlags = with stdenv.lib;
@@ -56,6 +60,7 @@ stdenv.mkDerivation rec {
5660 "--with-zlib=${zlib}"
5761 "--with-ofed=${rdma-core}"
5862 "--sysconfdir=/etc/slurm"
63+ "--with-pmix=${pmix}"
5964 ] ++ (optional (gtk2 == null) "--disable-gtktest")
6065 ++ (optional (!enableX11) "--disable-x11");
6166
pkgs/servers/computing/slurm/pmix-configure.patchnew file+13
@@ -0,0 +1,13 @@
1+diff --git a/configure b/configure
2+index 1cf53bc..ab68441 100755
3+--- a/configure
4++++ b/configure
5+@@ -21207,7 +21207,7 @@ rm -f conftest.err conftest.i conftest.$ac_ext
6+ as_fn_error $? "error processing $x_ac_cv_pmix_libdir: PMIx v3.x was already found in one of the previous paths" "$LINENO" 5
7+ fi
8+ _x_ac_pmix_v3_found="1"
9+- PMIX_V3_CPPFLAGS="-I$x_ac_cv_pmix_dir/include"
10++ PMIX_V3_CPPFLAGS="-I$x_ac_cv_pmix_dir/include -DPMIXP_V3_LIBPATH=\\\"$x_ac_cv_pmix_libdir\\\""
11+ if test "$ac_with_rpath" = "yes"; then
12+ PMIX_V3_LDFLAGS="-Wl,-rpath -Wl,$x_ac_cv_pmix_libdir -L$x_ac_cv_pmix_libdir"
13+ else
pkgs/top-level/all-packages.nix+2
@@ -6190,6 +6190,8 @@ in
61906190
61916191 pmacct = callPackage ../tools/networking/pmacct { };
61926192
6193+ pmix = callPackage ../development/libraries/pmix { };
6194+
61936195 polygraph = callPackage ../tools/networking/polygraph { };
61946196
61956197 progress = callPackage ../tools/misc/progress { };