@@ -1,56 +1,75 @@
1−{ lib, stdenv, fetchzip, makeWrapper, jre, pythonPackages, coreutils, hadoop
1+{ lib, stdenv, fetchzip, makeWrapper, jdk8, python3Packages, extraPythonPackages ? [], coreutils, hadoop
22, RSupport? true, R
33}:
44
55with lib;
66
7−stdenv.mkDerivation rec {
7+let
8+ spark = { pname, version, src }:
9+ stdenv.mkDerivation rec {
10+ inherit pname version src;
11+ nativeBuildInputs = [ makeWrapper ];
12+ buildInputs = [ jdk8 python3Packages.python ]
13+ ++ extraPythonPackages
14+ ++ optional RSupport R;
815
9− pname = "spark";
10− version = "2.4.4";
16+ untarDir = "${pname}-${version}";
17+ installPhase = ''
18+ mkdir -p $out/{lib/${untarDir}/conf,bin,/share/java}
19+ mv * $out/lib/${untarDir}
1120
12− src = fetchzip {
13− url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
14− sha256 = "1a9w5k0207fysgpxx6db3a00fs5hdc2ncx99x4ccy2s0v5ndc66g";
21+ cp $out/lib/${untarDir}/conf/log4j.properties{.template,}
22+
23+ cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF
24+ export JAVA_HOME="${jdk8}"
25+ export SPARK_HOME="$out/lib/${untarDir}"
26+ export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath)
27+ export PYSPARK_PYTHON="${python3Packages.python}/bin/${python3Packages.python.executable}"
28+ export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
29+ ${optionalString RSupport ''
30+ export SPARKR_R_SHELL="${R}/bin/R"
31+ export PATH="\$PATH:${R}/bin"''}
32+ EOF
33+
34+ for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
35+ makeWrapper "$n" "$out/bin/$(basename $n)"
36+ substituteInPlace "$n" --replace dirname ${coreutils.out}/bin/dirname
37+ done
38+ for n in $(find $out/lib/${untarDir}/sbin -type f); do
39+ # Spark deprecated scripts with "slave" in the name.
40+ # This line adds forward compatibility with the nixos spark module for
41+ # older versions of spark that don't have the new "worker" scripts.
42+ ln -s "$n" $(echo "$n" | sed -r 's/slave(s?).sh$/worker\1.sh/g') || true
43+ done
44+ ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
45+ '';
46+
47+ meta = {
48+ description = "Apache Spark is a fast and general engine for large-scale data processing";
49+ homepage = "http://spark.apache.org";
50+ license = lib.licenses.asl20;
51+ platforms = lib.platforms.all;
52+ maintainers = with maintainers; [ thoughtpolice offline kamilchm illustris ];
53+ repositories.git = "git://git.apache.org/spark.git";
54+ };
55+ };
56+in {
57+ spark3 = spark rec {
58+ pname = "spark";
59+ version = "3.1.2";
60+
61+ src = fetchzip {
62+ url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
63+ sha256 = "1bgh2y6jm7wqy6yc40rx68xkki31i3jiri2yixb1bm0i9pvsj9yf";
64+ };
1565 };
66+ spark2 = spark rec {
67+ pname = "spark";
68+ version = "2.4.8";
1669
17− nativeBuildInputs = [ makeWrapper ];
18− buildInputs = [ jre pythonPackages.python pythonPackages.numpy ]
19− ++ optional RSupport R;
20−
21− untarDir = "${pname}-${version}-bin-without-hadoop";
22− installPhase = ''
23− mkdir -p $out/{lib/${untarDir}/conf,bin,/share/java}
24− mv * $out/lib/${untarDir}
25−
26− sed -e 's/INFO, console/WARN, console/' < \
27− $out/lib/${untarDir}/conf/log4j.properties.template > \
28− $out/lib/${untarDir}/conf/log4j.properties
29−
30− cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF
31− export JAVA_HOME="${jre}"
32− export SPARK_HOME="$out/lib/${untarDir}"
33− export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath)
34− export PYSPARK_PYTHON="${pythonPackages.python}/bin/${pythonPackages.python.executable}"
35− export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
36− ${optionalString RSupport
37− ''export SPARKR_R_SHELL="${R}/bin/R"
38− export PATH=$PATH:"${R}/bin/R"''}
39− EOF
40−
41− for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
42− makeWrapper "$n" "$out/bin/$(basename $n)"
43− substituteInPlace "$n" --replace dirname ${coreutils.out}/bin/dirname
44− done
45− ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
46− '';
47−
48− meta = {
49− description = "Apache Spark is a fast and general engine for large-scale data processing";
50− homepage = "http://spark.apache.org";
51− license = lib.licenses.asl20;
52− platforms = lib.platforms.all;
53− maintainers = with maintainers; [ thoughtpolice offline kamilchm ];
54− repositories.git = "git://git.apache.org/spark.git";
70+ src = fetchzip {
71+ url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
72+ sha256 = "1mkyq0gz9fiav25vr0dba5ivp0wh0mh7kswwnx8pvsmb6wbwyfxv";
73+ };
5574 };
5675}