@@ -1,24 +1,25 @@
1−{ stdenv, pkgs, bazel_3, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
1+{ stdenv, bazel_3, buildBazelPackage, isPy3k, lib, fetchFromGitHub, symlinkJoin
22, addOpenGLRunpath
33# Python deps
4−, buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python
4+, buildPythonPackage, pythonOlder, pythonAtLeast, python
55# Python libraries
6−, numpy, tensorflow-tensorboard_2, backports_weakref, mock, enum34, absl-py
7−, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta
8−, functools32
6+, numpy, tensorflow-tensorboard_2, absl-py
7+, future, setuptools, wheel, keras-preprocessing, google-pasta
98, opt-einsum, astunparse, h5py
109, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_2
10+, dill, flatbuffers-python, tblib, typing-extensions
1111# Common deps
12−, git, swig, which, binutils, glibcLocales, cython
12+, git, pybind11, which, binutils, glibcLocales, cython, perl
1313# Common libraries
14−, jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2
15−, curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib
14+, jemalloc, openmpi, gast, grpc, sqlite, boringssl, jsoncpp
15+, curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib
1616# Upsteam by default includes cuda support since tensorflow 1.15. We could do
1717# that in nix as well. It would make some things easier and less confusing, but
1818# it would also make the default tensorflow package unfree. See
1919# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0
2020, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null
2121, mklSupport ? false, mkl ? null
22+, tensorboardSupport ? true
2223# XLA without CUDA is broken
2324, xlaSupport ? cudaSupport
2425# Default from ./configure script
@@ -39,7 +40,7 @@ assert ! (stdenv.isDarwin && cudaSupport);
3940assert mklSupport -> mkl != null;
4041
4142let
42− withTensorboard = pythonOlder "3.6";
43+ withTensorboard = (pythonOlder "3.6") || tensorboardSupport;
4344
4445 cudatoolkit_joined = symlinkJoin {
4546 name = "${cudatoolkit.name}-merged";
@@ -65,34 +66,40 @@ let
6566 includes_joined = symlinkJoin {
6667 name = "tensorflow-deps-merged";
6768 paths = [
68− pkgs.protobuf
6969 jsoncpp
7070 ];
7171 };
7272
7373 tfFeature = x: if x then "1" else "0";
7474
75− version = "2.3.2";
75+ version = "2.4.0";
7676 variant = if cudaSupport then "-gpu" else "";
7777 pname = "tensorflow${variant}";
7878
7979 pythonEnv = python.withPackages (_:
8080 [ # python deps needed during wheel build time (not runtime, see the buildPythonPackage part for that)
81− numpy
81+ # This list can likely be shortened, but each trial takes multiple hours so won't bother for now.
82+ absl-py
83+ astunparse
84+ dill
85+ flatbuffers-python
86+ gast
87+ google-pasta
88+ grpcio
89+ h5py
8290 keras-preprocessing
91+ numpy
92+ opt-einsum
8393 protobuf
84− wrapt
85− gast
86− astor
87− absl-py
88− termcolor
89− keras-applications
9094 setuptools
95+ six
96+ tblib
97+ tensorflow-estimator_2
98+ tensorflow-tensorboard_2
99+ termcolor
100+ typing-extensions
91101 wheel
92− ] ++ lib.optionals (!isPy3k)
93− [ future
94− functools32
95− mock
102+ wrapt
96103 ]);
97104
98105 bazel-build = buildBazelPackage {
@@ -103,27 +110,21 @@ let
103110 owner = "tensorflow";
104111 repo = "tensorflow";
105112 rev = "v${version}";
106− sha256 = "sha256-ncwIkqLDqrB33pB9/FTlBklsIJUEvnDUmyAeUfufCFs=";
113+ sha256 = "0yl06aypfxrcs35828xf04mkidz1x0j89v0q5h4d2xps1cb5rv3f";
107114 };
108115
109116 patches = [
110− # Fixes for NixOS jsoncpp
111− ./system-jsoncpp.patch
112−
117+ # Relax too strict Python packages versions dependencies.
113118 ./relax-dependencies.patch
114−
115− # see https://github.com/tensorflow/tensorflow/issues/40688
116− (fetchpatch {
117− url = "https://github.com/tensorflow/tensorflow/commit/75ea0b31477d6ba9e990e296bbbd8ca4e7eebadf.patch";
118− sha256 = "1xp1icacig0xm0nmb05sbrf4nw4xbln9fhc308birrv8286zx7wv";
119− })
119+ # Add missing `io_bazel_rules_docker` dependency.
120+ ./workspace.patch
120121 ];
121122
122123 # On update, it can be useful to steal the changes from gentoo
123124 # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow
124125
125126 nativeBuildInputs = [
126− swig which pythonEnv
127+ which pythonEnv cython perl
127128 ] ++ lib.optional cudaSupport addOpenGLRunpath;
128129
129130 buildInputs = [
@@ -135,19 +136,18 @@ let
135136 # libs taken from system through the TF_SYS_LIBS mechanism
136137 grpc
137138 sqlite
138− openssl
139+ boringssl
139140 jsoncpp
140− pkgs.protobuf
141141 curl
142+ pybind11
142143 snappy
143− flatbuffers
144+ flatbuffers-core
144145 icu
145146 double-conversion
146147 libpng
147− libjpeg
148+ libjpeg_turbo
148149 giflib
149− re2
150− pkgs.lmdb
150+ lmdb-core
151151 ] ++ lib.optionals cudaSupport [
152152 cudatoolkit
153153 cudnn
@@ -173,10 +173,17 @@ let
173173 # "com_github_googleapis_googleapis"
174174 # "com_github_googlecloudplatform_google_cloud_cpp"
175175 "com_github_grpc_grpc"
176− "com_google_protobuf"
177− "com_googlesource_code_re2"
176+ # Multiple issues with custom protobuf.
177+ # First `com_github_googleapis` fails to configure. Can be worked around by disabling `com_github_googleapis`
178+ # and related functionality, but then the next error is about "dangling symbolic link", and in general
179+ # looks like that's only the beginning: see
180+ # https://stackoverflow.com/questions/55578884/how-to-build-tensorflow-1-13-1-with-custom-protobuf
181+ # "com_google_protobuf"
182+ # Fails with the error: external/org_tensorflow/tensorflow/core/profiler/utils/tf_op_utils.cc:46:49: error: no matching function for call to 're2::RE2::FullMatch(absl::lts_2020_02_25::string_view&, re2::RE2&)'
183+ # "com_googlesource_code_re2"
178184 "curl"
179185 "cython"
186+ "dill_archive"
180187 "double_conversion"
181188 "enum34_archive"
182189 "flatbuffers"
@@ -198,8 +205,9 @@ let
198205 "pybind11"
199206 "six_archive"
200207 "snappy"
201− "swig"
208+ "tblib_archive"
202209 "termcolor_archive"
210+ "typing_extensions_archive"
203211 "wrapt"
204212 "zlib"
205213 ];
@@ -224,16 +232,13 @@ let
224232 TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities;
225233
226234 postPatch = ''
235+ # bazel 3.3 should work just as well as bazel 3.1
236+ rm -f .bazelversion
237+ '' + lib.optionalString (!withTensorboard) ''
227238 # Tensorboard pulls in a bunch of dependencies, some of which may
228239 # include security vulnerabilities. So we make it optional.
229240 # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
230− sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
231−
232− # numpy 1.19 added in https://github.com/tensorflow/tensorflow/commit/75ea0b31477d6ba9e990e296bbbd8ca4e7eebadf.patch
233− sed -i 's/numpy >= 1.16.0, < 1.19.0/numpy >= 1.16.0/' tensorflow/tools/pip_package/setup.py
234−
235− # bazel 3.3 should work just as well as bazel 3.1
236− rm -f .bazelversion
241+ sed -i '/tensorboard ~=/d' tensorflow/tools/pip_package/setup.py
237242 '';
238243
239244 # https://github.com/tensorflow/tensorflow/pull/39470
@@ -277,16 +282,15 @@ let
277282 bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow";
278283
279284 removeRulesCC = false;
285+ # Without this Bazel complaints about sandbox violations.
286+ dontAddBazelOpts = true;
280287
281288 fetchAttrs = {
282− # So that checksums don't depend on these.
283− TF_SYSTEM_LIBS = null;
284−
285289 # cudaSupport causes fetch of ncclArchive, resulting in different hashes
286290 sha256 = if cudaSupport then
287− "sha256-lEdPA9vhYO6vd5FgPMbFp2PkRvDBurPidYsxtJLXcbQ="
291+ "0vyy1hv0jy5pqwvnc8pxb9isgnbw07c4a4d4wn61db00np114crz"
288292 else
289− "sha256-ZEY/bWo5M3Juw1x3CwhXYXZHD4q5LzWDlhgXnh4P95U=";
293+ "0vczv5f9s4dxgwdkmf1y9b9ybh5d3y1nllqhb5q8aj9kq73izyn9";
290294 };
291295
292296 buildAttrs = {
@@ -329,15 +333,13 @@ let
329333 license = licenses.asl20;
330334 maintainers = with maintainers; [ jyp abbradar ];
331335 platforms = with platforms; linux ++ darwin;
332− # The py2 build fails due to some issue importing protobuf. Possibly related to the fix in
333− # https://github.com/akesandgren/easybuild-easyblocks/commit/1f2e517ddfd1b00a342c6abb55aef3fd93671a2b
334− broken = !(xlaSupport -> cudaSupport) || !isPy3k;
336+ broken = !(xlaSupport -> cudaSupport);
335337 };
336338 };
337339
338340in buildPythonPackage {
339341 inherit version pname;
340− disabled = isPy27;
342+ disabled = !isPy3k;
341343
342344 src = bazel-build.python;
343345
@@ -354,27 +356,23 @@ in buildPythonPackage {
354356 # tensorflow/tools/pip_package/setup.py
355357 propagatedBuildInputs = [
356358 absl-py
357− astor
359+ astunparse
360+ dill
361+ flatbuffers-python
358362 gast
359363 google-pasta
360− keras-applications
364+ grpcio
365+ h5py
361366 keras-preprocessing
362367 numpy
363− six
368+ opt-einsum
364369 protobuf
370+ six
371+ tblib
365372 tensorflow-estimator_2
366373 termcolor
374+ typing-extensions
367375 wrapt
368− grpcio
369− opt-einsum
370− astunparse
371− h5py
372− ] ++ lib.optionals (!isPy3k) [
373− mock
374− future
375− functools32
376− ] ++ lib.optionals (pythonOlder "3.4") [
377− backports_weakref enum34
378376 ] ++ lib.optionals withTensorboard [
379377 tensorflow-tensorboard_2
380378 ];