git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #153780 from ndl/submit/apache-beam

python3Packages.apache-beam: init at 2.35.0

commita142f0a0046ae65c05e8a2e403b9097054e798ad authorSamuel Ainsworth <skainsworth@gmail.com> dateSat 8 Jan 2022, 17:18 (5 years ago) parentca51f4b0341138
4 files changed+208
compare with master
unifiedsplit
pkgs/development/python-modules/apache-beam/default.nix+145 pkgs/development/python-modules/apache-beam/fix-cython.patch+41 pkgs/development/python-modules/apache-beam/relax-deps.patch+20 pkgs/top-level/python-packages.nix+2
pkgs/development/python-modules/apache-beam/default.nixnew file+145
@@ -0,0 +1,145 @@
1+{ buildPythonPackage
2+, cloudpickle
3+, crcmod
4+, cython
5+, dill
6+, fastavro
7+, fetchFromGitHub
8+, freezegun
9+, grpcio
10+, grpcio-tools
11+, hdfs
12+, httplib2
13+, lib
14+, mock
15+, mypy-protobuf
16+, numpy
17+, oauth2client
18+, orjson
19+, pandas
20+, parameterized
21+, proto-plus
22+, protobuf
23+, psycopg2
24+, pyarrow
25+, pydot
26+, pyhamcrest
27+, pymongo
28+, pytest-timeout
29+, pytest-xdist
30+, pytestCheckHook
31+, python
32+, python-dateutil
33+, pytz
34+, pyyaml
35+, requests
36+, requests-mock
37+, setuptools
38+, sqlalchemy
39+, tenacity
40+, typing-extensions
41+}:
42+
43+buildPythonPackage rec {
44+ pname = "apache-beam";
45+ version = "2.35.0";
46+
47+ src = fetchFromGitHub {
48+ owner = "apache";
49+ repo = "beam";
50+ rev = "v${version}";
51+ sha256 = "0qxkas33d8i6yj133plnadbfm74ak7arn7ldpziyiwdav3hj68sy";
52+ };
53+
54+ patches = [
55+ ./relax-deps.patch
56+ # Fixes https://issues.apache.org/jira/browse/BEAM-9324
57+ ./fix-cython.patch
58+ ];
59+
60+ sourceRoot = "source/sdks/python";
61+
62+ nativeBuildInputs = [
63+ cython
64+ grpcio-tools
65+ mypy-protobuf
66+ ];
67+
68+ propagatedBuildInputs = [
69+ cloudpickle
70+ crcmod
71+ cython
72+ dill
73+ fastavro
74+ grpcio
75+ hdfs
76+ httplib2
77+ numpy
78+ oauth2client
79+ orjson
80+ proto-plus
81+ protobuf
82+ pyarrow
83+ pydot
84+ pymongo
85+ python-dateutil
86+ pytz
87+ requests
88+ setuptools
89+ typing-extensions
90+ ];
91+
92+ pythonImportsCheck = [
93+ "apache_beam"
94+ ];
95+
96+ checkInputs = [
97+ freezegun
98+ mock
99+ pandas
100+ parameterized
101+ psycopg2
102+ pyhamcrest
103+ pytest-timeout
104+ pytest-xdist
105+ pytestCheckHook
106+ pyyaml
107+ requests-mock
108+ sqlalchemy
109+ tenacity
110+ ];
111+
112+ # Make sure we're running the tests for the actually installed
113+ # package, so that cython's .so files are available.
114+ preCheck = "cd $out/lib/${python.libPrefix}/site-packages";
115+
116+ disabledTestPaths = [
117+ # These tests depend on the availability of specific servers backends.
118+ "apache_beam/runners/portability/flink_runner_test.py"
119+ "apache_beam/runners/portability/samza_runner_test.py"
120+ "apache_beam/runners/portability/spark_runner_test.py"
121+ ];
122+
123+ disabledTests = [
124+ # The reasons of failures for these tests are unclear.
125+ # They reproduce in Docker with Ubuntu 22.04
126+ # (= they're not `nixpkgs`-specific) but given the upstream uses
127+ # quite elaborate testing infra with containers and multiple
128+ # different runners - I don't expect them to help debugging these
129+ # when running via our (= custom from their PoV) testing infra.
130+ "testBuildListUnpack"
131+ "testBuildTupleUnpack"
132+ "testBuildTupleUnpackWithCall"
133+ "test_convert_bare_types"
134+ "test_incomparable_default"
135+ "test_pardo_type_inference"
136+ "test_with_main_session"
137+ ];
138+
139+ meta = with lib; {
140+ description = "Unified model for defining both batch and streaming data-parallel processing pipelines";
141+ homepage = "https://beam.apache.org/";
142+ license = licenses.asl20;
143+ maintainers = with maintainers; [ ndl ];
144+ };
145+}
pkgs/development/python-modules/apache-beam/fix-cython.patchnew file+41
@@ -0,0 +1,41 @@
1+diff --git a/apache_beam/runners/worker/operations.py b/apache_beam/runners/worker/operations.py
2+index 3464c5750c..5921c72b90 100644
3+--- a/apache_beam/runners/worker/operations.py
4++++ b/apache_beam/runners/worker/operations.py
5+@@ -69,18 +69,6 @@ if TYPE_CHECKING:
6+ from apache_beam.runners.worker.statesampler import StateSampler
7+ from apache_beam.transforms.userstate import TimerSpec
8+
9+-# Allow some "pure mode" declarations.
10+-try:
11+- import cython
12+-except ImportError:
13+-
14+- class FakeCython(object):
15+- @staticmethod
16+- def cast(type, value):
17+- return value
18+-
19+- globals()['cython'] = FakeCython()
20+-
21+ _globally_windowed_value = GlobalWindows.windowed_value(None)
22+ _global_window_type = type(_globally_windowed_value.windows[0])
23+
24+@@ -149,7 +137,7 @@ class ConsumerSet(Receiver):
25+ # type: (WindowedValue) -> None
26+ self.update_counters_start(windowed_value)
27+ for consumer in self.consumers:
28+- cython.cast(Operation, consumer).process(windowed_value)
29++ consumer.process(windowed_value)
30+ self.update_counters_finish()
31+
32+ def try_split(self, fraction_of_remainder):
33+@@ -345,7 +333,7 @@ class Operation(object):
34+
35+ def output(self, windowed_value, output_index=0):
36+ # type: (WindowedValue, int) -> None
37+- cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
38++ self.receivers[output_index].receive(windowed_value)
39+
40+ def add_receiver(self, operation, output_index=0):
41+ # type: (Operation, int) -> None
pkgs/development/python-modules/apache-beam/relax-deps.patchnew file+20
@@ -0,0 +1,20 @@
1+diff --git a/setup.py b/setup.py
2+index 9429459622..2727b3becb 100644
3+--- a/setup.py
4++++ b/setup.py
5+@@ -136,12 +136,12 @@ REQUIRED_PACKAGES = [
6+ # version of dill. It is best to use the same version of dill on client and
7+ # server, therefore list of allowed versions is very narrow.
8+ # See: https://github.com/uqfoundation/dill/issues/341.
9+- 'dill>=0.3.1.1,<0.3.2',
10++ 'dill>=0.3.1.1',
11+ 'fastavro>=0.21.4,<2',
12+ 'grpcio>=1.29.0,<2',
13+ 'hdfs>=2.1.0,<3.0.0',
14+- 'httplib2>=0.8,<0.20.0',
15+- 'numpy>=1.14.3,<1.21.0',
16++ 'httplib2>=0.8',
17++ 'numpy>=1.14.3',
18+ 'pymongo>=3.8.0,<4.0.0',
19+ 'oauth2client>=2.0.1,<5',
20+ 'protobuf>=3.12.2,<4',
pkgs/top-level/python-packages.nix+2
@@ -550,6 +550,8 @@ in {
550550
551551 apache-airflow = callPackage ../development/python-modules/apache-airflow { };
552552
553+ apache-beam = callPackage ../development/python-modules/apache-beam { };
554+
553555 apcaccess = callPackage ../development/python-modules/apcaccess { };
554556
555557 apipkg = callPackage ../development/python-modules/apipkg { };