From fdf6f21b19c76794aa63d31eb1b71bd70b14a69a Mon Sep 17 00:00:00 2001 From: Vee Satayamas Date: Thu, 30 Jan 2020 09:02:29 +0000 Subject: [PATCH 1/3] Add a Dockerfile for training on CPU --- Dockerfile.cpu | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 Dockerfile.cpu diff --git a/Dockerfile.cpu b/Dockerfile.cpu new file mode 100644 index 00000000..7b41b8ae --- /dev/null +++ b/Dockerfile.cpu @@ -0,0 +1,51 @@ +FROM nvidia/cuda:9.0-devel +ENV LANG=C.UTF-8 +RUN apt update && \ + apt upgrade -y && \ + apt install -y ruby wget git cmake g++ libboost-all-dev \ + doxygen graphviz libblas-dev libopenblas-dev \ + libz-dev libssl-dev zlib1g-dev libbz2-dev liblzma-dev \ + libprotobuf9v5 protobuf-compiler libprotobuf-dev \ + python3-dev python3-numpy python3-setuptools +RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + wget https://apt.repos.intel.com/setup/intelproducts.list -O /etc/apt/sources.list.d/intelproducts.list && \ + apt update && \ + apt install -y intel-mkl-64bit-2020.0-088 +RUN cd / && \ + git clone https://github.com/aboSamoor/pycld2.git && \ + cd pycld2 && \ + git checkout e3ac86e && \ + python3 setup.py build && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/saffsd/langid.py.git && \ + cd langid.py && \ + git checkout 4153583 && \ + python3 setup.py build && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/Helsinki-NLP/OpusTools.git && \ + cd OpusTools && \ + git checkout d33b9f3 && \ + cd opustools_pkg && \ + python3 setup.py build && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/marian-nmt/marian-dev && \ + cd marian-dev && \ + git checkout 1.8.0 && \ + mkdir build && \ + cd build && \ + cmake .. \ + -DUSE_SENTENCEPIECE=on \ + -DCOMPILE_CPU=on \ + -DPROTOBUF_LIBRARY=/usr/lib/x86_64-linux-gnu/libprotobuf.so.9 \ + -DPROTOBUF_INCLUDE_DIR=/usr/include/google/protobuf \ + -DPROTOBUF_PROTOC_EXECUTABLE=/usr/bin/protoc && \ + make -j `nproc` +RUN cd / && \ + git clone --depth 1 https://github.com/moses-smt/mosesdecoder.git && \ + cd /mosesdecoder && \ + ./bjam -j8 From 683d9a5a66c4a07c2f716f12cafdd9ef135ad449 Mon Sep 17 00:00:00 2001 From: Vee Satayamas Date: Mon, 3 Feb 2020 15:41:34 +0700 Subject: [PATCH 2/3] Add Dockerfile for GPU --- Dockerfile.gpu | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Dockerfile.gpu diff --git a/Dockerfile.gpu b/Dockerfile.gpu new file mode 100644 index 00000000..afae2e2d --- /dev/null +++ b/Dockerfile.gpu @@ -0,0 +1,63 @@ +FROM nvidia/cuda:9.0-devel +ENV LANG=C.UTF-8 +RUN apt update && \ + apt upgrade -y && \ + apt install -y ruby wget git cmake g++ libboost-all-dev \ + doxygen graphviz libblas-dev libopenblas-dev \ + libz-dev libssl-dev zlib1g-dev libbz2-dev liblzma-dev \ + libprotobuf9v5 protobuf-compiler libprotobuf-dev \ + python3-dev python3-numpy python3-setuptools \ + cython3 +RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + wget https://apt.repos.intel.com/setup/intelproducts.list -O /etc/apt/sources.list.d/intelproducts.list && \ + apt update && \ + apt install -y intel-mkl-64bit-2020.0-088 +RUN cd / && \ + git clone https://github.com/aboSamoor/pycld2.git && \ + cd pycld2 && \ + git checkout e3ac86e && \ + python3 setup.py build && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/saffsd/langid.py.git && \ + cd langid.py && \ + git checkout 4153583 && \ + python3 setup.py build && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/Helsinki-NLP/OpusTools.git && \ + cd OpusTools && \ + git checkout d33b9f3 && \ + cd opustools_pkg && \ + python3 setup.py build && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/robertostling/eflomal.git && \ + cd eflomal && \ + git checkout 7b97f19 && \ + make install && \ + python3 setup.py install +RUN cd / && \ + git clone https://github.com/marian-nmt/marian-dev && \ + cd marian-dev && \ + git checkout 1.8.0 && \ + mkdir build && \ + cd build && \ + cmake .. \ + -DUSE_SENTENCEPIECE=on \ + -DPROTOBUF_LIBRARY=/usr/lib/x86_64-linux-gnu/libprotobuf.so.9 \ + -DPROTOBUF_INCLUDE_DIR=/usr/include/google/protobuf \ + -DPROTOBUF_PROTOC_EXECUTABLE=/usr/bin/protoc && \ + make -j `nproc` +RUN cd / && \ + git clone --depth 1 https://github.com/moses-smt/mosesdecoder.git && \ + cd /mosesdecoder && \ + ./bjam -j8 +RUN cd / && \ + git clone https://github.com/clab/fast_align.git && \ + cd fast_align && \ + git checkout cab1e9a && \ + cmake . && \ + make From 07dfd31259bd1bd33d880d8ddfabb0bade7a4d2c Mon Sep 17 00:00:00 2001 From: Vee Satayamas Date: Mon, 3 Feb 2020 16:03:26 +0700 Subject: [PATCH 3/3] Add more aligners --- Dockerfile.cpu | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 7b41b8ae..e3eec32b 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -6,7 +6,8 @@ RUN apt update && \ doxygen graphviz libblas-dev libopenblas-dev \ libz-dev libssl-dev zlib1g-dev libbz2-dev liblzma-dev \ libprotobuf9v5 protobuf-compiler libprotobuf-dev \ - python3-dev python3-numpy python3-setuptools + python3-dev python3-numpy python3-setuptools \ + cython3 RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ @@ -32,6 +33,12 @@ RUN cd / && \ cd opustools_pkg && \ python3 setup.py build && \ python3 setup.py install +RUN cd / && \ + git clone https://github.com/robertostling/eflomal.git && \ + cd eflomal && \ + git checkout 7b97f19 && \ + make install && \ + python3 setup.py install RUN cd / && \ git clone https://github.com/marian-nmt/marian-dev && \ cd marian-dev && \ @@ -49,3 +56,9 @@ RUN cd / && \ git clone --depth 1 https://github.com/moses-smt/mosesdecoder.git && \ cd /mosesdecoder && \ ./bjam -j8 +RUN cd / && \ + git clone https://github.com/clab/fast_align.git && \ + cd fast_align && \ + git checkout cab1e9a && \ + cmake . && \ + make