diff --git a/.gitignore b/.gitignore index d7d931f..5a73aac 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ *.swo wasm/test_page/node_modules -build-wasm +build-* models wasm/test_page/bergamot-translator-worker.* diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 0000000..583a588 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,55 @@ +# -*- mode: makefile-gmake; indent-tabs-mode: true; tab-width: 4 -*- +SHELL = bash +PWD = $(shell pwd) +WASM_IMAGE = local/bergamot-translator-build-wasm + +all: wasm-image compile-wasm + +# Build the Docker image for WASM builds +wasm-image: + docker build -t local/bergamot-translator-build-wasm ./wasm/ + +# Commands for compilation: +cmake_cmd = cmake + +wasm_cmake_cmd = ${cmake_cmd} +wasm_cmake_cmd += -DCOMPILE_WASM=on +wasm_cmake_cmd += -DProtobuf_INCLUDE_DIR=/usr/opt/protobuf-wasm-lib/dist/include +wasm_cmake_cmd += -DProtobuf_LIBRARY=/usr/opt/protobuf-wasm-lib/dist/lib/libprotobuf.a +wasm_cmake_cmd += -DPACKAGE_DIR=/repo/models + +make_cmd = make +#make_cmd += VERBOSE=1 + +# ... and running things on Docker +docker_mounts = ${PWD}/..:/repo +docker_mounts += ${HOME}/.ccache:/.ccache +run_on_docker = docker run --rm +run_on_docker += $(addprefix -v, ${docker_mounts}) +run_on_docker += ${INTERACTIVE_DOCKER_SESSION} + +${HOME}/.ccache: + mkdir -p $@ + +# Remove the bergamot-translator WASM build dir, forcing a clean compilation attempt +clean-wasm: BUILD_DIR = /repo/build-wasm-docker +clean-wasm: ${HOME}/.ccache + ${run_on_docker} ${WASM_IMAGE} bash -c '(rm -rf ${BUILD_DIR} || true)' + +# Compile bergamot-translator to WASM +compile-wasm: BUILD_DIR = /repo/build-wasm-docker +compile-wasm: ${HOME}/.ccache + ${run_on_docker} ${WASM_IMAGE} bash -c 'mkdir -p ${BUILD_DIR} && \ +cd ${BUILD_DIR} && \ +(emcmake ${wasm_cmake_cmd} .. && \ +(emmake ${make_cmd}) || \ +rm CMakeCache.txt)' + +# Start interactive shells for development / debugging purposes +native-shell: INTERACTIVE_DOCKER_SESSION = -it +native-shell: + ${run_on_docker} ${NATIVE_IMAGE} bash + +wasm-shell: INTERACTIVE_DOCKER_SESSION = -it +wasm-shell: + ${run_on_docker} ${WASM_IMAGE} bash diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..d98456a --- /dev/null +++ b/docker/README.md @@ -0,0 +1,27 @@ +## WASM + +Prepare docker image for WASM compilation: + +```bash +make wasm-image +``` + +Compile to wasm: + +```bash +make compile-wasm +``` + +## Debugging + +Remove the marian-decoder build dir, forcing the next compilation attempt to start from scratch: + +```bash +make clean-wasm +``` + +Enter a docker container shell for manually running commands: + +```bash +make wasm-shell +``` diff --git a/docker/wasm/Dockerfile b/docker/wasm/Dockerfile new file mode 100644 index 0000000..f309662 --- /dev/null +++ b/docker/wasm/Dockerfile @@ -0,0 +1,36 @@ +FROM emscripten/emsdk:2.0.9 + +# Install specific version of CMake +WORKDIR /usr +RUN wget https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Linux-x86_64.tar.gz -qO-\ + | tar xzf - --strip-components 1 + +# Install Python and Java (needed for Closure Compiler minification) +RUN apt-get update \ + && apt-get install -y \ + python3 \ + default-jre + +# Deps to compile protobuf from source + the protoc binary which we need natively +RUN apt-get update -y && apt-get --no-install-recommends -y install \ + protobuf-compiler \ + autoconf \ + autotools-dev \ + automake \ + autogen \ + libtool && ln -s /usr/bin/libtoolize /usr/bin/libtool \ + && mkdir -p /usr/opt \ + && cd /usr/opt \ + && git clone https://github.com/menduz/protobuf-wasm-lib + +RUN cd /usr/opt/protobuf-wasm-lib \ + && /bin/bash -c "BRANCH=v3.6.1 ./prepare.sh" +RUN cd /usr/opt/protobuf-wasm-lib/protobuf \ + && bash -x ../build.sh +RUN cp /usr/bin/protoc /usr/opt/protobuf-wasm-lib/dist/bin/protoc + +RUN apt-get --no-install-recommends -y install \ + libprotobuf-dev + +# Necessary for benchmarking +RUN pip3 install sacrebleu diff --git a/wasm/test_page/start_server.sh b/wasm/test_page/start_server.sh index b83344b..b0b5be1 100644 --- a/wasm/test_page/start_server.sh +++ b/wasm/test_page/start_server.sh @@ -1,8 +1,8 @@ #!/bin/bash -cp ../../build-wasm/wasm/bergamot-translator-worker.data . -cp ../../build-wasm/wasm/bergamot-translator-worker.js . -cp ../../build-wasm/wasm/bergamot-translator-worker.wasm . -cp ../../build-wasm/wasm/bergamot-translator-worker.worker.js . +cp ../../build-wasm-docker/wasm/bergamot-translator-worker.data . +cp ../../build-wasm-docker/wasm/bergamot-translator-worker.js . +cp ../../build-wasm-docker/wasm/bergamot-translator-worker.wasm . +cp ../../build-wasm-docker/wasm/bergamot-translator-worker.worker.js . npm install node bergamot-httpserver.js \ No newline at end of file