From c64deb50a888079bb11a579a99c4df317a93d21d Mon Sep 17 00:00:00 2001 From: abhi-agg <66322306+abhi-agg@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:30:39 +0100 Subject: [PATCH 01/14] Imported CI scripts from mozilla/bergamot-translator-old (#1) * CircleCI config, docs and badge * Increase CircleCI RAM from 4gb to 16gb Co-authored-by: Motin --- .circleci/config.yml | 19 ++++++++++++ .gitignore | 2 +- README.md | 2 ++ build-wasm.sh | 70 ++++++++++++++++++++++++++++++++++++++++++++ doc/CI.md | 22 ++++++++++++++ 5 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 .circleci/config.yml create mode 100755 build-wasm.sh create mode 100644 doc/CI.md diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..6bc00fe --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,19 @@ +version: 2.1 +jobs: + build: + docker: + - image: 'emscripten/emsdk:2.0.9' + resource_class: xlarge + + working_directory: ~/checkout + + steps: + - checkout + + - run: + name: Build WASM + command: bash build-wasm.sh + + - store_artifacts: + path: "build-wasm/wasm" + destination: "build-wasm/wasm" diff --git a/.gitignore b/.gitignore index 3acc4a5..840e69a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,6 @@ _deps wasm/test_page/node_modules -build-* +build-wasm models wasm/test_page/bergamot-translator-worker.* diff --git a/README.md b/README.md index 2b7f0a6..92fb874 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Bergamot Translator +[![CircleCI badge](https://img.shields.io/circleci/project/github/mozilla/bergamot-translator/main.svg?label=CircleCI)](https://circleci.com/gh/mozilla/bergamot-translator/) + Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt.github.io/) framework based) neural machine translation functionality in accordance with the [Bergamot](https://browser.mt/) project that focuses on improving client-side machine translation in a web browser. ## Build Instructions diff --git a/build-wasm.sh b/build-wasm.sh new file mode 100755 index 0000000..a4ea883 --- /dev/null +++ b/build-wasm.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Usage: ./build-wasm.sh + +set -e +set -x + +# Run script from the context of the script-containing directory +cd "$(dirname $0)" + +# This file replicates the instructions found in ./README.md under "Build WASM" +# with slight adjustments to be able to run the build script multiple times without having to clone all dependencies +# as per "As long as you don't update any submodule, just follow steps in `4.ii` to recompile." + +# 1. Download and Install Emscripten using following instructions (unless the EMSDK env var is already set) +if [ "$EMSDK" == "" ]; then + EMSDK_UPDATE_REQUIRED=0 + if [ ! -d "emsdk" ]; then + git clone https://github.com/emscripten-core/emsdk.git + EMSDK_UPDATE_REQUIRED=1 + else + cd emsdk + git fetch + # Only pull if necessary + if [ $(git rev-parse HEAD) != $(git rev-parse @{u}) ]; then + git pull --ff-only + EMSDK_UPDATE_REQUIRED=1 + fi + cd - + fi + if [ "$EMSDK_UPDATE_REQUIRED" == "1" ]; then + cd emsdk + ./emsdk install latest + ./emsdk activate latest + cd - + fi + source ./emsdk/emsdk_env.sh +fi + +# 3. Download models (only required if you want to package files in wasm binary) +if [ ! -d "bergamot-models" ]; then + git clone https://github.com/mozilla-applied-ml/bergamot-models +else + cd bergamot-models + git fetch + # Only pull if necessary + if [ $(git rev-parse HEAD) != $(git rev-parse @{u}) ]; then + git pull --ff-only + fi + cd - +fi +mkdir -p models +rm -rf models/* +cp -rf bergamot-models/* models +gunzip models/*/* + +# 4. Compile +# 1. Create a folder where you want to build all the artefacts (`build-wasm` in this case) +if [ ! -d "build-wasm" ]; then + mkdir build-wasm +fi +cd build-wasm + +# 2. Compile the artefacts +emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR="../models/" ../ +emmake make -j + +# The artefacts (.js and .wasm files) will be available in `wasm` folder of build directory ("build-wasm" in this case). + +exit 0 diff --git a/doc/CI.md b/doc/CI.md new file mode 100644 index 0000000..2f29b02 --- /dev/null +++ b/doc/CI.md @@ -0,0 +1,22 @@ +# Continuous Integration + +[Circle CI](https://circleci.com/) is used for continuous integration. Configured via `./.circleci/config.yml`. + +## Run Circle CI locally (requires Docker) + +1. [Install the CircleCI local cli](https://circleci.com/docs/2.0/local-cli/#installation) +2. Validate Circle CI configuration (useful exercise before pushing any changes to the configuration) + +```shell +circleci config validate -c .circleci/config.yml +``` + +3. To better mimic the starting point for CI, commit your changes and clone your repository into a clean directory then run CircleCI inside that directory: + +```shell +git clone . /tmp/$(basename $PWD) +cd /tmp/$(basename $PWD) +circleci build +``` + +Note: Steps related to caching and uploading/storing artifacts will report as failed locally. This is not necessarily a problem, they are designed to fail since the operations are not supported locally by the CircleCI build agent. From e96d7047a7042336908c06197180b4891e76a66d Mon Sep 17 00:00:00 2001 From: Andre Natal Date: Wed, 10 Mar 2021 19:57:13 -0800 Subject: [PATCH 02/14] Enable SIMD wormhole in circle ci build scripts - Modify the APIs that compile & instantiate WASM module --- build-wasm.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-wasm.sh b/build-wasm.sh index a4ea883..d490730 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -65,6 +65,11 @@ cd build-wasm emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR="../models/" ../ emmake make -j +# 3. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts +sed -i.bak 's/var result = WebAssembly.instantiateStreaming(response, info);/var result = WebAssembly.instantiateStreaming(response, info,{simdWormhole:true});/g' wasm/bergamot-translator-worker.js +sed -i.bak 's/return WebAssembly.instantiate(binary, info);/return WebAssembly.instantiate(binary, info, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js +sed -i.bak 's/var module = new WebAssembly.Module(bytes);/var module = new WebAssembly.Module(bytes, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + # The artefacts (.js and .wasm files) will be available in `wasm` folder of build directory ("build-wasm" in this case). exit 0 From 8c8913e2effa02b0e763fdb249d1074443cb44e1 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 11 Mar 2021 11:07:21 +0100 Subject: [PATCH 03/14] Use intgemm models in wasm test_page --- wasm/test_page/bergamot.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/test_page/bergamot.html b/wasm/test_page/bergamot.html index b4d2027..d720710 100644 --- a/wasm/test_page/bergamot.html +++ b/wasm/test_page/bergamot.html @@ -70,7 +70,7 @@ En consecuencia, durante el año 2011 se introdujeron 180 proyectos de ley que r // Set the Model Configuration as YAML formatted string. // For available configuration options, please check: https://marian-nmt.github.io/docs/cmd/marian-decoder/ const modelConfig = `models: - - /${languagePair}/model.${languagePair}.npz + - /${languagePair}/model.${languagePair}.intgemm.bin vocabs: - /${vocabLanguagePair}/vocab.${vocabLanguagePair}.spm - /${vocabLanguagePair}/vocab.${vocabLanguagePair}.spm From 4f124e7976a163b5d7dbead2e70a17eec11d81ed Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 11 Mar 2021 11:23:04 +0100 Subject: [PATCH 04/14] Enabled simdwormhole in github workflows --- .github/workflows/macos-custom-marian-wasm.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/macos-custom-marian-wasm.yml b/.github/workflows/macos-custom-marian-wasm.yml index 3e49369..373a262 100644 --- a/.github/workflows/macos-custom-marian-wasm.yml +++ b/.github/workflows/macos-custom-marian-wasm.yml @@ -33,6 +33,13 @@ jobs: working-directory: build-wasm run: emmake make -j2 + - name: Instantiate simd wormhole + working-directory: build-wasm + run: | + sed -i.bak 's/var result = WebAssembly.instantiateStreaming(response, info);/var result = WebAssembly.instantiateStreaming(response, info, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + sed -i.bak 's/return WebAssembly.instantiate(binary, info);/return WebAssembly.instantiate(binary, info, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + sed -i.bak 's/var module = new WebAssembly.Module(bytes);/var module = new WebAssembly.Module(bytes, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + - name: Check artifacts working-directory: build-wasm run: | From 6e7b7c71ec34883f7fdbf67ef7f56db0059ef328 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 11 Mar 2021 11:41:07 +0100 Subject: [PATCH 05/14] Updates README for enabling simdwormhole in WASM APIs --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 92fb874..299cc87 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,12 @@ Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt. emmake make -j ``` + 3. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts + ``` + sed -i.bak 's/var result = WebAssembly.instantiateStreaming(response, info);/var result = WebAssembly.instantiateStreaming(response, info, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + sed -i.bak 's/return WebAssembly.instantiate(binary, info);/return WebAssembly.instantiate(binary, info, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + sed -i.bak 's/var module = new WebAssembly.Module(bytes);/var module = new WebAssembly.Module(bytes, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js + ``` The artefacts (.js and .wasm files) will be available in `wasm` folder of build directory ("build-wasm" in this case). #### Recompiling From a2d66500976fdbd2b586e240d920b096fcedd529 Mon Sep 17 00:00:00 2001 From: Andre Natal Date: Wed, 10 Mar 2021 17:59:07 -0800 Subject: [PATCH 06/14] Patch to load just production ready models --- README.md | 2 +- build-wasm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 299cc87..cb1bc03 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt. ```bash mkdir models git clone https://github.com/mozilla-applied-ml/bergamot-models - cp -rf bergamot-models/* models + cp -rf bergamot-models/prod/* models gunzip models/*/* ``` diff --git a/build-wasm.sh b/build-wasm.sh index d490730..8349bf8 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -51,7 +51,7 @@ else fi mkdir -p models rm -rf models/* -cp -rf bergamot-models/* models +cp -rf bergamot-models/prod/* models gunzip models/*/* # 4. Compile From d1ecd007a696c133112f3b88fa3cdd92bfc2e61d Mon Sep 17 00:00:00 2001 From: Motin Date: Thu, 11 Mar 2021 10:27:47 +0200 Subject: [PATCH 07/14] Shallow clone bergamot-models repo since it has such a large history --- README.md | 2 +- build-wasm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cb1bc03..765d7cb 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt. If you want to package bergamot project specific models, please follow these instructions: ```bash mkdir models - git clone https://github.com/mozilla-applied-ml/bergamot-models + git clone --depth 1 --branch main --single-branch https://github.com/mozilla-applied-ml/bergamot-models cp -rf bergamot-models/prod/* models gunzip models/*/* ``` diff --git a/build-wasm.sh b/build-wasm.sh index 8349bf8..9b08aa7 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -39,7 +39,7 @@ fi # 3. Download models (only required if you want to package files in wasm binary) if [ ! -d "bergamot-models" ]; then - git clone https://github.com/mozilla-applied-ml/bergamot-models + git clone --depth 1 --branch main --single-branch https://github.com/mozilla-applied-ml/bergamot-models else cd bergamot-models git fetch From bf28edad82660db6a9524ae0e7f05dab007f78ce Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 11 Mar 2021 14:25:42 +0100 Subject: [PATCH 08/14] Improved wasm test_page - test page can load all 5 language pairs - Use intgemm.aplha* models - start_server.sh script automatically enable simdwormhole via APIs that instantiate WASM module --- wasm/test_page/bergamot.html | 5 ++++- wasm/test_page/start_server.sh | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/wasm/test_page/bergamot.html b/wasm/test_page/bergamot.html index d720710..4f1f2a0 100644 --- a/wasm/test_page/bergamot.html +++ b/wasm/test_page/bergamot.html @@ -32,6 +32,9 @@ + + + @@ -70,7 +73,7 @@ En consecuencia, durante el año 2011 se introdujeron 180 proyectos de ley que r // Set the Model Configuration as YAML formatted string. // For available configuration options, please check: https://marian-nmt.github.io/docs/cmd/marian-decoder/ const modelConfig = `models: - - /${languagePair}/model.${languagePair}.intgemm.bin + - /${languagePair}/model.${languagePair}.intgemm.alphas.bin vocabs: - /${vocabLanguagePair}/vocab.${vocabLanguagePair}.spm - /${vocabLanguagePair}/vocab.${vocabLanguagePair}.spm diff --git a/wasm/test_page/start_server.sh b/wasm/test_page/start_server.sh index b83344b..153624e 100644 --- a/wasm/test_page/start_server.sh +++ b/wasm/test_page/start_server.sh @@ -1,8 +1,17 @@ #!/bin/bash - +echo "Start: Copying artifacts in local folder------" cp ../../build-wasm/wasm/bergamot-translator-worker.data . cp ../../build-wasm/wasm/bergamot-translator-worker.js . cp ../../build-wasm/wasm/bergamot-translator-worker.wasm . cp ../../build-wasm/wasm/bergamot-translator-worker.worker.js . +echo "Done----" + +echo "Start: Enabling wormhole via APIs that compile and instantiate wasm module-------" +sed -i.bak 's/var result = WebAssembly.instantiateStreaming(response, info);/var result = WebAssembly.instantiateStreaming(response, info, {simdWormhole:true});/g' bergamot-translator-worker.js +sed -i.bak 's/return WebAssembly.instantiate(binary, info);/return WebAssembly.instantiate(binary, info, {simdWormhole:true});/g' bergamot-translator-worker.js +sed -i.bak 's/var module = new WebAssembly.Module(bytes);/var module = new WebAssembly.Module(bytes, {simdWormhole:true});/g' bergamot-translator-worker.js +echo "Done: Enabling wormhole via APIs that compile and instantiate wasm module--------" + npm install +echo "Start httpserver" node bergamot-httpserver.js \ No newline at end of file From f5dffeb5ca276886b7754644ea7f51d3e1c7d860 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Wed, 14 Apr 2021 12:03:33 +0200 Subject: [PATCH 09/14] Downgraded resource class to 'medium' for circle ci - Also restricted parallel make compilation to 3 --- .circleci/config.yml | 2 +- build-wasm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6bc00fe..6818bc2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ jobs: build: docker: - image: 'emscripten/emsdk:2.0.9' - resource_class: xlarge + resource_class: medium working_directory: ~/checkout diff --git a/build-wasm.sh b/build-wasm.sh index 9b08aa7..f92d8ba 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -63,7 +63,7 @@ cd build-wasm # 2. Compile the artefacts emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR="../models/" ../ -emmake make -j +emmake make -j3 # 3. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts sed -i.bak 's/var result = WebAssembly.instantiateStreaming(response, info);/var result = WebAssembly.instantiateStreaming(response, info,{simdWormhole:true});/g' wasm/bergamot-translator-worker.js From a7f6bb51d980d4dd7b95e577933750bbe0bef62f Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Wed, 14 Apr 2021 14:51:08 +0200 Subject: [PATCH 10/14] Minor cleanup in build-wasm.sh file --- build-wasm.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build-wasm.sh b/build-wasm.sh index f92d8ba..5348fe9 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -66,10 +66,8 @@ emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR="../models/" ../ emmake make -j3 # 3. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts -sed -i.bak 's/var result = WebAssembly.instantiateStreaming(response, info);/var result = WebAssembly.instantiateStreaming(response, info,{simdWormhole:true});/g' wasm/bergamot-translator-worker.js -sed -i.bak 's/return WebAssembly.instantiate(binary, info);/return WebAssembly.instantiate(binary, info, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js -sed -i.bak 's/var module = new WebAssembly.Module(bytes);/var module = new WebAssembly.Module(bytes, {simdWormhole:true});/g' wasm/bergamot-translator-worker.js +bash ../wasm/patch-artifacts-enable-wormhole.sh -# The artefacts (.js and .wasm files) will be available in `wasm` folder of build directory ("build-wasm" in this case). +# The artifacts (.js and .wasm files) will be available in `wasm` folder of build directory ("build-wasm" in this case). exit 0 From d8f7e51792fab8b34418734a5a8e55ab19e66ecb Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Tue, 4 May 2021 12:32:41 +0200 Subject: [PATCH 11/14] Minor README change - Changed "browsermt" to "mozilla" --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9dd47c4..eaf4abe 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt. ### Build Natively 1. Clone the repository using these instructions: ```bash - git clone https://github.com/browsermt/bergamot-translator + git clone https://github.com/mozilla/bergamot-translator cd bergamot-translator ``` 2. Compile @@ -34,7 +34,7 @@ Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt. 2. Clone the repository using these instructions: ```bash - git clone https://github.com/browsermt/bergamot-translator + git clone https://github.com/mozilla/bergamot-translator cd bergamot-translator ``` From c478a626a8084f0e7b210a25b4645a5a79b4d83d Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Tue, 4 May 2021 11:00:01 +0200 Subject: [PATCH 12/14] Updating ci scripts for the latest upstream changes - The upstream browsermt/bergamot-translator builds the wasm artifacts in top level build folder now --- .circleci/config.yml | 17 +++++++++++++++-- build-wasm.sh | 9 +++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6818bc2..6465469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,19 @@ jobs: name: Build WASM command: bash build-wasm.sh + - run: + name: Check artifacts + working_directory: build-wasm + command: | + ls -all bergamot* + if ls bergamot*.wasm &>/dev/null && ls bergamot*.js &>/dev/null && ls bergamot*.data &>/dev/null + then + echo "Artifacts Successfully Generated" + else + echo "Failure: Artifacts Not Present" + exit 1 + fi + - store_artifacts: - path: "build-wasm/wasm" - destination: "build-wasm/wasm" + path: "build-wasm" + destination: "build-wasm" diff --git a/build-wasm.sh b/build-wasm.sh index 5348fe9..fff92a4 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -37,7 +37,7 @@ if [ "$EMSDK" == "" ]; then source ./emsdk/emsdk_env.sh fi -# 3. Download models (only required if you want to package files in wasm binary) +# 3. Download models (required to perform inference using build artifacts) if [ ! -d "bergamot-models" ]; then git clone --depth 1 --branch main --single-branch https://github.com/mozilla-applied-ml/bergamot-models else @@ -53,21 +53,22 @@ mkdir -p models rm -rf models/* cp -rf bergamot-models/prod/* models gunzip models/*/* +find models \( -type f -name "model*" -or -type f -name "lex*" \) -delete # 4. Compile -# 1. Create a folder where you want to build all the artefacts (`build-wasm` in this case) +# 1. Create a folder where you want to build all the artifacts (`build-wasm` in this case) if [ ! -d "build-wasm" ]; then mkdir build-wasm fi cd build-wasm -# 2. Compile the artefacts +# 2. Compile the artifacts emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR="../models/" ../ emmake make -j3 # 3. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts bash ../wasm/patch-artifacts-enable-wormhole.sh -# The artifacts (.js and .wasm files) will be available in `wasm` folder of build directory ("build-wasm" in this case). +# The artifacts (.js and .wasm files) will be available in the build directory ("build-wasm" in this case). exit 0 From 743ebcd3bcfdf5e4b141a9ed3478eb79de19dece Mon Sep 17 00:00:00 2001 From: Motin Date: Tue, 4 May 2021 15:25:12 +0300 Subject: [PATCH 13/14] Extension desired changes (#129) * Enable worker file system * Avoid node.js-code in emscripten glue-code --- wasm/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wasm/CMakeLists.txt b/wasm/CMakeLists.txt index a785ba6..ba13f5c 100644 --- a/wasm/CMakeLists.txt +++ b/wasm/CMakeLists.txt @@ -20,6 +20,12 @@ if (NOT PACKAGE_DIR STREQUAL "") set(LINKER_FLAGS "${LINKER_FLAGS} --preload-file ${REALPATH_PACKAGE_DIR}@/") endif() +# Enable worker file system +set(LINKER_FLAGS "${LINKER_FLAGS} -lworkerfs.js") + +# Avoid node.js-code in emscripten glue-code +set(LINKER_FLAGS "${LINKER_FLAGS} -s ENVIRONMENT=web,worker") + set_target_properties(bergamot-translator-worker PROPERTIES SUFFIX ".js" LINK_FLAGS ${LINKER_FLAGS} From 8a6c7b44a30f4458a2fb414a1c98f4cf068c95f3 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Wed, 12 May 2021 08:59:32 +0200 Subject: [PATCH 14/14] Avoid packaging vocab files into wasm binary in CI builds - We don't need to package vocab files into wasm binary any more as a sync with upstream enabled passing vocabs as bytes --- .circleci/config.yml | 2 +- build-wasm.sh | 20 +------------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6465469..69ae356 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: working_directory: build-wasm command: | ls -all bergamot* - if ls bergamot*.wasm &>/dev/null && ls bergamot*.js &>/dev/null && ls bergamot*.data &>/dev/null + if ls bergamot*.wasm &>/dev/null && ls bergamot*.js &>/dev/null then echo "Artifacts Successfully Generated" else diff --git a/build-wasm.sh b/build-wasm.sh index fff92a4..bdc00d5 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -37,24 +37,6 @@ if [ "$EMSDK" == "" ]; then source ./emsdk/emsdk_env.sh fi -# 3. Download models (required to perform inference using build artifacts) -if [ ! -d "bergamot-models" ]; then - git clone --depth 1 --branch main --single-branch https://github.com/mozilla-applied-ml/bergamot-models -else - cd bergamot-models - git fetch - # Only pull if necessary - if [ $(git rev-parse HEAD) != $(git rev-parse @{u}) ]; then - git pull --ff-only - fi - cd - -fi -mkdir -p models -rm -rf models/* -cp -rf bergamot-models/prod/* models -gunzip models/*/* -find models \( -type f -name "model*" -or -type f -name "lex*" \) -delete - # 4. Compile # 1. Create a folder where you want to build all the artifacts (`build-wasm` in this case) if [ ! -d "build-wasm" ]; then @@ -63,7 +45,7 @@ fi cd build-wasm # 2. Compile the artifacts -emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR="../models/" ../ +emcmake cmake -DCOMPILE_WASM=on ../ emmake make -j3 # 3. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts