mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-10-05 17:38:40 +03:00
Remove wormhole references (#459)
* Remove warmhole references * Remove more references to the WORMHOLE * Update marian to wormhole removed marian * Whoops --------- Co-authored-by: Jelmer van der Linde <jelmer@ikhoefgeen.nl>
This commit is contained in:
parent
ca954670aa
commit
534ed37a3d
@ -1,52 +1,6 @@
|
||||
version: 2.1
|
||||
jobs:
|
||||
build-with-wormhole:
|
||||
docker:
|
||||
- image: 'emscripten/emsdk:3.1.8'
|
||||
resource_class: medium
|
||||
|
||||
working_directory: ~/checkout
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
- run:
|
||||
name: Build WASM WORMHOLE
|
||||
command: |
|
||||
bash build-wasm.sh WORMHOLE
|
||||
|
||||
- run:
|
||||
name: Check artifacts
|
||||
working_directory: build-wasm
|
||||
command: |
|
||||
ARTIFACT_BASE="bergamot-translator-worker"
|
||||
ARTIFACT_SUFFIX="with-wormhole"
|
||||
ARTIFACT_FINAL=$ARTIFACT_BASE-$ARTIFACT_SUFFIX
|
||||
|
||||
if [[ -f "$ARTIFACT_BASE.js" && -f "$ARTIFACT_BASE.wasm" ]]; then
|
||||
echo "Artifacts Successfully Generated"
|
||||
mkdir ../artifacts
|
||||
cp $ARTIFACT_BASE.wasm ../artifacts/$ARTIFACT_FINAL.wasm
|
||||
cp $ARTIFACT_BASE.js ../artifacts/$ARTIFACT_FINAL.js
|
||||
cd ../artifacts
|
||||
shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
|
||||
ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
|
||||
cp ../BERGAMOT_VERSION .
|
||||
else
|
||||
echo "Failure: Artifacts Not Present"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- artifacts/*
|
||||
|
||||
- store_artifacts:
|
||||
path: "artifacts"
|
||||
destination: "wasm-wormhole"
|
||||
|
||||
build-without-wormhole:
|
||||
build:
|
||||
docker:
|
||||
- image: 'emscripten/emsdk:3.1.8'
|
||||
resource_class: medium
|
||||
@ -66,8 +20,7 @@ jobs:
|
||||
working_directory: build-wasm
|
||||
command: |
|
||||
ARTIFACT_BASE="bergamot-translator-worker"
|
||||
ARTIFACT_SUFFIX="without-wormhole"
|
||||
ARTIFACT_FINAL=$ARTIFACT_BASE-$ARTIFACT_SUFFIX
|
||||
ARTIFACT_FINAL=$ARTIFACT_BASE
|
||||
|
||||
if [[ -f "$ARTIFACT_BASE.js" && -f "$ARTIFACT_BASE.wasm" ]]; then
|
||||
echo "Artifacts Successfully Generated"
|
||||
@ -75,8 +28,8 @@ jobs:
|
||||
cp $ARTIFACT_BASE.wasm ../artifacts/$ARTIFACT_FINAL.wasm
|
||||
cp $ARTIFACT_BASE.js ../artifacts/$ARTIFACT_FINAL.js
|
||||
cd ../artifacts
|
||||
shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
|
||||
ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
|
||||
shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize
|
||||
ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize
|
||||
else
|
||||
echo "Failure: Artifacts Not Present"
|
||||
exit 1
|
||||
@ -89,7 +42,7 @@ jobs:
|
||||
|
||||
- store_artifacts:
|
||||
path: "artifacts"
|
||||
destination: "wasm-without-wormhole"
|
||||
destination: "wasm"
|
||||
|
||||
publish_to_github:
|
||||
docker:
|
||||
@ -106,18 +59,13 @@ jobs:
|
||||
name: "Publish Release on GitHub"
|
||||
command: |
|
||||
export TAG_VERSION=$(cat ./artifacts/BERGAMOT_VERSION)
|
||||
cat ./artifacts/sha256-filesize-without-wormhole ./artifacts/sha256-filesize-with-wormhole >> ./artifacts/sha256-filesize
|
||||
rm ./artifacts/sha256-filesize-without-wormhole ./artifacts/sha256-filesize-with-wormhole ./artifacts/BERGAMOT_VERSION
|
||||
rm ./artifacts/BERGAMOT_VERSION
|
||||
ghr -t ${GHTOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${TAG_VERSION} ./artifacts/
|
||||
|
||||
workflows:
|
||||
build:
|
||||
jobs:
|
||||
- build-with-wormhole:
|
||||
filters:
|
||||
tags:
|
||||
only: /^v.*/
|
||||
- build-without-wormhole:
|
||||
- build:
|
||||
filters:
|
||||
tags:
|
||||
only: /^v.*/
|
||||
@ -128,7 +76,6 @@ workflows:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
requires:
|
||||
- build-without-wormhole
|
||||
- build-with-wormhole
|
||||
- build
|
||||
|
||||
|
||||
|
3
.github/workflows/build.yml
vendored
3
.github/workflows/build.yml
vendored
@ -236,7 +236,7 @@ jobs:
|
||||
run: |
|
||||
mkdir -p build-wasm
|
||||
cd build-wasm
|
||||
emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ..
|
||||
emcmake cmake -DCOMPILE_WASM=on ..
|
||||
|
||||
|
||||
- name: "Compile"
|
||||
@ -276,7 +276,6 @@ jobs:
|
||||
name: wasm-artefacts
|
||||
if-no-files-found: error
|
||||
path: |
|
||||
# Without wormhole
|
||||
${{github.workspace}}/build-wasm/bergamot-translator-worker.js
|
||||
${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm
|
||||
${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak
|
||||
|
2
3rd_party/marian-dev
vendored
2
3rd_party/marian-dev
vendored
@ -1 +1 @@
|
||||
Subproject commit 8dbde0fd8e690ad8791fb7fc94dba7674ee7c77e
|
||||
Subproject commit 300a50f4251d978dc197d15bb7b296597b1eb221
|
@ -113,7 +113,6 @@ message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}")
|
||||
|
||||
if(COMPILE_WASM)
|
||||
# See https://github.com/emscripten-core/emscripten/blob/main/src/settings.js
|
||||
set(WORMHOLE ON CACHE BOOL "Use WASM wormhole in intgemm https://bugzilla.mozilla.org/show_bug.cgi?id=1672160")
|
||||
list(APPEND WASM_COMPILE_FLAGS
|
||||
-O3
|
||||
# Preserve whitespaces in JS even for release builds; this doesn't increase wasm binary size
|
||||
|
@ -41,12 +41,7 @@ To build a version that translates with higher speeds on Firefox Nightly browser
|
||||
|
||||
The wasm artifacts (.js and .wasm files) will be available in the build directory ("build-wasm" in this case).
|
||||
|
||||
2. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts
|
||||
```bash
|
||||
bash ../wasm/patch-artifacts-enable-wormhole.sh
|
||||
```
|
||||
|
||||
3. Patch generated artifacts to import GEMM library from a separate wasm module
|
||||
2. Patch generated artifacts to import GEMM library from a separate wasm module
|
||||
```bash
|
||||
bash ../wasm/patch-artifacts-import-gemm-module.sh
|
||||
```
|
||||
@ -57,7 +52,7 @@ To build a version that runs on all browsers (including Firefox Nightly) but tra
|
||||
```bash
|
||||
mkdir build-wasm
|
||||
cd build-wasm
|
||||
emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ../
|
||||
emcmake cmake -DCOMPILE_WASM=on ../
|
||||
emmake make -j2
|
||||
```
|
||||
|
||||
|
@ -2,34 +2,6 @@
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# Usage
|
||||
Usage="Build translator to wasm (with/without wormhole).
|
||||
|
||||
Usage: $(basename "$0") [WORMHOLE]
|
||||
|
||||
where:
|
||||
WORMHOLE An optional string argument
|
||||
- when specified on command line, builds wasm artifacts with wormhole
|
||||
- when not specified (the default behaviour), builds wasm artifacts without wormhole."
|
||||
|
||||
if [ "$#" -gt 1 ]; then
|
||||
echo "Illegal number of parameters passed"
|
||||
echo "$Usage"
|
||||
exit
|
||||
fi
|
||||
|
||||
WORMHOLE=false
|
||||
|
||||
if [ "$#" -eq 1 ]; then
|
||||
if [ "$1" = "WORMHOLE" ]; then
|
||||
WORMHOLE=true
|
||||
else
|
||||
echo "Illegal parameter passed"
|
||||
echo "$Usage"
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
|
||||
# Run script from the context of the script-containing directory
|
||||
cd "$(dirname $0)"
|
||||
|
||||
@ -66,19 +38,10 @@ if [ ! -d ${BUILD_DIRECTORY} ]; then
|
||||
fi
|
||||
cd ${BUILD_DIRECTORY}
|
||||
|
||||
if [ "$WORMHOLE" = true ]; then
|
||||
emcmake cmake -DCOMPILE_WASM=on ../
|
||||
else
|
||||
emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ../
|
||||
fi
|
||||
emcmake cmake -DCOMPILE_WASM=on ../
|
||||
emmake make -j2
|
||||
|
||||
# 2. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts
|
||||
if [ "$WORMHOLE" = true ]; then
|
||||
bash ../wasm/patch-artifacts-enable-wormhole.sh
|
||||
fi
|
||||
|
||||
# 3. Import GEMM library from a separate wasm module
|
||||
# 2. Import GEMM library from a separate wasm module
|
||||
bash ../wasm/patch-artifacts-import-gemm-module.sh
|
||||
|
||||
# The artifacts (.js and .wasm files) will be available in the build directory
|
||||
|
@ -32,18 +32,8 @@ Alternatively refer to the file `test_page/js/worker.js` that demonstrates how t
|
||||
|
||||
Provide the folder containing the wasm artifacts as the first argument of `start_server.sh` script (`../../build-wasm` in this case).
|
||||
|
||||
* Open any of the browsers below
|
||||
* Firefox Nightly +87: make sure the following prefs are on (about:config)
|
||||
```
|
||||
dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled = true
|
||||
javascript.options.wasm_simd = true
|
||||
javascript.options.wasm_simd_wormhole = true
|
||||
```
|
||||
* Open any browser (tested with latest Chrome/Firefox/Safari)
|
||||
|
||||
* Chrome Canary +90: start with the following argument
|
||||
```
|
||||
--js-flags="--experimental-wasm-simd"
|
||||
```
|
||||
|
||||
* Browse to the following page:
|
||||
```
|
||||
|
@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
usage="Patch wasm artifacts to enable wormhole via APIs that compile and instantiate wasm module.
|
||||
|
||||
Usage: $(basename "$0") [WASM_ARTIFACTS_FOLDER]
|
||||
|
||||
where:
|
||||
WASM_ARTIFACTS_FOLDER Folder containing wasm artifacts
|
||||
(An optional argument, if unspecified the default is: current folder)"
|
||||
|
||||
if [ "$#" -gt 1 ]; then
|
||||
echo "Illegal number of parameters passed"
|
||||
echo "$usage"
|
||||
exit
|
||||
fi
|
||||
|
||||
# Parse wasm artifacts folder if provided via script argument or set it to default
|
||||
WASM_ARTIFACTS_FOLDER=$PWD
|
||||
if [ "$#" -eq 1 ]; then
|
||||
if [ ! -e "$1" ]; then
|
||||
echo "Error: Folder \""$1"\" doesn't exist"
|
||||
exit
|
||||
fi
|
||||
WASM_ARTIFACTS_FOLDER="$1"
|
||||
fi
|
||||
|
||||
WASM_ARTIFACTS="$WASM_ARTIFACTS_FOLDER/bergamot-translator-worker.js"
|
||||
if [ ! -e "$WASM_ARTIFACTS" ]; then
|
||||
echo "Error: Artifact \"$WASM_ARTIFACTS\" doesn't exist"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Patching \"$WASM_ARTIFACTS\" to enable wormhole via APIs that compile and instantiate wasm module"
|
||||
sed -i.bak 's/WebAssembly.instantiateStreaming[[:space:]]*([[:space:]]*response[[:space:]]*,[[:space:]]*info[[:space:]]*)/WebAssembly.instantiateStreaming(response, info, {simdWormhole:true})/g' $WASM_ARTIFACTS
|
||||
sed -i.bak 's/WebAssembly.instantiate[[:space:]]*([[:space:]]*binary[[:space:]]*,[[:space:]]*info[[:space:]]*)/WebAssembly.instantiate(binary, info, {simdWormhole:true})/g' $WASM_ARTIFACTS
|
||||
sed -i.bak 's/WebAssembly.Module[[:space:]]*([[:space:]]*bytes[[:space:]]*)/WebAssembly.Module(bytes, {simdWormhole:true})/g' $WASM_ARTIFACTS
|
||||
echo "Done"
|
Loading…
Reference in New Issue
Block a user