mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-09-11 05:35:33 +03:00
Merge remote-tracking branch 'origin/wasm-integration' into jp/absorb-batch-translator
Merging wasm-integration. Single thread codepath seems functional. Multithreading is broken.
This commit is contained in:
commit
10dcb8f548
4
.gitignore
vendored
4
.gitignore
vendored
@ -16,3 +16,7 @@ CTestTestfile.cmake
|
||||
_deps
|
||||
|
||||
|
||||
wasm/test_page/node_modules
|
||||
build-*
|
||||
models
|
||||
wasm/test_page/bergamot-translator-worker.*
|
||||
|
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -1,6 +1,6 @@
|
||||
[submodule "3rd_party/ssplit-cpp"]
|
||||
path = 3rd_party/ssplit-cpp
|
||||
url = https://github.com/ugermann/ssplit-cpp
|
||||
url = https://github.com/abhi-agg/ssplit-cpp
|
||||
[submodule "3rd_party/marian-dev"]
|
||||
path = 3rd_party/marian-dev
|
||||
url = https://github.com/browsermt/marian-dev
|
||||
|
6
3rd_party/CMakeLists.txt
vendored
6
3rd_party/CMakeLists.txt
vendored
@ -1,4 +1,10 @@
|
||||
add_subdirectory(marian-dev)
|
||||
|
||||
if(COMPILE_WASM)
|
||||
# This is a bad way of adding compilation flags. Will be improved soon.
|
||||
add_compile_options(${WASM_COMPILE_FLAGS})
|
||||
endif(COMPILE_WASM)
|
||||
|
||||
add_subdirectory(ssplit-cpp)
|
||||
|
||||
# Add include directories for 3rd party targets to be able to use it anywhere in the
|
||||
|
@ -8,13 +8,26 @@ project(bergamot_translator CXX C)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(BUILD_ARCH native CACHE STRING "Compile for this CPU architecture.")
|
||||
|
||||
# Custom CMake options to compile marian (a 3rd party submodule) for this project
|
||||
option(COMPILE_CUDA "Compile GPU version" OFF)
|
||||
option(USE_SENTENCEPIECE "Download and compile SentencePiece" ON)
|
||||
option(USE_STATIC_LIBS "Link statically against non-system libs" ON)
|
||||
option(USE_MKL "Compile with MKL support" ON)
|
||||
# Project specific cmake options
|
||||
option(COMPILE_WASM "Compile for WASM" OFF)
|
||||
option(COMPILE_THREAD_VARIANT "Compile with thread support" OFF)
|
||||
SET(PACKAGE_DIR "" CACHE STRING "Directory including all the files to be packaged (pre-loaded) in wasm builds")
|
||||
|
||||
# Set marian (3rd party submodule) cmake options to compile for this project
|
||||
SET(COMPILE_CUDA OFF CACHE BOOL "Compile GPU version")
|
||||
SET(USE_SENTENCEPIECE ON CACHE BOOL "Download and compile SentencePiece")
|
||||
SET(USE_STATIC_LIBS ON CACHE BOOL "Link statically against non-system libs")
|
||||
SET(USE_MKL OFF CACHE BOOL "Compile with MKL support")
|
||||
SET(COMPILE_DECODER_ONLY ON CACHE BOOL "Compile marian-decoder only")
|
||||
SET(COMPILE_WITH_PTHREADS OFF CACHE BOOL "Compile with pthreads support")
|
||||
SET(USE_WASM_COMPATIBLE_BLAS ON CACHE BOOL "Compile with a WASM compatible blas for decoder only builds")
|
||||
SET(COMPILE_LIBRARY_ONLY ON CACHE BOOL "Build only the Marian library and exclude all executables.")
|
||||
SET(COMPILE_WITHOUT_EXCEPTIONS ON CACHE BOOL "Compile without exceptions")
|
||||
if(COMPILE_WASM)
|
||||
# Set WORMHOLE to ON for marian whenever compiling for wasm platform
|
||||
SET(WORMHOLE ON CACHE BOOL "Use WASM wormhole in intgemm https://bugzilla.mozilla.org/show_bug.cgi?id=1672160")
|
||||
endif()
|
||||
|
||||
# Documentation: https://cliutils.gitlab.io/modern-cmake/chapters/projects/submodule.html
|
||||
# Ensures the submodules are set correctly during a build.
|
||||
@ -33,8 +46,22 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(NOT COMPILE_WASM)
|
||||
# Set BUILD_ARCH to native only while compiling for non wasm platform
|
||||
set(BUILD_ARCH native CACHE STRING "Compile for this CPU architecture.")
|
||||
endif()
|
||||
|
||||
if(COMPILE_WASM)
|
||||
list(APPEND WASM_COMPILE_FLAGS -pthread -O3 -g2 -fPIC -mssse3 -msimd128)
|
||||
list(APPEND WASM_COMPILE_FLAGS "SHELL:-s WASM=1" "SHELL:-s ASSERTIONS=0" "SHELL:-s DISABLE_EXCEPTION_CATCHING=1" "SHELL:-s LLD_REPORT_UNDEFINED" "SHELL:-s FORCE_FILESYSTEM=1" "SHELL:-s ALLOW_MEMORY_GROWTH=1")
|
||||
list(APPEND WASM_COMPILE_FLAGS -Wno-error=pthreads-mem-growth)
|
||||
endif(COMPILE_WASM)
|
||||
|
||||
add_subdirectory(3rd_party)
|
||||
add_subdirectory(src)
|
||||
add_subdirectory(app)
|
||||
|
||||
|
||||
if(NOT COMPILE_WASM)
|
||||
add_subdirectory(app)
|
||||
endif()
|
||||
if(COMPILE_WASM)
|
||||
add_subdirectory(wasm)
|
||||
endif(COMPILE_WASM)
|
||||
|
127
README.md
127
README.md
@ -3,63 +3,78 @@
|
||||
Bergamot translator provides a unified API for ([Marian NMT](https://marian-nmt.github.io/) framework based) neural machine translation functionality in accordance with the [Bergamot](https://browser.mt/) project that focuses on improving client-side machine translation in a web browser.
|
||||
|
||||
## Build Instructions
|
||||
```
|
||||
$ git clone https://github.com/browsermt/bergamot-translator
|
||||
$ cd bergamot-translator
|
||||
$ mkdir build
|
||||
$ cd build
|
||||
$ cmake ../
|
||||
$ make -j
|
||||
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Bergamot Translator
|
||||
|
||||
The build will generate the library that can be linked to any project. All the public header files are specified in `src` folder.
|
||||
|
||||
### `service-cli`
|
||||
|
||||
An executable `service-cli` is generated by the build in the `app` folder and
|
||||
provides command line interface to the underlying translator. The models
|
||||
required to run the command-line are available at
|
||||
[data.statmt.org/bergamot/models/](http://data.statmt.org/bergamot/models/).
|
||||
The following example uses an English to German tiny11 student model, available
|
||||
at:
|
||||
|
||||
* [data.statmt.org/bergamot/models/deen/ende.student.tiny11.tar.gz](http://data.statmt.org/bergamot/models/deen/ende.student.tiny11.tar.gz)
|
||||
### Build Natively
|
||||
|
||||
```bash
|
||||
MODEL_DIR=... # path to where the model-files are.
|
||||
ARGS=(
|
||||
-m $MODEL_DIR/model.intgemm.alphas.bin # Path to model file.
|
||||
--vocabs
|
||||
$MODEL_DIR/vocab.deen.spm # source-vocabulary
|
||||
$MODEL_DIR/vocab.deen.spm # target-vocabulary
|
||||
|
||||
# The following increases speed through one-best-decoding, shortlist and quantization.
|
||||
--beam-size 1 --skip-cost --shortlist $MODEL_DIR/lex.s2t.gz 50 50 --int8shiftAlphaAll
|
||||
|
||||
# Number of CPU threads (workers to launch). Parallelizes over cores and improves speed.
|
||||
# A value of 0 allows a path with no worker thread-launches and a single-thread.
|
||||
--cpu-threads 4
|
||||
|
||||
# Maximum size of a sentence allowed. If a sentence is above this length,
|
||||
# it's broken into pieces of less than or equal to this size.
|
||||
--max-length-break 1024
|
||||
|
||||
# Maximum number of tokens that can be fit in a batch. The optimal value
|
||||
# for the parameter is dependant on hardware and can be obtained by running
|
||||
# with variations and benchmarking.
|
||||
--mini-batch-words 1024
|
||||
|
||||
# Three modes are supported
|
||||
# - sentence: One sentence per line
|
||||
# - paragraph: One paragraph per line.
|
||||
# - wrapped_text: Paragraphs are separated by empty line.
|
||||
--ssplit-mode paragraph
|
||||
)
|
||||
|
||||
./app/service-cli "${ARGS[@]}" < path-to-input-file
|
||||
git clone --recursive https://github.com/browsermt/bergamot-translator
|
||||
cd bergamot-translator
|
||||
mkdir build
|
||||
cd build
|
||||
cmake ../
|
||||
make -j
|
||||
```
|
||||
|
||||
### Build WASM
|
||||
|
||||
To compile WASM, first download and Install Emscripten using following instructions:
|
||||
|
||||
1. Get the latest sdk: `git clone https://github.com/emscripten-core/emsdk.git`
|
||||
2. Enter the cloned directory: `cd emsdk`
|
||||
3. Install the lastest sdk tools: `./emsdk install latest`
|
||||
4. Activate the latest sdk tools: `./emsdk activate latest`
|
||||
5. Activate path variables: `source ./emsdk_env.sh`
|
||||
|
||||
After the successful installation of Emscripten, perform these steps:
|
||||
|
||||
```bash
|
||||
git clone --recursive https://github.com/browsermt/bergamot-translator
|
||||
cd bergamot-translator
|
||||
git checkout wasm-integration
|
||||
git submodule update --recursive
|
||||
mkdir build-wasm
|
||||
cd build-wasm
|
||||
emcmake cmake -DCOMPILE_WASM=on ../
|
||||
emmake make -j
|
||||
```
|
||||
|
||||
It should generate the artefacts (.js and .wasm files) in `wasm` folder inside build directory ("build-wasm" in this case).
|
||||
|
||||
Download the models from `https://github.com/mozilla-applied-ml/bergamot-models`, and place all the desired ones to package in a folder called `models`.
|
||||
|
||||
The build also allows packaging files into wasm binary (i.e. preloading in Emscripten’s virtual file system) using cmake
|
||||
option `PACKAGE_DIR`. The compile command below packages all the files in PATH directory (in these case, your models) into wasm binary.
|
||||
```bash
|
||||
emcmake cmake -DCOMPILE_WASM=on -DPACKAGE_DIR=/repo/models ../
|
||||
```
|
||||
Files packaged this way are preloaded in the root of the virtual file system.
|
||||
|
||||
To package the set of files expected by the test page:
|
||||
|
||||
```bash
|
||||
mkdir models
|
||||
git clone https://github.com/motin/bergamot-models
|
||||
cp -r bergamot-models/* models
|
||||
gunzip models/*/*
|
||||
```
|
||||
|
||||
After Editing Files:
|
||||
|
||||
```bash
|
||||
emmake make -j
|
||||
```
|
||||
|
||||
After Adding/Removing Files:
|
||||
|
||||
```bash
|
||||
emcmake cmake -DCOMPILE_WASM=on ../
|
||||
emmake make -j
|
||||
```
|
||||
|
||||
### Using Native version
|
||||
|
||||
The builds generate library that can be integrated to any project. All the public header files are specified in `src` folder. A short example of how to use the APIs is provided in `app/main.cpp` file
|
||||
|
||||
### Using WASM version
|
||||
|
||||
Please follow the `README` inside the `wasm` folder of this repository that demonstrates how to use the translator in JavaScript.
|
||||
|
@ -44,10 +44,10 @@ int main(int argc, char **argv) {
|
||||
"Prague, the University of Sheffield, University of Tartu, and "
|
||||
"Mozilla.");
|
||||
|
||||
auto futureResults = model->translate(std::move(texts), translationRequest);
|
||||
auto results = model->translate(std::move(texts), translationRequest);
|
||||
|
||||
// Resolve the future and get the actual result
|
||||
std::vector<TranslationResult> results = futureResults.get();
|
||||
//std::vector<TranslationResult> results = futureResults.get();
|
||||
|
||||
for (auto &result : results) {
|
||||
std::cout << "[original]: " << result.getOriginalText() << std::endl;
|
||||
|
84
doc/marian-integration.md
Normal file
84
doc/marian-integration.md
Normal file
@ -0,0 +1,84 @@
|
||||
# Marian Integration
|
||||
|
||||
This document summarizes the minimal build instructions develop for the
|
||||
marian-code powering bergamot-translator.
|
||||
|
||||
## Build Instructions
|
||||
|
||||
```
|
||||
$ git clone https://github.com/browsermt/bergamot-translator
|
||||
$ cd bergamot-translator
|
||||
$ mkdir build
|
||||
$ cd build
|
||||
$ cmake ../
|
||||
$ make -j
|
||||
|
||||
|
||||
The build will generate the library that can be linked to any project. All the
|
||||
public header files are specified in `src` folder.
|
||||
|
||||
## Command line apps
|
||||
|
||||
The following executables are created by the build:
|
||||
|
||||
1. `app/service-cli`: Extends marian to capability to work with string_views.
|
||||
`service-cli` exists to check if the underlying code, without the
|
||||
integration works or not.
|
||||
2. `app/bergamot-translator-app`: App which integreates service-cli's
|
||||
functionality into the translator agnostic API specified as part of the
|
||||
project. Integration failures are detected if same arguments work with
|
||||
`service-cli` and does not with `bergamot-translator-app`.
|
||||
3. `app/marian-decoder-new`: Helper executable to conveniently benchmark new
|
||||
implementation with the optimized upstream marian-decoder.
|
||||
|
||||
The models required to run the command-line are available at
|
||||
[data.statmt.org/bergamot/models/](http://data.statmt.org/bergamot/models/).
|
||||
The following example uses an English to German tiny11 student model, available
|
||||
at:
|
||||
|
||||
* [data.statmt.org/bergamot/models/deen/ende.student.tiny11.tar.gz](http://data.statmt.org/bergamot/models/deen/ende.student.tiny11.tar.gz)
|
||||
|
||||
<details>
|
||||
<summary> Example run of commandline: Click to expand </summary>
|
||||
<p>
|
||||
|
||||
```bash
|
||||
MODEL_DIR=... # path to where the model-files are.
|
||||
ARGS=(
|
||||
-m $MODEL_DIR/model.intgemm.alphas.bin # Path to model file.
|
||||
--vocabs
|
||||
$MODEL_DIR/vocab.deen.spm # source-vocabulary
|
||||
$MODEL_DIR/vocab.deen.spm # target-vocabulary
|
||||
|
||||
# The following increases speed through one-best-decoding, shortlist and quantization.
|
||||
--beam-size 1 --skip-cost --shortlist $MODEL_DIR/lex.s2t.gz 50 50 --int8shiftAlphaAll
|
||||
|
||||
# Number of CPU threads (workers to launch). Parallelizes over cores and improves speed.
|
||||
# A value of 0 allows a path with no worker thread-launches and a single-thread.
|
||||
--cpu-threads 4
|
||||
|
||||
# Maximum size of a sentence allowed. If a sentence is above this length,
|
||||
# it's broken into pieces of less than or equal to this size.
|
||||
--max-length-break 1024
|
||||
|
||||
# Maximum number of tokens that can be fit in a batch. The optimal value
|
||||
# for the parameter is dependant on hardware and can be obtained by running
|
||||
# with variations and benchmarking.
|
||||
--mini-batch-words 1024
|
||||
|
||||
# Three modes are supported
|
||||
# - sentence: One sentence per line
|
||||
# - paragraph: One paragraph per line.
|
||||
# - wrapped_text: Paragraphs are separated by empty line.
|
||||
--ssplit-mode paragraph
|
||||
)
|
||||
|
||||
./app/service-cli "${ARGS[@]}" < path-to-input-file
|
||||
./app/bergamot-translator-app "${ARGS[@]}" < path-to-input-file
|
||||
|
||||
```
|
||||
</p>
|
||||
|
||||
</summary>
|
||||
</details>
|
||||
|
55
docker/Makefile
Normal file
55
docker/Makefile
Normal file
@ -0,0 +1,55 @@
|
||||
# -*- mode: makefile-gmake; indent-tabs-mode: true; tab-width: 4 -*-
|
||||
SHELL = bash
|
||||
PWD = $(shell pwd)
|
||||
WASM_IMAGE = local/bergamot-translator-build-wasm
|
||||
|
||||
all: wasm-image compile-wasm
|
||||
|
||||
# Build the Docker image for WASM builds
|
||||
wasm-image:
|
||||
docker build -t local/bergamot-translator-build-wasm ./wasm/
|
||||
|
||||
# Commands for compilation:
|
||||
cmake_cmd = cmake
|
||||
|
||||
wasm_cmake_cmd = ${cmake_cmd}
|
||||
wasm_cmake_cmd += -DCOMPILE_WASM=on
|
||||
wasm_cmake_cmd += -DProtobuf_INCLUDE_DIR=/usr/opt/protobuf-wasm-lib/dist/include
|
||||
wasm_cmake_cmd += -DProtobuf_LIBRARY=/usr/opt/protobuf-wasm-lib/dist/lib/libprotobuf.a
|
||||
wasm_cmake_cmd += -DPACKAGE_DIR=/repo/models
|
||||
|
||||
make_cmd = make
|
||||
#make_cmd += VERBOSE=1
|
||||
|
||||
# ... and running things on Docker
|
||||
docker_mounts = ${PWD}/..:/repo
|
||||
docker_mounts += ${HOME}/.ccache:/.ccache
|
||||
run_on_docker = docker run --rm
|
||||
run_on_docker += $(addprefix -v, ${docker_mounts})
|
||||
run_on_docker += ${INTERACTIVE_DOCKER_SESSION}
|
||||
|
||||
${HOME}/.ccache:
|
||||
mkdir -p $@
|
||||
|
||||
# Remove the bergamot-translator WASM build dir, forcing a clean compilation attempt
|
||||
clean-wasm: BUILD_DIR = /repo/build-wasm-docker
|
||||
clean-wasm: ${HOME}/.ccache
|
||||
${run_on_docker} ${WASM_IMAGE} bash -c '(rm -rf ${BUILD_DIR} || true)'
|
||||
|
||||
# Compile bergamot-translator to WASM
|
||||
compile-wasm: BUILD_DIR = /repo/build-wasm-docker
|
||||
compile-wasm: ${HOME}/.ccache
|
||||
${run_on_docker} ${WASM_IMAGE} bash -c 'mkdir -p ${BUILD_DIR} && \
|
||||
cd ${BUILD_DIR} && \
|
||||
(emcmake ${wasm_cmake_cmd} .. && \
|
||||
(emmake ${make_cmd}) || \
|
||||
rm CMakeCache.txt)'
|
||||
|
||||
# Start interactive shells for development / debugging purposes
|
||||
native-shell: INTERACTIVE_DOCKER_SESSION = -it
|
||||
native-shell:
|
||||
${run_on_docker} ${NATIVE_IMAGE} bash
|
||||
|
||||
wasm-shell: INTERACTIVE_DOCKER_SESSION = -it
|
||||
wasm-shell:
|
||||
${run_on_docker} ${WASM_IMAGE} bash
|
27
docker/README.md
Normal file
27
docker/README.md
Normal file
@ -0,0 +1,27 @@
|
||||
## WASM
|
||||
|
||||
Prepare docker image for WASM compilation:
|
||||
|
||||
```bash
|
||||
make wasm-image
|
||||
```
|
||||
|
||||
Compile to wasm:
|
||||
|
||||
```bash
|
||||
make compile-wasm
|
||||
```
|
||||
|
||||
## Debugging
|
||||
|
||||
Remove the marian-decoder build dir, forcing the next compilation attempt to start from scratch:
|
||||
|
||||
```bash
|
||||
make clean-wasm
|
||||
```
|
||||
|
||||
Enter a docker container shell for manually running commands:
|
||||
|
||||
```bash
|
||||
make wasm-shell
|
||||
```
|
36
docker/wasm/Dockerfile
Normal file
36
docker/wasm/Dockerfile
Normal file
@ -0,0 +1,36 @@
|
||||
FROM emscripten/emsdk:2.0.9
|
||||
|
||||
# Install specific version of CMake
|
||||
WORKDIR /usr
|
||||
RUN wget https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Linux-x86_64.tar.gz -qO-\
|
||||
| tar xzf - --strip-components 1
|
||||
|
||||
# Install Python and Java (needed for Closure Compiler minification)
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y \
|
||||
python3 \
|
||||
default-jre
|
||||
|
||||
# Deps to compile protobuf from source + the protoc binary which we need natively
|
||||
RUN apt-get update -y && apt-get --no-install-recommends -y install \
|
||||
protobuf-compiler \
|
||||
autoconf \
|
||||
autotools-dev \
|
||||
automake \
|
||||
autogen \
|
||||
libtool && ln -s /usr/bin/libtoolize /usr/bin/libtool \
|
||||
&& mkdir -p /usr/opt \
|
||||
&& cd /usr/opt \
|
||||
&& git clone https://github.com/menduz/protobuf-wasm-lib
|
||||
|
||||
RUN cd /usr/opt/protobuf-wasm-lib \
|
||||
&& /bin/bash -c "BRANCH=v3.6.1 ./prepare.sh"
|
||||
RUN cd /usr/opt/protobuf-wasm-lib/protobuf \
|
||||
&& bash -x ../build.sh
|
||||
RUN cp /usr/bin/protoc /usr/opt/protobuf-wasm-lib/dist/bin/protoc
|
||||
|
||||
RUN apt-get --no-install-recommends -y install \
|
||||
libprotobuf-dev
|
||||
|
||||
# Necessary for benchmarking
|
||||
RUN pip3 install sacrebleu
|
@ -57,7 +57,7 @@ public:
|
||||
* entry of texts list will be moved to its corresponding TranslationResult
|
||||
* object).
|
||||
*/
|
||||
virtual std::future<std::vector<TranslationResult>>
|
||||
virtual std::vector<TranslationResult>
|
||||
translate(std::vector<std::string> &&texts, TranslationRequest request) = 0;
|
||||
|
||||
/* Check if the model can provide alignment information b/w original and
|
||||
|
@ -20,7 +20,11 @@ class TranslationResult {
|
||||
public:
|
||||
typedef std::vector<std::pair<std::string_view, std::string_view>>
|
||||
SentenceMappings;
|
||||
|
||||
#ifdef WASM_BINDINGS
|
||||
TranslationResult(const std::string &original, const std::string &translation)
|
||||
: originalText(original), translatedText(translation),
|
||||
sentenceMappings() {}
|
||||
#endif
|
||||
TranslationResult(const std::string &original, const std::string &translation,
|
||||
SentenceMappings &sentenceMappings)
|
||||
: originalText(original), translatedText(translation),
|
||||
@ -31,13 +35,29 @@ public:
|
||||
translatedText(std::move(other.translatedText)),
|
||||
sentenceMappings(std::move(other.sentenceMappings)) {}
|
||||
|
||||
#ifdef WASM_BINDINGS
|
||||
TranslationResult(const TranslationResult &other)
|
||||
: originalText(other.originalText),
|
||||
translatedText(other.translatedText),
|
||||
sentenceMappings(other.sentenceMappings) {}
|
||||
#endif
|
||||
|
||||
TranslationResult(std::string &&original, std::string &&translation,
|
||||
SentenceMappings &&sentenceMappings)
|
||||
: originalText(std::move(original)),
|
||||
translatedText(std::move(translation)),
|
||||
sentenceMappings(std::move(sentenceMappings)) {}
|
||||
|
||||
#ifndef WASM_BINDINGS
|
||||
TranslationResult &operator=(const TranslationResult &) = delete;
|
||||
#else
|
||||
TranslationResult &operator=(const TranslationResult &result) {
|
||||
originalText = result.originalText;
|
||||
translatedText = result.translatedText;
|
||||
sentenceMappings = result.sentenceMappings;
|
||||
return *this;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Return the original text. */
|
||||
const std::string &getOriginalText() const { return originalText; }
|
||||
|
@ -14,6 +14,22 @@ add_library(bergamot-translator STATIC
|
||||
batch.cpp
|
||||
sentence_ranges.cpp
|
||||
)
|
||||
if (COMPILE_DECODER_ONLY)
|
||||
# A dirty hack because of marian's bad cmake practices
|
||||
target_compile_definitions(bergamot-translator PUBLIC DECODER_ONLY)
|
||||
endif()
|
||||
|
||||
if(COMPILE_WASM)
|
||||
# A dirty hack because of marian's bad cmake practices
|
||||
target_compile_definitions(bergamot-translator PUBLIC USE_SSE2 WASM)
|
||||
# Enable code that is required for generating JS bindings
|
||||
target_compile_definitions(bergamot-translator PRIVATE WASM_BINDINGS)
|
||||
target_compile_options(bergamot-translator PRIVATE ${WASM_COMPILE_FLAGS})
|
||||
endif(COMPILE_WASM)
|
||||
|
||||
if (COMPILE_THREAD_VARIANT)
|
||||
target_compile_definitions(bergamot-translator PRIVATE WITH_PTHREADS)
|
||||
endif()
|
||||
|
||||
target_link_libraries(bergamot-translator marian ssplit)
|
||||
|
||||
|
@ -16,6 +16,8 @@
|
||||
#include "TranslationModel.h"
|
||||
#include "translator/parser.h"
|
||||
#include "translator/service.h"
|
||||
#include "translator/parser.h"
|
||||
|
||||
|
||||
std::shared_ptr<marian::Options> parseOptions(const std::string &config) {
|
||||
marian::Options options;
|
||||
@ -56,7 +58,7 @@ TranslationModel::TranslationModel(const std::string &config)
|
||||
|
||||
TranslationModel::~TranslationModel() {}
|
||||
|
||||
std::future<std::vector<TranslationResult>>
|
||||
std::vector<TranslationResult>
|
||||
TranslationModel::translate(std::vector<std::string> &&texts,
|
||||
TranslationRequest request) {
|
||||
// Implementing a non-async version first. Unpleasant, but should work.
|
||||
@ -92,8 +94,7 @@ TranslationModel::translate(std::vector<std::string> &&texts,
|
||||
);
|
||||
}
|
||||
|
||||
promise.set_value(std::move(translationResults));
|
||||
return future;
|
||||
return translationResults;
|
||||
}
|
||||
|
||||
bool TranslationModel::isAlignmentSupported() const { return false; }
|
||||
|
@ -55,7 +55,7 @@ public:
|
||||
* entry of texts list will be moved to its corresponding TranslationResult
|
||||
* object).
|
||||
*/
|
||||
std::future<std::vector<TranslationResult>>
|
||||
std::vector<TranslationResult>
|
||||
translate(std::vector<std::string> &&texts,
|
||||
TranslationRequest request) override;
|
||||
|
||||
|
@ -96,6 +96,8 @@ void BatchTranslator::translate(Batch &batch) {
|
||||
batch.completeBatch(histories);
|
||||
}
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
|
||||
void BatchTranslator::consumeFrom(PCQueue<Batch> &pcqueue) {
|
||||
Batch batch;
|
||||
Histories histories;
|
||||
@ -109,5 +111,7 @@ void BatchTranslator::consumeFrom(PCQueue<Batch> &pcqueue) {
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace bergamot
|
||||
} // namespace marian
|
||||
|
@ -8,11 +8,14 @@
|
||||
#include "common/utils.h"
|
||||
#include "data/shortlist.h"
|
||||
#include "definitions.h"
|
||||
#include "pcqueue.h"
|
||||
#include "request.h"
|
||||
#include "translator/history.h"
|
||||
#include "translator/scorers.h"
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
#include "pcqueue.h"
|
||||
#endif
|
||||
|
||||
namespace marian {
|
||||
namespace bergamot {
|
||||
|
||||
@ -30,7 +33,10 @@ public:
|
||||
std::string _identifier() { return "worker" + std::to_string(device_.no); }
|
||||
void translate(Batch &batch);
|
||||
void initialize();
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
void consumeFrom(PCQueue<Batch> &pcqueue);
|
||||
#endif
|
||||
|
||||
private:
|
||||
Ptr<Options> options_;
|
||||
@ -39,6 +45,11 @@ private:
|
||||
Ptr<ExpressionGraph> graph_;
|
||||
std::vector<Ptr<Scorer>> scorers_;
|
||||
Ptr<data::ShortlistGenerator const> slgen_;
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
PCQueue<PCItem> *pcqueue_;
|
||||
std::thread thread_;
|
||||
#endif
|
||||
};
|
||||
|
||||
} // namespace bergamot
|
||||
|
@ -57,12 +57,14 @@ void Batcher::addWholeRequest(Ptr<Request> request) {
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
void Batcher::produceTo(PCQueue<Batch> &pcqueue) {
|
||||
Batch batch;
|
||||
while (cleaveBatch(batch)) {
|
||||
pcqueue.ProduceSwap(batch);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace bergamot
|
||||
} // namespace marian
|
||||
|
@ -5,9 +5,12 @@
|
||||
#include "common/options.h"
|
||||
#include "data/corpus_base.h"
|
||||
#include "definitions.h"
|
||||
#include "pcqueue.h"
|
||||
#include "request.h"
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
#include "pcqueue.h"
|
||||
#endif
|
||||
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
@ -22,7 +25,9 @@ public:
|
||||
// which maintains priority among sentences from multiple concurrent requests.
|
||||
void addSentenceWithPriority(RequestSentence &sentence);
|
||||
void addWholeRequest(Ptr<Request> request);
|
||||
#ifdef WITH_PTHREADS
|
||||
void produceTo(PCQueue<Batch> &pcqueue);
|
||||
#endif
|
||||
|
||||
// Loads sentences with sentences compiled from (tentatively) multiple
|
||||
// requests optimizing for both padding and priority.
|
||||
|
@ -9,6 +9,7 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
#ifdef __APPLE__
|
||||
#include <mach/mach.h>
|
||||
#include <mach/mach_traps.h>
|
||||
@ -19,6 +20,7 @@
|
||||
#else
|
||||
#include <boost/interprocess/sync/interprocess_semaphore.hpp>
|
||||
#endif
|
||||
#endif // WITH_PTHREADS
|
||||
|
||||
#if __GNUC__ >= 3
|
||||
#define UTIL_UNLIKELY(x) __builtin_expect(!!(x), 0)
|
||||
@ -29,6 +31,7 @@
|
||||
namespace marian {
|
||||
namespace bergamot {
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
/* OS X Maverick and Boost interprocess were doing "Function not implemented."
|
||||
* So this is my own wrapper around the mach kernel APIs.
|
||||
*/
|
||||
@ -114,6 +117,20 @@ inline void WaitSemaphore(Semaphore &on) {
|
||||
}
|
||||
|
||||
#endif // Apple
|
||||
#else // WITH_PTHREADS
|
||||
// A dummy Semaphore class that does nothing
|
||||
class Semaphore {
|
||||
public:
|
||||
explicit Semaphore(unsigned int value) : count(value) {}
|
||||
~Semaphore() {}
|
||||
void wait() {}
|
||||
void post() {}
|
||||
private:
|
||||
unsigned int count;
|
||||
};
|
||||
|
||||
inline void WaitSemaphore(Semaphore &semaphore) { semaphore.wait(); }
|
||||
#endif // WITH_PTHREADS
|
||||
|
||||
/**
|
||||
* Producer consumer queue safe for multiple producers and multiple consumers.
|
||||
@ -134,7 +151,9 @@ public:
|
||||
void Produce(const T &val) {
|
||||
WaitSemaphore(empty_);
|
||||
{
|
||||
#ifdef WITH_PTHREADS
|
||||
std::lock_guard<std::mutex> produce_lock(produce_at_mutex_);
|
||||
#endif
|
||||
try {
|
||||
*produce_at_ = val;
|
||||
} catch (...) {
|
||||
@ -151,7 +170,9 @@ public:
|
||||
void ProduceSwap(T &val) {
|
||||
WaitSemaphore(empty_);
|
||||
{
|
||||
#ifdef WITH_PTHREADS
|
||||
std::lock_guard<std::mutex> produce_lock(produce_at_mutex_);
|
||||
#endif
|
||||
try {
|
||||
std::swap(*produce_at_, val);
|
||||
} catch (...) {
|
||||
@ -168,7 +189,9 @@ public:
|
||||
T &Consume(T &out) {
|
||||
WaitSemaphore(used_);
|
||||
{
|
||||
#ifdef WITH_PTHREADS
|
||||
std::lock_guard<std::mutex> consume_lock(consume_at_mutex_);
|
||||
#endif
|
||||
try {
|
||||
out = *consume_at_;
|
||||
} catch (...) {
|
||||
@ -186,7 +209,9 @@ public:
|
||||
T &ConsumeSwap(T &out) {
|
||||
WaitSemaphore(used_);
|
||||
{
|
||||
#ifdef WITH_PTHREADS
|
||||
std::lock_guard<std::mutex> consume_lock(consume_at_mutex_);
|
||||
#endif
|
||||
try {
|
||||
std::swap(out, *consume_at_);
|
||||
} catch (...) {
|
||||
@ -220,11 +245,15 @@ private:
|
||||
|
||||
// Index for next write in storage_.
|
||||
T *produce_at_;
|
||||
#ifdef WITH_PTHREADS
|
||||
std::mutex produce_at_mutex_;
|
||||
#endif
|
||||
|
||||
// Index for next read from storage_.
|
||||
T *consume_at_;
|
||||
#ifdef WITH_PTHREADS
|
||||
std::mutex consume_at_mutex_;
|
||||
#endif
|
||||
};
|
||||
|
||||
template <class T> struct UnboundedPage {
|
||||
|
@ -11,8 +11,12 @@ namespace bergamot {
|
||||
Service::Service(Ptr<Options> options)
|
||||
: requestId_(0), numWorkers_(options->get<int>("cpu-threads")),
|
||||
vocabs_(std::move(loadVocabularies(options))),
|
||||
text_processor_(vocabs_, options), batcher_(options),
|
||||
pcqueue_(2 * options->get<int>("cpu-threads")) {
|
||||
text_processor_(vocabs_, options), batcher_(options)
|
||||
#ifdef WITH_PTHREADS
|
||||
,
|
||||
pcqueue_(2 * options->get<int>("cpu-threads"))
|
||||
#endif // WITH_PTHREADS
|
||||
{
|
||||
|
||||
if (numWorkers_ == 0) {
|
||||
// In case workers are 0, a single-translator is created and initialized
|
||||
@ -21,6 +25,7 @@ Service::Service(Ptr<Options> options)
|
||||
translators_.emplace_back(deviceId, vocabs_, options);
|
||||
translators_.back().initialize();
|
||||
} else {
|
||||
#ifdef WITH_PTHREADS
|
||||
// If workers specified are greater than 0, translators_ are populated with
|
||||
// unitialized instances. These are then initialized inside
|
||||
// individual threads and set to consume from producer-consumer queue.
|
||||
@ -36,6 +41,7 @@ Service::Service(Ptr<Options> options)
|
||||
translator.consumeFrom(pcqueue_);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -70,7 +76,9 @@ std::future<Response> Service::translate(std::string &&input) {
|
||||
batcher_.addWholeRequest(request);
|
||||
|
||||
if (numWorkers_ > 0) {
|
||||
#ifdef WITH_PTHREADS
|
||||
batcher_.produceTo(pcqueue_);
|
||||
#endif
|
||||
} else {
|
||||
// Queue single-threaded
|
||||
Batch batch;
|
||||
@ -83,6 +91,7 @@ std::future<Response> Service::translate(std::string &&input) {
|
||||
}
|
||||
|
||||
void Service::stop() {
|
||||
#ifdef WITH_PTHREADS
|
||||
for (auto &worker : workers_) {
|
||||
Batch poison = Batch::poison();
|
||||
pcqueue_.ProduceSwap(poison);
|
||||
@ -93,6 +102,7 @@ void Service::stop() {
|
||||
}
|
||||
|
||||
workers_.clear(); // Takes care of idempotency.
|
||||
#endif
|
||||
}
|
||||
|
||||
Service::~Service() { stop(); }
|
||||
|
@ -3,7 +3,6 @@
|
||||
|
||||
#include "batch_translator.h"
|
||||
#include "batcher.h"
|
||||
#include "pcqueue.h"
|
||||
#include "response.h"
|
||||
#include "text_processor.h"
|
||||
|
||||
@ -12,6 +11,10 @@
|
||||
|
||||
#include "data/types.h"
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
#include "pcqueue.h"
|
||||
#endif
|
||||
|
||||
namespace marian {
|
||||
namespace bergamot {
|
||||
|
||||
@ -67,9 +70,12 @@ private:
|
||||
|
||||
TextProcessor text_processor_; // ORDER DEPENDENCY
|
||||
Batcher batcher_;
|
||||
PCQueue<Batch> pcqueue_;
|
||||
std::vector<BatchTranslator> translators_;
|
||||
|
||||
#ifdef WITH_PTHREADS
|
||||
PCQueue<Batch> pcqueue_;
|
||||
std::vector<std::thread> workers_;
|
||||
#endif
|
||||
};
|
||||
|
||||
std::vector<Ptr<const Vocab>> loadVocabularies(Ptr<Options> options);
|
||||
|
27
wasm/CMakeLists.txt
Normal file
27
wasm/CMakeLists.txt
Normal file
@ -0,0 +1,27 @@
|
||||
add_executable(bergamot-translator-worker
|
||||
bindings/TranslationModelBindings.cpp
|
||||
bindings/TranslationRequestBindings.cpp
|
||||
bindings/TranslationResultBindings.cpp
|
||||
)
|
||||
|
||||
# This header inclusion needs to go away later as path to public headers of bergamot
|
||||
# translator should be directly available from "bergamot-translator" target
|
||||
target_include_directories(bergamot-translator-worker
|
||||
PRIVATE ${CMAKE_SOURCE_DIR}/src/translator
|
||||
PRIVATE ${CMAKE_SOURCE_DIR}
|
||||
)
|
||||
# This compile definition is required for generating binding code properly
|
||||
target_compile_definitions(bergamot-translator-worker PRIVATE WASM_BINDINGS)
|
||||
target_compile_options(bergamot-translator-worker PRIVATE ${WASM_COMPILE_FLAGS})
|
||||
|
||||
set(LINKER_FLAGS "--bind -s ASSERTIONS=0 -s DISABLE_EXCEPTION_CATCHING=1 -s FORCE_FILESYSTEM=1 -s ALLOW_MEMORY_GROWTH=1 -s NO_DYNAMIC_EXECUTION=1")
|
||||
if (NOT PACKAGE_DIR STREQUAL "")
|
||||
set(LINKER_FLAGS "${LINKER_FLAGS} --preload-file ${PACKAGE_DIR}@/")
|
||||
endif()
|
||||
|
||||
set_target_properties(bergamot-translator-worker PROPERTIES
|
||||
SUFFIX ".js"
|
||||
LINK_FLAGS ${LINKER_FLAGS}
|
||||
)
|
||||
|
||||
target_link_libraries(bergamot-translator-worker bergamot-translator)
|
63
wasm/README.md
Normal file
63
wasm/README.md
Normal file
@ -0,0 +1,63 @@
|
||||
## Using Bergamot Translator in JavaScript
|
||||
The example file `bergamot.html` in the folder `test_page` demonstrates how to use the bergamot translator in JavaScript via a `<script>` tag.
|
||||
This example assumes that files were packaged in wasm binary.
|
||||
|
||||
A brief summary is here though:
|
||||
|
||||
```js
|
||||
// The model configuration as YAML formatted string. For available configuration options, please check: https://marian-nmt.github.io/docs/cmd/marian-decoder/
|
||||
// This example captures the most relevant options: model file, vocabulary files and shortlist file
|
||||
const modelConfig = "{\"models\":[\"/model.npz\"],\"vocabs\":[\"/vocab.esen.spm\",\"/vocab.esen.spm\"],\"shortlist\":[\"/lex.s2t\"],\"beam-size\":1}";
|
||||
|
||||
// Instantiate the TranslationModel
|
||||
const model = new Module.TranslationModel(modelConfig);
|
||||
|
||||
// Instantiate the arguments of translate() API i.e. TranslationRequest and input (vector<string>)
|
||||
const request = new Module.TranslationRequest();
|
||||
const input = new Module.VectorString;
|
||||
|
||||
// Initialize the input
|
||||
input.push_back("Hola"); input.push_back("Mundo");
|
||||
|
||||
// translate the input; the result is a vector<TranslationResult>
|
||||
const result = model.translate(input, request);
|
||||
|
||||
// Print original and translated text from each entry of vector<TranslationResult>
|
||||
for (let i = 0; i < result.size(); i++) {
|
||||
console.log(' original=' + result.get(i).getOriginalText() + ', translation=' + result.get(i).getTranslatedText());
|
||||
}
|
||||
|
||||
// Don't forget to clean up the instances
|
||||
model.delete();
|
||||
request.delete();
|
||||
input.delete();
|
||||
```
|
||||
|
||||
You can also see everything in action by following the next steps:
|
||||
* Start the test webserver (ensure you have the latest nodejs installed)
|
||||
```bash
|
||||
cd test_page
|
||||
bash start_server.sh
|
||||
```
|
||||
* Open any of the browsers below
|
||||
* Firefox Nightly +87: make sure the following prefs are on (about:config)
|
||||
```
|
||||
dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled = true
|
||||
javascript.options.wasm_simd = true
|
||||
javascript.options.wasm_simd_wormhole = true
|
||||
```
|
||||
|
||||
* Chrome Canary +90: start with the following argument
|
||||
```
|
||||
--js-flags="--experimental-wasm-simd"
|
||||
```
|
||||
|
||||
* Browse to the following page:
|
||||
```
|
||||
http://localhost:8000/bergamot.html
|
||||
```
|
||||
|
||||
* Run some translations:
|
||||
* Choose a model and press `Load Model`
|
||||
* Type a sentence to be translated in the `From` textbox and press `Translate`
|
||||
* See the results in the `To` and `Log` textboxes
|
23
wasm/bindings/TranslationModelBindings.cpp
Normal file
23
wasm/bindings/TranslationModelBindings.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
/*
|
||||
* TranslationModelBindings.cpp
|
||||
*
|
||||
* Bindings for TranslationModel class
|
||||
*/
|
||||
|
||||
#include <emscripten/bind.h>
|
||||
|
||||
#include "TranslationModel.h"
|
||||
|
||||
using namespace emscripten;
|
||||
|
||||
// Binding code
|
||||
EMSCRIPTEN_BINDINGS(translation_model) {
|
||||
class_<TranslationModel>("TranslationModel")
|
||||
.constructor<std::string>()
|
||||
.function("translate", &TranslationModel::translate)
|
||||
.function("isAlignmentSupported", &TranslationModel::isAlignmentSupported)
|
||||
;
|
||||
|
||||
register_vector<std::string>("VectorString");
|
||||
register_vector<TranslationResult>("VectorTranslationResult");
|
||||
}
|
17
wasm/bindings/TranslationRequestBindings.cpp
Normal file
17
wasm/bindings/TranslationRequestBindings.cpp
Normal file
@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Bindings for TranslationRequest class
|
||||
*
|
||||
*/
|
||||
|
||||
#include <emscripten/bind.h>
|
||||
|
||||
#include "TranslationRequest.h"
|
||||
|
||||
using namespace emscripten;
|
||||
|
||||
// Binding code
|
||||
EMSCRIPTEN_BINDINGS(translation_request) {
|
||||
class_<TranslationRequest>("TranslationRequest")
|
||||
.constructor<>()
|
||||
;
|
||||
}
|
20
wasm/bindings/TranslationResultBindings.cpp
Normal file
20
wasm/bindings/TranslationResultBindings.cpp
Normal file
@ -0,0 +1,20 @@
|
||||
/*
|
||||
* Bindings for TranslationResult class
|
||||
*
|
||||
*/
|
||||
|
||||
#include <emscripten/bind.h>
|
||||
#include <vector>
|
||||
|
||||
#include "TranslationResult.h"
|
||||
|
||||
using namespace emscripten;
|
||||
|
||||
// Binding code
|
||||
EMSCRIPTEN_BINDINGS(translation_result) {
|
||||
class_<TranslationResult>("TranslationResult")
|
||||
.constructor<std::string, std::string, TranslationResult::SentenceMappings>()
|
||||
.function("getOriginalText", &TranslationResult::getOriginalText)
|
||||
.function("getTranslatedText", &TranslationResult::getTranslatedText)
|
||||
;
|
||||
}
|
35
wasm/test_page/bergamot-httpserver.js
Normal file
35
wasm/test_page/bergamot-httpserver.js
Normal file
@ -0,0 +1,35 @@
|
||||
require(__dirname + '/helper.js');
|
||||
|
||||
var http = require('http');
|
||||
var express = require('express');
|
||||
var app = express();
|
||||
var server = http.createServer(app);
|
||||
var fs = require('fs');
|
||||
var url = require('url');
|
||||
const nocache = require('nocache');
|
||||
const cors = require('cors');
|
||||
|
||||
app.use(cors())
|
||||
app.use(nocache());
|
||||
app.get('/*.*' , cors(), function(req, res) {
|
||||
var options = url.parse(req.url, true);
|
||||
var mime = Helper.getMime(options);
|
||||
serveFile(res, options.pathname, mime);
|
||||
});
|
||||
|
||||
function serveFile(res, pathName, mime) {
|
||||
mime = mime || 'text/html';
|
||||
fs.readFile(__dirname + '/' + pathName, function (err, data) {
|
||||
if (err) {
|
||||
res.writeHead(500, {"Content-Type": "text/plain"});
|
||||
return res.end('Error loading ' + pathName + " with Error: " + err);
|
||||
}
|
||||
res.header('Cross-Origin-Embedder-Policy','require-corp');
|
||||
res.header('Cross-Origin-Opener-Policy','same-origin');
|
||||
res.writeHead(200, {"Content-Type": mime});
|
||||
res.end(data);
|
||||
});
|
||||
}
|
||||
|
||||
server.listen(8000);
|
||||
console.log('HTTP and BinaryJS server started on port 8000');
|
204
wasm/test_page/bergamot.html
Normal file
204
wasm/test_page/bergamot.html
Normal file
@ -0,0 +1,204 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<link rel="icon" href="data:,">
|
||||
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1">
|
||||
</head>
|
||||
<style>
|
||||
body, html, div {
|
||||
margin-left: 1%;
|
||||
margin-right: 1%;
|
||||
margin-bottom: 1%;
|
||||
margin-top: 1%;
|
||||
padding-left: 1%;
|
||||
padding-right: 1%;
|
||||
padding-bottom: 1%;
|
||||
padding-top: 1%;
|
||||
}
|
||||
|
||||
textarea, #to, #from {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
div {
|
||||
float: left;
|
||||
width: 80%;
|
||||
}
|
||||
</style>
|
||||
<body>
|
||||
|
||||
<div id="divradios">
|
||||
<label>Choose the model to use</label>
|
||||
<input type="radio" name="modellang" value="enes"/><label>English to Spanish</label>
|
||||
<input type="radio" name="modellang" value="esen" checked/><label>Spanish to English</label>
|
||||
<input type="button" id="load" value="Load Model"/>
|
||||
</div>
|
||||
|
||||
<div id="divtranslation">
|
||||
<label for="from">From</label>
|
||||
<textarea id="from" name="from">
|
||||
Una estrategia republicana para obstaculizar la reelección de Obama
|
||||
Los dirigentes republicanos justificaron su política por la necesidad de luchar contra el fraude electoral.
|
||||
Ahora bien, el Centro Brennan considera esto último un mito y afirma que el fraude electoral es menos frecuente en los Estados Unidos que el número de personas que mueren a causa de la caída de un rayo.
|
||||
De hecho, los abogados republicanos no han encontrado más que 300 casos de fraude electoral en los Estados Unidos en diez años.
|
||||
Una cosa es cierta: esas nuevas disposiciones afectarán negativamente a la tasa de participación.
|
||||
En ese sentido, estas medidas minarán en parte el sistema democrático americano.
|
||||
Al contrario de lo que ocurre en Canadá, los estados americanos son responsables de la organización de las elecciones federales en los Estados Unidos.
|
||||
Y en esa misma línea una mayoría de los gobiernos americanos promulgaron, a partir de 2009, nuevas leyes que dificultaban el proceso de inscripción o de votación.
|
||||
Este fenómeno se ha extendido tras las elecciones de noviembre de 2010, que vieron el aumento de 675 nuevos representantes republicanos en 26 estados.
|
||||
En consecuencia, durante el año 2011 se introdujeron 180 proyectos de ley que restringían el ejercicio del derecho de voto en 41 estados.
|
||||
</textarea>
|
||||
<br><br>
|
||||
<label for="to">To</label>
|
||||
<textarea id="to" name="to" readonly></textarea>
|
||||
<br><br>
|
||||
<input type="button" id="translate" value="Translate"/>
|
||||
</div>
|
||||
|
||||
<div id="divlog">
|
||||
<label for="log">Log:</label><br>
|
||||
<textarea id="log" name="log" rows="50" cols="75"></textarea>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
|
||||
var model, request, input = undefined;
|
||||
const loadModel = (from, to) => {
|
||||
|
||||
const languagePair = `${from}${to}`;
|
||||
|
||||
// Vocab files are re-used in both translation directions
|
||||
const vocabLanguagePair = from === "en" ? `${to}${from}` : languagePair;
|
||||
|
||||
// Set the Model Configuration as YAML formatted string.
|
||||
// For available configuration options, please check: https://marian-nmt.github.io/docs/cmd/marian-decoder/
|
||||
const modelConfig = `models:
|
||||
- /${languagePair}/model.${languagePair}.npz
|
||||
vocabs:
|
||||
- /${vocabLanguagePair}/vocab.${vocabLanguagePair}.spm
|
||||
- /${vocabLanguagePair}/vocab.${vocabLanguagePair}.spm
|
||||
beam-size: 1
|
||||
normalize: 1.0
|
||||
word-penalty: 0
|
||||
max-input-sentence-tokens: 128
|
||||
max-input-tokens: 1024
|
||||
workspace: 128
|
||||
max-length-factor: 2.0
|
||||
skip-cost: true
|
||||
cpu-threads: 1
|
||||
quiet: true
|
||||
quiet-translation: true
|
||||
shortlist:
|
||||
- /${languagePair}/lex.${languagePair}.s2t
|
||||
- 50
|
||||
- 50
|
||||
`;
|
||||
/*
|
||||
This config is not valid anymore in new APIs
|
||||
mini-batch: 32
|
||||
maxi-batch: 100
|
||||
maxi-batch-sort: src
|
||||
*/
|
||||
// TODO: Use in model config when wormhole is enabled:
|
||||
// gemm-precision: int8shift
|
||||
// TODO: Use in model config when loading of binary models is supported and we use model.intgemm.alphas.bin:
|
||||
// gemm-precision: int8shiftAlphaAll
|
||||
|
||||
console.debug("modelConfig: ", modelConfig);
|
||||
|
||||
// Instantiate the TranslationModel
|
||||
if (model) model.delete();
|
||||
model = new Module.TranslationModel(modelConfig);
|
||||
}
|
||||
|
||||
const translate = (sentences) => {
|
||||
|
||||
// Instantiate the arguments of translate() API i.e. TranslationRequest and input (vector<string>)
|
||||
var request = new Module.TranslationRequest();
|
||||
let input = new Module.VectorString;
|
||||
|
||||
// Initialize the input
|
||||
sentences.forEach(sentence => {
|
||||
// prevent empty sentences - it breaks the translation
|
||||
if (sentence.trim() === "") {
|
||||
return;
|
||||
}
|
||||
input.push_back(sentence.trim())
|
||||
})
|
||||
// Access input (just for debugging)
|
||||
console.log('Input size=', input.size());
|
||||
/*
|
||||
for (let i = 0; i < input.size(); i++) {
|
||||
console.log(' val:' + input.get(i));
|
||||
}
|
||||
*/
|
||||
|
||||
// Translate the input; the result is a vector<TranslationResult>
|
||||
let result = model.translate(input, request);
|
||||
// Access original and translated text from each entry of vector<TranslationResult>
|
||||
//console.log('Result size=', result.size(), ' - TimeDiff - ', (Date.now() - start)/1000);
|
||||
const translatedSentences = [];
|
||||
for (let i = 0; i < result.size(); i++) {
|
||||
translatedSentences.push(result.get(i).getTranslatedText());
|
||||
}
|
||||
console.log({ translatedSentences });
|
||||
request.delete();
|
||||
input.delete();
|
||||
return translatedSentences;
|
||||
}
|
||||
|
||||
document.querySelector("#load").addEventListener("click", () => {
|
||||
const lang = document.querySelector('input[name="modellang"]:checked').value;
|
||||
const from = lang.substring(0, 2);
|
||||
const to = lang.substring(2, 4);
|
||||
let start = Date.now();
|
||||
loadModel(from, to)
|
||||
log(`model ${from}${to} loaded in ${(Date.now() - start) / 1000} secs`);
|
||||
//log('Model Alignment:', model.isAlignmentSupported());
|
||||
});
|
||||
|
||||
const translateCall = () => {
|
||||
const text = document.querySelector('#from').value;
|
||||
const sentences = text.split("\n");
|
||||
let wordCount = 0;
|
||||
sentences.forEach(sentence => {
|
||||
wordCount += sentence.trim().split(" ").filter(word => word.trim() !== "").length;
|
||||
})
|
||||
const start = Date.now();
|
||||
const translatedSentences = translate(sentences);
|
||||
const secs = (Date.now() - start) / 1000;
|
||||
log(`Translation of ${translatedSentences.length} sentences (wordCount ${wordCount}) took ${secs} secs (${Math.round(wordCount / secs)} words per second)`);
|
||||
|
||||
document.querySelector('#to').value = translatedSentences.join("\n");
|
||||
}
|
||||
|
||||
document.querySelector("#translate").addEventListener("click", () => {
|
||||
translateCall();
|
||||
});
|
||||
|
||||
document.querySelector("#from").addEventListener('keyup', function(event) {
|
||||
if (event.keyCode === 13) {
|
||||
translateCall();
|
||||
}
|
||||
});
|
||||
|
||||
const log = (message) => {
|
||||
document.querySelector("#log").value += message + "\n";
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
let moduleLoadStart;
|
||||
var Module = {
|
||||
preRun: [function() {
|
||||
log(`Time until Module.preRun: ${(Date.now() - start) / 1000} secs`);
|
||||
moduleLoadStart = Date.now();
|
||||
}],
|
||||
onRuntimeInitialized: function() {
|
||||
log(`Wasm Runtime initialized (preRun -> onRuntimeInitialized) in ${(Date.now() - moduleLoadStart) / 1000} secs`);
|
||||
}
|
||||
};
|
||||
</script>
|
||||
<script src="bergamot-translator-worker.js"></script>
|
||||
</body>
|
||||
</html>
|
40
wasm/test_page/helper.js
Normal file
40
wasm/test_page/helper.js
Normal file
@ -0,0 +1,40 @@
|
||||
/*
|
||||
* @author - Based of a file from Gist here: https://gist.github.com/1757658
|
||||
*
|
||||
* @modified - Mike Newell - it was on Gist so I figure I can use it
|
||||
*
|
||||
* @Description - Added support for a few more mime types including the new
|
||||
* .ogv, .webm, and .mp4 file types for HTML5 video.
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* @modified - Andre Natal - removed unused types for the purpose of this use
|
||||
case
|
||||
*/
|
||||
|
||||
Helper = {
|
||||
|
||||
types: {
|
||||
"wasm" : "application/wasm"
|
||||
, "js" : "application/javascript"
|
||||
, "html" : "text/html"
|
||||
, "htm" : "text/html"
|
||||
, "ico" : "image/vnd.microsoft.icon",
|
||||
},
|
||||
|
||||
getMime: function(u) {
|
||||
|
||||
var ext = this.getExt(u.pathname).replace('.', '');
|
||||
|
||||
return this.types[ext.toLowerCase()] || 'application/octet-stream';
|
||||
|
||||
},
|
||||
|
||||
getExt: function(path) {
|
||||
var i = path.lastIndexOf('.');
|
||||
|
||||
return (i < 0) ? '' : path.substr(i);
|
||||
}
|
||||
|
||||
};
|
391
wasm/test_page/package-lock.json
generated
Normal file
391
wasm/test_page/package-lock.json
generated
Normal file
@ -0,0 +1,391 @@
|
||||
{
|
||||
"requires": true,
|
||||
"lockfileVersion": 1,
|
||||
"dependencies": {
|
||||
"accepts": {
|
||||
"version": "1.3.7",
|
||||
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
|
||||
"integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
|
||||
"requires": {
|
||||
"mime-types": "~2.1.24",
|
||||
"negotiator": "0.6.2"
|
||||
}
|
||||
},
|
||||
"array-flatten": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
|
||||
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
|
||||
},
|
||||
"body-parser": {
|
||||
"version": "1.19.0",
|
||||
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
|
||||
"integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
|
||||
"requires": {
|
||||
"bytes": "3.1.0",
|
||||
"content-type": "~1.0.4",
|
||||
"debug": "2.6.9",
|
||||
"depd": "~1.1.2",
|
||||
"http-errors": "1.7.2",
|
||||
"iconv-lite": "0.4.24",
|
||||
"on-finished": "~2.3.0",
|
||||
"qs": "6.7.0",
|
||||
"raw-body": "2.4.0",
|
||||
"type-is": "~1.6.17"
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
|
||||
"integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
|
||||
},
|
||||
"content-disposition": {
|
||||
"version": "0.5.3",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
|
||||
"integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
|
||||
"requires": {
|
||||
"safe-buffer": "5.1.2"
|
||||
}
|
||||
},
|
||||
"content-type": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
|
||||
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
|
||||
},
|
||||
"cookie": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
|
||||
"integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
|
||||
},
|
||||
"cookie-signature": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
|
||||
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
|
||||
},
|
||||
"cors": {
|
||||
"version": "2.8.5",
|
||||
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
|
||||
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
|
||||
"requires": {
|
||||
"object-assign": "^4",
|
||||
"vary": "^1"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
|
||||
"requires": {
|
||||
"ms": "2.0.0"
|
||||
}
|
||||
},
|
||||
"depd": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
|
||||
"integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
|
||||
},
|
||||
"destroy": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
|
||||
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
|
||||
},
|
||||
"ee-first": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
|
||||
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
|
||||
},
|
||||
"encodeurl": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
|
||||
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
|
||||
},
|
||||
"escape-html": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
|
||||
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
|
||||
},
|
||||
"etag": {
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
|
||||
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
|
||||
},
|
||||
"express": {
|
||||
"version": "4.17.1",
|
||||
"resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
|
||||
"integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
|
||||
"requires": {
|
||||
"accepts": "~1.3.7",
|
||||
"array-flatten": "1.1.1",
|
||||
"body-parser": "1.19.0",
|
||||
"content-disposition": "0.5.3",
|
||||
"content-type": "~1.0.4",
|
||||
"cookie": "0.4.0",
|
||||
"cookie-signature": "1.0.6",
|
||||
"debug": "2.6.9",
|
||||
"depd": "~1.1.2",
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"etag": "~1.8.1",
|
||||
"finalhandler": "~1.1.2",
|
||||
"fresh": "0.5.2",
|
||||
"merge-descriptors": "1.0.1",
|
||||
"methods": "~1.1.2",
|
||||
"on-finished": "~2.3.0",
|
||||
"parseurl": "~1.3.3",
|
||||
"path-to-regexp": "0.1.7",
|
||||
"proxy-addr": "~2.0.5",
|
||||
"qs": "6.7.0",
|
||||
"range-parser": "~1.2.1",
|
||||
"safe-buffer": "5.1.2",
|
||||
"send": "0.17.1",
|
||||
"serve-static": "1.14.1",
|
||||
"setprototypeof": "1.1.1",
|
||||
"statuses": "~1.5.0",
|
||||
"type-is": "~1.6.18",
|
||||
"utils-merge": "1.0.1",
|
||||
"vary": "~1.1.2"
|
||||
}
|
||||
},
|
||||
"finalhandler": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz",
|
||||
"integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
|
||||
"requires": {
|
||||
"debug": "2.6.9",
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"on-finished": "~2.3.0",
|
||||
"parseurl": "~1.3.3",
|
||||
"statuses": "~1.5.0",
|
||||
"unpipe": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"forwarded": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
||||
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
|
||||
},
|
||||
"fresh": {
|
||||
"version": "0.5.2",
|
||||
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
|
||||
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
|
||||
},
|
||||
"http-errors": {
|
||||
"version": "1.7.2",
|
||||
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
|
||||
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
|
||||
"requires": {
|
||||
"depd": "~1.1.2",
|
||||
"inherits": "2.0.3",
|
||||
"setprototypeof": "1.1.1",
|
||||
"statuses": ">= 1.5.0 < 2",
|
||||
"toidentifier": "1.0.0"
|
||||
}
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
|
||||
"requires": {
|
||||
"safer-buffer": ">= 2.1.2 < 3"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
|
||||
},
|
||||
"ipaddr.js": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
|
||||
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
|
||||
},
|
||||
"media-typer": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
|
||||
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
|
||||
},
|
||||
"merge-descriptors": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
|
||||
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
|
||||
},
|
||||
"methods": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
|
||||
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
||||
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.45.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz",
|
||||
"integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w=="
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.28",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.28.tgz",
|
||||
"integrity": "sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ==",
|
||||
"requires": {
|
||||
"mime-db": "1.45.0"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
|
||||
},
|
||||
"negotiator": {
|
||||
"version": "0.6.2",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
|
||||
"integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
|
||||
},
|
||||
"nocache": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/nocache/-/nocache-2.1.0.tgz",
|
||||
"integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q=="
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
|
||||
},
|
||||
"on-finished": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
|
||||
"integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
|
||||
"requires": {
|
||||
"ee-first": "1.1.1"
|
||||
}
|
||||
},
|
||||
"parseurl": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
|
||||
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
|
||||
},
|
||||
"path-to-regexp": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
|
||||
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
|
||||
},
|
||||
"proxy-addr": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
|
||||
"integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
|
||||
"requires": {
|
||||
"forwarded": "~0.1.2",
|
||||
"ipaddr.js": "1.9.1"
|
||||
}
|
||||
},
|
||||
"qs": {
|
||||
"version": "6.7.0",
|
||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
|
||||
"integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
|
||||
},
|
||||
"range-parser": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
|
||||
},
|
||||
"raw-body": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
|
||||
"integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
|
||||
"requires": {
|
||||
"bytes": "3.1.0",
|
||||
"http-errors": "1.7.2",
|
||||
"iconv-lite": "0.4.24",
|
||||
"unpipe": "1.0.0"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"send": {
|
||||
"version": "0.17.1",
|
||||
"resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
|
||||
"integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
|
||||
"requires": {
|
||||
"debug": "2.6.9",
|
||||
"depd": "~1.1.2",
|
||||
"destroy": "~1.0.4",
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"etag": "~1.8.1",
|
||||
"fresh": "0.5.2",
|
||||
"http-errors": "~1.7.2",
|
||||
"mime": "1.6.0",
|
||||
"ms": "2.1.1",
|
||||
"on-finished": "~2.3.0",
|
||||
"range-parser": "~1.2.1",
|
||||
"statuses": "~1.5.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ms": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
|
||||
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"serve-static": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
|
||||
"integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
|
||||
"requires": {
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"parseurl": "~1.3.3",
|
||||
"send": "0.17.1"
|
||||
}
|
||||
},
|
||||
"setprototypeof": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
|
||||
"integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
|
||||
},
|
||||
"statuses": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
|
||||
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
|
||||
},
|
||||
"toidentifier": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
|
||||
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
|
||||
},
|
||||
"type-is": {
|
||||
"version": "1.6.18",
|
||||
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
|
||||
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
|
||||
"requires": {
|
||||
"media-typer": "0.3.0",
|
||||
"mime-types": "~2.1.24"
|
||||
}
|
||||
},
|
||||
"unpipe": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
|
||||
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
|
||||
},
|
||||
"utils-merge": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
|
||||
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
|
||||
},
|
||||
"vary": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
|
||||
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
|
||||
}
|
||||
}
|
||||
}
|
7
wasm/test_page/package.json
Normal file
7
wasm/test_page/package.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"cors": "^2.8.5",
|
||||
"express": "^4.17.1",
|
||||
"nocache": "^2.1.0"
|
||||
}
|
||||
}
|
8
wasm/test_page/start_server.sh
Normal file
8
wasm/test_page/start_server.sh
Normal file
@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
cp ../../build-wasm-docker/wasm/bergamot-translator-worker.data .
|
||||
cp ../../build-wasm-docker/wasm/bergamot-translator-worker.js .
|
||||
cp ../../build-wasm-docker/wasm/bergamot-translator-worker.wasm .
|
||||
cp ../../build-wasm-docker/wasm/bergamot-translator-worker.worker.js .
|
||||
npm install
|
||||
node bergamot-httpserver.js
|
Loading…
Reference in New Issue
Block a user