diff --git a/.github/workflows/native-custom_marian-mac.yml b/.github/workflows/native-custom_marian-mac.yml deleted file mode 100644 index 1aae7e5..0000000 --- a/.github/workflows/native-custom_marian-mac.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Native (Custom Marian) MacOS - -on: - push: - branches: [ main, ci-sandbox ] - pull_request: - branches: [ main, ci-sandbox ] - -jobs: - build-macos: - name: MacOS - runs-on: macos-10.15 - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Configure CMake - run: | - mkdir -p build - cd build - cmake .. - - - name: Compile - working-directory: build - run: make -j2 - - - name: Print versions - working-directory: build - run: | - ./app/bergamot-translator-app --version diff --git a/.github/workflows/native-custom_marian-ubuntu.yml b/.github/workflows/native-custom_marian-ubuntu.yml deleted file mode 100644 index f051871..0000000 --- a/.github/workflows/native-custom_marian-ubuntu.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Native (Custom Marian) Ubuntu - -on: - push: - branches: [ main, ci-sandbox ] - pull_request: - branches: [ main, ci-sandbox ] - -jobs: - build-macos: - name: Ubuntu - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Configure CMake - run: | - mkdir -p build - cd build - cmake .. - - - name: Compile - working-directory: build - run: make -j2 - - - name: Print versions - working-directory: build - run: | - ./app/bergamot-translator-app --version diff --git a/.github/workflows/native-full_marian-mac.yml b/.github/workflows/native-full_marian-mac.yml deleted file mode 100644 index 1928c5c..0000000 --- a/.github/workflows/native-full_marian-mac.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Native (Full Marian) MacOS - -on: - push: - branches: [ main, ci-sandbox ] - pull_request: - branches: [ main, ci-sandbox ] - -jobs: - build-macos: - name: MacOS CPU-only - runs-on: macos-10.15 - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Install dependencies - run: brew install openblas protobuf - - # Openblas location is exported explicitly because openblas is keg-only, - # which means it was not symlinked into /usr/local/. - # CMake cannot find BLAS on GitHub runners if Marian is being compiled - # statically, hence USE_STATIC_LIBS=off - - name: Configure CMake - run: | - export LDFLAGS="-L/usr/local/opt/openblas/lib" - export CPPFLAGS="-I/usr/local/opt/openblas/include" - mkdir -p build - cd build - cmake .. \ - -DCOMPILE_CPU=on \ - -DCOMPILE_CUDA=off \ - -DCOMPILE_EXAMPLES=on \ - -DCOMPILE_SERVER=on \ - -DCOMPILE_TESTS=on \ - -DUSE_FBGEMM=on \ - -DUSE_SENTENCEPIECE=on \ - -DUSE_STATIC_LIBS=off \ - -DUSE_WASM_COMPATIBLE_SOURCE=off - - - name: Compile - working-directory: build - run: make -j2 - - - name: Run unit tests - working-directory: build - run: make test - - - name: Print versions - working-directory: build - run: | - ./marian --version - ./marian-decoder --version - ./marian-scorer --version - ./spm_encode --version - diff --git a/.github/workflows/native-full_marian-ubuntu.yml b/.github/workflows/native-full_marian-ubuntu.yml deleted file mode 100644 index e414f64..0000000 --- a/.github/workflows/native-full_marian-ubuntu.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: Native (Full Marian) Ubuntu - -on: - push: - branches: [ main, ci-test ] - pull_request: - branches: [ main, ci-test ] - -jobs: - build-ubuntu: - strategy: - matrix: - include: - # Ubuntu CPU-only build - - name: "Ubuntu CPU-only" - os: ubuntu-latest - cuda: "" - gcc: 8 - cpu: true - gpu: false - # GPU Builds are commented out, for bergamot-translator CI runs. - # Ubuntu GPU-only build - # - name: "Ubuntu GPU-only" - # os: ubuntu-latest - # cuda: "10.2" - # gcc: 7 - # cpu: false - # gpu: true - # Ubuntu 20.04 supports CUDA 11+ - #- name: "Ubuntu 20.04 CUDA 11.0 gcc-9" - #os: ubuntu-20.04 - #cuda: "11.0" - #gcc: 9 - #cpu: false - #gpu: true - # Ubuntu 18.04 supports CUDA 10.1+ - # - name: "Ubuntu 18.04 CUDA 10.2 gcc-8" - # os: ubuntu-18.04 - # cuda: "10.2" - # gcc: 8 - # cpu: true - # gpu: true - # Ubuntu 16.04 supports CUDA 8+ - # - name: "Ubuntu 16.04 CUDA 9.2 gcc-7" - # os: ubuntu-16.04 - # cuda: "9.2" - # gcc: 7 - # cpu: true - # gpu: true - - runs-on: ${{ matrix.os }} - name: ${{ matrix.name }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - # The following packages are already installed on GitHub-hosted runners: - # build-essential openssl libssl-dev - # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because - # it is installed together with libprotobuf-dev - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler libboost-all-dev g++-8 - - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html - - name: Install MKL - run: | - wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 - if: matrix.cpu == true - - # The script simplifies installation of different versions of CUDA - - name: Install CUDA - run: ./3rd_party/marian-dev/scripts/ci/install_cuda_ubuntu.sh ${{ matrix.cuda }} - if: matrix.gpu == true - - # Boost is installed on GitHub-hosted runners in a non-standard location - # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671 - - name: Configure CMake - run: | - mkdir -p build - cd build - CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \ - cmake .. \ - -DBoost_ARCHITECTURE=-x64 \ - -DCMAKE_BUILD_TYPE=Release \ - -DCOMPILE_CPU=${{ matrix.cpu }} \ - -DCOMPILE_CUDA=${{ matrix.gpu }} \ - -DCOMPILE_EXAMPLES=on \ - -DCOMPILE_SERVER=on \ - -DCOMPILE_TESTS=on \ - -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }} \ - -DUSE_FBGEMM=${{ matrix.cpu }} \ - -DUSE_SENTENCEPIECE=on \ - -DUSE_STATIC_LIBS=on \ - -DUSE_WASM_COMPATIBLE_SOURCE=off - - - name: Compile - working-directory: build - run: make -j2 - - - name: Run unit tests - working-directory: build - run: make test - # GitHub-hosted VMs do not have GPUs, so can not be run in CUDA builds - if: matrix.gpu == false - - - name: Print versions - working-directory: build - run: | - ./marian --version - ./marian-decoder --version - ./marian-scorer --version - ./marian-server --version - ./spm_encode --version - diff --git a/.github/workflows/native-mac.yml b/.github/workflows/native-mac.yml new file mode 100644 index 0000000..8df203d --- /dev/null +++ b/.github/workflows/native-mac.yml @@ -0,0 +1,108 @@ +name: Native MacOS + +on: + push: + branches: [ main, ci-sandbox ] + pull_request: + branches: [ main, ci-sandbox ] + +jobs: + build-macos: + strategy: + fail-fast: false + matrix: + include: + - name: "full-marian" + os: macos-10.15 + test_tags: "" + cmake: + CMAKE_BUILD_TYPE: "Release" + COMPILE_TESTS: "ON" + USE_WASM_COMPATIBLE_SOURCE: "OFF" + USE_FBGEMM: "OFF" + USE_STATIC_LIBS: "OFF" + COMPILE_SERVER: "OFF" + COMPILE_EXAMPLES: "OFF" + + - name: "minimal-marian" + os: macos-10.15 + test_tags: "'#wasm'" + cmake: + CMAKE_BUILD_TYPE: "Release" + COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and compile tests fail + USE_WASM_COMPATIBLE_SOURCE: "ON" + USE_FBGEMM: "OFF" + # explicitly set due to requirement of minimal marian being used + # within WASM. This is some yaml ugliness, but issok. + USE_STATIC_LIBS: "ON" + COMPILE_SERVER: "OFF" + COMPILE_EXAMPLES: "OFF" + + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install dependencies + run: | + brew update + brew install openblas protobuf coreutils + + # Openblas location is exported explicitly because openblas is keg-only, + # which means it was not symlinked into /usr/local/. + - name: Set BLAS Environment variables + run: | + echo "LDFLAGS=-L/usr/local/opt/openblas/lib" >> $GITHUB_ENV + echo "CPPFLAGS=-I/usr/local/opt/openblas/include" >> $GITHUB_ENV + if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF' + + # CMake cannot find BLAS on GitHub runners if Marian is being compiled + # statically, hence USE_STATIC_LIBS=off + - name: Configure CMake + run: | + mkdir -p build + cd build + cmake .. \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\ + -DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\ + -DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \ + -DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \ + -DUSE_STATIC_LIBS=${{ matrix.cmake.USE_STATIC_LIBS }} \ + -DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \ + -DUSE_FBGEMM=${{ matrix.cmake.USE_FBGEMM }} + + - name: Compile + working-directory: build + run: make -j2 + + - name: Run unit tests + working-directory: build + run: make test + if: matrix.cmake.COMPILE_TESTS == 'ON' + + - name: Print versions + working-directory: build + run: | + ./app/bergamot-translator-app --version + + - name: Install regression-test framework (BRT) + working-directory: bergamot-translator-tests + run : make install + + - name: Run regression-tests (BRT) + working-directory: bergamot-translator-tests + run : MARIAN=../build ./run_brt.sh ${{ matrix.test_tags }} + + - name: Upload regression-tests artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: brt-artifacts-${{ matrix.name }} + path: | + bergamot-translator-tests/**/*.expected + bergamot-translator-tests/**/*.log + bergamot-translator-tests/**/*.out diff --git a/.github/workflows/native-ubuntu.yml b/.github/workflows/native-ubuntu.yml new file mode 100644 index 0000000..dc8016b --- /dev/null +++ b/.github/workflows/native-ubuntu.yml @@ -0,0 +1,117 @@ +name: Native Ubuntu + +on: + push: + branches: [ main, ci-sandbox ] + pull_request: + branches: [ main, ci-sandbox ] + +jobs: + build-ubuntu: + strategy: + fail-fast: false + matrix: + include: + - name: "full-marian" + os: ubuntu-latest + gcc: 8 + cpu: 'ON' + gpu: 'OFF' + test_tags: "" + cmake: + CMAKE_BUILD_TYPE: "Release" + COMPILE_TESTS: "ON" + USE_WASM_COMPATIBLE_SOURCE: "OFF" + COMPILE_SERVER: "OFF" + COMPILE_EXAMPLES: "OFF" + + - name: "minimal-marian" + os: ubuntu-latest + gcc: 8 + cpu: 'ON' + gpu: 'OFF' + test_tags: "'#wasm'" + cmake: + CMAKE_BUILD_TYPE: "Release" + COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and COMPILE_TEST=ON fails. + USE_WASM_COMPATIBLE_SOURCE: "ON" + COMPILE_SERVER: "OFF" + COMPILE_EXAMPLES: "OFF" + + + runs-on: ${{ matrix.os }} + name: ${{ matrix.name }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + # The following packages are already installed on GitHub-hosted runners: + # build-essential openssl libssl-dev + # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because + # it is installed together with libprotobuf-dev + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + libgoogle-perftools-dev libprotobuf-dev protobuf-compiler \ + libboost-all-dev g++-${{ matrix.gcc }} + + # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html + - name: Install MKL + run: | + wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - + sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" + sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" + sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 + if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF' + + # Boost is installed on GitHub-hosted runners in a non-standard location + # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671 + - name: Configure CMake + run: | + mkdir -p build + cd build + CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \ + cmake .. \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\ + -DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\ + -DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \ + -DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \ + -DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \ + + - name: Compile bergamot-translator + working-directory: build + run: make -j2 + + - name: Run unit tests + working-directory: build + run: make test + # GitHub-hosted VMs do not have GPUs, so can not be run in CUDA builds + if: matrix.gpu == 'OFF' && matrix.cmake.COMPILE_TESTS == 'ON' + + - name: Print versions + working-directory: build + run: | + ./app/bergamot-translator-app --version + + + - name: Install regression-test framework (BRT) + working-directory: bergamot-translator-tests + run : make install + + - name: Run regression-tests (BRT) + working-directory: bergamot-translator-tests + run : MARIAN=../build ./run_brt.sh ${{ matrix.test_tags }} + + - name: Upload regression-tests artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: brt-artifacts-${{ matrix.name }} + path: | + bergamot-translator-tests/**/*.expected + bergamot-translator-tests/**/*.log + bergamot-translator-tests/**/*.out diff --git a/.github/workflows/wasm-custom_marian-ubuntu.yml b/.github/workflows/wasm-custom_marian-ubuntu.yml index d1364dc..7dfc839 100644 --- a/.github/workflows/wasm-custom_marian-ubuntu.yml +++ b/.github/workflows/wasm-custom_marian-ubuntu.yml @@ -2,9 +2,9 @@ name: WASM (Custom Marian) Ubuntu on: push: - branches: [ main ] + branches: [ main, ci-sandbox ] pull_request: - branches: [ main ] + branches: [ main, ci-sandbox ] jobs: build-wasm: diff --git a/.gitmodules b/.gitmodules index cc40735..8aa1014 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "3rd_party/ssplit-cpp"] path = 3rd_party/ssplit-cpp url = https://github.com/browsermt/ssplit-cpp +[submodule "bergamot-translator-tests"] + path = bergamot-translator-tests + url = https://github.com/browsermt/bergamot-translator-tests diff --git a/bergamot-translator-tests b/bergamot-translator-tests new file mode 160000 index 0000000..3771001 --- /dev/null +++ b/bergamot-translator-tests @@ -0,0 +1 @@ +Subproject commit 3771001720a8f01bba185ee5d5d908b7c266ef31