Moving small tests to GitHub CI (#93)

Adds regression-tests to the workflow for native minimal/custom marian and full builds. Co-authored-by: abhi-agg <66322306+abhi-agg@users.noreply.github.com>
2024-08-15 16:40:26 +03:00 · 2021-04-16 11:58:53 +01:00 · 2021-04-16 11:58:53 +01:00 · c00c263f8f
commit c00c263f8f
parent f1fc4f8041
9 changed files with 231 additions and 247 deletions
--- a/.github/workflows/native-custom_marian-mac.yml
+++ b/.github/workflows/native-custom_marian-mac.yml
@ -1,33 +0,0 @@
-name: Native (Custom Marian) MacOS
-
-on:
-  push:
-    branches: [ main, ci-sandbox ]
-  pull_request:
-    branches: [ main, ci-sandbox ]
-
-jobs:
-  build-macos:
-    name: MacOS
-    runs-on: macos-10.15
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-
-    - name: Configure CMake
-      run: |
-        mkdir -p build
-        cd build
-        cmake ..
-
-    - name: Compile
-      working-directory: build
-      run: make -j2
-
-    - name: Print versions
-      working-directory: build
-      run: |
-        ./app/bergamot-translator-app --version
--- a/.github/workflows/native-custom_marian-ubuntu.yml
+++ b/.github/workflows/native-custom_marian-ubuntu.yml
@ -1,33 +0,0 @@
-name: Native (Custom Marian) Ubuntu
-
-on:
-  push:
-    branches: [ main, ci-sandbox ]
-  pull_request:
-    branches: [ main, ci-sandbox ]
-
-jobs:
-  build-macos:
-    name: Ubuntu
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-
-    - name: Configure CMake
-      run: |
-        mkdir -p build
-        cd build
-        cmake ..
-
-    - name: Compile
-      working-directory: build
-      run: make -j2
-
-    - name: Print versions
-      working-directory: build
-      run: |
-        ./app/bergamot-translator-app --version
--- a/.github/workflows/native-full_marian-mac.yml
+++ b/.github/workflows/native-full_marian-mac.yml
@ -1,59 +0,0 @@
-name: Native (Full Marian) MacOS
-
-on:
-  push:
-    branches: [ main, ci-sandbox ]
-  pull_request:
-    branches: [ main, ci-sandbox ]
-
-jobs:
-  build-macos:
-    name: MacOS CPU-only
-    runs-on: macos-10.15
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-
-    - name: Install dependencies
-      run: brew install openblas protobuf
-
-    # Openblas location is exported explicitly because openblas is keg-only,
-    # which means it was not symlinked into /usr/local/.
-    # CMake cannot find BLAS on GitHub runners if Marian is being compiled
-    # statically, hence USE_STATIC_LIBS=off
-    - name: Configure CMake
-      run: |
-        export LDFLAGS="-L/usr/local/opt/openblas/lib"
-        export CPPFLAGS="-I/usr/local/opt/openblas/include"
-        mkdir -p build
-        cd build
-        cmake .. \
-          -DCOMPILE_CPU=on \
-          -DCOMPILE_CUDA=off \
-          -DCOMPILE_EXAMPLES=on \
-          -DCOMPILE_SERVER=on \
-          -DCOMPILE_TESTS=on \
-          -DUSE_FBGEMM=on \
-          -DUSE_SENTENCEPIECE=on \
-          -DUSE_STATIC_LIBS=off \
-          -DUSE_WASM_COMPATIBLE_SOURCE=off
-
-    - name: Compile
-      working-directory: build
-      run: make -j2
-
-    - name: Run unit tests
-      working-directory: build
-      run: make test
-
-    - name: Print versions
-      working-directory: build
-      run: |
-        ./marian --version
-        ./marian-decoder --version
-        ./marian-scorer --version
-        ./spm_encode --version
-
--- a/.github/workflows/native-full_marian-ubuntu.yml
+++ b/.github/workflows/native-full_marian-ubuntu.yml
@ -1,120 +0,0 @@
-name: Native (Full Marian) Ubuntu
-
-on:
-  push:
-    branches: [ main, ci-test ]
-  pull_request:
-    branches: [ main, ci-test ]
-
-jobs:
-  build-ubuntu:
-    strategy:
-      matrix:
-        include:
-          # Ubuntu CPU-only build
-          - name: "Ubuntu CPU-only"
-            os: ubuntu-latest
-            cuda: ""
-            gcc: 8
-            cpu: true
-            gpu: false
-          # GPU Builds are commented out, for bergamot-translator CI runs.
-          # Ubuntu GPU-only build
-          # - name: "Ubuntu GPU-only"
-          #   os: ubuntu-latest
-          #   cuda: "10.2"
-          #   gcc: 7
-          #   cpu: false
-          #   gpu: true
-          # Ubuntu 20.04 supports CUDA 11+
-          #- name: "Ubuntu 20.04 CUDA 11.0 gcc-9"
-            #os: ubuntu-20.04
-            #cuda: "11.0"
-            #gcc: 9
-            #cpu: false
-            #gpu: true
-          # Ubuntu 18.04 supports CUDA 10.1+
-          # - name: "Ubuntu 18.04 CUDA 10.2 gcc-8"
-          #   os: ubuntu-18.04
-          #   cuda: "10.2"
-          #   gcc: 8
-          #   cpu: true
-          #   gpu: true
-          # Ubuntu 16.04 supports CUDA 8+
-          # - name: "Ubuntu 16.04 CUDA 9.2 gcc-7"
-          #   os: ubuntu-16.04
-          #   cuda: "9.2"
-          #   gcc: 7
-          #   cpu: true
-          #   gpu: true
-
-    runs-on: ${{ matrix.os }}
-    name: ${{ matrix.name }}
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-
-    # The following packages are already installed on GitHub-hosted runners:
-    # build-essential openssl libssl-dev
-    # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because
-    # it is installed together with libprotobuf-dev
-    - name: Install dependencies
-      run: sudo apt-get update && sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler libboost-all-dev g++-8
-
-    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
-    - name: Install MKL
-      run: |
-        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
-        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
-        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
-        sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
-      if: matrix.cpu == true
-
-    # The script simplifies installation of different versions of CUDA
-    - name: Install CUDA
-      run: ./3rd_party/marian-dev/scripts/ci/install_cuda_ubuntu.sh ${{ matrix.cuda }}
-      if: matrix.gpu == true
-
-    # Boost is installed on GitHub-hosted runners in a non-standard location
-    # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
-    - name: Configure CMake
-      run: |
-        mkdir -p build
-        cd build
-        CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \
-        cmake .. \
-          -DBoost_ARCHITECTURE=-x64 \
-          -DCMAKE_BUILD_TYPE=Release \
-          -DCOMPILE_CPU=${{ matrix.cpu }} \
-          -DCOMPILE_CUDA=${{ matrix.gpu }} \
-          -DCOMPILE_EXAMPLES=on \
-          -DCOMPILE_SERVER=on \
-          -DCOMPILE_TESTS=on \
-          -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }} \
-          -DUSE_FBGEMM=${{ matrix.cpu }} \
-          -DUSE_SENTENCEPIECE=on \
-          -DUSE_STATIC_LIBS=on \
-          -DUSE_WASM_COMPATIBLE_SOURCE=off
-
-    - name: Compile
-      working-directory: build
-      run: make -j2
-
-    - name: Run unit tests
-      working-directory: build
-      run: make test
-      # GitHub-hosted VMs do not have GPUs, so can not be run in CUDA builds
-      if: matrix.gpu == false
-
-    - name: Print versions
-      working-directory: build
-      run: |
-        ./marian --version
-        ./marian-decoder --version
-        ./marian-scorer --version
-        ./marian-server --version
-        ./spm_encode --version
-
--- a/.github/workflows/native-mac.yml
+++ b/.github/workflows/native-mac.yml
@ -0,0 +1,108 @@
+name: Native MacOS
+
+on:
+  push:
+    branches: [ main, ci-sandbox ]
+  pull_request:
+    branches: [ main, ci-sandbox ]
+
+jobs:
+  build-macos:
+    strategy: 
+      fail-fast: false
+      matrix:
+        include:
+          - name: "full-marian"
+            os: macos-10.15
+            test_tags: ""
+            cmake: 
+              CMAKE_BUILD_TYPE: "Release"
+              COMPILE_TESTS: "ON"
+              USE_WASM_COMPATIBLE_SOURCE: "OFF"
+              USE_FBGEMM: "OFF"
+              USE_STATIC_LIBS: "OFF"
+              COMPILE_SERVER: "OFF"
+              COMPILE_EXAMPLES: "OFF"
+
+          - name: "minimal-marian"
+            os: macos-10.15
+            test_tags: "'#wasm'"
+            cmake: 
+              CMAKE_BUILD_TYPE: "Release"
+              COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and compile tests fail
+              USE_WASM_COMPATIBLE_SOURCE: "ON"
+              USE_FBGEMM: "OFF"
+              # explicitly set due to requirement of minimal marian being used
+              # within WASM. This is some yaml ugliness, but issok.
+              USE_STATIC_LIBS: "ON" 
+              COMPILE_SERVER: "OFF"
+              COMPILE_EXAMPLES: "OFF"
+        
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install dependencies
+      run: |
+          brew update
+          brew install openblas protobuf coreutils
+
+    # Openblas location is exported explicitly because openblas is keg-only,
+    # which means it was not symlinked into /usr/local/.
+    - name: Set BLAS Environment variables
+      run: |
+          echo "LDFLAGS=-L/usr/local/opt/openblas/lib" >> $GITHUB_ENV
+          echo "CPPFLAGS=-I/usr/local/opt/openblas/include" >> $GITHUB_ENV
+      if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF'
+
+    # CMake cannot find BLAS on GitHub runners if Marian is being compiled
+    # statically, hence USE_STATIC_LIBS=off
+    - name: Configure CMake
+      run: |
+        mkdir -p build
+        cd build
+        cmake .. \
+          -DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\
+          -DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\
+          -DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \
+          -DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \
+          -DUSE_STATIC_LIBS=${{ matrix.cmake.USE_STATIC_LIBS }} \
+          -DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \
+          -DUSE_FBGEMM=${{ matrix.cmake.USE_FBGEMM }}
+
+    - name: Compile
+      working-directory: build
+      run: make -j2
+
+    - name: Run unit tests
+      working-directory: build
+      run: make test
+      if: matrix.cmake.COMPILE_TESTS == 'ON'
+
+    - name: Print versions
+      working-directory: build
+      run: |
+        ./app/bergamot-translator-app --version
+
+    - name: Install regression-test framework (BRT)
+      working-directory: bergamot-translator-tests
+      run : make install
+
+    - name: Run regression-tests (BRT)
+      working-directory: bergamot-translator-tests
+      run : MARIAN=../build ./run_brt.sh ${{ matrix.test_tags }}
+
+    - name: Upload regression-tests artifacts
+      uses: actions/upload-artifact@v2
+      if: ${{ always() }}
+      with: 
+        name: brt-artifacts-${{ matrix.name }}
+        path: |
+            bergamot-translator-tests/**/*.expected
+            bergamot-translator-tests/**/*.log
+            bergamot-translator-tests/**/*.out
--- a/.github/workflows/native-ubuntu.yml
+++ b/.github/workflows/native-ubuntu.yml
@ -0,0 +1,117 @@
+name: Native Ubuntu
+
+on:
+  push:
+    branches: [ main, ci-sandbox ]
+  pull_request:
+    branches: [ main, ci-sandbox ]
+
+jobs:
+  build-ubuntu:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: "full-marian"
+            os: ubuntu-latest
+            gcc: 8
+            cpu: 'ON'
+            gpu: 'OFF'
+            test_tags: ""
+            cmake: 
+              CMAKE_BUILD_TYPE: "Release"
+              COMPILE_TESTS: "ON"
+              USE_WASM_COMPATIBLE_SOURCE: "OFF"
+              COMPILE_SERVER: "OFF"
+              COMPILE_EXAMPLES: "OFF"
+
+          - name: "minimal-marian"
+            os: ubuntu-latest
+            gcc: 8
+            cpu: 'ON'
+            gpu: 'OFF'
+            test_tags: "'#wasm'"
+            cmake:
+              CMAKE_BUILD_TYPE: "Release"
+              COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and COMPILE_TEST=ON fails.
+              USE_WASM_COMPATIBLE_SOURCE: "ON"
+              COMPILE_SERVER: "OFF"
+              COMPILE_EXAMPLES: "OFF"
+
+
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.name }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    # The following packages are already installed on GitHub-hosted runners:
+    # build-essential openssl libssl-dev
+    # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because
+    # it is installed together with libprotobuf-dev
+    - name: Install dependencies
+      run: |
+        sudo apt-get update 
+        sudo apt-get install -y \
+            libgoogle-perftools-dev libprotobuf-dev protobuf-compiler  \
+            libboost-all-dev g++-${{ matrix.gcc }} 
+
+    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
+    - name: Install MKL
+      run: |
+        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
+        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
+      if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF'
+
+    # Boost is installed on GitHub-hosted runners in a non-standard location
+    # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
+    - name: Configure CMake
+      run: |
+        mkdir -p build
+        cd build
+        CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \
+        cmake .. \
+          -DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\
+          -DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\
+          -DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \
+          -DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \
+          -DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \
+
+    - name: Compile bergamot-translator
+      working-directory: build
+      run: make -j2
+
+    - name: Run unit tests
+      working-directory: build
+      run: make test
+      # GitHub-hosted VMs do not have GPUs, so can not be run in CUDA builds
+      if: matrix.gpu == 'OFF' && matrix.cmake.COMPILE_TESTS == 'ON'
+
+    - name: Print versions
+      working-directory: build
+      run: |
+        ./app/bergamot-translator-app --version
+
+
+    - name: Install regression-test framework (BRT)
+      working-directory: bergamot-translator-tests
+      run : make install
+
+    - name: Run regression-tests (BRT)
+      working-directory: bergamot-translator-tests
+      run : MARIAN=../build ./run_brt.sh ${{ matrix.test_tags }}
+
+    - name: Upload regression-tests artifacts
+      uses: actions/upload-artifact@v2
+      if: ${{ always() }}
+      with: 
+        name: brt-artifacts-${{ matrix.name }}
+        path: |
+            bergamot-translator-tests/**/*.expected
+            bergamot-translator-tests/**/*.log
+            bergamot-translator-tests/**/*.out
--- a/.github/workflows/wasm-custom_marian-ubuntu.yml
+++ b/.github/workflows/wasm-custom_marian-ubuntu.yml
@ -2,9 +2,9 @@ name: WASM (Custom Marian) Ubuntu

 on:
  push:
-    branches: [ main ]
+    branches: [ main, ci-sandbox ]
  pull_request:
-    branches: [ main ]
+    branches: [ main, ci-sandbox ]

 jobs:
  build-wasm:
--- a/.gitmodules
+++ b/.gitmodules
@ -4,3 +4,6 @@
 [submodule "3rd_party/ssplit-cpp"]
 	path = 3rd_party/ssplit-cpp
 	url = https://github.com/browsermt/ssplit-cpp
+[submodule "bergamot-translator-tests"]
+	path = bergamot-translator-tests
+	url = https://github.com/browsermt/bergamot-translator-tests
--- a/1
+++ b/1
@ -0,0 +1 @@
+Subproject commit 3771001720a8f01bba185ee5d5d908b7c266ef31
				`@ -0,0 +1 @@`
				`Subproject commit 3771001720a8f01bba185ee5d5d908b7c266ef31`