diff --git a/.github/workflows/native-mac.yml b/.github/workflows/native-mac.yml deleted file mode 100644 index bbd3651..0000000 --- a/.github/workflows/native-mac.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Native MacOS - -on: - push: - branches: [ main, ci-sandbox ] - pull_request: - branches: [ main, ci-sandbox ] - -jobs: - build-macos: - strategy: - fail-fast: false - matrix: - include: - - name: "full-marian" - os: macos-10.15 - test_tags: "'#mac'" - cmake: - CMAKE_BUILD_TYPE: "Release" - COMPILE_TESTS: "ON" - USE_WASM_COMPATIBLE_SOURCE: "OFF" - USE_FBGEMM: "OFF" - USE_STATIC_LIBS: "OFF" - COMPILE_SERVER: "OFF" - COMPILE_EXAMPLES: "OFF" - USE_APPLE_ACCELERATE: "OFF" - - - name: "minimal-marian" - os: macos-10.15 - test_tags: "'#wasm'" - cmake: - CMAKE_BUILD_TYPE: "Release" - COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and compile tests fail - USE_WASM_COMPATIBLE_SOURCE: "ON" - USE_FBGEMM: "OFF" - # explicitly set due to requirement of minimal marian being used - # within WASM. This is some yaml ugliness, but issok. - USE_STATIC_LIBS: "ON" - COMPILE_SERVER: "OFF" - COMPILE_EXAMPLES: "OFF" - USE_APPLE_ACCELERATE: "OFF" - - - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Install dependencies - run: | - brew update - brew install openblas protobuf coreutils - - # Openblas location is exported explicitly because openblas is keg-only, - # which means it was not symlinked into /usr/local/. - - name: Set BLAS Environment variables - run: | - echo "LDFLAGS=-L/usr/local/opt/openblas/lib" >> $GITHUB_ENV - echo "CPPFLAGS=-I/usr/local/opt/openblas/include" >> $GITHUB_ENV - if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF' - - # CMake cannot find BLAS on GitHub runners if Marian is being compiled - # statically, hence USE_STATIC_LIBS=off - - name: Configure CMake - run: | - mkdir -p build - cd build - cmake -L .. \ - -DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\ - -DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\ - -DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \ - -DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \ - -DUSE_STATIC_LIBS=${{ matrix.cmake.USE_STATIC_LIBS }} \ - -DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \ - -DUSE_APPLE_ACCELERATE=${{ matrix.cmake.USE_APPLE_ACCELERATE }} \ - -DUSE_FBGEMM=${{ matrix.cmake.USE_FBGEMM }} - - - name: Compile - working-directory: build - run: make -j2 - - - name: Run unit tests - working-directory: build - run: make test - if: matrix.cmake.COMPILE_TESTS == 'ON' - - - name: Print versions - working-directory: build - run: | - ./app/bergamot --version - - - name: Install regression-test framework (BRT) - working-directory: bergamot-translator-tests - run : make install - - - name: Run regression-tests (BRT) - working-directory: bergamot-translator-tests - run : MARIAN=../build ./run_brt.sh ${{ matrix.test_tags }} - - - name: Upload regression-tests artifacts - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: brt-artifacts-${{ matrix.name }} - path: | - bergamot-translator-tests/**/*.expected - bergamot-translator-tests/**/*.log - bergamot-translator-tests/**/*.out diff --git a/.github/workflows/native-ubuntu.yml b/.github/workflows/native-ubuntu.yml deleted file mode 100644 index 0617201..0000000 --- a/.github/workflows/native-ubuntu.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: Native Ubuntu - -on: - push: - branches: [ main, ci-sandbox ] - pull_request: - branches: [ main, ci-sandbox ] - -jobs: - build-ubuntu: - strategy: - fail-fast: false - matrix: - include: - - name: "full-marian" - os: ubuntu-latest - gcc: 8 - force_recache: false - ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%" - cpu: 'ON' - gpu: 'OFF' - test_tags: "" - cmake: - CMAKE_BUILD_TYPE: "Release" - COMPILE_TESTS: "ON" - USE_WASM_COMPATIBLE_SOURCE: "OFF" - COMPILE_SERVER: "OFF" - COMPILE_EXAMPLES: "OFF" - CMAKE_C_COMPILER_LAUNCHER: "ccache" - CMAKE_CXX_COMPILER_LAUNCHER: "ccache" - - - name: "minimal-marian" - os: ubuntu-latest - gcc: 8 - force_recache: false - ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%" - cpu: 'ON' - gpu: 'OFF' - test_tags: "'#wasm'" - cmake: - CMAKE_BUILD_TYPE: "Release" - COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and COMPILE_TEST=ON fails. - USE_WASM_COMPATIBLE_SOURCE: "ON" - COMPILE_SERVER: "OFF" - COMPILE_EXAMPLES: "OFF" - CMAKE_C_COMPILER_LAUNCHER: "ccache" - CMAKE_CXX_COMPILER_LAUNCHER: "ccache" - - - name: "full-marian-force-recache" - os: ubuntu-latest - gcc: 8 - force_recache: true - ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%" - cpu: 'ON' - gpu: 'OFF' - test_tags: "" - cmake: - CMAKE_BUILD_TYPE: "Release" - COMPILE_TESTS: "ON" - USE_WASM_COMPATIBLE_SOURCE: "OFF" - COMPILE_SERVER: "OFF" - COMPILE_EXAMPLES: "OFF" - CMAKE_C_COMPILER_LAUNCHER: "ccache" - CMAKE_CXX_COMPILER_LAUNCHER: "ccache" - - - name: "minimal-marian-force-recache" - os: ubuntu-latest - gcc: 8 - force_recache: true - ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%" - cpu: 'ON' - gpu: 'OFF' - test_tags: "'#wasm'" - cmake: - CMAKE_BUILD_TYPE: "Release" - COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and COMPILE_TEST=ON fails. - USE_WASM_COMPATIBLE_SOURCE: "ON" - COMPILE_SERVER: "OFF" - COMPILE_EXAMPLES: "OFF" - CMAKE_C_COMPILER_LAUNCHER: "ccache" - CMAKE_CXX_COMPILER_LAUNCHER: "ccache" - - - runs-on: ${{ matrix.os }} - name: ${{ matrix.name }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - # The following packages are already installed on GitHub-hosted runners: - # build-essential openssl libssl-dev - # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because - # it is installed together with libprotobuf-dev - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libgoogle-perftools-dev libprotobuf-dev protobuf-compiler \ - libboost-all-dev g++-${{ matrix.gcc }} ccache - - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html - - name: Install MKL - run: | - wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 - if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF' - - - name: Generate ccache_vars - id: ccache_vars - shell: bash - run: | - echo "::set-output name=hash::$(${{ matrix.ccache_cmd }})" - echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" - - - name: Setup ccache environment variables - run: | - echo "CCACHE_COMPILERCHECK=${{ matrix.ccache_cmd }}" >> $GITHUB_ENV - echo "CCACHE_BASE_DIR=${{ github.workspace }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=2G" >> $GITHUB_ENV - - - name: Setup ccache recache on - run: | - echo "CCACHE_RECACHE=" >> $GITHUB_ENV - if: matrix.force_recache == true - - - name: Cache-op for build-cache through ccache - uses: actions/cache@v2 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} - restore-keys: | - ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}- - ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}- - ccache-${{ matrix.name }}- - - - name: Cache stats before build - run: | - ccache -s - ccache -z - - # Boost is installed on GitHub-hosted runners in a non-standard location - # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671 - - name: Configure CMake - run: | - mkdir -p build - cd build - CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \ - cmake -L .. \ - -DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\ - -DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\ - -DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \ - -DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \ - -DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \ - -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.cmake.CMAKE_C_COMPILER_LAUNCHER}} \ - -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.cmake.CMAKE_CXX_COMPILER_LAUNCHER}} - - - - name: Compile bergamot-translator - working-directory: build - run: make -j2 - - - name: Cache stats after build - run: | - ccache -s - - - name: Run unit tests - working-directory: build - run: make test - # GitHub-hosted VMs do not have GPUs, so can not be run in CUDA builds - if: matrix.gpu == 'OFF' && matrix.cmake.COMPILE_TESTS == 'ON' - - - name: Print versions - working-directory: build - run: | - ./app/bergamot --version - - - - name: Install regression-test framework (BRT) - working-directory: bergamot-translator-tests - run : make install - - - name: Run regression-tests (BRT) - working-directory: bergamot-translator-tests - run : MARIAN=../build ./run_brt.sh ${{ matrix.test_tags }} - - - name: Upload regression-tests artifacts - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: brt-artifacts-${{ matrix.name }} - path: | - bergamot-translator-tests/**/*.expected - bergamot-translator-tests/**/*.log - bergamot-translator-tests/**/*.out diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml new file mode 100644 index 0000000..eb3f4d4 --- /dev/null +++ b/.github/workflows/native.yml @@ -0,0 +1,238 @@ +name: native +'on': + push: + branches: + - main + - ci-sandbox + pull_request: + branches: + - '**' +env: + ccache_basedir: ${{ github.workspace }} + ccache_dir: "${{ github.workspace }}/.ccache" + ccache_compilercheck: content + ccache_compress: 'true' + ccache_compresslevel: 9 + ccache_maxsize: 200M + ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache +jobs: + ubuntu: + strategy: + fail-fast: false + matrix: + include: + - name: Ubuntu 18.04 full + os: ubuntu-18.04 + identifier: ubuntu_1804_full + cmake: -DCOMPILE_TESTS=on + brt_tags: '' + unittests: 'true' + - name: Ubuntu 18.04 minimal + os: ubuntu-18.04 + identifier: ubuntu_1804_minimal + cmake: -DCOMPILE_TESTS=off -DUSE_WASM_COMPATIBLE_SOURCE=on + brt_tags: "'#wasm'" + unittests: 'false' + - name: Ubuntu 20.04 full + os: ubuntu-20.04 + identifier: ubuntu_2004_full + cmake: -DCOMPILE_TESTS=on + brt_tags: '' + unittests: 'true' + - name: Ubuntu 20.04 minimal + os: ubuntu-20.04 + identifier: ubuntu_2004_minimal + cmake: -DCOMPILE_TESTS=off -DUSE_WASM_COMPATIBLE_SOURCE=on + brt_tags: "'#wasm'" + unittests: 'false' + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Install Dependencies + run: |- + sudo apt-get update + sudo apt-get install -y \ + libgoogle-perftools-dev libprotobuf-dev protobuf-compiler \ + libboost-all-dev ccache + - name: Install MKL + run: |- + wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - + sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" + sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" + sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 + - name: Generate ccache_vars for ccache based on machine + shell: bash + id: ccache_vars + run: |- + echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" + echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" + - name: Cache-op for build-cache through ccache + uses: actions/cache@v2 + with: + path: ${{ env.ccache_dir }} + key: ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} + restore-keys: |- + ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} + ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }} + ccache-${{ matrix.identifier }} + - name: ccache environment setup + run: |- + echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV + echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV + - name: ccache prolog + run: |- + ccache -s # Print current cache stats + ccache -z # Zero cache entry + - name: cmake + run: |- + mkdir -p build + cd build + cmake -L .. ${{ matrix.cmake }} ${{ env.ccache_cmake }} + - name: Build from source + working-directory: build + run: make -j2 + - name: ccache epilog + run: 'ccache -s # Print current cache stats' + - name: Print Versions + working-directory: build + run: ./app/bergamot --version + - name: Run unit tests + working-directory: build + run: make test + if: ${{ matrix.unittests == 'true' }} + - name: Install regression-test framework (BRT) + working-directory: bergamot-translator-tests + run: make install + - name: Run regression-tests (BRT) + working-directory: bergamot-translator-tests + id: brt_run + run: MARIAN=../build ./run_brt.sh ${{ matrix.brt_tags }} + - name: Print logs of unsuccessful BRTs + working-directory: bergamot-translator-tests + run: |- + grep "tests.*.sh" previous.log \ + | sed 's/^\s*-\s*//' \ + | xargs -I% bash -c 'echo %; tail -n20 %.log' + if: ${{ always() && steps.brt_run.outcome == 'failure' }} + - name: Upload regression-tests artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() && steps.brt_run.outcome != 'skipped' }} + with: + name: brt-${{ matrix.identifier }} + path: |- + bergamot-translator-tests/**/*.expected + bergamot-translator-tests/**/*.log + bergamot-translator-tests/**/*.out + mac: + strategy: + fail-fast: false + matrix: + include: + - name: MacOS 10.15 full + os: macos-10.15 + identifier: mac_1015_full + cmake: -DCOMPILE_TESTS=on -DUSE_APPLE_ACCELERATE=off -DUSE_FBGEMM=off -DUSE_STATIC_LIBS=off + brt_tags: "'#mac'" + unittests: 'true' + - name: MacOS 10.15 minimal + os: macos-10.15 + identifier: mac_1015_minimal + cmake: -DCOMPILE_TESTS=off -DUSE_APPLE_ACCELERATE=off -DUSE_FBGEMM=off -DUSE_STATIC_LIBS=on -DUSE_WASM_COMPATIBLE_SOURCE=on + brt_tags: "'#wasm'" + unittests: 'false' + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Install Dependencies + run: |- + brew update + brew install openblas protobuf ccache + brew install coreutils findutils + - name: Setup path with gnu + run: |- + echo "/usr/local/opt/coreutils/libexec/gnubin" >> $GITHUB_PATH + echo "/usr/local/opt/findutils/libexec/gnubin" >> $GITHUB_PATH + - name: Setup BLAS + run: |- + echo "LDFLAGS=-L/usr/local/opt/openblas/lib" >> $GITHUB_ENV + echo "CPPFLAGS=-I/usr/local/opt/openblas/include" >> $GITHUB_ENV + - name: Generate ccache_vars for ccache based on machine + shell: bash + id: ccache_vars + run: |- + echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" + echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" + - name: Cache-op for build-cache through ccache + uses: actions/cache@v2 + with: + path: ${{ env.ccache_dir }} + key: ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} + restore-keys: |- + ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} + ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }} + ccache-${{ matrix.identifier }} + - name: ccache environment setup + run: |- + echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV + echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV + - name: ccache prolog + run: |- + ccache -s # Print current cache stats + ccache -z # Zero cache entry + - name: cmake + run: |- + mkdir -p build + cd build + cmake -L .. ${{ matrix.cmake }} ${{ env.ccache_cmake }} + - name: Build from source + working-directory: build + run: make -j2 + - name: ccache epilog + run: 'ccache -s # Print current cache stats' + - name: Print Versions + working-directory: build + run: ./app/bergamot --version + - name: Run unit tests + working-directory: build + run: make test + if: ${{ matrix.unittests == 'true' }} + - name: Install regression-test framework (BRT) + working-directory: bergamot-translator-tests + run: make install + - name: Run regression-tests (BRT) + working-directory: bergamot-translator-tests + id: brt_run + run: MARIAN=../build ./run_brt.sh ${{ matrix.brt_tags }} + - name: Print logs of unsuccessful BRTs + working-directory: bergamot-translator-tests + run: |- + grep "tests.*.sh" previous.log \ + | sed 's/^\s*-\s*//' \ + | xargs -I% bash -c 'echo %; tail -n20 %.log' + if: ${{ always() && steps.brt_run.outcome == 'failure' }} + - name: Upload regression-tests artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() && steps.brt_run.outcome != 'skipped' }} + with: + name: brt-${{ matrix.identifier }} + path: |- + bergamot-translator-tests/**/*.expected + bergamot-translator-tests/**/*.log + bergamot-translator-tests/**/*.out +