bergamot-translator/.github/workflows/build.yml

name: "Build"
'on':
  push:
    branches:
      - main
      - ci-sandbox
    tags:
      - "v*.*.*"
  pull_request:
    branches:
      - '**'
env:
  qt_version: "6.2.1" # only used by build-macos
  emsdk_version: 3.1.8 # For use in emscripten build
  ccache_basedir: ${{ github.workspace }}
  ccache_dir: "${{ github.workspace }}/.ccache"
  ccache_compilercheck: content
  ccache_compress: 'true'
  ccache_compresslevel: 9
  ccache_maxsize: 200M
  ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache

jobs:
    build-wheels:
      strategy:
        matrix:
          os: [ubuntu-latest, macos-latest]
        fail-fast: false

      name: "cibuildwheel / ${{ matrix.os }}"
      runs-on: ${{ matrix.os }}

      steps:
        - uses: actions/checkout@v2
          with:
            submodules: recursive

        - name: Generate ccache_vars for ccache based on machine
          shell: bash
          id: ccache_vars
          run: |-
            echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
            echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"            

        - name: Cache-op for build-cache through ccache
          uses: actions/cache@v2
          with:
            path: ${{ env.ccache_dir }}
            key: ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
            restore-keys: |-
              ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
              ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}
              ccache-cibuildwheel-${{ matrix.os }}              

        - name: ccache environment setup
          run: |-
            mkdir -p ${{ env.ccache_dir }}            

        - name: Inject local version identifier for non tag builds
          if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
          run: |-
            echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV            

        - name: Apply MacOS patch
          if: ${{ startsWith(runner.os, 'mac') }}
          run: |
            patch -p1 < patches/01-marian-fstream-for-macos.patch            

        - name: Build wheels
          uses: pypa/cibuildwheel@v2.6.1
          # to supply options, put them in 'env', like:
          env:
            CIBW_ENVIRONMENT_LINUX:
              BUILD_ARCH=core-avx-i
              USE_CCACHE=1
              CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}
              CCACHE_COMPRESS=${{ env.ccache_compress }}
              CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}
              CCACHE_MAXSIZE=${{ env.ccache_maxsize }}
              PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }}
              CCACHE_DIR=/host/${{ env.ccache_dir }}
              CCACHE_BASEDIR=/host/${{ env.ccache_basedir }}

            CIBW_ENVIRONMENT_MACOS:
              BUILD_ARCH=core-avx-i
              USE_CCACHE=1
              CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}
              CCACHE_COMPRESS=${{ env.ccache_compress }}
              CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}
              CCACHE_MAXSIZE=${{ env.ccache_maxsize }}
              PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }}
              CCACHE_DIR=${{ env.ccache_dir }}
              CCACHE_BASEDIR=${{ env.ccache_basedir }}
              MACOSX_DEPLOYMENT_TARGET=10.9

            CIBW_BEFORE_BUILD_LINUX: |
              yum install -y ccache

              # Install Intel MKL.
              yum-config-manager -y --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo
              yum install -y intel-mkl

              chmod -R a+rwx /host/${{ env.ccache_dir }}

              ccache -s # Print current cache stats
              ccache -z # Zero cache entry              

            CIBW_BEFORE_BUILD_MACOS: |
              brew install openblas protobuf ccache boost pybind11
              chmod -R a+rwx ${{ env.ccache_dir }}
              ccache -s # Print current cache stats
              ccache -z # Zero cache entry              

            CIBW_BUILD: "cp{36,37,38,39,310}-*manylinux_x86_64 cp{36,37,38,39,310}-macosx_x86_64"

            CIBW_BEFORE_TEST: |
              ccache -s # Print current ccache stats              

            CIBW_TEST_COMMAND: |
              # The wheels are installed automatically and available.

              # Fetch models from translateLocally repository.
              python3 -m bergamot download -m en-de-tiny
              python3 -m bergamot download -m de-en-tiny
              python3 -m bergamot ls

              # Fetch models from opus repository.
              python3 -m bergamot download -m eng-fin-tiny -r opus
              python3 -m bergamot ls -r opus

              # Run the sample python script shipped with module
              python3 -m bergamot translate --model en-de-tiny <<< "Hello World"
              python3 -m bergamot translate --model en-de-tiny de-en-tiny <<< "Hello World"
              python3 -m bergamot translate --model eng-fin-tiny --repository opus <<< "Hello World"              


        - uses: actions/upload-artifact@v2
          with:
            name: wheels
            path: ./wheelhouse/*.whl

    upload-wheels:
      name: "Upload wheels to PyPI"
      runs-on: ubuntu-latest
      if: ${{ startsWith(github.ref, 'refs/tags/v') }}
      needs: [build-wheels]
      steps:
      - name: Download artifacts
        uses: actions/download-artifact@v2
        with:
          name: wheels

      - name: Publish wheels to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python3 -m pip install twine
          twine upload *.whl          


    build-wasm:
      name: "emscripten"
      runs-on: ubuntu-latest
      steps:

        - name: Checkout
          uses: actions/checkout@v2
          with:
            submodules: recursive

        - name: Set ccache environment for emcc
          run: |
            # We are hardcoding this to mtime instead of env pickup. Rest use content.
            echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV

            echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
            echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
            echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
            echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
            echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
            # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper
            echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV            

        # This need to be run before setup, so ccache build caching doesn't complain.
        - name: Obtain emsdk sources
          run: |
              git clone --depth 1 https://github.com/emscripten-core/emsdk.git              

        - name: Cache-op for build-cache through ccache
          uses: actions/cache@v2
          with:
            path: |
                ${{ env.ccache_dir }}
                ${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/                
            key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
            restore-keys: |-
              ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
              ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}
              ccache-${{ github.job }}-${{ env.emsdk_version }}              

        - name: Setup Emscripten toolchain
          run: |
              (cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
              (cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
              # mtime of this file is checked by ccache, we set it to avoid cache misses.
              touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten

              # These needs to be done in the activated shell.
              eval $(./emsdk/emsdk construct_env \
                  | sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \
                  | sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' );

              # This looks more permanent than version pinned, so keeping temporarily to avoid failures.
              echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH              

        - name: Generate ccache_vars for ccache based on machine
          shell: bash
          id: ccache_vars
          run: |-
            echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
            echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"            

        - name: Verify Emscripten setup
          run: |
              emcc --version
              emcmake cmake --version
              emmake make --version              

        - name: ccache prolog
          run: |-
            ccache -s # Print current cache stats
            ccache -z # Zero cache entry            

        - name: "Configure builds"
          run: |
            mkdir -p build-wasm
            cd build-wasm
            emcmake cmake -DCOMPILE_WASM=on ..            


        - name: "Compile"
          working-directory: build-wasm
          run: |
            emmake make -j2            

        - name: ccache epilog
          run: |
            ccache -s # Print current cache stats            

        - name: Import GEMM library from a separate wasm module
          working-directory: build-wasm
          run: bash ../wasm/patch-artifacts-import-gemm-module.sh

        # Setup nodejs-18, as nodejs-14 provided by emsdk fails when running
        # and newer version of node allows us to use fetch().
        - name: Setup nodejs
          uses: actions/setup-node@v3
          with:
            node-version: 18

        - name: Test run
          working-directory: wasm
          run: |
            cp ../build-wasm/bergamot-translator-worker.{js,wasm} ./
            npm install jsdom

            # --unhandled-rejections make the script exit with a non-zero code (at least on node-14).
            # So leaving this here.
            node --unhandled-rejections=strict node-test.js            

        # Upload both together.
        - name: Upload wasm artifact
          uses: actions/upload-artifact@v2
          with:
            name: wasm-artefacts
            if-no-files-found: error
            path: |
                ${{github.workspace}}/build-wasm/bergamot-translator-worker.js
                ${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm
                ${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak                


    upload-wasm:
      name: "Upload node package to NPM"
      runs-on: ubuntu-latest
      if: ${{ startsWith(github.ref, 'refs/tags/v') }}
      needs: [build-wasm]
      steps:
      - name: Download artifacts
        uses: actions/download-artifact@v2
        with:
          name: wasm-artefacts
          path: wasm/module/worker

      - uses: actions/setup-node@v3
        with:
          node-version: '18.x'
          registry-url: 'https://registry.npmjs.org'
      - run: npm ci
      - run: npm publish
        env:
          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}



  # Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model
    release-latest:
      name: Release Latest Build
      runs-on: ubuntu-latest
      needs: [build-wheels, build-wasm]
      if: github.ref == 'refs/heads/main'
      steps:
       - name: Download artifacts
         uses: actions/download-artifact@v2

       # Leave the below be, it will be useful.
       - name: List downloaded assets
         run: |
           find ./           

       - name: Update GitHub prerelease
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           automatic_release_tag: latest
           prerelease: true
           title: "Latest Build"
           files: |
                wheels/*.whl
                wasm-artefacts/bergamot-translator-worker.js
                wasm-artefacts/bergamot-translator-worker.wasm                

    release-version:
      name: Release version
      runs-on: ubuntu-latest
      needs: [build-wheels, build-wasm]
      permissions:
        contents: "write"
        packages: "write"
        pull-requests: "read"
      if: startsWith(github.ref, 'refs/tags/v')
      steps:
       - name: Download artifacts
         uses: actions/download-artifact@v2

       # Leave the below be, it will be useful.
       - name: List downloaded assets
         run: |
           find ./           

       - name: Update GitHub release
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           automatic_release_tag: ${{ github.ref_name }}
           prerelease: false
           title: "${{ github.ref_name }}"
           files: |
                wheels/*.whl
                wasm-artefacts/bergamot-translator-worker.js
                wasm-artefacts/bergamot-translator-worker.wasm                


    python-checks:
      name: "formatting and typechecks"
      runs-on: "ubuntu-latest"
      steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          submodules: recursive
      - name: Install Dependencies
        run: |-
            python3 -m pip install black isort pytype            
      - name: "Formatting checks: black, isort"
        run: |
            python3 -m black --diff --check bindings/python/ setup.py doc/conf.py
            python3 -m isort --profile black --diff --check bindings/python setup.py doc/conf.py            
      - name: "Static typing checks: pytype"
        run: |-
            python3 -m pytype bindings/python            

    docs:
      runs-on: ubuntu-latest
      needs: [build-wheels]
      steps:
        - name: Checkout
          uses: actions/checkout@v2
          with:
            submodules: recursive

        # Runs javascript to extract push events from both tags and branch (only main, due to workflow trigger)
        # converts refs/<>/<name> -> <name>
        # eg:
        #     refs/head/main   -> main
        #     refs/tags/v0.1.0 -> v0.1.0
        #
        - name: Download artifacts
          uses: actions/download-artifact@v2
        - name: Extract tag name
          id: tag
          uses: actions/github-script@0.2.0
          if: ${{ github.event_name == 'push' }}
          with:
            github-token: ${{ secrets.GITHUB_TOKEN }}
            script: |
              const args = context.payload.ref.split("/");
              [refs, category, ...rest] = args;
              return rest.join("/");              

        # Patches the BERGAMOT_VERSION file used by sphinx-docs at run time to
        # obtain names like 'main' or 'ci-sandbox' to not confuse with version
        # based documentation built separately.
        - name: Deploy-time patch version
          run: |
              echo ${{steps.tag.outputs.result }} > BERGAMOT_VERSION              

        - name: Set up Doxygen
          run: sudo apt-get install -y doxygen

        - name: Set up Python
          uses: actions/setup-python@v2
          with:
            python-version: 3.7

        - name: Set up dependency cache
          uses: actions/cache@v2
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('doc/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-              

        - name: Install dependencies
          working-directory: ./doc
          run: |
            python3 -m pip install -r requirements.txt
            python3 -m pip install --find-links=${{github.workspace}}/wheels bergamot            

        - name: Build documentation
          working-directory: ./doc
          run: sphinx-build -b html ./ build/


        - name: Deploy 🚀
          uses: JamesIves/github-pages-deploy-action@4.1.3
          if: ${{ github.event_name == 'push' && github.repository == 'browsermt/bergamot-translator' }}
          with:
            repository-name: 'browsermt/docs'
            branch: gh-pages # The branch the action should deploy to.
            folder: './doc/build/' # The folder the action should deploy.
            target-folder: '${{ steps.tag.outputs.result }}'
            ssh-key: ${{ secrets.BERGAMOT_SSH_PRIVATE_KEY }}

        # This artifact contains the HTML output of Sphinx only.
        # With index.html at the root of the produced zip file.
        # For use for maintainers to download the zip and check render of
        # documentation while generated at pull-request.
        - name: Upload documentation
          uses: actions/upload-artifact@v2
          if: ${{ github.event_name == 'pull_request'}}
          with:
            name: api-docs
            path: ./doc/build/
            if-no-files-found: error