mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-09-11 05:35:33 +03:00
Consolidate release artefacts (#329)
Brings in the previously wasm.yml into python.yml and the new file is renamed as build.yml. python.yml already had a version and pre-release jobs. These jobs downloaded artefacts from prior ran jobs (python wheel builds). The newly attached emscripten build now uploads artefacts of a WebAssembly binary and javascript file which are fed into the release and pre-release jobs in addition to the existing python builds.
This commit is contained in:
parent
91b2e0636d
commit
5e78260d52
@ -1,4 +1,4 @@
|
||||
name: "Python Bindings"
|
||||
name: "Build"
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
@ -11,6 +11,7 @@ name: "Python Bindings"
|
||||
- '**'
|
||||
env:
|
||||
qt_version: "6.2.1" # only used by build-macos
|
||||
emsdk_version: 2.0.9 # For use in emscripten build
|
||||
ccache_basedir: ${{ github.workspace }}
|
||||
ccache_dir: "${{ github.workspace }}/.ccache"
|
||||
ccache_compilercheck: content
|
||||
@ -230,16 +231,161 @@ jobs:
|
||||
with:
|
||||
path: ${{github.workspace}}/dist/bergamot-*.whl
|
||||
|
||||
build-wasm:
|
||||
name: "emscripten"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set ccache environment for emcc
|
||||
run: |
|
||||
# We are hardcoding this to mtime instead of env pickup. Rest use content.
|
||||
echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV
|
||||
|
||||
echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
|
||||
# https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper
|
||||
echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV
|
||||
|
||||
# This need to be run before setup, so ccache build caching doesn't complain.
|
||||
- name: Obtain emsdk sources
|
||||
run: |
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git
|
||||
|
||||
- name: Cache-op for build-cache through ccache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
${{ env.ccache_dir }}
|
||||
${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/
|
||||
key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
|
||||
restore-keys: |-
|
||||
ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
|
||||
ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}
|
||||
ccache-${{ github.job }}-${{ env.emsdk_version }}
|
||||
|
||||
- name: Setup Emscripten toolchain
|
||||
run: |
|
||||
(cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
|
||||
(cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
|
||||
# mtime of this file is checked by ccache, we set it to avoid cache misses.
|
||||
touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten
|
||||
|
||||
# These needs to be done in the activated shell.
|
||||
eval $(./emsdk/emsdk construct_env \
|
||||
| sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \
|
||||
| sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' );
|
||||
|
||||
# This looks more permanent than version pinned, so keeping temporarily to avoid failures.
|
||||
echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Generate ccache_vars for ccache based on machine
|
||||
shell: bash
|
||||
id: ccache_vars
|
||||
run: |-
|
||||
echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
|
||||
echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
|
||||
|
||||
- name: Verify Emscripten setup
|
||||
run: |
|
||||
emcc --version
|
||||
emcmake cmake --version
|
||||
emmake make --version
|
||||
|
||||
- name: ccache prolog
|
||||
run: |-
|
||||
ccache -s # Print current cache stats
|
||||
ccache -z # Zero cache entry
|
||||
|
||||
# WORMHOLE=off
|
||||
- name: "Configure builds for WORMHOLE=off"
|
||||
run: |
|
||||
mkdir -p build-wasm-without-wormhole
|
||||
cd build-wasm-without-wormhole
|
||||
emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ..
|
||||
|
||||
|
||||
- name: "Compile with WORMHOLE=off"
|
||||
working-directory: build-wasm-without-wormhole
|
||||
run: |
|
||||
emmake make -j2
|
||||
|
||||
- name: ccache epilog
|
||||
run: |
|
||||
ccache -s # Print current cache stats
|
||||
|
||||
- name: Import GEMM library from a separate wasm module
|
||||
working-directory: build-wasm-without-wormhole
|
||||
run: bash ../wasm/patch-artifacts-import-gemm-module.sh
|
||||
|
||||
|
||||
# WORMHOLE=on
|
||||
- name: "Configure builds for WORMHOLE=on"
|
||||
run: |
|
||||
mkdir -p build-wasm-with-wormhole
|
||||
cd build-wasm-with-wormhole
|
||||
emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=on ..
|
||||
|
||||
|
||||
- name: "Compile with WORMHOLE=on"
|
||||
working-directory: build-wasm-with-wormhole
|
||||
run: |
|
||||
emmake make -j2
|
||||
|
||||
- name: ccache epilog
|
||||
run: |
|
||||
ccache -s # Print current cache stats
|
||||
|
||||
- name: Instantiate simd wormhole
|
||||
working-directory: build-wasm-with-wormhole
|
||||
run: bash ../wasm/patch-artifacts-enable-wormhole.sh
|
||||
|
||||
- name: Import GEMM library from a separate wasm module
|
||||
working-directory: build-wasm-with-wormhole
|
||||
run: bash ../wasm/patch-artifacts-import-gemm-module.sh
|
||||
|
||||
# Rename the wormhole on builds
|
||||
- name: Rename artefacts with wormhole
|
||||
working-directory: build-wasm-with-wormhole
|
||||
run: |
|
||||
mv bergamot-translator-worker{,-with-wormhole}.js
|
||||
mv bergamot-translator-worker{,-with-wormhole}.js.bak
|
||||
mv bergamot-translator-worker{,-with-wormhole}.wasm
|
||||
|
||||
|
||||
# Upload both together.
|
||||
- name: Upload wasm artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: wasm-artefacts
|
||||
if-no-files-found: error
|
||||
path: |
|
||||
# Without wormhole
|
||||
${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js
|
||||
${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.wasm
|
||||
${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js.bak
|
||||
|
||||
${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js
|
||||
${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm
|
||||
${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js.bak
|
||||
|
||||
# Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model
|
||||
release-latest:
|
||||
name: Release Latest Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [python-ubuntu, python-macos]
|
||||
needs: [python-ubuntu, python-macos, build-wasm]
|
||||
if: github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
|
||||
|
||||
- name: Update GitHub prerelease
|
||||
uses: marvinpinto/action-automatic-releases@latest
|
||||
with:
|
||||
@ -248,12 +394,16 @@ jobs:
|
||||
prerelease: true
|
||||
title: "Latest Build"
|
||||
files: |
|
||||
${{github.workspace}}/artifact/*.whl
|
||||
artifact/*.whl
|
||||
wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js
|
||||
wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm
|
||||
wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js
|
||||
wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm
|
||||
|
||||
release-version:
|
||||
name: Release version
|
||||
runs-on: ubuntu-latest
|
||||
needs: [python-ubuntu, python-macos]
|
||||
needs: [python-ubuntu, python-macos, build-wasm]
|
||||
permissions:
|
||||
contents: "write"
|
||||
packages: "write"
|
||||
@ -271,7 +421,12 @@ jobs:
|
||||
prerelease: false
|
||||
title: "${{ github.ref_name }}"
|
||||
files: |
|
||||
${{github.workspace}}/artifact/*.whl
|
||||
artifact/*.whl
|
||||
wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js
|
||||
wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm
|
||||
wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js
|
||||
wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm
|
||||
|
||||
|
||||
python-checks:
|
||||
name: "formatting and typechecks"
|
123
.github/workflows/wasm.yml
vendored
123
.github/workflows/wasm.yml
vendored
@ -1,123 +0,0 @@
|
||||
name: WebAssembly
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, ci-sandbox ]
|
||||
pull_request:
|
||||
branches: [ '**' ]
|
||||
|
||||
env:
|
||||
ccache_basedir: ${{ github.workspace }}
|
||||
ccache_dir: "${{ github.workspace }}/.ccache"
|
||||
ccache_compilercheck: mtime
|
||||
ccache_compress: 'true'
|
||||
ccache_compresslevel: 9
|
||||
ccache_maxsize: 200M
|
||||
ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache
|
||||
emsdk_version: 2.0.9
|
||||
|
||||
jobs:
|
||||
build-wasm:
|
||||
name: "emscripten"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set ccache environment for emcc
|
||||
run: |
|
||||
echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
|
||||
# https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper
|
||||
echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV
|
||||
|
||||
# This need to be run before setup, so ccache build caching doesn't complain.
|
||||
- name: Obtain emsdk sources
|
||||
run: |
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git
|
||||
|
||||
- name: Cache-op for build-cache through ccache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
${{ env.ccache_dir }}
|
||||
${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/
|
||||
key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
|
||||
restore-keys: |-
|
||||
ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
|
||||
ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}
|
||||
ccache-${{ github.job }}-${{ env.emsdk_version }}
|
||||
|
||||
- name: Setup Emscripten toolchain
|
||||
run: |
|
||||
(cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
|
||||
(cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
|
||||
# mtime of this file is checked by ccache, we set it to avoid cache misses.
|
||||
touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten
|
||||
|
||||
# These needs to be done in the activated shell.
|
||||
eval $(./emsdk/emsdk construct_env \
|
||||
| sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \
|
||||
| sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' );
|
||||
|
||||
# This looks more permanent than version pinned, so keeping temporarily to avoid failures.
|
||||
echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Generate ccache_vars for ccache based on machine
|
||||
shell: bash
|
||||
id: ccache_vars
|
||||
run: |-
|
||||
echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
|
||||
echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
|
||||
|
||||
- name: Verify Emscripten setup
|
||||
run: |
|
||||
emcc --version
|
||||
emcmake cmake --version
|
||||
emmake make --version
|
||||
|
||||
- name: Configure builds
|
||||
run: |
|
||||
mkdir -p build-wasm
|
||||
cd build-wasm
|
||||
emcmake cmake -DCOMPILE_WASM=on ..
|
||||
|
||||
- name: ccache prolog
|
||||
run: |-
|
||||
ccache -s # Print current cache stats
|
||||
ccache -z # Zero cache entry
|
||||
|
||||
- name: Compile
|
||||
working-directory: build-wasm
|
||||
run: |
|
||||
emmake make -j2
|
||||
|
||||
- name: ccache epilog
|
||||
run: |
|
||||
ccache -s # Print current cache stats
|
||||
|
||||
- name: Instantiate simd wormhole
|
||||
working-directory: build-wasm
|
||||
run: bash ../wasm/patch-artifacts-enable-wormhole.sh
|
||||
|
||||
- name: Import GEMM library from a separate wasm module
|
||||
working-directory: build-wasm
|
||||
run: bash ../wasm/patch-artifacts-import-gemm-module.sh
|
||||
|
||||
- name: Upload wasm artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: bergamot-translator-worker
|
||||
if-no-files-found: error
|
||||
path: |
|
||||
${{github.workspace}}/build-wasm/bergamot-translator-worker.js
|
||||
${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm
|
||||
${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak
|
||||
|
Loading…
Reference in New Issue
Block a user