mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-08-15 16:40:26 +03:00
Enabling ccache on github builds for Ubuntu (#95)
* CI Changes to add tiny regression tests * Adding an inspect cache step * Removing ccache, pursue in another * Incorporating Nick's changes through submodule merge * Submodule now points to master * Restoring ccache enabled workflow file * Restoring ccache enabled CMakeLists * cache -> ccache typo fix * Moving CCACHE setup to GitHub runner file * Find also uses CCACHE dir * Updating CMakeLists not to override env * Cache compiler binary's contents * Changing a few names to trigger new build; Testing cache looks fun * USE_CCACHE=on, -L for inspection * Adding a ccache_cmd, but will only use in next commit * Using ccache_cmd * Removing " * Adding compiler hash script * Bunch of absolute paths * GITHUB_WORKSPACE typo * Nah, I'll keep -L and trigger another build * Trying something with compiler hash on cache key backup as well * builtin, bash it seems * Empty commit #1 * Move ccache stats to after compile * Reshuffling ccache vars * No comments * Updates to Github output set syntax * Empty Commit 1 * Empty Commit 2 * Empty commit 3 * /bin/bash -> bash; ccache_cmd for consistency * Adding ccache -s before and after build * Adding comments to compiler-hash script * Let's build cached and non-cached variants together for comparison * Fixing quotes, /bin/bash -> bash * Minor var/env adjustment * Adding ccache -z before the job * Reverting CMakeLists.txt without CCACHE * Switching to CMAKE_LANG_COMPILER_LAUNCHER instead of CMakeLists.txt rule * 5G -> 1G cache size * 1G -> 2G; Hyperparameter tuning
This commit is contained in:
Jerin Philip
GitHub
parent
6c7e6156ab
commit
77424a3df1
89
.github/workflows/native-ubuntu.yml
vendored
89
.github/workflows/native-ubuntu.yml
vendored
@ -15,6 +15,8 @@ jobs:
|
||||
- name: "full-marian"
|
||||
os: ubuntu-latest
|
||||
gcc: 8
|
||||
force_recache: false
|
||||
ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%"
|
||||
cpu: 'ON'
|
||||
gpu: 'OFF'
|
||||
test_tags: ""
|
||||
@ -24,10 +26,14 @@ jobs:
|
||||
USE_WASM_COMPATIBLE_SOURCE: "OFF"
|
||||
COMPILE_SERVER: "OFF"
|
||||
COMPILE_EXAMPLES: "OFF"
|
||||
CMAKE_C_COMPILER_LAUNCHER: "ccache"
|
||||
CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
|
||||
|
||||
- name: "minimal-marian"
|
||||
os: ubuntu-latest
|
||||
gcc: 8
|
||||
force_recache: false
|
||||
ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%"
|
||||
cpu: 'ON'
|
||||
gpu: 'OFF'
|
||||
test_tags: "'#wasm'"
|
||||
@ -37,6 +43,42 @@ jobs:
|
||||
USE_WASM_COMPATIBLE_SOURCE: "ON"
|
||||
COMPILE_SERVER: "OFF"
|
||||
COMPILE_EXAMPLES: "OFF"
|
||||
CMAKE_C_COMPILER_LAUNCHER: "ccache"
|
||||
CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
|
||||
|
||||
- name: "full-marian-force-recache"
|
||||
os: ubuntu-latest
|
||||
gcc: 8
|
||||
force_recache: true
|
||||
ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%"
|
||||
cpu: 'ON'
|
||||
gpu: 'OFF'
|
||||
test_tags: ""
|
||||
cmake:
|
||||
CMAKE_BUILD_TYPE: "Release"
|
||||
COMPILE_TESTS: "ON"
|
||||
USE_WASM_COMPATIBLE_SOURCE: "OFF"
|
||||
COMPILE_SERVER: "OFF"
|
||||
COMPILE_EXAMPLES: "OFF"
|
||||
CMAKE_C_COMPILER_LAUNCHER: "ccache"
|
||||
CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
|
||||
|
||||
- name: "minimal-marian-force-recache"
|
||||
os: ubuntu-latest
|
||||
gcc: 8
|
||||
force_recache: true
|
||||
ccache_cmd: "bash ${GITHUB_WORKSPACE}/scripts/ci/compiler-hash.sh %compiler%"
|
||||
cpu: 'ON'
|
||||
gpu: 'OFF'
|
||||
test_tags: "'#wasm'"
|
||||
cmake:
|
||||
CMAKE_BUILD_TYPE: "Release"
|
||||
COMPILE_TESTS: "OFF" # Minimal marian has no sqlite support and COMPILE_TEST=ON fails.
|
||||
USE_WASM_COMPATIBLE_SOURCE: "ON"
|
||||
COMPILE_SERVER: "OFF"
|
||||
COMPILE_EXAMPLES: "OFF"
|
||||
CMAKE_C_COMPILER_LAUNCHER: "ccache"
|
||||
CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
|
||||
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
@ -57,7 +99,7 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
libgoogle-perftools-dev libprotobuf-dev protobuf-compiler \
|
||||
libboost-all-dev g++-${{ matrix.gcc }}
|
||||
libboost-all-dev g++-${{ matrix.gcc }} ccache
|
||||
|
||||
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
|
||||
- name: Install MKL
|
||||
@ -68,6 +110,42 @@ jobs:
|
||||
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
|
||||
if: matrix.cmake.USE_WASM_COMPATIBLE_SOURCE == 'OFF'
|
||||
|
||||
- name: Generate ccache_vars
|
||||
id: ccache_vars
|
||||
shell: bash
|
||||
run: |
|
||||
echo "::set-output name=hash::$(${{ matrix.ccache_cmd }})"
|
||||
echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
|
||||
|
||||
- name: Setup ccache environment variables
|
||||
run: |
|
||||
echo "CCACHE_COMPILERCHECK=${{ matrix.ccache_cmd }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_BASE_DIR=${{ github.workspace }}" >> $GITHUB_ENV
|
||||
echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV
|
||||
echo "CCACHE_MAXSIZE=2G" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup ccache recache on
|
||||
run: |
|
||||
echo "CCACHE_RECACHE=" >> $GITHUB_ENV
|
||||
if: matrix.force_recache == true
|
||||
|
||||
- name: Cache-op for build-cache through ccache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ env.CCACHE_DIR }}
|
||||
key: ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
|
||||
restore-keys: |
|
||||
ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-
|
||||
ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-
|
||||
ccache-${{ matrix.name }}-
|
||||
|
||||
- name: Cache stats before build
|
||||
run: |
|
||||
ccache -s
|
||||
ccache -z
|
||||
|
||||
# Boost is installed on GitHub-hosted runners in a non-standard location
|
||||
# https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
|
||||
- name: Configure CMake
|
||||
@ -75,17 +153,24 @@ jobs:
|
||||
mkdir -p build
|
||||
cd build
|
||||
CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \
|
||||
cmake .. \
|
||||
cmake -L .. \
|
||||
-DCMAKE_BUILD_TYPE=${{ matrix.cmake.CMAKE_BUILD_TYPE }}\
|
||||
-DCOMPILE_TESTS=${{ matrix.cmake.COMPILE_TESTS }}\
|
||||
-DCOMPILE_EXAMPLES=${{ matrix.cmake.COMPILE_EXAMPLES }} \
|
||||
-DCOMPILE_SERVER=${{ matrix.cmake.COMPILE_SERVER }} \
|
||||
-DUSE_WASM_COMPATIBLE_SOURCE=${{ matrix.cmake.USE_WASM_COMPATIBLE_SOURCE }} \
|
||||
-DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.cmake.CMAKE_C_COMPILER_LAUNCHER}} \
|
||||
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.cmake.CMAKE_CXX_COMPILER_LAUNCHER}}
|
||||
|
||||
|
||||
- name: Compile bergamot-translator
|
||||
working-directory: build
|
||||
run: make -j2
|
||||
|
||||
- name: Cache stats after build
|
||||
run: |
|
||||
ccache -s
|
||||
|
||||
- name: Run unit tests
|
||||
working-directory: build
|
||||
run: make test
|
||||
|
||||
35
scripts/ci/compiler-hash.sh
Normal file
35
scripts/ci/compiler-hash.sh
Normal file
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Uses the command from https://stackoverflow.com/a/9355840/4565794.
|
||||
# -v displays the commands executed to run compilation. Of this cc1 additional
|
||||
# flags which contain the flags triggered by -march=native is what we need.
|
||||
# -E stop after preprocessing stage.
|
||||
|
||||
# Output on a linux machine with gcc-8 looks as follows:
|
||||
|
||||
# $ gcc -march=native -E -v - </dev/null 2>&1 | grep cc1
|
||||
# /usr/lib/gcc/x86_64-linux-gnu/8/cc1 -E -quiet -v -imultiarch x86_64-linux-gnu
|
||||
# - -march=skylake-avx512 -mmmx -mno-3dnow -msse -msse2 -msse3 -mssse3
|
||||
# -mno-sse4a -mcx16 -msahf -mmovbe -maes -mno-sha -mpclmul -mpopcnt -mabm
|
||||
# -mno-lwp -mfma -mno-fma4 -mno-xop -mbmi -mno-sgx -mbmi2 -mno-pconfig
|
||||
# -mno-wbnoinvd -mno-tbm -mavx -mavx2 -msse4.2 -msse4.1 -mlzcnt -mno-rtm
|
||||
# -mno-hle -mrdrnd -mf16c -mfsgsbase -mrdseed -mprfchw -madx -mfxsr -mxsave
|
||||
# -mxsaveopt -mavx512f -mno-avx512er -mavx512cd -mno-avx512pf -mno-prefetchwt1
|
||||
# -mclflushopt -mxsavec -mxsaves -mavx512dq -mavx512bw -mavx512vl
|
||||
# -mno-avx512ifma -mno-avx512vbmi -mno-avx5124fmaps -mno-avx5124vnniw -mclwb
|
||||
# -mno-mwaitx -mno-clzero -mpku -mno-rdpid -mno-gfni -mno-shstk
|
||||
# -mno-avx512vbmi2 -mavx512vnni -mno-vaes -mno-vpclmulqdq -mno-avx512bitalg
|
||||
# -mno-movdiri -mno-movdir64b --param l1-cache-size=32 --param
|
||||
# l1-cache-line-size=64 --param l2-cache-size=28160 -mtune=skylake-avx512
|
||||
# -fstack-protector-strong -Wformat -Wformat-security
|
||||
|
||||
# The sha256sum of the output is computed, and stripped to the first 8
|
||||
# characters for use in ccache and github cache store key. Can effectively be
|
||||
# considered as a hash of the compiler version and the flags activated by
|
||||
# -march=native.
|
||||
|
||||
COMPILER=$1
|
||||
|
||||
$COMPILER -march=native -E -v - < /dev/null 2>&1 | grep cc1 \
|
||||
| sha256sum | cut -c1-8
|
||||
|
||||
Loading…
Reference in New Issue
Block a user