Updates to GitHub CI workflows (#730)

* Update GitHub workflows
* Disable marian-server on Windows checks
* Rename GitHub workflows
* Remove CMakeSettings.json files used for CI only
This commit is contained in:
Roman Grundkiewicz 2020-09-16 10:27:38 +01:00 committed by GitHub
parent 97f094b444
commit 467b15e2b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 199 additions and 324 deletions

View File

@ -1,4 +1,4 @@
name: macOS CPU-only
name: MacOS
on:
push:
@ -8,7 +8,7 @@ on:
jobs:
build-macos:
name: MacOS CPU-only
runs-on: macos-10.15
steps:
@ -30,8 +30,15 @@ jobs:
export CPPFLAGS="-I/usr/local/opt/openblas/include"
mkdir -p build
cd build
cmake .. -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
-DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on -DUSE_STATIC_LIBS=off
cmake .. \
-DCOMPILE_CPU=on \
-DCOMPILE_CUDA=off \
-DCOMPILE_EXAMPLES=on \
-DCOMPILE_SERVER=on \
-DCOMPILE_TESTS=on \
-DUSE_FBGEMM=on \
-DUSE_SENTENCEPIECE=on \
-DUSE_STATIC_LIBS=off
- name: Compile
working-directory: build

View File

@ -1,75 +0,0 @@
name: Ubuntu 18.04 CPU-only
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build-ubuntu:
runs-on: ubuntu-18.04
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
# The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
- name: Install dependencies
run: sudo apt-get install --no-install-recommends libgoogle-perftools-dev libprotobuf10 libprotobuf-dev protobuf-compiler
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
- name: Install MKL
run: |
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get install --no-install-recommends intel-mkl-64bit-2020.0-088
- name: Print Boost paths
run: |
ls $BOOST_ROOT_1_69_0
ls $BOOST_ROOT_1_69_0/include
ls $BOOST_ROOT_1_69_0/lib
# Boost is already installed on GitHub-hosted runners in a non-standard location
# https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
- name: Configure CMake
run: |
mkdir -p build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release \
-DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
-DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
-DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
-DBoost_ARCHITECTURE=-x64
- name: Compile
working-directory: build
run: make -j2
- name: Run unit tests
working-directory: build
run: make test
- name: Print versions
working-directory: build
run: |
./marian --version
./marian-decoder --version
./marian-scorer --version
./spm_encode --version
- name: Prepare archive
working-directory: build
run: tar zcvf marian-ubuntu-release-static.tar.gz marian*
# Marian is built with FBGEMM, so there are some restrictions on what CPUs the executables can be run
- name: Upload archive
uses: actions/upload-artifact@v2
with:
name: marian-ubuntu-release-static.tar.gz
path: build/marian-ubuntu-release-static.tar.gz

View File

@ -1,4 +1,4 @@
name: Ubuntu CPU+CUDA
name: Ubuntu
on:
push:
@ -7,37 +7,48 @@ on:
branches: [ master ]
jobs:
build-ubuntu-cuda:
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.experimental }}
build-ubuntu:
strategy:
matrix:
include:
# Ubuntu CPU-only build
- name: "Ubuntu CPU-only"
os: ubuntu-latest
cuda: ""
gcc: 7
cpu: true
gpu: false
# Ubuntu GPU-only build
- name: "Ubuntu GPU-only"
os: ubuntu-latest
cuda: "10.2"
gcc: 7
cpu: false
gpu: true
# Ubuntu 20.04 supports CUDA 11+
#- os: ubuntu-20.04
#- name: "Ubuntu 20.04 CUDA 11.0 gcc-9"
#os: ubuntu-20.04
#cuda: "11.0"
#gcc: 9
#boost: false # ubuntu-20.04 image does not have Boost pre-installed yet
#experimental: true # continue even if the job fails
#cpu: false
#gpu: true
# Ubuntu 18.04 supports CUDA 10.1+
- os: ubuntu-18.04
- name: "Ubuntu 18.04 CUDA 10.2 gcc-8"
os: ubuntu-18.04
cuda: "10.2"
gcc: 8
boost: true
experimental: false
cpu: true
gpu: true
# Ubuntu 16.04 supports CUDA 8+
- os: ubuntu-16.04
cuda: "10.2"
- name: "Ubuntu 16.04 CUDA 9.2 gcc-7"
os: ubuntu-16.04
cuda: "9.2"
gcc: 7
boost: true
experimental: false
- os: ubuntu-16.04
cuda: 9.2
gcc: 7
boost: true
experimental: false
cpu: true
gpu: true
runs-on: ${{ matrix.os }}
name: ${{ matrix.name }}
steps:
- name: Checkout
@ -45,8 +56,10 @@ jobs:
with:
submodules: recursive
# The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
# The following packages are already installed on GitHub-hosted runners:
# build-essential openssl libssl-dev
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because
# it is installed together with libprotobuf-dev
- name: Install dependencies
run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler
@ -57,12 +70,14 @@ jobs:
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
if: matrix.cpu == true
# The script simplifies installation of different versions of CUDA
- name: Install CUDA
run: ./scripts/ci/install_cuda_ubuntu.sh ${{ matrix.cuda }}
if: matrix.gpu == true
# Boost is already installed on GitHub-hosted runners in a non-standard location
# Boost is installed on GitHub-hosted runners in a non-standard location
# https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
- name: Configure CMake
run: |
@ -70,21 +85,31 @@ jobs:
cd build
CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \
cmake .. \
-DCOMPILE_CPU=on -DCOMPILE_CUDA=on -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=${{ matrix.boost }} -DCOMPILE_TESTS=on \
-DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
-DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
-DBoost_ARCHITECTURE=-x64 \
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }}
-DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include \
-DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
-DBOOST_ROOT=$BOOST_ROOT_1_69_0 \
-DCMAKE_BUILD_TYPE=Release \
-DCOMPILE_CPU=${{ matrix.cpu }} \
-DCOMPILE_CUDA=${{ matrix.gpu }} \
-DCOMPILE_EXAMPLES=on \
-DCOMPILE_SERVER=on \
-DCOMPILE_TESTS=on \
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }} \
-DUSE_FBGEMM=${{ matrix.cpu }} \
-DUSE_SENTENCEPIECE=on \
-DUSE_STATIC_LIBS=on \
- name: Compile
working-directory: build
run: make -j2
# Unit tests are not run because GitHub-hosted runners do not have GPUs
# TODO: add a flag to CMake to compile unit tests only on CPU
#- name: Run unit tests
#working-directory: build
#run: make test
- name: Run unit tests
working-directory: build
run: make test
# GitHub-hosted VMs do not have GPUs, so can not be run in CUDA builds
if: matrix.gpu == false
- name: Print versions
working-directory: build
@ -92,5 +117,6 @@ jobs:
./marian --version
./marian-decoder --version
./marian-scorer --version
./marian-server --version
./spm_encode --version

View File

@ -1,67 +0,0 @@
name: Windows CPU-only
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build-windows:
runs-on: windows-2019
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
- name: Download MKL
run: |
# Wget can retry downloading files, so it is used instead of Invoke-WebRequest
C:\msys64\usr\bin\wget.exe -nv https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -O mkl.zip
Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
# Set the MKLROOT environment variable so that CMake can find MKL.
# GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
shell: powershell
- name: Prepare vcpkg
uses: lukka/run-vcpkg@v2
with:
vcpkgArguments: protobuf
vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
vcpkgDirectory: ${{ github.workspace }}/vcpkg/
vcpkgTriplet: x64-windows-static
# Build with a simplified CMake settings JSON file
- name: Run CMake
uses: lukka/run-cmake@v2
with:
buildDirectory: ${{ github.workspace }}/build/
cmakeAppendedArgs: -G Ninja
cmakeListsOrSettingsJson: CMakeSettingsJson
# JSON file must be in the same directory as the main CMakeLists.txt
cmakeSettingsJsonPath: ${{ github.workspace }}/_CMakeSettingsCI_CPU.json
useVcpkgToolchainFile: true
- name: Run unit tests
working-directory: build/Release/
run: ctest
#- name: Print versions
#working-directory: build/Release/
#run: |
#.\marian.exe --version
#.\marian-decoder.exe --version
#.\marian-scorer.exe --version
#.\spm_encode.exe --version
#shell: powershell
# Marian is built with FBGEMM, so there are some restrictions on what CPUs the executables can be run
- name: Upload archive
uses: actions/upload-artifact@v2
with:
name: marian-windows-release-static
path: build/Release/marian*.exe

View File

@ -1,64 +0,0 @@
name: Windows CPU+CUDA
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build-windows-cuda:
runs-on: windows-2019
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
- name: Install CUDA
run: |
.\scripts\ci\install_cuda_windows.ps1 "10.2"
# Set path to CUDA for subsequent steps so that CMake can find it
echo "::set-env name=CUDA_PATH::$env:CUDA_PATH"
echo "::add-path::$env:CUDA_PATH/bin"
shell: powershell
- name: Download MKL
run: |
# Wget can retry downloading files, so it is used instead of Invoke-WebRequest
C:\msys64\usr\bin\wget.exe -nv https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -O mkl.zip
Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
# Set the MKLROOT environment variable so that CMake can find MKL.
# GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
shell: powershell
- name: Prepare vcpkg
uses: lukka/run-vcpkg@v2
with:
vcpkgArguments: protobuf
vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
vcpkgDirectory: ${{ github.workspace }}/vcpkg/
vcpkgTriplet: x64-windows-static
# Build with a simplified CMake settings JSON file.
# On Windows+CUDA we compile with COMPILE_CUDA=on and USE_NCCL=off
- name: Run CMake
uses: lukka/run-cmake@v2
with:
buildDirectory: ${{ github.workspace }}/build/
cmakeAppendedArgs: -G Ninja
cmakeListsOrSettingsJson: CMakeSettingsJson
# JSON file must be in the same directory as the main CMakeLists.txt
cmakeSettingsJsonPath: ${{ github.workspace }}/_CMakeSettingsCI_GPU.json
useVcpkgToolchainFile: true
- name: Print versions
working-directory: build/Debug/
run: |
.\marian.exe --version
.\marian-decoder.exe --version
.\marian-scorer.exe --version
.\spm_encode.exe --version

130
.github/workflows/windows.yml vendored Normal file
View File

@ -0,0 +1,130 @@
name: Windows
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
env:
MKL_URL: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip"
jobs:
build-windows:
strategy:
matrix:
include:
# Windows CPU-only build
- name: "Windows CPU-only"
cuda: ""
gpu: false
# Windows CPU+GPU build
- name: "Windows CPU+CUDA"
cuda: "10.2"
gpu: true
runs-on: windows-2019
name: ${{ matrix.name }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
- name: Download MKL
run: |
# Wget retries downloading files and is faster than Invoke-WebRequest
C:\msys64\usr\bin\wget.exe -nv ${{ env.MKL_URL }} -O mkl.zip
Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
# Set the MKLROOT environment variable so that CMake can find MKL.
# GITHUB_WORKSPACE is an environment variable available on all
# GitHub-hosted runners
echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
shell: powershell
- name: Install CUDA
run: |
.\scripts\ci\install_cuda_windows.ps1 "10.2"
# Set path to CUDA for subsequent steps so that CMake can find it
echo "::set-env name=CUDA_PATH::$env:CUDA_PATH"
echo "::add-path::$env:CUDA_PATH/bin"
shell: powershell
if: matrix.gpu == true
- name: Prepare vcpkg
uses: lukka/run-vcpkg@v2
with:
vcpkgArguments: protobuf
vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
vcpkgDirectory: ${{ github.workspace }}/vcpkg/
vcpkgTriplet: x64-windows-static
# Windows CUDA builds use USE_NCCL=off due to compilation errors.
- name: Build Debug
uses: lukka/run-cmake@v2
with:
buildDirectory: ${{ github.workspace }}/build/Debug
cmakeAppendedArgs: '-G Ninja
-DCMAKE_BUILD_TYPE="Debug"
-DOPENSSL_USE_STATIC_LIBS="TRUE"
-DOPENSSL_MSVC_STATIC_RT="TRUE"
-DCOMPILE_CPU="TRUE"
-DCOMPILE_CUDA="${{ matrix.gpu }}"
-DCOMPILE_SERVER="FALSE"
-DCOMPILE_TESTS="TRUE"
-DUSE_FBGEMM="TRUE"
-DUSE_MPI="FALSE"
-DUSE_NCCL="FALSE"
-DUSE_SENTENCEPIECE="TRUE"
-DUSE_STATIC_LIBS="TRUE"'
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt
useVcpkgToolchainFile: true
# Building in Debug is sufficient for the all-in CPU+GPU compilation;
# its main purpose is to detect warnings that the Release build is not
# able to find sometimes.
if: matrix.gpu == true
# Windows CUDA builds use USE_NCCL=off due to compilation errors
# Boost is pre-installed on Azure/GitHub-hosted Windows runners
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#boost
# (not used yet)
- name: Build Release
uses: lukka/run-cmake@v2
with:
buildDirectory: ${{ github.workspace }}/build/
cmakeAppendedArgs: '-G Ninja
-DBOOST_ROOT="$(BOOST_ROOT_1_72_0)"
-DBOOST_INCLUDEDIR="$(BOOST_ROOT_1_72_0)/include"
-DBOOST_LIBRARYDIR="$(BOOST_ROOT_1_72_0)/lib"
-DCMAKE_BUILD_TYPE="Release"
-DOPENSSL_USE_STATIC_LIBS="TRUE"
-DOPENSSL_MSVC_STATIC_RT="TRUE"
-DCOMPILE_CPU="TRUE"
-DCOMPILE_CUDA="${{ matrix.gpu }}"
-DCOMPILE_SERVER="FALSE"
-DCOMPILE_TESTS="TRUE"
-DUSE_FBGEMM="TRUE"
-DUSE_MPI="FALSE"
-DUSE_NCCL="FALSE"
-DUSE_SENTENCEPIECE="TRUE"
-DUSE_STATIC_LIBS="TRUE"'
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt
useVcpkgToolchainFile: true
- name: Run unit tests
working-directory: build/
run: ctest
# Not run in GPU builds because GitHub-hosted VMs do not have GPUs
if: matrix.gpu == false
- name: Print versions
working-directory: build/
run: |
.\marian.exe --version
.\marian-decoder.exe --version
.\marian-scorer.exe --version
dir *.exe
shell: cmd

View File

@ -1,28 +0,0 @@
{
"configurations": [
{
"name": "Release",
"generator": "Ninja",
"configurationType": "Release",
"inheritEnvironments": [ "msvc_x64" ],
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"ctestCommandArgs": "",
"variables": [
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CUDA:BOOL", "value": "FALSE" },
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
{ "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
{ "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
]
}
]
}

View File

@ -1,54 +0,0 @@
{
"configurations": [
{
"name": "Release",
"generator": "Ninja",
"configurationType": "Release",
"inheritEnvironments": [ "msvc_x64" ],
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"ctestCommandArgs": "",
"variables": [
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
{ "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
{ "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
{ "name": "USE_NCCL:BOOL", "value": "FALSE" },
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
]
},
{
"name": "Debug",
"generator": "Ninja",
"configurationType": "Debug",
"inheritEnvironments": [ "msvc_x64" ],
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"ctestCommandArgs": "",
"variables": [
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
{ "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
{ "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
{ "name": "USE_NCCL:BOOL", "value": "FALSE" },
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
]
}
]
}