marian/azure-pipelines.yml
Roman Grundkiewicz a05124176d Merged PR 18531: Install GCC in Azure pipelines
This fixes Azure pipelines after recent changes in Azure-hosted runners removing GCC 8 and older on some Ubuntu images. GCC is now installed explicitly via `apt-get`.
2021-04-09 18:44:11 +00:00

447 lines
15 KiB
YAML

# Azure pipelines for Marian NMT
#
# The pipeline need to be added manually to the repository, for example:
# 1. Go to Your repository > Pipelines, click "New pipeline"
# 2. Choose "Azure Repos Git" and a repository
# 3. Choose "Existing Azure Pipelines YAML file" and specify path to this file
# 4. "More actions" > "Save"
# The pipeline CI trigger is set on the branch master only and PR trigger on a
# (non-draft) pull request to any branch
trigger:
- master
pool:
name: Azure Pipelines
variables:
BOOST_ROOT_WINDOWS: "C:/hostedtoolcache/windows/Boost/1.72.0/x86_64"
CUDA_PATH_WINDOWS: "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA"
MKL_DIR: "$(Build.SourcesDirectory)/mkl"
MKL_URL: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip"
VCPKG_COMMIT: 6185aa76504a5025f36754324abf307cc776f3da
VCPKG_DIR: "$(Build.SourcesDirectory)/vcpkg"
VCPKG_PACKAGES: "protobuf"
# The Visual Studio installation directory can be found using:
# pushd "C:\Program Files (x86)\Microsoft Visual Studio\Installer\"
# for /f "delims=" %%x in ('.\vswhere.exe -latest -property InstallationPath') do set VSPATH=%%x
# popd
VS_PATH: "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise"
stages:
- stage: Build
jobs:
######################################################################
- job: Windows
strategy:
matrix:
# Windows CPU-only build
"CPU":
cuda: false
cuda_version: ""
# Windows CPU+GPU build
"CUDA 10.2":
cuda: true
cuda_version: 10.2
pool:
vmImage: windows-latest
steps:
- checkout: self
submodules: true
- pwsh: .\scripts\ci\install_cuda_windows.ps1 "$(cuda_version)"
displayName: Install CUDA
condition: eq(variables.cuda, true)
- pwsh: |
C:\msys64\usr\bin\wget.exe -nv $(MKL_URL) -O mkl.zip
Expand-Archive -Force mkl.zip $(MKL_DIR)
displayName: Download MKL
## Cache for vcpkg packages. It does not work yet properly due to linker errors after restoring it.
#- task: Cache@2
# displayName: Cache
# inputs:
# # Change the first value (v0) to another value to clear the cache
# key: 'v0 | "$(VCPKG_PACKAGES)" | vcpkg | "$(Agent.OS)"'
# path: $(VCPKG_DIR)
# Boost is no longer pre-installed on Azure/GitHub-hosted Windows runners
- pwsh: |
Write-Host "Downloading Boost to $(BOOST_ROOT_WINDOWS)"
$Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe"
C:\msys64\usr\bin\wget.exe -nv $Url -O "$(Pipeline.Workspace)/boost.exe"
Start-Process -Wait -FilePath "$(Pipeline.Workspace)/boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=$(BOOST_ROOT_WINDOWS)"
displayName: Download Boost
- pwsh: |
git clone https://github.com/Microsoft/vcpkg.git $(VCPKG_DIR)
cd $(VCPKG_DIR)
git checkout $(VCPKG_COMMIT)
pushd
.\bootstrap-vcpkg.bat -disableMetrics
popd
# Install packages
.\vcpkg.exe install --triplet x64-windows-static $(VCPKG_PACKAGES)
# Clean to make the cache smaller
Remove-Item $(VCPKG_DIR)\downloads -Force -Recurse
Remove-Item $(VCPKG_DIR)\buildtrees -Force -Recurse
displayName: Prepare vcpkg
- script: |
:: Load VS environment
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
:: Create build directory
mkdir build
cd build
:: Run CMake
cmake .. -G Ninja ^
-DCMAKE_BUILD_TYPE="Debug" ^
-DCMAKE_C_COMPILER="cl.exe" ^
-DCMAKE_CXX_COMPILER="cl.exe" ^
-DCMAKE_MAKE_PROGRAM="ninja.exe" ^
-DCMAKE_TOOLCHAIN_FILE="$(VCPKG_DIR)\scripts\buildsystems\vcpkg.cmake" ^
-DVCPKG_TARGET_TRIPLET="x64-windows-static" ^
^
-DBOOST_ROOT="$(BOOST_ROOT)" ^
-DBOOST_INCLUDEDIR="$(BOOST_ROOT)/include" ^
-DBOOST_LIBRARYDIR="$(BOOST_ROOT)/lib" ^
-DOPENSSL_USE_STATIC_LIBS="TRUE" ^
-DOPENSSL_MSVC_STATIC_RT="TRUE" ^
^
-DCOMPILE_CPU="TRUE" ^
-DCOMPILE_CUDA="$(cuda)" ^
-DCOMPILE_EXAMPLES="FALSE" ^
-DCOMPILE_SERVER="TRUE" ^
-DCOMPILE_TESTS="TRUE" ^
^
-DUSE_CUDNN="FALSE" ^
-DUSE_FBGEMM="TRUE" ^
-DUSE_MPI="FALSE" ^
-DUSE_NCCL="FALSE" ^
-DUSE_SENTENCEPIECE="TRUE" ^
-DUSE_STATIC_LIBS="TRUE"
displayName: Configure CMake
env:
# Set envvars so that CMake can find the installed packages
MKLROOT: $(MKL_DIR)
CUDA_PATH: $(CUDA_PATH_WINDOWS)/v$(cuda_version)
BOOST_ROOT: $(BOOST_ROOT_WINDOWS)
- script: |
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
ninja
displayName: Compile
workingDirectory: build
- script: |
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
ctest
displayName: Run unit tests
workingDirectory: build
condition: eq(variables.cuda, false)
# Note that versions from Marian executables will not be printed for CUDA builds
- script: |
.\marian.exe --version
.\marian-decoder.exe --version
.\marian-scorer.exe --version
.\marian-server.exe --version
.\spm_encode.exe --version
displayName: Print versions
workingDirectory: build
######################################################################
- job: BuildUbuntu
displayName: Ubuntu
timeoutInMinutes: 90
strategy:
matrix:
################################################################
# Ubuntu CPU-only build
"CPU-only":
image: ubuntu-18.04
boost: true
cpu: true
gpu: false
cuda: ""
gcc: 7
unit_tests: true
examples: false
static: true
# Ubuntu GPU-only build
"GPU-only":
image: ubuntu-18.04
boost: true
cpu: false
gpu: true
cuda: 10.2
gcc: 7
unit_tests: false
examples: false
static: false
################################################################
# Ubuntu 20.04 supports CUDA 11+
#
# CPU is disabled because FBGEMM + GCC 9+ do not compile on machines with
# avx512_vnni, see https://github.com/marian-nmt/marian-dev/issues/709
#
"20.04 CUDA 11.1 gcc-9":
image: ubuntu-20.04
boost: false # ubuntu-20.04 does not have Boost pre-installed yet
cpu: false # the used fbgemm does not compile with gcc 9+
gpu: true
cuda: 11.1
gcc: 9
unit_tests: false # disable unit tests to minimize compilation time
examples: false # disable examples to minimize compilation time
static: false
################################################################
# Ubuntu 18.04 supports CUDA 10.1+
"18.04 CUDA 10.2 gcc-8":
image: ubuntu-18.04
boost: true
cpu: true
gpu: true
cuda: 10.2
gcc: 8
unit_tests: true
examples: true
static: true
################################################################
# Ubuntu 16.04 supports CUDA 8+
"16.04 CUDA 9.2 gcc-7":
image: ubuntu-16.04
boost: true
cpu: true
gpu: true
cuda: 9.2
gcc: 7
unit_tests: true
examples: true
static: false
pool:
vmImage: $(image)
steps:
- checkout: self
submodules: true
# The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
# GCC 8 and lower are no longer pre-installed
- bash: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler gcc-$(gcc) g++-$(gcc)
displayName: Install packages
# Boost is no longer pre-installed on Azure/GitHub-hosted runners
- bash: sudo apt-get install -y libboost-system-dev
displayName: Install Boost
condition: eq(variables.boost, true)
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
- bash: |
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
displayName: Install MKL
condition: eq(variables.cpu, true)
# The script simplifies installation of different versions of CUDA
- bash: ./scripts/ci/install_cuda_ubuntu.sh $(cuda)
displayName: Install CUDA
condition: eq(variables.gpu, true)
- bash: |
mkdir -p build
cd build
CC=/usr/bin/gcc-$(gcc) CXX=/usr/bin/g++-$(gcc) CUDAHOSTCXX=/usr/bin/g++-$(gcc) \
cmake .. \
-DCOMPILE_CPU=$(cpu) \
-DCOMPILE_CUDA=$(gpu) \
-DCOMPILE_EXAMPLES=$(examples) \
-DCOMPILE_SERVER=$(boost) \
-DCOMPILE_TESTS=$(unit_tests) \
-DUSE_FBGEMM=$(cpu) \
-DUSE_SENTENCEPIECE=on \
-DUSE_STATIC_LIBS=$(static) \
-DBoost_ARCHITECTURE=-x64 \
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-$(cuda)
displayName: Configure CMake
# Clean build/src/ to safe disk space on Azure-hosted VMs and stay below the 10GB limit
- bash: |
make -j3
rm -rf src/
displayName: Compile
workingDirectory: build
# Unit tests are run only for CPU-only builds because Azure-hosted runners do not have GPUs
# TODO: add an option to CMake to compile unit tests only for CPU
- bash: make test
displayName: Run unit tests
workingDirectory: build
condition: and(eq(variables.unit_tests, true), eq(variables.gpu, false))
- bash: |
./marian --version
./marian-decoder --version
./marian-scorer --version
./spm_encode --version
displayName: Print versions
workingDirectory: build
######################################################################
- job: BuildUbuntuMinimal
displayName: Ubuntu CPU+GPU gcc-5 cmake 3.5
pool:
vmImage: ubuntu-16.04
steps:
- checkout: self
submodules: true
# The script simplifies installation of different versions of CUDA.
# Ubuntu 16.04 on Azure-hosted VMs have GCC 5.5 as gcc-5, which is not compatible with CUDA 9.
# Downgrading to GCC 5.4 (the default gcc on Ubuntu 16.04) would be more work...
- bash: ./scripts/ci/install_cuda_ubuntu.sh "10.0"
displayName: Install CUDA
# CMake 3.5.1 is the minimum version supported
- bash: |
wget -nv https://cmake.org/files/v3.5/cmake-3.5.1-Linux-x86_64.tar.gz
tar zxf cmake-3.5.1-Linux-x86_64.tar.gz
./cmake-3.5.1-Linux-x86_64/bin/cmake --version
displayName: Download CMake
# GCC 5 is the minimum version supported
- bash: |
/usr/bin/gcc-5 --version
mkdir -p build
cd build
CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 CUDAHOSTCXX=/usr/bin/g++-5 \
../cmake-3.5.1-Linux-x86_64/bin/cmake .. \
-DCOMPILE_CPU=on \
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-10.0
displayName: Configure CMake
- bash: make -j3
displayName: Compile
workingDirectory: build
- bash: |
./marian --version
./marian-decoder --version
./marian-scorer --version
displayName: Print versions
workingDirectory: build
######################################################################
- job: BuildMacOS
displayName: macOS CPU clang
pool:
vmImage: macos-latest
steps:
- checkout: self
submodules: true
- bash: brew install boost openblas openssl protobuf
displayName: Install packages
# Openblas location is exported explicitly because openblas is keg-only, which means it was not symlinked into /usr/local/.
# CMake cannot find BLAS on Azure runners if Marian is being compiled statically, hence USE_STATIC_LIBS=off
- bash: |
export LDFLAGS="-L/usr/local/opt/openblas/lib"
export CPPFLAGS="-I/usr/local/opt/openblas/include"
mkdir -p build
cd build
cmake .. \
-DCOMPILE_CPU=on \
-DCOMPILE_CUDA=off \
-DCOMPILE_EXAMPLES=on \
-DCOMPILE_SERVER=on \
-DCOMPILE_TESTS=on \
-DUSE_FBGEMM=on \
-DUSE_SENTENCEPIECE=on \
-DUSE_STATIC_LIBS=off
displayName: Configure CMake
- bash: make -j2
displayName: Compile
workingDirectory: build
- bash: make test
displayName: Run unit tests
workingDirectory: build
- bash: |
./marian --version
./marian-decoder --version
./marian-scorer --version
./spm_encode --version
displayName: Print versions
workingDirectory: build
######################################################################
- job: BuildInstall
displayName: Linux CPU library install
pool:
vmImage: ubuntu-18.04
steps:
- checkout: self
submodules: true
# The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
- bash: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler
displayName: Install packages
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
- bash: |
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
displayName: Install MKL
- bash: /usr/bin/gcc-7 --version
displayName: Print GCC version
- bash: |
mkdir -p install
mkdir -p build
cd build
CC=/usr/bin/gcc-7 CXX=/usr/bin/g++-7 \
cmake .. \
-DCMAKE_INSTALL_PREFIX=../install \
-DCMAKE_BUILD_TYPE=slim \
-DCOMPILE_LIBRARY_ONLY=on \
-DCOMPILE_CUDA=off \
-DGENERATE_MARIAN_INSTALL_TARGETS=on \
-DUSE_FBGEMM=on \
-DUSE_SENTENCEPIECE=on
displayName: Configure CMake
- bash: make -j3 install
displayName: Compile & install
workingDirectory: build
- bash: |
test -e lib/libmarian.a
test -e lib/libfbgemm.a
test -e lib/libsentencepiece.a
ls -lah *
displayName: Check targets
workingDirectory: install