mirror of
https://github.com/marian-nmt/marian.git
synced 2024-08-15 01:00:23 +03:00
macos-10.15 will become unsupported in December 2022. Changes: * Upgrade Azure DevOps to macos-12 * Pull https://github.com/marian-nmt/sentencepiece/pull/14 * Fix clang 13 errors as in https://github.com/marian-nmt/marian-dev/pull/939
659 lines
24 KiB
YAML
659 lines
24 KiB
YAML
# Azure pipelines for Marian NMT
|
|
#
|
|
# The pipeline need to be added manually to the repository, for example:
|
|
# 1. Go to Your repository > Pipelines, click "New pipeline"
|
|
# 2. Choose "Azure Repos Git" and a repository
|
|
# 3. Choose "Existing Azure Pipelines YAML file" and specify path to this file
|
|
# 4. "More actions" > "Save"
|
|
|
|
parameters:
|
|
# Allow skipping the entire 'Build' stage
|
|
- name: runBuilds
|
|
displayName: Run builds? Uncheck to run regression tests only.
|
|
type: boolean
|
|
default: true
|
|
|
|
# The pipeline CI trigger is set on the branch master only and PR trigger on a
|
|
# (non-draft) pull request to any branch
|
|
trigger:
|
|
- master
|
|
|
|
pool:
|
|
name: Azure Pipelines
|
|
|
|
variables:
|
|
- group: marian-prod-tests
|
|
- name: BOOST_ROOT_WINDOWS
|
|
value: "C:/hostedtoolcache/windows/Boost/1.72.0/x86_64"
|
|
- name: BOOST_URL
|
|
value: "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe"
|
|
- name: CUDA_PATH_WINDOWS
|
|
value: "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA"
|
|
- name: MKL_DIR
|
|
value: "$(Build.SourcesDirectory)/mkl"
|
|
- name: MKL_URL
|
|
value: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip"
|
|
- name: VCPKG_COMMIT
|
|
value: 2022.03.10
|
|
- name: VCPKG_DIR
|
|
value: "$(Build.SourcesDirectory)/vcpkg"
|
|
- name: VCPKG_PACKAGES
|
|
value: "protobuf"
|
|
# The Visual Studio installation directory can be found using:
|
|
# pushd "C:\Program Files (x86)\Microsoft Visual Studio\Installer\"
|
|
# for /f "delims=" %%x in ('.\vswhere.exe -latest -property InstallationPath') do set VSPATH=%%x
|
|
# popd
|
|
- name: VS_PATH
|
|
value: "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise"
|
|
|
|
stages:
|
|
- stage: Builds
|
|
jobs:
|
|
|
|
######################################################################
|
|
- job: BuildWindows
|
|
condition: eq(${{ parameters.runBuilds }}, true)
|
|
displayName: Windows
|
|
|
|
strategy:
|
|
matrix:
|
|
# Windows CPU-only build
|
|
"CPU":
|
|
cuda: false
|
|
cuda_version: ""
|
|
# Windows CPU+GPU build
|
|
"CUDA 10.2":
|
|
cuda: true
|
|
cuda_version: 10.2
|
|
|
|
pool:
|
|
vmImage: windows-2019
|
|
|
|
steps:
|
|
- checkout: self
|
|
submodules: true
|
|
|
|
- pwsh: .\scripts\ci\install_cuda_windows.ps1 "$(cuda_version)"
|
|
displayName: Install CUDA
|
|
condition: eq(variables.cuda, true)
|
|
|
|
- pwsh: |
|
|
C:\msys64\usr\bin\wget.exe -nv $(MKL_URL) -O mkl.zip
|
|
Expand-Archive -Force mkl.zip $(MKL_DIR)
|
|
displayName: Download MKL
|
|
|
|
## Cache for Boost
|
|
#- task: Cache@2
|
|
# displayName: Cache Boost
|
|
# inputs:
|
|
# # Change the first value (v0) to another value to clear the cache
|
|
# key: '"v0" | "boost" | "$(BOOST_URL)" | "$(BOOST_ROOT_WINDOWS)" | "$(Agent.OS)"'
|
|
# path: $(BOOST_ROOT_WINDOWS)
|
|
# cacheHitVar: CACHE_BOOST_RESTORED
|
|
|
|
# Boost is no longer pre-installed on Azure/GitHub-hosted Windows runners
|
|
- pwsh: |
|
|
Write-Host "Downloading Boost to $(BOOST_ROOT_WINDOWS)"
|
|
C:\msys64\usr\bin\wget.exe -nv "$(BOOST_URL)" -O "$(Pipeline.Workspace)/boost.exe"
|
|
Start-Process -Wait -FilePath "$(Pipeline.Workspace)/boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=$(BOOST_ROOT_WINDOWS)"
|
|
displayName: Download Boost
|
|
condition: ne(variables.CACHE_BOOST_RESTORED, 'true')
|
|
|
|
## Cache for vcpkg packages. It does not work yet properly due to linker errors after restoring it.
|
|
#- task: Cache@2
|
|
# displayName: Cache vcpkg
|
|
# inputs:
|
|
# # Change the first value (v0) to another value to clear the cache
|
|
# key: '"v0" | "vcpkg" | "$(VCPKG_COMMIT)" | "$(VCPKG_PACKAGES)" | "$(Agent.OS)"'
|
|
# path: $(VCPKG_DIR)
|
|
# cacheHitVar: CACHE_VCPKG_RESTORED
|
|
|
|
- pwsh: |
|
|
git clone https://github.com/Microsoft/vcpkg.git $(VCPKG_DIR)
|
|
cd $(VCPKG_DIR)
|
|
git checkout $(VCPKG_COMMIT)
|
|
pushd
|
|
.\bootstrap-vcpkg.bat -disableMetrics
|
|
popd
|
|
# Install packages
|
|
.\vcpkg.exe install --triplet x64-windows-static $(VCPKG_PACKAGES)
|
|
# Clean to make the cache smaller
|
|
Remove-Item $(VCPKG_DIR)\downloads -Force -Recurse -ErrorAction SilentlyContinue
|
|
Remove-Item $(VCPKG_DIR)\buildtrees -Force -Recurse -ErrorAction SilentlyContinue
|
|
displayName: Prepare vcpkg
|
|
condition: ne(variables.CACHE_VCPKG_RESTORED, 'true')
|
|
|
|
- script: |
|
|
:: Load VS environment
|
|
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
|
|
:: Create build directory
|
|
mkdir build
|
|
cd build
|
|
:: Run CMake
|
|
cmake .. -G Ninja ^
|
|
-DCMAKE_BUILD_TYPE="Debug" ^
|
|
-DCMAKE_C_COMPILER="cl.exe" ^
|
|
-DCMAKE_CXX_COMPILER="cl.exe" ^
|
|
-DCMAKE_MAKE_PROGRAM="ninja.exe" ^
|
|
-DCMAKE_TOOLCHAIN_FILE="$(VCPKG_DIR)\scripts\buildsystems\vcpkg.cmake" ^
|
|
-DVCPKG_TARGET_TRIPLET="x64-windows-static" ^
|
|
^
|
|
-DBOOST_ROOT="$(BOOST_ROOT)" ^
|
|
-DBOOST_INCLUDEDIR="$(BOOST_ROOT)/include" ^
|
|
-DBOOST_LIBRARYDIR="$(BOOST_ROOT)/lib" ^
|
|
-DOPENSSL_USE_STATIC_LIBS="TRUE" ^
|
|
-DOPENSSL_MSVC_STATIC_RT="TRUE" ^
|
|
^
|
|
-DCOMPILE_CPU="TRUE" ^
|
|
-DCOMPILE_CUDA="$(cuda)" ^
|
|
-DCOMPILE_EXAMPLES="FALSE" ^
|
|
-DCOMPILE_SERVER="TRUE" ^
|
|
-DCOMPILE_TESTS="TRUE" ^
|
|
^
|
|
-DUSE_CUDNN="FALSE" ^
|
|
-DUSE_FBGEMM="TRUE" ^
|
|
-DUSE_MPI="FALSE" ^
|
|
-DUSE_NCCL="FALSE" ^
|
|
-DUSE_SENTENCEPIECE="TRUE" ^
|
|
-DUSE_STATIC_LIBS="TRUE"
|
|
displayName: Configure CMake
|
|
env:
|
|
# Set envvars so that CMake can find the installed packages
|
|
MKLROOT: $(MKL_DIR)
|
|
CUDA_PATH: $(CUDA_PATH_WINDOWS)/v$(cuda_version)
|
|
BOOST_ROOT: $(BOOST_ROOT_WINDOWS)
|
|
|
|
- script: |
|
|
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
|
|
ninja
|
|
displayName: Compile
|
|
workingDirectory: build
|
|
|
|
- script: |
|
|
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
|
|
ctest
|
|
displayName: Run unit tests
|
|
workingDirectory: build
|
|
condition: eq(variables.cuda, false)
|
|
|
|
# Note that versions from Marian executables will not be printed for CUDA builds
|
|
- script: |
|
|
.\marian.exe --version
|
|
.\marian-decoder.exe --version
|
|
.\marian-scorer.exe --version
|
|
.\marian-server.exe --version
|
|
.\spm_encode.exe --version
|
|
displayName: Print versions
|
|
workingDirectory: build
|
|
|
|
######################################################################
|
|
- job: BuildUbuntu
|
|
condition: eq(${{ parameters.runBuilds }}, true)
|
|
displayName: Ubuntu
|
|
timeoutInMinutes: 120
|
|
|
|
# Minimal tested configurations for marian-dev v1.11 and C++17:
|
|
# * Ubuntu 16.04, GCC 7.5, CMake 3.10.2, CUDA 9.2 (probably GCC 6 would work too)
|
|
# * Ubuntu 18.04, GCC 7.5, CMake 3.12.2, CUDA 10.0
|
|
strategy:
|
|
matrix:
|
|
################################################################
|
|
# Ubuntu CPU-only build
|
|
"CPU-only":
|
|
image: ubuntu-20.04
|
|
boost: true
|
|
cpu: true
|
|
gpu: false
|
|
cuda: ""
|
|
gcc: 9
|
|
unit_tests: true
|
|
examples: false
|
|
static: true
|
|
# Ubuntu GPU-only build
|
|
"GPU-only":
|
|
image: ubuntu-20.04
|
|
boost: true
|
|
cpu: false
|
|
gpu: true
|
|
cuda: 11.1
|
|
gcc: 9
|
|
unit_tests: false
|
|
examples: false
|
|
static: false
|
|
################################################################
|
|
# Ubuntu 22.04 supports CUDA 11+
|
|
#
|
|
# CPU is disabled because FBGEMM + GCC 9+ do not compile on machines with
|
|
# avx512_vnni, see https://github.com/marian-nmt/marian-dev/issues/709
|
|
"22.04 CUDA 11.7 gcc-11":
|
|
image: ubuntu-22.04
|
|
boost: false
|
|
cpu: false
|
|
gpu: true
|
|
cuda: 11.7
|
|
gcc: 11
|
|
unit_tests: false # disable unit tests to minimize compilation time
|
|
examples: false # disable examples to minimize compilation time
|
|
static: false
|
|
################################################################
|
|
# Ubuntu 20.04 supports CUDA 11+
|
|
"20.04 CUDA 11.1 gcc-9":
|
|
image: ubuntu-20.04
|
|
boost: true
|
|
cpu: true
|
|
gpu: true
|
|
cuda: 11.1
|
|
gcc: 9
|
|
unit_tests: true
|
|
examples: true
|
|
static: true
|
|
################################################################
|
|
# Ubuntu 16.04 is no longer available on Azure-hosted machines
|
|
|
|
pool:
|
|
vmImage: $(image)
|
|
|
|
steps:
|
|
- checkout: self
|
|
submodules: true
|
|
|
|
# The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev
|
|
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
|
|
# Note that installation of libunwind-dev is a bug fix for ubuntu-22.04 images on Azure/GitHub-hosted machines
|
|
# and is normally not required
|
|
- bash: sudo apt-get install -y libunwind-dev libgoogle-perftools-dev libprotobuf-dev protobuf-compiler
|
|
displayName: Install packages
|
|
|
|
# Boost is no longer pre-installed on Azure/GitHub-hosted runners
|
|
- bash: sudo apt-get install -y libboost-system-dev
|
|
displayName: Install Boost
|
|
condition: eq(variables.boost, true)
|
|
|
|
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
|
|
- bash: |
|
|
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
|
|
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
|
|
displayName: Install MKL
|
|
condition: eq(variables.cpu, true)
|
|
|
|
# The script simplifies installation of different versions of CUDA
|
|
- bash: ./scripts/ci/install_cuda_ubuntu.sh $(cuda)
|
|
displayName: Install CUDA
|
|
condition: eq(variables.gpu, true)
|
|
|
|
- bash: |
|
|
mkdir -p build
|
|
cd build
|
|
CC=/usr/bin/gcc-$(gcc) CXX=/usr/bin/g++-$(gcc) CUDAHOSTCXX=/usr/bin/g++-$(gcc) \
|
|
cmake .. \
|
|
-DCOMPILE_CPU=$(cpu) \
|
|
-DCOMPILE_CUDA=$(gpu) \
|
|
-DCOMPILE_EXAMPLES=$(examples) \
|
|
-DCOMPILE_SERVER=$(boost) \
|
|
-DCOMPILE_TESTS=$(unit_tests) \
|
|
-DUSE_FBGEMM=$(cpu) \
|
|
-DUSE_SENTENCEPIECE=on \
|
|
-DUSE_STATIC_LIBS=$(static) \
|
|
-DBoost_ARCHITECTURE=-x64 \
|
|
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-$(cuda)
|
|
displayName: Configure CMake
|
|
|
|
# Clean build/src/ to safe disk space on Azure-hosted VMs and stay below the 10GB limit
|
|
- bash: |
|
|
make -j3
|
|
rm -rf src/
|
|
displayName: Compile
|
|
workingDirectory: build
|
|
|
|
# Unit tests are run only for CPU-only builds because Azure-hosted runners do not have GPUs
|
|
# TODO: add an option to CMake to compile unit tests only for CPU
|
|
- bash: make test
|
|
displayName: Run unit tests
|
|
workingDirectory: build
|
|
condition: and(eq(variables.unit_tests, true), eq(variables.gpu, false))
|
|
|
|
- bash: |
|
|
./marian --version
|
|
./marian-decoder --version
|
|
./marian-scorer --version
|
|
./spm_encode --version
|
|
displayName: Print versions
|
|
workingDirectory: build
|
|
|
|
######################################################################
|
|
- job: BuildMacOS
|
|
condition: eq(${{ parameters.runBuilds }}, true)
|
|
displayName: macOS CPU clang
|
|
|
|
pool:
|
|
vmImage: macos-12
|
|
|
|
steps:
|
|
- checkout: self
|
|
submodules: true
|
|
|
|
- bash: brew install boost openblas openssl protobuf
|
|
displayName: Install packages
|
|
|
|
# Openblas location is exported explicitly because openblas is keg-only, which means it was not symlinked into /usr/local/.
|
|
# CMake cannot find BLAS on Azure runners if Marian is being compiled statically, hence USE_STATIC_LIBS=off
|
|
- bash: |
|
|
export LDFLAGS="-L/usr/local/opt/openblas/lib"
|
|
export CPPFLAGS="-I/usr/local/opt/openblas/include"
|
|
mkdir -p build
|
|
cd build
|
|
cmake .. \
|
|
-DCOMPILE_CPU=on \
|
|
-DCOMPILE_CUDA=off \
|
|
-DCOMPILE_EXAMPLES=on \
|
|
-DCOMPILE_SERVER=on \
|
|
-DCOMPILE_TESTS=on \
|
|
-DUSE_FBGEMM=on \
|
|
-DUSE_SENTENCEPIECE=on \
|
|
-DUSE_STATIC_LIBS=off
|
|
displayName: Configure CMake
|
|
|
|
- bash: make -j2
|
|
displayName: Compile
|
|
workingDirectory: build
|
|
|
|
- bash: make test
|
|
displayName: Run unit tests
|
|
workingDirectory: build
|
|
|
|
- bash: |
|
|
./marian --version
|
|
./marian-decoder --version
|
|
./marian-scorer --version
|
|
./spm_encode --version
|
|
displayName: Print versions
|
|
workingDirectory: build
|
|
|
|
######################################################################
|
|
- job: BuildInstall
|
|
condition: eq(${{ parameters.runBuilds }}, true)
|
|
displayName: Linux CPU library install
|
|
|
|
pool:
|
|
vmImage: ubuntu-20.04
|
|
|
|
steps:
|
|
- checkout: self
|
|
submodules: true
|
|
|
|
# The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev
|
|
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
|
|
- bash: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler
|
|
displayName: Install packages
|
|
|
|
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
|
|
- bash: |
|
|
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
|
|
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
|
|
displayName: Install MKL
|
|
|
|
- bash: /usr/bin/gcc-9 --version
|
|
displayName: Print GCC version
|
|
|
|
- bash: |
|
|
mkdir -p install
|
|
mkdir -p build
|
|
cd build
|
|
CC=/usr/bin/gcc-9 CXX=/usr/bin/g++-9 \
|
|
cmake .. \
|
|
-DCMAKE_INSTALL_PREFIX=../install \
|
|
-DCMAKE_BUILD_TYPE=slim \
|
|
-DCOMPILE_LIBRARY_ONLY=on \
|
|
-DCOMPILE_CUDA=off \
|
|
-DGENERATE_MARIAN_INSTALL_TARGETS=on \
|
|
-DUSE_FBGEMM=on \
|
|
-DUSE_SENTENCEPIECE=on
|
|
displayName: Configure CMake
|
|
|
|
- bash: make -j3 install
|
|
displayName: Compile & install
|
|
workingDirectory: build
|
|
|
|
- bash: |
|
|
test -e lib/libmarian.a
|
|
test -e lib/libfbgemm.a
|
|
test -e lib/libsentencepiece.a
|
|
ls -lah *
|
|
displayName: Check targets
|
|
workingDirectory: install
|
|
|
|
|
|
# Marian is built in the same job where the regression tests are run to make sure that executables
|
|
# are compiled and run on a machine with the same CPU architecture, which is required for
|
|
# compilations with FBGEMM.
|
|
- stage: Tests
|
|
jobs:
|
|
|
|
######################################################################
|
|
- job: TestWindows
|
|
displayName: Windows CPU+FBGEMM
|
|
|
|
pool:
|
|
vmImage: windows-2019
|
|
|
|
steps:
|
|
# Due to multiple checkouts this will be commonly cloned into D:\a\1\s\marian-dev
|
|
- checkout: self
|
|
submodules: true
|
|
|
|
- pwsh: |
|
|
C:\msys64\usr\bin\wget.exe -nv $(MKL_URL) -O mkl.zip
|
|
Expand-Archive -Force mkl.zip $(MKL_DIR)
|
|
displayName: Download MKL
|
|
|
|
# Cache for vcpkg packages
|
|
- task: Cache@2
|
|
displayName: Cache vcpkg
|
|
inputs:
|
|
# Change the first value (v0) to another value to clear the cache
|
|
key: '"v0" | "vcpkg" | "$(VCPKG_COMMIT)" | "$(VCPKG_PACKAGES)" | "$(Agent.OS)"'
|
|
path: $(VCPKG_DIR)
|
|
cacheHitVar: CACHE_VCPKG_RESTORED
|
|
|
|
- pwsh: |
|
|
git clone https://github.com/Microsoft/vcpkg.git $(VCPKG_DIR)
|
|
cd $(VCPKG_DIR)
|
|
git checkout $(VCPKG_COMMIT)
|
|
pushd
|
|
.\bootstrap-vcpkg.bat -disableMetrics
|
|
popd
|
|
# Install packages
|
|
.\vcpkg.exe install --triplet x64-windows-static $(VCPKG_PACKAGES)
|
|
# Clean to make the cache smaller
|
|
Remove-Item $(VCPKG_DIR)\downloads -Force -Recurse -ErrorAction SilentlyContinue
|
|
Remove-Item $(VCPKG_DIR)\buildtrees -Force -Recurse -ErrorAction SilentlyContinue
|
|
displayName: Prepare vcpkg
|
|
condition: ne(variables.CACHE_VCPKG_RESTORED, 'true')
|
|
|
|
- script: |
|
|
:: Load VS environment
|
|
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
|
|
:: Create build directory
|
|
mkdir build
|
|
cd build
|
|
:: Run CMake
|
|
cmake .. -G Ninja ^
|
|
-DCMAKE_BUILD_TYPE="Slim" ^
|
|
-DCMAKE_C_COMPILER="cl.exe" ^
|
|
-DCMAKE_CXX_COMPILER="cl.exe" ^
|
|
-DCMAKE_MAKE_PROGRAM="ninja.exe" ^
|
|
-DCMAKE_TOOLCHAIN_FILE="$(VCPKG_DIR)\scripts\buildsystems\vcpkg.cmake" ^
|
|
-DVCPKG_TARGET_TRIPLET="x64-windows-static" ^
|
|
-DDETERMINISTIC="TRUE" ^
|
|
^
|
|
-DCOMPILE_CPU="TRUE" ^
|
|
-DCOMPILE_CUDA="FALSE" ^
|
|
^
|
|
-DUSE_FBGEMM="TRUE" ^
|
|
-DUSE_SENTENCEPIECE="TRUE" ^
|
|
-DUSE_STATIC_LIBS="TRUE"
|
|
displayName: Configure CMake
|
|
env:
|
|
# Set envvars so that CMake can find the installed packages
|
|
MKLROOT: $(MKL_DIR)
|
|
workingDirectory: marian-dev
|
|
|
|
- script: |
|
|
pwd
|
|
call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64
|
|
ninja
|
|
displayName: Compile
|
|
workingDirectory: marian-dev/build
|
|
|
|
## Publish an artifact with Marian executables. Disabled because it is not portable due to
|
|
## fbgemm and not needed at the moment
|
|
#- pwsh: Compress-Archive -Path marian*.exe spm_*.exe -DestinationPath $(Build.SourcesDirectory)/marian-dev-ci_windows-x64_cpu.zip
|
|
#displayName: Prepare binaries
|
|
#workingDirectory: marian-dev/build
|
|
#- publish: marian-dev-ci_windows-x64_cpu.zip
|
|
#artifact: marian-dev-ci_windows-x64_cpu
|
|
#displayName: Publish binaries
|
|
|
|
# Running regression tests
|
|
# Due to multiple checkouts this will be commonly cloned into D:\a\1\s\marian-prod-tests
|
|
- checkout: git://Marian/marian-prod-tests
|
|
|
|
# Collect details about the CPU architecture, etc.
|
|
# Because the outputs goes to *.log files, they will be also included in the artifact with test outputs.
|
|
- script: bash -c "cat /proc/cpuinfo | tee cpuinfo.log"
|
|
displayName: Machine statistics
|
|
workingDirectory: marian-prod-tests
|
|
|
|
# The current SAS token will expire on 8/30/2023 and a new one will need to be set in Marian > Pipelines > Library
|
|
- bash: |
|
|
cd models
|
|
bash download-models.sh
|
|
ls
|
|
displayName: Prepare tests
|
|
env:
|
|
AWS_SECRET_SAS_TOKEN: $(blob-sas-token)
|
|
workingDirectory: marian-prod-tests
|
|
|
|
# Avoid using $(Build.SourcesDirectory) in bash tasks because on Windows pools it uses '\'
|
|
# instead of '/', which often breaks the job
|
|
- bash: MARIAN=../marian-dev/build TIMEOUT=10m bash ./run_mrt.sh '#cpu' '#basics' '#devops'
|
|
continueOnError: true
|
|
displayName: Run tests
|
|
workingDirectory: marian-prod-tests
|
|
|
|
- bash: |
|
|
# cut -c3- removes './' from paths making 7z to retain the directory structure
|
|
find . -type f \( -name "*.log" -o -name "*.out" -o -name "*.diff" \) -print | cut -c3- > listing.txt
|
|
echo "Creating an artifact with the following files:"
|
|
cat listing.txt
|
|
7z a -tzip ../regression-tests-ci_windows-x64_cpu.zip @listing.txt
|
|
displayName: Collect outputs
|
|
workingDirectory: marian-prod-tests
|
|
|
|
- publish: regression-tests-ci_windows-x64_cpu.zip
|
|
artifact: regression-tests-ci_windows-x64_cpu
|
|
displayName: Publish outputs
|
|
|
|
######################################################################
|
|
- job: TestLinux
|
|
displayName: Linux CPU+FBGEMM
|
|
|
|
pool:
|
|
vmImage: ubuntu-latest
|
|
|
|
steps:
|
|
# Due to multiple checkouts this will be commonly cloned into D:\a\1\s\marian-dev
|
|
- checkout: self
|
|
submodules: true
|
|
|
|
# The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev
|
|
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
|
|
- bash: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler gcc-9 g++-9
|
|
displayName: Install packages
|
|
|
|
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
|
|
- bash: |
|
|
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
|
|
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
|
|
displayName: Install MKL
|
|
|
|
# Note that COMPILE_CPU=on and USE_SENTENCEPUECE=on are set explicitly to make them detectable
|
|
# by the regression tests framework (not sure if it is still required)
|
|
- bash: |
|
|
mkdir -p install
|
|
mkdir -p build
|
|
cd build
|
|
CC=/usr/bin/gcc-9 CXX=/usr/bin/g++-9 \
|
|
cmake .. \
|
|
-DCMAKE_BUILD_TYPE=slim \
|
|
-DCOMPILE_CPU=on \
|
|
-DCOMPILE_CUDA=off \
|
|
-DDETERMINISTIC=on \
|
|
-DUSE_FBGEMM=on \
|
|
-DUSE_SENTENCEPIECE=on \
|
|
-DUSE_STATIC_LIBS=on
|
|
displayName: Configure CMake
|
|
workingDirectory: marian-dev
|
|
|
|
- bash: make -j3
|
|
displayName: Compile
|
|
workingDirectory: marian-dev/build
|
|
|
|
## Publish an artifact with Marian executables. Disabled because it is not portable due to
|
|
## fbgemm and not needed at the moment
|
|
#- bash: zip $(Build.SourcesDirectory)/marian-dev-ci_linux-x64-static_cpu.zip marian* spm_*
|
|
#displayName: Prepare binaries
|
|
#workingDirectory: marian-dev/build
|
|
#- publish: marian-dev-ci_linux-x64-static_cpu.zip
|
|
#artifact: marian-dev-ci_linux-x64-static_cpu
|
|
#displayName: Publish binaries
|
|
|
|
# Running regression tests
|
|
# Due to multiple checkouts this will be commonly cloned into D:\a\1\s\marian-prod-tests
|
|
- checkout: git://Marian/marian-prod-tests
|
|
|
|
# Collect details about the CPU architecture, etc.
|
|
# Because the outputs goes to *.log files, they will be also included in the artifact with test outputs.
|
|
- bash: |
|
|
echo ">>> lscpu"
|
|
lscpu | tee lscpu.log
|
|
echo ">>> cpuinfo"
|
|
cat /proc/cpuinfo | tee cpuinfo.log
|
|
/usr/bin/gcc-9 --version | tee gcc.log
|
|
displayName: Machine statistics
|
|
workingDirectory: marian-prod-tests
|
|
|
|
# The current SAS token will expire on 8/30/2023 and a new one will need to be set in Marian > Pipelines > Library
|
|
- bash: |
|
|
cd models
|
|
bash download-models.sh
|
|
ls
|
|
displayName: Prepare tests
|
|
env:
|
|
AWS_SECRET_SAS_TOKEN: $(blob-sas-token)
|
|
workingDirectory: marian-prod-tests
|
|
|
|
- bash: MARIAN=../marian-dev/build bash ./run_mrt.sh '#cpu' '#basics' '#devops'
|
|
continueOnError: true
|
|
displayName: Run tests
|
|
workingDirectory: marian-prod-tests
|
|
|
|
- bash: |
|
|
# cut -c3- removes './' from paths making 7z to retain the directory structure
|
|
find . -type f \( -name "*.log" -o -name "*.out" -o -name "*.diff" \) -print | cut -c3- > listing.txt
|
|
echo "Creating an artifact with the following files:"
|
|
cat listing.txt
|
|
7z a -tzip ../regression-tests-ci_linux-x64-static_cpu.zip @listing.txt
|
|
displayName: Collect outputs
|
|
workingDirectory: marian-prod-tests
|
|
|
|
- publish: regression-tests-ci_linux-x64-static_cpu.zip
|
|
artifact: regression-tests-ci_linux-x64-static_cpu
|
|
displayName: Publish outputs
|