mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-17 09:47:34 +03:00
b97645846a
* Add CUDA 11.x to Windows installation script * Update release.yml workflow
267 lines
10 KiB
YAML
267 lines
10 KiB
YAML
# This workflow builds packages with Marian executables and then attaches them
|
|
# to an existing GitHub release or drafts a new release if needed.
|
|
# There are two ways of utilizing the workflow:
|
|
#
|
|
# 1. Create a new tag locally and push it to the remote, for example:
|
|
#
|
|
# git tag 1.2.3
|
|
# git push origin 1.2.3
|
|
#
|
|
# The workflow, if built successfully, will open a new draft release (not seen
|
|
# for the public, available from under the /releases subpage on GitHub), use
|
|
# the last unreleased changes from CHANGELOG as the release description, and
|
|
# attach built packages as assets. Until the draft release is not published,
|
|
# the tag will be separated from the release, but they get merged into a single
|
|
# item with all assets. An admin user need to manually publish the draft
|
|
# release then. (Note: this behavior can be easily changed if needed).
|
|
#
|
|
# 2. Publish a new (non-draft) release with a tag via GitHub web interface. The
|
|
# workflow, if built successfully, will only attach built packages as
|
|
# additional assets. It is the responsibility of an admin user to add
|
|
# description to the release if it was not provided before..
|
|
#
|
|
name: Release
|
|
|
|
# The job is triggered by pushing a tag that follows the given pattern name
|
|
on:
|
|
push:
|
|
tags:
|
|
- "[0-9]+.[0-9]+.[0-9]+*"
|
|
|
|
env:
|
|
cuda_version: "11.2"
|
|
gcc_version: 8
|
|
|
|
jobs:
|
|
######################################################################
|
|
build-release-ubuntu:
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
# No builds with FBGEMM because they are not portable enough
|
|
# requiring same CPU architecture at compilation and runtime
|
|
- name: "Build Ubuntu CPU"
|
|
suffix: cpu
|
|
fbgemm: false
|
|
cuda: false
|
|
- name: "Build Ubuntu CPU+CUDA"
|
|
suffix: cuda11.2
|
|
fbgemm: false
|
|
cuda: true
|
|
|
|
runs-on: ubuntu-18.04
|
|
name: ${{ matrix.name }}
|
|
|
|
steps:
|
|
# Set env.github_tag_name and env.archive_name for subsequent steps
|
|
- name: Archive name
|
|
run: |
|
|
# Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/
|
|
TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-)
|
|
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
|
|
echo "github_tag_name=${TAG_NAME}" >> $GITHUB_ENV
|
|
echo "archive_name=${{ github.event.repository.name }}-${TAG_NAME}_linux-x64-static_${{ matrix.suffix }}" >> $GITHUB_ENV
|
|
shell: bash
|
|
|
|
- name: Checkout
|
|
uses: actions/checkout@v2
|
|
with:
|
|
submodules: recursive
|
|
|
|
# The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
|
|
# No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
|
|
# Boost is no longer pre-installed on GitHub-hosted runners
|
|
- name: Install dependencies
|
|
run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler libboost-system-dev gcc-${{ env.gcc_version }} g++-${{ env.gcc_version }}
|
|
|
|
# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
|
|
- name: Install MKL
|
|
run: |
|
|
wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
|
|
sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
|
|
sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
|
|
# The script simplifies installation of different versions of CUDA
|
|
- name: Install CUDA
|
|
run: ./scripts/ci/install_cuda_ubuntu.sh ${{ env.cuda_version }}
|
|
if: matrix.cuda == true
|
|
|
|
- name: Configure CMake
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
CC=/usr/bin/gcc-${{ env.gcc_version }} CXX=/usr/bin/g++-${{ env.gcc_version }} CUDAHOSTCXX=/usr/bin/g++-${{ env.gcc_version }} \
|
|
cmake .. \
|
|
-DBoost_ARCHITECTURE=-x64 \
|
|
-DCMAKE_BUILD_TYPE=Slim \
|
|
-DCOMPILE_CPU=on \
|
|
-DCOMPILE_CUDA=${{ matrix.cuda }} \
|
|
-DCOMPILE_KEPLER=${{ matrix.cuda }} \
|
|
-DCOMPILE_MAXWELL=${{ matrix.cuda }} \
|
|
-DCOMPILE_SERVER=on \
|
|
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ env.cuda_version }} \
|
|
-DUSE_FBGEMM=${{ matrix.fbgemm }} \
|
|
-DUSE_SENTENCEPIECE=on \
|
|
-DUSE_STATIC_LIBS=on \
|
|
- name: Compile
|
|
working-directory: build
|
|
run: make -j2
|
|
|
|
- name: Create archive
|
|
working-directory: build
|
|
run: tar zcvf ../${{ env.archive_name }}.tar.gz marian* ../README.md
|
|
|
|
# For testing only
|
|
#- name: Test archive
|
|
#run: tar zcvf ${{ env.archive_name }}.tar.gz README.md
|
|
|
|
- name: Upload archive
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: ${{ env.archive_name }}.tar.gz
|
|
path: ${{ env.archive_name }}.tar.gz
|
|
|
|
######################################################################
|
|
build-release-windows:
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
# No builds with FBGEMM because they are not portable enough
|
|
# requiring same CPU architecture at compilation and runtime
|
|
- name: "Build Windows CPU"
|
|
suffix: cpu
|
|
fbgemm: false
|
|
cuda: false
|
|
- name: "Build Windows CPU+CUDA"
|
|
suffix: cuda11.2
|
|
fbgemm: false
|
|
cuda: true
|
|
|
|
runs-on: windows-2019
|
|
name: ${{ matrix.name }}
|
|
|
|
steps:
|
|
# Set env.github_tag_name and env.archive_name for subsequent steps
|
|
- name: Archive name
|
|
run: |
|
|
# Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/
|
|
TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-)
|
|
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
|
|
echo "github_tag_name=${TAG_NAME}" >> $GITHUB_ENV
|
|
echo "archive_name=${{ github.event.repository.name }}-${TAG_NAME}_windows-x64_${{ matrix.suffix }}" >> $GITHUB_ENV
|
|
shell: bash
|
|
|
|
- name: Checkout
|
|
uses: actions/checkout@v2
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Download MKL
|
|
run: |
|
|
C:\msys64\usr\bin\wget.exe -nv https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -O mkl.zip
|
|
Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl
|
|
# Set the MKLROOT environment variable so that CMake can find MKL.
|
|
# GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
|
|
echo "MKLROOT=$env:GITHUB_WORKSPACE/mkl" | Out-File -FilePath $env:GITHUB_ENV -Append
|
|
shell: pwsh
|
|
|
|
- name: Install CUDA
|
|
run: |
|
|
.\scripts\ci\install_cuda_windows.ps1 '${{ env.cuda_version }}'
|
|
# Set path to CUDA for subsequent steps so that CMake can find it
|
|
echo "CUDA_PATH=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Append
|
|
echo "$env:CUDA_PATH/bin" | Out-File -FilePath $env:GITHUB_PATH -Append
|
|
shell: pwsh
|
|
if: matrix.cuda == true
|
|
|
|
- name: Prepare vcpkg
|
|
uses: lukka/run-vcpkg@v4
|
|
with:
|
|
vcpkgArguments: protobuf
|
|
vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
|
|
vcpkgDirectory: ${{ github.workspace }}/vcpkg/
|
|
vcpkgTriplet: x64-windows-static
|
|
|
|
# Build with a simplified CMake settings JSON file
|
|
- name: Run CMake
|
|
uses: lukka/run-cmake@v3
|
|
with:
|
|
buildDirectory: ${{ github.workspace }}/build/
|
|
cmakeAppendedArgs: '-G Ninja
|
|
-DCMAKE_BUILD_TYPE="Release"
|
|
-DOPENSSL_USE_STATIC_LIBS="TRUE"
|
|
-DOPENSSL_MSVC_STATIC_RT="TRUE"
|
|
-DCOMPILE_CPU="TRUE"
|
|
-DCOMPILE_CUDA="${{ matrix.cuda }}"
|
|
-DCOMPILE_SERVER="FALSE"
|
|
-DUSE_FBGEMM="${{ matrix.fbgemm }}"
|
|
-DUSE_MPI="FALSE"
|
|
-DUSE_NCCL="FALSE"
|
|
-DUSE_SENTENCEPIECE="TRUE"
|
|
-DUSE_STATIC_LIBS="TRUE"'
|
|
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
|
|
cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt
|
|
useVcpkgToolchainFile: true
|
|
|
|
- name: Create archive
|
|
run: |
|
|
cp ../README.md .
|
|
Compress-Archive -Path marian*.exe,README.md -DestinationPath ../${{ env.archive_name }}.zip
|
|
shell: pwsh
|
|
working-directory: build
|
|
|
|
# For testing only
|
|
#- name: Test archive
|
|
#run: |
|
|
#Compress-Archive -Path README.md -DestinationPath ${{ env.archive_name }}.zip
|
|
#shell: pwsh
|
|
|
|
- name: Upload archive
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: ${{ env.archive_name }}.zip
|
|
path: ${{ env.archive_name }}.zip
|
|
|
|
######################################################################
|
|
release:
|
|
needs: [build-release-ubuntu, build-release-windows]
|
|
|
|
runs-on: ubuntu-18.04
|
|
name: Release
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v2
|
|
with:
|
|
submodules: false # the repo is needed only for CHANGELOG, so submodules are not needed
|
|
|
|
# Extract entire markdown text from the [Unreleased] section to be used as the release description
|
|
- name: Extract changelog
|
|
run: |
|
|
cat CHANGELOG.md | sed -n "/^## \[Unreleased\]/,/^## \[[0-9]/p;/^## \[[0-9]/q" | sed '1d;$d' > RELEASE.md
|
|
cat RELEASE.md
|
|
# Downloads all artifacts to the current working directory
|
|
- name: Download archives
|
|
uses: actions/download-artifact@v2
|
|
|
|
# Artifacts are downloaded into directories with the same names as actual files.
|
|
# They are moved to a common directory ./assets/
|
|
- name: Prepare archives
|
|
run: |
|
|
mkdir -p assets
|
|
mv ${{ github.event.repository.name }}-*/* assets/
|
|
rmdir ${{ github.event.repository.name }}-*
|
|
ls -lh assets/
|
|
- name: Release
|
|
uses: softprops/action-gh-release@v1
|
|
if: startsWith(github.ref, 'refs/tags/')
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
with:
|
|
body_path: RELEASE.md
|
|
draft: true
|
|
files: |
|
|
assets/*.zip
|
|
assets/*.tar.gz
|
|
prerelease: false
|