mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 09:17:43 +03:00
739dc38660
While flailing about randomly trying to reset the Windows cache yesterday, I noticed a couple issues with the current script: - The `fork_point` calculation is just plain broken. Somehow our `set -euo pipefail` does not fail on subshell errors, but the existing command is just never going to work: it looks like `git` does not resolve refs on the `merge-base` command. It also looks like the `--fork-point` option is not what we want. I don't know how this happened. - `sort` on my machine and on CI do not seem to behave the same with respect to upper/lower case ordering. To make the script independent of the specific sort order on the machine (probably controlled by the locale), we now sort both the actual and the expected list. Finally, based on the failure to recognize a release commit once merged into master, I realized that of course computing the diff between a commit and itself will yield an empty diff. The `git_sha` step will now identify the "master" and "fork point" commits as the parent for a master build. CHANGELOG_BEGIN CHANGELOG_END
816 lines
35 KiB
YAML
816 lines
35 KiB
YAML
# Copyright (c) 2020 The DAML Authors. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Azure Pipelines file, see https://aka.ms/yaml
|
|
|
|
# Enable builds on all branches
|
|
trigger:
|
|
# Build every commit as our release process relies on
|
|
# the release process being built alone.
|
|
batch: false
|
|
branches:
|
|
include:
|
|
- master
|
|
|
|
# Enable PR triggers that target the master branch
|
|
pr:
|
|
autoCancel: true # cancel previous builds on push
|
|
branches:
|
|
include:
|
|
- master
|
|
|
|
jobs:
|
|
- job: git_sha
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [ "$(Build.Reason)" == "PullRequest" ]; then
|
|
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD^2)"
|
|
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
|
|
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
|
|
else
|
|
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD)"
|
|
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
|
|
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git rev-parse HEAD^1)"
|
|
fi
|
|
name: out
|
|
|
|
- job: check_standard_change_label
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
has_changed_infra_folder () {
|
|
git diff origin/master --name-only | grep -q '^infra/'
|
|
}
|
|
|
|
fail_if_missing_std_change_label () {
|
|
curl https://api.github.com/repos/digital-asset/daml/pulls/$PR -s | jq -r '.labels[].name' | grep -q '^Standard-Change$'
|
|
}
|
|
|
|
if has_changed_infra_folder; then
|
|
fail_if_missing_std_change_label
|
|
fi
|
|
env:
|
|
PR: $(System.PullRequest.PullRequestNumber)
|
|
|
|
- job: check_changelog_entry
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- checkout: self
|
|
- bash: ci/check-changelog.sh
|
|
|
|
- job: Linux
|
|
dependsOn:
|
|
- check_for_release
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
bazel build //release:release
|
|
./bazel-bin/release/release --release-dir "$(mktemp -d)"
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: macOS
|
|
dependsOn:
|
|
- check_for_release
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
vmImage: 'macOS-10.14'
|
|
variables:
|
|
nix-cache-key: $(Build.StagingDirectory)/macos-nix-key
|
|
nix-cache-path: /tmp/nix-cache/
|
|
bazel-repo-cache-key: $(Build.StagingDirectory)/bazel-repo-cache-key
|
|
bazel-repo-cache-path: $(Agent.BuildDirectory)/.bazel-cache/repo
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- bash: echo $(git log -n1 --pretty=format:%H dev-env nix azure-pipelines.yml) >> $(nix-cache-key)
|
|
displayName: nix cache key
|
|
- task: CacheBeta@0
|
|
inputs:
|
|
key: $(nix-cache-key) | v2
|
|
path: $(nix-cache-path)
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [[ -e $(nix-cache-path) ]]; then
|
|
DIR=$(pwd)
|
|
sudo mkdir /nix && sudo chown $USER /nix
|
|
cd /nix
|
|
tar xzf $(nix-cache-path)/nix.tar.gz
|
|
cd $DIR
|
|
curl -sfL https://nixos.org/releases/nix/nix-2.3.2/install | bash
|
|
fi
|
|
displayName: restore cache
|
|
- bash: echo $(git log -n1 --pretty=format:%H azure-pipelines.yml $(find . -name \*.bazel -or -name \*.bzl -or -name WORKSPACE -or -name BUILD)) >> $(bazel-repo-cache-key)
|
|
displayName: bazel repo cache key
|
|
- task: CacheBeta@0
|
|
inputs:
|
|
key: $(bazel-repo-cache-key)
|
|
path: $(bazel-repo-cache-path)
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [[ ! -e $(nix-cache-path) ]]; then
|
|
mkdir -p $(nix-cache-path)
|
|
cd /nix
|
|
GZIP=-9 tar czf $(nix-cache-path)/nix.tar.gz store var
|
|
fi
|
|
displayName: create nix cache
|
|
- bash: mkdir -p $(bazel-repo-cache-path)
|
|
displayName: ensure bazel repo cache exists
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: Windows
|
|
dependsOn:
|
|
- check_for_release
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'windows-pool'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-windows.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: perf
|
|
dependsOn:
|
|
- check_for_release
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 60
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- bash: ci/dev-env-install.sh
|
|
displayName: 'Build/Install the Developer Environment'
|
|
- bash: ci/configure-bazel.sh
|
|
displayName: 'Configure Bazel'
|
|
env:
|
|
IS_FORK: $(System.PullRequest.IsFork)
|
|
# to upload to the bazel cache
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
DAML_SDK_RELEASE_VERSION=$(release_tag) bazel run -- //ledger/sandbox-perf -foe true -i1 -f1 -wi 1 -bm avgt -rf csv -rff "$(Build.StagingDirectory)/sandbox-perf.csv"
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Perf test logs'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: check_for_release
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
pool:
|
|
name: "linux-pool"
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
./release.sh check
|
|
|
|
is_release_commit() {
|
|
changed="$(git diff-tree --no-commit-id --name-only -r $(branch_sha) $(fork_sha) | sort)"
|
|
stable=$(printf "LATEST\ndocs/source/support/release-notes.rst" | sort)
|
|
snapshot="LATEST"
|
|
[ "$snapshot" = "$changed" ] || [ "$stable" = "$changed" ]
|
|
}
|
|
|
|
if is_release_commit; then
|
|
echo "##vso[task.setvariable variable=is_release;isOutput=true]true"
|
|
echo "##vso[task.setvariable variable=trigger_sha;isOutput=true]$(branch_sha)"
|
|
echo "##vso[task.setvariable variable=release_sha;isOutput=true]$(cat LATEST | awk '{print $1}')"
|
|
echo "##vso[task.setvariable variable=release_tag;isOutput=true]$(cat LATEST | awk '{print $2}')"
|
|
else
|
|
echo "##vso[task.setvariable variable=is_release;isOutput=true]false"
|
|
fi
|
|
name: out
|
|
|
|
- job: windows_installer
|
|
dependsOn: [ "check_for_release", "Windows" ]
|
|
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
pool:
|
|
name: 'windows-pool'
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
- bash: ci/configure-bazel.sh
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
export DAML_SDK_RELEASE_VERSION=$(release_tag)
|
|
|
|
bazel build //release/windows-installer:windows-installer
|
|
INSTALLER=daml-sdk-$(release_tag)-windows.exe
|
|
mv "bazel-bin/release/windows-installer/daml-sdk-installer.exe" "$(Build.StagingDirectory)/$INSTALLER"
|
|
chmod +x "$(Build.StagingDirectory)/$INSTALLER"
|
|
cleanup () {
|
|
rm -f signing_key.pfx
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
|
|
MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
|
|
rm signing_key.pfx
|
|
echo "##vso[task.setvariable variable=installer;isOutput=true]$INSTALLER"
|
|
|
|
bazel build //release:sdk-release-tarball
|
|
TARBALL=daml-sdk-$(release_tag)-windows.tar.gz
|
|
cp bazel-bin/release/sdk-release-tarball.tar.gz '$(Build.StagingDirectory)'/$TARBALL
|
|
echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
|
|
|
|
name: publish
|
|
env:
|
|
SIGNING_KEY: $(microsoft-code-signing)
|
|
- task: PublishPipelineArtifact@0
|
|
inputs:
|
|
targetPath: $(Build.StagingDirectory)/$(publish.installer)
|
|
artifactName: $(publish.installer)
|
|
- task: PublishPipelineArtifact@0
|
|
inputs:
|
|
targetPath: $(Build.StagingDirectory)/$(publish.tarball)
|
|
artifactName: $(publish.tarball)
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: linux_tarball
|
|
dependsOn: [ "check_for_release", "Linux" ]
|
|
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
pool:
|
|
name: "linux-pool"
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
- bash: ci/configure-bazel.sh
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
export DAML_SDK_RELEASE_VERSION=$(release_tag)
|
|
bazel build //release:release
|
|
|
|
mkdir -p ~/.jfrog
|
|
cleanup() {
|
|
rm -f ~/.jfrog/jfrog-cli.conf
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$JFROG_CONFIG_CONTENT" > ~/.jfrog/jfrog-cli.conf
|
|
unset JFROG_CONFIG_CONTENT
|
|
|
|
./bazel-bin/release/release --release-dir "$(mktemp -d)" --upload
|
|
|
|
bazel build //release:sdk-release-tarball
|
|
TARBALL=daml-sdk-$(release_tag)-linux.tar.gz
|
|
cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL
|
|
echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
|
|
env:
|
|
JFROG_CONFIG_CONTENT: $(JFROG_CONFIG_CONTENT)
|
|
GPG_KEY: $(gpg-code-signing)
|
|
MAVEN_USERNAME: $(MAVEN_USERNAME)
|
|
MAVEN_PASSWORD: $(MAVEN_PASSWORD)
|
|
MAVEN_URL: $(MAVEN_URL)
|
|
NPM_TOKEN: $(NPM_TOKEN)
|
|
name: publish
|
|
- task: PublishPipelineArtifact@0
|
|
inputs:
|
|
targetPath: $(Build.StagingDirectory)/$(publish.tarball)
|
|
artifactName: $(publish.tarball)
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: macos_tarball
|
|
dependsOn: [ "check_for_release", "macOS" ]
|
|
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
pool:
|
|
vmImage: "macOS-10.14"
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
git checkout $(release_sha)
|
|
- bash: ci/configure-bazel.sh
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
export DAML_SDK_RELEASE_VERSION=$(release_tag)
|
|
|
|
bazel build //release:sdk-release-tarball
|
|
TARBALL=daml-sdk-$(release_tag)-macos.tar.gz
|
|
cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL
|
|
echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
|
|
name: publish
|
|
- task: PublishPipelineArtifact@0
|
|
inputs:
|
|
targetPath: $(Build.StagingDirectory)/$(publish.tarball)
|
|
artifactName: $(publish.tarball)
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: release
|
|
dependsOn: [ "check_for_release", "linux_tarball", "macos_tarball", "windows_installer" ]
|
|
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
pool:
|
|
vmImage: "Ubuntu-16.04"
|
|
variables:
|
|
linux-tarball: $[ dependencies.linux_tarball.outputs['publish.tarball'] ]
|
|
macos-tarball: $[ dependencies.macos_tarball.outputs['publish.tarball'] ]
|
|
windows-tarball: $[ dependencies.windows_installer.outputs['publish.tarball'] ]
|
|
windows-installer: $[ dependencies.windows_installer.outputs['publish.installer'] ]
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: |
|
|
set -euxo pipefail
|
|
if git tag v$(release_tag); then
|
|
git push origin v$(release_tag)
|
|
mkdir $(Build.StagingDirectory)/release
|
|
else
|
|
echo "##vso[task.setvariable variable=skip-github]TRUE"
|
|
fi
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(linux-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(macos-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(windows-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(windows-installer)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
KEY_FILE=$(mktemp)
|
|
GPG_DIR=$(mktemp -d)
|
|
cleanup() {
|
|
rm -rf $KEY_FILE $GPG_DIR
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GPG_KEY" | base64 -d > $KEY_FILE
|
|
gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
|
|
cd $(Build.StagingDirectory)/release
|
|
# Note: relies on our release artifacts not having spaces in their
|
|
# names. Creates a ${f}.asc with the signature for each $f.
|
|
for f in *; do
|
|
gpg --homedir $GPG_DIR -ab $f
|
|
done
|
|
env:
|
|
GPG_KEY: $(gpg-code-signing)
|
|
- task: GitHubRelease@0
|
|
inputs:
|
|
gitHubConnection: 'garyverhaegen-da'
|
|
repositoryName: '$(Build.Repository.Name)'
|
|
action: 'create'
|
|
target: '$(Build.SourceVersion)'
|
|
tagSource: 'auto'
|
|
assets: $(Build.StagingDirectory)/release/*
|
|
assetUploadMode: 'replace'
|
|
addChangeLog: false
|
|
isPrerelease: true
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: write_ledger_dump
|
|
dependsOn: [ "check_for_release" ]
|
|
pool:
|
|
vmImage: "Ubuntu-16.04"
|
|
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
git checkout $(release_sha)
|
|
export DAML_SDK_RELEASE_VERSION=$(release_tag)
|
|
|
|
sudo mkdir -p /nix
|
|
sudo chown $USER /nix
|
|
curl -sfL https://nixos.org/releases/nix/nix-2.3.2/install | bash
|
|
eval "$(dev-env/bin/dade-assist)"
|
|
GCS_KEY=$(mktemp)
|
|
cleanup () {
|
|
rm -f $GCS_KEY
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
|
|
gcloud auth activate-service-account --key-file=$GCS_KEY
|
|
export BOTO_CONFIG=/dev/null
|
|
|
|
bazel build //ledger/api-server-damlonx/reference-v2:reference-ledger-dump
|
|
gsutil cp bazel-bin/ledger/api-server-damlonx/reference-v2/reference-ledger-dump.out \
|
|
gs://daml-dumps/release/ledger/api-server-damlonx/reference-v2/reference-ledger-dump-$(release_tag)
|
|
env:
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- template: ci/tell-slack-failed.yml
|
|
|
|
- job: collect_build_data
|
|
condition: always()
|
|
dependsOn:
|
|
- Linux
|
|
- macOS
|
|
- Windows
|
|
- check_for_release
|
|
- linux_tarball
|
|
- macos_tarball
|
|
- windows_installer
|
|
- perf
|
|
- release
|
|
- check_standard_change_label
|
|
- write_ledger_dump
|
|
- git_sha
|
|
pool:
|
|
name: "linux-pool"
|
|
variables:
|
|
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
|
|
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
|
|
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
|
|
Linux.status: $[ dependencies.Linux.result ]
|
|
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
|
|
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
|
|
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
|
|
macOS.status: $[ dependencies.macOS.result ]
|
|
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
|
|
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
|
|
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
|
|
Windows.status: $[ dependencies.Windows.result ]
|
|
check_for_release.start: $[ dependencies.check_for_release.outputs['start.time'] ]
|
|
check_for_release.machine: $[ dependencies.check_for_release.outputs['start.machine'] ]
|
|
check_for_release.end: $[ dependencies.check_for_release.outputs['end.time'] ]
|
|
check_for_release.status: $[ dependencies.check_for_release.result ]
|
|
linux_tarball.start: $[ dependencies.linux_tarball.outputs['start.time'] ]
|
|
linux_tarball.machine: $[ dependencies.linux_tarball.outputs['start.machine'] ]
|
|
linux_tarball.end: $[ dependencies.linux_tarball.outputs['end.time'] ]
|
|
linux_tarball.status: $[ dependencies.linux_tarball.result ]
|
|
macos_tarball.start: $[ dependencies.macos_tarball.outputs['start.time'] ]
|
|
macos_tarball.machine: $[ dependencies.macos_tarball.outputs['start.machine'] ]
|
|
macos_tarball.end: $[ dependencies.macos_tarball.outputs['end.time'] ]
|
|
macos_tarball.status: $[ dependencies.macos_tarball.result ]
|
|
windows_installer.start: $[ dependencies.windows_installer.outputs['start.time'] ]
|
|
windows_installer.machine: $[ dependencies.windows_installer.outputs['start.machine'] ]
|
|
windows_installer.end: $[ dependencies.windows_installer.outputs['end.time'] ]
|
|
windows_installer.status: $[ dependencies.windows_installer.result ]
|
|
perf.start: $[ dependencies.perf.outputs['start.time'] ]
|
|
perf.machine: $[ dependencies.perf.outputs['start.machine'] ]
|
|
perf.end: $[ dependencies.perf.outputs['end.time'] ]
|
|
perf.status: $[ dependencies.perf.result ]
|
|
release.start: $[ dependencies.release.outputs['start.time'] ]
|
|
release.machine: $[ dependencies.release.outputs['start.machine'] ]
|
|
release.end: $[ dependencies.release.outputs['end.time'] ]
|
|
release.status: $[ dependencies.release.result ]
|
|
std_change.start: $[ dependencies.check_standard_change_label.outputs['start.time'] ]
|
|
std_change.machine: $[ dependencies.check_standard_change_label.outputs['start.machine'] ]
|
|
std_change.end: $[ dependencies.check_standard_change_label.outputs['end.time'] ]
|
|
std_change.status: $[ dependencies.check_standard_change_label.result ]
|
|
dump.start: $[ dependencies.write_ledger_dump.outputs['start.time'] ]
|
|
dump.machine: $[ dependencies.write_ledger_dump.outputs['start.machine'] ]
|
|
dump.end: $[ dependencies.write_ledger_dump.outputs['end.time'] ]
|
|
dump.status: $[ dependencies.write_ledger_dump.result ]
|
|
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
master_sha: $[ dependencies.git_sha.outputs['out.master'] ]
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
|
|
# Using expression syntax so we get an empty string if not set, rather
|
|
# than the raw $(VarName) string. Expression syntax works on the
|
|
# variables key, but not on the env one, so we need an extra indirection.
|
|
# Note: These Azure variables are only set for PR builds.
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
|
|
steps:
|
|
# some change in Azure configuration makes this fail recently (2020-01).
|
|
# Azure runs PR builds not on the PR commit, but on the GitHub-provided
|
|
# commit that would be the result of merging the PR. Recently, it looks
|
|
# like when it reaches the point of running this job (which has to run
|
|
# after the macOS one, which sometimes takes up to an hour), if master
|
|
# has changed in the meantime, Azure cannot find the commit it wants to
|
|
# build anymore. Therefore, we tell it not to checkout anything, and
|
|
# manually checkout the PR commit.
|
|
- checkout: none
|
|
- bash: |
|
|
set -euo pipefail
|
|
# Note: this is going to get the PR branch commit, not the
|
|
# result of the merge (i.e. this is not using the same commit as the
|
|
# other jobs in this build).
|
|
#
|
|
# We have seen errors getting this commit from GitHub, which we
|
|
# suspect are transient network errors, so adding simple exponential
|
|
# backoff to mitigate.
|
|
BACKOFF=1
|
|
tell_gary() {
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<@UEHSF89AQ> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> has failed to fetch its commit $(branch_sha) up to BACKOFF=$BACKOFF.\"}" \
|
|
$(Slack.team-daml-ci)
|
|
}
|
|
trap tell_gary EXIT
|
|
while ! git fetch origin $(branch_sha); do
|
|
if (( $BACKOFF > 500 )); then
|
|
echo "Could not get commit $(branch_sha); something is very wrong."
|
|
exit 1
|
|
else
|
|
sleep $BACKOFF
|
|
let "BACKOFF = $BACKOFF * 2"
|
|
fi
|
|
done
|
|
if (( $BACKOFF > 1 )); then
|
|
tell_gary
|
|
fi
|
|
trap - EXIT
|
|
git checkout $(branch_sha)
|
|
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
REPORT=$(mktemp)
|
|
cat >$REPORT <<END
|
|
{"jobs": {"Linux": {"start": "$(Linux.start)",
|
|
"machine": "$(Linux.machine)",
|
|
"end": "$(Linux.end)",
|
|
"status": "$(Linux.status)"},
|
|
"macOS": {"start": "$(macOS.start)",
|
|
"machine": "$(macOS.machine)",
|
|
"end": "$(macOS.end)",
|
|
"status": "$(macOS.status)"},
|
|
"Windows": {"start": "$(Windows.start)",
|
|
"machine": "$(Windows.machine)",
|
|
"end": "$(Windows.end)",
|
|
"status": "$(Windows.status)"},
|
|
"check_for_release": {"start": "$(check_for_release.start)",
|
|
"machine": "$(check_for_release.machine)",
|
|
"end": "$(check_for_release.end)",
|
|
"status": "$(check_for_release.status)"},
|
|
"linux_tarball": {"start": "$(linux_tarball.start)",
|
|
"machine": "$(linux_tarball.machine)",
|
|
"end": "$(linux_tarball.end)",
|
|
"status": "$(linux_tarball.status)"},
|
|
"macos_tarball": {"start": "$(macos_tarball.start)",
|
|
"machine": "$(macos_tarball.machine)",
|
|
"end": "$(macos_tarball.end)",
|
|
"status": "$(macos_tarball.status)"},
|
|
"windows_installer": {"start": "$(windows_installer.start)",
|
|
"machine": "$(windows_installer.machine)",
|
|
"end": "$(windows_installer.end)",
|
|
"status": "$(windows_installer.status)"},
|
|
"perf": {"start": "$(perf.start)",
|
|
"machine": "$(perf.machine)",
|
|
"end": "$(perf.end)",
|
|
"status": "$(perf.status)"},
|
|
"check_standard_change_label": {"start": "$(std_change.start)",
|
|
"machine": "$(std_change.machine)",
|
|
"end": "$(std_change.end)",
|
|
"status": "$(std_change.status)"},
|
|
"write_ledger_dump": {"start": "$(dump.start)",
|
|
"machine": "$(dump.machine)",
|
|
"end": "$(dump.end)",
|
|
"status": "$(dump.status)"},
|
|
"release": {"start": "$(release.start)",
|
|
"machine": "$(release.machine)",
|
|
"end": "$(release.end)",
|
|
"status": "$(release.status)"}},
|
|
"id": "$(Build.BuildId)",
|
|
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
|
|
"name": "$(Build.DefinitionName)",
|
|
"version": "$(Build.DefinitionVersion)",
|
|
"queued_by": "$(Build.QueuedBy)",
|
|
"reason": "$(Build.Reason)",
|
|
"branch": "$(Build.SourceBranch)",
|
|
"merge_commit": "$(Build.SourceVersion)",
|
|
"branch_commit": "$(branch_sha)",
|
|
"master_commit": "$(master_sha)",
|
|
"fork_point_commit": "$(fork_sha)",
|
|
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
|
|
"is_fork": "$(System.PullRequest.IsFork)",
|
|
"pr": "$PR_NUM",
|
|
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
|
|
"pr_source_branch": "$PR_BRANCH"}
|
|
END
|
|
# Test above JSON is well formed
|
|
cat $REPORT | jq '.'
|
|
REPORT_GZ=$(mktemp)
|
|
cat $REPORT | gzip -9 > $REPORT_GZ
|
|
GCS_KEY=$(mktemp)
|
|
cleanup() {
|
|
rm -rf $GCS_KEY
|
|
}
|
|
trap cleanup EXIT
|
|
# Application credentials will not be set for forks. We give up on
|
|
# tracking those for now. "Not set" in Azure world means set to the
|
|
# expression Azure would otherwise substitute, i.e. the literal value
|
|
# of the string in the `env:` block below.
|
|
if [[ "${GOOGLE_APPLICATION_CREDENTIALS_CONTENT:1:${#GOOGLE_APPLICATION_CREDENTIALS_CONTENT}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
|
|
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
|
|
gcloud auth activate-service-account --key-file=$GCS_KEY
|
|
BOTO_CONFIG=/dev/null gsutil cp $REPORT_GZ gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz
|
|
else
|
|
echo "Could not save build data: no credentials. Data was:"
|
|
cat $REPORT
|
|
fi
|
|
|
|
# Linux, macOS, perf, check_std_change_label and Windows are always
|
|
# required and should always succeed.
|
|
#
|
|
# windows_signing, release and write_ledger_dump only run on releases
|
|
# and are skipped otherwise.
|
|
if [[ "$(Linux.status)" != "Succeeded"
|
|
|| "$(macOS.status)" != "Succeeded"
|
|
|| "$(Windows.status)" != "Succeeded"
|
|
|| "$(perf.status)" != "Succeeded"
|
|
|| "$(dump.status)" == "Canceled"
|
|
|| "$(linux_tarball.status)" == "Canceled"
|
|
|| "$(macos_tarball.status)" == "Canceled"
|
|
|| "$(windows_installer.status)" == "Canceled"
|
|
|| "$(release.status)" == "Canceled" ]]; then
|
|
exit 1
|
|
fi
|
|
env:
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
# Commit message is always set
|
|
COMMIT_MSG: $(Build.SourceVersionMessage)
|
|
# Because these variables are always set (in the variables block),
|
|
# hopefully these should be set as expected (i.e. either correct
|
|
# value or empty string, but not $(Azure.Variable.Name)).
|
|
PR_NUM: $(pr.num)
|
|
PR_BRANCH: $(pr.branch)
|
|
|
|
- job: notify_user
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
dependsOn:
|
|
- git_sha
|
|
- collect_build_data
|
|
pool:
|
|
name: 'linux-pool'
|
|
variables:
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
status: $[ dependencies.collect_build_data.result ]
|
|
steps:
|
|
- checkout: none
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
# We have seen errors getting this commit from GitHub, which we
|
|
# suspect are transient network errors, so adding simple exponential
|
|
# backoff to mitigate.
|
|
BACKOFF=1
|
|
tell_gary() {
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<@UEHSF89AQ> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> has failed to fetch its commit $(branch_sha) up to BACKOFF=$BACKOFF.\"}" \
|
|
$(Slack.team-daml-ci)
|
|
}
|
|
trap tell_gary EXIT
|
|
while ! git fetch origin $(branch_sha); do
|
|
if (( $BACKOFF > 500 )); then
|
|
echo "Could not get commit $(branch_sha); something is very wrong."
|
|
exit 1
|
|
else
|
|
sleep $BACKOFF
|
|
let "BACKOFF = $BACKOFF * 2"
|
|
fi
|
|
done
|
|
if (( $BACKOFF > 1 )); then
|
|
tell_gary
|
|
fi
|
|
trap - EXIT
|
|
git checkout $(branch_sha)
|
|
|
|
|
|
tell_slack() {
|
|
local MESSAGE=$1
|
|
local USER_ID=$2
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<@${USER_ID}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> has completed with status ${MESSAGE}.\"}" \
|
|
$(Slack.team-daml-ci)
|
|
}
|
|
|
|
EMAIL=$(git log -n 1 --format=%ae)
|
|
user_registered() {
|
|
cat ci/slack_user_ids | grep $EMAIL
|
|
}
|
|
|
|
user_id() {
|
|
echo $(cat ci/slack_user_ids | grep $EMAIL | awk '{print $2}')
|
|
}
|
|
|
|
if user_registered; then
|
|
tell_slack "$(status)" "$(user_id)"
|
|
else
|
|
echo "User $(user_id) did not opt in for notifications."
|
|
fi
|