mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 17:28:46 +03:00
97e289e548
At the moment, if we try to run the build of a release commit after it has succeeded, the `git tag` step fails. We do not normally try to rebuild old commits that had succeeded, but sometimes Azure gets confused when we ask for rerunning specific jobs within a build, and reruns the whole build. This creates two (small, bubt annoying) problems: 1. It adds noise to the #team-daml channel as it notifies of the build failure, and 2. It marks the commit as failed with a red cross on the github list of commits, which obviously doesn't look great for release commits. This fixes that. Note that if a release does fail after the `git tag` step (e.g. some network error between Azure and GitHub), this **does not** change the necessary steps to remediate, as that situation would already be broken in the current setup. (Steps to remediate would essentially boil down to deleting the tag on GitHub before rerunning so the build can create it again.)
362 lines
16 KiB
YAML
362 lines
16 KiB
YAML
# Azure Pipelines file, see https://aka.ms/yaml
|
|
|
|
# Enable builds on all branches
|
|
trigger:
|
|
# Build every commit as our release process relies on
|
|
# the release process being built alone.
|
|
batch: false
|
|
branches:
|
|
include:
|
|
- master
|
|
|
|
# Enable PR triggers that target the master branch
|
|
pr:
|
|
autoCancel: true # cancel previous builds on push
|
|
branches:
|
|
include:
|
|
- master
|
|
|
|
jobs:
|
|
- job: Linux
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
name: linux
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: macOS
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
vmImage: 'macOS-10.14'
|
|
variables:
|
|
nix-cache-key: $(Build.StagingDirectory)/macos-nix-key
|
|
nix-cache-path: /tmp/nix-cache/
|
|
bazel-repo-cache-key: $(Build.StagingDirectory)/bazel-repo-cache-key
|
|
bazel-repo-cache-path: $(Agent.BuildDirectory)/.bazel-cache/repo
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: echo $(git log -n1 --pretty=format:%H dev-env nix azure-pipelines.yml) >> $(nix-cache-key)
|
|
displayName: nix cache key
|
|
- task: CacheBeta@0
|
|
inputs:
|
|
key: $(nix-cache-key)
|
|
path: $(nix-cache-path)
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [[ -e $(nix-cache-path) ]]; then
|
|
DIR=$(pwd)
|
|
sudo mkdir /nix && sudo chown $USER /nix
|
|
cd /nix
|
|
tar xzf $(nix-cache-path)/nix.tar.gz
|
|
cd $DIR
|
|
curl -sfL https://nixos.org/releases/nix/nix-2.2.1/install | bash
|
|
fi
|
|
displayName: restore cache
|
|
- bash: echo $(git log -n1 --pretty=format:%H azure-pipelines.yml $(find . -name \*.bazel -or -name \*.bzl -or -name WORKSPACE -or -name BUILD)) >> $(bazel-repo-cache-key)
|
|
displayName: bazel repo cache key
|
|
- task: CacheBeta@0
|
|
inputs:
|
|
key: $(bazel-repo-cache-key)
|
|
path: $(bazel-repo-cache-path)
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
name: macos
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [[ ! -e $(nix-cache-path) ]]; then
|
|
mkdir -p $(nix-cache-path)
|
|
cd /nix
|
|
GZIP=-9 tar czf $(nix-cache-path)/nix.tar.gz store var
|
|
fi
|
|
displayName: create nix cache
|
|
- bash: mkdir -p $(bazel-repo-cache-path)
|
|
displayName: ensure bazel repo cache exists
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: Windows
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'windows-pool'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- template: ci/build-windows.yml
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: perf
|
|
timeoutInMinutes: 60
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: ci/dev-env-install.sh
|
|
displayName: 'Build/Install the Developer Environment'
|
|
- bash: ci/configure-bazel.sh
|
|
displayName: 'Configure Bazel'
|
|
env:
|
|
IS_FORK: $(System.PullRequest.IsFork)
|
|
# to upload to the bazel cache
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
bazel run -- //ledger/sandbox-perf -foe true -i1 -f1 -wi 1 -bm avgt -rf csv -rff "$(Build.StagingDirectory)/sandbox-perf.csv"
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Perf test logs'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: hie_core_stack_86
|
|
timeoutInMinutes: 60
|
|
pool:
|
|
vmImage: 'ubuntu-latest'
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- task: CacheBeta@0
|
|
inputs:
|
|
key: stack-cache-v1 | $(Agent.OS) | $(Build.SourcesDirectory)/compiler/hie-core/stack.yaml | $(Build.SourcesDirectory)/compiler/hie-core/hie-core.cabal
|
|
path: .azure-cache
|
|
cacheHitVar: CACHE_RESTORED
|
|
displayName: "Cache stack artifacts"
|
|
- bash: |
|
|
mkdir -p ~/.stack
|
|
tar xzf .azure-cache/stack-root.tar.gz -C $HOME
|
|
displayName: "Unpack cache"
|
|
condition: eq(variables.CACHE_RESTORED, 'true')
|
|
- bash: |
|
|
sudo apt-get install -y g++ gcc libc6-dev libffi-dev libgmp-dev make zlib1g-dev
|
|
curl -sSL https://get.haskellstack.org/ | sh
|
|
displayName: 'Install Stack'
|
|
- bash: (cd compiler/hie-core; stack setup)
|
|
displayName: 'stack setup'
|
|
- bash: (cd compiler/hie-core; stack build --only-dependencies)
|
|
displayName: 'stack build --only-dependencies --test'
|
|
- bash: (cd compiler/hie-core; stack test || stack test || stack test)
|
|
# hie-core stack tests are flaky, see https://github.com/digital-asset/daml/issues/2606.
|
|
displayName: 'stack test'
|
|
- bash: |
|
|
mkdir -p .azure-cache
|
|
tar czf .azure-cache/stack-root.tar.gz -C $HOME .stack
|
|
displayName: "Pack cache"
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
- job: Windows_signing
|
|
# Signing is a separate job so that we can make sure that we only sign on releases.
|
|
# Since the release check is run on Linux, we do not have access to that information
|
|
# in the regular Windows step.
|
|
dependsOn: [ "Windows", "Linux" ]
|
|
pool:
|
|
name: 'windows-pool'
|
|
condition: and(succeeded(), eq(dependencies.Linux.outputs['release.has_released'], 'true'))
|
|
variables:
|
|
unsigned-installer: $[ dependencies.Windows.outputs['publish.artifact-unsigned-windows-installer'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(unsigned-installer)
|
|
targetPath: $(Build.StagingDirectory)/
|
|
- bash: |
|
|
set -euo pipefail
|
|
INSTALLER=daml-sdk-$(cat VERSION)-windows.exe
|
|
mv "$(Build.StagingDirectory)/$(unsigned-installer)" "$(Build.StagingDirectory)/$INSTALLER"
|
|
chmod +x "$(Build.StagingDirectory)/$INSTALLER"
|
|
echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
|
|
MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
|
|
rm signing_key.pfx
|
|
echo "##vso[task.setvariable variable=artifact-windows-installer;isOutput=true]$INSTALLER"
|
|
echo "##vso[task.setvariable variable=has_released;isOutput=true]true"
|
|
name: signing
|
|
env:
|
|
SIGNING_KEY: $(microsoft-code-signing)
|
|
- task: PublishPipelineArtifact@0
|
|
inputs:
|
|
targetPath: $(Build.StagingDirectory)/$(signing.artifact-windows-installer)
|
|
artifactName: $(signing.artifact-windows-installer)
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: release
|
|
dependsOn: [ "Linux", "macOS", "Windows", "Windows_signing", "perf"]
|
|
pool:
|
|
vmImage: "Ubuntu-16.04"
|
|
condition: and(succeeded(),
|
|
eq( dependencies.Linux.outputs['release.has_released'], 'true' ),
|
|
eq( dependencies.macOS.outputs['release.has_released'], 'true' ),
|
|
eq( dependencies.Windows.outputs['release.has_released'], 'true' ))
|
|
variables:
|
|
artifact-linux: $[ dependencies.Linux.outputs['publish.artifact'] ]
|
|
artifact-macos: $[ dependencies.macOS.outputs['publish.artifact'] ]
|
|
artifact-windows: $[ dependencies.Windows.outputs['publish.artifact'] ]
|
|
artifact-windows-installer: $[ dependencies.Windows_signing.outputs['signing.artifact-windows-installer'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: |
|
|
set -euxo pipefail
|
|
if git tag v$(cat VERSION); then
|
|
git push origin v$(cat VERSION)
|
|
mkdir $(Build.StagingDirectory)/release
|
|
else
|
|
echo "##vso[task.setvariable variable=skip-github]TRUE"
|
|
fi
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(artifact-linux)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(artifact-macos)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(artifact-windows)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(artifact-windows-installer)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: GitHubRelease@0
|
|
inputs:
|
|
gitHubConnection: 'garyverhaegen-da'
|
|
repositoryName: '$(Build.Repository.Name)'
|
|
action: 'create'
|
|
target: '$(Build.SourceVersion)'
|
|
tagSource: 'auto'
|
|
assets: $(Build.StagingDirectory)/release/*
|
|
assetUploadMode: 'replace'
|
|
addChangeLog: false
|
|
isPrerelease: true
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: collect_build_data
|
|
condition: always()
|
|
dependsOn: ["Linux", "macOS", "Windows", "Windows_signing", "perf", "release", "hie_core_stack_86"]
|
|
pool:
|
|
name: "linux-pool"
|
|
variables:
|
|
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
|
|
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
|
|
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
|
|
Linux.status: $[ dependencies.Linux.result ]
|
|
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
|
|
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
|
|
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
|
|
macOS.status: $[ dependencies.macOS.result ]
|
|
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
|
|
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
|
|
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
|
|
Windows.status: $[ dependencies.Windows.result ]
|
|
Windows_signing.start: $[ dependencies.Windows_signing.outputs['start.time'] ]
|
|
Windows_signing.machine: $[ dependencies.Windows_signing.outputs['start.machine'] ]
|
|
Windows_signing.end: $[ dependencies.Windows_signing.outputs['end.time'] ]
|
|
Windows_signing.status: $[ dependencies.Windows_signing.result ]
|
|
perf.start: $[ dependencies.perf.outputs['start.time'] ]
|
|
perf.machine: $[ dependencies.perf.outputs['start.machine'] ]
|
|
perf.end: $[ dependencies.perf.outputs['end.time'] ]
|
|
perf.status: $[ dependencies.perf.result ]
|
|
release.start: $[ dependencies.release.outputs['start.time'] ]
|
|
release.machine: $[ dependencies.release.outputs['start.machine'] ]
|
|
release.end: $[ dependencies.release.outputs['end.time'] ]
|
|
release.status: $[ dependencies.release.result ]
|
|
hie_core_stack_86.start: $[ dependencies.hie_core_stack_86.outputs['start.time'] ]
|
|
hie_core_stack_86.machine: $[ dependencies.hie_core_stack_86.outputs['start.machine'] ]
|
|
hie_core_stack_86.end: $[ dependencies.hie_core_stack_86.outputs['end.time'] ]
|
|
hie_core_stack_86.status: $[ dependencies.hie_core_stack_86.result ]
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
REPORT=$(mktemp)
|
|
cat >$REPORT <<END
|
|
{"jobs": {"Linux": {"start": "$(Linux.start)",
|
|
"machine": "$(Linux.machine)",
|
|
"end": "$(Linux.end)",
|
|
"status": "$(Linux.status)"},
|
|
"macOS": {"start": "$(macOS.start)",
|
|
"machine": "$(macOS.machine)",
|
|
"end": "$(macOS.end)",
|
|
"status": "$(macOS.status)"},
|
|
"Windows": {"start": "$(Windows.start)",
|
|
"machine": "$(Windows.machine)",
|
|
"end": "$(Windows.end)",
|
|
"status": "$(Windows.status)"},
|
|
"Windows_signing": {"start": "$(Windows_signing.start)",
|
|
"machine": "$(Windows_signing.machine)",
|
|
"end": "$(Windows_signing.end)",
|
|
"status": "$(Windows_signing.status)"},
|
|
"perf": {"start": "$(perf.start)",
|
|
"machine": "$(perf.machine)",
|
|
"end": "$(perf.end)",
|
|
"status": "$(perf.status)"},
|
|
"release": {"start": "$(release.start)",
|
|
"machine": "$(release.machine)",
|
|
"end": "$(release.end)",
|
|
"status": "$(release.status)"},
|
|
"hie_core_stack_86": {"start": "$(hie_core_stack_86.start)",
|
|
"machine": "$(hie_core_stack_86.machine)",
|
|
"end": "$(hie_core_stack_86.end)",
|
|
"status": "$(hie_core_stack_86.status)"}},
|
|
"id": "$(Build.BuildId)",
|
|
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
|
|
"name": "$(Build.DefinitionName)",
|
|
"version": "$(Build.DefinitionVersion)",
|
|
"queued_by": "$(Build.QueuedBy)",
|
|
"reason": "$(Build.Reason)",
|
|
"branch": "$(Build.SourceBranch)",
|
|
"commit_sha": "$(Build.SourceVersion)",
|
|
"commit_message": "$(Build.SourceVersionMessage)",
|
|
"is_fork": "$(System.PullRequest.IsFork)",
|
|
"pr": "$(System.PullRequest.PullRequestNumber)",
|
|
"pr_url": "https://github.com/digital-asset/daml/pull/$(System.PullRequest.PullRequestNumber)",
|
|
"pr_source_branch": "$(System.PullRequest.SourceBranch)"}
|
|
END
|
|
REPORT_GZ=$(mktemp)
|
|
cat $REPORT | gzip -9 > $REPORT_GZ
|
|
GCS_KEY=$(mktemp)
|
|
# Application credentials will not be set for forks. We give up on
|
|
# tracking those for now. "Not set" in Azure world means set to the
|
|
# expression Azure would otherwise substitute, i.e. the literal value
|
|
# of the string in the `env:` block below.
|
|
if [[ "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" != '$(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
|
|
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
|
|
gcloud auth activate-service-account --key-file=$GCS_KEY
|
|
BOTO_CONFIG=/dev/null gsutil cp $REPORT_GZ gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz
|
|
else
|
|
cat $REPORT
|
|
fi
|
|
|
|
if [[ "$(Linux.status)" == "Canceled"
|
|
|| "$(macOS.status)" == "Canceled"
|
|
|| "$(Windows.status)" == "Canceled"
|
|
|| "$(perf.status)" == "Canceled"
|
|
|| "$(Windows_signing.status)" == "Canceled"
|
|
|| "$(release.status)" == "Canceled" ]]; then
|
|
exit 1
|
|
fi
|
|
env:
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|