daml/azure-pipelines.yml
Moritz Kiefer a633dbd107
Adapt stack cache key to new format (#2375)
Azure currently emits a warning that the key format will switch to
single lines so this PR changes the key to the new format.
2019-08-02 15:22:05 +02:00

378 lines
15 KiB
YAML

# Azure Pipelines file, see https://aka.ms/yaml
# Enable builds on all branches
trigger:
# Build every commit as our release process relies on
# the release process being built alone.
batch: false
branches:
include:
- master
# Enable PR triggers that target the master branch
pr:
autoCancel: true # cancel previous builds on push
branches:
include:
- master
jobs:
- job: Linux
timeoutInMinutes: 360
pool:
name: 'linux-pool'
steps:
- template: ci/report-start.yml
- checkout: self
- template: ci/build-unix.yml
parameters:
name: linux
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: macOS
continueOnError: true
timeoutInMinutes: 360
pool:
vmImage: 'macOS-10.14'
variables:
cache-key: $(Build.StagingDirectory)/macos-nix-key
cache-path: /tmp/cache/
steps:
- template: ci/report-start.yml
- checkout: self
- bash: echo $(git log -n1 --pretty=format:%H dev-env nix azure-pipelines.yml) >> $(cache-key)
displayName: cache key
- task: CacheBeta@0
inputs:
key: $(cache-key)
path: $(cache-path)
- bash: |
set -euo pipefail
echo "##vso[task.setvariable variable=cacheFailed]true"
echo "##vso[task.setvariable variable=Agent.JobStatus]Succeeded"
condition: failed()
- bash: |
set -euo pipefail
if [[ -e $(cache-path) ]]; then
DIR=$(pwd)
sudo mkdir /nix && sudo chown $USER /nix
cd /nix
tar xzf $(cache-path)/nix.tar.gz
cd $DIR
curl -sfL https://nixos.org/releases/nix/nix-2.2.1/install | bash
fi
displayName: restore cache
condition: not(eq('true', variables['cacheFailed']))
- template: ci/build-unix.yml
parameters:
name: macos
- bash: |
set -euo pipefail
if [[ ! -e $(cache-path) ]]; then
mkdir -p $(cache-path)
cd /nix
GZIP=-9 tar czf $(cache-path)/nix.tar.gz store var
fi
displayName: create cache
condition: not(eq('true', variables['cacheFailed']))
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- bash: |
echo "##vso[task.setvariable variable=succeedBeforePostCache;isOutput=true]true"
name: cache
- job: Windows
timeoutInMinutes: 360
pool:
name: 'windows-pool'
steps:
- template: ci/report-start.yml
- template: ci/build-windows.yml
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: perf
timeoutInMinutes: 60
pool:
name: 'linux-pool'
steps:
- template: ci/report-start.yml
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel run -- //ledger/sandbox-perf -foe true -i1 -f1 -wi 1 -bm avgt -rf csv -rff "$(Build.StagingDirectory)/sandbox-perf.csv"
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)'
artifactName: 'Perf test logs'
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: hie_core_stack_86
continueOnError: true
timeoutInMinutes: 60
pool:
vmImage: 'ubuntu-latest'
steps:
- template: ci/report-start.yml
- checkout: self
- task: CacheBeta@0
inputs:
key: stack-cache-v1 | $(Agent.OS) | $(Build.SourcesDirectory)/compiler/hie-core/stack.yaml | $(Build.SourcesDirectory)/compiler/hie-core/hie-core.cabal
path: .azure-cache
cacheHitVar: CACHE_RESTORED
displayName: "Cache stack artifacts"
- bash: |
set -euo pipefail
echo "##vso[task.setvariable variable=cacheFailed]true"
echo "##vso[task.setvariable variable=Agent.JobStatus]Succeeded"
condition: failed()
- bash: |
mkdir -p ~/.stack
tar xzf .azure-cache/stack-root.tar.gz -C $HOME
displayName: "Unpack cache"
condition: eq(variables.CACHE_RESTORED, 'true')
- bash: |
sudo apt-get install -y g++ gcc libc6-dev libffi-dev libgmp-dev make zlib1g-dev
curl -sSL https://get.haskellstack.org/ | sh
displayName: 'Install Stack'
- bash: (cd compiler/hie-core; stack setup)
displayName: 'stack setup'
- bash: (cd compiler/hie-core; stack build --only-dependencies)
displayName: 'stack build --only-dependencies --test'
- bash: (cd compiler/hie-core; stack test)
displayName: 'stack test'
- bash: |
mkdir -p .azure-cache
tar czf .azure-cache/stack-root.tar.gz -C $HOME .stack
displayName: "Pack cache"
condition: not(eq('true', variables['cacheFailed']))
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- bash: |
echo "##vso[task.setvariable variable=succeedBeforePostCache;isOutput=true]true"
name: cache
- job: Windows_signing
# Signing is a separate job so that we can make sure that we only sign on releases.
# Since the release check is run on Linux, we do not have access to that information
# in the regular Windows step.
dependsOn: [ "Windows", "Linux" ]
pool:
name: 'windows-pool'
condition: and(succeeded(), eq(dependencies.Linux.outputs['release.has_released'], 'true'))
variables:
unsigned-installer: $[ dependencies.Windows.outputs['publish.artifact-unsigned-windows-installer'] ]
steps:
- template: ci/report-start.yml
- checkout: self
persistCredentials: true
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(unsigned-installer)
targetPath: $(Build.StagingDirectory)/
- bash: |
set -euo pipefail
INSTALLER=daml-sdk-$(cat VERSION)-windows.exe
mv "$(Build.StagingDirectory)/$(unsigned-installer)" "$(Build.StagingDirectory)/$INSTALLER"
chmod +x "$(Build.StagingDirectory)/$INSTALLER"
echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
rm signing_key.pfx
echo "##vso[task.setvariable variable=artifact-windows-installer;isOutput=true]$INSTALLER"
echo "##vso[task.setvariable variable=has_released;isOutput=true]true"
name: signing
env:
SIGNING_KEY: $(microsoft-code-signing)
- task: PublishPipelineArtifact@0
inputs:
targetPath: $(Build.StagingDirectory)/$(signing.artifact-windows-installer)
artifactName: $(signing.artifact-windows-installer)
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- bash: |
echo "##vsp[task.setvariable variable=succeedBeforePostCache;isOutput=true]true"
name: cache
- job: work_around_broken_cache
dependsOn: [ "macOS", "hie_core_stack_86" ]
pool:
vmImage: "Ubuntu-16.04"
variables:
mac: $[ dependencies.macOS.outputs['cache.succeedBeforePostCache'] ]
hie: $[ dependencies.hie_core_stack_86.outputs['cache.succeedBeforePostCache'] ]
steps:
- checkout: none
- bash: |
set -euo pipefail
echo "mac: $(mac)"
echo "hie: $(hie)"
if [[ "$(mac)" != "true" || "$(hie)" != "true" ]]; then
exit 1
fi
- job: release
dependsOn: [ "Linux", "macOS", "Windows", "Windows_signing", "perf"]
pool:
vmImage: "Ubuntu-16.04"
condition: and(succeeded(),
eq( dependencies.Linux.outputs['release.has_released'], 'true' ),
eq( dependencies.macOS.outputs['release.has_released'], 'true' ),
eq( dependencies.Windows.outputs['release.has_released'], 'true' ))
variables:
artifact-linux: $[ dependencies.Linux.outputs['publish.artifact'] ]
artifact-macos: $[ dependencies.macOS.outputs['publish.artifact'] ]
artifact-windows: $[ dependencies.Windows.outputs['publish.artifact'] ]
artifact-windows-installer: $[ dependencies.Windows_signing.outputs['signing.artifact-windows-installer'] ]
steps:
- template: ci/report-start.yml
- checkout: self
persistCredentials: true
- bash: |
set -euxo pipefail
git tag v$(cat VERSION)
git push origin v$(cat VERSION)
mkdir $(Build.StagingDirectory)/release
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(artifact-linux)
targetPath: $(Build.StagingDirectory)/release
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(artifact-macos)
targetPath: $(Build.StagingDirectory)/release
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(artifact-windows)
targetPath: $(Build.StagingDirectory)/release
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(artifact-windows-installer)
targetPath: $(Build.StagingDirectory)/release
- task: GitHubRelease@0
inputs:
gitHubConnection: 'garyverhaegen-da'
repositoryName: '$(Build.Repository.Name)'
action: 'create'
target: '$(Build.SourceVersion)'
tagSource: 'auto'
assets: $(Build.StagingDirectory)/release/*
assetUploadMode: 'replace'
addChangeLog: false
isPrerelease: true
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: collect_build_data
condition: always()
dependsOn: ["Linux", "macOS", "Windows", "Windows_signing", "perf", "release"]
pool:
name: "linux-pool"
variables:
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
Linux.status: $[ dependencies.Linux.result ]
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
macOS.status: $[ dependencies.macOS.result ]
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
Windows_signing.start: $[ dependencies.Windows_signing.outputs['start.time'] ]
Windows_signing.machine: $[ dependencies.Windows_signing.outputs['start.machine'] ]
Windows_signing.end: $[ dependencies.Windows_signing.outputs['end.time'] ]
Windows_signing.status: $[ dependencies.Windows_signing.result ]
perf.start: $[ dependencies.perf.outputs['start.time'] ]
perf.machine: $[ dependencies.perf.outputs['start.machine'] ]
perf.end: $[ dependencies.perf.outputs['end.time'] ]
perf.status: $[ dependencies.perf.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
release.status: $[ dependencies.release.result ]
steps:
- checkout: self
- bash: |
set -euo pipefail
REPORT=$(mktemp)
cat >$REPORT <<END
{"jobs": {"Linux": {"start": "$(Linux.start)",
"machine": "$(Linux.machine)",
"end": "$(Linux.end)",
"status": "$(Linux.status)"},
"macOS": {"start": "$(macOS.start)",
"machine": "$(macOS.machine)",
"end": "$(macOS.end)",
"status": "$(macOS.status)"},
"Windows": {"start": "$(Windows.start)",
"machine": "$(Windows.machine)",
"end": "$(Windows.end)",
"status": "$(Windows.status)"},
"Windows_signing": {"start": "$(Windows_signing.start)",
"machine": "$(Windows_signing.machine)",
"end": "$(Windows_signing.end)",
"status": "$(Windows_signing.status)"},
"perf": {"start": "$(perf.start)",
"machine": "$(perf.machine)",
"end": "$(perf.end)",
"status": "$(perf.status)"},
"release": {"start": "$(release.start)",
"machine": "$(release.machine)",
"end": "$(release.end)",
"status": "$(release.status)"}},
"id": "$(Build.BuildId)",
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
"name": "$(Build.DefinitionName)",
"version": "$(Build.DefinitionVersion)",
"queued_by": "$(Build.QueuedBy)",
"reason": "$(Build.Reason)",
"branch": "$(Build.SourceBranch)",
"commit_sha": "$(Build.SourceVersion)",
"commit_message": "$(Build.SourceVersionMessage)",
"is_fork": "$(System.PullRequest.IsFork)",
"pr": "$(System.PullRequest.PullRequestNumber)",
"pr_url": "https://github.com/digital-asset/daml/pull/$(System.PullRequest.PullRequestNumber)",
"pr_source_branch": "$(System.PullRequest.SourceBranch)"}
END
REPORT_GZ=$(mktemp)
cat $REPORT | gzip -9 > $REPORT_GZ
GCS_KEY=$(mktemp)
# Application credentials will not be set for forks. We give up on
# tracking those for now. "Not set" in Azure world means set to the
# expression Azure would otherwise substitute, i.e. the literal value
# of the string in the `env:` block below.
if [[ "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" != '$(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
gcloud auth activate-service-account --key-file=$GCS_KEY
BOTO_CONFIG=/dev/null gsutil cp $REPORT_GZ gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz
else
cat $REPORT
fi
if [[ "$(Linux.status)" == "Canceled"
|| "$(macOS.status)" == "Canceled"
|| "$(Windows.status)" == "Canceled"
|| "$(perf.status)" == "Canceled"
|| "$(Windows_signing.status)" == "Canceled"
|| "$(release.status)" == "Canceled" ]]; then
exit 1
fi
env:
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)