digital-asset/daml
1
1
Fork 0
mirror of https://github.com/digital-asset/daml.git synced 2024-09-20 01:07:18 +03:00
Code Issues Projects Releases Wiki Activity
5a019eaaa6
daml/ci/build.yml
Moritz Kiefer 5a019eaaa6
Split release process (PoC) (#11991) 
* Split release process (PoC)

This PR adds the necessary infrastructure for the new split release
process. Releases are still triggered via the LATEST file but you can
choose between the old and the new release process by adding
SPLIT-RELEASE at the end of the line.

As a first step this publishes all artifacts we currently publish to
Github releases to our GCP bucket.

changelog_begin
changelog_end

* drop report-start

changelog_begin
changelog_end

* fix gcp bucket

changelog_begin
changelog_end

* review feedback

changelog_begin
changelog_end

* Update azure-pipelines.yml

Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com>

* SPLIT-RELEASE ->SPLIT_RELEASE

changelog_begin
changelog_end

Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com>
2021-12-06 19:25:00 +01:00

607 lines
23 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
jobs:
- job: fix_bazel_cache
timeoutInMinutes: 120
condition: eq(variables['System.PullRequest.IsFork'], 'False')
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
eval "$(dev-env/bin/dade assist)"
bazel build //ci/cron:cron
key=$(mktemp)
cleanup="rm -rf $key ~/.config/gcloud"
trap "$cleanup" EXIT
echo "$GCRED" > $key
gcloud auth activate-service-account --key-file=$key
export BOTO_CONFIG=/dev/null
# 90 minutes should provide enough overlap for an hourly
# cronjob.
./bazel-bin/ci/cron/cron bazel-cache --age 90 --delete
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: tell-slack-failed.yml
- job: git_sha
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
source $(bash-lib)
if [ "$(Build.Reason)" == "PullRequest" ]; then
setvar branch "$(git rev-parse HEAD^2)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
else
setvar branch "$(git rev-parse HEAD)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git rev-parse HEAD^1)"
fi
name: out
- template: da-ghc-lib/compile.yml
parameters:
final_job_name: da_ghc_lib
- job: Linux
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux'
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel build //release:release
./bazel-bin/release/release --release-dir "$(mktemp -d)"
condition: and(succeeded(), ne(variables['is_release'], 'true'))
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: 'bazel-bin/docs/html.tar.gz'
artifactName: 'Docs bundle'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: macOS
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
timeoutInMinutes: 360
pool:
name: macOS-pool
demands: assignment -equals default
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
steps:
- template: report-start.yml
- template: clear-shared-segments-macos.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: macos
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- template: patch_bazel_windows/compile.yml
parameters:
final_job_name: patch_bazel_windows
- job: Windows
dependsOn:
- da_ghc_lib
- check_for_release
- patch_bazel_windows
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- name: skip_tests
value: $[ and(eq(variables.is_release, 'true'),
eq(variables['Build.SourceBranchName'], 'main')) ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: build-windows.yml
parameters:
release_tag: $(release_tag)
# Azure pipeline’s variable and parameter expansion is utter garbage.
# For whatever reason `env` values only seem to be able to use macro syntax
# and not runtime expression. is_release however is a runtime variable
# so template conditions won’t work. Therefore we define the variable here
# with a runtime expression, set the parameter to the (unexpanded) string "$(skip_tests)"
# and then splice that in via a template parameter.
skip_tests: $(skip_tests)
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)/logs'
artifactName: 'Bazel Logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_oracle
dependsOn:
- da_ghc_lib
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: clean-up.yml
- bash: |
source dev-env/lib/ensure-nix
ci/dev-env-push.py
displayName: 'Push Developer Environment build results'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
env:
# to upload to the Nix cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
IMAGE=$(cat ci/oracle_image)
docker pull $IMAGE
# Cleanup stray containers that might still be running from
# another build that didn’t get shut down cleanly.
docker rm -f oracle || true
# Oracle does not like if you connect to it via localhost if it’s running in the container.
# Interestingly it works if you use the external IP of the host so the issue is
# not the host it is listening on (it claims for that to be 0.0.0.0).
# --network host is a cheap escape hatch for this.
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
function cleanup() {
docker rm -f oracle
}
trap cleanup EXIT
testConnection() {
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
}
until testConnection
do
echo "Could not connect to Oracle, trying again..."
sleep 1
done
# Actually run some tests
# Note: Oracle tests all run sequentially because they all access the same Oracle instance,
# and we sometimes observe transient connection issues when running tests in parallel.
bazel test \
--config=oracle \
--test_strategy=exclusive \
//ledger-service/http-json-oracle/... \
//triggers/service:test-oracle \
//ledger/participant-integration-api:participant-integration-api-tests-oracle \
//ledger/ledger-on-sql:conformance-test-oracle
oracle_logs=$(Build.StagingDirectory)/oracle-logs
mkdir $oracle_logs
for path in $(docker exec oracle bash -c 'find /opt/oracle/diag/rdbms/ -type f'); do
# $path starts with a slash
mkdir -p $(dirname ${oracle_logs}${path})
docker exec oracle bash -c "cat $path" > ${oracle_logs}${path}
done
env:
DOCKER_LOGIN: $(DOCKER_LOGIN)
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
displayName: 'Build'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish the bazel test logs'
inputs:
pathtoPublish: 'bazel-testlogs/'
artifactName: 'Test logs Oracle'
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish Oracle image logs'
inputs:
pathtoPublish: '$(Build.StagingDirectory)/oracle-logs'
artifactName: 'Oracle image logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: platform_independence_test
condition: and(succeeded(),
eq(dependencies.check_for_release.outputs['out.is_release'], 'false'))
dependsOn:
- Windows
- Linux
- macOS
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-linux
targetPath: $(Build.StagingDirectory)/platform-independence/linux/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-windows
targetPath: $(Build.StagingDirectory)/platform-independence/windows/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-macos
targetPath: $(Build.StagingDirectory)/platform-independence/macos/
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
DIR1=$(mktemp -d)
DIR2=$(mktemp -d)
DIR3=$(mktemp -d)
trap "rm -rf $DIR1; rm -rf $DIR2; rm -rf $DIR3" EXIT
unzip -d $DIR1 $(Build.StagingDirectory)/platform-independence/linux/platform-independence.dar
unzip -d $DIR2 $(Build.StagingDirectory)/platform-independence/windows/platform-independence.dar
unzip -d $DIR3 $(Build.StagingDirectory)/platform-independence/macos/platform-independence.dar
# hie/hi files may differ.
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR2
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR3
displayName: 'Compare platform-independence dars of different platforms.'
- job: compatibility_ts_libs
dependsOn:
- da_ghc_lib
- check_for_release
condition: and(succeeded(),
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility_ts_libs.yml
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_linux
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
# For main and PRs targeting main, we simply check against the most recent stable tag.
- job: compatibility_stable_protobuf
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/check-protobuf-stability.sh
- template: tell-slack-failed.yml
- job: check_for_release
dependsOn:
- git_sha
variables:
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash-lib)
./release.sh check
changes_release_files() {
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
[ "LATEST" = "$changed" ]
}
changes_one_line_in_latest() {
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
add_one="1_0"
change_one="1_1"
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
}
added_line() {
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
}
if changes_release_files; then
if changes_one_line_in_latest; then
setvar is_release true
setvar trigger_sha $(branch_sha)
setvar release_sha "$(added_line | awk '{print $1}')"
RELEASE_TAG="$(added_line | awk '{print $2}')"
setvar release_tag "$RELEASE_TAG"
if [ $(added_line | awk '{print $3}') == "SPLIT_RELEASE" ]; then
setvar split_release_process true
else
setvar split_release_process false
fi
else
echo "Release commit should only add one version."
exit 1
fi
else
RELEASE_TAG="0.0.0"
setvar is_release false
fi
# This is the last snapshot that does not support Scala 2.13.
CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')"
if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then
setvar scala_2_13 true
else
setvar scala_2_13 false
fi
name: out
- job: collect_build_data
condition: always()
dependsOn:
- Linux
- Linux_oracle
- macOS
- Windows
- release
- git_sha
- compatibility_linux
- compatibility_stable_protobuf
- check_for_release
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
variables:
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
Linux.status: $[ dependencies.Linux.result ]
Linux_oracle.start: $[ dependencies.Linux_oracle.outputs['start.time'] ]
Linux_oracle.machine: $[ dependencies.Linux_oracle.outputs['start.machine'] ]
Linux_oracle.end: $[ dependencies.Linux_oracle.outputs['end.time'] ]
Linux_oracle.status: $[ dependencies.Linux_oracle.result ]
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
macOS.status: $[ dependencies.macOS.result ]
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
release.status: $[ dependencies.release.result ]
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
main_sha: $[ dependencies.git_sha.outputs['out.main'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
# Using expression syntax so we get an empty string if not set, rather
# than the raw $(VarName) string. Expression syntax works on the
# variables key, but not on the env one, so we need an extra indirection.
# Note: These Azure variables are only set for PR builds.
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
steps:
- template: bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
REPORT=$(mktemp)
cat >$REPORT <<END
{"jobs": {"Linux": {"start": "$(Linux.start)",
"machine": "$(Linux.machine)",
"end": "$(Linux.end)",
"status": "$(Linux.status)"},
"Linux_oracle": {"start": "$(Linux_oracle.start)",
"machine": "$(Linux_oracle.machine)",
"end": "$(Linux_oracle.end)",
"status": "$(Linux_oracle.status)"},
"macOS": {"start": "$(macOS.start)",
"machine": "$(macOS.machine)",
"end": "$(macOS.end)",
"status": "$(macOS.status)"},
"Windows": {"start": "$(Windows.start)",
"machine": "$(Windows.machine)",
"end": "$(Windows.end)",
"status": "$(Windows.status)"},
"release": {"start": "$(release.start)",
"machine": "$(release.machine)",
"end": "$(release.end)",
"status": "$(release.status)"},
"compatibility_linux": {"start": "$(compatibility_linux.start)",
"machine": "$(compatibility_linux.machine)",
"end": "$(compatibility_linux.end)",
"status": "$(compatibility_linux.status)"}},
"id": "$(Build.BuildId)",
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
"name": "$(Build.DefinitionName)",
"version": "$(Build.DefinitionVersion)",
"queued_by": "$(Build.QueuedBy)",
"reason": "$(Build.Reason)",
"branch": "$(Build.SourceBranch)",
"merge_commit": "$(Build.SourceVersion)",
"branch_commit": "$(branch_sha)",
"main_commit": "$(main_sha)",
"fork_point_commit": "$(fork_sha)",
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
"is_fork": "$(System.PullRequest.IsFork)",
"pr": "$PR_NUM",
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
"pr_source_branch": "$PR_BRANCH",
"is_release": $(is_release)}
END
# Test above JSON is well formed
cat $REPORT | jq '.'
REPORT_GZ=$(mktemp)
cat $REPORT | gzip -9 > $REPORT_GZ
# Application credentials will not be set for forks. We give up on
# tracking those for now. "Not set" in Azure world means set to the
# expression Azure would otherwise substitute, i.e. the literal value
# of the string in the `env:` block below.
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
gcs "$GCRED" cp "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
else
echo "Could not save build data: no credentials."
fi
# Linux, macOS and Windows are always required and should always
# succeed.
#
# release only run on releases and is skipped otherwise.
if [[ !("$(Linux.status)" == "Succeeded" || "$(Linux.status)" == "SucceededWithIssues")
|| "$(Linux_oracle.status)" != "Succeeded"
|| !("$(macOS.status)" == "Succeeded" || "$(macOS.status)" == "SucceededWithIssues")
|| "$(Windows.status)" != "Succeeded"
|| ("$(is_release)" == "false" && "$(compatibility_linux.status)" != "Succeeded")
|| "$(release.status)" == "Canceled" ]]; then
exit 1
fi
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
# Commit message is always set
COMMIT_MSG: $(Build.SourceVersionMessage)
# Because these variables are always set (in the variables block),
# hopefully these should be set as expected (i.e. either correct
# value or empty string, but not $(Azure.Variable.Name)).
PR_NUM: $(pr.num)
PR_BRANCH: $(pr.branch)
Reference in New Issue View Git Blame Copy Permalink