mirror of
https://github.com/digital-asset/daml.git
synced 2024-11-10 10:46:11 +03:00
361 lines
12 KiB
YAML
361 lines
12 KiB
YAML
# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
jobs:
|
||
|
||
- job: git_sha
|
||
pool:
|
||
name: 'ubuntu_20_04'
|
||
demands: assignment -equals default
|
||
steps:
|
||
- template: bash-lib.yml
|
||
parameters:
|
||
var_name: bash-lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
source $(bash-lib)
|
||
if [ "$(Build.Reason)" == "PullRequest" ]; then
|
||
setvar branch "$(git rev-parse HEAD^2)"
|
||
setvar main "$(git rev-parse HEAD^1)"
|
||
setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
|
||
else
|
||
setvar branch "$(git rev-parse HEAD)"
|
||
setvar main "$(git rev-parse HEAD^1)"
|
||
setvar fork_point "$(git rev-parse HEAD^1)"
|
||
fi
|
||
name: out
|
||
|
||
- job: Linux
|
||
dependsOn:
|
||
- check_for_release
|
||
variables:
|
||
- name: release_sha
|
||
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
||
- name: release_tag
|
||
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
||
- name: trigger_sha
|
||
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
||
- name: is_release
|
||
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
||
- template: job-variables.yml
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: 'ubuntu_20_04'
|
||
demands: assignment -equals default
|
||
steps:
|
||
- template: report-start.yml
|
||
- checkout: self
|
||
- bash: |
|
||
set -euo pipefail
|
||
git checkout $(release_sha)
|
||
name: checkout_release
|
||
condition: and(succeeded(), eq(variables.is_release, 'true'))
|
||
- template: clean-up.yml
|
||
- template: build-unix.yml
|
||
parameters:
|
||
release_tag: $(release_tag)
|
||
name: 'linux'
|
||
is_release: variables.is_release
|
||
- template: upload-bazel-metrics.yml
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
bazel build //release:release
|
||
./bazel-bin/release/release --release-dir "$(mktemp -d)"
|
||
condition: and(succeeded(), ne(variables['is_release'], 'true'))
|
||
- template: tell-slack-failed.yml
|
||
parameters:
|
||
trigger_sha: '$(trigger_sha)'
|
||
- template: report-end.yml
|
||
|
||
- template: macOS.yml
|
||
parameters:
|
||
job_name: macOS
|
||
name: macos
|
||
assignment: default
|
||
|
||
- template: macOS.yml
|
||
parameters:
|
||
job_name: m1
|
||
name: m1
|
||
assignment: m1-builds
|
||
|
||
- job: Windows
|
||
dependsOn:
|
||
- check_for_release
|
||
variables:
|
||
- name: release_sha
|
||
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
||
- name: release_tag
|
||
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
||
- name: trigger_sha
|
||
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
||
- name: is_release
|
||
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
||
- name: is_split_release
|
||
value: $[ dependencies.check_for_release.outputs['out.split_release_process'] ]
|
||
- name: skip_tests
|
||
value: $[ and(eq(variables.is_release, 'true'),
|
||
eq(variables['Build.SourceBranchName'], 'main')) ]
|
||
- template: job-variables.yml
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: 'windows-pool'
|
||
demands: assignment -equals default
|
||
steps:
|
||
- template: report-start.yml
|
||
- checkout: self
|
||
- bash: |
|
||
set -euo pipefail
|
||
|
||
for f in $(find /d/a/SourceRootMapping -type f); do
|
||
echo "-----"
|
||
echo $f
|
||
echo "-----"
|
||
cat $f
|
||
echo "-----"
|
||
done
|
||
name: workdirs
|
||
- bash: |
|
||
set -euo pipefail
|
||
git checkout $(release_sha)
|
||
name: checkout_release
|
||
condition: and(succeeded(), eq(variables.is_release, 'true'))
|
||
- template: build-windows.yml
|
||
parameters:
|
||
release_tag: $(release_tag)
|
||
# Azure pipeline’s variable and parameter expansion is utter garbage.
|
||
# For whatever reason `env` values only seem to be able to use macro syntax
|
||
# and not runtime expression. is_release however is a runtime variable
|
||
# so template conditions won’t work. Therefore we define the variable here
|
||
# with a runtime expression, set the parameter to the (unexpanded) string "$(skip_tests)"
|
||
# and then splice that in via a template parameter.
|
||
skip_tests: $(skip_tests)
|
||
is_release: variables.is_release
|
||
is_split_release: $(is_split_release)
|
||
- template: upload-bazel-metrics.yml
|
||
- task: PublishBuildArtifacts@1
|
||
condition: succeededOrFailed()
|
||
inputs:
|
||
pathtoPublish: '$(Build.StagingDirectory)/logs'
|
||
artifactName: 'Bazel Logs'
|
||
- template: tell-slack-failed.yml
|
||
parameters:
|
||
trigger_sha: '$(trigger_sha)'
|
||
- template: report-end.yml
|
||
|
||
- job: Linux_oracle
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: 'ubuntu_20_04'
|
||
demands: assignment -equals default
|
||
steps:
|
||
- template: report-start.yml
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- template: clean-up.yml
|
||
- bash: |
|
||
source dev-env/lib/ensure-nix
|
||
ci/dev-env-push.py
|
||
displayName: 'Push Developer Environment build results'
|
||
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
|
||
env:
|
||
# to upload to the Nix cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
|
||
IMAGE=$(cat ci/oracle_image)
|
||
docker pull $IMAGE
|
||
# Cleanup stray containers that might still be running from
|
||
# another build that didn’t get shut down cleanly.
|
||
docker rm -f oracle || true
|
||
# Oracle does not like if you connect to it via localhost if it’s running in the container.
|
||
# Interestingly it works if you use the external IP of the host so the issue is
|
||
# not the host it is listening on (it claims for that to be 0.0.0.0).
|
||
# --network host is a cheap escape hatch for this.
|
||
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||
function cleanup() {
|
||
docker rm -f oracle
|
||
}
|
||
trap cleanup EXIT
|
||
testConnection() {
|
||
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
|
||
}
|
||
until testConnection
|
||
do
|
||
echo "Could not connect to Oracle, trying again..."
|
||
sleep 1
|
||
done
|
||
# Actually run some tests
|
||
# Note: Oracle tests all run sequentially because they all access the same Oracle instance,
|
||
# and we sometimes observe transient connection issues when running tests in parallel.
|
||
bazel test \
|
||
--config=oracle \
|
||
--test_strategy=exclusive \
|
||
--test_tag_filters=+oracle \
|
||
//...
|
||
|
||
oracle_logs=$(Build.StagingDirectory)/oracle-logs
|
||
mkdir $oracle_logs
|
||
for path in $(docker exec oracle bash -c 'find /opt/oracle/diag/rdbms/ -type f'); do
|
||
# $path starts with a slash
|
||
mkdir -p $(dirname ${oracle_logs}${path})
|
||
docker exec oracle bash -c "cat $path" > ${oracle_logs}${path}
|
||
done
|
||
env:
|
||
DOCKER_LOGIN: $(DOCKER_LOGIN)
|
||
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
|
||
displayName: 'Build'
|
||
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
|
||
|
||
- task: PublishBuildArtifacts@1
|
||
condition: failed()
|
||
displayName: 'Publish the bazel test logs'
|
||
inputs:
|
||
pathtoPublish: 'bazel-testlogs/'
|
||
artifactName: 'Test logs Oracle'
|
||
|
||
- task: PublishBuildArtifacts@1
|
||
condition: failed()
|
||
displayName: 'Publish Oracle image logs'
|
||
inputs:
|
||
pathtoPublish: '$(Build.StagingDirectory)/oracle-logs'
|
||
artifactName: 'Oracle image logs'
|
||
|
||
- template: tell-slack-failed.yml
|
||
parameters:
|
||
trigger_sha: '$(trigger_sha)'
|
||
- template: report-end.yml
|
||
|
||
- job: platform_independence_test
|
||
condition: and(succeeded(),
|
||
eq(dependencies.check_for_release.outputs['out.is_release'], 'false'))
|
||
dependsOn:
|
||
- Windows
|
||
- Linux
|
||
- macOS
|
||
pool:
|
||
name: 'ubuntu_20_04'
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- task: DownloadPipelineArtifact@2
|
||
inputs:
|
||
artifactName: platform-independence-dar-linux
|
||
targetPath: $(Build.StagingDirectory)/platform-independence/linux/
|
||
- task: DownloadPipelineArtifact@2
|
||
inputs:
|
||
artifactName: platform-independence-dar-windows
|
||
targetPath: $(Build.StagingDirectory)/platform-independence/windows/
|
||
- task: DownloadPipelineArtifact@2
|
||
inputs:
|
||
artifactName: platform-independence-dar-macos
|
||
targetPath: $(Build.StagingDirectory)/platform-independence/macos/
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
|
||
DIR1=$(mktemp -d)
|
||
DIR2=$(mktemp -d)
|
||
DIR3=$(mktemp -d)
|
||
trap "rm -rf $DIR1; rm -rf $DIR2; rm -rf $DIR3" EXIT
|
||
|
||
unzip -d $DIR1 $(Build.StagingDirectory)/platform-independence/linux/platform-independence.dar
|
||
unzip -d $DIR2 $(Build.StagingDirectory)/platform-independence/windows/platform-independence.dar
|
||
unzip -d $DIR3 $(Build.StagingDirectory)/platform-independence/macos/platform-independence.dar
|
||
|
||
# hie/hi files may differ.
|
||
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR2
|
||
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR3
|
||
displayName: 'Compare platform-independence dars of different platforms.'
|
||
|
||
- job: compatibility_ts_libs
|
||
dependsOn:
|
||
- check_for_release
|
||
condition: and(succeeded(),
|
||
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- template: report-start.yml
|
||
- checkout: self
|
||
- template: clean-up.yml
|
||
- template: compatibility_ts_libs.yml
|
||
- template: tell-slack-failed.yml
|
||
- template: report-end.yml
|
||
|
||
- job: compatibility_linux
|
||
dependsOn:
|
||
- check_for_release
|
||
- compatibility_ts_libs
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- template: report-start.yml
|
||
- checkout: self
|
||
- template: clean-up.yml
|
||
- template: compatibility.yml
|
||
parameters:
|
||
test_flags: '--quick'
|
||
- template: tell-slack-failed.yml
|
||
- template: report-end.yml
|
||
|
||
# For main and PRs targeting main, we simply check against the most recent stable tag.
|
||
- job: compatibility_stable_protobuf
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
fetchTags: true
|
||
- bash: ci/check-protobuf-stability.sh
|
||
- template: tell-slack-failed.yml
|
||
|
||
- job: collect_build_data_failed
|
||
condition: failed()
|
||
dependsOn:
|
||
- Linux
|
||
- Linux_oracle
|
||
- macOS
|
||
- Windows
|
||
- release
|
||
- git_sha
|
||
- compatibility_linux
|
||
- compatibility_stable_protobuf
|
||
- check_for_release
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
|
||
- job: collect_build_data
|
||
condition: always()
|
||
dependsOn:
|
||
- collect_build_data_failed
|
||
variables:
|
||
failed_result: $[ dependencies.collect_build_data_failed.result ]
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- bash: "exit 1"
|
||
# Since 'collect_build_data_failed' only runs when 'failed()', if it was
|
||
# successful that means at least one of its dependencies failed, so we set
|
||
# the result of 'collect_build_data' to 'Failed' with 'exit 1'
|
||
condition: eq(variables.failed_result, 'Succeeded')
|