mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
56018b5d6e
* do a proper canton 3.x code drop in the canton-3x directory * copy the code from canton3 * address Gary's comments * fix canton-3x
397 lines
14 KiB
YAML
397 lines
14 KiB
YAML
# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Do not run on PRs
|
|
pr: none
|
|
|
|
# Do not run on merge to main
|
|
trigger: none
|
|
|
|
# Do run on a schedule (daily)
|
|
#
|
|
# Note: machines are killed every day at 4AM UTC, so we need to either:
|
|
# - run sufficiently before that that this doesn't get killed, or
|
|
# - run sufficiently after that that machines are initialized.
|
|
#
|
|
# Targeting 6AM UTC seems to fit that.
|
|
schedules:
|
|
- cron: "0 6 * * *"
|
|
displayName: daily checks and reporting
|
|
branches:
|
|
include:
|
|
- main
|
|
always: true
|
|
|
|
jobs:
|
|
- template: ../build-local-canton.yml
|
|
- job: compatibility_ts_libs
|
|
timeoutInMinutes: 60
|
|
pool:
|
|
name: ubuntu_20_04
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- template: ../clean-up.yml
|
|
- template: ../compatibility_ts_libs.yml
|
|
|
|
- job: compatibility
|
|
dependsOn:
|
|
- compatibility_ts_libs
|
|
- build_canton
|
|
timeoutInMinutes: 720
|
|
strategy:
|
|
matrix:
|
|
linux:
|
|
pool: ubuntu_20_04
|
|
# macos:
|
|
# pool: macOS-pool
|
|
pool:
|
|
name: $(pool)
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- template: ../get-local-canton.yml
|
|
- ${{ if eq(variables['pool'], 'macos-pool') }}:
|
|
- template: ../clear-shared-segments-macos.yml
|
|
- template: ../clean-up.yml
|
|
- template: ../compatibility.yml
|
|
|
|
- job: compatibility_windows
|
|
dependsOn:
|
|
- compatibility_ts_libs
|
|
- build_canton
|
|
timeoutInMinutes: 720
|
|
pool:
|
|
name: windows-pool
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- template: ../get-local-canton.yml
|
|
- template: ../compatibility-windows.yml
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Bazel Compatibility Logs'
|
|
|
|
- job: perf_speedy
|
|
timeoutInMinutes: 120
|
|
pool:
|
|
name: "ubuntu_20_04"
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- bash: ci/dev-env-install.sh
|
|
displayName: 'Build/Install the Developer Environment'
|
|
- bash: ci/configure-bazel.sh
|
|
displayName: 'Configure Bazel for root workspace'
|
|
env:
|
|
IS_FORK: $(System.PullRequest.IsFork)
|
|
# to upload to the bazel cache
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- template: ../bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(dev-env/bin/dade assist)"
|
|
source $(bash_lib)
|
|
|
|
OUT="$(Build.StagingDirectory)/perf-results-speedy.json"
|
|
|
|
START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
|
|
|
tmp=$(mktemp -d)/out.json
|
|
bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $tmp >&2
|
|
perf=$(printf '%.2f' $(cat $tmp | jq '.[0].primaryMetric.score'))
|
|
if [ "" = "$perf" ]; then exit 1; fi
|
|
sha=$(git rev-parse HEAD)
|
|
echo '{"current-perf": '$perf', "current-sha": "'$sha'"}' > "$OUT"
|
|
cat "$OUT"
|
|
|
|
gcs "$GCRED" cp "$OUT" gs://daml-data/perf/speedy/$START.json
|
|
|
|
setvar speedy_perf "$perf ms/op"
|
|
|
|
displayName: measure perf
|
|
name: out
|
|
env:
|
|
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- task: PublishPipelineArtifact@0
|
|
inputs:
|
|
targetPath: $(Build.StagingDirectory)/perf-results-speedy.json
|
|
artifactName: perf-speedy
|
|
|
|
- job: check_releases
|
|
timeoutInMinutes: 600
|
|
pool:
|
|
name: ubuntu_20_04
|
|
demands: assignment -equals default
|
|
condition: eq(variables['Build.SourceBranchName'], 'main')
|
|
steps:
|
|
- checkout: self
|
|
- bash: ci/dev-env-install.sh
|
|
displayName: 'Build/Install the Developer Environment'
|
|
- template: ../bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(dev-env/bin/dade assist)"
|
|
source $(bash_lib)
|
|
|
|
export AUTH="$(get_gh_auth_header)"
|
|
|
|
wrap_gcloud "$GCRED" "ci/cron/check-releases.sh"
|
|
displayName: check releases
|
|
env:
|
|
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
|
|
- template: ../blackduck.yml
|
|
|
|
- job: run_notices_pr_build
|
|
timeoutInMinutes: 60
|
|
dependsOn: ["blackduck_scan"]
|
|
condition: and(succeeded(),
|
|
eq(variables['Build.SourceBranchName'], 'main'))
|
|
pool:
|
|
name: 'ubuntu_20_04'
|
|
demands: assignment -equals default
|
|
variables:
|
|
need_to_build: $[ dependencies.blackduck_scan.outputs['out.need_to_build'] ]
|
|
steps:
|
|
- bash: |
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
if [ "$(need_to_build)" == "true" ]; then
|
|
branch="notices-update-$(Build.BuildId)"
|
|
az extension add --name azure-devops
|
|
trap "az devops logout" EXIT
|
|
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
|
az pipelines build queue --branch "$branch" \
|
|
--definition-name "PRs" \
|
|
--org "https://dev.azure.com/digitalasset" \
|
|
--project daml
|
|
fi
|
|
|
|
- job: update_canton
|
|
timeoutInMinutes: 60
|
|
pool:
|
|
name: ubuntu_20_04
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- template: ../bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
source $(bash_lib)
|
|
|
|
git fetch
|
|
git checkout origin/main
|
|
|
|
get_canton_version_for_major_version() {
|
|
major=$1
|
|
curl -u $AUTH \
|
|
--fail \
|
|
--location \
|
|
--silent \
|
|
https://digitalasset.jfrog.io/artifactory/api/storage/assembly/canton \
|
|
| jq -r '.children[].uri' \
|
|
| sed -e 's/^\///' \
|
|
| grep -P "^${major}\.\d+\.\d+" \
|
|
| sort -V \
|
|
| tail -1
|
|
}
|
|
|
|
canton_version_to_commitish() {
|
|
canton_version=$1
|
|
STABLE_REGEX="\d+\.\d+\.\d+"
|
|
SNAPSHOT_REGEX="^${STABLE_REGEX}-snapshot\.\d{8}\.\d+(\.\d+)?\.v[0-9a-f]{8}$"
|
|
if (echo "$1" | grep -q -P "${SNAPSHOT_REGEX}"); then
|
|
# Extracting commit (short) sha from snapshot version
|
|
echo "${canton_version: -8}"
|
|
else
|
|
# relying on stable version tag
|
|
echo "v$canton_version"
|
|
fi
|
|
}
|
|
|
|
canton2_version=$(get_canton_version_for_major_version 2)
|
|
canton3_version=$(get_canton_version_for_major_version 3)
|
|
|
|
### update binaries ###
|
|
|
|
ee_url="https://digitalasset.jfrog.io/artifactory/assembly/canton/$canton2_version/canton-enterprise-$canton2_version.tar.gz"
|
|
ee_tmp=$(mktemp)
|
|
curl -u $AUTH --fail --location --silent "$ee_url" > $ee_tmp
|
|
ee_sha=$(sha256sum $ee_tmp | awk '{print $1}')
|
|
ee_target_url=https://digitalasset.jfrog.io/artifactory/assembly/daml/canton-backup/$canton2_version/$ee_sha/canton-enterprise-$canton2_version.tar.gz
|
|
if ! curl -u $AUTH -f -I $ee_target_url; then
|
|
curl -u $AUTH \
|
|
-f \
|
|
-X PUT \
|
|
-H "X-Checksum-MD5: $(md5sum $ee_tmp | awk '{print $1}')" \
|
|
-H "X-Checksum-SHA1: $(sha1sum $ee_tmp | awk '{print $1}')" \
|
|
-H "X-Checksum-SHA256: $(sha256sum $ee_tmp | awk '{print $1}')" \
|
|
-T $ee_tmp \
|
|
$ee_target_url
|
|
fi
|
|
|
|
sed -i 's|SKIP_DEV_CANTON_TESTS=.*|SKIP_DEV_CANTON_TESTS=false|' build.sh
|
|
sed -i "s|CANTON_ENTERPRISE_VERSION=.*|CANTON_ENTERPRISE_VERSION=$canton2_version|" test-common/canton/BUILD.bazel
|
|
sed -i "s|CANTON_ENTERPRISE_SHA=.*|CANTON_ENTERPRISE_SHA=$ee_sha|" test-common/canton/BUILD.bazel
|
|
sed -i "s|CANTON_ENTERPRISE_URL=.*|CANTON_ENTERPRISE_URL=$ee_target_url|" test-common/canton/BUILD.bazel
|
|
|
|
rm -f arbitrary_canton_sha
|
|
|
|
### code drop ###
|
|
|
|
copy_canton_code() {
|
|
canton_version=$1
|
|
canton_dir=$2
|
|
|
|
commitish=$(canton_version_to_commitish $canton_version)
|
|
tmp=$(mktemp -d)
|
|
trap "rm -rf ${tmp}" EXIT
|
|
|
|
git clone https://$GITHUB_TOKEN@github.com/DACH-NY/canton $tmp
|
|
git -C $tmp checkout $commitish
|
|
for path in community daml-common-staging README.md; do
|
|
src=$tmp/$path
|
|
dst=$canton_dir/$path
|
|
rm -rf $dst
|
|
mkdir -p $(dirname $dst)
|
|
cp -rf $src $dst
|
|
git add $dst
|
|
done
|
|
}
|
|
|
|
copy_canton_code $canton2_version canton
|
|
copy_canton_code $canton3_version canton-3x
|
|
|
|
### create PR ###
|
|
|
|
commit_date=$(git -C $tmp log -n1 --format=%cd --date=format:%Y%m%d HEAD)
|
|
number_of_commits=$(git -C $tmp rev-list --count HEAD)
|
|
commit_sha_8=$(git -C $tmp log -n1 --format=%h --abbrev=8 HEAD)
|
|
canton_tag=$commit_date.$number_of_commits.0.v$commit_sha_8
|
|
branch="canton-update-$canton_tag-$canton2_version-$canton3_version"
|
|
|
|
if git diff --exit-code origin/main -- canton build.sh arbitrary_canton_sha test-common/canton/BUILD.bazel >/dev/null; then
|
|
echo "Already up-to-date with latest Canton source & snapshot."
|
|
else
|
|
if [ "main" = "$(Build.SourceBranchName)" ]; then
|
|
git add build.sh test-common/canton/BUILD.bazel
|
|
git ls-files | grep arbitrary_canton_sha >&2 && git rm -f arbitrary_canton_sha
|
|
open_pr "$branch" "update canton to $canton_tag/$canton2_version/$canton3_version"
|
|
az extension add --name azure-devops
|
|
trap "az devops logout" EXIT
|
|
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
|
az pipelines build queue --branch "$branch" \
|
|
--definition-name "PRs" \
|
|
--org "https://dev.azure.com/digitalasset" \
|
|
--project daml
|
|
else
|
|
echo "Would open PR if this were running on main."
|
|
fi
|
|
fi
|
|
env:
|
|
GITHUB_TOKEN: $(CANTON_READONLY_TOKEN)
|
|
AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
|
|
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
|
|
- job: compat_versions_pr
|
|
condition: eq(variables['Build.SourceBranchName'], 'main')
|
|
pool:
|
|
name: ubuntu_20_04
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: ci/dev-env-install.sh
|
|
- template: ../bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
## refresh tags, in case someone deleted one
|
|
git fetch --prune --prune-tags
|
|
|
|
## get latest version, in case it's already up to date
|
|
git fetch origin
|
|
git merge origin/main
|
|
|
|
source $(bash_lib)
|
|
|
|
cp .bazelrc compatibility/
|
|
compatibility/update-versions.sh
|
|
git add compatibility/versions.bzl compatibility/maven_install.json
|
|
|
|
|
|
if ! git diff --cached --quiet; then
|
|
BRANCH=update-compat-versions-$(date -I)
|
|
TITLE="update compat versions ($(date -I))"
|
|
open_pr "$BRANCH" "$TITLE"
|
|
trigger_azure $(System.AccessToken) PRs --branch $BRANCH
|
|
trigger_azure $(System.AccessToken) digital-asset.daml-daily-compat --branch $BRANCH
|
|
else
|
|
echo "No changes"
|
|
fi
|
|
|
|
- job: report
|
|
dependsOn: [compatibility_ts_libs, compatibility, compatibility_windows,
|
|
perf_speedy, check_releases,
|
|
blackduck_scan, run_notices_pr_build, update_canton, compat_versions_pr]
|
|
condition: and(succeededOrFailed(),
|
|
eq(variables['Build.SourceBranchName'], 'main'))
|
|
pool:
|
|
name: ubuntu_20_04
|
|
demands: assignment -equals default
|
|
variables:
|
|
compatibility_ts_libs: $[ dependencies.compatibility_ts_libs.result ]
|
|
compatibility: $[ dependencies.compatibility.result ]
|
|
compatibility_windows: $[ dependencies.compatibility_windows.result ]
|
|
perf_speedy: $[ dependencies.perf_speedy.result ]
|
|
speedy_perf: $[ dependencies.perf_speedy.outputs['out.speedy_perf'] ]
|
|
check_releases: $[ dependencies.check_releases.result ]
|
|
blackduck_scan: $[ dependencies.blackduck_scan.result ]
|
|
run_notices_pr_build: $[ dependencies.run_notices_pr_build.result ]
|
|
update_canton: $[ dependencies.update_canton.result ]
|
|
compat_versions_pr: $[ dependencies.compat_versions_pr.result ]
|
|
steps:
|
|
- template: ../bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
source $(bash_lib)
|
|
|
|
COMMIT_TITLE="$(escape_slack "$(git log --pretty=format:%s -n1)")"
|
|
COMMIT_LINK="<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|${COMMIT_TITLE}>"
|
|
|
|
if [[ "$(compatibility_ts_libs)" == "Succeeded"
|
|
&& "$(compatibility)" == "Succeeded"
|
|
&& "$(compatibility_windows)" == "Succeeded"
|
|
&& "$(perf_speedy)" == "Succeeded"
|
|
&& "$(check_releases)" == "Succeeded"
|
|
&& "$(update_canton)" == "Succeeded"
|
|
&& ("$(blackduck_scan)" == "Succeeded" || "$(blackduck_scan)" == "Skipped")
|
|
&& ("$(run_notices_pr_build)" == "Succeeded" || "$(run_notices_pr_build)" == "Skipped")
|
|
&& "$(compat_versions_pr)" == "Succeeded"
|
|
]]; then
|
|
MSG="Daily tests passed: $COMMIT_LINK"
|
|
REPORT='```speedy_perf: $(speedy_perf)```
|
|
'
|
|
tell_slack "$(echo -e "$MSG\n$REPORT")" "$(Slack.ci-failures-daml)"
|
|
else
|
|
tell_slack "Daily tests failed: $COMMIT_LINK." "$(Slack.ci-failures-daml)"
|
|
fi
|