digital-asset/daml
1
1
Fork 0
mirror of https://github.com/digital-asset/daml.git synced 2024-09-20 01:07:18 +03:00
Code Issues Projects Releases Wiki Activity
5d0d5027ff
daml/ci/cron/daily-compat.yml
Gary Verhaegen 5d0d5027ff
cleanup: remove unused credentials (#18814)
2024-03-22 16:43:39 +01:00

313 lines
11 KiB
YAML
Raw Blame History

# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Do not run on PRs
pr: none
# Do not run on merge to main
trigger: none
# Do run on a schedule (daily)
#
# Note: machines are killed every day at 4AM UTC, so we need to either:
# - run sufficiently before that that this doesn't get killed, or
# - run sufficiently after that that machines are initialized.
#
# Targeting 6AM UTC seems to fit that.
schedules:
- cron: "0 6 * * *"
displayName: daily checks and reporting
branches:
include:
- main
- main-2.x
always: true
jobs:
- job: compatibility_ts_libs
timeoutInMinutes: 60
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
- template: ../clean-up.yml
- template: ../compatibility_ts_libs.yml
- job: compatibility_linux
dependsOn: compatibility_ts_libs
timeoutInMinutes: 720
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
- template: ../clean-up.yml
- template: ../compatibility.yml
- job: compatibility_windows
dependsOn: compatibility_ts_libs
timeoutInMinutes: 720
pool:
name: windows-pool
demands: assignment -equals default
steps:
- checkout: self
- template: ../compatibility-windows.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)'
artifactName: 'Bazel Compatibility Logs'
- job: perf_speedy
timeoutInMinutes: 120
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
# TODO(https://github.com/digital-asset/daml/issues/18222): re-enable once
# the benchmark has been ported to Daml 3.
condition: False
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel for root workspace'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: ../bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
cd sdk
eval "$(dev-env/bin/dade assist)"
source $(bash_lib)
OUT="$(Build.StagingDirectory)/perf-results-speedy.json"
START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
tmp=$(mktemp -d)/out.json
bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $tmp >&2
perf=$(printf '%.2f' $(cat $tmp | jq '.[0].primaryMetric.score'))
if [ "" = "$perf" ]; then exit 1; fi
sha=$(git rev-parse HEAD)
echo '{"current-perf": '$perf', "current-sha": "'$sha'"}' > "$OUT"
cat "$OUT"
gcs "$GCRED" cp "$OUT" gs://daml-data/perf/speedy/$START.json
setvar speedy_perf "$perf ms/op"
displayName: measure perf
name: out
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- task: PublishPipelineArtifact@0
inputs:
targetPath: $(Build.StagingDirectory)/perf-results-speedy.json
artifactName: perf-speedy
- job: check_releases
timeoutInMinutes: 600
pool:
name: ubuntu_20_04
demands: assignment -equals default
condition: eq(variables['Build.SourceBranchName'], 'main')
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: ../bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(sdk/dev-env/bin/dade assist)"
source $(bash_lib)
export AUTH="$(get_gh_auth_header)"
wrap_gcloud "$GCRED" "ci/cron/check-releases.sh"
displayName: check releases
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: ../blackduck.yml
- job: run_notices_pr_build
timeoutInMinutes: 60
dependsOn: ["blackduck_scan"]
condition: and(succeeded(),
or(eq(variables['Build.SourceBranchName'], 'main'),
eq(variables['Build.SourceBranchName'], 'main-2.x')))
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
variables:
need_to_build: $[ dependencies.blackduck_scan.outputs['out.need_to_build'] ]
steps:
- bash: |
cd sdk
eval "$(./dev-env/bin/dade-assist)"
if [ "$(need_to_build)" == "true" ]; then
branch="notices-update-$(Build.BuildId)"
az extension add --name azure-devops
trap "az devops logout" EXIT
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
az pipelines build queue --branch "$branch" \
--definition-name "PRs" \
--org "https://dev.azure.com/digitalasset" \
--project daml
fi
- job: update_canton
condition: not(eq(variables['Build.SourceBranchName'], 'not-main'))
timeoutInMinutes: 60
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
persistCredentials: true
- template: ../bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
cd sdk
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
git fetch
git checkout origin/$(Build.SourceBranchName)
version=$(ci/refresh-canton.sh)
branch="main-canton-update-$version"
if git diff --exit-code origin/$(Build.SourceBranchName) -- canton build.sh >/dev/null; then
echo "Already up-to-date with latest Canton source & snapshot."
else
if [ "main" = "$(Build.SourceBranchName)" ]; then
git add build.sh canton
open_pr "$branch" "update canton to $version" "tell-slack: canton" "" "$(Build.SourceBranchName)"
az extension add --name azure-devops
trap "az devops logout" EXIT
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
az pipelines build queue --branch "$branch" \
--definition-name "PRs" \
--org "https://dev.azure.com/digitalasset" \
--project daml
else
echo "Would open PR if this were running on main."
fi
fi
env:
GITHUB_TOKEN: $(CANTON_READONLY_TOKEN)
- job: compat_versions_pr
condition: or(eq(variables['Build.SourceBranchName'], 'main'),
eq(variables['Build.SourceBranchName'], 'main-2.x'))
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
persistCredentials: true
- bash: ci/dev-env-install.sh
- template: ../bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
cd sdk
eval "$(./dev-env/bin/dade-assist)"
## refresh tags, in case someone deleted one
git fetch --prune --prune-tags
## get latest version, in case it's already up to date
git fetch origin
git -c user.name="Azure Pipelines Daml Build" \
-c user.email="support@digitalasset.com" \
merge origin/main
source $(bash_lib)
cp .bazelrc compatibility/
compatibility/update-versions.sh
git add compatibility/versions.bzl compatibility/maven_install.json
if ! git diff --cached --quiet; then
BRANCH=update-compat-versions-$(date -I)-$(Build.SourceBranchName)
TITLE="update compat versions ($(date -I) $(Build.SourceBranchName))"
open_pr "$BRANCH" "$TITLE" "" "" "$(Build.SourceBranchName)"
trigger_azure $(System.AccessToken) PRs --branch $BRANCH
trigger_azure $(System.AccessToken) digital-asset.daml-daily-compat --branch $BRANCH
else
echo "No changes"
fi
- job: report
dependsOn: [compatibility_ts_libs, compatibility_linux, compatibility_windows,
perf_speedy, check_releases,
blackduck_scan, run_notices_pr_build, update_canton,
compat_versions_pr]
condition: and(succeededOrFailed(),
or(eq(variables['Build.SourceBranchName'], 'main'),
eq(variables['Build.SourceBranchName'], 'main-2.x')))
pool:
name: ubuntu_20_04
demands: assignment -equals default
variables:
compatibility_ts_libs: $[ dependencies.compatibility_ts_libs.result ]
compatibility_linux: $[ dependencies.compatibility_linux.result ]
compatibility_windows: $[ dependencies.compatibility_windows.result ]
perf_speedy: $[ dependencies.perf_speedy.result ]
speedy_perf: $[ dependencies.perf_speedy.outputs['out.speedy_perf'] ]
check_releases: $[ dependencies.check_releases.result ]
blackduck_scan: $[ dependencies.blackduck_scan.result ]
run_notices_pr_build: $[ dependencies.run_notices_pr_build.result ]
update_canton: $[ dependencies.update_canton.result ]
compat_versions_pr: $[ dependencies.compat_versions_pr.result ]
steps:
- template: ../bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
cd sdk
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
COMMIT_TITLE="$(escape_slack "$(git log --pretty=format:%s -n1)")"
COMMIT_LINK="<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|${COMMIT_TITLE}>"
if [[ "$(compatibility_ts_libs)" == "Succeeded"
&& "$(compatibility_linux)" == "Succeeded"
&& "$(compatibility_windows)" == "Succeeded"
&& ("$(perf_speedy)" == "Succeeded" || "$(perf_speedy)" == "Skipped")
&& "$(check_releases)" == "Succeeded"
&& "$(update_canton)" == "Succeeded"
&& ("$(blackduck_scan)" == "Succeeded" || "$(blackduck_scan)" == "Skipped")
&& ("$(run_notices_pr_build)" == "Succeeded" || "$(run_notices_pr_build)" == "Skipped")
&& "$(compat_versions_pr)" == "Succeeded"
]]; then
MSG="Daily tests passed ($(Build.SourceBranchName)): $COMMIT_LINK"
if [ "$(perf_speedy)" == "Skipped" ]; then
REPORT="Perf test was skipped."
else
REPORT='```speedy_perf: $(speedy_perf)```'
fi
tell_slack "$(echo -e "$MSG\n$REPORT\n")" "$(Slack.ci-failures-daml)"
else
tell_slack "Daily tests failed ($(Build.SourceBranchName)): $COMMIT_LINK." "$(Slack.ci-failures-daml)"
fi
Reference in New Issue View Git Blame Copy Permalink