From ca976fe3d761ebff3b7adc45c92a7d7b5be5b5d9 Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Thu, 3 Feb 2022 20:05:35 +0100 Subject: [PATCH] nightly split releases (#12744) Because there's no reason not to. The only obstacle to automating the normal release process is that we need an explicit manual validation step for our audit log when creating a release, but split-releases are created in the Assemblty repo, so we can have the audit log over there. The diff is going to be very messy because there's a lot of stuff moving around. The only thing that is not just moving a job to a separate file is the `ci/daily-snapshot.yml` file, which is 100% new and is meant as a new Azure Pipelines entrypoint (which I will create after this gets approved). I have made a reasonable effort to create individual commits that simplify reviewing, but I expect it's still going to be kind of a mess. I'm open to opening separate PRs to ~bump my stats~ move one job at a time if that makes reviewing (and testing) easier. CHANGELOG_BEGIN CHANGELOG_END --- azure-pipelines.yml | 88 +--------------------------------- ci/build.yml | 67 -------------------------- ci/check-for-release-job.yml | 71 ++++++++++++++++++++++++++++ ci/daily-snapshot.yml | 43 +++++++++++++++++ ci/prs.yml | 2 + ci/split-release-job.yml | 91 ++++++++++++++++++++++++++++++++++++ 6 files changed, 209 insertions(+), 153 deletions(-) create mode 100644 ci/check-for-release-job.yml create mode 100644 ci/daily-snapshot.yml create mode 100644 ci/split-release-job.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5998027cecc..2e64ce384f8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,7 @@ pr: none jobs: - template: ci/build.yml +- template: ci/check-for-release-job.yml - job: release dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ] @@ -158,92 +159,7 @@ jobs: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml -- job: split_release - dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ] - condition: and(succeeded(), - eq(dependencies.check_for_release.outputs['out.is_release'], 'true'), - eq(dependencies.check_for_release.outputs['out.split_release_process'], 'true'), - eq(variables['Build.SourceBranchName'], 'main')) - pool: - name: 'ubuntu_20_04' - demands: assignment -equals default - variables: - release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] - release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ] - trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] - steps: - - checkout: self - persistCredentials: true - - bash: | - set -euo pipefail - git checkout $(release_sha) - name: checkout_release - - template: ci/bash-lib.yml - parameters: - var_name: bash-lib - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: linux-release - targetPath: $(Build.StagingDirectory)/release-artifacts - condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE'))) - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: macos-release - targetPath: $(Build.StagingDirectory)/release-artifacts - condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE'))) - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: windows-release - targetPath: $(Build.StagingDirectory)/release-artifacts - condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE'))) - - bash: | - set -euo pipefail - # Note: this gets dev-env from the release commit, not the trigger commit - eval "$(./dev-env/bin/dade-assist)" - KEY_FILE=$(mktemp) - GPG_DIR=$(mktemp -d) - cleanup() { - rm -rf $KEY_FILE $GPG_DIR - } - trap cleanup EXIT - echo "$GPG_KEY" | base64 -d > $KEY_FILE - gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE - # For now we only sign artifactory artifacts here and leave signing of artifacts - # published to GH to the assembly repo. - cd $(Build.StagingDirectory)/release-artifacts/artifactory - for f in *; do - gpg --homedir $GPG_DIR -ab $f - done - env: - GPG_KEY: $(gpg-code-signing) - - bash: | - set -eou pipefail - # Note: this gets dev-env from the release commit, not the trigger commit - eval "$(./dev-env/bin/dade-assist)" - mkdir -p $(Build.StagingDirectory)/split-release - ./ci/assembly-split-release-artifacts.sh $(release_tag) $(Build.StagingDirectory)/release-artifacts $(Build.StagingDirectory)/split-release - - bash: | - set -euo pipefail - # Note: this gets dev-env from the release commit, not the trigger commit - eval "$(./dev-env/bin/dade-assist)" - source $(bash-lib) - cd $(Build.StagingDirectory)/split-release/split-release - for f in "damlc-*" daml-libs/daml-script; do - gcs "$GCRED" cp -r "$f" "gs://daml-binaries/split-releases/$(release_tag)/" - done - name: gcs_for_canton - env: - GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) - - bash: | - set -euo pipefail - # Note: this gets dev-env from the release commit, not the trigger commit - eval "$(./dev-env/bin/dade-assist)" - ./ci/publish-artifactory.sh $(Build.StagingDirectory) $(release_tag) split - env: - AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD) - - template: ci/tell-slack-failed.yml - parameters: - trigger_sha: '$(trigger_sha)' +- template: ci/split-release-job.yml - job: compat_versions_pr dependsOn: diff --git a/ci/build.yml b/ci/build.yml index ce676256015..d4775684e3f 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -406,73 +406,6 @@ jobs: - bash: ci/check-protobuf-stability.sh - template: tell-slack-failed.yml -- job: check_for_release - dependsOn: - - git_sha - variables: - branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] - fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] - pool: - name: "ubuntu_20_04" - demands: assignment -equals default - steps: - - template: bash-lib.yml - parameters: - var_name: bash-lib - - bash: | - set -euo pipefail - eval "$(./dev-env/bin/dade-assist)" - source $(bash-lib) - - ./release.sh check - - changes_release_files() { - changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)" - [ "LATEST" = "$changed" ] - } - - changes_one_line_in_latest() { - changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')" - add_one="1_0" - change_one="1_1" - [[ "$add_one" == "$changed" || "$change_one" == "$changed" ]] - } - - added_line() { - echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)" - } - - if changes_release_files; then - if changes_one_line_in_latest; then - setvar is_release true - setvar trigger_sha $(branch_sha) - setvar release_sha "$(added_line | awk '{print $1}')" - RELEASE_TAG="$(added_line | awk '{print $2}')" - setvar release_tag "$RELEASE_TAG" - if [ "$(added_line | awk '{print $3}')" == "SPLIT_RELEASE" ]; then - setvar split_release_process true - else - setvar split_release_process false - fi - else - echo "Release commit should only add one version." - exit 1 - fi - else - RELEASE_TAG="0.0.0" - setvar is_release false - fi - - # This is the last snapshot that does not support Scala 2.13. - CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')" - - if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then - setvar scala_2_13 true - else - setvar scala_2_13 false - fi - name: out - - job: collect_build_data condition: always() dependsOn: diff --git a/ci/check-for-release-job.yml b/ci/check-for-release-job.yml new file mode 100644 index 00000000000..2adf6a41340 --- /dev/null +++ b/ci/check-for-release-job.yml @@ -0,0 +1,71 @@ +# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +jobs: +- job: check_for_release + dependsOn: + - git_sha + variables: + branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] + fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] + pool: + name: "ubuntu_20_04" + demands: assignment -equals default + steps: + - template: bash-lib.yml + parameters: + var_name: bash-lib + - bash: | + set -euo pipefail + eval "$(./dev-env/bin/dade-assist)" + source $(bash-lib) + + ./release.sh check + + changes_release_files() { + changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)" + [ "LATEST" = "$changed" ] + } + + changes_one_line_in_latest() { + changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')" + add_one="1_0" + change_one="1_1" + [[ "$add_one" == "$changed" || "$change_one" == "$changed" ]] + } + + added_line() { + echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)" + } + + if changes_release_files; then + if changes_one_line_in_latest; then + setvar is_release true + setvar trigger_sha $(branch_sha) + setvar release_sha "$(added_line | awk '{print $1}')" + RELEASE_TAG="$(added_line | awk '{print $2}')" + setvar release_tag "$RELEASE_TAG" + if [ "$(added_line | awk '{print $3}')" == "SPLIT_RELEASE" ]; then + setvar split_release_process true + else + setvar split_release_process false + fi + else + echo "Release commit should only add one version." + exit 1 + fi + else + RELEASE_TAG="0.0.0" + setvar is_release false + fi + + # This is the last snapshot that does not support Scala 2.13. + CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')" + + if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then + setvar scala_2_13 true + else + setvar scala_2_13 false + fi + name: out + diff --git a/ci/daily-snapshot.yml b/ci/daily-snapshot.yml new file mode 100644 index 00000000000..166cb97c283 --- /dev/null +++ b/ci/daily-snapshot.yml @@ -0,0 +1,43 @@ +# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +pr: none +trigger: none +schedules: +- cron: "0 3 * * *" + displayName: Daily split snapshot + branches: + include: + - main + +jobs: +- job: check_for_release + dependsOn: + - git_sha + variables: + branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] + fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] + pool: + name: "ubuntu_20_04" + demands: assignment -equals default + steps: + - template: bash-lib.yml + parameters: + var_name: bash-lib + - bash: | + set -euo pipefail + eval "$(./dev-env/bin/dade-assist)" + source $(bash-lib) + + prefix=$(head -1 LATEST | awk '{print $2}' | sed -e 's/\([^-]*\).*/\1/') + release=$(./release.sh snapshot HEAD $prefix | awk '{print $2}') + + setvar is_release true + setvar trigger_sha "$(branch_sha)" + setvar release_sha "$(branch_sha)" + setvar release_tag "$release" + setvar split_release_process true + setvar scala_2_13 true + name: out +- template: ci/build.yml +- template: ci/split-release-job.yml diff --git a/ci/prs.yml b/ci/prs.yml index 931ab28dc97..8508eb8218b 100644 --- a/ci/prs.yml +++ b/ci/prs.yml @@ -12,6 +12,8 @@ pr: jobs: - template: build.yml +- template: check-for-release-job.yml + - job: check_standard_change_label dependsOn: diff --git a/ci/split-release-job.yml b/ci/split-release-job.yml new file mode 100644 index 00000000000..6b572a75aed --- /dev/null +++ b/ci/split-release-job.yml @@ -0,0 +1,91 @@ +# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +jobs: +- job: split_release + dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ] + condition: and(succeeded(), + eq(dependencies.check_for_release.outputs['out.is_release'], 'true'), + eq(dependencies.check_for_release.outputs['out.split_release_process'], 'true'), + eq(variables['Build.SourceBranchName'], 'main')) + pool: + name: 'ubuntu_20_04' + demands: assignment -equals default + variables: + release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] + release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ] + trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] + steps: + - checkout: self + persistCredentials: true + - bash: | + set -euo pipefail + git checkout $(release_sha) + name: checkout_release + - template: ci/bash-lib.yml + parameters: + var_name: bash-lib + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: linux-release + targetPath: $(Build.StagingDirectory)/release-artifacts + condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE'))) + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: macos-release + targetPath: $(Build.StagingDirectory)/release-artifacts + condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE'))) + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: windows-release + targetPath: $(Build.StagingDirectory)/release-artifacts + condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE'))) + - bash: | + set -euo pipefail + # Note: this gets dev-env from the release commit, not the trigger commit + eval "$(./dev-env/bin/dade-assist)" + KEY_FILE=$(mktemp) + GPG_DIR=$(mktemp -d) + cleanup() { + rm -rf $KEY_FILE $GPG_DIR + } + trap cleanup EXIT + echo "$GPG_KEY" | base64 -d > $KEY_FILE + gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE + # For now we only sign artifactory artifacts here and leave signing of artifacts + # published to GH to the assembly repo. + cd $(Build.StagingDirectory)/release-artifacts/artifactory + for f in *; do + gpg --homedir $GPG_DIR -ab $f + done + env: + GPG_KEY: $(gpg-code-signing) + - bash: | + set -eou pipefail + # Note: this gets dev-env from the release commit, not the trigger commit + eval "$(./dev-env/bin/dade-assist)" + mkdir -p $(Build.StagingDirectory)/split-release + ./ci/assembly-split-release-artifacts.sh $(release_tag) $(Build.StagingDirectory)/release-artifacts $(Build.StagingDirectory)/split-release + - bash: | + set -euo pipefail + # Note: this gets dev-env from the release commit, not the trigger commit + eval "$(./dev-env/bin/dade-assist)" + source $(bash-lib) + cd $(Build.StagingDirectory)/split-release/split-release + for f in "damlc-*" daml-libs/daml-script; do + gcs "$GCRED" cp -r "$f" "gs://daml-binaries/split-releases/$(release_tag)/" + done + name: gcs_for_canton + env: + GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) + - bash: | + set -euo pipefail + # Note: this gets dev-env from the release commit, not the trigger commit + eval "$(./dev-env/bin/dade-assist)" + ./ci/publish-artifactory.sh $(Build.StagingDirectory) $(release_tag) split + env: + AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD) + - template: ci/tell-slack-failed.yml + parameters: + trigger_sha: '$(trigger_sha)' +