mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-11 04:45:57 +03:00
nightly split releases (#12744)
Because there's no reason not to. The only obstacle to automating the normal release process is that we need an explicit manual validation step for our audit log when creating a release, but split-releases are created in the Assemblty repo, so we can have the audit log over there. The diff is going to be very messy because there's a lot of stuff moving around. The only thing that is not just moving a job to a separate file is the `ci/daily-snapshot.yml` file, which is 100% new and is meant as a new Azure Pipelines entrypoint (which I will create after this gets approved). I have made a reasonable effort to create individual commits that simplify reviewing, but I expect it's still going to be kind of a mess. I'm open to opening separate PRs to ~bump my stats~ move one job at a time if that makes reviewing (and testing) easier. CHANGELOG_BEGIN CHANGELOG_END
This commit is contained in:
Gary Verhaegen
GitHub
parent
9e1fe24f71
commit
ca976fe3d7
@ -18,6 +18,7 @@ pr: none
|
||||
|
||||
jobs:
|
||||
- template: ci/build.yml
|
||||
- template: ci/check-for-release-job.yml
|
||||
|
||||
- job: release
|
||||
dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
|
||||
@ -158,92 +159,7 @@ jobs:
|
||||
trigger_sha: '$(trigger_sha)'
|
||||
- template: ci/report-end.yml
|
||||
|
||||
- job: split_release
|
||||
dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
|
||||
condition: and(succeeded(),
|
||||
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
||||
eq(dependencies.check_for_release.outputs['out.split_release_process'], 'true'),
|
||||
eq(variables['Build.SourceBranchName'], 'main'))
|
||||
pool:
|
||||
name: 'ubuntu_20_04'
|
||||
demands: assignment -equals default
|
||||
variables:
|
||||
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
||||
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
||||
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
||||
steps:
|
||||
- checkout: self
|
||||
persistCredentials: true
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
git checkout $(release_sha)
|
||||
name: checkout_release
|
||||
- template: ci/bash-lib.yml
|
||||
parameters:
|
||||
var_name: bash-lib
|
||||
- task: DownloadPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: linux-release
|
||||
targetPath: $(Build.StagingDirectory)/release-artifacts
|
||||
condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
|
||||
- task: DownloadPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: macos-release
|
||||
targetPath: $(Build.StagingDirectory)/release-artifacts
|
||||
condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
|
||||
- task: DownloadPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: windows-release
|
||||
targetPath: $(Build.StagingDirectory)/release-artifacts
|
||||
condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
KEY_FILE=$(mktemp)
|
||||
GPG_DIR=$(mktemp -d)
|
||||
cleanup() {
|
||||
rm -rf $KEY_FILE $GPG_DIR
|
||||
}
|
||||
trap cleanup EXIT
|
||||
echo "$GPG_KEY" | base64 -d > $KEY_FILE
|
||||
gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
|
||||
# For now we only sign artifactory artifacts here and leave signing of artifacts
|
||||
# published to GH to the assembly repo.
|
||||
cd $(Build.StagingDirectory)/release-artifacts/artifactory
|
||||
for f in *; do
|
||||
gpg --homedir $GPG_DIR -ab $f
|
||||
done
|
||||
env:
|
||||
GPG_KEY: $(gpg-code-signing)
|
||||
- bash: |
|
||||
set -eou pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
mkdir -p $(Build.StagingDirectory)/split-release
|
||||
./ci/assembly-split-release-artifacts.sh $(release_tag) $(Build.StagingDirectory)/release-artifacts $(Build.StagingDirectory)/split-release
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
source $(bash-lib)
|
||||
cd $(Build.StagingDirectory)/split-release/split-release
|
||||
for f in "damlc-*" daml-libs/daml-script; do
|
||||
gcs "$GCRED" cp -r "$f" "gs://daml-binaries/split-releases/$(release_tag)/"
|
||||
done
|
||||
name: gcs_for_canton
|
||||
env:
|
||||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
./ci/publish-artifactory.sh $(Build.StagingDirectory) $(release_tag) split
|
||||
env:
|
||||
AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
|
||||
- template: ci/tell-slack-failed.yml
|
||||
parameters:
|
||||
trigger_sha: '$(trigger_sha)'
|
||||
- template: ci/split-release-job.yml
|
||||
|
||||
- job: compat_versions_pr
|
||||
dependsOn:
|
||||
|
||||
67
ci/build.yml
67
ci/build.yml
@ -406,73 +406,6 @@ jobs:
|
||||
- bash: ci/check-protobuf-stability.sh
|
||||
- template: tell-slack-failed.yml
|
||||
|
||||
- job: check_for_release
|
||||
dependsOn:
|
||||
- git_sha
|
||||
variables:
|
||||
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
||||
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
||||
pool:
|
||||
name: "ubuntu_20_04"
|
||||
demands: assignment -equals default
|
||||
steps:
|
||||
- template: bash-lib.yml
|
||||
parameters:
|
||||
var_name: bash-lib
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
source $(bash-lib)
|
||||
|
||||
./release.sh check
|
||||
|
||||
changes_release_files() {
|
||||
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
|
||||
[ "LATEST" = "$changed" ]
|
||||
}
|
||||
|
||||
changes_one_line_in_latest() {
|
||||
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
|
||||
add_one="1_0"
|
||||
change_one="1_1"
|
||||
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
|
||||
}
|
||||
|
||||
added_line() {
|
||||
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
|
||||
}
|
||||
|
||||
if changes_release_files; then
|
||||
if changes_one_line_in_latest; then
|
||||
setvar is_release true
|
||||
setvar trigger_sha $(branch_sha)
|
||||
setvar release_sha "$(added_line | awk '{print $1}')"
|
||||
RELEASE_TAG="$(added_line | awk '{print $2}')"
|
||||
setvar release_tag "$RELEASE_TAG"
|
||||
if [ "$(added_line | awk '{print $3}')" == "SPLIT_RELEASE" ]; then
|
||||
setvar split_release_process true
|
||||
else
|
||||
setvar split_release_process false
|
||||
fi
|
||||
else
|
||||
echo "Release commit should only add one version."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
RELEASE_TAG="0.0.0"
|
||||
setvar is_release false
|
||||
fi
|
||||
|
||||
# This is the last snapshot that does not support Scala 2.13.
|
||||
CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')"
|
||||
|
||||
if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then
|
||||
setvar scala_2_13 true
|
||||
else
|
||||
setvar scala_2_13 false
|
||||
fi
|
||||
name: out
|
||||
|
||||
- job: collect_build_data
|
||||
condition: always()
|
||||
dependsOn:
|
||||
|
||||
71
ci/check-for-release-job.yml
Normal file
71
ci/check-for-release-job.yml
Normal file
@ -0,0 +1,71 @@
|
||||
# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
jobs:
|
||||
- job: check_for_release
|
||||
dependsOn:
|
||||
- git_sha
|
||||
variables:
|
||||
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
||||
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
||||
pool:
|
||||
name: "ubuntu_20_04"
|
||||
demands: assignment -equals default
|
||||
steps:
|
||||
- template: bash-lib.yml
|
||||
parameters:
|
||||
var_name: bash-lib
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
source $(bash-lib)
|
||||
|
||||
./release.sh check
|
||||
|
||||
changes_release_files() {
|
||||
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
|
||||
[ "LATEST" = "$changed" ]
|
||||
}
|
||||
|
||||
changes_one_line_in_latest() {
|
||||
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
|
||||
add_one="1_0"
|
||||
change_one="1_1"
|
||||
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
|
||||
}
|
||||
|
||||
added_line() {
|
||||
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
|
||||
}
|
||||
|
||||
if changes_release_files; then
|
||||
if changes_one_line_in_latest; then
|
||||
setvar is_release true
|
||||
setvar trigger_sha $(branch_sha)
|
||||
setvar release_sha "$(added_line | awk '{print $1}')"
|
||||
RELEASE_TAG="$(added_line | awk '{print $2}')"
|
||||
setvar release_tag "$RELEASE_TAG"
|
||||
if [ "$(added_line | awk '{print $3}')" == "SPLIT_RELEASE" ]; then
|
||||
setvar split_release_process true
|
||||
else
|
||||
setvar split_release_process false
|
||||
fi
|
||||
else
|
||||
echo "Release commit should only add one version."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
RELEASE_TAG="0.0.0"
|
||||
setvar is_release false
|
||||
fi
|
||||
|
||||
# This is the last snapshot that does not support Scala 2.13.
|
||||
CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')"
|
||||
|
||||
if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then
|
||||
setvar scala_2_13 true
|
||||
else
|
||||
setvar scala_2_13 false
|
||||
fi
|
||||
name: out
|
||||
|
||||
43
ci/daily-snapshot.yml
Normal file
43
ci/daily-snapshot.yml
Normal file
@ -0,0 +1,43 @@
|
||||
# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
pr: none
|
||||
trigger: none
|
||||
schedules:
|
||||
- cron: "0 3 * * *"
|
||||
displayName: Daily split snapshot
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
- job: check_for_release
|
||||
dependsOn:
|
||||
- git_sha
|
||||
variables:
|
||||
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
||||
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
||||
pool:
|
||||
name: "ubuntu_20_04"
|
||||
demands: assignment -equals default
|
||||
steps:
|
||||
- template: bash-lib.yml
|
||||
parameters:
|
||||
var_name: bash-lib
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
source $(bash-lib)
|
||||
|
||||
prefix=$(head -1 LATEST | awk '{print $2}' | sed -e 's/\([^-]*\).*/\1/')
|
||||
release=$(./release.sh snapshot HEAD $prefix | awk '{print $2}')
|
||||
|
||||
setvar is_release true
|
||||
setvar trigger_sha "$(branch_sha)"
|
||||
setvar release_sha "$(branch_sha)"
|
||||
setvar release_tag "$release"
|
||||
setvar split_release_process true
|
||||
setvar scala_2_13 true
|
||||
name: out
|
||||
- template: ci/build.yml
|
||||
- template: ci/split-release-job.yml
|
||||
@ -12,6 +12,8 @@ pr:
|
||||
|
||||
jobs:
|
||||
- template: build.yml
|
||||
- template: check-for-release-job.yml
|
||||
|
||||
|
||||
- job: check_standard_change_label
|
||||
dependsOn:
|
||||
|
||||
91
ci/split-release-job.yml
Normal file
91
ci/split-release-job.yml
Normal file
@ -0,0 +1,91 @@
|
||||
# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
jobs:
|
||||
- job: split_release
|
||||
dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
|
||||
condition: and(succeeded(),
|
||||
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
||||
eq(dependencies.check_for_release.outputs['out.split_release_process'], 'true'),
|
||||
eq(variables['Build.SourceBranchName'], 'main'))
|
||||
pool:
|
||||
name: 'ubuntu_20_04'
|
||||
demands: assignment -equals default
|
||||
variables:
|
||||
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
||||
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
||||
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
||||
steps:
|
||||
- checkout: self
|
||||
persistCredentials: true
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
git checkout $(release_sha)
|
||||
name: checkout_release
|
||||
- template: ci/bash-lib.yml
|
||||
parameters:
|
||||
var_name: bash-lib
|
||||
- task: DownloadPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: linux-release
|
||||
targetPath: $(Build.StagingDirectory)/release-artifacts
|
||||
condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
|
||||
- task: DownloadPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: macos-release
|
||||
targetPath: $(Build.StagingDirectory)/release-artifacts
|
||||
condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
|
||||
- task: DownloadPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: windows-release
|
||||
targetPath: $(Build.StagingDirectory)/release-artifacts
|
||||
condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
KEY_FILE=$(mktemp)
|
||||
GPG_DIR=$(mktemp -d)
|
||||
cleanup() {
|
||||
rm -rf $KEY_FILE $GPG_DIR
|
||||
}
|
||||
trap cleanup EXIT
|
||||
echo "$GPG_KEY" | base64 -d > $KEY_FILE
|
||||
gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
|
||||
# For now we only sign artifactory artifacts here and leave signing of artifacts
|
||||
# published to GH to the assembly repo.
|
||||
cd $(Build.StagingDirectory)/release-artifacts/artifactory
|
||||
for f in *; do
|
||||
gpg --homedir $GPG_DIR -ab $f
|
||||
done
|
||||
env:
|
||||
GPG_KEY: $(gpg-code-signing)
|
||||
- bash: |
|
||||
set -eou pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
mkdir -p $(Build.StagingDirectory)/split-release
|
||||
./ci/assembly-split-release-artifacts.sh $(release_tag) $(Build.StagingDirectory)/release-artifacts $(Build.StagingDirectory)/split-release
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
source $(bash-lib)
|
||||
cd $(Build.StagingDirectory)/split-release/split-release
|
||||
for f in "damlc-*" daml-libs/daml-script; do
|
||||
gcs "$GCRED" cp -r "$f" "gs://daml-binaries/split-releases/$(release_tag)/"
|
||||
done
|
||||
name: gcs_for_canton
|
||||
env:
|
||||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
# Note: this gets dev-env from the release commit, not the trigger commit
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
./ci/publish-artifactory.sh $(Build.StagingDirectory) $(release_tag) split
|
||||
env:
|
||||
AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
|
||||
- template: ci/tell-slack-failed.yml
|
||||
parameters:
|
||||
trigger_sha: '$(trigger_sha)'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user