mirror of
https://github.com/digital-asset/daml.git
synced 2024-10-05 16:57:28 +03:00
[release] faster releases - experiment (#18812)
When we build a release, it is always a "past" commit - typically, one that has already been tested twice: once when the corresponding PR was run, and then again as a "main"-branch commit. Release branches don't run, but their protection rules enforce linear merges. Either way, we know we're building a _good_ commit, and, assuming our builds and tests are hermetic, testing that commit again when we make a release is a pure waste of time and CPU resources. The other case, where we make an ad-hoc release from a branch that has not been merged, has a similar issue: we do not necessarily want to run the full test suite, because part of the reason we need that commit may be that it doesn't succeed as is. Based on that observation, I wondered what might be the minimal set of things we actually need to build when making a release. This PR is an experiment in trying to find that out.
This commit is contained in:
Gary Verhaegen
GitHub
parent
714b0c7f4f
commit
7c83265ef9
@ -18,7 +18,6 @@ mkdir -p $OUTPUT_DIR/split-release
|
||||
|
||||
TARBALL=daml-sdk-$RELEASE_TAG-$NAME.tar.gz
|
||||
EE_TARBALL=daml-sdk-$RELEASE_TAG-$NAME-ee.tar.gz
|
||||
bazel build //release:sdk-release-tarball-ce //release:sdk-release-tarball-ee
|
||||
cp bazel-bin/release/sdk-release-tarball-ce.tar.gz $OUTPUT_DIR/github/$TARBALL
|
||||
# Used for the non-split release process.
|
||||
cp bazel-bin/release/sdk-release-tarball-ee.tar.gz $OUTPUT_DIR/artifactory/$EE_TARBALL
|
||||
@ -26,34 +25,27 @@ cp bazel-bin/release/sdk-release-tarball-ee.tar.gz $OUTPUT_DIR/artifactory/$EE_T
|
||||
cp bazel-bin/release/sdk-release-tarball-ee.tar.gz $OUTPUT_DIR/split-release/$EE_TARBALL
|
||||
|
||||
|
||||
bazel build //compiler/damlc:damlc-dist
|
||||
cp bazel-bin/compiler/damlc/damlc-dist.tar.gz $OUTPUT_DIR/split-release/damlc-$RELEASE_TAG-$NAME.tar.gz
|
||||
|
||||
# Platform independent artifacts are only built on Linux.
|
||||
if [[ "$NAME" == "linux-intel" ]]; then
|
||||
bazel build //release:protobufs
|
||||
PROTOS_ZIP=protobufs-$RELEASE_TAG.zip
|
||||
cp bazel-bin/release/protobufs.zip $OUTPUT_DIR/github/$PROTOS_ZIP
|
||||
|
||||
SCRIPT=daml-script-$RELEASE_TAG.jar
|
||||
bazel build //daml-script/runner:daml-script-binary_distribute.jar
|
||||
cp bazel-bin/daml-script/runner/daml-script-binary_distribute.jar $OUTPUT_DIR/artifactory/$SCRIPT
|
||||
|
||||
mkdir -p $OUTPUT_DIR/split-release/daml-libs/daml-script
|
||||
bazel build //daml-script/daml:daml-script-dars
|
||||
cp bazel-bin/daml-script/daml/*.dar $OUTPUT_DIR/split-release/daml-libs/daml-script/
|
||||
bazel build //daml-script/daml3:daml3-script-dars
|
||||
cp bazel-bin/daml-script/daml3/*.dar $OUTPUT_DIR/split-release/daml-libs/daml-script/
|
||||
|
||||
mkdir -p $OUTPUT_DIR/split-release/docs
|
||||
|
||||
bazel build //docs:sphinx-source-tree //docs:pdf-fonts-tar //docs:non-sphinx-html-docs //docs:sphinx-source-tree-deps
|
||||
cp bazel-bin/docs/sphinx-source-tree.tar.gz $OUTPUT_DIR/split-release/docs/sphinx-source-tree-$RELEASE_TAG.tar.gz
|
||||
cp bazel-bin/docs/sphinx-source-tree-deps.tar.gz $OUTPUT_DIR/split-release/docs/sphinx-source-tree-deps-$RELEASE_TAG.tar.gz
|
||||
cp bazel-bin/docs/pdf-fonts-tar.tar.gz $OUTPUT_DIR/split-release/docs/pdf-fonts-$RELEASE_TAG.tar.gz
|
||||
cp bazel-bin/docs/non-sphinx-html-docs.tar.gz $OUTPUT_DIR/split-release/docs/non-sphinx-html-docs-$RELEASE_TAG.tar.gz
|
||||
|
||||
bazel build //test-evidence:generate-security-test-evidence-files
|
||||
cp bazel-bin/test-evidence/daml-security-test-evidence.csv $OUTPUT_DIR/github/daml-security-test-evidence-$RELEASE_TAG.csv
|
||||
cp bazel-bin/test-evidence/daml-security-test-evidence.json $OUTPUT_DIR/github/daml-security-test-evidence-$RELEASE_TAG.json
|
||||
fi
|
||||
|
||||
50
sdk/ci/build-release.sh
Executable file
50
sdk/ci/build-release.sh
Executable file
@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
|
||||
cd $DIR/..
|
||||
|
||||
execution_log_postfix=${1:-}${2:-}
|
||||
|
||||
export LC_ALL=en_US.UTF-8
|
||||
|
||||
ARTIFACT_DIRS="${BUILD_ARTIFACTSTAGINGDIRECTORY:-$PWD}"
|
||||
mkdir -p "${ARTIFACT_DIRS}/logs"
|
||||
|
||||
if [ "${1:-}" = "_m1" ]; then
|
||||
bazel="arch -arm64 bazel"
|
||||
else
|
||||
bazel=bazel
|
||||
fi
|
||||
|
||||
if [ "$(uname)-$(uname -m)" = "Linux-x86_64" ]; then
|
||||
# These are platform-independent so we only need to build them once
|
||||
extra_build_targets="//release:protobufs \
|
||||
//daml-script/runner:daml-script-binary_distribute.jar \
|
||||
//daml-script/daml:daml-script-dars \
|
||||
//daml-script/daml3:daml3-script-dars \
|
||||
//docs:sphinx-source-tree \
|
||||
//docs:pdf-fonts-tar \
|
||||
//docs:non-sphinx-html-docs \
|
||||
//docs:sphinx-source-tree-deps \
|
||||
//test-evidence:generate-security-test-evidence-files"
|
||||
else
|
||||
extra_build_targets=
|
||||
fi
|
||||
|
||||
$bazel build \
|
||||
//compiler/damlc/tests:platform-independence.dar \
|
||||
//release:sdk-release-tarball-ce \
|
||||
//release:sdk-release-tarball-ee \
|
||||
//compiler/damlc:damlc-dist \
|
||||
//release:protobufs \
|
||||
$extra_build_targets \
|
||||
--profile build-profile.json \
|
||||
--experimental_profile_include_target_label \
|
||||
--build_event_json_file build-events.json \
|
||||
--build_event_publish_all_actions \
|
||||
--execution_log_json_file "$ARTIFACT_DIRS/logs/build_execution${execution_log_postfix}.json.gz"
|
||||
71
sdk/ci/build-releases.ps1
Normal file
71
sdk/ci/build-releases.ps1
Normal file
@ -0,0 +1,71 @@
|
||||
Set-StrictMode -Version latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# See https://github.com/lukesampson/scoop/issues/3859
|
||||
Set-Strictmode -Off
|
||||
.\dev-env\windows\bin\dadew.ps1 install
|
||||
Set-StrictMode -Version latest
|
||||
.\dev-env\windows\bin\dadew.ps1 sync
|
||||
.\dev-env\windows\bin\dadew.ps1 enable
|
||||
|
||||
if (!(Test-Path .\.bazelrc.local)) {
|
||||
Set-Content -Path .\.bazelrc.local -Value 'build --config windows'
|
||||
}
|
||||
|
||||
$ARTIFACT_DIRS = if ("$env:BUILD_ARTIFACTSTAGINGDIRECTORY") { $env:BUILD_ARTIFACTSTAGINGDIRECTORY } else { Get-Location }
|
||||
|
||||
if (!(Test-Path ${ARTIFACT_DIRS}/logs)) {
|
||||
mkdir -p ${ARTIFACT_DIRS}/logs
|
||||
} elseif (Test-Path ${ARTIFACT_DIRS}/logs -PathType Leaf) {
|
||||
throw ("Cannot create directory '${ARTIFACT_DIRS}/logs'. Conflicting file.")
|
||||
}
|
||||
|
||||
# If a previous build was forcefully terminated, then stack's lock file might
|
||||
# not have been cleaned up properly leading to errors of the form
|
||||
#
|
||||
# user error (hTryLock: lock already exists: C:\Users\u\AppData\Roaming\stack\pantry\hackage\hackage-security-lock)
|
||||
#
|
||||
# The package cache might be corrupted and just removing the lock might lead to
|
||||
# errors as below, so we just nuke the entire stack cache.
|
||||
#
|
||||
# Failed populating package index cache
|
||||
# IncompletePayload 56726464 844
|
||||
#
|
||||
if (Test-Path -Path $env:appdata\stack\pantry\hackage\hackage-security-lock) {
|
||||
Write-Output ">> Nuking stack directory"
|
||||
Remove-Item -ErrorAction Continue -Force -Recurse -Path $env:appdata\stack
|
||||
}
|
||||
|
||||
function bazel() {
|
||||
Write-Output ">> bazel $args"
|
||||
$global:lastexitcode = 0
|
||||
$backupErrorActionPreference = $script:ErrorActionPreference
|
||||
$script:ErrorActionPreference = "Continue"
|
||||
& bazel.exe @args 2>&1 | %{ "$_" }
|
||||
$script:ErrorActionPreference = $backupErrorActionPreference
|
||||
if ($global:lastexitcode -ne 0 -And $args[0] -ne "shutdown") {
|
||||
Write-Output "<< bazel $args (failed, exit code: $global:lastexitcode)"
|
||||
throw ("Bazel returned non-zero exit code: $global:lastexitcode")
|
||||
}
|
||||
Write-Output "<< bazel $args (ok)"
|
||||
}
|
||||
|
||||
# ScalaCInvoker, a Bazel worker, created by rules_scala opens some of the bazel execroot's files,
|
||||
# which later causes issues on Bazel init (source forest creation) on Windows. A shutdown closes workers,
|
||||
# which is a workaround for this problem.
|
||||
bazel shutdown
|
||||
|
||||
# Prefetch nodejs_dev_env to avoid permission denied errors on external/nodejs_dev_env/nodejs_dev_env/node.exe
|
||||
# It isn’t clear where exactly those errors are coming from.
|
||||
bazel fetch @nodejs_dev_env//...
|
||||
|
||||
bazel build `
|
||||
//compiler/damlc/tests:platform-independence.dar `
|
||||
//release:sdk-release-tarball-ce `
|
||||
//release:sdk-release-tarball-ee `
|
||||
`-`-profile build-profile.json `
|
||||
`-`-experimental_profile_include_target_label `
|
||||
`-`-build_event_json_file build-events.json `
|
||||
`-`-build_event_publish_all_actions
|
||||
|
||||
bazel shutdown
|
||||
Loading…
Reference in New Issue
Block a user