From cd33c2015cc863fdf36c6e20f38cf2d1330def7b Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Tue, 9 Feb 2021 11:42:34 +0100 Subject: [PATCH] ci: use setvar to set variables (#8664) My goal here is to investigate the new warning Azure has been showing for the past few days: > ##[warning]%25 detected in ##vso command. In March 2021, the agent command parser will be updated to unescape this to %. To opt out of this behavior, set a job level variable DECODE_PERCENTS to false. Setting to true will force this behavior immediately. More information can be found at https://github.com/microsoft/azure-pipelines-agent/blob/master/docs/design/percentEncoding.md As far as I'm aware we are not deliberately passing in any `%25` in any of our `vso` commands, so I was a bit surprised by this. CHANGELOG_BEGIN CHANGELOG_END --- azure-pipelines.yml | 11 +++++------ ci/bash-lib.yml | 5 +++++ ci/build-unix.yml | 20 ++++++++++++-------- ci/build-windows.yml | 9 +++++++-- ci/build.yml | 25 ++++++++++++++----------- ci/patch_bazel_windows/compile.yml | 8 ++++---- ci/report-end.yml | 6 +++++- ci/report-start.yml | 8 ++++++-- 8 files changed, 58 insertions(+), 34 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 19b86ac599..3fca8c81c8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -70,14 +70,18 @@ jobs: - checkout: self persistCredentials: true - template: ci/clean-up.yml + - template: ci/bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail + source $(bash-lib) if git tag v$(release_tag) $(release_sha); then git push origin v$(release_tag) mkdir $(Build.StagingDirectory)/release mkdir $(Build.StagingDirectory)/artifactory else - echo "##vso[task.setvariable variable=skip-github]TRUE" + setvar skip-github TRUE fi - task: DownloadPipelineArtifact@0 inputs: @@ -262,11 +266,6 @@ jobs: source $(bash_lib) - setvar() { - echo "Setting '$1' to '$2'" - echo "##vso[task.setvariable variable=$1;isOutput=true]$2" - } - DELAY=1 while ! curl --fail -I https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/$(release_tag)/ledger-api-test-tool-$(release_tag).jar; do sleep $DELAY diff --git a/ci/bash-lib.yml b/ci/bash-lib.yml index 93dd8bdce7..332b6bc4da 100644 --- a/ci/bash-lib.yml +++ b/ci/bash-lib.yml @@ -142,6 +142,11 @@ steps: rm -rf $gpg_dir $key return $res } + setvar() { + echo "Setting '$1' to '$2'" + echo "##vso[task.setvariable variable=$1;isOutput=true]$2" + } + END echo "##vso[task.setvariable variable=${{parameters.var_name}}]$TMP" displayName: install Bash lib diff --git a/ci/build-unix.yml b/ci/build-unix.yml index e46a782930..eba0e98d89 100644 --- a/ci/build-unix.yml +++ b/ci/build-unix.yml @@ -60,51 +60,55 @@ steps: eq(${{parameters.is_release}}, 'true'), eq(variables['Build.SourceBranchName'], 'main'), eq('${{parameters.name}}', 'linux')) + - template: bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail + source $(bash-lib) eval "$(./dev-env/bin/dade-assist)" TARBALL=daml-sdk-${{parameters.release_tag}}-${{parameters.name}}.tar.gz cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL - echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" + setvar tarball "$TARBALL" PROTOS_ZIP=protobufs-${{parameters.release_tag}}.zip cp bazel-bin/release/protobufs.zip $(Build.StagingDirectory)/$PROTOS_ZIP - echo "##vso[task.setvariable variable=protos-zip;isOutput=true]$PROTOS_ZIP" + setvar protos-zip "$PROTOS_ZIP" DAML_ON_SQL=daml-on-sql-${{parameters.release_tag}}.jar ## Not built by default bazel build //ledger/daml-on-sql:daml-on-sql-binary_deploy.jar cp bazel-bin/ledger/daml-on-sql/daml-on-sql-binary_deploy.jar $(Build.StagingDirectory)/$DAML_ON_SQL - echo "##vso[task.setvariable variable=daml-on-sql;isOutput=true]$DAML_ON_SQL" + setvar daml-on-sql "$DAML_ON_SQL" JSON_API=http-json-${{parameters.release_tag}}.jar ## Not built by default bazel build //ledger-service/http-json:http-json-binary_deploy.jar cp bazel-bin/ledger-service/http-json/http-json-binary_deploy.jar $(Build.StagingDirectory)/$JSON_API - echo "##vso[task.setvariable variable=json-api;isOutput=true]$JSON_API" + setvar json-api "$JSON_API" TRIGGER=daml-trigger-runner-${{parameters.release_tag}}.jar bazel build //triggers/runner:trigger-runner_deploy.jar cp bazel-bin/triggers/runner/trigger-runner_deploy.jar $(Build.StagingDirectory)/$TRIGGER - echo "##vso[task.setvariable variable=trigger-runner;isOutput=true]$TRIGGER" + setvar trigger-runner "$TRIGGER" TRIGGER_SERVICE=trigger-service-${{parameters.release_tag}}.jar ## Not built by default bazel build //triggers/service:trigger-service-binary_deploy.jar cp bazel-bin/triggers/service/trigger-service-binary_deploy.jar $(Build.StagingDirectory)/$TRIGGER_SERVICE - echo "##vso[task.setvariable variable=trigger-service;isOutput=true]$TRIGGER_SERVICE" + setvar trigger-service "$TRIGGER_SERVICE" OAUTH2_MIDDLEWARE=oauth2-middleware-${{parameters.release_tag}}.jar ## Not built by default bazel build //triggers/service/auth:oauth2-middleware-binary_deploy.jar cp bazel-bin/triggers/service/auth/oauth2-middleware-binary_deploy.jar $(Build.StagingDirectory)/$OAUTH2_MIDDLEWARE - echo "##vso[task.setvariable variable=oauth2-middleware;isOutput=true]$OAUTH2_MIDDLEWARE" + setvar oauth2-middleware "$OAUTH2_MIDDLEWARE" SCRIPT=daml-script-${{parameters.release_tag}}.jar bazel build //daml-script/runner:script-runner_deploy.jar cp bazel-bin/daml-script/runner/script-runner_deploy.jar $(Build.StagingDirectory)/$SCRIPT - echo "##vso[task.setvariable variable=script-runner;isOutput=true]$SCRIPT" + setvar script-runner "$SCRIPT" env: DAML_SDK_RELEASE_VERSION: ${{parameters.release_tag}} name: publish diff --git a/ci/build-windows.yml b/ci/build-windows.yml index 6ecff7dce1..392345c578 100644 --- a/ci/build-windows.yml +++ b/ci/build-windows.yml @@ -28,8 +28,13 @@ steps: pathtoPublish: 'bazel-testlogs/' artifactName: 'Test logs' + - template: bash-lib.yml + parameters: + var_name: bash-lib + - bash: | set -euo pipefail + source $(bash-lib) INSTALLER=daml-sdk-${{parameters.release_tag}}-windows.exe mv "bazel-bin/release/windows-installer/daml-sdk-installer.exe" "$(Build.StagingDirectory)/$INSTALLER" chmod +wx "$(Build.StagingDirectory)/$INSTALLER" @@ -41,10 +46,10 @@ steps: MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER" rm signing_key.pfx trap - EXIT - echo "##vso[task.setvariable variable=installer;isOutput=true]$INSTALLER" + setvar installer "$INSTALLER" TARBALL=daml-sdk-${{parameters.release_tag}}-windows.tar.gz cp bazel-bin/release/sdk-release-tarball.tar.gz '$(Build.StagingDirectory)'/$TARBALL - echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" + setvar tarball "$TARBALL" name: publish env: SIGNING_KEY: $(microsoft-code-signing) diff --git a/ci/build.yml b/ci/build.yml index ab5a20a376..bfa6eaa7e1 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -7,16 +7,20 @@ jobs: name: 'ubuntu_20_04' demands: assignment -equals default steps: + - template: bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail + source $(bash-lib) if [ "$(Build.Reason)" == "PullRequest" ]; then - echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD^2)" - echo "##vso[task.setvariable variable=main;isOutput=true]$(git rev-parse HEAD^1)" - echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))" + setvar branch "$(git rev-parse HEAD^2)" + setvar main "$(git rev-parse HEAD^1)" + setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))" else - echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD)" - echo "##vso[task.setvariable variable=main;isOutput=true]$(git rev-parse HEAD^1)" - echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git rev-parse HEAD^1)" + setvar branch "$(git rev-parse HEAD)" + setvar main "$(git rev-parse HEAD^1)" + setvar fork_point "$(git rev-parse HEAD^1)" fi name: out @@ -359,8 +363,12 @@ jobs: name: "ubuntu_20_04" demands: assignment -equals default steps: + - template: bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail + source $(bash-lib) ./release.sh check @@ -376,11 +384,6 @@ jobs: [[ "$add_one" == "$changed" || "$change_one" == "$changed" ]] } - setvar() { - echo "Setting '$1' to '$2'" - echo "##vso[task.setvariable variable=$1;isOutput=true]$2" - } - added_line() { echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)" } diff --git a/ci/patch_bazel_windows/compile.yml b/ci/patch_bazel_windows/compile.yml index 5a68ada929..f5c5773d66 100644 --- a/ci/patch_bazel_windows/compile.yml +++ b/ci/patch_bazel_windows/compile.yml @@ -11,8 +11,12 @@ jobs: name: ubuntu_20_04 steps: - checkout: self + - template: ../bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail + source $(bash-lib) CACHE_KEY="$(find ci/patch_bazel_windows -type f -print0 | sort -z | xargs -r0 md5sum | md5sum | awk '{print $1}')" TARGET="patch_bazel_windows/bazel-$CACHE_KEY.zip" @@ -24,10 +28,6 @@ jobs: else SHOULD_RUN=true fi - setvar() { - echo "$1: $2" - echo "##vso[task.setvariable variable=$1;isOutput=true]$2" - } setvar cache_key $CACHE_KEY setvar should_run $SHOULD_RUN setvar target $TARGET diff --git a/ci/report-end.yml b/ci/report-end.yml index 054e35351f..e4e98c894d 100644 --- a/ci/report-end.yml +++ b/ci/report-end.yml @@ -2,9 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 steps: + - template: bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail - echo "##vso[task.setvariable variable=time;isOutput=true]$(date -u +"%Y-%m-%dT%H:%M:%S+00:00")" + source $(bash-lib) + setvar time "$(date -u +"%Y-%m-%dT%H:%M:%S+00:00")" condition: always() name: end diff --git a/ci/report-start.yml b/ci/report-start.yml index 2d7f73b019..c86d2a1aea 100644 --- a/ci/report-start.yml +++ b/ci/report-start.yml @@ -2,10 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 steps: + - template: bash-lib.yml + parameters: + var_name: bash-lib - bash: | set -euo pipefail - echo "##vso[task.setvariable variable=time;isOutput=true]$(date -u +"%Y-%m-%dT%H:%M:%S+00:00")" - echo "##vso[task.setvariable variable=machine;isOutput=true]$(Agent.MachineName)" + source $(bash-lib) + setvar time "$(date -u +"%Y-%m-%dT%H:%M:%S+00:00")" + setvar machine "$(Agent.MachineName)" condition: always() name: start