fix tarball generation (#4738)

The existing approach is a historical accident. The reason for the
additional tarball/install jobs was that, in my original attempt, the
build steps would still build the current commit, as opposed to the
target commit.

This is not such an issue on Linux, but setting up the build environment
on Windows and macOS _again_ for no good reason is a pure waste of time
(and effort in getting it right). Now that the build steps build the
target commit (with the env var set), we can go back to the way things
were previously: just take the build products directly from the build
step.

CHANGELOG_BEGIN
CHANGELOG_END

azure-pipelines.yml

@@ -84,7 +84,6 @@ jobs:
       - checkout: self
       - bash: |
           set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
           git checkout $(release_sha)
           git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
         name: checkout_release
@@ -92,6 +91,8 @@ jobs:
       - template: ci/build-unix.yml
         parameters:
           release_tag: $(release_tag)
+          name: 'linux'
+          is_release: $(is_release)
       - bash: |
           set -euo pipefail
           eval "$(./dev-env/bin/dade-assist)"
@@ -118,7 +119,6 @@ jobs:
       - checkout: self
       - bash: |
           set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
           git checkout $(release_sha)
           git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
         name: checkout_release
@@ -132,6 +132,8 @@ jobs:
       - template: ci/build-unix.yml
         parameters:
           release_tag: $(release_tag)
+          name: macos
+          is_release: $(is_release)
       - bash: mkdir -p $(bazel-repo-cache-path)
         displayName: ensure bazel repo cache exists
       - template: ci/tell-slack-failed.yml
@@ -153,7 +155,6 @@ jobs:
       - checkout: self
       - bash: |
           set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
           git checkout $(release_sha)
           git checkout $(trigger_sha) -- docs/source/support/release-notes.rst
         name: checkout_release
@@ -161,6 +162,7 @@ jobs:
       - template: ci/build-windows.yml
         parameters:
           release_tag: $(release_tag)
+          is_release: $(is_release)
       - template: ci/tell-slack-failed.yml
       - template: ci/report-end.yml
 
@@ -236,168 +238,18 @@ jobs:
           fi
         name: out
 
-  - job: release_windows_installer
-    dependsOn: [ "check_for_release", "Windows" ]
-    condition: and(succeeded(),
-                   eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
-                   eq(variables['Build.SourceBranchName'], 'master'))
-    pool:
-      name: 'windows-pool'
-    variables:
-      release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
-      release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
-    steps:
-      - template: ci/report-start.yml
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-          git checkout $(release_sha)
-      - bash: ci/configure-bazel.sh
-        env:
-          IS_FORK: $(System.PullRequest.IsFork)
-          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-
-          export DAML_SDK_RELEASE_VERSION=$(release_tag)
-
-          bazel build //release/windows-installer:windows-installer
-          INSTALLER=daml-sdk-$(release_tag)-windows.exe
-          mv "bazel-bin/release/windows-installer/daml-sdk-installer.exe" "$(Build.StagingDirectory)/$INSTALLER"
-          chmod +x "$(Build.StagingDirectory)/$INSTALLER"
-          cleanup () {
-              rm -f signing_key.pfx
-          }
-          trap cleanup EXIT
-          echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
-          MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
-          rm signing_key.pfx
-          echo "##vso[task.setvariable variable=installer;isOutput=true]$INSTALLER"
-
-          bazel build //release:sdk-release-tarball
-          TARBALL=daml-sdk-$(release_tag)-windows.tar.gz
-          cp bazel-bin/release/sdk-release-tarball.tar.gz '$(Build.StagingDirectory)'/$TARBALL
-          echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
-
-        name: publish
-        env:
-          SIGNING_KEY: $(microsoft-code-signing)
-      - task: PublishPipelineArtifact@0
-        inputs:
-          targetPath: $(Build.StagingDirectory)/$(publish.installer)
-          artifactName: $(publish.installer)
-      - task: PublishPipelineArtifact@0
-        inputs:
-          targetPath: $(Build.StagingDirectory)/$(publish.tarball)
-          artifactName: $(publish.tarball)
-      - template: ci/tell-slack-failed.yml
-      - template: ci/report-end.yml
-
-  - job: release_linux_tarball
-    dependsOn: [ "check_for_release", "Linux" ]
-    condition: and(succeeded(),
-                   eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
-                   eq(variables['Build.SourceBranchName'], 'master'))
-    variables:
-      release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
-      release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
-    pool:
-      name: "linux-pool"
-    steps:
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-          git checkout $(release_sha)
-      - bash: ci/configure-bazel.sh
-        env:
-          IS_FORK: $(System.PullRequest.IsFork)
-          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-
-          export DAML_SDK_RELEASE_VERSION=$(release_tag)
-          bazel build //release:release
-
-          mkdir -p ~/.jfrog
-          cleanup() {
-              rm -f ~/.jfrog/jfrog-cli.conf
-          }
-          trap cleanup EXIT
-          echo "$JFROG_CONFIG_CONTENT" > ~/.jfrog/jfrog-cli.conf
-          unset JFROG_CONFIG_CONTENT
-
-          ./bazel-bin/release/release --release-dir "$(mktemp -d)" --upload
-
-          bazel build //release:sdk-release-tarball
-          TARBALL=daml-sdk-$(release_tag)-linux.tar.gz
-          cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL
-          echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
-        env:
-            JFROG_CONFIG_CONTENT: $(JFROG_CONFIG_CONTENT)
-            GPG_KEY: $(gpg-code-signing)
-            MAVEN_USERNAME: $(MAVEN_USERNAME)
-            MAVEN_PASSWORD: $(MAVEN_PASSWORD)
-            MAVEN_URL: $(MAVEN_URL)
-            NPM_TOKEN: $(NPM_TOKEN)
-        name: publish
-      - task: PublishPipelineArtifact@0
-        inputs:
-          targetPath: $(Build.StagingDirectory)/$(publish.tarball)
-          artifactName: $(publish.tarball)
-      - template: ci/tell-slack-failed.yml
-      - template: ci/report-end.yml
-
-  - job: release_macos_tarball
-    dependsOn: [ "check_for_release", "macOS" ]
-    condition: and(succeeded(),
-                   eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
-                   eq(variables['Build.SourceBranchName'], 'master'))
-    variables:
-      release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
-      release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
-    pool:
-      vmImage: "macOS-10.14"
-    steps:
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-          git checkout $(release_sha)
-      - bash: ci/configure-bazel.sh
-        env:
-          IS_FORK: $(System.PullRequest.IsFork)
-          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-
-          export DAML_SDK_RELEASE_VERSION=$(release_tag)
-
-          bazel build //release:sdk-release-tarball
-          TARBALL=daml-sdk-$(release_tag)-macos.tar.gz
-          cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL
-          echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
-        name: publish
-      - task: PublishPipelineArtifact@0
-        inputs:
-          targetPath: $(Build.StagingDirectory)/$(publish.tarball)
-          artifactName: $(publish.tarball)
-      - template: ci/tell-slack-failed.yml
-      - template: ci/report-end.yml
-
   - job: release
-    dependsOn: [ "check_for_release", "release_linux_tarball", "release_macos_tarball", "release_windows_installer" ]
+    dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
     condition: and(succeeded(),
                    eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
                    eq(variables['Build.SourceBranchName'], 'master'))
     pool:
       vmImage: "Ubuntu-16.04"
     variables:
-      linux-tarball: $[ dependencies.release_linux_tarball.outputs['publish.tarball'] ]
-      macos-tarball: $[ dependencies.release_macos_tarball.outputs['publish.tarball'] ]
-      windows-tarball: $[ dependencies.release_windows_installer.outputs['publish.tarball'] ]
-      windows-installer: $[ dependencies.release_windows_installer.outputs['publish.installer'] ]
+      linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ]
+      macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ]
+      windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ]
+      windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ]
       release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
       release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
     steps:
@@ -509,9 +361,6 @@ jobs:
       - macOS
       - Windows
       - check_for_release
-      - release_linux_tarball
-      - release_macos_tarball
-      - release_windows_installer
       - perf
       - release
       - check_standard_change_label
@@ -536,18 +385,6 @@ jobs:
       check_for_release.machine: $[ dependencies.check_for_release.outputs['start.machine'] ]
       check_for_release.end: $[ dependencies.check_for_release.outputs['end.time'] ]
       check_for_release.status: $[ dependencies.check_for_release.result ]
-      release_linux_tarball.start: $[ dependencies.release_linux_tarball.outputs['start.time'] ]
-      release_linux_tarball.machine: $[ dependencies.release_linux_tarball.outputs['start.machine'] ]
-      release_linux_tarball.end: $[ dependencies.release_linux_tarball.outputs['end.time'] ]
-      release_linux_tarball.status: $[ dependencies.release_linux_tarball.result ]
-      release_macos_tarball.start: $[ dependencies.release_macos_tarball.outputs['start.time'] ]
-      release_macos_tarball.machine: $[ dependencies.release_macos_tarball.outputs['start.machine'] ]
-      release_macos_tarball.end: $[ dependencies.release_macos_tarball.outputs['end.time'] ]
-      release_macos_tarball.status: $[ dependencies.release_macos_tarball.result ]
-      release_windows_installer.start: $[ dependencies.release_windows_installer.outputs['start.time'] ]
-      release_windows_installer.machine: $[ dependencies.release_windows_installer.outputs['start.machine'] ]
-      release_windows_installer.end: $[ dependencies.release_windows_installer.outputs['end.time'] ]
-      release_windows_installer.status: $[ dependencies.release_windows_installer.result ]
       perf.start: $[ dependencies.perf.outputs['start.time'] ]
       perf.machine: $[ dependencies.perf.outputs['start.machine'] ]
       perf.end: $[ dependencies.perf.outputs['end.time'] ]
@@ -638,18 +475,6 @@ jobs:
                                           "machine": "$(check_for_release.machine)",
                                           "end": "$(check_for_release.end)",
                                           "status": "$(check_for_release.status)"},
-                    "linux_tarball": {"start": "$(linux_tarball.start)",
-                                      "machine": "$(linux_tarball.machine)",
-                                      "end": "$(linux_tarball.end)",
-                                      "status": "$(linux_tarball.status)"},
-                    "release_macos_tarball": {"start": "$(release_macos_tarball.start)",
-                                      "machine": "$(release_macos_tarball.machine)",
-                                      "end": "$(release_macos_tarball.end)",
-                                      "status": "$(release_macos_tarball.status)"},
-                    "release_windows_installer": {"start": "$(release_windows_installer.start)",
-                                          "machine": "$(release_windows_installer.machine)",
-                                          "end": "$(release_windows_installer.end)",
-                                          "status": "$(release_windows_installer.status)"},
                     "perf": {"start": "$(perf.start)",
                              "machine": "$(perf.machine)",
                              "end": "$(perf.end)",
@@ -715,9 +540,6 @@ jobs:
               || "$(Windows.status)" != "Succeeded"
               || "$(perf.status)" != "Succeeded"
               || "$(dump.status)" == "Canceled"
-              || "$(linux_tarball.status)" == "Canceled"
-              || "$(release_macos_tarball.status)" == "Canceled"
-              || "$(release_windows_installer.status)" == "Canceled"
               || "$(release.status)" == "Canceled" ]]; then
               exit 1
           fi

ci/build-unix.yml

@@ -2,6 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 parameters:
+  is_release: ''
+  name: ''
   release_tag: ''
 
 steps:
@@ -40,3 +42,47 @@ steps:
     inputs:
       pathtoPublish: 'bazel-testlogs/'
       artifactName: 'Test logs'
+
+  - bash: |
+      set -euo pipefail
+      eval "$(./dev-env/bin/dade-assist)"
+
+      mkdir -p ~/.jfrog
+      cleanup() {
+          rm -f ~/.jfrog/jfrog-cli.conf
+      }
+      trap cleanup EXIT
+      echo "$JFROG_CONFIG_CONTENT" > ~/.jfrog/jfrog-cli.conf
+      unset JFROG_CONFIG_CONTENT
+
+      ./bazel-bin/release/release --release-dir "$(mktemp -d)" --upload
+    env:
+      DAML_SDK_RELEASE_VERSION: ${{parameters.release_tag}}
+      JFROG_CONFIG_CONTENT: $(JFROG_CONFIG_CONTENT)
+      GPG_KEY: $(gpg-code-signing)
+      MAVEN_USERNAME: $(MAVEN_USERNAME)
+      MAVEN_PASSWORD: $(MAVEN_PASSWORD)
+      MAVEN_URL: "https://oss.sonatype.org"
+      NPM_TOKEN: $(NPM_TOKEN)
+    name: publish_npm_mvn
+    condition: and(succeeded(),
+                   eq('${{parameters.is_release}}', 'true'),
+                   eq(variables['Build.SourceBranchName'], 'master'),
+                   eq('${{parameters.name}}', 'linux'))
+  - bash: |
+      set -euo pipefail
+      eval "$(./dev-env/bin/dade-assist)"
+      TARBALL=daml-sdk-${{parameters.release_tag}}-${{parameters.name}}.tar.gz
+      cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL
+      echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
+    name: publish
+    condition: and(succeeded(),
+                   eq('${{parameters.is_release}}', 'true'),
+                   eq(variables['Build.SourceBranchName'], 'master'))
+  - task: PublishPipelineArtifact@0
+    inputs:
+      targetPath: $(Build.StagingDirectory)/$(publish.tarball)
+      artifactName: $(publish.tarball)
+    condition: and(succeeded(),
+                   eq('${{parameters.is_release}}', 'true'),
+                   eq(variables['Build.SourceBranchName'], 'master'))

ci/build-windows.yml

@@ -3,6 +3,7 @@
 
 parameters:
   release_tag: ''
+  is_release: ''
 
 steps:
   - bash: ci/configure-bazel.sh
@@ -26,3 +27,41 @@ steps:
     inputs:
       pathtoPublish: 'bazel-testlogs/'
       artifactName: 'Test logs'
+
+  - bash: |
+      export DAML_SDK_RELEASE_VERSION=${{parameters.release_tag}}
+      INSTALLER=daml-sdk-${{parameters.release_tag}}-windows.exe
+      mv "bazel-bin/release/windows-installer/daml-sdk-installer.exe" "$(Build.StagingDirectory)/$INSTALLER"
+      chmod +x "$(Build.StagingDirectory)/$INSTALLER"
+      cleanup () {
+          rm -f signing_key.pfx
+      }
+      trap cleanup EXIT
+      echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
+      MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
+      rm signing_key.pfx
+      echo "##vso[task.setvariable variable=installer;isOutput=true]$INSTALLER"
+      bazel build //release:sdk-release-tarball
+      TARBALL=daml-sdk-${{parameters.release_tag}}-windows.tar.gz
+      cp bazel-bin/release/sdk-release-tarball.tar.gz '$(Build.StagingDirectory)'/$TARBALL
+      echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
+    name: publish
+    env:
+      SIGNING_KEY: $(microsoft-code-signing)
+    condition: and(succeeded(),
+                   eq('${{parameters.is_release}}', 'true'),
+                   eq(variables['Build.SourceBranchName'], 'master'))
+  - task: PublishPipelineArtifact@0
+    condition: and(succeeded(),
+                   eq('${{parameters.is_release}}', 'true'),
+                   eq(variables['Build.SourceBranchName'], 'master'))
+    inputs:
+      targetPath: $(Build.StagingDirectory)/$(publish.installer)
+      artifactName: $(publish.installer)
+  - task: PublishPipelineArtifact@0
+    condition: and(succeeded(),
+                   eq('${{parameters.is_release}}', 'true'),
+                   eq(variables['Build.SourceBranchName'], 'master'))
+    inputs:
+      targetPath: $(Build.StagingDirectory)/$(publish.tarball)
+      artifactName: $(publish.tarball)