automated ghc-lib build (#6188)

automated ghc-lib build

This PR aims at automating the build of ghc-lib. The current process
still has a few manual steps; it needs to be updated because Bintray is
going away, so this seemed like a good opportunity to fully automate it.

This works like the "patch bazel on Windows" jobs: the filename will
contain a hash of the `ci/da-ghc-lib` folder, and the job will run only
if the corresponding filename does not yet exist on the GCS bucket. PRs
aiming at changing the ghc-lib version will need to run twice: once to
create the artifacts, and once to change the `stack-snapshot.yaml` file
to match.

CHANGELOG_BEGIN
CHANGELOG_END

azure-pipelines.yml

@@ -82,8 +82,13 @@ jobs:
       - checkout: self
       - bash: ci/check-changelog.sh $(fork_sha)
 
+  - template: ci/da-ghc-lib/compile.yml
+    parameters:
+      final_job_name: da_ghc_lib
+
   - job: Linux
     dependsOn:
+      - da_ghc_lib
       - check_for_release
     variables:
       release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
@@ -121,6 +126,7 @@ jobs:
 
   - job: macOS
     dependsOn:
+      - da_ghc_lib
       - check_for_release
     timeoutInMinutes: 360
     pool:
@@ -155,6 +161,7 @@ jobs:
 
   - job: Windows
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - patch_bazel_windows
     variables:
@@ -191,6 +198,7 @@ jobs:
 
   - job: compatibility_ts_libs
     dependsOn:
+      - da_ghc_lib
       - check_for_release
     timeoutInMinutes: 60
     pool:
@@ -205,6 +213,7 @@ jobs:
 
   - job: compatibility_linux
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - compatibility_ts_libs
     timeoutInMinutes: 60
@@ -222,6 +231,7 @@ jobs:
 
   - job: compatibility_macos
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - compatibility_ts_libs
     timeoutInMinutes: 60
@@ -238,6 +248,7 @@ jobs:
 
   - job: compatibility_windows
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - compatibility_ts_libs
       - patch_bazel_windows
@@ -470,7 +481,9 @@ jobs:
       - template: ci/report-end.yml
 
   - job: write_ledger_dump
-    dependsOn: [ "check_for_release" ]
+    dependsOn:
+      - check_for_release
+      - da_ghc_lib
     pool:
       vmImage: "Ubuntu-16.04"
     condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),

ci/da-ghc-lib/compile.yml

@@ -0,0 +1,73 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+parameters:
+- name: final_job_name
+
+jobs:
+
+- job: ${{ parameters.final_job_name }}
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  variables:
+    ghc-lib-sha: 'd2eadb7ef4a6d4f33a6a04056d2e57f11a59ef0f'
+    base-sha: '9c787d4d24f2b515934c8503ee2bbd7cfac4da20'
+    patches: '4650347db9a7e8c65831ab83b80da43b28c3deba 833ca63be2ab14871874ccb6974921e8952802e9'
+    flavor: 'ghc-8.8.1'
+  steps:
+  - checkout: self
+  - bash: |
+      set -euo pipefail
+
+      CACHE_KEY="$(find ci/da-ghc-lib -type f -print0 | sort -z | xargs -r0 md5sum | md5sum | awk '{print $1}')"
+      LIB="da-ghc-lib/ghc-lib-$CACHE_KEY.tar.gz"
+      PARSER="da-ghc-lib/ghc-lib-parser-$CACHE_KEY.tar.gz"
+      URL_BASE="https://daml-binaries.da-ext.net"
+      LIB_STATUS="$(curl -Is "$URL_BASE/$LIB" | head -1 | awk '{print $2}')"
+      PARSER_STATUS="$(curl -Is "$URL_BASE/$PARSER" | head -1 | awk '{print $2}')"
+      if [ "200" = "$LIB_STATUS" ] && [ "200" = "$PARSER_STATUS" ]; then
+          echo "Both artifacts exist; no need to run."
+          exit 0
+      fi
+
+      DIR="$(pwd)"
+      WORKDIR="$(mktemp -d -p "$(pwd)")"
+      cleanup() {
+          rm -rf "$WORKDIR"
+      }
+      trap cleanup EXIT
+
+      cd $WORKDIR
+
+      git clone https://github.com/digital-asset/ghc-lib.git
+      cd ghc-lib
+      git checkout $(ghc-lib-sha)
+      stack runhaskell --package extra \
+                       --package optparse-applicative \
+                       CI.hs \
+                       -- \
+                         --da \
+                         --merge-base-sha=$(base-sha) \
+                         $(for p in $(patches); do echo -n "--patch $p "; done) \
+                         --gen-flavor=$(flavor) \
+            | tee log
+      VERSION=$(cat log | tail -1)
+
+      GCS_KEY=$(mktemp)
+      cleanup() {
+          rm -rf $GCS_KEY
+          rm -rf "$WORKDIR"
+      }
+      trap cleanup EXIT
+      # This will break on external PRs.
+      echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
+      gcloud auth activate-service-account --key-file=$GCS_KEY
+      BOTO_CONFIG=/dev/null gsutil cp -n "ghc-lib-${VERSION}.tar.gz" "gs://daml-binaries/$LIB"
+      BOTO_CONFIG=/dev/null gsutil cp -n "ghc-lib-parser-${VERSION}.tar.gz" "gs://daml-binaries/$PARSER"
+      echo "Please add(/replace) the following lines to stack-snapshot.yaml:"
+      echo "  - archive: $URL_BASE/$LIB"
+      echo "    sha256: \"$(sha256sum ghc-lib-${VERSION}.tar.gz | awk '{print $1}')\""
+      echo "  - archive: $URL_BASE/$PARSER"
+      echo "    sha256: \"$(sha256sum ghc-lib-parser-${VERSION}.tar.gz | awk '{print $1}')\""
+    env:
+      GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)

stack-snapshot.yaml

@@ -3,10 +3,10 @@
 
 resolver: lts-14.1
 packages:
-  - archive: http://digitalassetsdk.bintray.com/ghc-lib/ghc-lib-parser-8.8.1.20200225.tar.gz
-    sha256: "31b6f1fc15d257e66d3264e62e229203087a2e799a9638f6d5ff6cc43ec0b4a4"
-  - archive: http://digitalassetsdk.bintray.com/ghc-lib/ghc-lib-8.8.1.20200225.tar.gz
-    sha256: "2cd22db09fdcb9bb121bd6f85024aba437ca2796ebe16453d15cbf2cb192023c"
+  - archive: https://daml-binaries.da-ext.net/da-ghc-lib/ghc-lib-48295232a0bebf0bf80b90447d0f5890.tar.gz
+    sha256: "32ef1e2aebdd473681e38fbbf4747b88a1e03d00ea90adb9917b613659acad9f"
+  - archive: https://daml-binaries.da-ext.net/da-ghc-lib/ghc-lib-parser-48295232a0bebf0bf80b90447d0f5890.tar.gz
+    sha256: "dbaaca05c677794261e102f1a7d6a4409d368a47feb214b6378fa1169d269ec3"
   - github: digital-asset/hlint
     commit: "3e78bce69749b22a80fec1e8eb853cc0c100c18e"
     sha256: "cf39f2b378485afc77ffdad4dbb057d5d9b4dfc5a38c76ddc44e920e537fb0fa"