patching Bazel on Windows (infra bits, no patch yet) (#5918)

patch Bazel on Windows (ci setup)

We have a weird, intermittent bug on Windows where Bazel gets into a
broken state. To investigate, we need to patch Bazel to add more debug
output than present in the official distribution. This PR adds the basic
infrastructure we need to download the Bazel source code, apply a patch,
compile it, and make that binary available to the rest of the build.
This is for Windows only as we already have the ability to do similar
things on Linux and macOS through Nix.

This PR does not contain any intresting patch to Bazel, just the minimum
that we can check we are actually using the patched version.

CHANGELOG_BEGIN
CHANGELOG_END

.dadew

@@ -7,7 +7,7 @@
         "msys2",
         "curl-7.65.3",
         "cacert",
-        "vcredist-14.0.23026",
+        "vcredist-14.24.28127.4",
         "bazel",
         "nodejs-10.16.3",
         "python-3.8.2",

azure-pipelines.yml

@@ -149,9 +149,14 @@ jobs:
           trigger_sha: '$(trigger_sha)'
       - template: ci/report-end.yml
 
+  - template: ci/patch_bazel_windows/compile.yml
+    parameters:
+      final_job_name: patch_bazel_windows
+
   - job: Windows
     dependsOn:
       - check_for_release
+      - patch_bazel_windows
     variables:
       release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
       release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
@@ -218,6 +223,7 @@ jobs:
   - job: compatibility_windows
     dependsOn:
       - check_for_release
+      - patch_bazel_windows
     timeoutInMinutes: 60
     pool:
       name: 'windows-pool'

ci/patch_bazel_windows/compile.yml

@@ -0,0 +1,116 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+parameters:
+- name: final_job_name
+
+jobs:
+
+- job: patch_bazel_pre_check
+  pool:
+    name: linux-pool
+  steps:
+  - checkout: self
+  - bash: |
+      set -euo pipefail
+
+      CACHE_KEY="$(md5sum $(find ci/patch_bazel_windows -type f) | md5sum | awk '{print $1}')"
+      TARGET="patch_bazel_windows/bazel-$CACHE_KEY.zip"
+      TARGET_URL="https://daml-binaries.da-ext.net/$TARGET"
+
+      if [ "200" = "$(curl -Is "$TARGET_URL" | head -1 | awk '{print $2}')" ]; then
+          SHOULD_RUN=false
+      else
+          SHOULD_RUN=true
+      fi
+      setvar() {
+          echo "$1: $2"
+          echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
+      }
+      setvar cache_key $CACHE_KEY
+      setvar should_run $SHOULD_RUN
+      setvar target $TARGET
+      setvar target_url $TARGET_URL
+    name: out
+
+- job: patch_bazel_compile
+  dependsOn:
+    - patch_bazel_pre_check
+  variables:
+    cache_key: $[ dependencies.patch_bazel_pre_check.outputs['out.cache_key'] ]
+    should_run: $[ dependencies.patch_bazel_pre_check.outputs['out.should_run'] ]
+    bazel_base_version: 2.1.0
+  pool:
+    vmImage: windows-2019
+  steps:
+  - checkout: self
+    condition: eq(variables.should_run, 'true')
+  - bash: |
+      git clone https://github.com/bazelbuild/bazel.git
+      cd bazel
+      git checkout $(bazel_base_version)
+      git apply --ignore-space-change --ignore-whitespace --whitespace=nowarn < ../ci/patch_bazel_windows/patch-bazel
+    condition: eq(variables.should_run, 'true')
+  - powershell: |
+      choco install msys2 --noprogress --yes
+      choco install zip --noprogress --yes
+    condition: eq(variables.should_run, 'true')
+  - powershell: |
+      C:\tools\msys64\usr\bin\pacman -S zip --noconfirm
+    condition: eq(variables.should_run, 'true')
+  - bash: |
+      set -euo pipefail
+      cd bazel
+      bazel build src/main/cpp:client src:package-zip_jdk_minimal -c opt --stamp --embed_label $(bazel_base_version)-patched-$(cache_key)
+      # Note (MK) For some reason, the `zip` from chocolatey seems to result in
+      # a “zip file structure invalid” error. I’ve tried adding msys to PATH so the Bazel
+      # rules pick up `zip` from msys but that broke other things. So for now
+      # we skip the final Bazel rule to build the self-extracting exe and instead
+      # call `zip` from msys separately.
+      /c/tools/msys64/msys2_shell.cmd -defterm -no-start -here -c "cat bazel-bin/src/main/cpp/client.exe bazel-bin/src/package_jdk_minimal.zip > bazel.exe && zip -A bazel.exe"
+      mkdir '$(Build.StagingDirectory)\patched-bazel'
+      zip bazel.zip bazel.exe
+      cp bazel.zip '$(Build.StagingDirectory)\patched-bazel'
+    condition: eq(variables.should_run, 'true')
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: $(Build.StagingDirectory)/patched-bazel
+      artifactName: patched-bazel
+    condition: eq(variables.should_run, 'true')
+
+- job: ${{ parameters.final_job_name }}
+  dependsOn:
+    - patch_bazel_compile
+    - patch_bazel_pre_check
+  variables:
+    cache_key: $[ dependencies.patch_bazel_pre_check.outputs['out.cache_key'] ]
+    target: $[ dependencies.patch_bazel_pre_check.outputs['out.target'] ]
+    target_url: $[ dependencies.patch_bazel_pre_check.outputs['out.target_url'] ]
+    should_run: $[ dependencies.patch_bazel_pre_check.outputs['out.should_run'] ]
+  pool:
+    name: linux-pool
+  steps:
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: patched-bazel
+      path: $(Build.StagingDirectory)/patched-bazel
+    condition: eq(variables.should_run, 'true')
+  - bash: |
+      set -euo pipefail
+
+      SOURCE='$(Build.StagingDirectory)/patched-bazel/bazel.zip'
+
+      GCS_KEY=$(mktemp)
+      cleanup() {
+          rm -rf $GCS_KEY
+      }
+      trap cleanup EXIT
+      # This will break on external PRs.
+      echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
+      gcloud auth activate-service-account --key-file=$GCS_KEY
+      BOTO_CONFIG=/dev/null gsutil cp "$SOURCE" "gs://daml-binaries/$(target)"
+      echo "url: $(target_url)"
+      echo "hash: $(sha256sum "$(Build.StagingDirectory)/patched-bazel/bazel.zip" | awk '{print $1}')"
+    env:
+      GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
+    condition: eq(variables.should_run, 'true')

ci/patch_bazel_windows/patch-bazel

@@ -0,0 +1,13 @@
+diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+index 0fa1b5db2e..e6a1494ae5 100644
+--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
++++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+@@ -144,7 +144,7 @@ public final class RemoteModule extends BlazeModule {
+     env.getEventBus().register(this);
+     String invocationId = env.getCommandId().toString();
+     String buildRequestId = env.getBuildRequestId();
+-    env.getReporter().handle(Event.info(String.format("Invocation ID: %s", invocationId)));
++    env.getReporter().handle(Event.info(String.format("Invocation id: %s", invocationId)));
+ 
+     Path logDir =
+         env.getOutputBase().getRelative(env.getRuntime().getProductName() + "-remote-logs");

dev-env/windows/manifests/bazel.json

@@ -5,13 +5,13 @@
   "bin": "bazel.exe",
   "architecture": {
     "64bit": {
-      "url": "https://github.com/bazelbuild/bazel/releases/download/2.1.0/bazel-2.1.0-windows-x86_64.zip",
+      "url": "https://daml-binaries.da-ext.net/patch_bazel_windows/bazel-fd12df4ebdd9cfe98bd623c5a827a288.zip",
-      "hash": "0c8492b49310f73cdfbc4df9173b5f2cd2aa547d8dfbe46b47b93f3d712e5864"
+      "hash": "e1d742eeb0752d74af5257edf1d7bc0a332ddb3a76420aef137bc33738ecc165"
     }
   },
   "depends": [
     "msys2",
-    "vcredist-14.0.23026"
+    "vcredist-14.24.28127.4"
   ],
   "env_set": {
     "BAZEL_SH": "$(appdir msys2)\\current\\usr\\bin\\bash.exe"

dev-env/windows/manifests/vcredist-14.0.23026.json

@@ -1,20 +0,0 @@
-{
-  "homepage": "https://support.microsoft.com/pl-pl/help/2977003.",
-  "version": "14.0.23026",
-  "license": {
-    "url": "https://docs.microsoft.com/en-us/visualstudio/productinfo/2015-redistribution-vs"
-  },
-  "architecture": {
-  "64bit": {
-    "url": "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe",
-    "hash": "5eea714e1f22f1875c1cb7b1738b0c0b1f02aec5ecb95f0fdb1c5171c6cd93a3"
-    }
-  },
-  "pre_install": "copy-item $dir\\vc_redist.x64.exe $dir\\uninstall.exe",
-  "installer": {
-    "script": "Start-Process \"$dir\\vc_redist.x64.exe\" -ArgumentList \"/Install\", \"/Quiet\", \"/NoRestart\" -NoNewWindow -Wait"
-  },
-  "uninstaller": {
-    "script": "Start-Process \"$dir\\uninstall.exe\" -ArgumentList \"/Uninstall\", \"/Quiet\", \"/NoRestart\" -NoNewWindow -Wait"
-  }
-}

dev-env/windows/manifests/vcredist-14.24.28127.4.json

@@ -0,0 +1,22 @@
+{
+    "homepage": "https://www.visualstudio.com/downloads/",
+    "description": "Microsoft Visual C++ Redistributable for Visual Studio 2005/2008/2010/2012/2013/2015-2019.",
+    "version": "14.24.28127.4",
+    "license": {
+        "identifier": "Freeware",
+        "url": "https://www.microsoft.com/en-us/legal/intellectualproperty/copyright/default.aspx"
+    },
+    "url": [
+        "https://download.visualstudio.microsoft.com/download/pr/3b070396-b7fb-4eee-aa8b-102a23c3e4f4/40EA2955391C9EAE3E35619C4C24B5AAF3D17AEAA6D09424EE9672AA9372AEED/VC_redist.x64.exe",
+        "https://download.visualstudio.microsoft.com/download/pr/9307e627-aaac-42cb-a32a-a39e166ee8cb/E59AE3E886BD4571A811FE31A47959AE5C40D87C583F786816C60440252CD7EC/VC_redist.x86.exe"
+    ],
+    "hash": [
+        "40ea2955391c9eae3e35619c4c24b5aaf3d17aeaa6d09424ee9672aa9372aeed",
+        "e59ae3e886bd4571a811fe31a47959ae5c40d87c583f786816c60440252cd7ec"
+    ],
+    "post_install": [
+        "Invoke-ExternalCommand -FilePath \"$dir\\vc_redist.x64.exe\" -ArgumentList \"/fo /quiet /norestart\" -RunAs | Out-Null",
+        "Invoke-ExternalCommand -FilePath \"$dir\\vc_redist.x86.exe\" -ArgumentList \"/fo /quiet /norestart\" -RunAs | Out-Null"
+    ],
+    "notes": "You can now remove all vcredist installers with 'scoop uninstall vcredist vcredist2005 vcredist2008 vcredist2010 vcredist2012 vcredist2013'"
+}

infra/binaries.tf

@@ -0,0 +1,91 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+resource "google_storage_bucket" "binaries" {
+  project = "${local.project}"
+  name    = "daml-binaries"
+  labels  = "${local.labels}"
+
+  # SLA is enough for a cache and is cheaper than MULTI_REGIONAL
+  # see https://cloud.google.com/storage/docs/storage-classes
+  storage_class = "REGIONAL"
+
+  # Use a normal region since the storage_class is regional
+  location = "${local.region}"
+}
+
+resource "google_storage_bucket_acl" "binaries" {
+  bucket      = "${google_storage_bucket.binaries.name}"
+  default_acl = "publicread"
+  role_entity = [
+    "OWNER:project-owners-${data.google_project.current.number}",
+    "OWNER:project-editors-${data.google_project.current.number}",
+    "READER:project-viewers-${data.google_project.current.number}",
+    "READER:allUsers",
+  ]
+}
+
+// allow rw access for CI writer (see writer.tf)
+resource "google_storage_bucket_iam_member" "binaries-ci-rw" {
+  bucket = "${google_storage_bucket.binaries.name}"
+
+  # https://cloud.google.com/storage/docs/access-control/iam-roles
+  role   = "roles/storage.objectAdmin"
+  member = "serviceAccount:${google_service_account.writer.email}"
+}
+
+
+output binaries_ip {
+  description = "The external IP assigned to the global fowarding rule."
+  value       = "${google_compute_global_address.binaries.address}"
+}
+
+resource "google_compute_backend_bucket" "binaries" {
+  project     = "${local.project}"
+  name        = "binaries-backend"
+  bucket_name = "${google_storage_bucket.binaries.name}"
+  enable_cdn  = true
+}
+
+resource "google_compute_global_address" "binaries" {
+  project    = "${local.project}"
+  name       = "binaries-address"
+  ip_version = "IPV4"
+}
+
+resource "google_compute_url_map" "binaries" {
+  project         = "${local.project}"
+  name            = "binaries"
+  default_service = "${google_compute_backend_bucket.binaries.self_link}"
+}
+
+resource "google_compute_target_http_proxy" "binaries" {
+  project = "${local.project}"
+  name    = "binaries-http-proxy"
+  url_map = "${google_compute_url_map.binaries.self_link}"
+}
+
+resource "google_compute_global_forwarding_rule" "binaries-http" {
+  project    = "${local.project}"
+  name       = "binaries-http"
+  target     = "${google_compute_target_http_proxy.binaries.self_link}"
+  ip_address = "${google_compute_global_address.binaries.address}"
+  port_range = "80"
+  depends_on = ["google_compute_global_address.binaries"]
+}
+
+resource "google_compute_target_https_proxy" "binaries" {
+  project          = "${local.project}"
+  name             = "binaries-https-proxy"
+  url_map          = "${google_compute_url_map.binaries.self_link}"
+  ssl_certificates = ["${local.ssl_certificate}"]
+}
+
+resource "google_compute_global_forwarding_rule" "https" {
+  project    = "${local.project}"
+  name       = "binaries-https"
+  target     = "${google_compute_target_https_proxy.binaries.self_link}"
+  ip_address = "${google_compute_global_address.binaries.address}"
+  port_range = "443"
+  depends_on = ["google_compute_global_address.binaries"]
+}