From bda565fa4470d77a803fb4d9f0f6de41e4bf282a Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Tue, 12 May 2020 23:16:04 +0200 Subject: [PATCH] patching Bazel on Windows (infra bits, no patch yet) (#5918) patch Bazel on Windows (ci setup) We have a weird, intermittent bug on Windows where Bazel gets into a broken state. To investigate, we need to patch Bazel to add more debug output than present in the official distribution. This PR adds the basic infrastructure we need to download the Bazel source code, apply a patch, compile it, and make that binary available to the rest of the build. This is for Windows only as we already have the ability to do similar things on Linux and macOS through Nix. This PR does not contain any intresting patch to Bazel, just the minimum that we can check we are actually using the patched version. CHANGELOG_BEGIN CHANGELOG_END --- .dadew | 2 +- azure-pipelines.yml | 6 + ci/patch_bazel_windows/compile.yml | 116 ++++++++++++++++++ ci/patch_bazel_windows/patch-bazel | 13 ++ dev-env/windows/manifests/bazel.json | 6 +- .../manifests/vcredist-14.0.23026.json | 20 --- .../manifests/vcredist-14.24.28127.4.json | 22 ++++ infra/binaries.tf | 91 ++++++++++++++ 8 files changed, 252 insertions(+), 24 deletions(-) create mode 100644 ci/patch_bazel_windows/compile.yml create mode 100644 ci/patch_bazel_windows/patch-bazel delete mode 100644 dev-env/windows/manifests/vcredist-14.0.23026.json create mode 100644 dev-env/windows/manifests/vcredist-14.24.28127.4.json create mode 100644 infra/binaries.tf diff --git a/.dadew b/.dadew index 65945cd4769..6cb952dcb4c 100644 --- a/.dadew +++ b/.dadew @@ -7,7 +7,7 @@ "msys2", "curl-7.65.3", "cacert", - "vcredist-14.0.23026", + "vcredist-14.24.28127.4", "bazel", "nodejs-10.16.3", "python-3.8.2", diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c779c3c13ba..a2b8d104e47 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -149,9 +149,14 @@ jobs: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml + - template: ci/patch_bazel_windows/compile.yml + parameters: + final_job_name: patch_bazel_windows + - job: Windows dependsOn: - check_for_release + - patch_bazel_windows variables: release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ] @@ -218,6 +223,7 @@ jobs: - job: compatibility_windows dependsOn: - check_for_release + - patch_bazel_windows timeoutInMinutes: 60 pool: name: 'windows-pool' diff --git a/ci/patch_bazel_windows/compile.yml b/ci/patch_bazel_windows/compile.yml new file mode 100644 index 00000000000..891dc5789f1 --- /dev/null +++ b/ci/patch_bazel_windows/compile.yml @@ -0,0 +1,116 @@ +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +parameters: +- name: final_job_name + +jobs: + +- job: patch_bazel_pre_check + pool: + name: linux-pool + steps: + - checkout: self + - bash: | + set -euo pipefail + + CACHE_KEY="$(md5sum $(find ci/patch_bazel_windows -type f) | md5sum | awk '{print $1}')" + TARGET="patch_bazel_windows/bazel-$CACHE_KEY.zip" + TARGET_URL="https://daml-binaries.da-ext.net/$TARGET" + + if [ "200" = "$(curl -Is "$TARGET_URL" | head -1 | awk '{print $2}')" ]; then + SHOULD_RUN=false + else + SHOULD_RUN=true + fi + setvar() { + echo "$1: $2" + echo "##vso[task.setvariable variable=$1;isOutput=true]$2" + } + setvar cache_key $CACHE_KEY + setvar should_run $SHOULD_RUN + setvar target $TARGET + setvar target_url $TARGET_URL + name: out + +- job: patch_bazel_compile + dependsOn: + - patch_bazel_pre_check + variables: + cache_key: $[ dependencies.patch_bazel_pre_check.outputs['out.cache_key'] ] + should_run: $[ dependencies.patch_bazel_pre_check.outputs['out.should_run'] ] + bazel_base_version: 2.1.0 + pool: + vmImage: windows-2019 + steps: + - checkout: self + condition: eq(variables.should_run, 'true') + - bash: | + git clone https://github.com/bazelbuild/bazel.git + cd bazel + git checkout $(bazel_base_version) + git apply --ignore-space-change --ignore-whitespace --whitespace=nowarn < ../ci/patch_bazel_windows/patch-bazel + condition: eq(variables.should_run, 'true') + - powershell: | + choco install msys2 --noprogress --yes + choco install zip --noprogress --yes + condition: eq(variables.should_run, 'true') + - powershell: | + C:\tools\msys64\usr\bin\pacman -S zip --noconfirm + condition: eq(variables.should_run, 'true') + - bash: | + set -euo pipefail + cd bazel + bazel build src/main/cpp:client src:package-zip_jdk_minimal -c opt --stamp --embed_label $(bazel_base_version)-patched-$(cache_key) + # Note (MK) For some reason, the `zip` from chocolatey seems to result in + # a “zip file structure invalid” error. I’ve tried adding msys to PATH so the Bazel + # rules pick up `zip` from msys but that broke other things. So for now + # we skip the final Bazel rule to build the self-extracting exe and instead + # call `zip` from msys separately. + /c/tools/msys64/msys2_shell.cmd -defterm -no-start -here -c "cat bazel-bin/src/main/cpp/client.exe bazel-bin/src/package_jdk_minimal.zip > bazel.exe && zip -A bazel.exe" + mkdir '$(Build.StagingDirectory)\patched-bazel' + zip bazel.zip bazel.exe + cp bazel.zip '$(Build.StagingDirectory)\patched-bazel' + condition: eq(variables.should_run, 'true') + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.StagingDirectory)/patched-bazel + artifactName: patched-bazel + condition: eq(variables.should_run, 'true') + +- job: ${{ parameters.final_job_name }} + dependsOn: + - patch_bazel_compile + - patch_bazel_pre_check + variables: + cache_key: $[ dependencies.patch_bazel_pre_check.outputs['out.cache_key'] ] + target: $[ dependencies.patch_bazel_pre_check.outputs['out.target'] ] + target_url: $[ dependencies.patch_bazel_pre_check.outputs['out.target_url'] ] + should_run: $[ dependencies.patch_bazel_pre_check.outputs['out.should_run'] ] + pool: + name: linux-pool + steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: patched-bazel + path: $(Build.StagingDirectory)/patched-bazel + condition: eq(variables.should_run, 'true') + - bash: | + set -euo pipefail + + SOURCE='$(Build.StagingDirectory)/patched-bazel/bazel.zip' + + GCS_KEY=$(mktemp) + cleanup() { + rm -rf $GCS_KEY + } + trap cleanup EXIT + # This will break on external PRs. + echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY + gcloud auth activate-service-account --key-file=$GCS_KEY + BOTO_CONFIG=/dev/null gsutil cp "$SOURCE" "gs://daml-binaries/$(target)" + echo "url: $(target_url)" + echo "hash: $(sha256sum "$(Build.StagingDirectory)/patched-bazel/bazel.zip" | awk '{print $1}')" + env: + GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) + condition: eq(variables.should_run, 'true') diff --git a/ci/patch_bazel_windows/patch-bazel b/ci/patch_bazel_windows/patch-bazel new file mode 100644 index 00000000000..3145e986ad6 --- /dev/null +++ b/ci/patch_bazel_windows/patch-bazel @@ -0,0 +1,13 @@ +diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +index 0fa1b5db2e..e6a1494ae5 100644 +--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java ++++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +@@ -144,7 +144,7 @@ public final class RemoteModule extends BlazeModule { + env.getEventBus().register(this); + String invocationId = env.getCommandId().toString(); + String buildRequestId = env.getBuildRequestId(); +- env.getReporter().handle(Event.info(String.format("Invocation ID: %s", invocationId))); ++ env.getReporter().handle(Event.info(String.format("Invocation id: %s", invocationId))); + + Path logDir = + env.getOutputBase().getRelative(env.getRuntime().getProductName() + "-remote-logs"); diff --git a/dev-env/windows/manifests/bazel.json b/dev-env/windows/manifests/bazel.json index 8a1308bc897..e9107b863f9 100644 --- a/dev-env/windows/manifests/bazel.json +++ b/dev-env/windows/manifests/bazel.json @@ -5,13 +5,13 @@ "bin": "bazel.exe", "architecture": { "64bit": { - "url": "https://github.com/bazelbuild/bazel/releases/download/2.1.0/bazel-2.1.0-windows-x86_64.zip", - "hash": "0c8492b49310f73cdfbc4df9173b5f2cd2aa547d8dfbe46b47b93f3d712e5864" + "url": "https://daml-binaries.da-ext.net/patch_bazel_windows/bazel-fd12df4ebdd9cfe98bd623c5a827a288.zip", + "hash": "e1d742eeb0752d74af5257edf1d7bc0a332ddb3a76420aef137bc33738ecc165" } }, "depends": [ "msys2", - "vcredist-14.0.23026" + "vcredist-14.24.28127.4" ], "env_set": { "BAZEL_SH": "$(appdir msys2)\\current\\usr\\bin\\bash.exe" diff --git a/dev-env/windows/manifests/vcredist-14.0.23026.json b/dev-env/windows/manifests/vcredist-14.0.23026.json deleted file mode 100644 index 235ec861bb3..00000000000 --- a/dev-env/windows/manifests/vcredist-14.0.23026.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "homepage": "https://support.microsoft.com/pl-pl/help/2977003.", - "version": "14.0.23026", - "license": { - "url": "https://docs.microsoft.com/en-us/visualstudio/productinfo/2015-redistribution-vs" - }, - "architecture": { - "64bit": { - "url": "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe", - "hash": "5eea714e1f22f1875c1cb7b1738b0c0b1f02aec5ecb95f0fdb1c5171c6cd93a3" - } - }, - "pre_install": "copy-item $dir\\vc_redist.x64.exe $dir\\uninstall.exe", - "installer": { - "script": "Start-Process \"$dir\\vc_redist.x64.exe\" -ArgumentList \"/Install\", \"/Quiet\", \"/NoRestart\" -NoNewWindow -Wait" - }, - "uninstaller": { - "script": "Start-Process \"$dir\\uninstall.exe\" -ArgumentList \"/Uninstall\", \"/Quiet\", \"/NoRestart\" -NoNewWindow -Wait" - } -} diff --git a/dev-env/windows/manifests/vcredist-14.24.28127.4.json b/dev-env/windows/manifests/vcredist-14.24.28127.4.json new file mode 100644 index 00000000000..39ba3c192eb --- /dev/null +++ b/dev-env/windows/manifests/vcredist-14.24.28127.4.json @@ -0,0 +1,22 @@ +{ + "homepage": "https://www.visualstudio.com/downloads/", + "description": "Microsoft Visual C++ Redistributable for Visual Studio 2005/2008/2010/2012/2013/2015-2019.", + "version": "14.24.28127.4", + "license": { + "identifier": "Freeware", + "url": "https://www.microsoft.com/en-us/legal/intellectualproperty/copyright/default.aspx" + }, + "url": [ + "https://download.visualstudio.microsoft.com/download/pr/3b070396-b7fb-4eee-aa8b-102a23c3e4f4/40EA2955391C9EAE3E35619C4C24B5AAF3D17AEAA6D09424EE9672AA9372AEED/VC_redist.x64.exe", + "https://download.visualstudio.microsoft.com/download/pr/9307e627-aaac-42cb-a32a-a39e166ee8cb/E59AE3E886BD4571A811FE31A47959AE5C40D87C583F786816C60440252CD7EC/VC_redist.x86.exe" + ], + "hash": [ + "40ea2955391c9eae3e35619c4c24b5aaf3d17aeaa6d09424ee9672aa9372aeed", + "e59ae3e886bd4571a811fe31a47959ae5c40d87c583f786816c60440252cd7ec" + ], + "post_install": [ + "Invoke-ExternalCommand -FilePath \"$dir\\vc_redist.x64.exe\" -ArgumentList \"/fo /quiet /norestart\" -RunAs | Out-Null", + "Invoke-ExternalCommand -FilePath \"$dir\\vc_redist.x86.exe\" -ArgumentList \"/fo /quiet /norestart\" -RunAs | Out-Null" + ], + "notes": "You can now remove all vcredist installers with 'scoop uninstall vcredist vcredist2005 vcredist2008 vcredist2010 vcredist2012 vcredist2013'" +} diff --git a/infra/binaries.tf b/infra/binaries.tf new file mode 100644 index 00000000000..22d9b8f7b00 --- /dev/null +++ b/infra/binaries.tf @@ -0,0 +1,91 @@ +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +resource "google_storage_bucket" "binaries" { + project = "${local.project}" + name = "daml-binaries" + labels = "${local.labels}" + + # SLA is enough for a cache and is cheaper than MULTI_REGIONAL + # see https://cloud.google.com/storage/docs/storage-classes + storage_class = "REGIONAL" + + # Use a normal region since the storage_class is regional + location = "${local.region}" +} + +resource "google_storage_bucket_acl" "binaries" { + bucket = "${google_storage_bucket.binaries.name}" + default_acl = "publicread" + role_entity = [ + "OWNER:project-owners-${data.google_project.current.number}", + "OWNER:project-editors-${data.google_project.current.number}", + "READER:project-viewers-${data.google_project.current.number}", + "READER:allUsers", + ] +} + +// allow rw access for CI writer (see writer.tf) +resource "google_storage_bucket_iam_member" "binaries-ci-rw" { + bucket = "${google_storage_bucket.binaries.name}" + + # https://cloud.google.com/storage/docs/access-control/iam-roles + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.writer.email}" +} + + +output binaries_ip { + description = "The external IP assigned to the global fowarding rule." + value = "${google_compute_global_address.binaries.address}" +} + +resource "google_compute_backend_bucket" "binaries" { + project = "${local.project}" + name = "binaries-backend" + bucket_name = "${google_storage_bucket.binaries.name}" + enable_cdn = true +} + +resource "google_compute_global_address" "binaries" { + project = "${local.project}" + name = "binaries-address" + ip_version = "IPV4" +} + +resource "google_compute_url_map" "binaries" { + project = "${local.project}" + name = "binaries" + default_service = "${google_compute_backend_bucket.binaries.self_link}" +} + +resource "google_compute_target_http_proxy" "binaries" { + project = "${local.project}" + name = "binaries-http-proxy" + url_map = "${google_compute_url_map.binaries.self_link}" +} + +resource "google_compute_global_forwarding_rule" "binaries-http" { + project = "${local.project}" + name = "binaries-http" + target = "${google_compute_target_http_proxy.binaries.self_link}" + ip_address = "${google_compute_global_address.binaries.address}" + port_range = "80" + depends_on = ["google_compute_global_address.binaries"] +} + +resource "google_compute_target_https_proxy" "binaries" { + project = "${local.project}" + name = "binaries-https-proxy" + url_map = "${google_compute_url_map.binaries.self_link}" + ssl_certificates = ["${local.ssl_certificate}"] +} + +resource "google_compute_global_forwarding_rule" "https" { + project = "${local.project}" + name = "binaries-https" + target = "${google_compute_target_https_proxy.binaries.self_link}" + ip_address = "${google_compute_global_address.binaries.address}" + port_range = "443" + depends_on = ["google_compute_global_address.binaries"] +}