From fef712bf60401330bcf4378710f4b7d5519f772d Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Wed, 27 Jan 2021 17:38:34 +0100 Subject: [PATCH] Upgrade linux nodes to 20.04 (#8617) CHANGELOG_BEGIN - Our Linux binaries are now built on Ubuntu 20.04 instead of 16.04. We do not expect any user-level impact, but please reach out if you do notice any issue that might be caused by this. CHANGELOG_END --- .bazelrc | 2 +- azure-cron.yml | 8 +- azure-pipelines.yml | 4 +- ci/build.yml | 14 +- ci/cron/daily-compat.yml | 12 +- ci/patch_bazel_windows/compile.yml | 8 +- ci/prs.yml | 8 +- dev-env/windows/manifests/bazel.json | 4 +- infra/main.tf | 2 + infra/vsts_agent_linux.tf | 2 - infra/vsts_agent_ubuntu_20_04.tf | 80 +++++++++ infra/vsts_agent_ubuntu_20_04_startup.sh | 198 +++++++++++++++++++++++ 12 files changed, 311 insertions(+), 31 deletions(-) create mode 100644 infra/vsts_agent_ubuntu_20_04.tf create mode 100644 infra/vsts_agent_ubuntu_20_04_startup.sh diff --git a/.bazelrc b/.bazelrc index e0d6af2ffa..e382bab0ae 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,7 +1,7 @@ # Bazel distributed cache, can be temporarily disabled by passing the following # flag: --noremote_accept_cached build:darwin --remote_http_cache=https://bazel-cache.da-ext.net -build:linux --remote_http_cache=https://bazel-cache.da-ext.net +build:linux --remote_http_cache=https://bazel-cache.da-ext.net/ubuntu_20_04 build --remote_upload_local_results=false # Enable the disk cache in addition to the http cache. build:linux --disk_cache=.bazel-cache/disk diff --git a/azure-cron.yml b/azure-cron.yml index 29cd9235b8..33636abc44 100644 --- a/azure-cron.yml +++ b/azure-cron.yml @@ -31,7 +31,7 @@ jobs: - job: docs timeoutInMinutes: 120 pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - checkout: self @@ -51,7 +51,7 @@ jobs: - job: docker_image timeoutInMinutes: 60 pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - checkout: self @@ -106,7 +106,7 @@ jobs: - job: vscode_marketplace timeoutInMinutes: 10 pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - checkout: self @@ -173,7 +173,7 @@ jobs: - job: download_stats timeoutInMinutes: 10 pool: - name: "linux-pool" + name: "ubuntu_20_04" demands: assignment -equals default steps: - checkout: self diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 78358648de..ad6454797e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ jobs: - job: check_perf_test pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default steps: - bash: | @@ -244,7 +244,7 @@ jobs: - release - check_for_release pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default variables: release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ] diff --git a/ci/build.yml b/ci/build.yml index c213f8704a..7eda4d2838 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -4,7 +4,7 @@ jobs: - job: git_sha pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - bash: | @@ -35,7 +35,7 @@ jobs: is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ] timeoutInMinutes: 360 pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - template: report-start.yml @@ -144,7 +144,7 @@ jobs: - da_ghc_lib timeoutInMinutes: 360 pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - template: report-start.yml @@ -205,7 +205,7 @@ jobs: - check_for_release timeoutInMinutes: 360 pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default steps: - template: report-start.yml @@ -222,7 +222,7 @@ jobs: - compatibility_ts_libs timeoutInMinutes: 360 pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default steps: - template: report-start.yml @@ -285,7 +285,7 @@ jobs: branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] pool: - name: "linux-pool" + name: "ubuntu_20_04" demands: assignment -equals default steps: - bash: | @@ -342,7 +342,7 @@ jobs: - compatibility_windows - Linux_scala_2_13 pool: - name: "linux-pool" + name: "ubuntu_20_04" demands: assignment -equals default variables: Linux.start: $[ dependencies.Linux.outputs['start.time'] ] diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml index 348aa2df93..8443af4b45 100644 --- a/ci/cron/daily-compat.yml +++ b/ci/cron/daily-compat.yml @@ -26,7 +26,7 @@ jobs: - job: compatibility_ts_libs timeoutInMinutes: 60 pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default steps: - checkout: self @@ -40,7 +40,7 @@ jobs: strategy: matrix: linux: - pool: linux-pool + pool: ubuntu_20_04 macos: pool: macOS-pool pool: @@ -73,7 +73,7 @@ jobs: - job: perf_speedy timeoutInMinutes: 120 pool: - name: "linux-pool" + name: "ubuntu_20_04" demands: assignment -equals default steps: - checkout: self @@ -121,7 +121,7 @@ jobs: - job: perf_http_json timeoutInMinutes: 120 pool: - name: "linux-pool" + name: "ubuntu_20_04" demands: assignment -equals default steps: - checkout: self @@ -179,7 +179,7 @@ jobs: - job: check_releases timeoutInMinutes: 240 pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default steps: - checkout: self @@ -203,7 +203,7 @@ jobs: timeoutInMinutes: 1200 condition: eq(variables['Build.SourceBranchName'], 'main') pool: - name: linux-pool + name: ubuntu_20_04 demands: assignment -equals default steps: - checkout: self diff --git a/ci/patch_bazel_windows/compile.yml b/ci/patch_bazel_windows/compile.yml index ce16cd0d2a..001ed5b495 100644 --- a/ci/patch_bazel_windows/compile.yml +++ b/ci/patch_bazel_windows/compile.yml @@ -8,7 +8,7 @@ jobs: - job: patch_bazel_pre_check pool: - name: linux-pool + name: ubuntu_20_04 steps: - checkout: self - bash: | @@ -63,8 +63,10 @@ jobs: condition: eq(variables.should_run, 'true') - bash: | set -euo pipefail + BAZEL=$(mktemp) + curl -sL https://github.com/bazelbuild/bazel/releases/download/$(bazel_base_version)/bazel-$(bazel_base_version)-windows-x86_64.exe > $BAZEL cd bazel - bazel build src/main/cpp:client src:package-zip_jdk_minimal -c opt --stamp --embed_label $(bazel_base_version)-patched-$(cache_key) + $BAZEL build src/main/cpp:client src:package-zip_jdk_minimal -c opt --stamp --embed_label $(bazel_base_version)-patched-$(cache_key) # Note (MK) For some reason, the `zip` from chocolatey seems to result in # a “zip file structure invalid” error. I’ve tried adding msys to PATH so the Bazel # rules pick up `zip` from msys but that broke other things. So for now @@ -92,7 +94,7 @@ jobs: target_url: $[ dependencies.patch_bazel_pre_check.outputs['out.target_url'] ] should_run: $[ dependencies.patch_bazel_pre_check.outputs['out.should_run'] ] pool: - name: linux-pool + name: ubuntu_20_04 steps: - task: DownloadPipelineArtifact@2 inputs: diff --git a/ci/prs.yml b/ci/prs.yml index c2b727e7d2..2feb7db55f 100644 --- a/ci/prs.yml +++ b/ci/prs.yml @@ -21,7 +21,7 @@ jobs: branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] condition: eq(variables['Build.Reason'], 'PullRequest') pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - checkout: self @@ -51,7 +51,7 @@ jobs: pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ] condition: eq(variables['Build.Reason'], 'PullRequest') pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default steps: - checkout: self @@ -86,7 +86,7 @@ jobs: - collect_build_data - check_for_release pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default variables: branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] @@ -138,7 +138,7 @@ jobs: - check_for_release - check_changelog_entry pool: - name: 'linux-pool' + name: 'ubuntu_20_04' demands: assignment -equals default variables: pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ] diff --git a/dev-env/windows/manifests/bazel.json b/dev-env/windows/manifests/bazel.json index a0e8e20218..887b7df035 100644 --- a/dev-env/windows/manifests/bazel.json +++ b/dev-env/windows/manifests/bazel.json @@ -5,8 +5,8 @@ "bin": "bazel.exe", "architecture": { "64bit": { - "url": "https://daml-binaries.da-ext.net/patch_bazel_windows/bazel-e0202b0f2f3c2d8c3b477ddb59bd30ce.zip", - "hash": "7177fbc5660626ea68713d6fab026581deced2e20aad0723e2b5ed3e2eb1f2dc" + "url": "https://daml-binaries.da-ext.net/patch_bazel_windows/bazel-f6f61bf596011e671cb3b327b0f29686.zip", + "hash": "5b9224a94da96403bffd0309e011d63504807386718d806d0aaf769f674626f5" } }, "depends": [ diff --git a/infra/main.tf b/infra/main.tf index 678e61b7fc..4b26dda90d 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -49,3 +49,5 @@ locals { ssl_certificate_hoogle = "https://www.googleapis.com/compute/v1/projects/da-dev-gcp-daml-language/global/sslCertificates/daml-lang-hoogle-app-service-https-cert" } + +resource "secret_resource" "vsts-token" {} diff --git a/infra/vsts_agent_linux.tf b/infra/vsts_agent_linux.tf index 9ef2cfd123..6db036e71e 100644 --- a/infra/vsts_agent_linux.tf +++ b/infra/vsts_agent_linux.tf @@ -1,8 +1,6 @@ # Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -resource "secret_resource" "vsts-token" {} - data "template_file" "vsts-agent-linux-startup" { template = "${file("${path.module}/vsts_agent_linux_startup.sh")}" diff --git a/infra/vsts_agent_ubuntu_20_04.tf b/infra/vsts_agent_ubuntu_20_04.tf new file mode 100644 index 0000000000..7a419809d3 --- /dev/null +++ b/infra/vsts_agent_ubuntu_20_04.tf @@ -0,0 +1,80 @@ +# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +data "template_file" "vsts-agent-ubuntu_20_04-startup" { + template = "${file("${path.module}/vsts_agent_ubuntu_20_04_startup.sh")}" + + vars = { + vsts_token = "${secret_resource.vsts-token.value}" + vsts_account = "digitalasset" + vsts_pool = "ubuntu_20_04" + } +} + +resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04" { + provider = "google-beta" + name = "vsts-agent-ubuntu-20-04" + base_instance_name = "vsts-agent-ubuntu-20-04" + region = "us-east1" + target_size = 1 + + version { + name = "vsts-agent-ubuntu-20-04" + instance_template = "${google_compute_instance_template.vsts-agent-ubuntu_20_04.self_link}" + } + + update_policy { + type = "PROACTIVE" + minimal_action = "REPLACE" + max_surge_fixed = 3 + min_ready_sec = 60 + } +} + +resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" { + name_prefix = "vsts-agent-ubuntu-20-04-" + machine_type = "c2-standard-8" + labels = "${local.machine-labels}" + + disk { + disk_size_gb = 200 + disk_type = "pd-ssd" + source_image = "ubuntu-os-cloud/ubuntu-2004-lts" + } + + lifecycle { + create_before_destroy = true + } + + metadata { + startup-script = "${data.template_file.vsts-agent-ubuntu_20_04-startup.rendered}" + + shutdown-script = < /etc/resolv.conf + +# delete self +rm -vf "$0" + +## Install system dependencies +apt-get update -q +apt-get install -qy \ + curl sudo \ + bzip2 rsync \ + jq liblttng-ust0 libcurl4 libkrb5-3 zlib1g \ + git \ + netcat \ + apt-transport-https \ + software-properties-common + +# Install dependencies for Chrome (to run Puppeteer tests on the gsg) +# list taken from: https://github.com/puppeteer/puppeteer/blob/a3d1536a6b6e282a43521bea28aef027a7133df8/docs/troubleshooting.md#chrome-headless-doesnt-launch-on-unix +# see https://github.com/digital-asset/daml/pull/5540 for context +apt-get install -qy \ + gconf-service \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libfontconfig1 \ + libgbm-dev \ + libgcc1 \ + libgconf-2-4 \ + libgdk-pixbuf2.0-0 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libstdc++6 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + ca-certificates \ + fonts-liberation \ + libappindicator1 \ + libnss3 \ + lsb-release \ + xdg-utils \ + wget + +curl -sSL https://dl.google.com/cloudagents/install-logging-agent.sh | bash + +#install docker +DOCKER_VERSION="5:20.10.2~3-0~ubuntu-$(lsb_release -cs)" +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +apt-key fingerprint 0EBFCD88 +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -qy docker-ce=$DOCKER_VERSION docker-ce-cli=$DOCKER_VERSION containerd.io + +#Start docker daemon +systemctl enable docker + +## Install the VSTS agent +groupadd --gid 3000 vsts +useradd \ + --create-home \ + --gid 3000 \ + --shell /bin/bash \ + --uid 3000 \ + vsts +#add docker group to user +usermod -aG docker vsts + +su --login vsts <<'AGENT_SETUP' +set -euo pipefail + +VSTS_ACCOUNT=${vsts_account} +VSTS_POOL=${vsts_pool} +VSTS_TOKEN=${vsts_token} + +mkdir -p ~/agent +cd ~/agent +echo 'assignment=default' > .capabilities + +echo Determining matching VSTS agent... +VSTS_AGENT_RESPONSE=$(curl -sSfL \ + -u "user:$VSTS_TOKEN" \ + -H 'Accept:application/json;api-version=3.0-preview' \ + "https://$VSTS_ACCOUNT.visualstudio.com/_apis/distributedtask/packages/agent?platform=linux-x64") + +VSTS_AGENT_URL=$(echo "$VSTS_AGENT_RESPONSE" \ + | jq -r '.value | map([.version.major,.version.minor,.version.patch,.downloadUrl]) | sort | .[length-1] | .[3]') + +if [ -z "$VSTS_AGENT_URL" -o "$VSTS_AGENT_URL" == "null" ]; then + echo 1>&2 error: could not determine a matching VSTS agent - check that account \'$VSTS_ACCOUNT\' is correct and the token is valid for that account + exit 1 +fi + +echo Downloading and installing VSTS agent... +curl -sSfL "$VSTS_AGENT_URL" | tar -xz --no-same-owner + +set +u +source ./env.sh +set -u + +./config.sh \ + --acceptTeeEula \ + --agent "$(hostname)" \ + --auth PAT \ + --pool "$VSTS_POOL" \ + --replace \ + --token "$VSTS_TOKEN" \ + --unattended \ + --url "https://$VSTS_ACCOUNT.visualstudio.com" +AGENT_SETUP + +## Hardening + +chown --recursive root:root /home/vsts/agent/{*.sh,bin,externals} + +## Install Nix + +# This needs to run inside of a user with sudo access +echo "vsts ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/nix_installation +su --command "sh <(curl -sSfL https://nixos.org/nix/install) --daemon" --login vsts +rm /etc/sudoers.d/nix_installation + +# Note: the "hydra.da-int.net" string is now part of the name of the key for +# legacy reasons; it bears no relation to the DNS hostname of the current +# cache. +cat < /etc/nix/nix.conf +binary-cache-public-keys = hydra.da-int.net-2:91tXuJGf/ExbAz7IWsMsxQ5FsO6lG/EGM5QVt+xhZu0= hydra.da-int.net-1:6Oy2+KYvI7xkAOg0gJisD7Nz/6m8CmyKMbWfSKUe03g= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs= +binary-caches = https://nix-cache.da-ext.net https://cache.nixos.org +build-users-group = nixbld +cores = 1 +max-jobs = 0 +sandbox = relaxed +NIX_CONF + +systemctl restart nix-daemon + +# Warm up local caches by building dev-env and current daml main +# This is allowed to fail, as we still want to have CI machines +# around, even when their caches are only warmed up halfway +su --login vsts <<'CACHE_WARMUP' +# user-wide bazel disk cache override +echo "build:linux --disk_cache=~/.bazel-cache" > ~/.bazelrc + +# clone and build +( + git clone https://github.com/digital-asset/daml + cd daml + git checkout upgrade-linux-nodes-to-20.04 + ./ci/dev-env-install.sh + ./build.sh "_$(uname)" +) || true +CACHE_WARMUP + +# Remove /home/vsts/daml folder that might be present from cache warmup +rm -R /home/vsts/daml || true + +## Finish + +# run the fake local webserver, taken from the docker image +web-server() { + while true; do + printf 'HTTP/1.1 302 Found\r\nLocation: https://%s.visualstudio.com/_admin/_AgentPool\r\n\r\n' "${vsts_account}" | nc -l -p 80 -q 0 > /dev/null + done +} +web-server & + +# Start the VSTS agent +su --login --command "cd /home/vsts/agent && exec ./run.sh" - vsts