Upgrade linux nodes to 20.04 (#8617)

CHANGELOG_BEGIN

- Our Linux binaries are now built on Ubuntu 20.04 instead of 16.04. We
  do not expect any user-level impact, but please reach out if you
  do notice any issue that might be caused by this.

CHANGELOG_END

.bazelrc

@@ -1,7 +1,7 @@
 # Bazel distributed cache, can be temporarily disabled by passing the following
 # flag: --noremote_accept_cached
 build:darwin --remote_http_cache=https://bazel-cache.da-ext.net
-build:linux --remote_http_cache=https://bazel-cache.da-ext.net
+build:linux --remote_http_cache=https://bazel-cache.da-ext.net/ubuntu_20_04
 build --remote_upload_local_results=false
 # Enable the disk cache in addition to the http cache.
 build:linux --disk_cache=.bazel-cache/disk

azure-cron.yml

@@ -31,7 +31,7 @@ jobs:
   - job: docs
     timeoutInMinutes: 120
     pool:
-      name: 'linux-pool'
+      name: 'ubuntu_20_04'
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -51,7 +51,7 @@ jobs:
   - job: docker_image
     timeoutInMinutes: 60
     pool:
-      name: 'linux-pool'
+      name: 'ubuntu_20_04'
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -106,7 +106,7 @@ jobs:
   - job: vscode_marketplace
     timeoutInMinutes: 10
     pool:
-      name: 'linux-pool'
+      name: 'ubuntu_20_04'
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -173,7 +173,7 @@ jobs:
   - job: download_stats
     timeoutInMinutes: 10
     pool:
-      name: "linux-pool"
+      name: "ubuntu_20_04"
       demands: assignment -equals default
     steps:
       - checkout: self

azure-pipelines.yml

@@ -21,7 +21,7 @@ jobs:
 
 - job: check_perf_test
   pool:
-    name: linux-pool
+    name: ubuntu_20_04
     demands: assignment -equals default
   steps:
   - bash: |
@@ -244,7 +244,7 @@ jobs:
   - release
   - check_for_release
   pool:
-    name: linux-pool
+    name: ubuntu_20_04
     demands: assignment -equals default
   variables:
     release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]

ci/build.yml

@@ -4,7 +4,7 @@
 jobs:
 - job: git_sha
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   steps:
     - bash: |
@@ -35,7 +35,7 @@ jobs:
     is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
   timeoutInMinutes: 360
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   steps:
     - template: report-start.yml
@@ -144,7 +144,7 @@ jobs:
     - da_ghc_lib
   timeoutInMinutes: 360
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   steps:
     - template: report-start.yml
@@ -205,7 +205,7 @@ jobs:
     - check_for_release
   timeoutInMinutes: 360
   pool:
-    name: linux-pool
+    name: ubuntu_20_04
     demands: assignment -equals default
   steps:
     - template: report-start.yml
@@ -222,7 +222,7 @@ jobs:
     - compatibility_ts_libs
   timeoutInMinutes: 360
   pool:
-    name: linux-pool
+    name: ubuntu_20_04
     demands: assignment -equals default
   steps:
     - template: report-start.yml
@@ -285,7 +285,7 @@ jobs:
     branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
     fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
   pool:
-    name: "linux-pool"
+    name: "ubuntu_20_04"
     demands: assignment -equals default
   steps:
     - bash: |
@@ -342,7 +342,7 @@ jobs:
     - compatibility_windows
     - Linux_scala_2_13
   pool:
-    name: "linux-pool"
+    name: "ubuntu_20_04"
     demands: assignment -equals default
   variables:
     Linux.start: $[ dependencies.Linux.outputs['start.time'] ]

ci/cron/daily-compat.yml

@@ -26,7 +26,7 @@ jobs:
   - job: compatibility_ts_libs
     timeoutInMinutes: 60
     pool:
-      name: linux-pool
+      name: ubuntu_20_04
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -40,7 +40,7 @@ jobs:
     strategy:
       matrix:
         linux:
-          pool: linux-pool
+          pool: ubuntu_20_04
         macos:
           pool: macOS-pool
     pool:
@@ -73,7 +73,7 @@ jobs:
   - job: perf_speedy
     timeoutInMinutes: 120
     pool:
-      name: "linux-pool"
+      name: "ubuntu_20_04"
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -121,7 +121,7 @@ jobs:
   - job: perf_http_json
     timeoutInMinutes: 120
     pool:
-      name: "linux-pool"
+      name: "ubuntu_20_04"
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -179,7 +179,7 @@ jobs:
   - job: check_releases
     timeoutInMinutes: 240
     pool:
-      name: linux-pool
+      name: ubuntu_20_04
       demands: assignment -equals default
     steps:
       - checkout: self
@@ -203,7 +203,7 @@ jobs:
     timeoutInMinutes: 1200
     condition: eq(variables['Build.SourceBranchName'], 'main')
     pool:
-      name: linux-pool
+      name: ubuntu_20_04
       demands: assignment -equals default
     steps:
       - checkout: self

ci/patch_bazel_windows/compile.yml

@@ -8,7 +8,7 @@ jobs:
 
 - job: patch_bazel_pre_check
   pool:
-    name: linux-pool
+    name: ubuntu_20_04
   steps:
   - checkout: self
   - bash: |
@@ -63,8 +63,10 @@ jobs:
     condition: eq(variables.should_run, 'true')
   - bash: |
       set -euo pipefail
+      BAZEL=$(mktemp)
+      curl -sL https://github.com/bazelbuild/bazel/releases/download/$(bazel_base_version)/bazel-$(bazel_base_version)-windows-x86_64.exe > $BAZEL
       cd bazel
-      bazel build src/main/cpp:client src:package-zip_jdk_minimal -c opt --stamp --embed_label $(bazel_base_version)-patched-$(cache_key)
+      $BAZEL build src/main/cpp:client src:package-zip_jdk_minimal -c opt --stamp --embed_label $(bazel_base_version)-patched-$(cache_key)
       # Note (MK) For some reason, the `zip` from chocolatey seems to result in
       # a “zip file structure invalid” error. I’ve tried adding msys to PATH so the Bazel
       # rules pick up `zip` from msys but that broke other things. So for now
@@ -92,7 +94,7 @@ jobs:
     target_url: $[ dependencies.patch_bazel_pre_check.outputs['out.target_url'] ]
     should_run: $[ dependencies.patch_bazel_pre_check.outputs['out.should_run'] ]
   pool:
-    name: linux-pool
+    name: ubuntu_20_04
   steps:
   - task: DownloadPipelineArtifact@2
     inputs:

ci/prs.yml

@@ -21,7 +21,7 @@ jobs:
     branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
   condition: eq(variables['Build.Reason'], 'PullRequest')
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   steps:
     - checkout: self
@@ -51,7 +51,7 @@ jobs:
     pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
   condition: eq(variables['Build.Reason'], 'PullRequest')
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   steps:
     - checkout: self
@@ -86,7 +86,7 @@ jobs:
     - collect_build_data
     - check_for_release
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   variables:
     branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
@@ -138,7 +138,7 @@ jobs:
     - check_for_release
     - check_changelog_entry
   pool:
-    name: 'linux-pool'
+    name: 'ubuntu_20_04'
     demands: assignment -equals default
   variables:
     pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]

dev-env/windows/manifests/bazel.json

@@ -5,8 +5,8 @@
   "bin": "bazel.exe",
   "architecture": {
     "64bit": {
-      "url": "https://daml-binaries.da-ext.net/patch_bazel_windows/bazel-e0202b0f2f3c2d8c3b477ddb59bd30ce.zip",
-      "hash": "7177fbc5660626ea68713d6fab026581deced2e20aad0723e2b5ed3e2eb1f2dc"
+      "url": "https://daml-binaries.da-ext.net/patch_bazel_windows/bazel-f6f61bf596011e671cb3b327b0f29686.zip",
+      "hash": "5b9224a94da96403bffd0309e011d63504807386718d806d0aaf769f674626f5"
     }
   },
   "depends": [

infra/main.tf

@@ -49,3 +49,5 @@ locals {
 
   ssl_certificate_hoogle = "https://www.googleapis.com/compute/v1/projects/da-dev-gcp-daml-language/global/sslCertificates/daml-lang-hoogle-app-service-https-cert"
 }
+
+resource "secret_resource" "vsts-token" {}

infra/vsts_agent_linux.tf

@@ -1,8 +1,6 @@
 # Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-resource "secret_resource" "vsts-token" {}
-
 data "template_file" "vsts-agent-linux-startup" {
   template = "${file("${path.module}/vsts_agent_linux_startup.sh")}"
 

infra/vsts_agent_ubuntu_20_04.tf

@@ -0,0 +1,80 @@
+# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+data "template_file" "vsts-agent-ubuntu_20_04-startup" {
+  template = "${file("${path.module}/vsts_agent_ubuntu_20_04_startup.sh")}"
+
+  vars = {
+    vsts_token   = "${secret_resource.vsts-token.value}"
+    vsts_account = "digitalasset"
+    vsts_pool    = "ubuntu_20_04"
+  }
+}
+
+resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04" {
+  provider           = "google-beta"
+  name               = "vsts-agent-ubuntu-20-04"
+  base_instance_name = "vsts-agent-ubuntu-20-04"
+  region             = "us-east1"
+  target_size        = 1
+
+  version {
+    name              = "vsts-agent-ubuntu-20-04"
+    instance_template = "${google_compute_instance_template.vsts-agent-ubuntu_20_04.self_link}"
+  }
+
+  update_policy {
+    type            = "PROACTIVE"
+    minimal_action  = "REPLACE"
+    max_surge_fixed = 3
+    min_ready_sec   = 60
+  }
+}
+
+resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" {
+  name_prefix  = "vsts-agent-ubuntu-20-04-"
+  machine_type = "c2-standard-8"
+  labels       = "${local.machine-labels}"
+
+  disk {
+    disk_size_gb = 200
+    disk_type    = "pd-ssd"
+    source_image = "ubuntu-os-cloud/ubuntu-2004-lts"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  metadata {
+    startup-script = "${data.template_file.vsts-agent-ubuntu_20_04-startup.rendered}"
+
+    shutdown-script = <<EOS
+#!/usr/bin/env bash
+set -euo pipefail
+cd /home/vsts/agent
+su vsts <<SHUTDOWN_AGENT
+export VSTS_AGENT_INPUT_TOKEN='${secret_resource.vsts-token.value}'
+./config.sh remove --unattended --auth PAT
+SHUTDOWN_AGENT
+    EOS
+  }
+
+  network_interface {
+    network = "default"
+
+    // Ephemeral IP to get access to the Internet
+    access_config {}
+  }
+
+  service_account {
+    email  = "log-writer@da-dev-gcp-daml-language.iam.gserviceaccount.com"
+    scopes = ["cloud-platform"]
+  }
+
+  scheduling {
+    automatic_restart   = false
+    on_host_maintenance = "TERMINATE"
+    preemptible         = false
+  }
+}

infra/vsts_agent_ubuntu_20_04_startup.sh

@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Agent startup script
+set -euo pipefail
+
+## Hardening
+
+# Commit sepukku on failure
+trap "shutdown -h now" EXIT
+
+# replace the default nameserver to not use the metadata server
+echo "nameserver 8.8.8.8" > /etc/resolv.conf
+
+# delete self
+rm -vf "$0"
+
+## Install system dependencies
+apt-get update -q
+apt-get install -qy \
+  curl sudo \
+  bzip2 rsync \
+  jq liblttng-ust0 libcurl4 libkrb5-3 zlib1g \
+  git \
+  netcat \
+  apt-transport-https \
+  software-properties-common
+
+# Install dependencies for Chrome (to run Puppeteer tests on the gsg)
+# list taken from: https://github.com/puppeteer/puppeteer/blob/a3d1536a6b6e282a43521bea28aef027a7133df8/docs/troubleshooting.md#chrome-headless-doesnt-launch-on-unix
+# see https://github.com/digital-asset/daml/pull/5540 for context
+apt-get install -qy \
+    gconf-service \
+    libasound2 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libc6 \
+    libcairo2 \
+    libcups2 \
+    libdbus-1-3 \
+    libexpat1 \
+    libfontconfig1 \
+    libgbm-dev \
+    libgcc1 \
+    libgconf-2-4 \
+    libgdk-pixbuf2.0-0 \
+    libglib2.0-0 \
+    libgtk-3-0 \
+    libnspr4 \
+    libpango-1.0-0 \
+    libpangocairo-1.0-0 \
+    libstdc++6 \
+    libx11-6 \
+    libx11-xcb1 \
+    libxcb1 \
+    libxcomposite1 \
+    libxcursor1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxi6 \
+    libxrandr2 \
+    libxrender1 \
+    libxss1 \
+    libxtst6 \
+    ca-certificates \
+    fonts-liberation \
+    libappindicator1 \
+    libnss3 \
+    lsb-release \
+    xdg-utils \
+    wget
+
+curl -sSL https://dl.google.com/cloudagents/install-logging-agent.sh | bash
+
+#install docker
+DOCKER_VERSION="5:20.10.2~3-0~ubuntu-$(lsb_release -cs)"
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+apt-key fingerprint 0EBFCD88
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+apt-get update
+apt-get install -qy docker-ce=$DOCKER_VERSION docker-ce-cli=$DOCKER_VERSION containerd.io
+
+#Start docker daemon
+systemctl enable docker
+
+## Install the VSTS agent
+groupadd --gid 3000 vsts
+useradd \
+  --create-home \
+  --gid 3000 \
+  --shell /bin/bash \
+  --uid 3000 \
+  vsts
+#add docker group to user
+usermod -aG docker vsts
+
+su --login vsts <<'AGENT_SETUP'
+set -euo pipefail
+
+VSTS_ACCOUNT=${vsts_account}
+VSTS_POOL=${vsts_pool}
+VSTS_TOKEN=${vsts_token}
+
+mkdir -p ~/agent
+cd ~/agent
+echo 'assignment=default' > .capabilities
+
+echo Determining matching VSTS agent...
+VSTS_AGENT_RESPONSE=$(curl -sSfL \
+  -u "user:$VSTS_TOKEN" \
+  -H 'Accept:application/json;api-version=3.0-preview' \
+  "https://$VSTS_ACCOUNT.visualstudio.com/_apis/distributedtask/packages/agent?platform=linux-x64")
+
+VSTS_AGENT_URL=$(echo "$VSTS_AGENT_RESPONSE" \
+  | jq -r '.value | map([.version.major,.version.minor,.version.patch,.downloadUrl]) | sort | .[length-1] | .[3]')
+
+if [ -z "$VSTS_AGENT_URL" -o "$VSTS_AGENT_URL" == "null" ]; then
+  echo 1>&2 error: could not determine a matching VSTS agent - check that account \'$VSTS_ACCOUNT\' is correct and the token is valid for that account
+  exit 1
+fi
+
+echo Downloading and installing VSTS agent...
+curl -sSfL "$VSTS_AGENT_URL" | tar -xz --no-same-owner
+
+set +u
+source ./env.sh
+set -u
+
+./config.sh \
+  --acceptTeeEula \
+  --agent "$(hostname)" \
+  --auth PAT \
+  --pool "$VSTS_POOL" \
+  --replace \
+  --token "$VSTS_TOKEN" \
+  --unattended \
+  --url "https://$VSTS_ACCOUNT.visualstudio.com"
+AGENT_SETUP
+
+## Hardening
+
+chown --recursive root:root /home/vsts/agent/{*.sh,bin,externals}
+
+## Install Nix
+
+# This needs to run inside of a user with sudo access
+echo "vsts ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/nix_installation
+su --command "sh <(curl -sSfL https://nixos.org/nix/install) --daemon" --login vsts
+rm /etc/sudoers.d/nix_installation
+
+# Note: the "hydra.da-int.net" string is now part of the name of the key for
+# legacy reasons; it bears no relation to the DNS hostname of the current
+# cache.
+cat <<NIX_CONF > /etc/nix/nix.conf
+binary-cache-public-keys = hydra.da-int.net-2:91tXuJGf/ExbAz7IWsMsxQ5FsO6lG/EGM5QVt+xhZu0= hydra.da-int.net-1:6Oy2+KYvI7xkAOg0gJisD7Nz/6m8CmyKMbWfSKUe03g= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=
+binary-caches = https://nix-cache.da-ext.net https://cache.nixos.org
+build-users-group = nixbld
+cores = 1
+max-jobs = 0
+sandbox = relaxed
+NIX_CONF
+
+systemctl restart nix-daemon
+
+# Warm up local caches by building dev-env and current daml main
+# This is allowed to fail, as we still want to have CI machines
+# around, even when their caches are only warmed up halfway
+su --login vsts <<'CACHE_WARMUP'
+# user-wide bazel disk cache override
+echo "build:linux --disk_cache=~/.bazel-cache" > ~/.bazelrc
+
+# clone and build
+(
+  git clone https://github.com/digital-asset/daml
+  cd daml
+  git checkout upgrade-linux-nodes-to-20.04
+  ./ci/dev-env-install.sh
+  ./build.sh "_$(uname)"
+) || true
+CACHE_WARMUP
+
+# Remove /home/vsts/daml folder that might be present from cache warmup
+rm -R /home/vsts/daml || true
+
+## Finish
+
+# run the fake local webserver, taken from the docker image
+web-server() {
+  while true; do
+    printf 'HTTP/1.1 302 Found\r\nLocation: https://%s.visualstudio.com/_admin/_AgentPool\r\n\r\n' "${vsts_account}" | nc -l -p 80 -q 0 > /dev/null
+  done
+}
+web-server &
+
+# Start the VSTS agent
+su --login --command "cd /home/vsts/agent && exec ./run.sh" - vsts