ci: set up Ubuntu nodes on Azure (#16610)

infra/README.md

@@ -1,6 +1,6 @@
 # Daml
 
-This is the terraform code used by the Daml repository to deploy supporting
+This is the Terraform code used by the Daml repository to deploy supporting
 infrastructure such as the Bazel caches, Nix caches and Azure Pipeline (VSTS)
 Agents.
 
@@ -10,6 +10,25 @@ To deploy the infrastructure changes, you will to get access to the
 `da-dev-gcp-daml-language` Google project from DA IT. Then run
 `gcloud auth login` to configure the local credentials.
 
+You also need access to the `9114f3e0-9963-4368-9a0a-117bcdbf0055` subscription
+on Azure. To authenticate with Azure, run:
+
+```
+az login
+```
+
+Terraform will use the appropriate subscription by default. If you want to be
+able to inspect your Terraform state through the CLI, you may want to select
+that subspription as the default using:
+
+```
+az account set --subscription 9114f3e0-9963-4368-9a0a-117bcdbf0055
+```
+
+Remember, though, that the goal is for the Terraform files to exactly describe
+the state of the resources in the cloud, so please refrain from making any
+manual changes using the CLI.
+
 ## Deployment
 
 All the infrastructure is currently deployed using

infra/main.tf

@@ -20,6 +20,10 @@ terraform {
       source  = "hashicorp/google-beta"
       version = "4.43.0"
     }
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "3.31.0"
+    }
   }
 }
 
@@ -38,6 +42,20 @@ provider "google-beta" {
 provider "secret" {
 }
 
+provider "azurerm" {
+  features {
+    virtual_machine {
+      graceful_shutdown = true
+    }
+  }
+  subscription_id = "9114f3e0-9963-4368-9a0a-117bcdbf0055"
+}
+
+resource "azurerm_resource_group" "daml-ci" {
+  name     = "daml-ci"
+  location = "East US"
+}
+
 data "google_project" "current" {
   project_id = local.project
 }

infra/ubuntu.tf

@@ -0,0 +1,35 @@
+# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+locals {
+  ubuntu = {
+    gcp = [
+      {
+        name       = "ci-u1",
+        disk_size  = 400,
+        size       = 20,
+        assignment = "default",
+      },
+      {
+        name       = "ci-u2",
+        disk_size  = 400,
+        size       = 0,
+        assignment = "default",
+      },
+    ],
+    azure = [
+      {
+        name       = "du1",
+        disk_size  = 400,
+        size       = 5,
+        assignment = "default",
+      },
+      {
+        name       = "du2",
+        disk_size  = 400,
+        size       = 0,
+        assignment = "default",
+      },
+    ]
+  }
+}

infra/ubuntu_azure.tf

@@ -0,0 +1,93 @@
+# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+resource "azurerm_linux_virtual_machine_scale_set" "ubuntu" {
+  count               = length(local.ubuntu.azure)
+  name                = "ubuntu"
+  resource_group_name = azurerm_resource_group.daml-ci.name
+  location            = azurerm_resource_group.daml-ci.location
+  sku                 = "Standard_D4_v2"
+  instances           = local.ubuntu.azure[count.index].size
+
+  admin_username                  = "adminuser"
+  disable_password_authentication = true
+  admin_ssh_key {
+    username   = "adminuser"
+    public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCygLXwXmrkMYpxLtyolmOqodZ6w6DZINDLrCnpEpyykWbUdEmbTVYclF92dRLTd84TyQO5lfL7eAUAi6KWYE0DNIXV/Jl93iM+80/i3QqIMjcydzkkkSNJHDPECJoVx+ftm0tCOWZXLudsHgHkMu0Vx2R9XvfUB+MY5sU50NV6wwmAAyZezW8l51/vcPeLb5YX7hV8+VjF9zv5f7SxZGsfALYB2CwddwPoO+/xrnj/Vz9jWsO5y4I7ia1tAs3QOabtz9UPfvVxIoDnEAojgVnsb7GvB4SvxraEQHNLwXcRmzCPPyznOhiAdGKd1kpEMnbD7lKQrKdHX2PdVJZB/PF1ekv62HD6ZuzKmrcB7qUpj6qkDCfaPCw1psTefXFjK53Q2LffZVhanOw+Oq7J1Gdo7phl40ipQyHHr/jp0pMmQ7mZhXnbrQE4H4csMoMbzWH9WcJ+qowBUHMb55Ai0WcVho0/w7+FPAiVDyUobxpaZnqBOV+/n/hC9kkkC1bfokP6oFEi6w4m1/g1LlgWLo+ex9H2ebOt9yiUBsWXwWUyrvbtCANpo510Ss9rCj9NS9vu7iH3GV9JcpaGs1AF7NXNduwI+LCiYK+smBo0T1I8Sq/TpoYtDCAhoGZth4sppetEgMNOFsri7ZZiu0NiJLpEhhVou06CMM/KSwBU2PzeSw== Azure Self Hosted Runners"
+  }
+
+  computer_name_prefix = "daml-ubuntu"
+
+  # save a bit of energy for the planet
+  overprovision = false
+
+  custom_data = base64encode(templatefile("${path.module}/ubuntu_startup.sh", {
+    vsts_token   = secret_resource.vsts-token.value
+    vsts_account = "digitalasset"
+    vsts_pool    = "ubuntu_20_04"
+    size         = local.ubuntu.azure[count.index].disk_size
+    gcp_logging  = ""
+    assignment   = local.ubuntu.azure[count.index].assignment
+  }))
+
+  source_image_reference {
+    publisher = "canonical"
+    offer     = "0001-com-ubuntu-server-focal"
+    sku       = "20_04-lts"
+    version   = "latest"
+  }
+
+  os_disk {
+    caching              = "ReadOnly"
+    storage_account_type = "Standard_LRS"
+    disk_size_gb         = local.ubuntu.azure[count.index].disk_size
+  }
+
+  network_interface {
+    name    = "default"
+    primary = true
+
+    ip_configuration {
+      name      = "default"
+      primary   = true
+      subnet_id = one(azurerm_virtual_network.ubuntu.subnet).id
+    }
+  }
+
+  # required to get console output in Azure UI
+  boot_diagnostics {
+    storage_account_uri = null
+  }
+
+}
+
+resource "azurerm_virtual_network" "ubuntu" {
+  name                = "ubuntu"
+  location            = azurerm_resource_group.daml-ci.location
+  resource_group_name = azurerm_resource_group.daml-ci.name
+  address_space       = ["10.0.0.0/16"]
+
+  subnet {
+    name           = "subnet"
+    address_prefix = "10.0.1.0/24"
+    security_group = azurerm_network_security_group.ubuntu.id
+  }
+}
+
+resource "azurerm_network_security_group" "ubuntu" {
+  name                = "ubuntu"
+  location            = azurerm_resource_group.daml-ci.location
+  resource_group_name = azurerm_resource_group.daml-ci.name
+
+  security_rule {
+    name                       = "deny-inbound"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Deny"
+    protocol                   = "*"
+    source_port_range          = "*"
+    destination_port_range     = "*"
+    source_address_prefix      = "*"
+    destination_address_prefix = "*"
+  }
+}

infra/ubuntu_gcp.tf

@@ -1,31 +1,16 @@
 # Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-locals {
-  ubuntu = [
-    {
-      name      = "ci-u1",
-      disk_size = 400,
-      size      = 30,
-    },
-    {
-      name      = "ci-u2",
-      disk_size = 400,
-      size      = 0,
-    },
-  ]
-}
-
 resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04" {
-  count              = length(local.ubuntu)
+  count              = length(local.ubuntu.gcp)
   provider           = google-beta
-  name               = local.ubuntu[count.index].name
+  name               = local.ubuntu.gcp[count.index].name
-  base_instance_name = local.ubuntu[count.index].name
+  base_instance_name = local.ubuntu.gcp[count.index].name
   region             = "us-east1"
-  target_size        = local.ubuntu[count.index].size
+  target_size        = local.ubuntu.gcp[count.index].size
 
   version {
-    name              = local.ubuntu[count.index].name
+    name              = local.ubuntu.gcp[count.index].name
     instance_template = google_compute_instance_template.vsts-agent-ubuntu_20_04[count.index].self_link
   }
 
@@ -43,13 +28,13 @@ resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04
 }
 
 resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" {
-  count        = length(local.ubuntu)
+  count        = length(local.ubuntu.gcp)
-  name_prefix  = "${local.ubuntu[count.index].name}-"
+  name_prefix  = "${local.ubuntu.gcp[count.index].name}-"
   machine_type = "c2-standard-8"
   labels       = local.machine-labels
 
   disk {
-    disk_size_gb = local.ubuntu[count.index].disk_size
+    disk_size_gb = local.ubuntu.gcp[count.index].disk_size
     disk_type    = "pd-ssd"
     source_image = "ubuntu-os-cloud/ubuntu-2004-lts"
   }
@@ -59,11 +44,17 @@ resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" {
   }
 
   metadata = {
-    startup-script = templatefile("${path.module}/vsts_agent_ubuntu_20_04_startup.sh", {
+    startup-script = templatefile("${path.module}/ubuntu_startup.sh", {
       vsts_token   = secret_resource.vsts-token.value
       vsts_account = "digitalasset"
       vsts_pool    = "ubuntu_20_04"
-      size         = local.ubuntu[count.index].disk_size
+      size         = local.ubuntu.gcp[count.index].disk_size
+      gcp_logging  = <<EOF
+# Taken from https://cloud.google.com/logging/docs/agent/logging/installation
+curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+curl -sSL https://dl.google.com/cloudagents/add-logging-agent-repo.sh | bash -s -- --also-install
+EOF
+      assignment   = local.ubuntu.gcp[count.index].assignment
     })
 
     shutdown-script = nonsensitive("#!/usr/bin/env bash\nset -euo pipefail\ncd /home/vsts/agent\nsu vsts <<SHUTDOWN_AGENT\nexport VSTS_AGENT_INPUT_TOKEN='${secret_resource.vsts-token.value}'\n./config.sh remove --unattended --auth PAT\nSHUTDOWN_AGENT\n    ")

infra/ubuntu_startup.sh

@@ -72,10 +72,7 @@ apt-get install -qy \
     xdg-utils \
     wget
 
-# Taken from https://cloud.google.com/logging/docs/agent/logging/installation
+${gcp_logging}
-curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-curl -sSL https://dl.google.com/cloudagents/add-logging-agent-repo.sh | bash -s -- --also-install
-
 #install docker
 # BEGIN Installing Docker per https://docs.docker.com/engine/install/ubuntu/
 apt-get -y install apt-transport-https \
@@ -159,7 +156,7 @@ VSTS_TOKEN=${vsts_token}
 
 mkdir -p ~/agent
 cd ~/agent
-echo 'assignment=default' > .capabilities
+echo 'assignment=${assignment}' > .capabilities
 
 echo Determining matching VSTS agent...
 VSTS_AGENT_RESPONSE=$(curl -sSfL \

nix/default.nix

@@ -204,6 +204,7 @@ in rec {
     # used to set up the webide CI pipeline in azure-cron.yml
     docker-credential-gcr = pkgs.docker-credential-gcr;
     terraform = pkgs.terraform_1.withPlugins (p: with p; [
+      azurerm
       google
       google-beta
       secret