From 7d69a5975c9b566666ce5a949a258f24e79c28b6 Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Wed, 29 Mar 2023 15:21:14 +0200 Subject: [PATCH] ci: set up Ubuntu nodes on Azure (#16610) --- infra/README.md | 21 ++++- infra/main.tf | 18 ++++ infra/ubuntu.tf | 35 +++++++ infra/ubuntu_azure.tf | 93 +++++++++++++++++++ ...ts_agent_ubuntu_20_04.tf => ubuntu_gcp.tf} | 41 ++++---- ...ntu_20_04_startup.sh => ubuntu_startup.sh} | 7 +- nix/default.nix | 1 + 7 files changed, 185 insertions(+), 31 deletions(-) create mode 100644 infra/ubuntu.tf create mode 100644 infra/ubuntu_azure.tf rename infra/{vsts_agent_ubuntu_20_04.tf => ubuntu_gcp.tf} (65%) rename infra/{vsts_agent_ubuntu_20_04_startup.sh => ubuntu_startup.sh} (95%) diff --git a/infra/README.md b/infra/README.md index e267ce35e2..51bb144ec5 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,6 +1,6 @@ # Daml -This is the terraform code used by the Daml repository to deploy supporting +This is the Terraform code used by the Daml repository to deploy supporting infrastructure such as the Bazel caches, Nix caches and Azure Pipeline (VSTS) Agents. @@ -10,6 +10,25 @@ To deploy the infrastructure changes, you will to get access to the `da-dev-gcp-daml-language` Google project from DA IT. Then run `gcloud auth login` to configure the local credentials. +You also need access to the `9114f3e0-9963-4368-9a0a-117bcdbf0055` subscription +on Azure. To authenticate with Azure, run: + +``` +az login +``` + +Terraform will use the appropriate subscription by default. If you want to be +able to inspect your Terraform state through the CLI, you may want to select +that subspription as the default using: + +``` +az account set --subscription 9114f3e0-9963-4368-9a0a-117bcdbf0055 +``` + +Remember, though, that the goal is for the Terraform files to exactly describe +the state of the resources in the cloud, so please refrain from making any +manual changes using the CLI. + ## Deployment All the infrastructure is currently deployed using diff --git a/infra/main.tf b/infra/main.tf index 18d0eb8e0a..6adfa4d2af 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -20,6 +20,10 @@ terraform { source = "hashicorp/google-beta" version = "4.43.0" } + azurerm = { + source = "hashicorp/azurerm" + version = "3.31.0" + } } } @@ -38,6 +42,20 @@ provider "google-beta" { provider "secret" { } +provider "azurerm" { + features { + virtual_machine { + graceful_shutdown = true + } + } + subscription_id = "9114f3e0-9963-4368-9a0a-117bcdbf0055" +} + +resource "azurerm_resource_group" "daml-ci" { + name = "daml-ci" + location = "East US" +} + data "google_project" "current" { project_id = local.project } diff --git a/infra/ubuntu.tf b/infra/ubuntu.tf new file mode 100644 index 0000000000..47d6c280ba --- /dev/null +++ b/infra/ubuntu.tf @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +locals { + ubuntu = { + gcp = [ + { + name = "ci-u1", + disk_size = 400, + size = 20, + assignment = "default", + }, + { + name = "ci-u2", + disk_size = 400, + size = 0, + assignment = "default", + }, + ], + azure = [ + { + name = "du1", + disk_size = 400, + size = 5, + assignment = "default", + }, + { + name = "du2", + disk_size = 400, + size = 0, + assignment = "default", + }, + ] + } +} diff --git a/infra/ubuntu_azure.tf b/infra/ubuntu_azure.tf new file mode 100644 index 0000000000..6c5fac7765 --- /dev/null +++ b/infra/ubuntu_azure.tf @@ -0,0 +1,93 @@ +# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +resource "azurerm_linux_virtual_machine_scale_set" "ubuntu" { + count = length(local.ubuntu.azure) + name = "ubuntu" + resource_group_name = azurerm_resource_group.daml-ci.name + location = azurerm_resource_group.daml-ci.location + sku = "Standard_D4_v2" + instances = local.ubuntu.azure[count.index].size + + admin_username = "adminuser" + disable_password_authentication = true + admin_ssh_key { + username = "adminuser" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCygLXwXmrkMYpxLtyolmOqodZ6w6DZINDLrCnpEpyykWbUdEmbTVYclF92dRLTd84TyQO5lfL7eAUAi6KWYE0DNIXV/Jl93iM+80/i3QqIMjcydzkkkSNJHDPECJoVx+ftm0tCOWZXLudsHgHkMu0Vx2R9XvfUB+MY5sU50NV6wwmAAyZezW8l51/vcPeLb5YX7hV8+VjF9zv5f7SxZGsfALYB2CwddwPoO+/xrnj/Vz9jWsO5y4I7ia1tAs3QOabtz9UPfvVxIoDnEAojgVnsb7GvB4SvxraEQHNLwXcRmzCPPyznOhiAdGKd1kpEMnbD7lKQrKdHX2PdVJZB/PF1ekv62HD6ZuzKmrcB7qUpj6qkDCfaPCw1psTefXFjK53Q2LffZVhanOw+Oq7J1Gdo7phl40ipQyHHr/jp0pMmQ7mZhXnbrQE4H4csMoMbzWH9WcJ+qowBUHMb55Ai0WcVho0/w7+FPAiVDyUobxpaZnqBOV+/n/hC9kkkC1bfokP6oFEi6w4m1/g1LlgWLo+ex9H2ebOt9yiUBsWXwWUyrvbtCANpo510Ss9rCj9NS9vu7iH3GV9JcpaGs1AF7NXNduwI+LCiYK+smBo0T1I8Sq/TpoYtDCAhoGZth4sppetEgMNOFsri7ZZiu0NiJLpEhhVou06CMM/KSwBU2PzeSw== Azure Self Hosted Runners" + } + + computer_name_prefix = "daml-ubuntu" + + # save a bit of energy for the planet + overprovision = false + + custom_data = base64encode(templatefile("${path.module}/ubuntu_startup.sh", { + vsts_token = secret_resource.vsts-token.value + vsts_account = "digitalasset" + vsts_pool = "ubuntu_20_04" + size = local.ubuntu.azure[count.index].disk_size + gcp_logging = "" + assignment = local.ubuntu.azure[count.index].assignment + })) + + source_image_reference { + publisher = "canonical" + offer = "0001-com-ubuntu-server-focal" + sku = "20_04-lts" + version = "latest" + } + + os_disk { + caching = "ReadOnly" + storage_account_type = "Standard_LRS" + disk_size_gb = local.ubuntu.azure[count.index].disk_size + } + + network_interface { + name = "default" + primary = true + + ip_configuration { + name = "default" + primary = true + subnet_id = one(azurerm_virtual_network.ubuntu.subnet).id + } + } + + # required to get console output in Azure UI + boot_diagnostics { + storage_account_uri = null + } + +} + +resource "azurerm_virtual_network" "ubuntu" { + name = "ubuntu" + location = azurerm_resource_group.daml-ci.location + resource_group_name = azurerm_resource_group.daml-ci.name + address_space = ["10.0.0.0/16"] + + subnet { + name = "subnet" + address_prefix = "10.0.1.0/24" + security_group = azurerm_network_security_group.ubuntu.id + } +} + +resource "azurerm_network_security_group" "ubuntu" { + name = "ubuntu" + location = azurerm_resource_group.daml-ci.location + resource_group_name = azurerm_resource_group.daml-ci.name + + security_rule { + name = "deny-inbound" + priority = 100 + direction = "Inbound" + access = "Deny" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "*" + } +} diff --git a/infra/vsts_agent_ubuntu_20_04.tf b/infra/ubuntu_gcp.tf similarity index 65% rename from infra/vsts_agent_ubuntu_20_04.tf rename to infra/ubuntu_gcp.tf index a6938a1c9b..40d815c905 100644 --- a/infra/vsts_agent_ubuntu_20_04.tf +++ b/infra/ubuntu_gcp.tf @@ -1,31 +1,16 @@ # Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -locals { - ubuntu = [ - { - name = "ci-u1", - disk_size = 400, - size = 30, - }, - { - name = "ci-u2", - disk_size = 400, - size = 0, - }, - ] -} - resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04" { - count = length(local.ubuntu) + count = length(local.ubuntu.gcp) provider = google-beta - name = local.ubuntu[count.index].name - base_instance_name = local.ubuntu[count.index].name + name = local.ubuntu.gcp[count.index].name + base_instance_name = local.ubuntu.gcp[count.index].name region = "us-east1" - target_size = local.ubuntu[count.index].size + target_size = local.ubuntu.gcp[count.index].size version { - name = local.ubuntu[count.index].name + name = local.ubuntu.gcp[count.index].name instance_template = google_compute_instance_template.vsts-agent-ubuntu_20_04[count.index].self_link } @@ -43,13 +28,13 @@ resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04 } resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" { - count = length(local.ubuntu) - name_prefix = "${local.ubuntu[count.index].name}-" + count = length(local.ubuntu.gcp) + name_prefix = "${local.ubuntu.gcp[count.index].name}-" machine_type = "c2-standard-8" labels = local.machine-labels disk { - disk_size_gb = local.ubuntu[count.index].disk_size + disk_size_gb = local.ubuntu.gcp[count.index].disk_size disk_type = "pd-ssd" source_image = "ubuntu-os-cloud/ubuntu-2004-lts" } @@ -59,11 +44,17 @@ resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" { } metadata = { - startup-script = templatefile("${path.module}/vsts_agent_ubuntu_20_04_startup.sh", { + startup-script = templatefile("${path.module}/ubuntu_startup.sh", { vsts_token = secret_resource.vsts-token.value vsts_account = "digitalasset" vsts_pool = "ubuntu_20_04" - size = local.ubuntu[count.index].disk_size + size = local.ubuntu.gcp[count.index].disk_size + gcp_logging = < .capabilities +echo 'assignment=${assignment}' > .capabilities echo Determining matching VSTS agent... VSTS_AGENT_RESPONSE=$(curl -sSfL \ diff --git a/nix/default.nix b/nix/default.nix index 1d36e78b8e..5005a9d9c7 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -204,6 +204,7 @@ in rec { # used to set up the webide CI pipeline in azure-cron.yml docker-credential-gcr = pkgs.docker-credential-gcr; terraform = pkgs.terraform_1.withPlugins (p: with p; [ + azurerm google google-beta secret