mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
ci: set up Ubuntu nodes on Azure (#16610)
This commit is contained in:
parent
629439d876
commit
7d69a5975c
@ -1,6 +1,6 @@
|
||||
# Daml
|
||||
|
||||
This is the terraform code used by the Daml repository to deploy supporting
|
||||
This is the Terraform code used by the Daml repository to deploy supporting
|
||||
infrastructure such as the Bazel caches, Nix caches and Azure Pipeline (VSTS)
|
||||
Agents.
|
||||
|
||||
@ -10,6 +10,25 @@ To deploy the infrastructure changes, you will to get access to the
|
||||
`da-dev-gcp-daml-language` Google project from DA IT. Then run
|
||||
`gcloud auth login` to configure the local credentials.
|
||||
|
||||
You also need access to the `9114f3e0-9963-4368-9a0a-117bcdbf0055` subscription
|
||||
on Azure. To authenticate with Azure, run:
|
||||
|
||||
```
|
||||
az login
|
||||
```
|
||||
|
||||
Terraform will use the appropriate subscription by default. If you want to be
|
||||
able to inspect your Terraform state through the CLI, you may want to select
|
||||
that subspription as the default using:
|
||||
|
||||
```
|
||||
az account set --subscription 9114f3e0-9963-4368-9a0a-117bcdbf0055
|
||||
```
|
||||
|
||||
Remember, though, that the goal is for the Terraform files to exactly describe
|
||||
the state of the resources in the cloud, so please refrain from making any
|
||||
manual changes using the CLI.
|
||||
|
||||
## Deployment
|
||||
|
||||
All the infrastructure is currently deployed using
|
||||
|
@ -20,6 +20,10 @@ terraform {
|
||||
source = "hashicorp/google-beta"
|
||||
version = "4.43.0"
|
||||
}
|
||||
azurerm = {
|
||||
source = "hashicorp/azurerm"
|
||||
version = "3.31.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -38,6 +42,20 @@ provider "google-beta" {
|
||||
provider "secret" {
|
||||
}
|
||||
|
||||
provider "azurerm" {
|
||||
features {
|
||||
virtual_machine {
|
||||
graceful_shutdown = true
|
||||
}
|
||||
}
|
||||
subscription_id = "9114f3e0-9963-4368-9a0a-117bcdbf0055"
|
||||
}
|
||||
|
||||
resource "azurerm_resource_group" "daml-ci" {
|
||||
name = "daml-ci"
|
||||
location = "East US"
|
||||
}
|
||||
|
||||
data "google_project" "current" {
|
||||
project_id = local.project
|
||||
}
|
||||
|
35
infra/ubuntu.tf
Normal file
35
infra/ubuntu.tf
Normal file
@ -0,0 +1,35 @@
|
||||
# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
locals {
|
||||
ubuntu = {
|
||||
gcp = [
|
||||
{
|
||||
name = "ci-u1",
|
||||
disk_size = 400,
|
||||
size = 20,
|
||||
assignment = "default",
|
||||
},
|
||||
{
|
||||
name = "ci-u2",
|
||||
disk_size = 400,
|
||||
size = 0,
|
||||
assignment = "default",
|
||||
},
|
||||
],
|
||||
azure = [
|
||||
{
|
||||
name = "du1",
|
||||
disk_size = 400,
|
||||
size = 5,
|
||||
assignment = "default",
|
||||
},
|
||||
{
|
||||
name = "du2",
|
||||
disk_size = 400,
|
||||
size = 0,
|
||||
assignment = "default",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
93
infra/ubuntu_azure.tf
Normal file
93
infra/ubuntu_azure.tf
Normal file
@ -0,0 +1,93 @@
|
||||
# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
resource "azurerm_linux_virtual_machine_scale_set" "ubuntu" {
|
||||
count = length(local.ubuntu.azure)
|
||||
name = "ubuntu"
|
||||
resource_group_name = azurerm_resource_group.daml-ci.name
|
||||
location = azurerm_resource_group.daml-ci.location
|
||||
sku = "Standard_D4_v2"
|
||||
instances = local.ubuntu.azure[count.index].size
|
||||
|
||||
admin_username = "adminuser"
|
||||
disable_password_authentication = true
|
||||
admin_ssh_key {
|
||||
username = "adminuser"
|
||||
public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCygLXwXmrkMYpxLtyolmOqodZ6w6DZINDLrCnpEpyykWbUdEmbTVYclF92dRLTd84TyQO5lfL7eAUAi6KWYE0DNIXV/Jl93iM+80/i3QqIMjcydzkkkSNJHDPECJoVx+ftm0tCOWZXLudsHgHkMu0Vx2R9XvfUB+MY5sU50NV6wwmAAyZezW8l51/vcPeLb5YX7hV8+VjF9zv5f7SxZGsfALYB2CwddwPoO+/xrnj/Vz9jWsO5y4I7ia1tAs3QOabtz9UPfvVxIoDnEAojgVnsb7GvB4SvxraEQHNLwXcRmzCPPyznOhiAdGKd1kpEMnbD7lKQrKdHX2PdVJZB/PF1ekv62HD6ZuzKmrcB7qUpj6qkDCfaPCw1psTefXFjK53Q2LffZVhanOw+Oq7J1Gdo7phl40ipQyHHr/jp0pMmQ7mZhXnbrQE4H4csMoMbzWH9WcJ+qowBUHMb55Ai0WcVho0/w7+FPAiVDyUobxpaZnqBOV+/n/hC9kkkC1bfokP6oFEi6w4m1/g1LlgWLo+ex9H2ebOt9yiUBsWXwWUyrvbtCANpo510Ss9rCj9NS9vu7iH3GV9JcpaGs1AF7NXNduwI+LCiYK+smBo0T1I8Sq/TpoYtDCAhoGZth4sppetEgMNOFsri7ZZiu0NiJLpEhhVou06CMM/KSwBU2PzeSw== Azure Self Hosted Runners"
|
||||
}
|
||||
|
||||
computer_name_prefix = "daml-ubuntu"
|
||||
|
||||
# save a bit of energy for the planet
|
||||
overprovision = false
|
||||
|
||||
custom_data = base64encode(templatefile("${path.module}/ubuntu_startup.sh", {
|
||||
vsts_token = secret_resource.vsts-token.value
|
||||
vsts_account = "digitalasset"
|
||||
vsts_pool = "ubuntu_20_04"
|
||||
size = local.ubuntu.azure[count.index].disk_size
|
||||
gcp_logging = ""
|
||||
assignment = local.ubuntu.azure[count.index].assignment
|
||||
}))
|
||||
|
||||
source_image_reference {
|
||||
publisher = "canonical"
|
||||
offer = "0001-com-ubuntu-server-focal"
|
||||
sku = "20_04-lts"
|
||||
version = "latest"
|
||||
}
|
||||
|
||||
os_disk {
|
||||
caching = "ReadOnly"
|
||||
storage_account_type = "Standard_LRS"
|
||||
disk_size_gb = local.ubuntu.azure[count.index].disk_size
|
||||
}
|
||||
|
||||
network_interface {
|
||||
name = "default"
|
||||
primary = true
|
||||
|
||||
ip_configuration {
|
||||
name = "default"
|
||||
primary = true
|
||||
subnet_id = one(azurerm_virtual_network.ubuntu.subnet).id
|
||||
}
|
||||
}
|
||||
|
||||
# required to get console output in Azure UI
|
||||
boot_diagnostics {
|
||||
storage_account_uri = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
resource "azurerm_virtual_network" "ubuntu" {
|
||||
name = "ubuntu"
|
||||
location = azurerm_resource_group.daml-ci.location
|
||||
resource_group_name = azurerm_resource_group.daml-ci.name
|
||||
address_space = ["10.0.0.0/16"]
|
||||
|
||||
subnet {
|
||||
name = "subnet"
|
||||
address_prefix = "10.0.1.0/24"
|
||||
security_group = azurerm_network_security_group.ubuntu.id
|
||||
}
|
||||
}
|
||||
|
||||
resource "azurerm_network_security_group" "ubuntu" {
|
||||
name = "ubuntu"
|
||||
location = azurerm_resource_group.daml-ci.location
|
||||
resource_group_name = azurerm_resource_group.daml-ci.name
|
||||
|
||||
security_rule {
|
||||
name = "deny-inbound"
|
||||
priority = 100
|
||||
direction = "Inbound"
|
||||
access = "Deny"
|
||||
protocol = "*"
|
||||
source_port_range = "*"
|
||||
destination_port_range = "*"
|
||||
source_address_prefix = "*"
|
||||
destination_address_prefix = "*"
|
||||
}
|
||||
}
|
@ -1,31 +1,16 @@
|
||||
# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
locals {
|
||||
ubuntu = [
|
||||
{
|
||||
name = "ci-u1",
|
||||
disk_size = 400,
|
||||
size = 30,
|
||||
},
|
||||
{
|
||||
name = "ci-u2",
|
||||
disk_size = 400,
|
||||
size = 0,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04" {
|
||||
count = length(local.ubuntu)
|
||||
count = length(local.ubuntu.gcp)
|
||||
provider = google-beta
|
||||
name = local.ubuntu[count.index].name
|
||||
base_instance_name = local.ubuntu[count.index].name
|
||||
name = local.ubuntu.gcp[count.index].name
|
||||
base_instance_name = local.ubuntu.gcp[count.index].name
|
||||
region = "us-east1"
|
||||
target_size = local.ubuntu[count.index].size
|
||||
target_size = local.ubuntu.gcp[count.index].size
|
||||
|
||||
version {
|
||||
name = local.ubuntu[count.index].name
|
||||
name = local.ubuntu.gcp[count.index].name
|
||||
instance_template = google_compute_instance_template.vsts-agent-ubuntu_20_04[count.index].self_link
|
||||
}
|
||||
|
||||
@ -43,13 +28,13 @@ resource "google_compute_region_instance_group_manager" "vsts-agent-ubuntu_20_04
|
||||
}
|
||||
|
||||
resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" {
|
||||
count = length(local.ubuntu)
|
||||
name_prefix = "${local.ubuntu[count.index].name}-"
|
||||
count = length(local.ubuntu.gcp)
|
||||
name_prefix = "${local.ubuntu.gcp[count.index].name}-"
|
||||
machine_type = "c2-standard-8"
|
||||
labels = local.machine-labels
|
||||
|
||||
disk {
|
||||
disk_size_gb = local.ubuntu[count.index].disk_size
|
||||
disk_size_gb = local.ubuntu.gcp[count.index].disk_size
|
||||
disk_type = "pd-ssd"
|
||||
source_image = "ubuntu-os-cloud/ubuntu-2004-lts"
|
||||
}
|
||||
@ -59,11 +44,17 @@ resource "google_compute_instance_template" "vsts-agent-ubuntu_20_04" {
|
||||
}
|
||||
|
||||
metadata = {
|
||||
startup-script = templatefile("${path.module}/vsts_agent_ubuntu_20_04_startup.sh", {
|
||||
startup-script = templatefile("${path.module}/ubuntu_startup.sh", {
|
||||
vsts_token = secret_resource.vsts-token.value
|
||||
vsts_account = "digitalasset"
|
||||
vsts_pool = "ubuntu_20_04"
|
||||
size = local.ubuntu[count.index].disk_size
|
||||
size = local.ubuntu.gcp[count.index].disk_size
|
||||
gcp_logging = <<EOF
|
||||
# Taken from https://cloud.google.com/logging/docs/agent/logging/installation
|
||||
curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
|
||||
curl -sSL https://dl.google.com/cloudagents/add-logging-agent-repo.sh | bash -s -- --also-install
|
||||
EOF
|
||||
assignment = local.ubuntu.gcp[count.index].assignment
|
||||
})
|
||||
|
||||
shutdown-script = nonsensitive("#!/usr/bin/env bash\nset -euo pipefail\ncd /home/vsts/agent\nsu vsts <<SHUTDOWN_AGENT\nexport VSTS_AGENT_INPUT_TOKEN='${secret_resource.vsts-token.value}'\n./config.sh remove --unattended --auth PAT\nSHUTDOWN_AGENT\n ")
|
@ -72,10 +72,7 @@ apt-get install -qy \
|
||||
xdg-utils \
|
||||
wget
|
||||
|
||||
# Taken from https://cloud.google.com/logging/docs/agent/logging/installation
|
||||
curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
|
||||
curl -sSL https://dl.google.com/cloudagents/add-logging-agent-repo.sh | bash -s -- --also-install
|
||||
|
||||
${gcp_logging}
|
||||
#install docker
|
||||
# BEGIN Installing Docker per https://docs.docker.com/engine/install/ubuntu/
|
||||
apt-get -y install apt-transport-https \
|
||||
@ -159,7 +156,7 @@ VSTS_TOKEN=${vsts_token}
|
||||
|
||||
mkdir -p ~/agent
|
||||
cd ~/agent
|
||||
echo 'assignment=default' > .capabilities
|
||||
echo 'assignment=${assignment}' > .capabilities
|
||||
|
||||
echo Determining matching VSTS agent...
|
||||
VSTS_AGENT_RESPONSE=$(curl -sSfL \
|
@ -204,6 +204,7 @@ in rec {
|
||||
# used to set up the webide CI pipeline in azure-cron.yml
|
||||
docker-credential-gcr = pkgs.docker-credential-gcr;
|
||||
terraform = pkgs.terraform_1.withPlugins (p: with p; [
|
||||
azurerm
|
||||
google
|
||||
google-beta
|
||||
secret
|
||||
|
Loading…
Reference in New Issue
Block a user