collect historical download data (#2003)

azure-cron.yml

@@ -188,3 +188,24 @@ jobs:
           fi
         env:
           MARKETPLACE_TOKEN: $(VSCODE_MARKETPLACE_TOKEN)
+
+  - job: download_stats
+    timeoutInMinutes: 10
+    pool:
+      name: "linux-pool"
+    steps:
+      - checkout: self
+      - bash: |
+          set -euo pipefail
+
+          eval "$(dev-env/bin/dade-assist)"
+
+          STATS=$(mktemp)
+          curl https://api.github.com/repos/digital-asset/daml/releases -s | gzip -9 > $STATS
+
+          GCS_KEY=$(mktemp)
+          echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
+          gcloud auth activate-service-account --key-file=$GCS_KEY
+          BOTO_CONFIG=/dev/null gsutil cp $STATS gs://daml-data/downloads/$(date -u +%Y%m%d_%H%M%SZ).json.gz
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)

infra/data_bucket.tf

@@ -0,0 +1,34 @@
+# Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+resource "google_storage_bucket" "data" {
+  project = "${local.project}"
+  name    = "daml-data"
+  labels  = "${local.labels}"
+
+  # SLA is enough for a cache and is cheaper than MULTI_REGIONAL
+  # see https://cloud.google.com/storage/docs/storage-classes
+  storage_class = "REGIONAL"
+
+  # Use a normal region since the storage_class is regional
+  location = "${local.region}"
+}
+
+resource "google_storage_bucket_acl" "data" {
+  bucket = "${google_storage_bucket.data.name}"
+
+  role_entity = [
+    "OWNER:project-owners-${data.google_project.current.number}",
+    "OWNER:project-editors-${data.google_project.current.number}",
+    "READER:project-viewers-${data.google_project.current.number}",
+  ]
+}
+
+// allow rw access for CI writer (see writer.tf)
+resource "google_storage_bucket_iam_member" "data" {
+  bucket = "${google_storage_bucket.data.name}"
+
+  # https://cloud.google.com/storage/docs/access-control/iam-roles
+  role   = "roles/storage.objectAdmin"
+  member = "serviceAccount:${google_service_account.writer.email}"
+}

infra/main.tf

@@ -18,6 +18,10 @@ provider "google-beta" {
   region  = "us-east4"
 }
 
+data "google_project" "current" {
+  project_id = "${local.project}"
+}
+
 locals {
   labels = {
     cost-allocation = "daml-language"