Fix k8s yaml errors and document how to prevent them. (#2298)

Signed-off-by: Spencer Schrock <sschrock@google.com>

Signed-off-by: Spencer Schrock <sschrock@google.com>

cron/k8s/README.md

@@ -3,6 +3,12 @@
 Currently there is no automation to sync changes to these files to the GKE cluster.
 Changes must be manually applied with `kubectl` by a user with permissions to modify the cluster.
 
+Before committing any changes, it is recommended to check your YAML files for errors with [yamllint](yamllint.readthedocs.io). To check all YAML files in this directory, run:
+```
+yamllint -d relaxed .
+```
+Note: `relaxed` is a more  tolerant, predefined config which ignores things like line length.
+
 ## Installing `kubectl`
 
 Follow instructions

cron/k8s/cii.yaml

@@ -26,14 +26,14 @@ spec:
         spec:
           restartPolicy: Never
           containers:
-          - name: cii-worker
-            image: gcr.io/openssf/scorecard-cii-worker:stable
-            args: ["--config=/etc/scorecard/config.yaml"]
-            imagePullPolicy: Always
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: cii-worker
+              image: gcr.io/openssf/scorecard-cii-worker:stable
+              args: ["--config=/etc/scorecard/config.yaml"]
+              imagePullPolicy: Always
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
           volumes:
             - name: config-volume
               configMap:

cron/k8s/controller.release.yaml

@@ -50,37 +50,37 @@ spec:
         spec:
           restartPolicy: Never
           containers:
-          - name: controller
-            image: gcr.io/openssf/scorecard-batch-controller:latest
-            args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.release.csv"]
-            imagePullPolicy: Always
-            env:
-            - name: SCORECARD_REQUEST_TOPIC_URL
-              value: "gcppubsub://projects/openssf/topics/scorecard-batch-requests-releasetest"
-            - name: SCORECARD_DATA_BUCKET_URL
-              value: "gs://ossf-scorecard-data-releasetest2"
-            # Raw results.
-            - name: SCORECARD_RAW_RESULT_DATA_BUCKET_URL
-              value: "gs://ossf-scorecard-rawdata-releasetest"
-            - name: SCORECARD_SHARD_SIZE
-              value: "5"
-            resources:
-              limits:
-                memory: 1Gi
-              requests:
-                memory: 1Gi
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: controller
+              image: gcr.io/openssf/scorecard-batch-controller:latest
+              args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.release.csv"]
+              imagePullPolicy: Always
+              env:
+                - name: SCORECARD_REQUEST_TOPIC_URL
+                  value: "gcppubsub://projects/openssf/topics/scorecard-batch-requests-releasetest"
+                - name: SCORECARD_DATA_BUCKET_URL
+                  value: "gs://ossf-scorecard-data-releasetest2"
+                # Raw results.
+                - name: SCORECARD_RAW_RESULT_DATA_BUCKET_URL
+                  value: "gs://ossf-scorecard-rawdata-releasetest"
+                - name: SCORECARD_SHARD_SIZE
+                  value: "5"
+              resources:
+                limits:
+                  memory: 1Gi
+                requests:
+                  memory: 1Gi
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
+            - name: worker-update
+              image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee
+              command:
+                - "kubectl"
+                - "rollout"
+                - "restart"
+                - "deployment/scorecard-batch-worker-releasetest"
           volumes:
             - name: config-volume
               configMap:
                 name: scorecard-config
-          - name: worker-update
-            image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee
-            command:
-              - "kubectl"
-              - "rollout"
-              - "restart"
-              - "deployment/scorecard-batch-worker-releasetest"

cron/k8s/controller.yaml

@@ -50,27 +50,27 @@ spec:
         spec:
           restartPolicy: Never
           containers:
-          - name: controller
-            image: gcr.io/openssf/scorecard-batch-controller:stable
-            args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.csv"]
-            imagePullPolicy: Always
-            resources:
-              limits:
-                memory: 1Gi
-              requests:
-                memory: 1Gi
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: controller
+              image: gcr.io/openssf/scorecard-batch-controller:stable
+              args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.csv"]
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  memory: 1Gi
+                requests:
+                  memory: 1Gi
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
+            - name: worker-update
+              image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee
+              command:
+                - "kubectl"
+                - "rollout"
+                - "restart"
+                - "deployment/scorecard-batch-worker"
           volumes:
             - name: config-volume
               configMap:
                 name: scorecard-config
-          - name: worker-update
-            image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee
-            command:
-              - "kubectl"
-              - "rollout"
-              - "restart"
-              - "deployment/scorecard-batch-worker"

cron/k8s/transfer-raw.yaml

@@ -25,24 +25,24 @@ spec:
       template:
         spec:
           containers:
-          - name: bq-rawdata-transfer
-            image: gcr.io/openssf/scorecard-bq-transfer:latest
-            args: ["--config=/etc/scorecard/config.yaml"]
-            imagePullPolicy: Always
-            resources:
-              limits:
-                memory: 1Gi
-              requests:
-                memory: 1Gi
-            env:
-              - name: SCORECARD_BIGQUERY_TABLE
-                value: "scorecard-rawdata"
-              - name: SCORECARD_DATA_BUCKET_URL
-                value: "gs://ossf-scorecard-rawdata"
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: bq-rawdata-transfer
+              image: gcr.io/openssf/scorecard-bq-transfer:latest
+              args: ["--config=/etc/scorecard/config.yaml"]
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  memory: 1Gi
+                requests:
+                  memory: 1Gi
+              env:
+                - name: SCORECARD_BIGQUERY_TABLE
+                  value: "scorecard-rawdata"
+                - name: SCORECARD_DATA_BUCKET_URL
+                  value: "gs://ossf-scorecard-rawdata"
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
           volumes:
             - name: config-volume
               configMap:

cron/k8s/transfer.release-raw.yaml

@@ -26,26 +26,26 @@ spec:
         spec:
           restartPolicy: OnFailure
           containers:
-          - name: bq-transfer-rawdata-releasetest
-            image: gcr.io/openssf/scorecard-bq-transfer:latest
-            args: ["--config=/etc/scorecard/config.yaml"]
-            imagePullPolicy: Always
-            env:
-            - name: SCORECARD_DATA_BUCKET_URL
-              value: "gs://ossf-scorecard-rawdata-releasetest"
-            - name: SCORECARD_BIGQUERY_TABLE
-              value: "scorecard-rawdata-releasetest"
-            - name: SCORECARD_COMPLETION_THRESHOLD
-              value: "0.9"
-            resources:
-              limits:
-                memory: 1Gi
-              requests:
-                memory: 1Gi
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: bq-transfer-rawdata-releasetest
+              image: gcr.io/openssf/scorecard-bq-transfer:latest
+              args: ["--config=/etc/scorecard/config.yaml"]
+              imagePullPolicy: Always
+              env:
+                - name: SCORECARD_DATA_BUCKET_URL
+                  value: "gs://ossf-scorecard-rawdata-releasetest"
+                - name: SCORECARD_BIGQUERY_TABLE
+                  value: "scorecard-rawdata-releasetest"
+                - name: SCORECARD_COMPLETION_THRESHOLD
+                  value: "0.9"
+              resources:
+                limits:
+                  memory: 1Gi
+                requests:
+                  memory: 1Gi
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
           volumes:
             - name: config-volume
               configMap:

cron/k8s/transfer.release.yaml

@@ -26,28 +26,28 @@ spec:
         spec:
           restartPolicy: OnFailure
           containers:
-          - name: bq-transfer-releasetest-v2
-            image: gcr.io/openssf/scorecard-bq-transfer:latest
-            args: ["--config=/etc/scorecard/config.yaml"]
-            imagePullPolicy: Always
-            env:
-            - name: SCORECARD_DATA_BUCKET_URL
-              value: "gs://ossf-scorecard-data-releasetest2"
-            - name: SCORECARD_BIGQUERY_TABLE
-              value: "scorecard_releasetest2"
-            - name: SCORECARD_COMPLETION_THRESHOLD
-              value: "0.9"
-            - name: SCORECARD_WEBHOOK_URL
-              value: "http://10.4.8.246/"
-            resources:
-              limits:
-                memory: 1Gi
-              requests:
-                memory: 1Gi
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: bq-transfer-releasetest-v2
+              image: gcr.io/openssf/scorecard-bq-transfer:latest
+              args: ["--config=/etc/scorecard/config.yaml"]
+              imagePullPolicy: Always
+              env:
+                - name: SCORECARD_DATA_BUCKET_URL
+                  value: "gs://ossf-scorecard-data-releasetest2"
+                - name: SCORECARD_BIGQUERY_TABLE
+                  value: "scorecard_releasetest2"
+                - name: SCORECARD_COMPLETION_THRESHOLD
+                  value: "0.9"
+                - name: SCORECARD_WEBHOOK_URL
+                  value: "http://10.4.8.246/"
+              resources:
+                limits:
+                  memory: 1Gi
+                requests:
+                  memory: 1Gi
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
           volumes:
             - name: config-volume
               configMap:

cron/k8s/transfer.yaml

@@ -25,19 +25,19 @@ spec:
       template:
         spec:
           containers:
-          - name: bq-transfer-v2
-            image: gcr.io/openssf/scorecard-bq-transfer:latest
-            args: ["--config=/etc/scorecard/config.yaml"]
-            imagePullPolicy: Always
-            resources:
-              limits:
-                memory: 1Gi
-              requests:
-                memory: 1Gi
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/scorecard
-                readOnly: true
+            - name: bq-transfer-v2
+              image: gcr.io/openssf/scorecard-bq-transfer:latest
+              args: ["--config=/etc/scorecard/config.yaml"]
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  memory: 1Gi
+                requests:
+                  memory: 1Gi
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/scorecard
+                  readOnly: true
           volumes:
             - name: config-volume
               configMap:

cron/k8s/worker.release.yaml

@@ -27,34 +27,34 @@ spec:
         app.kubernetes.io/name: worker-releasetest
     spec:
       containers:
-      - name: worker
-        image: gcr.io/openssf/scorecard-batch-worker:latest
-        args: ["--ignoreRuntimeErrors=false", "--config=/etc/scorecard/config.yaml"]
-        imagePullPolicy: Always
-        env:
-        - name: SCORECARD_DATA_BUCKET_URL
-          value: "gs://ossf-scorecard-data-releasetest2"
-        - name: SCORECARD_RAW_RESULT_DATA_BUCKET_URL
-          value: "gs://ossf-scorecard-rawdata-releasetest"
-        - name: SCORECARD_REQUEST_SUBSCRIPTION_URL
-          value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest"
-        - name: SCORECARD_METRIC_EXPORTER
-          value: "printer"
-        - name: GITHUB_AUTH_SERVER
-          value: "10.4.4.210:80"
-        - name: "SCORECARD_API_RESULTS_BUCKET_URL"
-          value: "gs://ossf-scorecard-cron-releasetest-results"
-        resources:
-          requests:
-            memory: 5Gi
-            ephemeral-storage: 100Gi
-          limits:
-            memory: 12Gi
-            ephemeral-storage: 500Gi
-        volumeMounts:
-          - name: config-volume
-            mountPath: /etc/scorecard
-            readOnly: true
+        - name: worker
+          image: gcr.io/openssf/scorecard-batch-worker:latest
+          args: ["--ignoreRuntimeErrors=false", "--config=/etc/scorecard/config.yaml"]
+          imagePullPolicy: Always
+          env:
+            - name: SCORECARD_DATA_BUCKET_URL
+              value: "gs://ossf-scorecard-data-releasetest2"
+            - name: SCORECARD_RAW_RESULT_DATA_BUCKET_URL
+              value: "gs://ossf-scorecard-rawdata-releasetest"
+            - name: SCORECARD_REQUEST_SUBSCRIPTION_URL
+              value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest"
+            - name: SCORECARD_METRIC_EXPORTER
+              value: "printer"
+            - name: GITHUB_AUTH_SERVER
+              value: "10.4.4.210:80"
+            - name: "SCORECARD_API_RESULTS_BUCKET_URL"
+              value: "gs://ossf-scorecard-cron-releasetest-results"
+          resources:
+            requests:
+              memory: 5Gi
+              ephemeral-storage: 100Gi
+            limits:
+              memory: 12Gi
+              ephemeral-storage: 500Gi
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/scorecard
+              readOnly: true
       volumes:
         - name: config-volume
           configMap:

cron/k8s/worker.yaml

@@ -27,24 +27,24 @@ spec:
         app.kubernetes.io/name: worker
     spec:
       containers:
-      - name: worker
-        image: gcr.io/openssf/scorecard-batch-worker:stable
-        args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"]
-        imagePullPolicy: Always
-        env:
-        - name: GITHUB_AUTH_SERVER
-          value: "10.4.4.210:80"
-        resources:
-          requests:
-            memory: 5Gi
-            ephemeral-storage: 100Gi
-          limits:
-            memory: 12Gi
-            ephemeral-storage: 500Gi
-        volumeMounts:
-          - name: config-volume
-            mountPath: /etc/scorecard
-            readOnly: true
+        - name: worker
+          image: gcr.io/openssf/scorecard-batch-worker:stable
+          args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"]
+          imagePullPolicy: Always
+          env:
+            - name: GITHUB_AUTH_SERVER
+              value: "10.4.4.210:80"
+          resources:
+            requests:
+              memory: 5Gi
+              ephemeral-storage: 100Gi
+            limits:
+              memory: 12Gi
+              ephemeral-storage: 500Gi
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/scorecard
+              readOnly: true
       volumes:
         - name: config-volume
           configMap: