From 35511342c80aebb7a23f34ec6060912bc2fcb5cc Mon Sep 17 00:00:00 2001 From: naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Tue, 15 Feb 2022 16:31:08 +0000 Subject: [PATCH] :seedling: Parallelize the builds - Created a workflow with multiple jobs for each of the docker builds - Created a workflow with multiple jobs for each of the ko builds - Removed the reference to dockerbuild and kobuild in the build-targets make target - This should reduce the time required to finish the CI builds as it makes it parallel. --- .github/workflows/docker.yml | 276 +++++++++++++++++++++++++++++++++++ Makefile | 4 +- 2 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..5b68fa68 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,276 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: docker-build +on: [push, pull_request] +jobs: + scorecard: + name: scorecard-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make scorecard-docker + cron-controller: + name: cron-controller-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make cron-controller-docker + cron-worker: + name: cron-worker-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make cron-worker-docker + cron-cii-worker: + name: cron-cii--worker-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make cron-cii-worker-docker + cron-bq-transfer: + name: cron-bq-transfer-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make cron-bq-transfer-docker + cron-webhook: + name: cron-webhook-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make cron-webhook-docker + cron-github-server: + name: cron-github-server-docker + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + with: + version: 3.17.3 + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Clone the code + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0 + with: + go-version: '^1.17' + - name: docker build + run: make cron-github-server-docker diff --git a/Makefile b/Makefile index 8d27b82a..1894dfa0 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ $(PROTOC): ##@ Build ################################## make all ################################### all: ## Runs build, test and verify -all-targets = build check-linter check-osv unit-test validate-docs add-projects validate-projects +all-targets = build check-linter check-osv unit-test validate-docs add-projects validate-projects ko-build-everything .PHONY: all all-targets-update-dependencies $(all-targets) update-dependencies tree-status all-targets-update-dependencies: $(all-targets) | update-dependencies all: update-dependencies all-targets-update-dependencies tree-status @@ -91,7 +91,7 @@ build-cron: build-controller build-worker build-cii-worker \ build-shuffler build-bq-transfer build-github-server \ build-webhook build-add-script build-validate-script build-update-script -build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron ko-build-everything dockerbuild +build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron .PHONY: build $(build-targets) build: ## Build all binaries and images in the repo. build: $(build-targets)