🌱 Parallelize builds

- parallelize builds
This commit is contained in:
naveen 2022-02-16 23:54:22 +00:00 committed by Naveen
parent 1aff6db9f6
commit bba55d4257
2 changed files with 617 additions and 18 deletions

View File

@ -15,25 +15,16 @@
name: build
on: [push, pull_request]
jobs:
validate:
name: Validate
unit-test:
name: unit-test
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
# In order:
# * Module download cache
# * Build cache (Linux)
# * Build cache (Mac)
# * Build cache (Windows)
path: |
~/go/pkg/mod
~/.cache/go-build
@ -50,17 +41,625 @@ jobs:
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: Install tools
run: make install
- name: Run presubmit tests
run: |
go env -w GOFLAGS=-mod=mod
make -j 5 all
- name: Run unit-tests
run: make unit-test
- name: Upload codecoverage
uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # 2.1.0
with:
files: ./unit-coverage.out
verbose: true
generate-mocks:
name: generate-mocks
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: generate mocks
run: |
go env -w GOFLAGS=-mod=mod
make install
make generate-mocks
generate-docs:
name: generate-docs
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: generate docs
run: |
go env -w GOFLAGS=-mod=mod
make generate-docs
build-proto:
name: build-proto
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build-proto
run: |
go env -w GOFLAGS=-mod=mod
make build-proto
build-scorecard:
name: build-scorecard
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: Run build
run: |
go env -w GOFLAGS=-mod=mod
make build-scorecard
build-controller:
name: build-controller
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build cron
run: |
go env -w GOFLAGS=-mod=mod
make build-controller
build-worker:
name: build-worker
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build worker
run: |
go env -w GOFLAGS=-mod=mod
make build-worker
build-cii-worker:
name: build-cii-worker
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build cii-worker
run: |
go env -w GOFLAGS=-mod=mod
make build-cii-worker
build-shuffler:
name: build-shuffler
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build shuffler
run: |
go env -w GOFLAGS=-mod=mod
make build-shuffler
build-bq-transfer:
name: build-bq-transfer
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build bq transfer
run: |
go env -w GOFLAGS=-mod=mod
make build-bq-transfer
build-github-server:
name: build-github-server
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build bq transfer
run: |
go env -w GOFLAGS=-mod=mod
make build-github-server
build-webhook:
name: build-webhook
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build webhook
run: |
go env -w GOFLAGS=-mod=mod
make build-webhook
build-add-script:
name: build-add-script
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build-add-script
run: |
go env -w GOFLAGS=-mod=mod
make build-add-script
build-validate-script:
name: build-validate-script
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build-validate-script
run: |
go env -w GOFLAGS=-mod=mod
make build-validate-script
build-update-script:
name: build-update-script
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: build-validate-script
run: |
go env -w GOFLAGS=-mod=mod
make build-update-script
check-linter:
name: check-linter
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Install Protoc
uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2
with:
version: 3.17.3
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: Run build
run: |
go env -w GOFLAGS=-mod=mod
make check-linter
validate-docs:
name: validate-docs
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: Run build
run: |
go env -w GOFLAGS=-mod=mod
make validate-docs
add-projects:
name: add-projects
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: Run build
run: |
go env -w GOFLAGS=-mod=mod
make add-projects
validate-projects:
name: validate-projects
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed #v2.1.7
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2.2.0
with:
go-version: '^1.17'
- name: Run build
run: |
go env -w GOFLAGS=-mod=mod
make validate-projects
license-check:
name: license boilerplate check
runs-on: ubuntu-latest

View File

@ -40,7 +40,7 @@ $(PROTOC):
##@ Build
################################## make all ###################################
all: ## Runs build, test and verify
all-targets = build check-linter check-osv unit-test validate-docs add-projects validate-projects
all-targets = build check-linter check-osv validate-docs add-projects validate-projects
.PHONY: all all-targets-update-dependencies $(all-targets) update-dependencies tree-status
all-targets-update-dependencies: $(all-targets) | update-dependencies
all: update-dependencies all-targets-update-dependencies tree-status