diff --git a/Makefile b/Makefile index c9790476..454fc769 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,11 @@ SHELL := /bin/bash -GINKGO := ginkgo GIT_HASH := $(shell git rev-parse HEAD) GIT_VERSION ?= $(shell git describe --tags --always --dirty) SOURCE_DATE_EPOCH=$(shell git log --date=iso8601-strict -1 --pretty=%ct) -GOLANGCI_LINT := golangci-lint -PROTOC_GEN_GO := protoc-gen-go -MOCKGEN := mockgen -PROTOC := $(shell which protoc) -GORELEASER := goreleaser IMAGE_NAME = scorecard OUTPUT = output PLATFORM="linux/amd64,linux/arm64,linux/386,linux/arm" LDFLAGS=$(shell ./scripts/version-ldflags) -KOCACHE_PATH=/tmp/ko - -define create_kocache_path - mkdir -p $(KOCACHE_PATH) -endef @@ -30,24 +19,72 @@ help: ## Display this help ############################################################################### -##@ Development +##@ Tools ################################ make install ################################# -.PHONY: install -install: ## Installs all dependencies needed to compile Scorecard -install: | $(PROTOC) - @echo Installing tools from tools/tools.go - cd tools; cat tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install % +TOOLS_DIR := tools +TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin) +GOBIN := $(shell go env GOBIN) +# Golang binaries. + +GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint +$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint + +KO := $(TOOLS_BIN_DIR)/ko +$(KO): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/google/ko + +STUNNING_TRIBBLE := $(TOOLS_BIN_DIR)/stunning-tribble +$(STUNNING_TRIBBLE): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/naveensrinivasan/stunning-tribble + +MOCKGEN := $(TOOLS_BIN_DIR)/mockgen +$(MOCKGEN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/golang/mock/mockgen + +GINKGO := $(TOOLS_BIN_DIR)/ginkgo +$(GINKGO): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/onsi/ginkgo/v2/ginkgo + +GORELEASER := $(TOOLS_BIN_DIR)/goreleaser +$(GORELEASER): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/goreleaser/goreleaser + +PROTOC_GEN_GO := $(TOOLS_BIN_DIR)/protoc-gen-go +$(PROTOC_GEN_GO): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install google.golang.org/protobuf/cmd/protoc-gen-go + +# Non-Golang binaries. +# TODO: Figure out how to install these binaries automatically. + +PROTOC := $(shell which protoc) $(PROTOC): ifeq (,$(PROTOC)) $(error download and install protobuf compiler package - https://developers.google.com/protocol-buffers/docs/downloads) endif + +# Installs required binaries into $(TOOLS_BIN_DIR) wherever possible. +# Keeping a local copy instead of a global install allows for: +# i) Controlling the binary version Scorecard depends on leading to consistent +# behavior across users. +# ii) Avoids installing a whole bunch of otherwise unnecessary tools in the user's workspace. +.PHONY: install +install: ## Installs required binaries. +install: $(GOLANGCI_LINT) \ + $(KO) \ + $(STUNNING_TRIBBLE) \ + $(PROTOC_GEN_GO) $(PROTOC) \ + $(MOCKGEN) \ + $(GINKGO) \ + $(GORELEASER) + ############################################################################### ##@ Build ################################## make all ################################### all: ## Runs build, test and verify -all-targets = build check-linter check-osv validate-docs add-projects validate-projects +all-targets = build check-linter check-osv validate-docs add-projects validate-projects .PHONY: all all-targets-update-dependencies $(all-targets) update-dependencies tree-status all-targets-update-dependencies: $(all-targets) | update-dependencies all: update-dependencies all-targets-update-dependencies tree-status @@ -58,23 +95,22 @@ update-dependencies: ## Update go dependencies for all modules cd tools; go mod tidy && go mod verify; cd ../ cd attestor; go mod tidy && go mod verify; cd ../ -$(GOLANGCI_LINT): install check-linter: ## Install and run golang linter -check-linter: $(GOLANGCI_LINT) +check-linter: | $(GOLANGCI_LINT) # Run golangci-lint linter - golangci-lint run -c .golangci.yml + $(GOLANGCI_LINT) run -c .golangci.yml check-osv: ## Checks osv.dev for any vulnerabilities -check-osv: $(install) +check-osv: | $(STUNNING_TRIBBLE) # Run stunning-tribble for checking the dependencies have any OSV go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \ - | stunning-tribble + | $(STUNNING_TRIBBLE) # Checking the tools which also has go.mod cd tools; go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \ - | stunning-tribble ; cd .. + | $(STUNNING_TRIBBLE) ; cd .. # Checking the attestor module for vulns cd attestor; go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \ - | stunning-tribble ; cd .. + | $(STUNNING_TRIBBLE) ; cd .. add-projects: ## Adds new projects to ./cron/internal/data/projects.csv add-projects: ./cron/internal/data/projects.csv | build-add-script @@ -99,35 +135,40 @@ build-cron: build-controller build-worker build-cii-worker \ build-shuffler build-bq-transfer build-github-server \ build-webhook build-add-script build-validate-script build-update-script -build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron +build-targets = generate-mocks generate-docs build-scorecard build-cron build-proto .PHONY: build $(build-targets) build: ## Build all binaries and images in the repo. build: $(build-targets) build-proto: ## Compiles and generates all required protobufs build-proto: cron/internal/data/request.pb.go cron/internal/data/metadata.pb.go -cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) install - protoc --go_out=../../../ cron/internal/data/request.proto -cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) install - protoc --go_out=../../../ cron/internal/data/metadata.proto +cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) $(PROTOC_GEN_GO) + $(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=../../../ cron/internal/data/request.proto +cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) $(PROTOC_GEN_GO) + $(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=../../../ cron/internal/data/metadata.proto generate-mocks: ## Compiles and generates all mocks using mockgen. -generate-mocks: clients/mockclients/repo_client.go clients/mockclients/repo.go clients/mockclients/cii_client.go checks/mockclients/vulnerabilities.go cmd/packagemanager_mockclient.go -clients/mockclients/repo_client.go: clients/repo_client.go +generate-mocks: clients/mockclients/repo_client.go \ + clients/mockclients/repo.go \ + clients/mockclients/cii_client.go \ + checks/mockclients/vulnerabilities.go \ + cmd/packagemanager_mockclient.go +clients/mockclients/repo_client.go: clients/repo_client.go | $(MOCKGEN) # Generating MockRepoClient $(MOCKGEN) -source=clients/repo_client.go -destination=clients/mockclients/repo_client.go -package=mockrepo -copyright_file=clients/mockclients/license.txt -clients/mockclients/repo.go: clients/repo.go +clients/mockclients/repo.go: clients/repo.go | $(MOCKGEN) # Generating MockRepo $(MOCKGEN) -source=clients/repo.go -destination=clients/mockclients/repo.go -package=mockrepo -copyright_file=clients/mockclients/license.txt -clients/mockclients/cii_client.go: clients/cii_client.go +clients/mockclients/cii_client.go: clients/cii_client.go | $(MOCKGEN) # Generating MockCIIClient $(MOCKGEN) -source=clients/cii_client.go -destination=clients/mockclients/cii_client.go -package=mockrepo -copyright_file=clients/mockclients/license.txt -checks/mockclients/vulnerabilities.go: clients/vulnerabilities.go +checks/mockclients/vulnerabilities.go: clients/vulnerabilities.go | $(MOCKGEN) # Generating MockCIIClient $(MOCKGEN) -source=clients/vulnerabilities.go -destination=clients/mockclients/vulnerabilities.go -package=mockrepo -copyright_file=clients/mockclients/license.txt -cmd/packagemanager_mockclient.go: cmd/packagemanager_client.go +cmd/packagemanager_mockclient.go: cmd/packagemanager_client.go | $(MOCKGEN) # Generating MockPackageManagerClient $(MOCKGEN) -source=cmd/packagemanager_client.go -destination=cmd/packagemanager_mockclient.go -package=cmd -copyright_file=clients/mockclients/license.txt + generate-docs: ## Generates docs generate-docs: validate-docs docs/checks.md docs/checks.md: docs/checks/internal/checks.yaml docs/checks/internal/*.go docs/checks/internal/generate/*.go @@ -196,66 +237,6 @@ cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/ # Run go build on the update script cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update -ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko -.PHONY: ko-build-everything $(ko-targets) -ko-build-everything: $(ko-targets) - -scorecard-ko: - $(call create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) LDFLAGS="$(LDFLAGS)" \ - KO_CACHE=$(KOCACHE_PATH) ko build -B \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4 -cron-controller-ko: - $(call_create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-controller LDFLAGS="$(LDFLAGS)" \ - KOCACHE=$(KOCACHE_PATH) ko build -B \ - --push=false \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/controller -cron-worker-ko: - $(call_create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker LDFLAGS="$(LDFLAGS)" \ - KOCACHE=$(KOCACHE_PATH) ko build -B \ - --push=false \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/worker -cron-cii-worker-ko: - $(call_create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker LDFLAGS="$(LDFLAGS)" \ - KOCACHE=$(KOCACHE_PATH) ko build -B \ - --push=false \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/cii -cron-bq-transfer-ko: - $(call_create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer LDFLAGS="$(LDFLAGS)" \ - KOCACHE=$(KOCACHE_PATH) ko build -B \ - --push=false \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/bq -cron-webhook-ko: - $(call_create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook LDFLAGS="$(LDFLAGS)" \ - KOCACHE=$(KOCACHE_PATH) ko build -B \ - --push=false \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/webhook -cron-github-server-ko: - $(call_create_kocache_path) - KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server LDFLAGS="$(LDFLAGS)" \ - KOCACHE=$(KOCACHE_PATH) ko build -B \ - --push=false \ - --sbom=none \ - --platform=$(PLATFORM)\ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server - docker-targets = scorecard-docker cron-controller-docker cron-worker-docker cron-cii-worker-docker cron-bq-transfer-docker cron-webhook-docker cron-github-server-docker .PHONY: dockerbuild $(docker-targets) dockerbuild: $(docker-targets) @@ -290,8 +271,6 @@ unit-test: ## Runs unit test without e2e unit-test-attestor: ## Runs unit tests on scorecard-attestor cd attestor; SKIP_GINKGO=1 go test -covermode=atomic -coverprofile=unit-coverage.out `go list ./...`; cd ..; -$(GINKGO): install - check-env: ifndef GITHUB_AUTH_TOKEN $(error GITHUB_AUTH_TOKEN is undefined) @@ -307,3 +286,99 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO) # Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this. TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./... ############################################################################### + +##@ TODO(#744) +################################## make ko-images ############################# +ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko +.PHONY: ko-images $(ko-targets) +ko-images: $(ko-targets) + +KOCACHE_PATH=/tmp/ko + +$(KOCACHE_PATH): + mkdir -p $(KOCACHE_PATH) + +scorecard-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/${IMAGE_NAME} + LDFLAGS="$(LDFLAGS)" \ + KO_CACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --sbom=none \ + --platform=$(PLATFORM) \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4 + +cron-controller-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-controller \ + LDFLAGS="$(LDFLAGS)" \ + KOCACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --push=false \ + --sbom=none \ + --platform=$(PLATFORM) \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4/cron/internal/controller + +cron-worker-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker \ + LDFLAGS="$(LDFLAGS)" \ + KOCACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --push=false \ + --sbom=none \ + --platform=$(PLATFORM) \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4/cron/internal/worker + +cron-cii-worker-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker \ + LDFLAGS="$(LDFLAGS)" \ + KOCACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --push=false \ + --sbom=none \ + --platform=$(PLATFORM)\ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4/cron/internal/cii + +cron-bq-transfer-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer \ + LDFLAGS="$(LDFLAGS)" \ + KOCACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --push=false \ + --sbom=none \ + --platform=$(PLATFORM) \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4/cron/internal/bq + +cron-webhook-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook \ + LDFLAGS="$(LDFLAGS)" \ + KOCACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --push=false \ + --sbom=none \ + --platform=$(PLATFORM) \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4/cron/internal/webhook + +cron-github-server-ko: | $(KO) $(KOCACHE_PATH) + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server \ + LDFLAGS="$(LDFLAGS)" \ + KOCACHE=$(KOCACHE_PATH) \ + $(KO) build -B \ + --push=false \ + --sbom=none \ + --platform=$(PLATFORM) \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) \ + github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server + +###############################################################################