mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
Modify tool installation (#2288)
Signed-off-by: Azeem Shaikh <azeemshaikh38@gmail.com> Signed-off-by: Azeem Shaikh <azeemshaikh38@gmail.com>
This commit is contained in:
parent
0f87094997
commit
08c2ee5f42
269
Makefile
269
Makefile
@ -1,22 +1,11 @@
|
||||
SHELL := /bin/bash
|
||||
GINKGO := ginkgo
|
||||
GIT_HASH := $(shell git rev-parse HEAD)
|
||||
GIT_VERSION ?= $(shell git describe --tags --always --dirty)
|
||||
SOURCE_DATE_EPOCH=$(shell git log --date=iso8601-strict -1 --pretty=%ct)
|
||||
GOLANGCI_LINT := golangci-lint
|
||||
PROTOC_GEN_GO := protoc-gen-go
|
||||
MOCKGEN := mockgen
|
||||
PROTOC := $(shell which protoc)
|
||||
GORELEASER := goreleaser
|
||||
IMAGE_NAME = scorecard
|
||||
OUTPUT = output
|
||||
PLATFORM="linux/amd64,linux/arm64,linux/386,linux/arm"
|
||||
LDFLAGS=$(shell ./scripts/version-ldflags)
|
||||
KOCACHE_PATH=/tmp/ko
|
||||
|
||||
define create_kocache_path
|
||||
mkdir -p $(KOCACHE_PATH)
|
||||
endef
|
||||
|
||||
|
||||
|
||||
@ -30,18 +19,66 @@ help: ## Display this help
|
||||
|
||||
###############################################################################
|
||||
|
||||
##@ Development
|
||||
##@ Tools
|
||||
################################ make install #################################
|
||||
.PHONY: install
|
||||
install: ## Installs all dependencies needed to compile Scorecard
|
||||
install: | $(PROTOC)
|
||||
@echo Installing tools from tools/tools.go
|
||||
cd tools; cat tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install %
|
||||
TOOLS_DIR := tools
|
||||
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)
|
||||
GOBIN := $(shell go env GOBIN)
|
||||
|
||||
# Golang binaries.
|
||||
|
||||
GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
|
||||
$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint
|
||||
|
||||
KO := $(TOOLS_BIN_DIR)/ko
|
||||
$(KO): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/google/ko
|
||||
|
||||
STUNNING_TRIBBLE := $(TOOLS_BIN_DIR)/stunning-tribble
|
||||
$(STUNNING_TRIBBLE): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/naveensrinivasan/stunning-tribble
|
||||
|
||||
MOCKGEN := $(TOOLS_BIN_DIR)/mockgen
|
||||
$(MOCKGEN): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/golang/mock/mockgen
|
||||
|
||||
GINKGO := $(TOOLS_BIN_DIR)/ginkgo
|
||||
$(GINKGO): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/onsi/ginkgo/v2/ginkgo
|
||||
|
||||
GORELEASER := $(TOOLS_BIN_DIR)/goreleaser
|
||||
$(GORELEASER): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/goreleaser/goreleaser
|
||||
|
||||
PROTOC_GEN_GO := $(TOOLS_BIN_DIR)/protoc-gen-go
|
||||
$(PROTOC_GEN_GO): $(TOOLS_DIR)/go.mod
|
||||
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install google.golang.org/protobuf/cmd/protoc-gen-go
|
||||
|
||||
# Non-Golang binaries.
|
||||
# TODO: Figure out how to install these binaries automatically.
|
||||
|
||||
PROTOC := $(shell which protoc)
|
||||
$(PROTOC):
|
||||
ifeq (,$(PROTOC))
|
||||
$(error download and install protobuf compiler package - https://developers.google.com/protocol-buffers/docs/downloads)
|
||||
endif
|
||||
|
||||
# Installs required binaries into $(TOOLS_BIN_DIR) wherever possible.
|
||||
# Keeping a local copy instead of a global install allows for:
|
||||
# i) Controlling the binary version Scorecard depends on leading to consistent
|
||||
# behavior across users.
|
||||
# ii) Avoids installing a whole bunch of otherwise unnecessary tools in the user's workspace.
|
||||
.PHONY: install
|
||||
install: ## Installs required binaries.
|
||||
install: $(GOLANGCI_LINT) \
|
||||
$(KO) \
|
||||
$(STUNNING_TRIBBLE) \
|
||||
$(PROTOC_GEN_GO) $(PROTOC) \
|
||||
$(MOCKGEN) \
|
||||
$(GINKGO) \
|
||||
$(GORELEASER)
|
||||
|
||||
###############################################################################
|
||||
|
||||
##@ Build
|
||||
@ -58,23 +95,22 @@ update-dependencies: ## Update go dependencies for all modules
|
||||
cd tools; go mod tidy && go mod verify; cd ../
|
||||
cd attestor; go mod tidy && go mod verify; cd ../
|
||||
|
||||
$(GOLANGCI_LINT): install
|
||||
check-linter: ## Install and run golang linter
|
||||
check-linter: $(GOLANGCI_LINT)
|
||||
check-linter: | $(GOLANGCI_LINT)
|
||||
# Run golangci-lint linter
|
||||
golangci-lint run -c .golangci.yml
|
||||
$(GOLANGCI_LINT) run -c .golangci.yml
|
||||
|
||||
check-osv: ## Checks osv.dev for any vulnerabilities
|
||||
check-osv: $(install)
|
||||
check-osv: | $(STUNNING_TRIBBLE)
|
||||
# Run stunning-tribble for checking the dependencies have any OSV
|
||||
go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
|
||||
| stunning-tribble
|
||||
| $(STUNNING_TRIBBLE)
|
||||
# Checking the tools which also has go.mod
|
||||
cd tools; go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
|
||||
| stunning-tribble ; cd ..
|
||||
| $(STUNNING_TRIBBLE) ; cd ..
|
||||
# Checking the attestor module for vulns
|
||||
cd attestor; go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
|
||||
| stunning-tribble ; cd ..
|
||||
| $(STUNNING_TRIBBLE) ; cd ..
|
||||
|
||||
add-projects: ## Adds new projects to ./cron/internal/data/projects.csv
|
||||
add-projects: ./cron/internal/data/projects.csv | build-add-script
|
||||
@ -99,35 +135,40 @@ build-cron: build-controller build-worker build-cii-worker \
|
||||
build-shuffler build-bq-transfer build-github-server \
|
||||
build-webhook build-add-script build-validate-script build-update-script
|
||||
|
||||
build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron
|
||||
build-targets = generate-mocks generate-docs build-scorecard build-cron build-proto
|
||||
.PHONY: build $(build-targets)
|
||||
build: ## Build all binaries and images in the repo.
|
||||
build: $(build-targets)
|
||||
|
||||
build-proto: ## Compiles and generates all required protobufs
|
||||
build-proto: cron/internal/data/request.pb.go cron/internal/data/metadata.pb.go
|
||||
cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) install
|
||||
protoc --go_out=../../../ cron/internal/data/request.proto
|
||||
cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) install
|
||||
protoc --go_out=../../../ cron/internal/data/metadata.proto
|
||||
cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) $(PROTOC_GEN_GO)
|
||||
$(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=../../../ cron/internal/data/request.proto
|
||||
cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) $(PROTOC_GEN_GO)
|
||||
$(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=../../../ cron/internal/data/metadata.proto
|
||||
|
||||
generate-mocks: ## Compiles and generates all mocks using mockgen.
|
||||
generate-mocks: clients/mockclients/repo_client.go clients/mockclients/repo.go clients/mockclients/cii_client.go checks/mockclients/vulnerabilities.go cmd/packagemanager_mockclient.go
|
||||
clients/mockclients/repo_client.go: clients/repo_client.go
|
||||
generate-mocks: clients/mockclients/repo_client.go \
|
||||
clients/mockclients/repo.go \
|
||||
clients/mockclients/cii_client.go \
|
||||
checks/mockclients/vulnerabilities.go \
|
||||
cmd/packagemanager_mockclient.go
|
||||
clients/mockclients/repo_client.go: clients/repo_client.go | $(MOCKGEN)
|
||||
# Generating MockRepoClient
|
||||
$(MOCKGEN) -source=clients/repo_client.go -destination=clients/mockclients/repo_client.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
|
||||
clients/mockclients/repo.go: clients/repo.go
|
||||
clients/mockclients/repo.go: clients/repo.go | $(MOCKGEN)
|
||||
# Generating MockRepo
|
||||
$(MOCKGEN) -source=clients/repo.go -destination=clients/mockclients/repo.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
|
||||
clients/mockclients/cii_client.go: clients/cii_client.go
|
||||
clients/mockclients/cii_client.go: clients/cii_client.go | $(MOCKGEN)
|
||||
# Generating MockCIIClient
|
||||
$(MOCKGEN) -source=clients/cii_client.go -destination=clients/mockclients/cii_client.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
|
||||
checks/mockclients/vulnerabilities.go: clients/vulnerabilities.go
|
||||
checks/mockclients/vulnerabilities.go: clients/vulnerabilities.go | $(MOCKGEN)
|
||||
# Generating MockCIIClient
|
||||
$(MOCKGEN) -source=clients/vulnerabilities.go -destination=clients/mockclients/vulnerabilities.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
|
||||
cmd/packagemanager_mockclient.go: cmd/packagemanager_client.go
|
||||
cmd/packagemanager_mockclient.go: cmd/packagemanager_client.go | $(MOCKGEN)
|
||||
# Generating MockPackageManagerClient
|
||||
$(MOCKGEN) -source=cmd/packagemanager_client.go -destination=cmd/packagemanager_mockclient.go -package=cmd -copyright_file=clients/mockclients/license.txt
|
||||
|
||||
generate-docs: ## Generates docs
|
||||
generate-docs: validate-docs docs/checks.md
|
||||
docs/checks.md: docs/checks/internal/checks.yaml docs/checks/internal/*.go docs/checks/internal/generate/*.go
|
||||
@ -196,66 +237,6 @@ cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/
|
||||
# Run go build on the update script
|
||||
cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update
|
||||
|
||||
ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko
|
||||
.PHONY: ko-build-everything $(ko-targets)
|
||||
ko-build-everything: $(ko-targets)
|
||||
|
||||
scorecard-ko:
|
||||
$(call create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) LDFLAGS="$(LDFLAGS)" \
|
||||
KO_CACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4
|
||||
cron-controller-ko:
|
||||
$(call_create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-controller LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/controller
|
||||
cron-worker-ko:
|
||||
$(call_create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/worker
|
||||
cron-cii-worker-ko:
|
||||
$(call_create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/cii
|
||||
cron-bq-transfer-ko:
|
||||
$(call_create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/bq
|
||||
cron-webhook-ko:
|
||||
$(call_create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/webhook
|
||||
cron-github-server-ko:
|
||||
$(call_create_kocache_path)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) ko build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server
|
||||
|
||||
docker-targets = scorecard-docker cron-controller-docker cron-worker-docker cron-cii-worker-docker cron-bq-transfer-docker cron-webhook-docker cron-github-server-docker
|
||||
.PHONY: dockerbuild $(docker-targets)
|
||||
dockerbuild: $(docker-targets)
|
||||
@ -290,8 +271,6 @@ unit-test: ## Runs unit test without e2e
|
||||
unit-test-attestor: ## Runs unit tests on scorecard-attestor
|
||||
cd attestor; SKIP_GINKGO=1 go test -covermode=atomic -coverprofile=unit-coverage.out `go list ./...`; cd ..;
|
||||
|
||||
$(GINKGO): install
|
||||
|
||||
check-env:
|
||||
ifndef GITHUB_AUTH_TOKEN
|
||||
$(error GITHUB_AUTH_TOKEN is undefined)
|
||||
@ -307,3 +286,99 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO)
|
||||
# Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this.
|
||||
TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
|
||||
###############################################################################
|
||||
|
||||
##@ TODO(#744)
|
||||
################################## make ko-images #############################
|
||||
ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko
|
||||
.PHONY: ko-images $(ko-targets)
|
||||
ko-images: $(ko-targets)
|
||||
|
||||
KOCACHE_PATH=/tmp/ko
|
||||
|
||||
$(KOCACHE_PATH):
|
||||
mkdir -p $(KOCACHE_PATH)
|
||||
|
||||
scorecard-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/${IMAGE_NAME}
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KO_CACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM) \
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4
|
||||
|
||||
cron-controller-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-controller \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM) \
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4/cron/internal/controller
|
||||
|
||||
cron-worker-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM) \
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4/cron/internal/worker
|
||||
|
||||
cron-cii-worker-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM)\
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4/cron/internal/cii
|
||||
|
||||
cron-bq-transfer-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM) \
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4/cron/internal/bq
|
||||
|
||||
cron-webhook-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM) \
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4/cron/internal/webhook
|
||||
|
||||
cron-github-server-ko: | $(KO) $(KOCACHE_PATH)
|
||||
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
|
||||
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
KOCACHE=$(KOCACHE_PATH) \
|
||||
$(KO) build -B \
|
||||
--push=false \
|
||||
--sbom=none \
|
||||
--platform=$(PLATFORM) \
|
||||
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
|
||||
github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server
|
||||
|
||||
###############################################################################
|
||||
|
Loading…
Reference in New Issue
Block a user