mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
Refactor Makefile and add proto compile support. (#458)
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
Azeem Shaikh
GitHub
parent
f73f94bd0c
commit
ba3b5c5979
@ -23,7 +23,7 @@ COPY . ./
|
||||
FROM base AS build
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
RUN CGO_ENABLED=0 make build
|
||||
RUN CGO_ENABLED=0 make build-scorecard
|
||||
|
||||
FROM gcr.io/distroless/base:nonroot
|
||||
COPY --from=build /src/scorecard /
|
||||
|
||||
217
Makefile
217
Makefile
@ -1,10 +1,15 @@
|
||||
SHELL := /bin/bash
|
||||
GOBIN ?= $(GOPATH)/bin
|
||||
GINKGO ?= $(GOBIN)/ginkgo
|
||||
GINKGO := $(GOBIN)/ginkgo
|
||||
GOLANGGCI_LINT := $(GOBIN)/golangci-lint
|
||||
PROTOC_GEN_GO := $(GOBIN)/protoc-gen-go
|
||||
PROTOC := $(shell which protoc)
|
||||
IMAGE_NAME = scorecard
|
||||
OUTPUT = output
|
||||
FOCUS_DISK_TEST="E2E TEST:Disk Cache|E2E TEST:executable"
|
||||
IGNORED_CI_TEST="E2E TEST:blob|E2E TEST:Disk Cache|E2E TEST:executable"
|
||||
|
||||
############################### make help #####################################
|
||||
.PHONY: help
|
||||
help: ## Display this help
|
||||
@awk \
|
||||
@ -21,61 +26,125 @@ help: ## Display this help
|
||||
printf "\n%s%s%s\n", col, substr($$0, 5), nocol \
|
||||
} \
|
||||
' $(MAKEFILE_LIST)
|
||||
###############################################################################
|
||||
|
||||
all: ## Runs build, test and verify
|
||||
.PHONY: all
|
||||
all: build test check-projects build-cron build-scripts verify projects-update generate-docs
|
||||
################################ make install #################################
|
||||
.PHONY: install
|
||||
install: ## Installs all dependencies needed to compile Scorecard
|
||||
install: | $(GINKGO) $(GOLANGGCI_LINT) $(PROTOC_GEN_GO) $(PROTOC)
|
||||
|
||||
.PHONY: build
|
||||
build: ## Runs go build and generates executable
|
||||
CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"'
|
||||
|
||||
.PHONY: build-cron
|
||||
build-cron: ## Runs go build on the cronjob
|
||||
cd cron && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o scorecardcron
|
||||
|
||||
|
||||
.PHONY: build-scripts
|
||||
build-scripts: ## Runs go build on the scripts
|
||||
cd scripts && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o validate
|
||||
|
||||
.PHONY: test
|
||||
test: ## Runs unit test
|
||||
# ignoring e2e tests
|
||||
go test -covermode atomic `go list ./... | grep -v e2e`
|
||||
|
||||
golangci-lint:
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.40.0
|
||||
|
||||
lint: golangci-lint ## Runs golangci-lint linter
|
||||
golangci-lint run -n
|
||||
|
||||
check-env:
|
||||
ifndef GITHUB_AUTH_TOKEN
|
||||
$(error GITHUB_AUTH_TOKEN is undefined)
|
||||
endif
|
||||
|
||||
e2e: ## Runs e2e tests
|
||||
.PHONY: e2e
|
||||
# export GITHUB_AUTH_TOKEN with personal access token to run the e2e
|
||||
e2e: build check-env ginkgo
|
||||
$(GINKGO) --skip="E2E TEST:executable" -p -v -cover ./...
|
||||
|
||||
ginkgo:
|
||||
$(GINKGO):
|
||||
go get -u github.com/onsi/ginkgo/ginkgo@v1.16.2
|
||||
|
||||
unexport USE_DISK_CACHE
|
||||
unexport USE_BLOB_CACHE
|
||||
ci-e2e: ## Runs ci e2e tests
|
||||
.PHONY: ci-e2e
|
||||
# export GITHUB_AUTH_TOKEN with personal access token to run the e2e
|
||||
ci-e2e: build check-env e2e-cron
|
||||
$(GOLANGGCI_LINT):
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.40.0
|
||||
|
||||
$(PROTOC_GEN_GO):
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go
|
||||
|
||||
$(PROTOC):
|
||||
ifeq (,$(PROTOC))
|
||||
$(error download and install protobuf compiler package - https://developers.google.com/protocol-buffers/docs/downloads)
|
||||
endif
|
||||
###############################################################################
|
||||
|
||||
################################## make all ###################################
|
||||
all: ## Runs build, test and verify
|
||||
all-targets = update-dependencies build check-linter unit-test validate-projects tree-status
|
||||
.PHONY: all $(all-targets)
|
||||
all: $(all-targets)
|
||||
|
||||
update-dependencies: ## Update go dependencies for all modules
|
||||
# Update root go modules
|
||||
go mod tidy && go mod verify
|
||||
# Update ./scripts/ go modules
|
||||
cd scripts && go mod tidy && go mod verify
|
||||
# Update ./scripts/update go modules
|
||||
cd ./scripts/update && go mod tidy && go mod verify
|
||||
|
||||
check-linter: ## Install and run golang linter
|
||||
check-linter: | $(GOLANGGCI_LINT)
|
||||
# Run golangci-lint linter
|
||||
golangci-lint run -n
|
||||
|
||||
validate-projects: ## Validates ./cron/projects.txt
|
||||
validate-projects: build-scripts
|
||||
# Validate ./cron/projects.txt
|
||||
./scripts/validate ./cron/projects.txt
|
||||
|
||||
tree-status: ## Verify tree is clean and all changes are committed
|
||||
# Verify the tree is clean and all changes are commited
|
||||
./scripts/tree-status
|
||||
###############################################################################
|
||||
|
||||
############################### make build ################################
|
||||
build-targets = build-proto generate-docs build-scorecard build-cron build-scripts build-update dockerbuild
|
||||
.PHONY: build $(build-targets)
|
||||
build: ## Build all binaries and images in the reepo.
|
||||
build: $(build-targets)
|
||||
|
||||
build-proto: ## Compiles and generates all required protobufs
|
||||
build-proto: cron/data/request.pb.go
|
||||
cron/data/request.pb.go: cron/data/request.proto | $(PROTOC_GEN_GO) $(PROTOC)
|
||||
protoc --go_out=../../../ cron/data/request.proto
|
||||
|
||||
generate-docs: ## Generates docs
|
||||
generate-docs: checks/checks.md
|
||||
checks/checks.md: checks/checks.yaml checks/main/*.go
|
||||
# Generating checks.md
|
||||
cd ./checks/main && go run main.go
|
||||
|
||||
build-scorecard: ## Runs go build on repo
|
||||
# Run go build and generate scorecard executable
|
||||
CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"'
|
||||
|
||||
build-cron: ## Runs go build on the cron job
|
||||
# Run go build on the cronjob
|
||||
cd cron && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o scorecardcron
|
||||
|
||||
build-scripts: ## Runs go build on the scripts
|
||||
build-scripts: scripts/validate
|
||||
scripts/validate: scripts/*.go
|
||||
# Run go build on the scripts
|
||||
cd scripts && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o validate
|
||||
|
||||
build-update: ## Runs go build on scripts/update
|
||||
build-update: scripts/update/projects-update
|
||||
scripts/update/projects-update: scripts/update/*.go
|
||||
# Run go build on projects-update
|
||||
cd scripts/update && CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o projects-update
|
||||
|
||||
dockerbuild: ## Runs docker build
|
||||
# Build all Docker images in the Repo
|
||||
$(call ndef, GITHUB_AUTH_TOKEN)
|
||||
DOCKER_BUILDKIT=1 docker build . --file Dockerfile --tag $(IMAGE_NAME)
|
||||
DOCKER_BUILDKIT=1 docker build . --file cron/Dockerfile --tag $(IMAGE_NAME)cron
|
||||
###############################################################################
|
||||
|
||||
################################# make test ###################################
|
||||
test-targets = unit-test e2e ci-e2e test-disk-cache e2e-cron
|
||||
.PHONY: test $(test-targets)
|
||||
test: $(test-targets)
|
||||
|
||||
unit-test: ## Runs unit test without e2e
|
||||
# Run unit tests, ignoring e2e tests
|
||||
go test -covermode atomic `go list ./... | grep -v e2e`
|
||||
|
||||
e2e: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
|
||||
e2e: build-scorecard check-env | $(GINKGO)
|
||||
# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
|
||||
ginkgo --skip="E2E TEST:executable" -p -v -cover ./...
|
||||
|
||||
ci-e2e: ## Runs CI e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
|
||||
ci-e2e: build-scorecard check-env e2e-cron | $(GINKGO)
|
||||
# Run CI e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
|
||||
$(call ndef, GITHUB_AUTH_TOKEN)
|
||||
@echo Ignoring these test for ci-e2e $(IGNORED_CI_TEST)
|
||||
ginkgo -p -v -cover --skip=$(IGNORED_CI_TEST) ./e2e/...
|
||||
|
||||
.PHONY: test-disk-cache
|
||||
test-disk-cache: build ## Runs disk cache tests
|
||||
test-disk-cache: ## Runs disk cache tests
|
||||
test-disk-cache: build-scorecard | $(GINKGO)
|
||||
# Runs disk cache tests
|
||||
$(call ndef, GITHUB_AUTH_TOKEN)
|
||||
# Start with clean cache
|
||||
rm -rf $(OUTPUT)
|
||||
@ -95,50 +164,12 @@ test-disk-cache: build ## Runs disk cache tests
|
||||
--metadata=openssf --format json > ./$(OUTPUT)/results.json
|
||||
USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" ginkgo -p -v -cover --focus=$(FOCUS_DISK_TEST) ./e2e/...
|
||||
|
||||
e2e-cron: ## validates cron
|
||||
e2e-cron: ## Runs a e2e test cron job and validates its functionality
|
||||
# Validate cron
|
||||
GCS_BUCKET=ossf-scorecards-dev go run ./cron/main.go ./e2e/cron-projects.txt
|
||||
|
||||
|
||||
# Verification targets
|
||||
.PHONY: verify
|
||||
verify: verify-go-mod verify-go-mod-cron verify-go-mod-scripts lint ## Run all verification targets
|
||||
|
||||
.PHONY: verify-go-mod
|
||||
verify-go-mod: ## Verify the go modules
|
||||
go mod tidy && \
|
||||
go mod verify
|
||||
./scripts/tree-status
|
||||
|
||||
verify-go-mod-cron: ## Verify the go modules for cron
|
||||
cd cron && \
|
||||
go mod tidy && \
|
||||
go mod verify
|
||||
./scripts/tree-status
|
||||
|
||||
verify-go-mod-scripts: ## Verify the go modules for scripts
|
||||
cd scripts && \
|
||||
go mod tidy && \
|
||||
go mod verify
|
||||
./scripts/tree-status
|
||||
|
||||
.PHONY: dockerbuild
|
||||
dockerbuild: ## Runs docker build
|
||||
$(call ndef, GITHUB_AUTH_TOKEN)
|
||||
DOCKER_BUILDKIT=1 docker build . --file Dockerfile --tag $(IMAGE_NAME)
|
||||
DOCKER_BUILDKIT=1 docker build . --file cron/Dockerfile --tag $(IMAGE_NAME)cron
|
||||
|
||||
.PHONY: check-projects
|
||||
check-projects: ## Validates ./cron/projects.txt
|
||||
cd ./scripts && go build -o validate
|
||||
./scripts/validate ./cron/projects.txt
|
||||
|
||||
.PHONY: projects-update
|
||||
projects-update: ## builds the scripts/update binary
|
||||
cd ./scripts/update && go mod tidy && go mod verify
|
||||
cd ./scripts/update && CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o projects-update .
|
||||
./scripts/tree-status
|
||||
|
||||
.PHONY: generate-docs
|
||||
generate-docs: ## generate docs
|
||||
cd ./checks/main && go run main.go
|
||||
./scripts/tree-status
|
||||
check-env:
|
||||
ifndef GITHUB_AUTH_TOKEN
|
||||
$(error GITHUB_AUTH_TOKEN is undefined)
|
||||
endif
|
||||
###############################################################################
|
||||
|
||||
@ -24,7 +24,7 @@ COPY . ./
|
||||
FROM base AS build
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
RUN CGO_ENABLED=0 make build
|
||||
RUN CGO_ENABLED=0 make build-scorecard
|
||||
|
||||
FROM base AS cron
|
||||
ARG TARGETOS
|
||||
|
||||
@ -8,15 +8,4 @@ to install necessary binaries.
|
||||
|
||||
## Compile
|
||||
|
||||
Use the command below to compile:
|
||||
|
||||
```
|
||||
protoc --go_out=$DST_DIR request.proto
|
||||
```
|
||||
|
||||
NOTE: $DST_DIR should contain `github.com/ossf/scorecard/cron/data` directory
|
||||
structure.
|
||||
|
||||
## Future work
|
||||
|
||||
Update Makefile to compile and generate proto files, when we run `make all`.
|
||||
Run `make build-proto` to compile proto.
|
||||
|
||||
@ -14,14 +14,13 @@
|
||||
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.15.8
|
||||
// source: cron/data/request.proto
|
||||
|
||||
package data
|
||||
|
||||
import (
|
||||
proto "github.com/golang/protobuf/proto"
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
|
||||
@ -36,10 +35,6 @@ const (
|
||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||
)
|
||||
|
||||
// This is a compile-time assertion that a sufficiently up-to-date version
|
||||
// of the legacy proto package is being used.
|
||||
const _ = proto.ProtoPackageIsVersion4
|
||||
|
||||
type ScorecardBatchRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
||||
5
go.mod
5
go.mod
@ -5,7 +5,6 @@ go 1.16
|
||||
require (
|
||||
github.com/aws/aws-sdk-go v1.36.30 // indirect
|
||||
github.com/bradleyfalzon/ghinstallation v1.1.1
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/go-github/v32 v32.1.0
|
||||
github.com/jszwec/csvutil v1.5.0
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
@ -22,8 +21,8 @@ require (
|
||||
gocloud.dev v0.22.0
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19
|
||||
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 // indirect
|
||||
golang.org/x/tools v0.1.1-0.20210504170620-03ebc2c9fca8 // indirect
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect
|
||||
golang.org/x/tools v0.1.1 // indirect
|
||||
google.golang.org/protobuf v1.26.0
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
|
||||
12
go.sum
12
go.sum
@ -407,7 +407,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
@ -585,10 +585,10 @@ golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c=
|
||||
golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -660,8 +660,8 @@ golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4f
|
||||
golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.1-0.20210504170620-03ebc2c9fca8 h1:rTLms91GhM16y4sUcNGLdel0jJ8jXdQeXuN+7evgYiQ=
|
||||
golang.org/x/tools v0.1.1-0.20210504170620-03ebc2c9fca8/go.mod h1:sH/Eidr0EddymY8HZSakBo32zU3fG5ovDq874hJLjVg=
|
||||
golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user