diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9de4cb71..bfbb85e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -696,54 +696,6 @@ jobs: command: | go env -w GOFLAGS=-mod=mod make build-validate-script - build-update-script: - name: build-update-script - runs-on: ubuntu-latest - needs: build-proto - permissions: - contents: read - steps: - - name: Harden Runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1 - with: - egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - - name: Install Protoc - uses: arduino/setup-protoc@149f6c87b92550901b26acd1632e11c3662e381f # v1.3.0 - with: - version: ${{ env.PROTOC_VERSION }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache builds - # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Clone the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0 - with: - go-version: ${{ env.GO_VERSION }} - check-latest: true - cache: true - - name: build-validate-script - uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 30 - command: | - go env -w GOFLAGS=-mod=mod - make build-update-script validate-docs: name: validate-docs runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index d67e51d8..8c8cf9eb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ clients/githubrepo/roundtripper/tokens/server/github-auth-server clients/githubrepo/roundtripper/tokens/server/github-auth-server.docker cron/internal/data/add/add cron/internal/data/validate/validate -cron/internal/data/update/projects-update cron/internal/controller/controller cron/internal/controller/controller.docker cron/internal/worker/worker diff --git a/Makefile b/Makefile index 31a9d0c0..720cdcca 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ tree-status: | all-targets-update-dependencies ## Verify tree is clean and all c ## Build all cron-related targets build-cron: build-controller build-worker build-cii-worker \ build-shuffler build-bq-transfer build-github-server \ - build-webhook build-add-script build-validate-script build-update-script + build-webhook build-add-script build-validate-script build-targets = generate-mocks generate-docs build-scorecard build-cron build-proto build-attestor .PHONY: build $(build-targets) @@ -295,12 +295,6 @@ cron/internal/data/validate/validate: cron/internal/data/validate/*.go cron/data # Run go build on the validate script cd cron/internal/data/validate && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o validate -build-update-script: ## Runs go build on the update script -build-update-script: cron/internal/data/update/projects-update -cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/data/*.go - # Run go build on the update script - cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update - docker-targets = scorecard-docker cron-controller-docker cron-worker-docker cron-cii-worker-docker cron-bq-transfer-docker cron-webhook-docker cron-github-server-docker .PHONY: dockerbuild $(docker-targets) dockerbuild: $(docker-targets) diff --git a/cron/internal/data/update/dependency.go b/cron/internal/data/update/dependency.go deleted file mode 100644 index e0757aad..00000000 --- a/cron/internal/data/update/dependency.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2021 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "context" - "fmt" - "io" - "log" - "os" - "os/exec" - "regexp" - "strings" - - "github.com/go-git/go-git/v5" - "github.com/google/go-github/v38/github" - "golang.org/x/tools/go/vcs" //nolint:staticcheck // TODO(https://github.com/ossf/scorecard/issues/3262) - - "github.com/ossf/scorecard/v4/clients/githubrepo" - "github.com/ossf/scorecard/v4/cron/data" -) - -var ( - // TODO = move them outside the sourcecode. - bazelRepos = []repositoryDepsURL{ - { - Owner: "envoyproxy", - Repo: "envoy", - File: "bazel/repository_locations.bzl", - }, - { - Owner: "envoyproxy", - Repo: "envoy", - File: "api/bazel/repository_locations.bzl", - }, - { - Owner: "grpc", - Repo: "grpc", - File: "bazel/grpc_deps.bzl", - }, - } - // TODO = move them outside the sourcecode. - gorepos = []repositoryDepsURL{ - { - Owner: "ossf", - Repo: "scorecard", - }, - { - Owner: "sigstore", - Repo: "cosign", - }, - { - Owner: "kubernetes", - Repo: "kubernetes", - Vendor: true, - }, - } -) - -type repositoryDepsURL struct { - Owner, Repo, File string - Vendor bool -} - -// Programmatically gets Envoy's dependencies and add to projects. -// Re-using a checker type. -func getBazelDeps(repo repositoryDepsURL) []data.RepoFormat { - client := github.NewClient(nil) - ctx := context.Background() - depRepos := []data.RepoFormat{} - fo, _, _, err := client.Repositories.GetContents(ctx, repo.Owner, repo.Repo, repo.File, nil) - if err != nil { - // If we can't get content, gracefully fail but alert. - log.Panicf("Failed to get repository content %s", err) - return depRepos - } - - fc, err := fo.GetContent() - if err != nil { - // If we can't get content, gracefully fail, but alert. - log.Panicf("Failed to get repository content %s", err) - return depRepos - } - - // Match all patterns of github.com/{}/{}. - re := regexp.MustCompile(`github\\.com/[^\/]*/[^\/"]*`) - - // TODO: Replace with a starlark interpreter that can be used for any project. - for _, match := range re.FindAllString(fc, -1) { - repo := data.RepoFormat{} - repo.Repo = strings.TrimSuffix(match, ".git") - if _, err := githubrepo.MakeGithubRepo(repo.Repo); err != nil { - log.Panicf("error during repo.Set: %v", err) - return depRepos - } - depRepos = append(depRepos, repo) - } - return depRepos -} - -// GetGoDeps returns go repo dependencies. -func getGoDeps(repo repositoryDepsURL) []data.RepoFormat { - repoURLs := []data.RepoFormat{} - pwd, err := os.Getwd() - if err != nil { - log.Default().Println(err) - return nil - } - //nolint - defer os.Chdir(pwd) - // creating temp dir for git clone - gitDir, err := os.MkdirTemp(pwd, "") - if err != nil { - log.Default().Println("Cannot create temporary dir", err) - return nil - } - defer os.RemoveAll(gitDir) - - // cloning git repo to get `go list -m all` out for getting all the dependencies - _, err = git.PlainClone(gitDir, false, - &git.CloneOptions{URL: fmt.Sprintf("http://github.com/%s/%s", repo.Owner, repo.Repo)}) - if err != nil { - log.Default().Println(err) - return nil - } - - if err := os.Chdir(gitDir); err != nil { - log.Default().Println(err) - return nil - } - - var cmd *exec.Cmd - if repo.Vendor { - cmd = exec.Command("go", "list", "-e", "mod=vendor", "all") - } else { - cmd = exec.Command("go", "list", "-m", "all") - } - var out bytes.Buffer - cmd.Stdout = &out - err = cmd.Run() - if err != nil { - log.Default().Println(err) - return nil - } - - /* - example output of go list -m all - gopkg.in/resty.v1 v1.12.0 - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 - */ - for _, l := range strings.Split(out.String(), "\n") { - dependency := strings.Split(l, " ")[0] - if strings.HasPrefix(dependency, "github.com") { - repoURLs = parseGoModURL(dependency, repoURLs) - } else { - dependency = getVanityRepoURL(dependency) - repoURLs = parseGoModURL(dependency, repoURLs) - } - } - return repoURLs -} - -// getVanityRepoURL returns actual git repository for the go vanity URL -// https://github.com/GoogleCloudPlatform/govanityurls. -func getVanityRepoURL(u string) string { - repo, err := vcs.RepoRootForImportDynamic(u, false) - if err != nil { - log.Default().Println("unable to parse the vanity URL", u, err) - return "" - } - return repo.Repo -} - -func parseGoModURL(dependency string, repoURLs []data.RepoFormat) []data.RepoFormat { - repoURL := data.RepoFormat{} - splitURL := strings.Split(dependency, "/") - //nolint:gomnd - if len(splitURL) < 3 { - return repoURLs - } - u := fmt.Sprintf("%s/%s/%s", splitURL[0], splitURL[1], splitURL[2]) - if _, err := githubrepo.MakeGithubRepo(u); err != nil { - return repoURLs - } - repoURL.Repo = u - repoURLs = append(repoURLs, repoURL) - return repoURLs -} - -func getDependencies(in io.Reader) (oldRepos, newRepos []data.RepoFormat, e error) { - iter, err := data.MakeIteratorFrom(in) - if err != nil { - return nil, nil, fmt.Errorf("error during data.MakeIterator: %w", err) - } - - // Read all project repositores into a map. - m := make(map[string][]string) - oldRepos = make([]data.RepoFormat, 0) - for iter.HasNext() { - repo, err := iter.Next() - if err != nil { - return nil, nil, fmt.Errorf("error during iter.Next: %w", err) - } - oldRepos = append(oldRepos, repo) - // We do not handle duplicates. - m[repo.Repo] = repo.Metadata - } - - // Create a list of project dependencies that are not already present. - newRepos = []data.RepoFormat{} - for _, repo := range bazelRepos { - for _, item := range getBazelDeps(repo) { - if _, ok := m[item.Repo]; !ok { - // Also add to m to avoid dupes. - m[item.Repo] = item.Metadata - newRepos = append(newRepos, item) - } - } - } - for _, repo := range gorepos { - for _, item := range getGoDeps(repo) { - if _, ok := m[item.Repo]; !ok { - // Also add to m to avoid dupes. - m[item.Repo] = item.Metadata - newRepos = append(newRepos, item) - } - } - } - return oldRepos, newRepos, nil -} diff --git a/cron/internal/data/update/main.go b/cron/internal/data/update/main.go deleted file mode 100644 index e1d03b59..00000000 --- a/cron/internal/data/update/main.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package main updates projects repositories with a projects dependencies. -package main - -import ( - "bytes" - "os" - - "github.com/ossf/scorecard/v4/cron/data" -) - -// Adds "project=${PROJECT},dependency=true" to the repositories metadata. -// Args: -// -// file path to old_projects.csv new_projects.csv -func main() { - if len(os.Args) != 3 { - panic("must provide 2 arguments") - } - - inFile, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0o644) - if err != nil { - panic(err) - } - defer inFile.Close() - oldRepos, newRepos, err := getDependencies(inFile) - if err != nil { - panic(err) - } - - var buf bytes.Buffer - if err := data.SortAndAppendTo(&buf, oldRepos, newRepos); err != nil { - panic(err) - } - - projects, err := os.OpenFile(os.Args[2], os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - panic(err) - } - if _, err := projects.Write(buf.Bytes()); err != nil { - panic(err) - } -} diff --git a/go.mod b/go.mod index 6b2406fc..72391249 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( github.com/mcuadros/go-jsonschema-generator v0.0.0-20200330054847-ba7a369d4303 github.com/onsi/ginkgo/v2 v2.11.0 github.com/otiai10/copy v1.12.0 - golang.org/x/tools/go/vcs v0.1.0-deprecated sigs.k8s.io/release-utils v0.6.0 ) diff --git a/go.sum b/go.sum index 8010cb9a..82350d32 100644 --- a/go.sum +++ b/go.sum @@ -2990,8 +2990,6 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= -golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4= -golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8= golang.org/x/vuln v0.0.0-20230303230808-d3042fecc4e3 h1:9GJsAwSzB/ztwMwsEm3ihUgCXHCULbNsubxqIrdKa44= golang.org/x/vuln v0.0.0-20230303230808-d3042fecc4e3/go.mod h1:LTLnfk/dpXDNKsX6aCg/cI4LyCVnTyrQhgV/yLJuly0= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=