mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
🌱 Delete unused project-update functionality. (#3269)
Signed-off-by: Spencer Schrock <sschrock@google.com>
This commit is contained in:
parent
8d865ff041
commit
7cc6482680
48
.github/workflows/main.yml
vendored
48
.github/workflows/main.yml
vendored
@ -696,54 +696,6 @@ jobs:
|
||||
command: |
|
||||
go env -w GOFLAGS=-mod=mod
|
||||
make build-validate-script
|
||||
build-update-script:
|
||||
name: build-update-script
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-proto
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1
|
||||
with:
|
||||
egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
|
||||
|
||||
- name: Install Protoc
|
||||
uses: arduino/setup-protoc@149f6c87b92550901b26acd1632e11c3662e381f # v1.3.0
|
||||
with:
|
||||
version: ${{ env.PROTOC_VERSION }}
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Cache builds
|
||||
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
|
||||
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/Library/Caches/go-build
|
||||
%LocalAppData%\go-build
|
||||
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-
|
||||
- name: Clone the code
|
||||
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
check-latest: true
|
||||
cache: true
|
||||
- name: build-validate-script
|
||||
uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
|
||||
with:
|
||||
max_attempts: 3
|
||||
retry_on: error
|
||||
timeout_minutes: 30
|
||||
command: |
|
||||
go env -w GOFLAGS=-mod=mod
|
||||
make build-update-script
|
||||
validate-docs:
|
||||
name: validate-docs
|
||||
runs-on: ubuntu-latest
|
||||
|
1
.gitignore
vendored
1
.gitignore
vendored
@ -8,7 +8,6 @@ clients/githubrepo/roundtripper/tokens/server/github-auth-server
|
||||
clients/githubrepo/roundtripper/tokens/server/github-auth-server.docker
|
||||
cron/internal/data/add/add
|
||||
cron/internal/data/validate/validate
|
||||
cron/internal/data/update/projects-update
|
||||
cron/internal/controller/controller
|
||||
cron/internal/controller/controller.docker
|
||||
cron/internal/worker/worker
|
||||
|
8
Makefile
8
Makefile
@ -120,7 +120,7 @@ tree-status: | all-targets-update-dependencies ## Verify tree is clean and all c
|
||||
## Build all cron-related targets
|
||||
build-cron: build-controller build-worker build-cii-worker \
|
||||
build-shuffler build-bq-transfer build-github-server \
|
||||
build-webhook build-add-script build-validate-script build-update-script
|
||||
build-webhook build-add-script build-validate-script
|
||||
|
||||
build-targets = generate-mocks generate-docs build-scorecard build-cron build-proto build-attestor
|
||||
.PHONY: build $(build-targets)
|
||||
@ -295,12 +295,6 @@ cron/internal/data/validate/validate: cron/internal/data/validate/*.go cron/data
|
||||
# Run go build on the validate script
|
||||
cd cron/internal/data/validate && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o validate
|
||||
|
||||
build-update-script: ## Runs go build on the update script
|
||||
build-update-script: cron/internal/data/update/projects-update
|
||||
cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/data/*.go
|
||||
# Run go build on the update script
|
||||
cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update
|
||||
|
||||
docker-targets = scorecard-docker cron-controller-docker cron-worker-docker cron-cii-worker-docker cron-bq-transfer-docker cron-webhook-docker cron-github-server-docker
|
||||
.PHONY: dockerbuild $(docker-targets)
|
||||
dockerbuild: $(docker-targets)
|
||||
|
@ -1,243 +0,0 @@
|
||||
// Copyright 2021 OpenSSF Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/google/go-github/v38/github"
|
||||
"golang.org/x/tools/go/vcs" //nolint:staticcheck // TODO(https://github.com/ossf/scorecard/issues/3262)
|
||||
|
||||
"github.com/ossf/scorecard/v4/clients/githubrepo"
|
||||
"github.com/ossf/scorecard/v4/cron/data"
|
||||
)
|
||||
|
||||
var (
|
||||
// TODO = move them outside the sourcecode.
|
||||
bazelRepos = []repositoryDepsURL{
|
||||
{
|
||||
Owner: "envoyproxy",
|
||||
Repo: "envoy",
|
||||
File: "bazel/repository_locations.bzl",
|
||||
},
|
||||
{
|
||||
Owner: "envoyproxy",
|
||||
Repo: "envoy",
|
||||
File: "api/bazel/repository_locations.bzl",
|
||||
},
|
||||
{
|
||||
Owner: "grpc",
|
||||
Repo: "grpc",
|
||||
File: "bazel/grpc_deps.bzl",
|
||||
},
|
||||
}
|
||||
// TODO = move them outside the sourcecode.
|
||||
gorepos = []repositoryDepsURL{
|
||||
{
|
||||
Owner: "ossf",
|
||||
Repo: "scorecard",
|
||||
},
|
||||
{
|
||||
Owner: "sigstore",
|
||||
Repo: "cosign",
|
||||
},
|
||||
{
|
||||
Owner: "kubernetes",
|
||||
Repo: "kubernetes",
|
||||
Vendor: true,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
type repositoryDepsURL struct {
|
||||
Owner, Repo, File string
|
||||
Vendor bool
|
||||
}
|
||||
|
||||
// Programmatically gets Envoy's dependencies and add to projects.
|
||||
// Re-using a checker type.
|
||||
func getBazelDeps(repo repositoryDepsURL) []data.RepoFormat {
|
||||
client := github.NewClient(nil)
|
||||
ctx := context.Background()
|
||||
depRepos := []data.RepoFormat{}
|
||||
fo, _, _, err := client.Repositories.GetContents(ctx, repo.Owner, repo.Repo, repo.File, nil)
|
||||
if err != nil {
|
||||
// If we can't get content, gracefully fail but alert.
|
||||
log.Panicf("Failed to get repository content %s", err)
|
||||
return depRepos
|
||||
}
|
||||
|
||||
fc, err := fo.GetContent()
|
||||
if err != nil {
|
||||
// If we can't get content, gracefully fail, but alert.
|
||||
log.Panicf("Failed to get repository content %s", err)
|
||||
return depRepos
|
||||
}
|
||||
|
||||
// Match all patterns of github.com/{}/{}.
|
||||
re := regexp.MustCompile(`github\\.com/[^\/]*/[^\/"]*`)
|
||||
|
||||
// TODO: Replace with a starlark interpreter that can be used for any project.
|
||||
for _, match := range re.FindAllString(fc, -1) {
|
||||
repo := data.RepoFormat{}
|
||||
repo.Repo = strings.TrimSuffix(match, ".git")
|
||||
if _, err := githubrepo.MakeGithubRepo(repo.Repo); err != nil {
|
||||
log.Panicf("error during repo.Set: %v", err)
|
||||
return depRepos
|
||||
}
|
||||
depRepos = append(depRepos, repo)
|
||||
}
|
||||
return depRepos
|
||||
}
|
||||
|
||||
// GetGoDeps returns go repo dependencies.
|
||||
func getGoDeps(repo repositoryDepsURL) []data.RepoFormat {
|
||||
repoURLs := []data.RepoFormat{}
|
||||
pwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Default().Println(err)
|
||||
return nil
|
||||
}
|
||||
//nolint
|
||||
defer os.Chdir(pwd)
|
||||
// creating temp dir for git clone
|
||||
gitDir, err := os.MkdirTemp(pwd, "")
|
||||
if err != nil {
|
||||
log.Default().Println("Cannot create temporary dir", err)
|
||||
return nil
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
|
||||
// cloning git repo to get `go list -m all` out for getting all the dependencies
|
||||
_, err = git.PlainClone(gitDir, false,
|
||||
&git.CloneOptions{URL: fmt.Sprintf("http://github.com/%s/%s", repo.Owner, repo.Repo)})
|
||||
if err != nil {
|
||||
log.Default().Println(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.Chdir(gitDir); err != nil {
|
||||
log.Default().Println(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
var cmd *exec.Cmd
|
||||
if repo.Vendor {
|
||||
cmd = exec.Command("go", "list", "-e", "mod=vendor", "all")
|
||||
} else {
|
||||
cmd = exec.Command("go", "list", "-m", "all")
|
||||
}
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
log.Default().Println(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
example output of go list -m all
|
||||
gopkg.in/resty.v1 v1.12.0
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
|
||||
*/
|
||||
for _, l := range strings.Split(out.String(), "\n") {
|
||||
dependency := strings.Split(l, " ")[0]
|
||||
if strings.HasPrefix(dependency, "github.com") {
|
||||
repoURLs = parseGoModURL(dependency, repoURLs)
|
||||
} else {
|
||||
dependency = getVanityRepoURL(dependency)
|
||||
repoURLs = parseGoModURL(dependency, repoURLs)
|
||||
}
|
||||
}
|
||||
return repoURLs
|
||||
}
|
||||
|
||||
// getVanityRepoURL returns actual git repository for the go vanity URL
|
||||
// https://github.com/GoogleCloudPlatform/govanityurls.
|
||||
func getVanityRepoURL(u string) string {
|
||||
repo, err := vcs.RepoRootForImportDynamic(u, false)
|
||||
if err != nil {
|
||||
log.Default().Println("unable to parse the vanity URL", u, err)
|
||||
return ""
|
||||
}
|
||||
return repo.Repo
|
||||
}
|
||||
|
||||
func parseGoModURL(dependency string, repoURLs []data.RepoFormat) []data.RepoFormat {
|
||||
repoURL := data.RepoFormat{}
|
||||
splitURL := strings.Split(dependency, "/")
|
||||
//nolint:gomnd
|
||||
if len(splitURL) < 3 {
|
||||
return repoURLs
|
||||
}
|
||||
u := fmt.Sprintf("%s/%s/%s", splitURL[0], splitURL[1], splitURL[2])
|
||||
if _, err := githubrepo.MakeGithubRepo(u); err != nil {
|
||||
return repoURLs
|
||||
}
|
||||
repoURL.Repo = u
|
||||
repoURLs = append(repoURLs, repoURL)
|
||||
return repoURLs
|
||||
}
|
||||
|
||||
func getDependencies(in io.Reader) (oldRepos, newRepos []data.RepoFormat, e error) {
|
||||
iter, err := data.MakeIteratorFrom(in)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("error during data.MakeIterator: %w", err)
|
||||
}
|
||||
|
||||
// Read all project repositores into a map.
|
||||
m := make(map[string][]string)
|
||||
oldRepos = make([]data.RepoFormat, 0)
|
||||
for iter.HasNext() {
|
||||
repo, err := iter.Next()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("error during iter.Next: %w", err)
|
||||
}
|
||||
oldRepos = append(oldRepos, repo)
|
||||
// We do not handle duplicates.
|
||||
m[repo.Repo] = repo.Metadata
|
||||
}
|
||||
|
||||
// Create a list of project dependencies that are not already present.
|
||||
newRepos = []data.RepoFormat{}
|
||||
for _, repo := range bazelRepos {
|
||||
for _, item := range getBazelDeps(repo) {
|
||||
if _, ok := m[item.Repo]; !ok {
|
||||
// Also add to m to avoid dupes.
|
||||
m[item.Repo] = item.Metadata
|
||||
newRepos = append(newRepos, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, repo := range gorepos {
|
||||
for _, item := range getGoDeps(repo) {
|
||||
if _, ok := m[item.Repo]; !ok {
|
||||
// Also add to m to avoid dupes.
|
||||
m[item.Repo] = item.Metadata
|
||||
newRepos = append(newRepos, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return oldRepos, newRepos, nil
|
||||
}
|
@ -1,56 +0,0 @@
|
||||
// Copyright 2021 OpenSSF Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package main updates projects repositories with a projects dependencies.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
|
||||
"github.com/ossf/scorecard/v4/cron/data"
|
||||
)
|
||||
|
||||
// Adds "project=${PROJECT},dependency=true" to the repositories metadata.
|
||||
// Args:
|
||||
//
|
||||
// file path to old_projects.csv new_projects.csv
|
||||
func main() {
|
||||
if len(os.Args) != 3 {
|
||||
panic("must provide 2 arguments")
|
||||
}
|
||||
|
||||
inFile, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0o644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer inFile.Close()
|
||||
oldRepos, newRepos, err := getDependencies(inFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := data.SortAndAppendTo(&buf, oldRepos, newRepos); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
projects, err := os.OpenFile(os.Args[2], os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := projects.Write(buf.Bytes()); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
1
go.mod
1
go.mod
@ -51,7 +51,6 @@ require (
|
||||
github.com/mcuadros/go-jsonschema-generator v0.0.0-20200330054847-ba7a369d4303
|
||||
github.com/onsi/ginkgo/v2 v2.11.0
|
||||
github.com/otiai10/copy v1.12.0
|
||||
golang.org/x/tools/go/vcs v0.1.0-deprecated
|
||||
sigs.k8s.io/release-utils v0.6.0
|
||||
)
|
||||
|
||||
|
2
go.sum
2
go.sum
@ -2990,8 +2990,6 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
|
||||
golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
|
||||
golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
|
||||
golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=
|
||||
golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8=
|
||||
golang.org/x/vuln v0.0.0-20230303230808-d3042fecc4e3 h1:9GJsAwSzB/ztwMwsEm3ihUgCXHCULbNsubxqIrdKa44=
|
||||
golang.org/x/vuln v0.0.0-20230303230808-d3042fecc4e3/go.mod h1:LTLnfk/dpXDNKsX6aCg/cI4LyCVnTyrQhgV/yLJuly0=
|
||||
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
Loading…
Reference in New Issue
Block a user