ossf/scorecard
1
1
Fork 0
mirror of https://github.com/ossf/scorecard.git synced 2024-08-16 11:50:37 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

🌱 Delete unused project-update functionality. (#3269)

Browse Source 
Signed-off-by: Spencer Schrock <sschrock@google.com>
This commit is contained in:
Spencer Schrock 2023-07-18 14:58:19 -07:00 committed by GitHub
parent 8d865ff041
commit 7cc6482680
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.github/workflows/main.yml vendored
View File

@ -696,54 +696,6 @@ jobs:
command: |
go env -w GOFLAGS=-mod=mod
make build-validate-script
build-update-script:
name: build-update-script
runs-on: ubuntu-latest
needs: build-proto
permissions:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1
with:
egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
- name: Install Protoc
uses: arduino/setup-protoc@149f6c87b92550901b26acd1632e11c3662e381f # v1.3.0
with:
version: ${{ env.PROTOC_VERSION }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Cache builds
# https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Clone the code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0
with:
go-version: ${{ env.GO_VERSION }}
check-latest: true
cache: true
- name: build-validate-script
uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
with:
max_attempts: 3
retry_on: error
timeout_minutes: 30
command: |
go env -w GOFLAGS=-mod=mod
make build-update-script
validate-docs:
name: validate-docs
runs-on: ubuntu-latest

1
.gitignore vendored
View File

@ -8,7 +8,6 @@ clients/githubrepo/roundtripper/tokens/server/github-auth-server
clients/githubrepo/roundtripper/tokens/server/github-auth-server.docker
cron/internal/data/add/add
cron/internal/data/validate/validate
cron/internal/data/update/projects-update
cron/internal/controller/controller
cron/internal/controller/controller.docker
cron/internal/worker/worker

8
Makefile
View File

@ -120,7 +120,7 @@ tree-status: | all-targets-update-dependencies ## Verify tree is clean and all c
## Build all cron-related targets
build-cron: build-controller build-worker build-cii-worker \
build-shuffler build-bq-transfer build-github-server \
build-webhook build-add-script build-validate-script build-update-script
build-webhook build-add-script build-validate-script
build-targets = generate-mocks generate-docs build-scorecard build-cron build-proto build-attestor
.PHONY: build $(build-targets)
@ -295,12 +295,6 @@ cron/internal/data/validate/validate: cron/internal/data/validate/*.go cron/data
# Run go build on the validate script
cd cron/internal/data/validate && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o validate
build-update-script: ## Runs go build on the update script
build-update-script: cron/internal/data/update/projects-update
cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/data/*.go
# Run go build on the update script
cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update
docker-targets = scorecard-docker cron-controller-docker cron-worker-docker cron-cii-worker-docker cron-bq-transfer-docker cron-webhook-docker cron-github-server-docker
.PHONY: dockerbuild $(docker-targets)
dockerbuild: $(docker-targets)

243
cron/internal/data/update/dependency.go
View File

@ -1,243 +0,0 @@
// Copyright 2021 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"regexp"
"strings"
"github.com/go-git/go-git/v5"
"github.com/google/go-github/v38/github"
"golang.org/x/tools/go/vcs" //nolint:staticcheck // TODO(https://github.com/ossf/scorecard/issues/3262)
"github.com/ossf/scorecard/v4/clients/githubrepo"
"github.com/ossf/scorecard/v4/cron/data"
)
var (
// TODO = move them outside the sourcecode.
bazelRepos = []repositoryDepsURL{
{
Owner: "envoyproxy",
Repo: "envoy",
File: "bazel/repository_locations.bzl",
},
{
Owner: "envoyproxy",
Repo: "envoy",
File: "api/bazel/repository_locations.bzl",
},
{
Owner: "grpc",
Repo: "grpc",
File: "bazel/grpc_deps.bzl",
},
}
// TODO = move them outside the sourcecode.
gorepos = []repositoryDepsURL{
{
Owner: "ossf",
Repo: "scorecard",
},
{
Owner: "sigstore",
Repo: "cosign",
},
{
Owner: "kubernetes",
Repo: "kubernetes",
Vendor: true,
},
}
)
type repositoryDepsURL struct {
Owner, Repo, File string
Vendor bool
}
// Programmatically gets Envoy's dependencies and add to projects.
// Re-using a checker type.
func getBazelDeps(repo repositoryDepsURL) []data.RepoFormat {
client := github.NewClient(nil)
ctx := context.Background()
depRepos := []data.RepoFormat{}
fo, _, _, err := client.Repositories.GetContents(ctx, repo.Owner, repo.Repo, repo.File, nil)
if err != nil {
// If we can't get content, gracefully fail but alert.
log.Panicf("Failed to get repository content %s", err)
return depRepos
}
fc, err := fo.GetContent()
if err != nil {
// If we can't get content, gracefully fail, but alert.
log.Panicf("Failed to get repository content %s", err)
return depRepos
}
// Match all patterns of github.com/{}/{}.
re := regexp.MustCompile(`github\\.com/[^\/]*/[^\/"]*`)
// TODO: Replace with a starlark interpreter that can be used for any project.
for _, match := range re.FindAllString(fc, -1) {
repo := data.RepoFormat{}
repo.Repo = strings.TrimSuffix(match, ".git")
if _, err := githubrepo.MakeGithubRepo(repo.Repo); err != nil {
log.Panicf("error during repo.Set: %v", err)
return depRepos
}
depRepos = append(depRepos, repo)
}
return depRepos
}
// GetGoDeps returns go repo dependencies.
func getGoDeps(repo repositoryDepsURL) []data.RepoFormat {
repoURLs := []data.RepoFormat{}
pwd, err := os.Getwd()
if err != nil {
log.Default().Println(err)
return nil
}
//nolint
defer os.Chdir(pwd)
// creating temp dir for git clone
gitDir, err := os.MkdirTemp(pwd, "")
if err != nil {
log.Default().Println("Cannot create temporary dir", err)
return nil
}
defer os.RemoveAll(gitDir)
// cloning git repo to get `go list -m all` out for getting all the dependencies
_, err = git.PlainClone(gitDir, false,
&git.CloneOptions{URL: fmt.Sprintf("http://github.com/%s/%s", repo.Owner, repo.Repo)})
if err != nil {
log.Default().Println(err)
return nil
}
if err := os.Chdir(gitDir); err != nil {
log.Default().Println(err)
return nil
}
var cmd *exec.Cmd
if repo.Vendor {
cmd = exec.Command("go", "list", "-e", "mod=vendor", "all")
} else {
cmd = exec.Command("go", "list", "-m", "all")
}
var out bytes.Buffer
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
log.Default().Println(err)
return nil
}
/*
example output of go list -m all
gopkg.in/resty.v1 v1.12.0
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
*/
for _, l := range strings.Split(out.String(), "\n") {
dependency := strings.Split(l, " ")[0]
if strings.HasPrefix(dependency, "github.com") {
repoURLs = parseGoModURL(dependency, repoURLs)
} else {
dependency = getVanityRepoURL(dependency)
repoURLs = parseGoModURL(dependency, repoURLs)
}
}
return repoURLs
}
// getVanityRepoURL returns actual git repository for the go vanity URL
// https://github.com/GoogleCloudPlatform/govanityurls.
func getVanityRepoURL(u string) string {
repo, err := vcs.RepoRootForImportDynamic(u, false)
if err != nil {
log.Default().Println("unable to parse the vanity URL", u, err)
return ""
}
return repo.Repo
}
func parseGoModURL(dependency string, repoURLs []data.RepoFormat) []data.RepoFormat {
repoURL := data.RepoFormat{}
splitURL := strings.Split(dependency, "/")
//nolint:gomnd
if len(splitURL) < 3 {
return repoURLs
}
u := fmt.Sprintf("%s/%s/%s", splitURL[0], splitURL[1], splitURL[2])
if _, err := githubrepo.MakeGithubRepo(u); err != nil {
return repoURLs
}
repoURL.Repo = u
repoURLs = append(repoURLs, repoURL)
return repoURLs
}
func getDependencies(in io.Reader) (oldRepos, newRepos []data.RepoFormat, e error) {
iter, err := data.MakeIteratorFrom(in)
if err != nil {
return nil, nil, fmt.Errorf("error during data.MakeIterator: %w", err)
}
// Read all project repositores into a map.
m := make(map[string][]string)
oldRepos = make([]data.RepoFormat, 0)
for iter.HasNext() {
repo, err := iter.Next()
if err != nil {
return nil, nil, fmt.Errorf("error during iter.Next: %w", err)
}
oldRepos = append(oldRepos, repo)
// We do not handle duplicates.
m[repo.Repo] = repo.Metadata
}
// Create a list of project dependencies that are not already present.
newRepos = []data.RepoFormat{}
for _, repo := range bazelRepos {
for _, item := range getBazelDeps(repo) {
if _, ok := m[item.Repo]; !ok {
// Also add to m to avoid dupes.
m[item.Repo] = item.Metadata
newRepos = append(newRepos, item)
}
}
}
for _, repo := range gorepos {
for _, item := range getGoDeps(repo) {
if _, ok := m[item.Repo]; !ok {
// Also add to m to avoid dupes.
m[item.Repo] = item.Metadata
newRepos = append(newRepos, item)
}
}
}
return oldRepos, newRepos, nil
}

56
cron/internal/data/update/main.go
View File

@ -1,56 +0,0 @@
// Copyright 2021 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package main updates projects repositories with a projects dependencies.
package main
import (
"bytes"
"os"
"github.com/ossf/scorecard/v4/cron/data"
)
// Adds "project=${PROJECT},dependency=true" to the repositories metadata.
// Args:
//
// file path to old_projects.csv new_projects.csv
func main() {
if len(os.Args) != 3 {
panic("must provide 2 arguments")
}
inFile, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0o644)
if err != nil {
panic(err)
}
defer inFile.Close()
oldRepos, newRepos, err := getDependencies(inFile)
if err != nil {
panic(err)
}
var buf bytes.Buffer
if err := data.SortAndAppendTo(&buf, oldRepos, newRepos); err != nil {
panic(err)
}
projects, err := os.OpenFile(os.Args[2], os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
panic(err)
}
if _, err := projects.Write(buf.Bytes()); err != nil {
panic(err)
}
}

1
go.mod
View File

@ -51,7 +51,6 @@ require (
github.com/mcuadros/go-jsonschema-generator v0.0.0-20200330054847-ba7a369d4303
github.com/onsi/ginkgo/v2 v2.11.0
github.com/otiai10/copy v1.12.0
golang.org/x/tools/go/vcs v0.1.0-deprecated
sigs.k8s.io/release-utils v0.6.0
)

2
go.sum
View File

@ -2990,8 +2990,6 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=
golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8=
golang.org/x/vuln v0.0.0-20230303230808-d3042fecc4e3 h1:9GJsAwSzB/ztwMwsEm3ihUgCXHCULbNsubxqIrdKa44=
golang.org/x/vuln v0.0.0-20230303230808-d3042fecc4e3/go.mod h1:LTLnfk/dpXDNKsX6aCg/cI4LyCVnTyrQhgV/yLJuly0=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=