mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-17 11:57:12 +03:00
Add pypi and ruby gems package support. (#226)
Adds some more package managers to https://github.com/ossf/scorecard/issues/33 Co-authored-by: Naveen <172697+naveensrinivasan@users.noreply.github.com>
This commit is contained in:
parent
47eda487f5
commit
a44dd6a758
2
Makefile
2
Makefile
@ -100,7 +100,7 @@ verify-go-mod: ## Verify the go modules
|
|||||||
export GO111MODULE=on && \
|
export GO111MODULE=on && \
|
||||||
go mod tidy && \
|
go mod tidy && \
|
||||||
go mod verify
|
go mod verify
|
||||||
./hack/tree-status
|
./scripts/tree-status
|
||||||
|
|
||||||
.PHONY: dockerbuild
|
.PHONY: dockerbuild
|
||||||
dockerbuild: ## Runs docker build
|
dockerbuild: ## Runs docker build
|
||||||
|
@ -116,7 +116,11 @@ Signed-Tags: Fail 10
|
|||||||
|
|
||||||
### Package manager support
|
### Package manager support
|
||||||
|
|
||||||
scorecard has an option to provide `--npm` package name and it would fetch the corresponding GitHub code.
|
scorecard has an option to provide either `--npm` / `--pypi` / `--rubygems`
|
||||||
|
package name and it would run the checks on the corresponding GitHub source
|
||||||
|
code.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
``` shell
|
``` shell
|
||||||
./scorecard --npm=angular
|
./scorecard --npm=angular
|
||||||
|
115
cmd/root.go
115
cmd/root.go
@ -44,6 +44,8 @@ var (
|
|||||||
logLevel = zap.LevelFlag("verbosity", zap.InfoLevel, "override the default log level")
|
logLevel = zap.LevelFlag("verbosity", zap.InfoLevel, "override the default log level")
|
||||||
format string
|
format string
|
||||||
npm string
|
npm string
|
||||||
|
pypi string
|
||||||
|
rubygems string
|
||||||
showDetails bool
|
showDetails bool
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -54,7 +56,8 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var rootCmd = &cobra.Command{
|
var rootCmd = &cobra.Command{
|
||||||
Use: "./scorecard --repo=<repo_url> [--checks=check1,...] [--show-details] or ./scorecard --npm=<npm packagename> [--checks=check1,...] [--show-details]",
|
Use: `./scorecard --repo=<repo_url> [--checks=check1,...] [--show-details]
|
||||||
|
or ./scorecard --{npm,pypi,rubgems}=<package_name> [--checks=check1,...] [--show-details]`,
|
||||||
Short: "Security Scorecards",
|
Short: "Security Scorecards",
|
||||||
Long: "A program that shows security scorecard for an open source software.",
|
Long: "A program that shows security scorecard for an open source software.",
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
@ -77,8 +80,24 @@ var rootCmd = &cobra.Command{
|
|||||||
log.Fatalf("invalid format flag %s. allowed values are: [default, csv, json]", format)
|
log.Fatalf("invalid format flag %s. allowed values are: [default, csv, json]", format)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(npm) != 0 {
|
if npm != "" {
|
||||||
if git, err := fetchGitRepoistoryFromNPM(npm); err != nil {
|
if git, err := fetchGitRepositoryFromNPM(npm); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
} else {
|
||||||
|
if err := cmd.Flags().Set("repo", git); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if pypi != "" {
|
||||||
|
if git, err := fetchGitRepositoryFromPYPI(pypi); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
} else {
|
||||||
|
if err := cmd.Flags().Set("repo", git); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if rubygems != "" {
|
||||||
|
if git, err := fetchGitRepositoryFromRubyGems(rubygems); err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
} else {
|
} else {
|
||||||
if err := cmd.Flags().Set("repo", git); err != nil {
|
if err := cmd.Flags().Set("repo", git); err != nil {
|
||||||
@ -108,7 +127,6 @@ var rootCmd = &cobra.Command{
|
|||||||
for _, c := range enabledChecks {
|
for _, c := range enabledChecks {
|
||||||
if format == formatDefault {
|
if format == formatDefault {
|
||||||
fmt.Fprintf(os.Stderr, "Starting [%s]\n", c.Name)
|
fmt.Fprintf(os.Stderr, "Starting [%s]\n", c.Name)
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@ -149,6 +167,18 @@ type npmSearchResults struct {
|
|||||||
} `json:"objects"`
|
} `json:"objects"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type pypiSearchResults struct {
|
||||||
|
Info struct {
|
||||||
|
ProjectUrls struct {
|
||||||
|
Source string `json:"Source"`
|
||||||
|
} `json:"project_urls"`
|
||||||
|
} `json:"info"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rubyGemsSearchResults struct {
|
||||||
|
SourceCodeURI string `json:"source_code_uri"`
|
||||||
|
}
|
||||||
|
|
||||||
type record struct {
|
type record struct {
|
||||||
Repo string
|
Repo string
|
||||||
Date string
|
Date string
|
||||||
@ -228,35 +258,94 @@ func displayResult(result bool) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gets the GitHub repository URL for the npm package
|
// Gets the GitHub repository URL for the npm package.
|
||||||
func fetchGitRepoistoryFromNPM(packageName string) (string, error) {
|
//nolint:noctx,goerr113
|
||||||
npmsearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1"
|
func fetchGitRepositoryFromNPM(packageName string) (string, error) {
|
||||||
|
npmSearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1"
|
||||||
const timeout = 10
|
const timeout = 10
|
||||||
client := &http.Client{
|
client := &http.Client{
|
||||||
Timeout: timeout * time.Second,
|
Timeout: timeout * time.Second,
|
||||||
}
|
}
|
||||||
resp, err := client.Get(fmt.Sprintf(npmsearchURL, packageName))
|
resp, err := client.Get(fmt.Sprintf(npmSearchURL, packageName))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", fmt.Errorf("failed to get npm package json: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
v := &npmSearchResults{}
|
v := &npmSearchResults{}
|
||||||
err = json.NewDecoder(resp.Body).Decode(v)
|
err = json.NewDecoder(resp.Body).Decode(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", fmt.Errorf("failed to parse npm package json: %v", err)
|
||||||
}
|
}
|
||||||
if len(v.Objects) == 0 {
|
if len(v.Objects) == 0 {
|
||||||
return "", fmt.Errorf("could not find search results for npm package %s", packageName)
|
return "", fmt.Errorf("could not find source repo for npm package: %s", packageName)
|
||||||
}
|
}
|
||||||
return v.Objects[0].Package.Links.Repository, nil
|
return v.Objects[0].Package.Links.Repository, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Gets the GitHub repository URL for the pypi package.
|
||||||
|
//nolint:noctx,goerr113
|
||||||
|
func fetchGitRepositoryFromPYPI(packageName string) (string, error) {
|
||||||
|
pypiSearchURL := "https://pypi.org/pypi/%s/json"
|
||||||
|
const timeout = 10
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: timeout * time.Second,
|
||||||
|
}
|
||||||
|
resp, err := client.Get(fmt.Sprintf(pypiSearchURL, packageName))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to get pypi package json: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer resp.Body.Close()
|
||||||
|
v := &pypiSearchResults{}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(v)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to parse pypi package json: %v", err)
|
||||||
|
}
|
||||||
|
if v.Info.ProjectUrls.Source == "" {
|
||||||
|
return "", fmt.Errorf("could not find source repo for pypi package: %s", packageName)
|
||||||
|
}
|
||||||
|
return v.Info.ProjectUrls.Source, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gets the GitHub repository URL for the rubygems package.
|
||||||
|
//nolint:noctx,goerr113
|
||||||
|
func fetchGitRepositoryFromRubyGems(packageName string) (string, error) {
|
||||||
|
rubyGemsSearchURL := "https://rubygems.org/api/v1/gems/%s.json"
|
||||||
|
const timeout = 10
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: timeout * time.Second,
|
||||||
|
}
|
||||||
|
resp, err := client.Get(fmt.Sprintf(rubyGemsSearchURL, packageName))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to get ruby gem json: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer resp.Body.Close()
|
||||||
|
v := &rubyGemsSearchResults{}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(v)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to parse ruby gem json: %v", err)
|
||||||
|
}
|
||||||
|
if v.SourceCodeURI == "" {
|
||||||
|
return "", fmt.Errorf("could not find source repo for ruby gem: %s", packageName)
|
||||||
|
}
|
||||||
|
return v.SourceCodeURI, nil
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
// Add the zap flag manually
|
// Add the zap flag manually
|
||||||
rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)
|
rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)
|
||||||
rootCmd.Flags().Var(&repo, "repo", "repository to check")
|
rootCmd.Flags().Var(&repo, "repo", "repository to check")
|
||||||
rootCmd.Flags().StringVar(&npm, "npm", "", "npm package to check. If the npm package has a GitHub repository")
|
rootCmd.Flags().StringVar(
|
||||||
|
&npm, "npm", "",
|
||||||
|
"npm package to check, given that the npm package has a GitHub repository")
|
||||||
|
rootCmd.Flags().StringVar(
|
||||||
|
&pypi, "pypi", "",
|
||||||
|
"pypi package to check, given that the pypi package has a GitHub repository")
|
||||||
|
rootCmd.Flags().StringVar(
|
||||||
|
&rubygems, "rubygems", "",
|
||||||
|
"rubygems package to check, given that the rubygems package has a GitHub repository")
|
||||||
rootCmd.Flags().StringVar(&format, "format", formatDefault, "output format. allowed values are [default, csv, json]")
|
rootCmd.Flags().StringVar(&format, "format", formatDefault, "output format. allowed values are [default, csv, json]")
|
||||||
rootCmd.Flags().StringSliceVar(
|
rootCmd.Flags().StringSliceVar(
|
||||||
&metaData, "metadata", []string{}, "metadata for the project.It can be multiple separated by commas")
|
&metaData, "metadata", []string{}, "metadata for the project.It can be multiple separated by commas")
|
||||||
|
@ -1,27 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Copyright 2020 Security Scorecard Authors
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
tmp=$(mktemp -d)
|
|
||||||
trap "rm -rf $tmp" EXIT
|
|
||||||
git clone https://github.com/google/oss-fuzz --depth=1 $tmp
|
|
||||||
cat $tmp/projects/*/Dockerfile | grep "git clone" | grep -o "github.com/\S*" | sort | uniq > $tmp/repos.txt
|
|
||||||
|
|
||||||
ossfuzz_file=checks/ossfuzz.go
|
|
||||||
echo "package checks" > $ossfuzz_file
|
|
||||||
echo "// GENERATED CODE, DO NOT EDIT" >> $ossfuzz_file
|
|
||||||
echo "var fuzzRepos=\`" >> $ossfuzz_file
|
|
||||||
cat $tmp/repos.txt >> $ossfuzz_file
|
|
||||||
echo "\`" >> $ossfuzz_file
|
|
1
go.mod
1
go.mod
@ -17,5 +17,6 @@ require (
|
|||||||
go.uber.org/zap v1.16.0
|
go.uber.org/zap v1.16.0
|
||||||
gocloud.dev v0.22.0
|
gocloud.dev v0.22.0
|
||||||
golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19
|
golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19
|
||||||
|
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c // indirect
|
||||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
|
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
|
||||||
)
|
)
|
||||||
|
3
go.sum
3
go.sum
@ -641,8 +641,9 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
|
|||||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE=
|
|
||||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c h1:dS09fXwOFF9cXBnIzZexIuUBj95U1NyQjkEhkgidDow=
|
||||||
|
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
Loading…
Reference in New Issue
Block a user