From a44dd6a758226307e84e2bdea20ee1bc83cda963 Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 1 Mar 2021 08:21:20 -0800 Subject: [PATCH] Add pypi and ruby gems package support. (#226) Adds some more package managers to https://github.com/ossf/scorecard/issues/33 Co-authored-by: Naveen <172697+naveensrinivasan@users.noreply.github.com> --- Makefile | 2 +- README.md | 6 +- cmd/root.go | 115 ++++++++++++++++++++++++++++++---- gen_ossfuzz_repos.sh | 27 -------- go.mod | 1 + go.sum | 3 +- {hack => scripts}/tree-status | 0 7 files changed, 111 insertions(+), 43 deletions(-) delete mode 100755 gen_ossfuzz_repos.sh rename {hack => scripts}/tree-status (100%) diff --git a/Makefile b/Makefile index 7f876726..5175da4e 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ verify-go-mod: ## Verify the go modules export GO111MODULE=on && \ go mod tidy && \ go mod verify - ./hack/tree-status + ./scripts/tree-status .PHONY: dockerbuild dockerbuild: ## Runs docker build diff --git a/README.md b/README.md index 41bbec03..fa98c203 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,11 @@ Signed-Tags: Fail 10 ### Package manager support -scorecard has an option to provide `--npm` package name and it would fetch the corresponding GitHub code. +scorecard has an option to provide either `--npm` / `--pypi` / `--rubygems` +package name and it would run the checks on the corresponding GitHub source +code. + +For example: ``` shell ./scorecard --npm=angular diff --git a/cmd/root.go b/cmd/root.go index d1f4c03e..b8685844 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -44,6 +44,8 @@ var ( logLevel = zap.LevelFlag("verbosity", zap.InfoLevel, "override the default log level") format string npm string + pypi string + rubygems string showDetails bool ) @@ -54,7 +56,8 @@ const ( ) var rootCmd = &cobra.Command{ - Use: "./scorecard --repo= [--checks=check1,...] [--show-details] or ./scorecard --npm= [--checks=check1,...] [--show-details]", + Use: `./scorecard --repo= [--checks=check1,...] [--show-details] +or ./scorecard --{npm,pypi,rubgems}= [--checks=check1,...] [--show-details]`, Short: "Security Scorecards", Long: "A program that shows security scorecard for an open source software.", Run: func(cmd *cobra.Command, args []string) { @@ -77,8 +80,24 @@ var rootCmd = &cobra.Command{ log.Fatalf("invalid format flag %s. allowed values are: [default, csv, json]", format) } - if len(npm) != 0 { - if git, err := fetchGitRepoistoryFromNPM(npm); err != nil { + if npm != "" { + if git, err := fetchGitRepositoryFromNPM(npm); err != nil { + log.Fatal(err) + } else { + if err := cmd.Flags().Set("repo", git); err != nil { + log.Fatal(err) + } + } + } else if pypi != "" { + if git, err := fetchGitRepositoryFromPYPI(pypi); err != nil { + log.Fatal(err) + } else { + if err := cmd.Flags().Set("repo", git); err != nil { + log.Fatal(err) + } + } + } else if rubygems != "" { + if git, err := fetchGitRepositoryFromRubyGems(rubygems); err != nil { log.Fatal(err) } else { if err := cmd.Flags().Set("repo", git); err != nil { @@ -108,7 +127,6 @@ var rootCmd = &cobra.Command{ for _, c := range enabledChecks { if format == formatDefault { fmt.Fprintf(os.Stderr, "Starting [%s]\n", c.Name) - } } ctx := context.Background() @@ -149,6 +167,18 @@ type npmSearchResults struct { } `json:"objects"` } +type pypiSearchResults struct { + Info struct { + ProjectUrls struct { + Source string `json:"Source"` + } `json:"project_urls"` + } `json:"info"` +} + +type rubyGemsSearchResults struct { + SourceCodeURI string `json:"source_code_uri"` +} + type record struct { Repo string Date string @@ -228,35 +258,94 @@ func displayResult(result bool) string { } } -// Gets the GitHub repository URL for the npm package -func fetchGitRepoistoryFromNPM(packageName string) (string, error) { - npmsearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1" +// Gets the GitHub repository URL for the npm package. +//nolint:noctx,goerr113 +func fetchGitRepositoryFromNPM(packageName string) (string, error) { + npmSearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1" const timeout = 10 client := &http.Client{ Timeout: timeout * time.Second, } - resp, err := client.Get(fmt.Sprintf(npmsearchURL, packageName)) - + resp, err := client.Get(fmt.Sprintf(npmSearchURL, packageName)) if err != nil { - return "", err + return "", fmt.Errorf("failed to get npm package json: %v", err) } defer resp.Body.Close() v := &npmSearchResults{} err = json.NewDecoder(resp.Body).Decode(v) if err != nil { - return "", err + return "", fmt.Errorf("failed to parse npm package json: %v", err) } if len(v.Objects) == 0 { - return "", fmt.Errorf("could not find search results for npm package %s", packageName) + return "", fmt.Errorf("could not find source repo for npm package: %s", packageName) } return v.Objects[0].Package.Links.Repository, nil } + +// Gets the GitHub repository URL for the pypi package. +//nolint:noctx,goerr113 +func fetchGitRepositoryFromPYPI(packageName string) (string, error) { + pypiSearchURL := "https://pypi.org/pypi/%s/json" + const timeout = 10 + client := &http.Client{ + Timeout: timeout * time.Second, + } + resp, err := client.Get(fmt.Sprintf(pypiSearchURL, packageName)) + if err != nil { + return "", fmt.Errorf("failed to get pypi package json: %v", err) + } + + defer resp.Body.Close() + v := &pypiSearchResults{} + err = json.NewDecoder(resp.Body).Decode(v) + if err != nil { + return "", fmt.Errorf("failed to parse pypi package json: %v", err) + } + if v.Info.ProjectUrls.Source == "" { + return "", fmt.Errorf("could not find source repo for pypi package: %s", packageName) + } + return v.Info.ProjectUrls.Source, nil +} + +// Gets the GitHub repository URL for the rubygems package. +//nolint:noctx,goerr113 +func fetchGitRepositoryFromRubyGems(packageName string) (string, error) { + rubyGemsSearchURL := "https://rubygems.org/api/v1/gems/%s.json" + const timeout = 10 + client := &http.Client{ + Timeout: timeout * time.Second, + } + resp, err := client.Get(fmt.Sprintf(rubyGemsSearchURL, packageName)) + if err != nil { + return "", fmt.Errorf("failed to get ruby gem json: %v", err) + } + + defer resp.Body.Close() + v := &rubyGemsSearchResults{} + err = json.NewDecoder(resp.Body).Decode(v) + if err != nil { + return "", fmt.Errorf("failed to parse ruby gem json: %v", err) + } + if v.SourceCodeURI == "" { + return "", fmt.Errorf("could not find source repo for ruby gem: %s", packageName) + } + return v.SourceCodeURI, nil +} + func init() { // Add the zap flag manually rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine) rootCmd.Flags().Var(&repo, "repo", "repository to check") - rootCmd.Flags().StringVar(&npm, "npm", "", "npm package to check. If the npm package has a GitHub repository") + rootCmd.Flags().StringVar( + &npm, "npm", "", + "npm package to check, given that the npm package has a GitHub repository") + rootCmd.Flags().StringVar( + &pypi, "pypi", "", + "pypi package to check, given that the pypi package has a GitHub repository") + rootCmd.Flags().StringVar( + &rubygems, "rubygems", "", + "rubygems package to check, given that the rubygems package has a GitHub repository") rootCmd.Flags().StringVar(&format, "format", formatDefault, "output format. allowed values are [default, csv, json]") rootCmd.Flags().StringSliceVar( &metaData, "metadata", []string{}, "metadata for the project.It can be multiple separated by commas") diff --git a/gen_ossfuzz_repos.sh b/gen_ossfuzz_repos.sh deleted file mode 100755 index 4391136d..00000000 --- a/gen_ossfuzz_repos.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright 2020 Security Scorecard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -tmp=$(mktemp -d) -trap "rm -rf $tmp" EXIT -git clone https://github.com/google/oss-fuzz --depth=1 $tmp -cat $tmp/projects/*/Dockerfile | grep "git clone" | grep -o "github.com/\S*" | sort | uniq > $tmp/repos.txt - -ossfuzz_file=checks/ossfuzz.go -echo "package checks" > $ossfuzz_file -echo "// GENERATED CODE, DO NOT EDIT" >> $ossfuzz_file -echo "var fuzzRepos=\`" >> $ossfuzz_file -cat $tmp/repos.txt >> $ossfuzz_file -echo "\`" >> $ossfuzz_file diff --git a/go.mod b/go.mod index 15feac3e..e399de58 100644 --- a/go.mod +++ b/go.mod @@ -17,5 +17,6 @@ require ( go.uber.org/zap v1.16.0 gocloud.dev v0.22.0 golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19 + golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c // indirect gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect ) diff --git a/go.sum b/go.sum index 1010f884..13c35417 100644 --- a/go.sum +++ b/go.sum @@ -641,8 +641,9 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c h1:dS09fXwOFF9cXBnIzZexIuUBj95U1NyQjkEhkgidDow= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/hack/tree-status b/scripts/tree-status similarity index 100% rename from hack/tree-status rename to scripts/tree-status