Add pypi and ruby gems package support. (#226)

Adds some more package managers to
https://github.com/ossf/scorecard/issues/33

Co-authored-by: Naveen <172697+naveensrinivasan@users.noreply.github.com>

Makefile

@@ -100,7 +100,7 @@ verify-go-mod: ## Verify the go modules
 	export GO111MODULE=on && \
 		go mod tidy && \
 		go mod verify
-	./hack/tree-status 
+	./scripts/tree-status 
 
 .PHONY: dockerbuild
 dockerbuild: ## Runs docker build

README.md

@@ -116,7 +116,11 @@ Signed-Tags: Fail 10
 
 ### Package manager support
 
-scorecard has an option to provide `--npm` package name and it would fetch the corresponding GitHub code.
+scorecard has an option to provide either `--npm` / `--pypi` / `--rubygems`
+package name and it would run the checks on the corresponding GitHub source
+code.
+
+For example:
 
 ``` shell
 ./scorecard --npm=angular

cmd/root.go

@@ -44,6 +44,8 @@ var (
 	logLevel    = zap.LevelFlag("verbosity", zap.InfoLevel, "override the default log level")
 	format      string
 	npm         string
+	pypi        string
+	rubygems    string
 	showDetails bool
 )
 
@@ -54,7 +56,8 @@ const (
 )
 
 var rootCmd = &cobra.Command{
-	Use:   "./scorecard --repo=<repo_url> [--checks=check1,...] [--show-details] or ./scorecard --npm=<npm packagename> [--checks=check1,...] [--show-details]",
+	Use: `./scorecard --repo=<repo_url> [--checks=check1,...] [--show-details]
+or ./scorecard --{npm,pypi,rubgems}=<package_name> [--checks=check1,...] [--show-details]`,
 	Short: "Security Scorecards",
 	Long:  "A program that shows security scorecard for an open source software.",
 	Run: func(cmd *cobra.Command, args []string) {
@@ -77,8 +80,24 @@ var rootCmd = &cobra.Command{
 			log.Fatalf("invalid format flag %s. allowed values are: [default, csv, json]", format)
 		}
 
-		if len(npm) != 0 {
-			if git, err := fetchGitRepoistoryFromNPM(npm); err != nil {
+		if npm != "" {
+			if git, err := fetchGitRepositoryFromNPM(npm); err != nil {
+				log.Fatal(err)
+			} else {
+				if err := cmd.Flags().Set("repo", git); err != nil {
+					log.Fatal(err)
+				}
+			}
+		} else if pypi != "" {
+			if git, err := fetchGitRepositoryFromPYPI(pypi); err != nil {
+				log.Fatal(err)
+			} else {
+				if err := cmd.Flags().Set("repo", git); err != nil {
+					log.Fatal(err)
+				}
+			}
+		} else if rubygems != "" {
+			if git, err := fetchGitRepositoryFromRubyGems(rubygems); err != nil {
 				log.Fatal(err)
 			} else {
 				if err := cmd.Flags().Set("repo", git); err != nil {
@@ -108,7 +127,6 @@ var rootCmd = &cobra.Command{
 		for _, c := range enabledChecks {
 			if format == formatDefault {
 				fmt.Fprintf(os.Stderr, "Starting [%s]\n", c.Name)
-
 			}
 		}
 		ctx := context.Background()
@@ -149,6 +167,18 @@ type npmSearchResults struct {
 	} `json:"objects"`
 }
 
+type pypiSearchResults struct {
+	Info struct {
+		ProjectUrls struct {
+			Source string `json:"Source"`
+		} `json:"project_urls"`
+	} `json:"info"`
+}
+
+type rubyGemsSearchResults struct {
+	SourceCodeURI string `json:"source_code_uri"`
+}
+
 type record struct {
 	Repo     string
 	Date     string
@@ -228,35 +258,94 @@ func displayResult(result bool) string {
 	}
 }
 
-// Gets the GitHub repository URL for the npm package
-func fetchGitRepoistoryFromNPM(packageName string) (string, error) {
-	npmsearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1"
+// Gets the GitHub repository URL for the npm package.
+//nolint:noctx,goerr113
+func fetchGitRepositoryFromNPM(packageName string) (string, error) {
+	npmSearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1"
 	const timeout = 10
 	client := &http.Client{
 		Timeout: timeout * time.Second,
 	}
-	resp, err := client.Get(fmt.Sprintf(npmsearchURL, packageName))
-
+	resp, err := client.Get(fmt.Sprintf(npmSearchURL, packageName))
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to get npm package json: %v", err)
 	}
 
 	defer resp.Body.Close()
 	v := &npmSearchResults{}
 	err = json.NewDecoder(resp.Body).Decode(v)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to parse npm package json: %v", err)
 	}
 	if len(v.Objects) == 0 {
-		return "", fmt.Errorf("could not find search results for npm package %s", packageName)
+		return "", fmt.Errorf("could not find source repo for npm package: %s", packageName)
 	}
 	return v.Objects[0].Package.Links.Repository, nil
 }
+
+// Gets the GitHub repository URL for the pypi package.
+//nolint:noctx,goerr113
+func fetchGitRepositoryFromPYPI(packageName string) (string, error) {
+	pypiSearchURL := "https://pypi.org/pypi/%s/json"
+	const timeout = 10
+	client := &http.Client{
+		Timeout: timeout * time.Second,
+	}
+	resp, err := client.Get(fmt.Sprintf(pypiSearchURL, packageName))
+	if err != nil {
+		return "", fmt.Errorf("failed to get pypi package json: %v", err)
+	}
+
+	defer resp.Body.Close()
+	v := &pypiSearchResults{}
+	err = json.NewDecoder(resp.Body).Decode(v)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse pypi package json: %v", err)
+	}
+	if v.Info.ProjectUrls.Source == "" {
+		return "", fmt.Errorf("could not find source repo for pypi package: %s", packageName)
+	}
+	return v.Info.ProjectUrls.Source, nil
+}
+
+// Gets the GitHub repository URL for the rubygems package.
+//nolint:noctx,goerr113
+func fetchGitRepositoryFromRubyGems(packageName string) (string, error) {
+	rubyGemsSearchURL := "https://rubygems.org/api/v1/gems/%s.json"
+	const timeout = 10
+	client := &http.Client{
+		Timeout: timeout * time.Second,
+	}
+	resp, err := client.Get(fmt.Sprintf(rubyGemsSearchURL, packageName))
+	if err != nil {
+		return "", fmt.Errorf("failed to get ruby gem json: %v", err)
+	}
+
+	defer resp.Body.Close()
+	v := &rubyGemsSearchResults{}
+	err = json.NewDecoder(resp.Body).Decode(v)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse ruby gem json: %v", err)
+	}
+	if v.SourceCodeURI == "" {
+		return "", fmt.Errorf("could not find source repo for ruby gem: %s", packageName)
+	}
+	return v.SourceCodeURI, nil
+}
+
 func init() {
 	// Add the zap flag manually
 	rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)
 	rootCmd.Flags().Var(&repo, "repo", "repository to check")
-	rootCmd.Flags().StringVar(&npm, "npm", "", "npm package to check. If the npm package has a GitHub repository")
+	rootCmd.Flags().StringVar(
+		&npm, "npm", "",
+		"npm package to check, given that the npm package has a GitHub repository")
+	rootCmd.Flags().StringVar(
+		&pypi, "pypi", "",
+		"pypi package to check, given that the pypi package has a GitHub repository")
+	rootCmd.Flags().StringVar(
+		&rubygems, "rubygems", "",
+		"rubygems package to check, given that the rubygems package has a GitHub repository")
 	rootCmd.Flags().StringVar(&format, "format", formatDefault, "output format. allowed values are [default, csv, json]")
 	rootCmd.Flags().StringSliceVar(
 		&metaData, "metadata", []string{}, "metadata for the project.It can be multiple separated by commas")

gen_ossfuzz_repos.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-# Copyright 2020 Security Scorecard Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-tmp=$(mktemp -d)
-trap "rm -rf $tmp" EXIT
-git clone https://github.com/google/oss-fuzz --depth=1 $tmp
-cat $tmp/projects/*/Dockerfile | grep "git clone" | grep -o "github.com/\S*" | sort | uniq > $tmp/repos.txt
-
-ossfuzz_file=checks/ossfuzz.go
-echo "package checks" > $ossfuzz_file
-echo "// GENERATED CODE, DO NOT EDIT" >> $ossfuzz_file
-echo "var fuzzRepos=\`" >> $ossfuzz_file
-cat $tmp/repos.txt >> $ossfuzz_file
-echo "\`" >> $ossfuzz_file

go.mod

@@ -17,5 +17,6 @@ require (
 	go.uber.org/zap v1.16.0
 	gocloud.dev v0.22.0
 	golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19
+	golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c // indirect
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 )

go.sum

@@ -641,8 +641,9 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c h1:dS09fXwOFF9cXBnIzZexIuUBj95U1NyQjkEhkgidDow=
+golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=