From f997b2720d4a64ebccdd280ba533906aae315684 Mon Sep 17 00:00:00 2001 From: raghavkaul <8695110+raghavkaul@users.noreply.github.com> Date: Wed, 24 May 2023 14:43:36 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Gitlab:=20Add=20projects=20to=20cro?= =?UTF-8?q?n=20(#2936)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cron: add gitlab projects * support gitlab client * simplify gitlab detection Signed-off-by: Raghav Kaul * fix MakeGitlabRepo * shortcut when repo url is github.com * fixes add-projects, validate-projects Signed-off-by: Raghav Kaul * Move gitlab repos to release controller Signed-off-by: Raghav Kaul * Add csv headers Signed-off-by: Raghav Kaul * Use gitlab.WithBaseURL Signed-off-by: Raghav Kaul * formatting & logging Signed-off-by: Raghav Kaul * remove spurious test Signed-off-by: Raghav Kaul * consolidate logic Signed-off-by: Raghav Kaul * Turn on experimental flag Signed-off-by: Raghav Kaul * Add projects Signed-off-by: Raghav Kaul * Update client Signed-off-by: Raghav Kaul * update Signed-off-by: Raghav Kaul * update Signed-off-by: Raghav Kaul * update Signed-off-by: Raghav Kaul * update Signed-off-by: Raghav Kaul --------- Signed-off-by: Raghav Kaul --- checker/client.go | 27 +- clients/gitlabrepo/client.go | 21 +- clients/gitlabrepo/client_test.go | 2 +- clients/gitlabrepo/commits_test.go | 2 +- clients/gitlabrepo/contributors_test.go | 2 +- clients/gitlabrepo/repo.go | 9 +- clients/gitlabrepo/repo_test.go | 12 +- cron/config/config.yaml | 2 +- cron/config/config_test.go | 2 +- cron/data/iterator.go | 12 +- cron/data/iterator_test.go | 59 +++- cron/data/testdata/basic-gitlab-only.csv | 3 + cron/data/testdata/basic-with-gitlab.csv | 6 + cron/data/testdata/failing_urls.csv | 2 +- cron/data/writer_test.go | 10 + .../data/gitlab-projects-selected.csv | 253 ++++++++++++++++++ cron/internal/data/validate/main.go | 2 +- cron/internal/emulator/config.yaml | 2 +- cron/internal/worker/main.go | 57 +++- cron/k8s/controller.release.yaml | 6 +- cron/k8s/worker.release.yaml | 2 + e2e/ci_tests_test.go | 5 +- e2e/code_review_test.go | 5 +- e2e/license_test.go | 4 +- e2e/maintained_test.go | 2 +- e2e/security_policy_test.go | 4 +- 26 files changed, 441 insertions(+), 72 deletions(-) create mode 100644 cron/data/testdata/basic-gitlab-only.csv create mode 100644 cron/data/testdata/basic-with-gitlab.csv create mode 100644 cron/internal/data/gitlab-projects-selected.csv diff --git a/checker/client.go b/checker/client.go index b6931a7d..7a28b277 100644 --- a/checker/client.go +++ b/checker/client.go @@ -57,29 +57,14 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge _, experimental := os.LookupEnv("SCORECARD_EXPERIMENTAL") var repoClient clients.RepoClient - //nolint:nestif - if experimental && glrepo.DetectGitLab(repoURI) { + if experimental { repo, makeRepoError = glrepo.MakeGitlabRepo(repoURI) - if makeRepoError != nil { - return repo, - nil, - nil, - nil, - nil, - fmt.Errorf("getting local directory client: %w", makeRepoError) + if repo != nil && makeRepoError == nil { + repoClient, makeRepoError = glrepo.CreateGitlabClient(ctx, repo.Host()) } + } - var err error - repoClient, err = glrepo.CreateGitlabClientWithToken(ctx, os.Getenv("GITLAB_AUTH_TOKEN"), repo) - if err != nil { - return repo, - nil, - nil, - nil, - nil, - fmt.Errorf("error creating gitlab client: %w", err) - } - } else { + if makeRepoError != nil || repo == nil { repo, makeRepoError = ghrepo.MakeGithubRepo(repoURI) if makeRepoError != nil { return repo, @@ -87,7 +72,7 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge nil, nil, nil, - fmt.Errorf("getting local directory client: %w", makeRepoError) + fmt.Errorf("error making github repo: %w", makeRepoError) } repoClient = ghrepo.CreateGithubRepoClient(ctx, logger) } diff --git a/clients/gitlabrepo/client.go b/clients/gitlabrepo/client.go index 5b16a0fc..1254118f 100644 --- a/clients/gitlabrepo/client.go +++ b/clients/gitlabrepo/client.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "log" + "os" "time" "github.com/xanzy/go-gitlab" @@ -250,8 +251,13 @@ func (client *Client) Close() error { return nil } -func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients.Repo) (clients.RepoClient, error) { - client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.Host())) +func CreateGitlabClient(ctx context.Context, host string) (clients.RepoClient, error) { + token := os.Getenv("GITLAB_AUTH_TOKEN") + return CreateGitlabClientWithToken(ctx, token, host) +} + +func CreateGitlabClientWithToken(ctx context.Context, token, host string) (clients.RepoClient, error) { + client, err := gitlab.NewClient(token, gitlab.WithBaseURL(host)) if err != nil { return nil, fmt.Errorf("could not create gitlab client with error: %w", err) } @@ -308,14 +314,3 @@ func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients func CreateOssFuzzRepoClient(ctx context.Context, logger *log.Logger) (clients.RepoClient, error) { return nil, fmt.Errorf("%w, oss fuzz currently only supported for github repos", clients.ErrUnsupportedFeature) } - -// DetectGitLab: check whether the repoURI is a GitLab URI -// Makes HTTP request to GitLab API. -func DetectGitLab(repoURI string) bool { - var repo repoURL - if err := repo.parse(repoURI); err != nil { - return false - } - - return repo.IsValid() == nil -} diff --git a/clients/gitlabrepo/client_test.go b/clients/gitlabrepo/client_test.go index e4caa108..8a92d634 100644 --- a/clients/gitlabrepo/client_test.go +++ b/clients/gitlabrepo/client_test.go @@ -41,7 +41,7 @@ func Test_InitRepo(t *testing.T) { t.Error("couldn't make gitlab repo", err) } - client, err := CreateGitlabClientWithToken(context.Background(), "", repo) + client, err := CreateGitlabClient(context.Background(), repo.Host()) if err != nil { t.Error("couldn't make gitlab client", err) } diff --git a/clients/gitlabrepo/commits_test.go b/clients/gitlabrepo/commits_test.go index cb29f7c1..93c179fa 100644 --- a/clients/gitlabrepo/commits_test.go +++ b/clients/gitlabrepo/commits_test.go @@ -42,7 +42,7 @@ func Test_Setup(t *testing.T) { t.Error("couldn't make gitlab repo", err) } - client, err := CreateGitlabClientWithToken(context.Background(), "", repo) + client, err := CreateGitlabClient(context.Background(), repo.Host()) if err != nil { t.Error("couldn't make gitlab client", err) } diff --git a/clients/gitlabrepo/contributors_test.go b/clients/gitlabrepo/contributors_test.go index cbb1ced3..50eab3d7 100644 --- a/clients/gitlabrepo/contributors_test.go +++ b/clients/gitlabrepo/contributors_test.go @@ -48,7 +48,7 @@ func Test_ContributorsSetup(t *testing.T) { t.Error("couldn't make gitlab repo", err) } - client, err := CreateGitlabClientWithToken(context.Background(), "", repo) + client, err := CreateGitlabClientWithToken(context.Background(), "", repo.Host()) if err != nil { t.Error("couldn't make gitlab client", err) } diff --git a/clients/gitlabrepo/repo.go b/clients/gitlabrepo/repo.go index 24dcaba9..65a44783 100644 --- a/clients/gitlabrepo/repo.go +++ b/clients/gitlabrepo/repo.go @@ -17,6 +17,7 @@ package gitlabrepo import ( + "errors" "fmt" "net/url" "strings" @@ -38,6 +39,8 @@ type repoURL struct { metadata []string } +var errInvalidGitlabRepoURL = errors.New("repo is not a gitlab repo") + // Parses input string into repoURL struct /* * Accepted input string formats are as follows: @@ -98,6 +101,10 @@ func (r *repoURL) IsValid() error { return nil } + if strings.EqualFold(r.host, "github.com") { + return fmt.Errorf("%w: %s", errInvalidGitlabRepoURL, r.host) + } + client, err := gitlab.NewClient("", gitlab.WithBaseURL(fmt.Sprintf("%s://%s", r.scheme, r.host))) if err != nil { return sce.WithMessage(err, @@ -141,7 +148,7 @@ func MakeGitlabRepo(input string) (clients.Repo, error) { return nil, fmt.Errorf("error during parse: %w", err) } if err := repo.IsValid(); err != nil { - return nil, fmt.Errorf("error n IsValid: %w", err) + return nil, fmt.Errorf("error in IsValid: %w", err) } return &repo, nil } diff --git a/clients/gitlabrepo/repo_test.go b/clients/gitlabrepo/repo_test.go index 798be39d..87b36894 100644 --- a/clients/gitlabrepo/repo_test.go +++ b/clients/gitlabrepo/repo_test.go @@ -119,7 +119,7 @@ func TestRepoURL_IsValid(t *testing.T) { } } -func TestRepoURL_DetectGitlab(t *testing.T) { +func TestRepoURL_MakeGitLabRepo(t *testing.T) { tests := []struct { repouri string expected bool @@ -157,9 +157,13 @@ func TestRepoURL_DetectGitlab(t *testing.T) { if tt.flagRequired && os.Getenv("TEST_GITLAB_EXTERNAL") == "" { continue } - g := DetectGitLab(tt.repouri) - if g != tt.expected { - t.Errorf("got %s isgitlab: %t expected %t", tt.repouri, g, tt.expected) + g, err := MakeGitlabRepo(tt.repouri) + if (g != nil) != (err == nil) { + t.Errorf("got gitlabrepo: %s with err %s", g, err) + } + isGitlab := g != nil && err == nil + if isGitlab != tt.expected { + t.Errorf("got %s isgitlab: %t expected %t", tt.repouri, isGitlab, tt.expected) } } } diff --git a/cron/config/config.yaml b/cron/config/config.yaml index ed5453e1..9fd9cf8a 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -45,7 +45,7 @@ additional-params: # TODO(#859): Re-add Contributors after fixing inconsistencies. # TODO: Dependency-Update-Tool and SAST are search heavy # TODO: Vulnerabilities is slow on repos with lots of dependencies - blacklisted-checks: CI-Tests,Contributors,Dependency-Update-Tool + blacklisted-checks: CI-Tests,Contributors,Dependency-Update-Tool,Webhooks cii-data-bucket-url: gs://ossf-scorecard-cii-data # Raw results. raw-bigquery-table: scorecard-rawdata diff --git a/cron/config/config_test.go b/cron/config/config_test.go index 54afb660..536393a2 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -34,7 +34,7 @@ const ( prodCompletionThreshold = 0.99 prodWebhookURL = "" prodCIIDataBucket = "gs://ossf-scorecard-cii-data" - prodBlacklistedChecks = "CI-Tests,Contributors,Dependency-Update-Tool" + prodBlacklistedChecks = "CI-Tests,Contributors,Dependency-Update-Tool,Webhooks" prodShardSize int = 10 prodMetricExporter string = "stackdriver" prodMetricStackdriverPrefix string = "scorecard-cron" diff --git a/cron/data/iterator.go b/cron/data/iterator.go index 00d61ee3..66f0982c 100644 --- a/cron/data/iterator.go +++ b/cron/data/iterator.go @@ -24,6 +24,7 @@ import ( "github.com/jszwec/csvutil" "github.com/ossf/scorecard/v4/clients/githubrepo" + "github.com/ossf/scorecard/v4/clients/gitlabrepo" ) // Iterator interface is used to iterate through list of input repos for the cron job. @@ -83,9 +84,14 @@ func (reader *csvIterator) Next() (RepoFormat, error) { if reader.err != nil { return reader.next, fmt.Errorf("reader has error: %w", reader.err) } - // Sanity check valid GitHub URL. - if _, err := githubrepo.MakeGithubRepo(reader.next.Repo); err != nil { - return reader.next, fmt.Errorf("invalid GitHub URL: %w", err) + + repoURI := reader.next.Repo + + // validate gitlab or github url + if _, err := gitlabrepo.MakeGitlabRepo(repoURI); err != nil { + if _, err := githubrepo.MakeGithubRepo(repoURI); err != nil { + return reader.next, fmt.Errorf("invalid URL, neither github nor gitlab: %w", err) + } } return reader.next, nil } diff --git a/cron/data/iterator_test.go b/cron/data/iterator_test.go index 8091d615..69572b87 100644 --- a/cron/data/iterator_test.go +++ b/cron/data/iterator_test.go @@ -64,6 +64,63 @@ func TestCsvIterator(t *testing.T) { }, }, }, + { + name: "BasicGitlabOnly", + filename: "testdata/basic-gitlab-only.csv", + outcomes: []outcome{ + { + hasError: false, + repo: RepoFormat{ + Repo: "gitlab.com/owner1/repo1", + }, + }, + { + hasError: false, + repo: RepoFormat{ + Repo: "gitlab.com/owner3/path1/repo2", + Metadata: []string{"meta"}, + }, + }, + }, + }, + { + name: "BasicWithGitlab", + filename: "testdata/basic-with-gitlab.csv", + outcomes: []outcome{ + { + hasError: false, + repo: RepoFormat{ + Repo: "github.com/owner1/repo1", + }, + }, + { + hasError: false, + repo: RepoFormat{ + Repo: "github.com/owner2/repo2", + }, + }, + { + hasError: false, + repo: RepoFormat{ + Repo: "github.com/owner3/repo3", + Metadata: []string{"meta"}, + }, + }, + { + hasError: false, + repo: RepoFormat{ + Repo: "gitlab.com/owner1/repo1", + }, + }, + { + hasError: false, + repo: RepoFormat{ + Repo: "gitlab.com/owner3/path1/repo2", + Metadata: []string{"meta"}, + }, + }, + }, + }, { name: "Comment", filename: "testdata/comment.csv", @@ -95,7 +152,7 @@ func TestCsvIterator(t *testing.T) { outcomes: []outcome{ { hasError: true, - expectedErr: sce.ErrorUnsupportedHost, + expectedErr: sce.ErrorInvalidURL, }, { hasError: true, diff --git a/cron/data/testdata/basic-gitlab-only.csv b/cron/data/testdata/basic-gitlab-only.csv new file mode 100644 index 00000000..8d33822e --- /dev/null +++ b/cron/data/testdata/basic-gitlab-only.csv @@ -0,0 +1,3 @@ +repo,metadata +gitlab.com/owner1/repo1, +gitlab.com/owner3/path1/repo2,meta diff --git a/cron/data/testdata/basic-with-gitlab.csv b/cron/data/testdata/basic-with-gitlab.csv new file mode 100644 index 00000000..8a1c1d2b --- /dev/null +++ b/cron/data/testdata/basic-with-gitlab.csv @@ -0,0 +1,6 @@ +repo,metadata +github.com/owner1/repo1, +github.com/owner2/repo2, +github.com/owner3/repo3,meta +gitlab.com/owner1/repo1, +gitlab.com/owner3/path1/repo2,meta diff --git a/cron/data/testdata/failing_urls.csv b/cron/data/testdata/failing_urls.csv index f3c4a5d0..5ca11f25 100644 --- a/cron/data/testdata/failing_urls.csv +++ b/cron/data/testdata/failing_urls.csv @@ -1,4 +1,4 @@ repo,metadata -gitlab.com/owner1/repo1, +gitlab.com//repo1, github.com/owner2/, github.com//repo3,meta diff --git a/cron/data/writer_test.go b/cron/data/writer_test.go index f847d12e..e9ea51d2 100644 --- a/cron/data/writer_test.go +++ b/cron/data/writer_test.go @@ -34,16 +34,26 @@ func TestCsvWriter(t *testing.T) { Repo: "github.com/owner1/repo1", Metadata: []string{"meta1"}, }, + { + Repo: "gitlab.com/owner3/repo3", + Metadata: []string{"meta3"}, + }, }, newRepos: []RepoFormat{ { Repo: "github.com/owner2/repo2", Metadata: []string{"meta2"}, }, + { + Repo: "gitlab.com/owner4/repo4", + Metadata: []string{"meta4"}, + }, }, out: `repo,metadata github.com/owner1/repo1,meta1 github.com/owner2/repo2,meta2 +gitlab.com/owner3/repo3,meta3 +gitlab.com/owner4/repo4,meta4 `, }, } diff --git a/cron/internal/data/gitlab-projects-selected.csv b/cron/internal/data/gitlab-projects-selected.csv new file mode 100644 index 00000000..dd22ca97 --- /dev/null +++ b/cron/internal/data/gitlab-projects-selected.csv @@ -0,0 +1,253 @@ +repo,metadata +https://gitlab.com/gitlab-org/gitlab-foss, +https://gitlab.com/gitlab-org/gitlab, +https://gitlab.com/CalcProgrammer1/OpenRGB, +https://gitlab.com/gitlab-org/gitlab-runner, +https://gitlab.com/fdroid/fdroidclient, +https://gitlab.com/bramw/baserow, +https://gitlab.com/AuroraOSS/AuroraStore, +https://gitlab.com/graphviz/graphviz, +https://gitlab.com/pgjones/quart, +https://gitlab.com/libeigen/eigen, +https://gitlab.com/gitlab-org/gitlab-development-kit, +https://gitlab.com/gitlab-org/omnibus-gitlab, +https://gitlab.com/tezos/tezos, +https://gitlab.com/mayan-edms/mayan-edms, +https://gitlab.com/meltano/meltano, +https://gitlab.com/gitlab-com/runbooks, +https://gitlab.com/antora/antora, +https://gitlab.com/pycqa/flake8, +https://gitlab.com/meno/dropzone, +https://gitlab.com/pages/hugo, +https://gitlab.com/sequoia-pgp/sequoia, +https://gitlab.com/gableroux/unity3d-gitlab-ci-example, +https://gitlab.com/gitlab-org/gitaly, +https://gitlab.com/gitlab-org/cli, +https://gitlab.com/postgres-ai/database-lab, +https://gitlab.com/timvisee/ffsend, +https://gitlab.com/leanlabsio/kanban, +https://gitlab.com/pgjones/hypercorn, +https://gitlab.com/Rich-Harris/buble, +https://gitlab.com/cznic/sqlite, +https://gitlab.com/postgres-ai/postgres-checkup, +https://gitlab.com/mojo42/Jirafeau, +https://gitlab.com/eidheim/Simple-Web-Server, +https://gitlab.com/NebulousLabs/Sia, +https://gitlab.com/akihe/radamsa, +https://gitlab.com/procps-ng/procps, +https://gitlab.com/jam-systems/jam, +https://gitlab.com/catamphetamine/libphonenumber-js, +https://gitlab.com/olaris/olaris-server, +https://gitlab.com/stavros/harbormaster, +https://gitlab.com/conradsnicta/armadillo-code, +https://gitlab.com/gitlab-org/gitlab-shell, +https://gitlab.com/dalibo/postgresql_anonymizer, +https://gitlab.com/fatihacet/gitlab-vscode-extension, +https://gitlab.com/brinkervii/grapejuice, +https://gitlab.com/gitlab-org/gitlab-runner-docker-cleanup, +https://gitlab.com/gitlab-org/gitlab-ui, +https://gitlab.com/ajak/tuir, +https://gitlab.com/kornelski/babel-preset-php, +https://gitlab.com/mailman/hyperkitty, +https://gitlab.com/wg1/jpeg-xl, +https://gitlab.com/nsnam/ns-3-dev, +https://gitlab.com/axet/android-book-reader, +https://gitlab.com/shodan-public/nrich, +https://gitlab.com/bloom42/bloom, +https://gitlab.com/lfortran/lfortran, +https://gitlab.com/gitlab-org/gitlab-triage, +https://gitlab.com/esr/reposurgeon, +https://gitlab.com/leinardi/gkraken, +https://gitlab.com/QEF/q-e, +https://gitlab.com/eidheim/Simple-WebSocket-Server, +https://gitlab.com/signald/signald, +https://gitlab.com/chaica/feed2toot, +https://gitlab.com/gitlab-org/gitlab-pages, +https://gitlab.com/pulsechaincom/go-pulse, +https://gitlab.com/GoogleDriveIndex/Google-Drive-Index, +https://gitlab.com/antonok/enum_dispatch, +https://gitlab.com/gitlab-org/gitlab-workhorse, +https://gitlab.com/petsc/petsc, +https://gitlab.com/eternal-twin/etwin, +https://gitlab.com/mattbas/python-lottie, +https://gitlab.com/gitlab-org/docker-distribution-pruner, +https://gitlab.com/rosie-pattern-language/rosie, +https://gitlab.com/BuildStream/buildstream, +https://gitlab.com/kicad/libraries/kicad-footprints, +https://gitlab.com/dmfay/massive-js, +https://gitlab.com/nanuchi/go-full-course-youtube, +https://gitlab.com/sublime-music/sublime-music, +https://gitlab.com/gitlab-org/opstrace/opstrace, +https://gitlab.com/gitlab-org/release-cli, +https://gitlab.com/gitlab-org/ci-cd/docker-machine, +https://gitlab.com/catamphetamine/react-phone-number-input, +https://gitlab.com/IvanSanchez/Leaflet.GridLayer.GoogleMutant, +https://gitlab.com/klamonte/jexer, +https://gitlab.com/woob/woob, +https://gitlab.com/crates.rs/crates.rs, +https://gitlab.com/stavros/python-yeelight, +https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image, +https://gitlab.com/dbsystel/gitlab-ci-python-library, +https://gitlab.com/DerManu/QCustomPlot, +https://gitlab.com/juhani/go-semrel-gitlab, +https://gitlab.com/postgres-ai/joe, +https://gitlab.com/altek/accountant, +https://gitlab.com/formschema/native, +https://gitlab.com/gardenappl/readability-cli, +https://gitlab.com/doctormo/python-crontab, +https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent, +https://gitlab.com/mattbas/glaxnimate, +https://gitlab.com/mailman/postorius, +https://gitlab.com/cznic/ql, +https://gitlab.com/gitlab-org/release-tools, +https://gitlab.com/gitlab-org/gitlab-svgs, +https://gitlab.com/bzip2/bzip2, +https://gitlab.com/Molcas/OpenMolcas, +https://gitlab.com/anarcat/wallabako, +https://gitlab.com/gpsd/gpsd, +https://gitlab.com/xiliumhq/chromiumembedded/cefglue, +https://gitlab.com/weitzman/drupal-test-traits, +https://gitlab.com/DavidGriffith/frotz, +https://gitlab.com/sane-project/backends, +https://gitlab.com/palisade/palisade-release, +https://gitlab.com/thorchain/thornode, +https://gitlab.com/susurrus/serialport-rs, +https://gitlab.com/purelb/purelb, +https://gitlab.com/libtiff/libtiff, +https://gitlab.com/gilrs-project/gilrs, +https://gitlab.com/altom/altunity/altunitytester, +https://gitlab.com/tglman/persy, +https://gitlab.com/esr/loccount, +https://gitlab.com/WhyNotHugo/darkman, +https://gitlab.com/remram44/taguette, +https://gitlab.com/goobook/goobook, +https://gitlab.com/edneville/please, +https://gitlab.com/gitlab-org/cloud-native/gitlab-operator, +https://gitlab.com/hyper-expanse/open-source/semantic-delivery-gitlab, +https://gitlab.com/bitcoin-cash-node/bitcoin-cash-node, +https://gitlab.com/tspiteri/rug, +https://gitlab.com/libxc/libxc, +https://gitlab.com/amatos/rest-countries, +https://gitlab.com/m2crypto/m2crypto, +https://gitlab.com/ttyperacer/terminal-typeracer, +https://gitlab.com/glatteis/earthwalker, +https://gitlab.com/mattia.basaglia/python-lottie, +https://gitlab.com/john.carroll.p/rschedule, +https://gitlab.com/open-source-keir/financial-modelling/trading/barter-rs, +https://gitlab.com/portmod/portmod, +https://gitlab.com/librespacefoundation/polaris/polaris, +https://gitlab.com/allianceauth/allianceauth, +https://gitlab.com/gitlab-org/incubation-engineering/ai-assist/dokter, +https://gitlab.com/joneshf/purty, +https://gitlab.com/cerfacs/batman, +https://gitlab.com/lightmeter/controlcenter, +https://gitlab.com/autokent/pdf-parse, +https://gitlab.com/inkscape/extensions, +https://gitlab.com/vstconsulting/polemarch, +https://gitlab.com/stuko/ovito, +https://gitlab.com/php-ai/php-ml, +https://gitlab.com/cmocka/cmocka, +https://gitlab.com/kashell/Kawa, +https://gitlab.com/francoisjacquet/rosariosis, +https://gitlab.com/catamphetamine/read-excel-file, +https://gitlab.com/oer/emacs-reveal, +https://gitlab.com/xiayesuifeng/v2rayxplus, +https://gitlab.com/gitmate/open-source/IGitt, +https://gitlab.com/subnetzero/iridium, +https://gitlab.com/yorickpeterse/oga, +https://gitlab.com/mbryant/functiontrace, +https://gitlab.com/pyspread/pyspread, +https://gitlab.com/pavel.krupala/pyqt-node-editor, +https://gitlab.com/dslackw/colored, +https://gitlab.com/mikler/glaber, +https://gitlab.com/drutopia/drutopia, +https://gitlab.com/cznic/ccgo, +https://gitlab.com/broj42/nuxt-cookie-control, +https://gitlab.com/orobardet/gitlab-ci-linter, +https://gitlab.com/AdrianDC/gitlabci-local, +https://gitlab.com/virtio-fs/virtiofsd, +https://gitlab.com/ternaris/rosbags, +https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate, +https://gitlab.com/asuran-rs/asuran, +https://gitlab.com/librespacefoundation/satnogs/satnogs-network, +https://gitlab.com/maxlefou/hugo.386, +https://gitlab.com/mattia.basaglia/tgs, +https://gitlab.com/opennota/findimagedupes, +https://gitlab.com/html-validate/html-validate, +https://gitlab.com/oer/org-re-reveal, +https://gitlab.com/BrightOpen/Samotop, +https://gitlab.com/Friz64/erupt, +https://gitlab.com/nyx-space/nyx, +https://gitlab.com/msvechla/vaultbot, +https://gitlab.com/profclems/glab, +https://gitlab.com/pwoolcoc/soup, +https://gitlab.com/yawning/obfs4, +https://gitlab.com/gitlab-org/gitlab-exporter, +https://gitlab.com/gitlab-com/support/toolbox/fast-stats, +https://gitlab.com/lv2/lv2, +https://gitlab.com/remcohaszing/eslint-formatter-gitlab, +https://gitlab.com/eidheim/tiny-process-library, +https://gitlab.com/Linaro/tuxmake, +https://gitlab.com/sdurobotics/ur_rtde, +https://gitlab.com/pulsechaincom/pls-faucet, +https://gitlab.com/aa900031/nestjs-command, +https://gitlab.com/philbooth/bfj, +https://gitlab.com/stp-team/systemtestportal-webapp, +https://gitlab.com/cunity/gitlab-emulator, +https://gitlab.com/tractor-team/tractor, +https://gitlab.com/wrobell/remt, +https://gitlab.com/parrot_parrot/ms-teams-replace-background, +https://gitlab.com/p8n/panopticon, +https://gitlab.com/gitlab-org/terraform-provider-gitlab, +https://gitlab.com/dslackw/slpkg, +https://gitlab.com/gtk-kt/gtk-kt, +https://gitlab.com/ProjectWARP/warp-go, +https://gitlab.com/vmware/idem/idem, +https://gitlab.com/shackra/goimapnotify, +https://gitlab.com/Cwiiis/ferris, +https://gitlab.com/kicad/libraries/kicad-footprint-generator, +https://gitlab.com/broj42/nuxt-gmaps, +https://gitlab.com/lansharkconsulting/django/django-encrypted-model-fields, +https://gitlab.com/pycqa/flake8-docstrings, +https://gitlab.com/xmpp-rs/xmpp-rs, +https://gitlab.com/trantor/trantor, +https://gitlab.com/osnvr/os-nvr, +https://gitlab.com/etherlab.org/ethercat, +https://gitlab.com/inbitcoin/lighter, +https://gitlab.com/hoppr/hoppr, +https://gitlab.com/nomadic-labs/tezos, +https://gitlab.com/dee-see/graphql-path-enum, +https://gitlab.com/ilpianista/arch-audit, +https://gitlab.com/lely_industries/lely-core, +https://gitlab.com/okannen/static_init, +https://gitlab.com/TNThieding/exif, +https://gitlab.com/under-test/undertest, +https://gitlab.com/wholegrain/website-carbon-badges, +https://gitlab.com/broj42/nuxt-lazy-load, +https://gitlab.com/catamphetamine/country-flag-icons, +https://gitlab.com/golang-commonmark/markdown, +https://gitlab.com/williamyaoh/shrinkwraprs, +https://gitlab.com/torkleyy/err-derive, +https://gitlab.com/pavanello-research-group/dftpy, +https://gitlab.com/BVollmerhaus/blurwal, +https://gitlab.com/citrus-rs/citrus, +https://gitlab.com/gitlab-org/gl-openshift/gitlab-runner-operator, +https://gitlab.com/Oslandia/py3dtiles, +https://gitlab.com/Nulide/findmydeviceserver, +https://gitlab.com/clickable/clickable, +https://gitlab.com/microo8/plgo, +https://gitlab.com/cordite/cordite, +https://gitlab.com/shyft-os/shyft, +https://gitlab.com/antora/antora-lunr-extension, +https://gitlab.com/cerlane/SoftPosit, +https://gitlab.com/akita/mgpusim, +https://gitlab.com/lobaro/iot-dashboard, +https://gitlab.com/secml/secml, +https://gitlab.com/gitlab-org/container-registry, +https://gitlab.com/gitlab-org/labkit, +https://gitlab.com/isard/isardvdi, +https://gitlab.com/lmco/hoppr/hoppr, +https://gitlab.com/JacobLinCool/bahamut-automation, +https://gitlab.com/hoppr/hoppr-cop, +https://gitlab.com/hoppr/hoppr-cyclonedx-models, +https://gitlab.com/hoppr/droppr, diff --git a/cron/internal/data/validate/main.go b/cron/internal/data/validate/main.go index 96c10d56..82407f2a 100644 --- a/cron/internal/data/validate/main.go +++ b/cron/internal/data/validate/main.go @@ -24,7 +24,7 @@ import ( // Validates data.Iterator used by production PubSub cron job. // * Check for no duplicates in repoURLs. -// * Check repoURL is a valid GitHub URL. +// * Check repoURL is a valid GitHub/GitLab URL. func main() { if len(os.Args) != 2 { panic("must provide single argument") diff --git a/cron/internal/emulator/config.yaml b/cron/internal/emulator/config.yaml index 5569d6b4..9c12e725 100644 --- a/cron/internal/emulator/config.yaml +++ b/cron/internal/emulator/config.yaml @@ -40,7 +40,7 @@ additional-params: # TODO(#859): Re-add Contributors after fixing inconsistencies. # TODO: Dependency-Update-Tool and SAST are search heavy # TODO: Vulnerabilities is slow on repos with lots of dependencies - blacklisted-checks: CI-Tests,Contributors,Dependency-Update-Tool,SAST,Vulnerabilities + blacklisted-checks: CI-Tests,Contributors,Dependency-Update-Tool,SAST,Vulnerabilities,Webhooks cii-data-bucket-url: gs://ossf-scorecard-cii-data # Raw results. raw-bigquery-table: scorecard-rawdata diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go index 6ed479dc..39e4f044 100644 --- a/cron/internal/worker/main.go +++ b/cron/internal/worker/main.go @@ -30,6 +30,7 @@ import ( "github.com/ossf/scorecard/v4/clients" "github.com/ossf/scorecard/v4/clients/githubrepo" githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats" + "github.com/ossf/scorecard/v4/clients/gitlabrepo" "github.com/ossf/scorecard/v4/clients/ossfuzz" "github.com/ossf/scorecard/v4/cron/config" "github.com/ossf/scorecard/v4/cron/data" @@ -49,14 +50,41 @@ const ( rawResultsFile = "raw.json" ) -var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored") +var ( + ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored") + + // TODO, should probably be its own config/env var, as the checks we want to run + // per-platform will differ based on API cost/efficiency/implementation. + gitlabDisabledChecks = []string{ + "Binary-Artifacts", + "Branch-Protection", + "CII-Best-Practices", + "CI-Tests", + "Code-Review", + "Contributors", + "Dangerous-Workflow", + "Dependency-Update-Tool", + "Fuzzing", + "License", + "Maintained", + "Packaging", + "Pinned-Dependencies", + "SAST", + // "Security-Policy", + "Signed-Releases", + "Token-Permissions", + "Vulnerabilities", + "Webhooks", + } +) type ScorecardWorker struct { ctx context.Context logger *log.Logger checkDocs docs.Doc exporter monitoring.Exporter - repoClient clients.RepoClient + githubClient clients.RepoClient + gitlabClient clients.RepoClient ciiClient clients.CIIBestPracticesClient ossFuzzRepoClient clients.RepoClient vulnsClient clients.VulnerabilitiesClient @@ -91,7 +119,11 @@ func newScorecardWorker() (*ScorecardWorker, error) { sw.ctx = context.Background() sw.logger = log.NewLogger(log.InfoLevel) - sw.repoClient = githubrepo.CreateGithubRepoClient(sw.ctx, sw.logger) + sw.githubClient = githubrepo.CreateGithubRepoClient(sw.ctx, sw.logger) + // TODO(raghavkaul): Read GitLab auth token from environment + if sw.gitlabClient, err = gitlabrepo.CreateGitlabClient(sw.ctx, "https://gitlab.com"); err != nil { + return nil, fmt.Errorf("gitlabrepo.CreateGitlabClient: %w", err) + } sw.ciiClient = clients.BlobCIIBestPracticesClient(ciiDataBucketURL) if sw.ossFuzzRepoClient, err = ossfuzz.CreateOSSFuzzClientEager(ossfuzz.StatusURL); err != nil { return nil, fmt.Errorf("ossfuzz.CreateOSSFuzzClientEager: %w", err) @@ -119,7 +151,7 @@ func (sw *ScorecardWorker) Close() { func (sw *ScorecardWorker) Process(ctx context.Context, req *data.ScorecardBatchRequest, bucketURL string) error { return processRequest(ctx, req, sw.blacklistedChecks, bucketURL, sw.rawBucketURL, sw.apiBucketURL, - sw.checkDocs, sw.repoClient, sw.ossFuzzRepoClient, sw.ciiClient, sw.vulnsClient, sw.logger) + sw.checkDocs, sw.githubClient, sw.gitlabClient, sw.ossFuzzRepoClient, sw.ciiClient, sw.vulnsClient, sw.logger) } func (sw *ScorecardWorker) PostProcess() { @@ -131,7 +163,7 @@ func processRequest(ctx context.Context, batchRequest *data.ScorecardBatchRequest, blacklistedChecks []string, bucketURL, rawBucketURL, apiBucketURL string, checkDocs docs.Doc, - repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, + githubClient, gitlabClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, vulnsClient clients.VulnerabilitiesClient, logger *log.Logger, @@ -143,10 +175,16 @@ func processRequest(ctx context.Context, // TODO: run Scorecard for each repo in a separate thread. for _, repoReq := range batchRequest.GetRepos() { logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repoReq.Url)) - repo, err := githubrepo.MakeGithubRepo(*repoReq.Url) - if err != nil { + var repo clients.Repo + var err error + repoClient := githubClient + disabledChecks := blacklistedChecks + if repo, err = gitlabrepo.MakeGitlabRepo(*repoReq.Url); err == nil { // repo is a gitlab url + repoClient = gitlabClient + disabledChecks = gitlabDisabledChecks + } else if repo, err = githubrepo.MakeGithubRepo(*repoReq.Url); err != nil { // TODO(log): Previously Warn. Consider logging an error here. - logger.Info(fmt.Sprintf("invalid GitHub URL: %v", err)) + logger.Info(fmt.Sprintf("URL was neither valid GitLab nor GitHub: %v", err)) continue } repo.AppendMetadata(repoReq.Metadata...) @@ -161,7 +199,8 @@ func processRequest(ctx context.Context, if err != nil { return fmt.Errorf("error during policy.GetEnabled: %w", err) } - for _, check := range blacklistedChecks { + + for _, check := range disabledChecks { delete(checksToRun, check) } diff --git a/cron/k8s/controller.release.yaml b/cron/k8s/controller.release.yaml index 7cffad6b..c03984a2 100644 --- a/cron/k8s/controller.release.yaml +++ b/cron/k8s/controller.release.yaml @@ -52,7 +52,11 @@ spec: containers: - name: controller image: gcr.io/openssf/scorecard-batch-controller:latest - args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.release.csv"] + args: [ + "--config=/etc/scorecard/config.yaml", + "cron/internal/data/projects.release.csv", + "cron/internal/data/gitlab-projects-selected.csv" + ] imagePullPolicy: Always env: - name: GOMEMLIMIT diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index 50558847..d5497df9 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -54,6 +54,8 @@ spec: key: installation_id - name: "SCORECARD_API_RESULTS_BUCKET_URL" value: "gs://ossf-scorecard-cron-releasetest-results" + - name: "SCORECARD_EXPERIMENTAL" + value: "true" volumeMounts: - name: config-volume mountPath: /etc/scorecard diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index 945f0782..9730e4e4 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -16,7 +16,6 @@ package e2e import ( "context" - "os" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -109,7 +108,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { dl := scut.TestDetailLogger{} repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/gitlab-org/gitlab") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) // url to commit is https://gitlab.com/gitlab-org/gitlab/-/commit/8ae23fa220d73fa07501aabd94214c9e83fe61a0 err = repoClient.InitRepo(repo, "8ae23fa220d73fa07501aabd94214c9e83fe61a0", 0) @@ -138,7 +137,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { dl := scut.TestDetailLogger{} repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/fdroid/fdroidclient") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) // url to commit is https://gitlab.com/fdroid/fdroidclient/-/commit/a1d33881902cee33586a4fd4ee1538042a7bdedf err = repoClient.InitRepo(repo, "a1d33881902cee33586a4fd4ee1538042a7bdedf", 0) diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go index 69e5e174..871d75c4 100644 --- a/e2e/code_review_test.go +++ b/e2e/code_review_test.go @@ -16,7 +16,6 @@ package e2e import ( "context" - "os" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -157,7 +156,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { dl := scut.TestDetailLogger{} repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/fdroid/fdroidclient") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) err = repoClient.InitRepo(repo, "1f7ed43c120047102862d9d1d644f5b2de7a47f2", 0) Expect(err).Should(BeNil()) @@ -185,7 +184,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { dl := scut.TestDetailLogger{} repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/gitlab-org/gitlab") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) err = repoClient.InitRepo(repo, clients.HeadSHA, 0) // err = repoClient.InitRepo(repo, "0b5ba5049f3e5b8e945305acfa45c44d63df21b1", 0) diff --git a/e2e/license_test.go b/e2e/license_test.go index 0c4c7501..e4374968 100644 --- a/e2e/license_test.go +++ b/e2e/license_test.go @@ -125,7 +125,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { dl := scut.TestDetailLogger{} repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/N8BWert/scorecard-check-license-e2e") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) @@ -153,7 +153,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { dl := scut.TestDetailLogger{} repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/N8BWert/scorecard-check-license-e2e") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) err = repoClient.InitRepo(repo, "c3a8778e73ea95f937c228a34ee57d5e006f7304", 0) Expect(err).Should(BeNil()) diff --git a/e2e/maintained_test.go b/e2e/maintained_test.go index 7e7576ed..a20bcaf6 100644 --- a/e2e/maintained_test.go +++ b/e2e/maintained_test.go @@ -63,7 +63,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() { repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/gitlab-org/gitlab") Expect(err).Should(BeNil()) repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), - os.Getenv("GITLAB_AUTH_TOKEN"), repo) + os.Getenv("GITLAB_AUTH_TOKEN"), repo.Host()) Expect(err).Should(BeNil()) err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) diff --git a/e2e/security_policy_test.go b/e2e/security_policy_test.go index 7dba22ee..fc6a5ab0 100644 --- a/e2e/security_policy_test.go +++ b/e2e/security_policy_test.go @@ -180,7 +180,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { // project url is gitlab.com/bramw/baserow. repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/ossf-test/baserow") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) @@ -211,7 +211,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { // project url is gitlab.com/bramw/baserow. repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/ossf-test/baserow") Expect(err).Should(BeNil()) - repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + repoClient, err := gitlabrepo.CreateGitlabClient(context.Background(), repo.Host()) Expect(err).Should(BeNil()) // url to commit is https://gitlab.com/bramw/baserow/-/commit/28e6224b7d86f7b30bad6adb6b42f26a814c2f58 err = repoClient.InitRepo(repo, "28e6224b7d86f7b30bad6adb6b42f26a814c2f58", 0)