From e2715fd7f08bb4adb877b9312891f9526882fc1e Mon Sep 17 00:00:00 2001 From: raghavkaul <8695110+raghavkaul@users.noreply.github.com> Date: Thu, 16 Mar 2023 15:18:32 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20GitLab:=20Security=20Policy=20check?= =?UTF-8?q?=20(#2754)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add tarballHandler for GitLab, enabling repo download Signed-off-by: Raghav Kaul * Abstract OrgSecurityPolicy details to RepoClient instead of checker Signed-off-by: Raghav Kaul * Remove Org() from RepoClient Signed-off-by: Raghav Kaul * Rename Signed-off-by: Raghav Kaul * Don't run as part of CI tests that depend on external sites Signed-off-by: Raghav Kaul --------- Signed-off-by: Raghav Kaul --- Makefile | 2 +- checks/raw/security_policy.go | 21 +- checks/raw/security_policy_test.go | 2 - clients/githubrepo/client.go | 15 ++ clients/githubrepo/repo.go | 13 -- clients/gitlabrepo/client.go | 13 +- clients/gitlabrepo/repo.go | 12 -- clients/gitlabrepo/repo_test.go | 32 ++- clients/gitlabrepo/tarball.go | 264 +++++++++++++++++++++++ clients/gitlabrepo/tarball_test.go | 177 +++++++++++++++ clients/gitlabrepo/testdata/basic.tar.gz | Bin 0 -> 268 bytes clients/localdir/client.go | 4 + clients/localdir/repo.go | 5 - clients/mockclients/repo.go | 15 -- clients/mockclients/repo_client.go | 10 + clients/ossfuzz/client.go | 6 + clients/repo.go | 1 - clients/repo_client.go | 2 + e2e/e2e_suite_test.go | 3 + e2e/security_policy_test.go | 64 ++++++ 20 files changed, 586 insertions(+), 75 deletions(-) create mode 100644 clients/gitlabrepo/tarball.go create mode 100644 clients/gitlabrepo/tarball_test.go create mode 100644 clients/gitlabrepo/testdata/basic.tar.gz diff --git a/Makefile b/Makefile index b9d5e4c8..9f8eccc4 100644 --- a/Makefile +++ b/Makefile @@ -335,7 +335,7 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO) TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./... e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN - TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./... + TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./... e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./... diff --git a/checks/raw/security_policy.go b/checks/raw/security_policy.go index eba2e0f3..c4da3f29 100644 --- a/checks/raw/security_policy.go +++ b/checks/raw/security_policy.go @@ -25,10 +25,8 @@ import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/checks/fileparser" "github.com/ossf/scorecard/v4/clients" - "github.com/ossf/scorecard/v4/clients/githubrepo" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/log" ) type securityPolicyFilesWithURI struct { @@ -62,21 +60,18 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) // Check if present in parent org. // https#://docs.github.com/en/github/building-a-strong-community/creating-a-default-community-health-file. - // TODO(1491): Make this non-GitHub specific. - logger := log.NewLogger(log.InfoLevel) - // HAD TO HARD CODE TO 30 - dotGitHubClient := githubrepo.CreateGithubRepoClient(c.Ctx, logger) - err = dotGitHubClient.InitRepo(c.Repo.Org(), clients.HeadSHA, 0) + client, err := c.RepoClient.GetOrgRepoClient(c.Ctx) + switch { case err == nil: - defer dotGitHubClient.Close() - data.uri = dotGitHubClient.URI() - err = fileparser.OnAllFilesDo(dotGitHubClient, isSecurityPolicyFile, &data) + defer client.Close() + data.uri = client.URI() + err = fileparser.OnAllFilesDo(client, isSecurityPolicyFile, &data) if err != nil { - return checker.SecurityPolicyData{}, err + return checker.SecurityPolicyData{}, fmt.Errorf("unable to create github client: %w", err) } - case errors.Is(err, sce.ErrRepoUnreachable): + case errors.Is(err, sce.ErrRepoUnreachable), errors.Is(err, clients.ErrUnsupportedFeature): break default: return checker.SecurityPolicyData{}, err @@ -91,7 +86,7 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) if data.files[idx].File.Type == finding.FileTypeURL { filePattern = strings.Replace(filePattern, data.uri+"/", "", 1) } - err := fileparser.OnMatchingFileContentDo(dotGitHubClient, fileparser.PathMatcher{ + err := fileparser.OnMatchingFileContentDo(client, fileparser.PathMatcher{ Pattern: filePattern, CaseSensitive: false, }, checkSecurityPolicyFileContent, &data.files[idx].File, &data.files[idx].Information) diff --git a/checks/raw/security_policy_test.go b/checks/raw/security_policy_test.go index a934ef32..51fefd34 100644 --- a/checks/raw/security_policy_test.go +++ b/checks/raw/security_policy_test.go @@ -138,8 +138,6 @@ func TestSecurityPolicy(t *testing.T) { mockRepo := mockrepo.NewMockRepo(ctrl) mockRepoClient.EXPECT().ListFiles(gomock.Any()).Return(tt.files, nil).AnyTimes() - mockRepo.EXPECT().Org().Return(nil).AnyTimes() - // // the revised Security Policy will immediate go for the // file contents once found. This test will return that // mock file, but this specific unit test is not testing diff --git a/clients/githubrepo/client.go b/clients/githubrepo/client.go index a8b44e7d..b707d018 100644 --- a/clients/githubrepo/client.go +++ b/clients/githubrepo/client.go @@ -195,6 +195,21 @@ func (client *Client) GetCreatedAt() (time.Time, error) { return client.repo.CreatedAt.Time, nil } +func (client *Client) GetOrgRepoClient(ctx context.Context) (clients.RepoClient, error) { + dotGithubRepo, err := MakeGithubRepo(fmt.Sprintf("%s/.github", client.repourl.owner)) + if err != nil { + return nil, fmt.Errorf("error during MakeGithubRepo: %w", err) + } + + logger := log.NewLogger(log.InfoLevel) + c := CreateGithubRepoClient(ctx, logger) + if err := c.InitRepo(dotGithubRepo, clients.HeadSHA, 0); err != nil { + return nil, fmt.Errorf("error during InitRepo: %w", err) + } + + return c, nil +} + // ListWebhooks implements RepoClient.ListWebhooks. func (client *Client) ListWebhooks() ([]clients.Webhook, error) { return client.webhook.listWebhooks() diff --git a/clients/githubrepo/repo.go b/clients/githubrepo/repo.go index 295ee21b..638fe428 100644 --- a/clients/githubrepo/repo.go +++ b/clients/githubrepo/repo.go @@ -23,10 +23,6 @@ import ( sce "github.com/ossf/scorecard/v4/errors" ) -const ( - githubOrgRepo = ".github" -) - type repoURL struct { host, owner, repo, defaultBranch, commitSHA string metadata []string @@ -85,15 +81,6 @@ func (r *repoURL) String() string { return fmt.Sprintf("%s-%s-%s", r.host, r.owner, r.repo) } -// Org implements Repo.Org. -func (r *repoURL) Org() clients.Repo { - return &repoURL{ - host: r.host, - owner: r.owner, - repo: githubOrgRepo, - } -} - // IsValid implements Repo.IsValid. func (r *repoURL) IsValid() error { switch r.host { diff --git a/clients/gitlabrepo/client.go b/clients/gitlabrepo/client.go index e58aca0f..f42436de 100644 --- a/clients/gitlabrepo/client.go +++ b/clients/gitlabrepo/client.go @@ -51,6 +51,7 @@ type Client struct { webhook *webhookHandler languages *languagesHandler licenses *licensesHandler + tarball *tarballHandler ctx context.Context commitDepth int } @@ -128,6 +129,9 @@ func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string, commitD // Init languagesHandler client.licenses.init(client.repourl) + // Init tarballHandler + client.tarball.init(client.ctx, client.repourl, repo, commitSHA) + return nil } @@ -140,11 +144,11 @@ func (client *Client) LocalPath() (string, error) { } func (client *Client) ListFiles(predicate func(string) (bool, error)) ([]string, error) { - return nil, nil + return client.tarball.listFiles(predicate) } func (client *Client) GetFileContent(filename string) ([]byte, error) { - return nil, nil + return client.tarball.getFileContent(filename) } func (client *Client) ListCommits() ([]clients.Commit, error) { @@ -183,6 +187,10 @@ func (client *Client) GetCreatedAt() (time.Time, error) { return client.project.getCreatedAt() } +func (client *Client) GetOrgRepoClient(ctx context.Context) (clients.RepoClient, error) { + return nil, fmt.Errorf("GetOrgRepoClient (GitLab): %w", clients.ErrUnsupportedFeature) +} + func (client *Client) ListWebhooks() ([]clients.Webhook, error) { return client.webhook.listWebhooks() } @@ -269,6 +277,7 @@ func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients glClient: client, }, licenses: &licensesHandler{}, + tarball: &tarballHandler{}, }, nil } diff --git a/clients/gitlabrepo/repo.go b/clients/gitlabrepo/repo.go index 5c223ac1..2d072f14 100644 --- a/clients/gitlabrepo/repo.go +++ b/clients/gitlabrepo/repo.go @@ -27,10 +27,6 @@ import ( sce "github.com/ossf/scorecard/v4/errors" ) -const ( - gitlabOrgProj = ".gitlab" -) - type repoURL struct { scheme string host string @@ -95,14 +91,6 @@ func (r *repoURL) String() string { return fmt.Sprintf("%s-%s_%s", r.host, r.owner, r.project) } -func (r *repoURL) Org() clients.Repo { - return &repoURL{ - host: r.host, - owner: r.owner, - project: gitlabOrgProj, - } -} - // IsValid implements Repo.IsValid. func (r *repoURL) IsValid() error { if strings.Contains(r.host, "gitlab.") { diff --git a/clients/gitlabrepo/repo_test.go b/clients/gitlabrepo/repo_test.go index de99f02b..798be39d 100644 --- a/clients/gitlabrepo/repo_test.go +++ b/clients/gitlabrepo/repo_test.go @@ -16,6 +16,7 @@ package gitlabrepo import ( "fmt" + "os" "testing" "github.com/google/go-cmp/cmp" @@ -25,10 +26,11 @@ import ( func TestRepoURL_IsValid(t *testing.T) { t.Parallel() tests := []struct { - name string - inputURL string - expected repoURL - wantErr bool + name string + inputURL string + expected repoURL + wantErr bool + flagRequired bool }{ { name: "github repository", @@ -73,7 +75,6 @@ func TestRepoURL_IsValid(t *testing.T) { inputURL: "https://gitlab.com/ossf-test/scorecard-check-binary-artifacts-e2e/", wantErr: false, }, - { name: "valid hosted gitlab project", expected: repoURL{ @@ -82,12 +83,16 @@ func TestRepoURL_IsValid(t *testing.T) { owner: "webmaster-team", project: "webml", }, - inputURL: "https://salsa.debian.org/webmaster-team/webwml", - wantErr: false, + inputURL: "https://salsa.debian.org/webmaster-team/webwml", + wantErr: false, + flagRequired: true, }, } for _, tt := range tests { tt := tt // Re-initializing variable so it is not changed while executing the closure blow + if tt.flagRequired && os.Getenv("TEST_GITLAB_EXTERNAL") == "" { + continue + } t.Run(tt.name, func(t *testing.T) { t.Parallel() r := repoURL{ @@ -116,8 +121,9 @@ func TestRepoURL_IsValid(t *testing.T) { func TestRepoURL_DetectGitlab(t *testing.T) { tests := []struct { - repouri string - expected bool + repouri string + expected bool + flagRequired bool }{ { repouri: "github.com/ossf/scorecard", @@ -136,8 +142,9 @@ func TestRepoURL_DetectGitlab(t *testing.T) { expected: true, }, { - repouri: "https://salsa.debian.org/webmaster-team/webml", - expected: true, + repouri: "https://salsa.debian.org/webmaster-team/webml", + expected: true, + flagRequired: true, }, { // Invalid repo @@ -147,6 +154,9 @@ func TestRepoURL_DetectGitlab(t *testing.T) { } for _, tt := range tests { + if tt.flagRequired && os.Getenv("TEST_GITLAB_EXTERNAL") == "" { + continue + } g := DetectGitLab(tt.repouri) if g != tt.expected { t.Errorf("got %s isgitlab: %t expected %t", tt.repouri, g, tt.expected) diff --git a/clients/gitlabrepo/tarball.go b/clients/gitlabrepo/tarball.go new file mode 100644 index 00000000..4646f192 --- /dev/null +++ b/clients/gitlabrepo/tarball.go @@ -0,0 +1,264 @@ +// Copyright 2021 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitlabrepo + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/xanzy/go-gitlab" + + sce "github.com/ossf/scorecard/v4/errors" +) + +const ( + repoDir = "project*" + repoFilename = "gitlabproject*.tar.gz" +) + +var ( + errTarballNotFound = errors.New("tarball not found") + errTarballCorrupted = errors.New("corrupted tarball") + errZipSlip = errors.New("ZipSlip path detected") +) + +func extractAndValidateArchivePath(path, dest string) (string, error) { + const splitLength = 2 + // The tarball will have a top-level directory which contains all the repository files. + // Discard the directory and only keep the actual files. + names := strings.SplitN(path, "/", splitLength) + if len(names) < splitLength { + return dest, nil + } + if names[1] == "" { + return dest, nil + } + // Check for ZipSlip: https://snyk.io/research/zip-slip-vulnerability + cleanpath := filepath.Join(dest, names[1]) + if !strings.HasPrefix(cleanpath, filepath.Clean(dest)+string(os.PathSeparator)) { + return "", fmt.Errorf("%w: %s", errZipSlip, names[1]) + } + return cleanpath, nil +} + +type tarballHandler struct { + errSetup error + once *sync.Once + ctx context.Context + repo *gitlab.Project + repourl *repoURL + commitSHA string + tempDir string + tempTarFile string + files []string +} + +func (handler *tarballHandler) init(ctx context.Context, repourl *repoURL, repo *gitlab.Project, commitSHA string) { + handler.errSetup = nil + handler.once = new(sync.Once) + handler.ctx = ctx + handler.repo = repo + handler.repourl = repourl + handler.commitSHA = commitSHA +} + +func (handler *tarballHandler) setup() error { + handler.once.Do(func() { + // cleanup any previous state. + if err := handler.cleanup(); err != nil { + handler.errSetup = sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return + } + + // setup tem dir/files and download repo tarball. + if err := handler.getTarball(); errors.Is(err, errTarballNotFound) { + log.Printf("unable to get tarball %v. Skipping...", err) + return + } else if err != nil { + handler.errSetup = sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return + } + + // extract file names and content from tarball. + if err := handler.extractTarball(); errors.Is(err, errTarballCorrupted) { + log.Printf("unable to extract tarball %v. Skipping...", err) + } else if err != nil { + handler.errSetup = sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + }) + return handler.errSetup +} + +func (handler *tarballHandler) getTarball() error { + url := fmt.Sprintf("%s/api/v4/projects/%d/repository/archive.tar.gz?sha=%s", + handler.repourl.Host(), handler.repo.ID, handler.commitSHA) + req, err := http.NewRequestWithContext(handler.ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("http.NewRequestWithContext: %w", err) + } + req.Header.Set("PRIVATE-TOKEN", os.Getenv("GITLAB_AUTH_TOKEN")) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("http.DefaultClient.Do: %w", err) + } + defer resp.Body.Close() + + // Handler 400/404 errors. + switch resp.StatusCode { + case http.StatusNotFound, http.StatusBadRequest: + return fmt.Errorf("%w: %s", errTarballNotFound, url) + } + + // Create a temp file. This automatically appends a random number to the name. + tempDir, err := os.MkdirTemp("", repoDir) + if err != nil { + return fmt.Errorf("os.MkdirTemp: %w", err) + } + repoFile, err := os.CreateTemp(tempDir, repoFilename) + if err != nil { + return fmt.Errorf("os.CreateTemp: %w", err) + } + defer repoFile.Close() + if _, err := io.Copy(repoFile, resp.Body); err != nil { + // If the incomming tarball is corrupted or the server times out. + return fmt.Errorf("%w io.Copy: %v", errTarballNotFound, err) + } + + handler.tempDir = tempDir + handler.tempTarFile = repoFile.Name() + return nil +} + +// nolint: gocognit +func (handler *tarballHandler) extractTarball() error { + in, err := os.OpenFile(handler.tempTarFile, os.O_RDONLY, 0o644) + if err != nil { + return fmt.Errorf("os.OpenFile: %w", err) + } + gz, err := gzip.NewReader(in) + if err != nil { + return fmt.Errorf("%w: gzip.NewReader %v %v", errTarballCorrupted, handler.tempTarFile, err) + } + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("%w tarReader.Next: %v", errTarballCorrupted, err) + } + + switch header.Typeflag { + case tar.TypeDir: + dirpath, err := extractAndValidateArchivePath(header.Name, handler.tempDir) + if err != nil { + return err + } + if dirpath == filepath.Clean(handler.tempDir) { + continue + } + + if err := os.Mkdir(dirpath, 0o755); err != nil { + return fmt.Errorf("error during os.Mkdir: %w", err) + } + case tar.TypeReg: + if header.Size <= 0 { + continue + } + filenamepath, err := extractAndValidateArchivePath(header.Name, handler.tempDir) + if err != nil { + return err + } + + if _, err := os.Stat(filepath.Dir(filenamepath)); os.IsNotExist(err) { + if err := os.Mkdir(filepath.Dir(filenamepath), 0o755); err != nil { + return fmt.Errorf("os.Mkdir: %w", err) + } + } + outFile, err := os.Create(filenamepath) + if err != nil { + return fmt.Errorf("os.Create: %w", err) + } + + //nolint: gosec + // Potential for DoS vulnerability via decompression bomb. + // Since such an attack will only impact a single shard, ignoring this for now. + if _, err := io.Copy(outFile, tr); err != nil { + return fmt.Errorf("%w io.Copy: %v", errTarballCorrupted, err) + } + outFile.Close() + handler.files = append(handler.files, + strings.TrimPrefix(filenamepath, filepath.Clean(handler.tempDir)+string(os.PathSeparator))) + case tar.TypeXGlobalHeader, tar.TypeSymlink: + continue + default: + log.Printf("Unknown file type %s: '%s'", header.Name, string(header.Typeflag)) + continue + } + } + return nil +} + +func (handler *tarballHandler) listFiles(predicate func(string) (bool, error)) ([]string, error) { + if err := handler.setup(); err != nil { + return nil, fmt.Errorf("error during tarballHandler.setup: %w", err) + } + ret := make([]string, 0) + for _, file := range handler.files { + matches, err := predicate(file) + if err != nil { + return nil, err + } + if matches { + ret = append(ret, file) + } + } + return ret, nil +} + +func (handler *tarballHandler) getFileContent(filename string) ([]byte, error) { + if err := handler.setup(); err != nil { + fmt.Printf("err: %v\n", err) + return nil, fmt.Errorf("error during tarballHandler.setup: %w", err) + } + fmt.Printf("handler.tempDir: %v\n", handler.tempDir) + fmt.Printf("filename: %v\n", filename) + content, err := os.ReadFile(filepath.Join(handler.tempDir, filename)) + if err != nil { + return content, fmt.Errorf("os.ReadFile: %w", err) + } + return content, nil +} + +func (handler *tarballHandler) cleanup() error { + if err := os.RemoveAll(handler.tempDir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("os.Remove: %w", err) + } + + // Remove old file so we don't iterate through them. + handler.files = nil + return nil +} diff --git a/clients/gitlabrepo/tarball_test.go b/clients/gitlabrepo/tarball_test.go new file mode 100644 index 00000000..c9c4d571 --- /dev/null +++ b/clients/gitlabrepo/tarball_test.go @@ -0,0 +1,177 @@ +// Copyright 2021 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitlabrepo + +import ( + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +type listfileTest struct { + predicate func(string) (bool, error) + err error + outcome []string +} + +type getcontentTest struct { + err error + filename string + output []byte +} + +func isSortedString(x, y string) bool { + return x < y +} + +func setup(inputFile string) (tarballHandler, error) { + tempDir, err := os.MkdirTemp("", repoDir) + if err != nil { + return tarballHandler{}, fmt.Errorf("test failed to create TempDir: %w", err) + } + tempFile, err := os.CreateTemp(tempDir, repoFilename) + if err != nil { + return tarballHandler{}, fmt.Errorf("test failed to create TempFile: %w", err) + } + testFile, err := os.OpenFile(inputFile, os.O_RDONLY, 0o644) + if err != nil { + return tarballHandler{}, fmt.Errorf("unable to open testfile: %w", err) + } + if _, err := io.Copy(tempFile, testFile); err != nil { + return tarballHandler{}, fmt.Errorf("unable to do io.Copy: %w", err) + } + tarballHandler := tarballHandler{ + tempDir: tempDir, + tempTarFile: tempFile.Name(), + once: new(sync.Once), + } + tarballHandler.once.Do(func() { + // We don't want to run the code in tarballHandler.setup(), so if we execute tarballHandler.once.Do() right + // here, it won't get executed later when setup() is called. + }) + return tarballHandler, nil +} + +// nolint: gocognit +func TestExtractTarball(t *testing.T) { + t.Parallel() + testcases := []struct { + name string + inputFile string + listfileTests []listfileTest + getcontentTests []getcontentTest + }{ + { + name: "Basic", + inputFile: "testdata/basic.tar.gz", + listfileTests: []listfileTest{ + { + // Returns all files in the tarball. + predicate: func(string) (bool, error) { return true, nil }, + outcome: []string{"file0", "dir1/file1", "dir1/dir2/file2"}, + }, + { + // Skips all files inside `dir1/dir2` directory. + predicate: func(fn string) (bool, error) { return !strings.HasPrefix(fn, "dir1/dir2"), nil }, + outcome: []string{"file0", "dir1/file1"}, + }, + { + // Skips all files. + predicate: func(fn string) (bool, error) { return false, nil }, + outcome: []string{}, + }, + }, + getcontentTests: []getcontentTest{ + { + filename: "file0", + output: []byte("content0\n"), + }, + { + filename: "dir1/file1", + output: []byte("content1\n"), + }, + { + filename: "dir1/dir2/file2", + output: []byte("content2\n"), + }, + { + filename: "does/not/exist", + err: os.ErrNotExist, + }, + }, + }, + } + + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.name, func(t *testing.T) { + t.Parallel() + + // Setup + handler, err := setup(testcase.inputFile) + if err != nil { + t.Fatalf("test setup failed: %v", err) + } + + // Extract tarball. + if err := handler.extractTarball(); err != nil { + t.Fatalf("test failed: %v", err) + } + + // Test ListFiles API. + for _, listfiletest := range testcase.listfileTests { + matchedFiles, err := handler.listFiles(listfiletest.predicate) + if !errors.Is(err, listfiletest.err) { + t.Errorf("test failed: expected - %v, got - %v", listfiletest.err, err) + continue + } + if !cmp.Equal(listfiletest.outcome, + matchedFiles, + cmpopts.SortSlices(isSortedString)) { + t.Errorf("test failed: expected - %q, got - %q", listfiletest.outcome, matchedFiles) + } + } + + // Test GetFileContent API. + for _, getcontenttest := range testcase.getcontentTests { + content, err := handler.getFileContent(getcontenttest.filename) + if getcontenttest.err != nil && !errors.Is(err, getcontenttest.err) { + t.Errorf("test failed: expected - %v, got - %v", getcontenttest.err, err) + } + if getcontenttest.err == nil && !cmp.Equal(getcontenttest.output, content) { + t.Errorf("test failed: expected - %s, got - %s", string(getcontenttest.output), string(content)) + } + } + + // Test that files get deleted. + if err := handler.cleanup(); err != nil { + t.Errorf("test failed: %v", err) + } + if _, err := os.Stat(handler.tempDir); !os.IsNotExist(err) { + t.Errorf("%v", err) + } + if len(handler.files) != 0 { + t.Error("client.files not cleaned up!") + } + }) + } +} diff --git a/clients/gitlabrepo/testdata/basic.tar.gz b/clients/gitlabrepo/testdata/basic.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1bc288650a96788c839f7fee50969b50ee2a9339 GIT binary patch literal 268 zcmV+n0rUPJiwFP!000001MSs8YQr!P1yHSf3g4ibu|_(NY1Lq8Vk0Yy^z@aC8*pd~ z21|kDzl9dIL4t4niGGT~kHf1fM<-6SBBMpDQ<}vIxAXHwerFv?TGMlC#TwN}!82W* zr`Uz2QQepK$La6y#?T+Ta5;qGJk~f@CGjW!kN(r^GJE8Jt@$T^qMQ7UaZWXbj