From aa634bd2518328ceb1c80305c2c9008f1c176da4 Mon Sep 17 00:00:00 2001 From: naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:22:41 +0000 Subject: [PATCH] :seedling: Fixes the broken e2e Fixes for broken e2e --- .github/workflows/integration.yml | 5 ++-- e2e/branch_protection_test.go | 44 +++++-------------------------- e2e/ci_tests_test.go | 11 +++----- e2e/code_review_test.go | 11 +++----- e2e/contributors_test.go | 28 -------------------- e2e/packaging_test.go | 10 ++----- e2e/pinned_dependencies_test.go | 11 +++----- e2e/sast_test.go | 10 +++---- 8 files changed, 25 insertions(+), 105 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2643aadd..6a9f53ee 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -86,9 +86,8 @@ jobs: env: GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }} run: | - echo "Skipping ci-e2e. Will be re-enabled soon." - #go env -w GOFLAGS=-mod=mod - #make ci-e2e + go env -w GOFLAGS=-mod=mod + make ci-e2e - name: install tools run: make install diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go index 3102c9e5..68a560cc 100644 --- a/e2e/branch_protection_test.go +++ b/e2e/branch_protection_test.go @@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { Context("E2E TEST:Validating branch protection", func() { It("Should fail to return branch protection on other repositories", func() { dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("apache/airflow") + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) err = repoClient.InitRepo(repo) @@ -43,48 +43,18 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { } expected := scut.TestReturn{ Error: nil, - Score: 1, - NumberOfWarn: 3, - NumberOfInfo: 0, + Score: 9, + NumberOfWarn: 1, + NumberOfInfo: 8, NumberOfDebug: 0, } result := checks.BranchProtection(&req) // UPGRADEv2: to remove. // Old version. - Expect(result.Error).ShouldNot(BeNil()) - Expect(result.Pass).Should(BeFalse()) + Expect(result.Error).Should(BeNil()) + Expect(result.Pass).Should(BeTrue()) // New version. - Expect(scut.ValidateTestReturn(nil, "branch protection not accessible", &expected, &result, &dl)).Should(BeTrue()) - }) - Context("E2E TEST:Validating branch protection", func() { - It("Should fail to return branch protection on other repositories", func() { - dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e") - Expect(err).Should(BeNil()) - repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) - Expect(err).Should(BeNil()) - req := checker.CheckRequest{ - Ctx: context.Background(), - RepoClient: repoClient, - Repo: repo, - Dlogger: &dl, - } - expected := scut.TestReturn{ - Error: nil, - Score: 9, - NumberOfWarn: 1, - NumberOfInfo: 8, - NumberOfDebug: 0, - } - result := checks.BranchProtection(&req) - // UPGRADEv2: to remove. - // Old version. - Expect(result.Error).Should(BeNil()) - Expect(result.Pass).Should(BeTrue()) - // New version. - Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue()) - }) + Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue()) }) }) }) diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index d4983af5..58d4f10f 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { Context("E2E TEST:Validating use of CI tests", func() { It("Should return use of CI tests", func() { dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("apache/airflow") + repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) err = repoClient.InitRepo(repo) @@ -43,17 +43,12 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { } expected := scut.TestReturn{ Error: nil, - Score: checker.MaxResultScore, + Score: checker.InconclusiveResultScore, NumberOfWarn: 0, NumberOfInfo: 0, - NumberOfDebug: 30, + NumberOfDebug: 0, } result := checks.CITests(&req) - // UPGRADEv2: to remove. - // Old version. - Expect(result.Error).Should(BeNil()) - Expect(result.Pass).Should(BeTrue()) - // New version. Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue()) }) }) diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go index 082863d9..44f0930c 100644 --- a/e2e/code_review_test.go +++ b/e2e/code_review_test.go @@ -32,7 +32,7 @@ var _ = Describe("E2E TEST:CodeReview", func() { Context("E2E TEST:Validating use of code reviews", func() { It("Should return use of code reviews", func() { dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("apache/airflow") + repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) err = repoClient.InitRepo(repo) @@ -46,17 +46,12 @@ var _ = Describe("E2E TEST:CodeReview", func() { } expected := scut.TestReturn{ Error: nil, - Score: checker.MaxResultScore, + Score: checker.MinResultScore, NumberOfWarn: 0, - NumberOfInfo: 0, + NumberOfInfo: 3, NumberOfDebug: 0, } result := checks.DoesCodeReview(&req) - // UPGRADEv2: to remove. - // Old version. - Expect(result.Error).Should(BeNil()) - Expect(result.Pass).Should(BeTrue()) - // New version. Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) }) }) diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go index 04fafcd8..0bf513a0 100644 --- a/e2e/contributors_test.go +++ b/e2e/contributors_test.go @@ -56,33 +56,5 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() { // New version. Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue()) }) - It("Should return valid project contributors", func() { - dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("apache/airflow") - Expect(err).Should(BeNil()) - repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) - Expect(err).Should(BeNil()) - checkRequest := checker.CheckRequest{ - Ctx: context.Background(), - RepoClient: repoClient, - Repo: repo, - Dlogger: &dl, - } - expected := scut.TestReturn{ - Error: nil, - Score: checker.MaxResultScore, - NumberOfWarn: 0, - NumberOfInfo: 1, - NumberOfDebug: 0, - } - result := checks.Contributors(&checkRequest) - // UPGRADEv2: to remove. - // Old version. - Expect(result.Error).Should(BeNil()) - Expect(result.Pass).Should(BeTrue()) - // New version. - Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue()) - }) }) }) diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index 3cf44db8..95e51444 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -43,18 +43,12 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() { } expected := scut.TestReturn{ Error: nil, - Score: checker.MaxResultScore, - NumberOfWarn: 0, + Score: checker.InconclusiveResultScore, + NumberOfWarn: 1, NumberOfInfo: 2, NumberOfDebug: 2, } result := checks.Packaging(&req) - - // UPGRADEv2: to remove. - // Old version. - Expect(result.Error).Should(BeNil()) - Expect(result.Pass).Should(BeTrue()) - // New version. Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue()) }) }) diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go index 6cb780d5..97868369 100644 --- a/e2e/pinned_dependencies_test.go +++ b/e2e/pinned_dependencies_test.go @@ -45,17 +45,12 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { } expected := scut.TestReturn{ Error: nil, - Score: checker.MinResultScore, - NumberOfWarn: 154, - NumberOfInfo: 0, + Score: 3, + NumberOfWarn: 150, + NumberOfInfo: 2, NumberOfDebug: 0, } result := checks.PinnedDependencies(&req) - // UPGRADEv2: to remove. - // Old version. - Expect(result.Error).Should(BeNil()) - Expect(result.Pass).Should(BeFalse()) - // New version. Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue()) }) }) diff --git a/e2e/sast_test.go b/e2e/sast_test.go index d68a7b8d..4ca8b7be 100644 --- a/e2e/sast_test.go +++ b/e2e/sast_test.go @@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() { Context("E2E TEST:Validating use of SAST tools", func() { It("Should return use of SAST tools", func() { dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("apache/airflow") + repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) err = repoClient.InitRepo(repo) @@ -43,10 +43,10 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() { } expected := scut.TestReturn{ Error: nil, - Score: 7, - NumberOfWarn: 1, - NumberOfInfo: 1, - NumberOfDebug: 1, + Score: 0, + NumberOfWarn: 2, + NumberOfInfo: 0, + NumberOfDebug: 0, } result := checks.SAST(&req) // UPGRADEv2: to remove.