mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-17 11:57:12 +03:00
🌱 Fixes the broken e2e
Fixes for broken e2e
This commit is contained in:
parent
fd238d0e40
commit
aa634bd251
5
.github/workflows/integration.yml
vendored
5
.github/workflows/integration.yml
vendored
@ -86,9 +86,8 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
|
GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
echo "Skipping ci-e2e. Will be re-enabled soon."
|
go env -w GOFLAGS=-mod=mod
|
||||||
#go env -w GOFLAGS=-mod=mod
|
make ci-e2e
|
||||||
#make ci-e2e
|
|
||||||
|
|
||||||
- name: install tools
|
- name: install tools
|
||||||
run: make install
|
run: make install
|
||||||
|
@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
|
|||||||
Context("E2E TEST:Validating branch protection", func() {
|
Context("E2E TEST:Validating branch protection", func() {
|
||||||
It("Should fail to return branch protection on other repositories", func() {
|
It("Should fail to return branch protection on other repositories", func() {
|
||||||
dl := scut.TestDetailLogger{}
|
dl := scut.TestDetailLogger{}
|
||||||
repo, err := githubrepo.MakeGithubRepo("apache/airflow")
|
repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e")
|
||||||
Expect(err).Should(BeNil())
|
Expect(err).Should(BeNil())
|
||||||
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
||||||
err = repoClient.InitRepo(repo)
|
err = repoClient.InitRepo(repo)
|
||||||
@ -43,48 +43,18 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
|
|||||||
}
|
}
|
||||||
expected := scut.TestReturn{
|
expected := scut.TestReturn{
|
||||||
Error: nil,
|
Error: nil,
|
||||||
Score: 1,
|
Score: 9,
|
||||||
NumberOfWarn: 3,
|
NumberOfWarn: 1,
|
||||||
NumberOfInfo: 0,
|
NumberOfInfo: 8,
|
||||||
NumberOfDebug: 0,
|
NumberOfDebug: 0,
|
||||||
}
|
}
|
||||||
result := checks.BranchProtection(&req)
|
result := checks.BranchProtection(&req)
|
||||||
// UPGRADEv2: to remove.
|
// UPGRADEv2: to remove.
|
||||||
// Old version.
|
// Old version.
|
||||||
Expect(result.Error).ShouldNot(BeNil())
|
Expect(result.Error).Should(BeNil())
|
||||||
Expect(result.Pass).Should(BeFalse())
|
Expect(result.Pass).Should(BeTrue())
|
||||||
// New version.
|
// New version.
|
||||||
Expect(scut.ValidateTestReturn(nil, "branch protection not accessible", &expected, &result, &dl)).Should(BeTrue())
|
Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue())
|
||||||
})
|
|
||||||
Context("E2E TEST:Validating branch protection", func() {
|
|
||||||
It("Should fail to return branch protection on other repositories", func() {
|
|
||||||
dl := scut.TestDetailLogger{}
|
|
||||||
repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e")
|
|
||||||
Expect(err).Should(BeNil())
|
|
||||||
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
|
||||||
err = repoClient.InitRepo(repo)
|
|
||||||
Expect(err).Should(BeNil())
|
|
||||||
req := checker.CheckRequest{
|
|
||||||
Ctx: context.Background(),
|
|
||||||
RepoClient: repoClient,
|
|
||||||
Repo: repo,
|
|
||||||
Dlogger: &dl,
|
|
||||||
}
|
|
||||||
expected := scut.TestReturn{
|
|
||||||
Error: nil,
|
|
||||||
Score: 9,
|
|
||||||
NumberOfWarn: 1,
|
|
||||||
NumberOfInfo: 8,
|
|
||||||
NumberOfDebug: 0,
|
|
||||||
}
|
|
||||||
result := checks.BranchProtection(&req)
|
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
// Old version.
|
|
||||||
Expect(result.Error).Should(BeNil())
|
|
||||||
Expect(result.Pass).Should(BeTrue())
|
|
||||||
// New version.
|
|
||||||
Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue())
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
|
|||||||
Context("E2E TEST:Validating use of CI tests", func() {
|
Context("E2E TEST:Validating use of CI tests", func() {
|
||||||
It("Should return use of CI tests", func() {
|
It("Should return use of CI tests", func() {
|
||||||
dl := scut.TestDetailLogger{}
|
dl := scut.TestDetailLogger{}
|
||||||
repo, err := githubrepo.MakeGithubRepo("apache/airflow")
|
repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow")
|
||||||
Expect(err).Should(BeNil())
|
Expect(err).Should(BeNil())
|
||||||
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
||||||
err = repoClient.InitRepo(repo)
|
err = repoClient.InitRepo(repo)
|
||||||
@ -43,17 +43,12 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
|
|||||||
}
|
}
|
||||||
expected := scut.TestReturn{
|
expected := scut.TestReturn{
|
||||||
Error: nil,
|
Error: nil,
|
||||||
Score: checker.MaxResultScore,
|
Score: checker.InconclusiveResultScore,
|
||||||
NumberOfWarn: 0,
|
NumberOfWarn: 0,
|
||||||
NumberOfInfo: 0,
|
NumberOfInfo: 0,
|
||||||
NumberOfDebug: 30,
|
NumberOfDebug: 0,
|
||||||
}
|
}
|
||||||
result := checks.CITests(&req)
|
result := checks.CITests(&req)
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
// Old version.
|
|
||||||
Expect(result.Error).Should(BeNil())
|
|
||||||
Expect(result.Pass).Should(BeTrue())
|
|
||||||
// New version.
|
|
||||||
Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
|
Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -32,7 +32,7 @@ var _ = Describe("E2E TEST:CodeReview", func() {
|
|||||||
Context("E2E TEST:Validating use of code reviews", func() {
|
Context("E2E TEST:Validating use of code reviews", func() {
|
||||||
It("Should return use of code reviews", func() {
|
It("Should return use of code reviews", func() {
|
||||||
dl := scut.TestDetailLogger{}
|
dl := scut.TestDetailLogger{}
|
||||||
repo, err := githubrepo.MakeGithubRepo("apache/airflow")
|
repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow")
|
||||||
Expect(err).Should(BeNil())
|
Expect(err).Should(BeNil())
|
||||||
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
||||||
err = repoClient.InitRepo(repo)
|
err = repoClient.InitRepo(repo)
|
||||||
@ -46,17 +46,12 @@ var _ = Describe("E2E TEST:CodeReview", func() {
|
|||||||
}
|
}
|
||||||
expected := scut.TestReturn{
|
expected := scut.TestReturn{
|
||||||
Error: nil,
|
Error: nil,
|
||||||
Score: checker.MaxResultScore,
|
Score: checker.MinResultScore,
|
||||||
NumberOfWarn: 0,
|
NumberOfWarn: 0,
|
||||||
NumberOfInfo: 0,
|
NumberOfInfo: 3,
|
||||||
NumberOfDebug: 0,
|
NumberOfDebug: 0,
|
||||||
}
|
}
|
||||||
result := checks.DoesCodeReview(&req)
|
result := checks.DoesCodeReview(&req)
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
// Old version.
|
|
||||||
Expect(result.Error).Should(BeNil())
|
|
||||||
Expect(result.Pass).Should(BeTrue())
|
|
||||||
// New version.
|
|
||||||
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
|
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -56,33 +56,5 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() {
|
|||||||
// New version.
|
// New version.
|
||||||
Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
|
Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
|
||||||
})
|
})
|
||||||
It("Should return valid project contributors", func() {
|
|
||||||
dl := scut.TestDetailLogger{}
|
|
||||||
repo, err := githubrepo.MakeGithubRepo("apache/airflow")
|
|
||||||
Expect(err).Should(BeNil())
|
|
||||||
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
|
||||||
err = repoClient.InitRepo(repo)
|
|
||||||
Expect(err).Should(BeNil())
|
|
||||||
checkRequest := checker.CheckRequest{
|
|
||||||
Ctx: context.Background(),
|
|
||||||
RepoClient: repoClient,
|
|
||||||
Repo: repo,
|
|
||||||
Dlogger: &dl,
|
|
||||||
}
|
|
||||||
expected := scut.TestReturn{
|
|
||||||
Error: nil,
|
|
||||||
Score: checker.MaxResultScore,
|
|
||||||
NumberOfWarn: 0,
|
|
||||||
NumberOfInfo: 1,
|
|
||||||
NumberOfDebug: 0,
|
|
||||||
}
|
|
||||||
result := checks.Contributors(&checkRequest)
|
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
// Old version.
|
|
||||||
Expect(result.Error).Should(BeNil())
|
|
||||||
Expect(result.Pass).Should(BeTrue())
|
|
||||||
// New version.
|
|
||||||
Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -43,18 +43,12 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() {
|
|||||||
}
|
}
|
||||||
expected := scut.TestReturn{
|
expected := scut.TestReturn{
|
||||||
Error: nil,
|
Error: nil,
|
||||||
Score: checker.MaxResultScore,
|
Score: checker.InconclusiveResultScore,
|
||||||
NumberOfWarn: 0,
|
NumberOfWarn: 1,
|
||||||
NumberOfInfo: 2,
|
NumberOfInfo: 2,
|
||||||
NumberOfDebug: 2,
|
NumberOfDebug: 2,
|
||||||
}
|
}
|
||||||
result := checks.Packaging(&req)
|
result := checks.Packaging(&req)
|
||||||
|
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
// Old version.
|
|
||||||
Expect(result.Error).Should(BeNil())
|
|
||||||
Expect(result.Pass).Should(BeTrue())
|
|
||||||
// New version.
|
|
||||||
Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue())
|
Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -45,17 +45,12 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() {
|
|||||||
}
|
}
|
||||||
expected := scut.TestReturn{
|
expected := scut.TestReturn{
|
||||||
Error: nil,
|
Error: nil,
|
||||||
Score: checker.MinResultScore,
|
Score: 3,
|
||||||
NumberOfWarn: 154,
|
NumberOfWarn: 150,
|
||||||
NumberOfInfo: 0,
|
NumberOfInfo: 2,
|
||||||
NumberOfDebug: 0,
|
NumberOfDebug: 0,
|
||||||
}
|
}
|
||||||
result := checks.PinnedDependencies(&req)
|
result := checks.PinnedDependencies(&req)
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
// Old version.
|
|
||||||
Expect(result.Error).Should(BeNil())
|
|
||||||
Expect(result.Pass).Should(BeFalse())
|
|
||||||
// New version.
|
|
||||||
Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue())
|
Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() {
|
|||||||
Context("E2E TEST:Validating use of SAST tools", func() {
|
Context("E2E TEST:Validating use of SAST tools", func() {
|
||||||
It("Should return use of SAST tools", func() {
|
It("Should return use of SAST tools", func() {
|
||||||
dl := scut.TestDetailLogger{}
|
dl := scut.TestDetailLogger{}
|
||||||
repo, err := githubrepo.MakeGithubRepo("apache/airflow")
|
repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow")
|
||||||
Expect(err).Should(BeNil())
|
Expect(err).Should(BeNil())
|
||||||
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
|
||||||
err = repoClient.InitRepo(repo)
|
err = repoClient.InitRepo(repo)
|
||||||
@ -43,10 +43,10 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() {
|
|||||||
}
|
}
|
||||||
expected := scut.TestReturn{
|
expected := scut.TestReturn{
|
||||||
Error: nil,
|
Error: nil,
|
||||||
Score: 7,
|
Score: 0,
|
||||||
NumberOfWarn: 1,
|
NumberOfWarn: 2,
|
||||||
NumberOfInfo: 1,
|
NumberOfInfo: 0,
|
||||||
NumberOfDebug: 1,
|
NumberOfDebug: 0,
|
||||||
}
|
}
|
||||||
result := checks.SAST(&req)
|
result := checks.SAST(&req)
|
||||||
// UPGRADEv2: to remove.
|
// UPGRADEv2: to remove.
|
||||||
|
Loading…
Reference in New Issue
Block a user