Add monitoring for Scorecard errors (#597)

Co-authored-by: Azeem Shaikh <azeems@google.com>

checker/check_result.go

@@ -14,7 +14,11 @@
 
 package checker
 
-import "errors"
+import (
+	"errors"
+
+	scorecarderrors "github.com/ossf/scorecard/errors"
+)
 
 const MaxResultConfidence = 10
 
@@ -35,7 +39,7 @@ func MakeInconclusiveResult(name string, err error) CheckResult {
 		Name:       name,
 		Pass:       false,
 		Confidence: 0,
-		Error:      err,
+		Error:      scorecarderrors.MakeZeroConfidenceError(err),
 	}
 }
 
@@ -61,7 +65,7 @@ func MakeRetryResult(name string, err error) CheckResult {
 		Name:        name,
 		Pass:        false,
 		ShouldRetry: true,
-		Error:       err,
+		Error:       scorecarderrors.MakeRetryError(err),
 	}
 }
 

checker/check_runner.go

@@ -23,6 +23,7 @@ import (
 	opencensusstats "go.opencensus.io/stats"
 	"go.opencensus.io/tag"
 
+	scorecarderrors "github.com/ossf/scorecard/errors"
 	"github.com/ossf/scorecard/stats"
 )
 
@@ -46,9 +47,18 @@ func (l *logger) Logf(s string, f ...interface{}) {
 	l.messages = append(l.messages, fmt.Sprintf(s, f...))
 }
 
-func logStats(ctx context.Context, startTime time.Time) {
+func logStats(ctx context.Context, startTime time.Time, result CheckResult) error {
 	runTimeInSecs := time.Now().Unix() - startTime.Unix()
 	opencensusstats.Record(ctx, stats.CheckRuntimeInSec.M(runTimeInSecs))
+
+	if result.Error != nil {
+		ctx, err := tag.New(ctx, tag.Upsert(stats.ErrorName, scorecarderrors.GetErrorName(result.Error)))
+		if err != nil {
+			return fmt.Errorf("%w", err)
+		}
+		opencensusstats.Record(ctx, stats.CheckErrors.M(1))
+	}
+	return nil
 }
 
 func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
@@ -56,7 +66,7 @@ func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
 	if err != nil {
 		panic(err)
 	}
-	defer logStats(ctx, time.Now())
+	startTime := time.Now()
 
 	var res CheckResult
 	var l logger
@@ -73,6 +83,10 @@ func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
 		break
 	}
 	res.Details = l.messages
+
+	if err := logStats(ctx, startTime, res); err != nil {
+		panic(err)
+	}
 	return res
 }
 

cron/worker/main.go

@@ -135,6 +135,7 @@ func startMetricsExporter() (monitoring.Exporter, error) {
 
 	if err := view.Register(
 		&stats.CheckRuntime,
+		&stats.CheckErrorCount,
 		&stats.RepoRuntime,
 		&stats.OutgoingHTTPRequests); err != nil {
 		return nil, fmt.Errorf("error during view.Register: %w", err)

errors/names.go

@@ -0,0 +1,44 @@
+// Copyright 2020 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package errors
+
+import (
+	"errors"
+)
+
+const (
+	// RetryError occurs when checks fail after exhausting all retry attempts.
+	RetryError = "RetryError"
+	// ZeroConfidenceError shows an inconclusive result.
+	ZeroConfidenceError = "ZeroConfidenceError"
+	// UnknownError for all error types not handled.
+	UnknownError = "UnknownError"
+)
+
+var (
+	errRetry          *ErrRetry
+	errZeroConfidence *ErrZeroConfidence
+)
+
+func GetErrorName(err error) string {
+	switch {
+	case errors.As(err, &errRetry):
+		return RetryError
+	case errors.As(err, &errZeroConfidence):
+		return ZeroConfidenceError
+	default:
+		return UnknownError
+	}
+}

errors/types.go

@@ -0,0 +1,55 @@
+// Copyright 2020 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package errors
+
+import (
+	"fmt"
+)
+
+type (
+	ErrRetry          struct{ wrappedError }
+	ErrZeroConfidence struct{ wrappedError }
+)
+
+func MakeRetryError(err error) error {
+	return &ErrRetry{
+		wrappedError{
+			msg:        "unable to run check, retry",
+			innerError: err,
+		},
+	}
+}
+
+func MakeZeroConfidenceError(err error) error {
+	return &ErrZeroConfidence{
+		wrappedError{
+			msg:        "check result was unknown",
+			innerError: err,
+		},
+	}
+}
+
+type wrappedError struct {
+	innerError error
+	msg        string
+}
+
+func (err *wrappedError) Error() string {
+	return fmt.Sprintf("%s: %v", err.msg, err.innerError)
+}
+
+func (err *wrappedError) Unwrap() error {
+	return err.innerError
+}

stats/measures.go

@@ -20,6 +20,8 @@ var (
 	// CheckRuntimeInSec measures the CPU runtime in seconds per check.
 	CheckRuntimeInSec = stats.Int64("CheckRuntimeInSec", "Measures the CPU runtime in seconds for a check",
 		stats.UnitSeconds)
+	// CheckErrors measures the count of errors per check.
+	CheckErrors = stats.Int64("CheckErrors", "Measures the count of errors", stats.UnitDimensionless)
 	// RepoRuntimeInSec measures the CPU runtime in seconds per repo.
 	RepoRuntimeInSec = stats.Int64("RepoRuntimeInSec", "Measures the CPU runtime in seconds for a repo",
 		stats.UnitSeconds)

stats/tags.go

@@ -19,6 +19,8 @@ import "go.opencensus.io/tag"
 var (
 	// CheckName is the tag key for the check name.
 	CheckName = tag.MustNewKey("checkName")
+	// ErrorName is the tag key for errors.
+	ErrorName = tag.MustNewKey("errorName")
 	// Repo is the tag key for the repo name.
 	Repo = tag.MustNewKey("repo")
 	// RequestTag is the tag key for the request type.

stats/views.go

@@ -45,6 +45,15 @@ var (
 			1<<15),
 	}
 
+	// CheckErrorCount tracks error count stats for checks.
+	CheckErrorCount = view.View{
+		Name:        "CheckErrorCount",
+		Description: "Error count by type per check",
+		Measure:     CheckErrors,
+		TagKeys:     []tag.Key{CheckName, ErrorName},
+		Aggregation: view.Count(),
+	}
+
 	// RepoRuntime tracks CPU runtime stats for repos.
 	RepoRuntime = view.View{
 		Name:        "RepoRuntime",