Add HTTP stats (#484)

Co-authored-by: Azeem Shaikh <azeems@google.com>

cron/main.go

@@ -53,7 +53,10 @@ func startMetricsExporter() (*stackdriver.Exporter, error) {
 	if err := exporter.StartMetricsExporter(); err != nil {
 		return nil, fmt.Errorf("error in StartMetricsExporter: %w", err)
 	}
-	if err := view.Register(&stats.CheckRuntime); err != nil {
+
+	if err := view.Register(
+		&stats.CheckRuntime,
+		&stats.OutgoingHTTPRequests); err != nil {
 		return nil, fmt.Errorf("error during view.Register: %w", err)
 	}
 	return exporter, nil
@@ -106,7 +109,9 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	defer exporter.Flush()
 	defer exporter.StopMetricsExporter()
+
 	for _, r := range inputRepos {
 		fmt.Println(r.Repo)
 

cron/monitoring/exporter.go

@@ -26,7 +26,10 @@ func NewStackDriverExporter() (*stackdriver.Exporter, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error getting ProjectID: %w", err)
 	}
-	exporter, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
+	exporter, err := stackdriver.NewExporter(stackdriver.Options{
+		ProjectID:    projectID,
+		MetricPrefix: "scorecard-cron",
+	})
 	if err != nil {
 		return nil, fmt.Errorf("error during stackdriver.NewExporter: %w", err)
 	}

roundtripper/census.go

@@ -0,0 +1,54 @@
+// Copyright 2020 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roundtripper
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/ossf/scorecard/stats"
+	"go.opencensus.io/plugin/ochttp"
+	opencensusstats "go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+)
+
+func MakeCensusTransport(innerTransport http.RoundTripper) http.RoundTripper {
+	return &censusTransport{
+		innerTransport: &ochttp.Transport{
+			Base: innerTransport,
+		},
+	}
+}
+
+// censusTransport is a monitoring aware http.Transport.
+type censusTransport struct {
+	innerTransport http.RoundTripper
+}
+
+// Roundtrip handles context update and measurement recording.
+func (ct *censusTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	ctx, err := tag.New(r.Context(), tag.Upsert(stats.RequestTag, "requested"))
+	if err != nil {
+		return nil, fmt.Errorf("error during tag.New: %w", err)
+	}
+	defer opencensusstats.Record(ctx, stats.HTTPRequests.M(1))
+
+	r = r.WithContext(ctx)
+	resp, err := ct.innerTransport.RoundTrip(r)
+	if err != nil {
+		return nil, fmt.Errorf("error in RoundTrip: %w", err)
+	}
+	return resp, nil
+}

roundtripper/rate_limit.go

@@ -15,11 +15,15 @@
 package roundtripper
 
 import (
+	"fmt"
 	"net/http"
 	"strconv"
 	"time"
 
+	"github.com/ossf/scorecard/stats"
 	"github.com/pkg/errors"
+	opencensusstats "go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 	"go.uber.org/zap"
 )
 
@@ -38,11 +42,18 @@ type rateLimitTransport struct {
 
 // Roundtrip handles caching and ratelimiting of responses from GitHub.
 func (gh *rateLimitTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	// Reaching here implies we couldn't reply using cache.
+	ctx, err := tag.New(r.Context(), tag.Upsert(stats.RequestTag, "actual"))
+	if err != nil {
+		return nil, fmt.Errorf("error during tag.New: %w", err)
+	}
+	defer opencensusstats.Record(ctx, stats.HTTPRequests.M(1))
+
+	r = r.WithContext(ctx)
 	resp, err := gh.innerTransport.RoundTrip(r)
 	if err != nil {
 		return nil, errors.Wrap(err, "error in round trip")
 	}
-
 	rateLimit := resp.Header.Get("X-RateLimit-Remaining")
 	remaining, err := strconv.Atoi(rateLimit)
 	if err != nil {

roundtripper/roundtripper.go

@@ -60,11 +60,10 @@ func NewTransport(ctx context.Context, logger *zap.SugaredLogger) http.RoundTrip
 		}
 	}
 
-	// Wrap that with the rate limiter
-	rateLimit := MakeRateLimitedTransport(transport, logger)
-
-	// Wrap that with a HTTP cache
-	return cachedTransportFactory(rateLimit)
+	// Wrap that with the rate limiter, HTTP cache and census-enabled transport.
+	return MakeCensusTransport(
+		cachedTransportFactory(
+			MakeRateLimitedTransport(transport, logger)))
 }
 
 func cachedTransportFactory(innerTransport http.RoundTripper) http.RoundTripper {

stats/measures.go

@@ -16,4 +16,7 @@ package stats
 
 import "go.opencensus.io/stats"
 
-var CPURuntimeInSec = stats.Int64("CPURuntimeInSec", "Measures the CPU runtime in seconds", stats.UnitSeconds)
+var (
+	CPURuntimeInSec = stats.Int64("CPURuntimeInSec", "Measures the CPU runtime in seconds", stats.UnitSeconds)
+	HTTPRequests    = stats.Int64("HTTPRequests", "Measures the count of HTTP requests", stats.UnitDimensionless)
+)

stats/tags.go

@@ -17,6 +17,7 @@ package stats
 import "go.opencensus.io/tag"
 
 var (
-	CheckName = tag.MustNewKey("checkName")
-	Repo      = tag.MustNewKey("repo")
+	CheckName  = tag.MustNewKey("checkName")
+	Repo       = tag.MustNewKey("repo")
+	RequestTag = tag.MustNewKey("requestTag")
 )

stats/views.go

@@ -15,29 +15,40 @@
 package stats
 
 import (
+	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
 )
 
-var CheckRuntime = view.View{
-	Name:        "CheckRuntime",
-	Description: "CPU runtime stats per repo per check",
-	Measure:     CPURuntimeInSec,
-	TagKeys:     []tag.Key{Repo, CheckName},
-	Aggregation: view.Distribution(
-		1<<2,
-		1<<3,
-		1<<4,
-		1<<5,
-		1<<6,
-		1<<7,
-		1<<8,
-		1<<9,
-		1<<10,
-		1<<11,
-		1<<12,
-		1<<13,
-		1<<14,
-		1<<15,
-		1<<16),
-}
+var (
+	CheckRuntime = view.View{
+		Name:        "CheckRuntime",
+		Description: "CPU runtime stats per repo per check",
+		Measure:     CPURuntimeInSec,
+		TagKeys:     []tag.Key{Repo, CheckName},
+		Aggregation: view.Distribution(
+			1<<2,
+			1<<3,
+			1<<4,
+			1<<5,
+			1<<6,
+			1<<7,
+			1<<8,
+			1<<9,
+			1<<10,
+			1<<11,
+			1<<12,
+			1<<13,
+			1<<14,
+			1<<15,
+			1<<16),
+	}
+
+	OutgoingHTTPRequests = view.View{
+		Name:        "OutgoingHTTPRequests",
+		Description: "HTTPRequests made per repo per check per URL path",
+		Measure:     HTTPRequests,
+		TagKeys:     []tag.Key{Repo, CheckName, ochttp.KeyClientPath, RequestTag},
+		Aggregation: view.Count(),
+	}
+)