Add GitHub token server (#1132)

Co-authored-by: Azeem Shaikh <azeems@google.com>

.gitignore

@@ -1,6 +1,7 @@
 # binary.
 scorecard
 gitblobcache
+clients/githubrepo/roundtripper/tokens/server/github-auth-server
 cron/data/add/add
 cron/data/validate/validate
 cron/data/update/projects-update

Makefile

@@ -90,7 +90,7 @@ tree-status: ## Verify tree is clean and all changes are committed
 ###############################################################################
 
 ################################## make build #################################
-build-targets = generate-docs build-proto build-scorecard build-pubsub build-bq-transfer \
+build-targets = generate-docs build-proto build-scorecard build-pubsub build-bq-transfer build-github-server \
 	build-add-script build-validate-script build-update-script dockerbuild
 .PHONY: build $(build-targets)
 build: ## Build all binaries and images in the repo.
@@ -127,6 +127,12 @@ build-bq-transfer: ./cron/bq/*.go
 	# Run go build on the Copier cron job
 	cd cron/bq && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o data-transfer
 
+build-github-server: ## Runs go build on the GitHub auth server
+build-github-server: ./clients/githubrepo/roundtripper/tokens/*
+	# Run go build on the GitHub auth server
+	cd clients/githubrepo/roundtripper/tokens/server && \
+		CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o github-auth-server
+
 build-webhook: ## Runs go build on the cron webhook
 	# Run go build on the cron webhook
 	cd cron/webhook && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o webhook
@@ -157,6 +163,7 @@ dockerbuild: ## Runs docker build
 	DOCKER_BUILDKIT=1 docker build . --file cron/worker/Dockerfile --tag $(IMAGE_NAME)-batch-worker
 	DOCKER_BUILDKIT=1 docker build . --file cron/bq/Dockerfile --tag $(IMAGE_NAME)-bq-transfer
 	DOCKER_BUILDKIT=1 docker build . --file cron/webhook/Dockerfile --tag ${IMAGE_NAME}-webhook
+	DOCKER_BUILDKIT=1 docker build . --file clients/githubrepo/roundtripper/tokens/server/Dockerfile --tag ${IMAGE_NAME}-github-server
 ###############################################################################
 
 ################################# make test ###################################

clients/githubrepo/roundtripper/roundtripper.go

@@ -21,48 +21,36 @@ import (
 	"net/http"
 	"os"
 	"strconv"
-	"strings"
 
 	"github.com/bradleyfalzon/ghinstallation/v2"
 	"go.uber.org/zap"
-)
 
-// GithubAuthTokens are for making requests to GiHub's API.
-var GithubAuthTokens = []string{"GITHUB_AUTH_TOKEN", "GITHUB_TOKEN", "GH_TOKEN", "GH_AUTH_TOKEN"}
+	"github.com/ossf/scorecard/v3/clients/githubrepo/roundtripper/tokens"
+)
 
 const (
-
-	// GithubAppKeyPath is the path to file for GitHub App key.
-	GithubAppKeyPath = "GITHUB_APP_KEY_PATH"
-	// GithubAppID is the app ID for the GitHub App.
-	GithubAppID = "GITHUB_APP_ID"
-	// GithubAppInstallationID is the installation ID for the GitHub App.
-	GithubAppInstallationID = "GITHUB_APP_INSTALLATION_ID"
+	// githubAppKeyPath is the path to file for GitHub App key.
+	githubAppKeyPath = "GITHUB_APP_KEY_PATH"
+	// githubAppID is the app ID for the GitHub App.
+	githubAppID = "GITHUB_APP_ID"
+	// githubAppInstallationID is the installation ID for the GitHub App.
+	githubAppInstallationID = "GITHUB_APP_INSTALLATION_ID"
 )
 
-func readGitHubTokens() (string, bool) {
-	for _, name := range GithubAuthTokens {
-		if token, exists := os.LookupEnv(name); exists && token != "" {
-			return token, exists
-		}
-	}
-	return "", false
-}
-
 // NewTransport returns a configured http.Transport for use with GitHub.
 func NewTransport(ctx context.Context, logger *zap.SugaredLogger) http.RoundTripper {
 	transport := http.DefaultTransport
 
 	// nolint
-	if token, exists := readGitHubTokens(); exists {
+	if tokenAccessor := tokens.MakeTokenAccessor(); tokenAccessor != nil {
 		// Use GitHub PAT
-		transport = MakeGitHubTransport(transport, strings.Split(token, ","))
-	} else if keyPath := os.Getenv(GithubAppKeyPath); keyPath != "" { // Also try a GITHUB_APP
-		appID, err := strconv.Atoi(os.Getenv(GithubAppID))
+		transport = makeGitHubTransport(transport, tokenAccessor)
+	} else if keyPath := os.Getenv(githubAppKeyPath); keyPath != "" { // Also try a GITHUB_APP
+		appID, err := strconv.Atoi(os.Getenv(githubAppID))
 		if err != nil {
 			log.Panic(err)
 		}
-		installationID, err := strconv.Atoi(os.Getenv(GithubAppInstallationID))
+		installationID, err := strconv.Atoi(os.Getenv(githubAppInstallationID))
 		if err != nil {
 			log.Panic(err)
 		}

clients/githubrepo/roundtripper/tokens/accessor.go

@@ -0,0 +1,52 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tokens defines interface to access GitHub PATs.
+package tokens
+
+import (
+	"os"
+	"strings"
+)
+
+// githubAuthServer is the RPC URL for the token server.
+const githubAuthServer = "GITHUB_AUTH_SERVER"
+
+// TokenAccessor interface defines a `retrieve-once` data structure.
+// Implementations of this interface must be thread-safe.
+type TokenAccessor interface {
+	Next() (uint64, string)
+	Release(uint64)
+}
+
+func readGitHubTokens() (string, bool) {
+	githubAuthTokens := []string{"GITHUB_AUTH_TOKEN", "GITHUB_TOKEN", "GH_TOKEN", "GH_AUTH_TOKEN"}
+	for _, name := range githubAuthTokens {
+		if token, exists := os.LookupEnv(name); exists && token != "" {
+			return token, exists
+		}
+	}
+	return "", false
+}
+
+// MakeTokenAccessor is a factory function of TokenAccessor.
+func MakeTokenAccessor() TokenAccessor {
+	if value, exists := readGitHubTokens(); exists {
+		return makeRoundRobinAccessor(strings.Split(value, ","))
+	}
+	if value, exists := os.LookupEnv(githubAuthServer); exists {
+		return makeRPCAccessor(value)
+	}
+	return nil
+}

clients/githubrepo/roundtripper/tokens/round_robin.go

@@ -0,0 +1,61 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tokens
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+const expiryTimeInSec = 30
+
+// roundRobinAccessor implements TokenAccessor.
+type roundRobinAccessor struct {
+	accessTokens []string
+	accessState  []int64
+	counter      uint64
+}
+
+// Next implements TokenAccessor.Next.
+func (tokens *roundRobinAccessor) Next() (uint64, string) {
+	c := atomic.AddUint64(&tokens.counter, 1)
+	l := len(tokens.accessTokens)
+	index := c % uint64(l)
+
+	// If selected accessToken is unavailable, wait.
+	for !atomic.CompareAndSwapInt64(&tokens.accessState[index], 0, time.Now().Unix()) {
+		currVal := tokens.accessState[index]
+		expired := time.Now().After(time.Unix(currVal, 0).Add(expiryTimeInSec * time.Second))
+		if !expired {
+			continue
+		}
+		if atomic.CompareAndSwapInt64(&tokens.accessState[index], currVal, time.Now().Unix()) {
+			break
+		}
+	}
+	return index, tokens.accessTokens[index]
+}
+
+// Release implements TokenAccessor.Release.
+func (tokens *roundRobinAccessor) Release(id uint64) {
+	atomic.SwapInt64(&tokens.accessState[id], 0)
+}
+
+func makeRoundRobinAccessor(accessTokens []string) TokenAccessor {
+	return &roundRobinAccessor{
+		accessTokens: accessTokens,
+		accessState:  make([]int64, len(accessTokens)),
+	}
+}

clients/githubrepo/roundtripper/tokens/rpc.go

@@ -0,0 +1,50 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tokens
+
+// Token is used for GitHub token server RPC request/response.
+type Token struct {
+	Value string
+	ID    uint64
+}
+
+// TokenOverRPC is an RPC handler implementation for Golang RPCs.
+type TokenOverRPC struct {
+	client TokenAccessor
+}
+
+// Next requests for the next available GitHub token.
+// Server blocks the call until a token becomes available.
+func (accessor *TokenOverRPC) Next(args struct{}, token *Token) error {
+	id, val := accessor.client.Next()
+	*token = Token{
+		ID:    id,
+		Value: val,
+	}
+	return nil
+}
+
+// Release inserts the token at `index` back into the token pool to be used by another client.
+func (accessor *TokenOverRPC) Release(id uint64, reply *struct{}) error {
+	accessor.client.Release(id)
+	return nil
+}
+
+// NewTokenOverRPC creates a new instance of TokenOverRPC.
+func NewTokenOverRPC(client TokenAccessor) *TokenOverRPC {
+	return &TokenOverRPC{
+		client: client,
+	}
+}

clients/githubrepo/roundtripper/tokens/rpc_client.go

@@ -0,0 +1,52 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tokens
+
+import (
+	"log"
+	"net/rpc"
+)
+
+// rpcAccessor implements TokenAccessor.
+type rpcAccessor struct {
+	client *rpc.Client
+}
+
+// Next implements TokenAccessor.Next.
+func (accessor *rpcAccessor) Next() (uint64, string) {
+	var token Token
+	if err := accessor.client.Call("TokenOverRPC.Next", struct{}{}, &token); err != nil {
+		log.Printf("error during RPC call Next: %v", err)
+		return 0, ""
+	}
+	return token.ID, token.Value
+}
+
+// Release implements TokenAccessor.Release.
+func (accessor *rpcAccessor) Release(id uint64) {
+	if err := accessor.client.Call("TokenOverRPC.Release", id, struct{}{}); err != nil {
+		log.Printf("error during RPC call Release: %v", err)
+	}
+}
+
+func makeRPCAccessor(serverURL string) TokenAccessor {
+	client, err := rpc.DialHTTP("tcp", serverURL)
+	if err != nil {
+		panic(err)
+	}
+	return &rpcAccessor{
+		client: client,
+	}
+}

clients/githubrepo/roundtripper/tokens/server/Dockerfile

@@ -0,0 +1,29 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang@sha256:3c4de86eec9cbc619cdd72424abd88326ffcf5d813a8338a7743c55e5898734f AS base
+WORKDIR /src
+ENV CGO_ENABLED=0
+COPY go.* ./
+RUN go mod download
+COPY . ./
+
+FROM base AS authserver
+ARG TARGETOS
+ARG TARGETARCH
+RUN CGO_ENABLED=0 make build-github-server
+
+FROM gcr.io/distroless/base:nonroot@sha256:a74f307185001c69bc362a40dbab7b67d410a872678132b187774fa21718fa13
+COPY --from=authserver /src/clients/githubrepo/roundtripper/tokens/server/github-auth-server clients/githubrepo/roundtripper/tokens/server/github-auth-server
+ENTRYPOINT ["clients/githubrepo/roundtripper/tokens/server/github-auth-server"]

clients/githubrepo/roundtripper/tokens/server/cloudbuild.yaml

@@ -0,0 +1,21 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '.',
+    '-t', 'gcr.io/openssf/scorecard-github-server:latest',
+    '-t', 'gcr.io/openssf/scorecard-github-server:$COMMIT_SHA',
+    '-f', 'clients/githubrepo/roundtripper/tokens/server/Dockerfile']
+images: ['gcr.io/openssf/scorecard-github-server']

clients/githubrepo/roundtripper/tokens/server/main.go

@@ -0,0 +1,47 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main implements the GitHub token server.
+package main
+
+import (
+	"net"
+	"net/http"
+	"net/rpc"
+
+	"github.com/ossf/scorecard/v3/clients/githubrepo/roundtripper/tokens"
+)
+
+func main() {
+	// Sanity check
+	tokenAccessor := tokens.MakeTokenAccessor()
+	if tokenAccessor == nil {
+		panic("")
+	}
+
+	rpcAccessor := tokens.NewTokenOverRPC(tokenAccessor)
+	if err := rpc.Register(rpcAccessor); err != nil {
+		panic(err)
+	}
+	rpc.HandleHTTP()
+
+	l, err := net.Listen("tcp", "127.0.0.1:8080")
+	if err != nil {
+		panic(err)
+	}
+
+	if err := http.Serve(l, nil); err != nil {
+		panic(err)
+	}
+}

clients/githubrepo/roundtripper/transport.go

@@ -18,41 +18,33 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"sync/atomic"
-	"time"
 
 	"go.opencensus.io/stats"
 	"go.opencensus.io/tag"
 
+	"github.com/ossf/scorecard/v3/clients/githubrepo/roundtripper/tokens"
 	githubstats "github.com/ossf/scorecard/v3/clients/githubrepo/stats"
 )
 
-const expiryTimeInSec = 30
-
-// MakeGitHubTransport wraps input RoundTripper with GitHub authorization logic.
-func MakeGitHubTransport(innerTransport http.RoundTripper, accessTokens []string) http.RoundTripper {
+// makeGitHubTransport wraps input RoundTripper with GitHub authorization logic.
+func makeGitHubTransport(innerTransport http.RoundTripper, accessor tokens.TokenAccessor) http.RoundTripper {
 	return &githubTransport{
 		innerTransport: innerTransport,
-		tokens:         makeTokenAccessor(accessTokens),
+		tokens:         accessor,
 	}
 }
 
 // githubTransport handles authorization using GitHub personal access tokens (PATs) during HTTP requests.
 type githubTransport struct {
 	innerTransport http.RoundTripper
-	tokens         tokenAccessor
-}
-
-type tokenAccessor interface {
-	next() (uint64, string)
-	release(uint64)
+	tokens         tokens.TokenAccessor
 }
 
 func (gt *githubTransport) RoundTrip(r *http.Request) (*http.Response, error) {
-	index, token := gt.tokens.next()
-	defer gt.tokens.release(index)
+	id, token := gt.tokens.Next()
+	defer gt.tokens.Release(id)
 
-	ctx, err := tag.New(r.Context(), tag.Upsert(githubstats.TokenIndex, fmt.Sprint(index)))
+	ctx, err := tag.New(r.Context(), tag.Upsert(githubstats.TokenIndex, fmt.Sprint(id)))
 	if err != nil {
 		return nil, fmt.Errorf("error updating context: %w", err)
 	}
@@ -74,39 +66,3 @@ func (gt *githubTransport) RoundTrip(r *http.Request) (*http.Response, error) {
 	}
 	return resp, nil
 }
-
-func makeTokenAccessor(accessTokens []string) tokenAccessor {
-	return &roundRobinAccessor{
-		accessTokens: accessTokens,
-		accessState:  make([]int64, len(accessTokens)),
-	}
-}
-
-type roundRobinAccessor struct {
-	accessTokens []string
-	accessState  []int64
-	counter      uint64
-}
-
-func (roundRobin *roundRobinAccessor) next() (uint64, string) {
-	c := atomic.AddUint64(&roundRobin.counter, 1)
-	l := len(roundRobin.accessTokens)
-	index := c % uint64(l)
-
-	// If selected accessToken is unavailable, wait.
-	for !atomic.CompareAndSwapInt64(&roundRobin.accessState[index], 0, time.Now().Unix()) {
-		currVal := roundRobin.accessState[index]
-		expired := time.Now().After(time.Unix(currVal, 0).Add(expiryTimeInSec * time.Second))
-		if !expired {
-			continue
-		}
-		if atomic.CompareAndSwapInt64(&roundRobin.accessState[index], currVal, time.Now().Unix()) {
-			break
-		}
-	}
-	return index, roundRobin.accessTokens[index]
-}
-
-func (roundRobin *roundRobinAccessor) release(index uint64) {
-	atomic.SwapInt64(&roundRobin.accessState[index], 0)
-}

cron/k8s/auth.yaml

@@ -0,0 +1,56 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: scorecard-github-server
+spec:
+  selector:
+    app.kubernetes.io/name: github-auth-server
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 8080
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: github-auth-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: github-auth-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: github-auth-server
+    spec:
+      containers:
+      - name: github-auth-server
+        image: gcr.io/openssf/scorecard-github-server:stable
+        imagePullPolicy: Always
+        env:
+        - name: GITHUB_AUTH_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: github
+              key: token
+  strategy:
+    type: "RollingUpdate"
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0

cron/webhook/main.go

@@ -36,6 +36,7 @@ var images = []string{
 	"gcr.io/openssf/scorecard-batch-controller",
 	"gcr.io/openssf/scorecard-batch-worker",
 	"gcr.io/openssf/scorecard-bq-transfer",
+	"gcr.io/openssf/scorecard-github-server",
 }
 
 func scriptHandler(w http.ResponseWriter, r *http.Request) {