mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-17 11:57:12 +03:00
🌱 implement basic rate limiting for best practices worker. (#4090)
We are getting connection reset requests from bestpractices.dev and 429 errors from our GCS bucket for too many writes. The GCS limit (1000 QPS) is much higher, so just use the bestpractices.dev limit of 1 QPS. https://github.com/coreinfrastructure/best-practices-badge/blob/main/docs/api.md The construct was taken from https://go.dev/wiki/RateLimiting which "works well for rates up to tens of operations per second." Signed-off-by: Spencer Schrock <sschrock@google.com>
This commit is contained in:
parent
256d5a3b50
commit
cc7132d631
@ -24,6 +24,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ossf/scorecard/v5/clients"
|
||||
"github.com/ossf/scorecard/v5/cron/config"
|
||||
@ -95,9 +96,13 @@ func main() {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
throttle := time.NewTicker(time.Second) // bestpractices.dev wants 1 QPS
|
||||
defer throttle.Stop()
|
||||
|
||||
pageNum := 1
|
||||
pageResp, err := getPage(ctx, pageNum)
|
||||
for err == nil && len(pageResp) > 0 {
|
||||
<-throttle.C
|
||||
if err := writeToCIIDataBucket(ctx, pageResp, ciiDataBucket); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user