mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-17 11:57:12 +03:00
parent
fc7157e38a
commit
8fdb0e767e
@ -36,20 +36,17 @@ const (
|
|||||||
// TransferStatusFilename file identifies if shard transfer to BigQuery is completed.
|
// TransferStatusFilename file identifies if shard transfer to BigQuery is completed.
|
||||||
TransferStatusFilename string = ".transfer_complete"
|
TransferStatusFilename string = ".transfer_complete"
|
||||||
projectID string = "SCORECARD_PROJECT_ID"
|
projectID string = "SCORECARD_PROJECT_ID"
|
||||||
resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL"
|
|
||||||
requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL"
|
requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL"
|
||||||
requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
|
requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
|
||||||
bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET"
|
bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET"
|
||||||
bigqueryTable string = "SCORECARD_BIGQUERY_TABLE"
|
|
||||||
completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD"
|
completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD"
|
||||||
shardSize string = "SCORECARD_SHARD_SIZE"
|
shardSize string = "SCORECARD_SHARD_SIZE"
|
||||||
webhookURL string = "SCORECARD_WEBHOOK_URL"
|
webhookURL string = "SCORECARD_WEBHOOK_URL"
|
||||||
metricExporter string = "SCORECARD_METRIC_EXPORTER"
|
metricExporter string = "SCORECARD_METRIC_EXPORTER"
|
||||||
ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL"
|
ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL"
|
||||||
blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS"
|
blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS"
|
||||||
// v2 results.
|
bigqueryTable string = "SCORECARD_BIGQUERY_TABLE"
|
||||||
bigqueryTableV2 string = "SCORECARD_BIGQUERY_TABLEV2"
|
resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL"
|
||||||
resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2"
|
|
||||||
// Raw results.
|
// Raw results.
|
||||||
rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE"
|
rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE"
|
||||||
rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL"
|
rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL"
|
||||||
@ -64,7 +61,7 @@ var (
|
|||||||
configYAML []byte
|
configYAML []byte
|
||||||
)
|
)
|
||||||
|
|
||||||
//nolint
|
// nolint
|
||||||
type config struct {
|
type config struct {
|
||||||
ProjectID string `yaml:"project-id"`
|
ProjectID string `yaml:"project-id"`
|
||||||
ResultDataBucketURL string `yaml:"result-data-bucket-url"`
|
ResultDataBucketURL string `yaml:"result-data-bucket-url"`
|
||||||
@ -78,9 +75,6 @@ type config struct {
|
|||||||
BlacklistedChecks string `yaml:"blacklisted-checks"`
|
BlacklistedChecks string `yaml:"blacklisted-checks"`
|
||||||
MetricExporter string `yaml:"metric-exporter"`
|
MetricExporter string `yaml:"metric-exporter"`
|
||||||
ShardSize int `yaml:"shard-size"`
|
ShardSize int `yaml:"shard-size"`
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
ResultDataBucketURLV2 string `yaml:"result-data-bucket-url-v2"`
|
|
||||||
BigQueryTableV2 string `yaml:"bigquery-table-v2"`
|
|
||||||
// Raw results.
|
// Raw results.
|
||||||
RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
|
RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
|
||||||
RawBigQueryTable string `yaml:"raw-bigquery-table"`
|
RawBigQueryTable string `yaml:"raw-bigquery-table"`
|
||||||
@ -190,18 +184,6 @@ func GetCompletionThreshold() (float64, error) {
|
|||||||
return getFloat64ConfigValue(completionThreshold, configYAML, "CompletionThreshold", "completion-threshold")
|
return getFloat64ConfigValue(completionThreshold, configYAML, "CompletionThreshold", "completion-threshold")
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBigQueryTableV2 returns the table name to transfer cron job results.
|
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
func GetBigQueryTableV2() (string, error) {
|
|
||||||
return getStringConfigValue(bigqueryTableV2, configYAML, "BigQueryTableV2", "bigquery-table-v2")
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetResultDataBucketURLV2 returns the bucketURL for storing cron job results.
|
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
func GetResultDataBucketURLV2() (string, error) {
|
|
||||||
return getStringConfigValue(resultDataBucketURLV2, configYAML, "ResultDataBucketURLV2", "result-data-bucket-url-v2")
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetRawBigQueryTable returns the table name to transfer cron job results.
|
// GetRawBigQueryTable returns the table name to transfer cron job results.
|
||||||
func GetRawBigQueryTable() (string, error) {
|
func GetRawBigQueryTable() (string, error) {
|
||||||
return getStringConfigValue(rawBigqueryTable, configYAML,
|
return getStringConfigValue(rawBigqueryTable, configYAML,
|
||||||
|
@ -13,11 +13,10 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
project-id: openssf
|
project-id: openssf
|
||||||
result-data-bucket-url: gs://ossf-scorecard-data
|
|
||||||
request-topic-url: gcppubsub://projects/openssf/topics/scorecard-batch-requests
|
request-topic-url: gcppubsub://projects/openssf/topics/scorecard-batch-requests
|
||||||
request-subscription-url: gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker
|
request-subscription-url: gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker
|
||||||
bigquery-dataset: scorecardcron
|
bigquery-dataset: scorecardcron
|
||||||
bigquery-table: scorecard
|
bigquery-table: scorecard-v2
|
||||||
completion-threshold: 0.99
|
completion-threshold: 0.99
|
||||||
shard-size: 10
|
shard-size: 10
|
||||||
webhook-url:
|
webhook-url:
|
||||||
@ -26,9 +25,7 @@ cii-data-bucket-url: gs://ossf-scorecard-cii-data
|
|||||||
# TODO(#859): Re-add Contributors after fixing inconsistencies.
|
# TODO(#859): Re-add Contributors after fixing inconsistencies.
|
||||||
blacklisted-checks: SAST,CI-Tests,Contributors
|
blacklisted-checks: SAST,CI-Tests,Contributors
|
||||||
metric-exporter: stackdriver
|
metric-exporter: stackdriver
|
||||||
# UPGRADEv2: to remove.
|
result-data-bucket-url: gs://ossf-scorecard-data2
|
||||||
result-data-bucket-url-v2: gs://ossf-scorecard-data2
|
|
||||||
bigquery-table-v2: scorecard-v2
|
|
||||||
# Raw results.
|
# Raw results.
|
||||||
raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
|
raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
|
||||||
raw-bigquery-table: scorecard-rawdata
|
raw-bigquery-table: scorecard-rawdata
|
||||||
|
@ -25,20 +25,17 @@ import (
|
|||||||
const (
|
const (
|
||||||
testEnvVar string = "TEST_ENV_VAR"
|
testEnvVar string = "TEST_ENV_VAR"
|
||||||
prodProjectID = "openssf"
|
prodProjectID = "openssf"
|
||||||
prodBucket = "gs://ossf-scorecard-data"
|
prodBucket = "gs://ossf-scorecard-data2"
|
||||||
prodTopic = "gcppubsub://projects/openssf/topics/scorecard-batch-requests"
|
prodTopic = "gcppubsub://projects/openssf/topics/scorecard-batch-requests"
|
||||||
prodSubscription = "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker"
|
prodSubscription = "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker"
|
||||||
prodBigQueryDataset = "scorecardcron"
|
prodBigQueryDataset = "scorecardcron"
|
||||||
prodBigQueryTable = "scorecard"
|
prodBigQueryTable = "scorecard-v2"
|
||||||
prodCompletionThreshold = 0.99
|
prodCompletionThreshold = 0.99
|
||||||
prodWebhookURL = ""
|
prodWebhookURL = ""
|
||||||
prodCIIDataBucket = "gs://ossf-scorecard-cii-data"
|
prodCIIDataBucket = "gs://ossf-scorecard-cii-data"
|
||||||
prodBlacklistedChecks = "SAST,CI-Tests,Contributors"
|
prodBlacklistedChecks = "SAST,CI-Tests,Contributors"
|
||||||
prodShardSize int = 10
|
prodShardSize int = 10
|
||||||
prodMetricExporter string = "stackdriver"
|
prodMetricExporter string = "stackdriver"
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
prodBucketV2 = "gs://ossf-scorecard-data2"
|
|
||||||
prodBigQueryTableV2 = "scorecard-v2"
|
|
||||||
// Raw results.
|
// Raw results.
|
||||||
prodRawBucket = "gs://ossf-scorecard-rawdata"
|
prodRawBucket = "gs://ossf-scorecard-rawdata"
|
||||||
prodRawBigQueryTable = "scorecard-rawdata"
|
prodRawBigQueryTable = "scorecard-rawdata"
|
||||||
@ -75,9 +72,6 @@ func TestYAMLParsing(t *testing.T) {
|
|||||||
BlacklistedChecks: prodBlacklistedChecks,
|
BlacklistedChecks: prodBlacklistedChecks,
|
||||||
ShardSize: prodShardSize,
|
ShardSize: prodShardSize,
|
||||||
MetricExporter: prodMetricExporter,
|
MetricExporter: prodMetricExporter,
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
ResultDataBucketURLV2: prodBucketV2,
|
|
||||||
BigQueryTableV2: prodBigQueryTableV2,
|
|
||||||
RawResultDataBucketURL: prodRawBucket,
|
RawResultDataBucketURL: prodRawBucket,
|
||||||
RawBigQueryTable: prodRawBigQueryTable,
|
RawBigQueryTable: prodRawBigQueryTable,
|
||||||
},
|
},
|
||||||
|
@ -118,11 +118,6 @@ func main() {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
bucket2, err := config.GetResultDataBucketURLV2()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rawBucket, err := config.GetRawResultDataBucketURL()
|
rawBucket, err := config.GetRawResultDataBucketURL()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
@ -150,17 +145,6 @@ func main() {
|
|||||||
panic(fmt.Errorf("error writing to BlobStore: %w", err))
|
panic(fmt.Errorf("error writing to BlobStore: %w", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// UPGRADEv2: to remove.
|
|
||||||
*metadata.ShardLoc = bucket2 + "/" + data.GetBlobFilename("", t)
|
|
||||||
metadataJSON, err = protojson.Marshal(&metadata)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Errorf("error during protojson.Marshal2: %w", err))
|
|
||||||
}
|
|
||||||
err = data.WriteToBlobStore(ctx, bucket2, data.GetShardMetadataFilename(t), metadataJSON)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Errorf("error writing to BlobStore2: %w", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Raw data.
|
// Raw data.
|
||||||
*metadata.ShardLoc = rawBucket + "/" + data.GetBlobFilename("", t)
|
*metadata.ShardLoc = rawBucket + "/" + data.GetBlobFilename("", t)
|
||||||
metadataJSON, err = protojson.Marshal(&metadata)
|
metadataJSON, err = protojson.Marshal(&metadata)
|
||||||
|
@ -93,7 +93,6 @@ func AsJSON(r *pkg.ScorecardResult, showDetails bool, logLevel log.Level, writer
|
|||||||
Metadata: r.Metadata,
|
Metadata: r.Metadata,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for _, checkResult := range r.Checks {
|
for _, checkResult := range r.Checks {
|
||||||
tmpResult := jsonCheckResult{
|
tmpResult := jsonCheckResult{
|
||||||
Name: checkResult.Name,
|
Name: checkResult.Name,
|
||||||
@ -142,7 +141,6 @@ func AsJSON2(r *pkg.ScorecardResult, showDetails bool,
|
|||||||
AggregateScore: jsonFloatScore(score),
|
AggregateScore: jsonFloatScore(score),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for _, checkResult := range r.Checks {
|
for _, checkResult := range r.Checks {
|
||||||
doc, e := checkDocs.GetCheck(checkResult.Name)
|
doc, e := checkDocs.GetCheck(checkResult.Name)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
|
@ -33,9 +33,4 @@ spec:
|
|||||||
memory: 1Gi
|
memory: 1Gi
|
||||||
requests:
|
requests:
|
||||||
memory: 1Gi
|
memory: 1Gi
|
||||||
env:
|
|
||||||
- name: SCORECARD_BIGQUERY_TABLE
|
|
||||||
value: "scorecard-v2"
|
|
||||||
- name: SCORECARD_DATA_BUCKET_URL
|
|
||||||
value: "gs://ossf-scorecard-data2"
|
|
||||||
restartPolicy: OnFailure
|
restartPolicy: OnFailure
|
||||||
|
@ -36,6 +36,8 @@ spec:
|
|||||||
value: "scorecard_releasetest2"
|
value: "scorecard_releasetest2"
|
||||||
- name: SCORECARD_COMPLETION_THRESHOLD
|
- name: SCORECARD_COMPLETION_THRESHOLD
|
||||||
value: "0.9"
|
value: "0.9"
|
||||||
|
- name: SCORECARD_WEBHOOK_URL
|
||||||
|
value: "http://10.4.8.246/"
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
memory: 1Gi
|
memory: 1Gi
|
||||||
|
@ -1,45 +0,0 @@
|
|||||||
# Copyright 2021 Security Scorecard Authors
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
apiVersion: batch/v1
|
|
||||||
kind: CronJob
|
|
||||||
metadata:
|
|
||||||
name: scorecard-bq-transfer-releasetest
|
|
||||||
spec:
|
|
||||||
# Daily at 9am and 6pm PST.
|
|
||||||
schedule: "0 1,16 * * *"
|
|
||||||
concurrencyPolicy: "Forbid"
|
|
||||||
jobTemplate:
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
restartPolicy: OnFailure
|
|
||||||
containers:
|
|
||||||
- name: bq-transfer-releasetest
|
|
||||||
image: gcr.io/openssf/scorecard-bq-transfer:latest
|
|
||||||
imagePullPolicy: Always
|
|
||||||
env:
|
|
||||||
- name: SCORECARD_DATA_BUCKET_URL
|
|
||||||
value: "gs://ossf-scorecard-data-releasetest"
|
|
||||||
- name: SCORECARD_BIGQUERY_TABLE
|
|
||||||
value: "scorecard_releasetest"
|
|
||||||
- name: SCORECARD_COMPLETION_THRESHOLD
|
|
||||||
value: "0.9"
|
|
||||||
- name: SCORECARD_WEBHOOK_URL
|
|
||||||
value: "http://10.4.8.246/"
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: 1Gi
|
|
||||||
requests:
|
|
||||||
memory: 1Gi
|
|
@ -1,36 +0,0 @@
|
|||||||
# Copyright 2021 Security Scorecard Authors
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
apiVersion: batch/v1
|
|
||||||
kind: CronJob
|
|
||||||
metadata:
|
|
||||||
name: scorecard-bq-transfer
|
|
||||||
spec:
|
|
||||||
# At 02:00UTC on Monday and Thursday.
|
|
||||||
schedule: "0 2 * * 1,4"
|
|
||||||
concurrencyPolicy: "Forbid"
|
|
||||||
jobTemplate:
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: bq-transfer
|
|
||||||
image: gcr.io/openssf/scorecard-bq-transfer:stable
|
|
||||||
imagePullPolicy: Always
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: 1Gi
|
|
||||||
requests:
|
|
||||||
memory: 1Gi
|
|
||||||
restartPolicy: OnFailure
|
|
@ -33,16 +33,11 @@ spec:
|
|||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
env:
|
env:
|
||||||
- name: SCORECARD_DATA_BUCKET_URL
|
- name: SCORECARD_DATA_BUCKET_URL
|
||||||
value: "gs://ossf-scorecard-data-releasetest"
|
|
||||||
# UPGRADEv2: to remove.
|
|
||||||
- name: SCORECARD_DATA_BUCKET_URLV2
|
|
||||||
value: "gs://ossf-scorecard-data-releasetest2"
|
value: "gs://ossf-scorecard-data-releasetest2"
|
||||||
- name: RAW_SCORECARD_DATA_BUCKET_URL
|
- name: RAW_SCORECARD_DATA_BUCKET_URL
|
||||||
value: "gs://ossf-scorecard-rawdata-releasetest"
|
value: "gs://ossf-scorecard-rawdata-releasetest"
|
||||||
- name: SCORECARD_REQUEST_SUBSCRIPTION_URL
|
- name: SCORECARD_REQUEST_SUBSCRIPTION_URL
|
||||||
value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest"
|
value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest"
|
||||||
- name: SCORECARD_BLACKLISTED_CHECKS
|
|
||||||
value: "SAST,CI-Tests,Contributors"
|
|
||||||
- name: SCORECARD_METRIC_EXPORTER
|
- name: SCORECARD_METRIC_EXPORTER
|
||||||
value: "printer"
|
value: "printer"
|
||||||
- name: GITHUB_AUTH_SERVER
|
- name: GITHUB_AUTH_SERVER
|
||||||
|
@ -48,7 +48,7 @@ var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to tru
|
|||||||
// nolint: gocognit
|
// nolint: gocognit
|
||||||
func processRequest(ctx context.Context,
|
func processRequest(ctx context.Context,
|
||||||
batchRequest *data.ScorecardBatchRequest,
|
batchRequest *data.ScorecardBatchRequest,
|
||||||
blacklistedChecks []string, bucketURL, bucketURL2, rawBucketURL string,
|
blacklistedChecks []string, bucketURL, rawBucketURL string,
|
||||||
checkDocs docs.Doc,
|
checkDocs docs.Doc,
|
||||||
repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
|
repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
|
||||||
ciiClient clients.CIIBestPracticesClient,
|
ciiClient clients.CIIBestPracticesClient,
|
||||||
@ -59,22 +59,17 @@ func processRequest(ctx context.Context,
|
|||||||
fmt.Sprintf("shard-%07d", batchRequest.GetShardNum()),
|
fmt.Sprintf("shard-%07d", batchRequest.GetShardNum()),
|
||||||
batchRequest.GetJobTime().AsTime())
|
batchRequest.GetJobTime().AsTime())
|
||||||
// Sanity check - make sure we are not re-processing an already processed request.
|
// Sanity check - make sure we are not re-processing an already processed request.
|
||||||
exists1, err := data.BlobExists(ctx, bucketURL, filename)
|
existsScore, err := data.BlobExists(ctx, bucketURL, filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error during BlobExists: %w", err)
|
return fmt.Errorf("error during BlobExists: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
exists2, err := data.BlobExists(ctx, bucketURL2, filename)
|
existsRaw, err := data.BlobExists(ctx, rawBucketURL, filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error during BlobExists: %w", err)
|
return fmt.Errorf("error during BlobExists: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
exists3, err := data.BlobExists(ctx, rawBucketURL, filename)
|
if existsScore && existsRaw {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error during BlobExists: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if exists1 && exists2 && exists3 {
|
|
||||||
logger.Info(fmt.Sprintf("Already processed shard %s. Nothing to do.", filename))
|
logger.Info(fmt.Sprintf("Already processed shard %s. Nothing to do.", filename))
|
||||||
// We have already processed this request, nothing to do.
|
// We have already processed this request, nothing to do.
|
||||||
return nil
|
return nil
|
||||||
@ -140,7 +135,7 @@ func processRequest(ctx context.Context,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := data.WriteToBlobStore(ctx, bucketURL2, filename, buffer2.Bytes()); err != nil {
|
if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer2.Bytes()); err != nil {
|
||||||
return fmt.Errorf("error during WriteToBlobStore2: %w", err)
|
return fmt.Errorf("error during WriteToBlobStore2: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -197,11 +192,6 @@ func main() {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
bucketURL2, err := config.GetResultDataBucketURLV2()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rawBucketURL, err := config.GetRawResultDataBucketURL()
|
rawBucketURL, err := config.GetRawResultDataBucketURL()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
@ -252,7 +242,7 @@ func main() {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err := processRequest(ctx, req, blacklistedChecks,
|
if err := processRequest(ctx, req, blacklistedChecks,
|
||||||
bucketURL, bucketURL2, rawBucketURL, checkDocs,
|
bucketURL, rawBucketURL, checkDocs,
|
||||||
repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
|
repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
|
||||||
// TODO(log): Previously Warn. Consider logging an error here.
|
// TODO(log): Previously Warn. Consider logging an error here.
|
||||||
logger.Info(fmt.Sprintf("error processing request: %v", err))
|
logger.Info(fmt.Sprintf("error processing request: %v", err))
|
||||||
|
Loading…
Reference in New Issue
Block a user