mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-19 21:18:09 +03:00
137 lines
3.7 KiB
Go
137 lines
3.7 KiB
Go
// Copyright 2021 Security Scorecard Authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package data
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"gocloud.dev/blob"
|
|
|
|
// Needed to link in GCP drivers.
|
|
_ "gocloud.dev/blob/gcsblob"
|
|
|
|
"github.com/ossf/scorecard/cron/config"
|
|
)
|
|
|
|
const (
|
|
// filePrefixFormat uses ISO 8601 standard, i.e - YYYY-MM-DDTHH:MM:SS.
|
|
// This format guarantees that lexicographically sorted files are chronologically sorted.
|
|
filePrefixFormat = "2006.01.02/150405/"
|
|
)
|
|
|
|
var (
|
|
errShortBlobName = errors.New("input key length is shorter than expected")
|
|
errParseBlobName = errors.New("error parsing input blob name")
|
|
)
|
|
|
|
func GetBlobKeys(ctx context.Context, bucketURL string) ([]string, error) {
|
|
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error from blob.OpenBucket: %w", err)
|
|
}
|
|
defer bucket.Close()
|
|
|
|
keys := make([]string, 0)
|
|
iter := bucket.List(nil)
|
|
for {
|
|
next, err := iter.Next(ctx)
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error during iter.Next: %w", err)
|
|
}
|
|
keys = append(keys, next.Key)
|
|
}
|
|
return keys, nil
|
|
}
|
|
|
|
func GetBlobContent(ctx context.Context, bucketURL, key string) ([]byte, error) {
|
|
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error from blob.OpenBucket: %w", err)
|
|
}
|
|
defer bucket.Close()
|
|
|
|
keyData, err := bucket.ReadAll(ctx, key)
|
|
if err != nil {
|
|
return keyData, fmt.Errorf("error during bucket.ReadAll: %w", err)
|
|
}
|
|
return keyData, nil
|
|
}
|
|
|
|
func BlobExists(ctx context.Context, bucketURL, key string) (bool, error) {
|
|
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
|
if err != nil {
|
|
return false, fmt.Errorf("error from blob.OpenBucket: %w", err)
|
|
}
|
|
defer bucket.Close()
|
|
|
|
ret, err := bucket.Exists(ctx, key)
|
|
if err != nil {
|
|
return ret, fmt.Errorf("error during bucket.Exists: %w", err)
|
|
}
|
|
return ret, nil
|
|
}
|
|
|
|
func WriteToBlobStore(ctx context.Context, bucketURL, filename string, data []byte) error {
|
|
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
|
if err != nil {
|
|
return fmt.Errorf("error from blob.OpenBucket: %w", err)
|
|
}
|
|
defer bucket.Close()
|
|
|
|
blobWriter, err := bucket.NewWriter(ctx, filename, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("error from bucket.NewWriter: %w", err)
|
|
}
|
|
if _, err = blobWriter.Write(data); err != nil {
|
|
return fmt.Errorf("error from blobWriter.Write: %w", err)
|
|
}
|
|
if err := blobWriter.Close(); err != nil {
|
|
return fmt.Errorf("error from blobWriter.Close: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func GetBlobFilename(filename string, datetime time.Time) string {
|
|
return datetime.Format(filePrefixFormat) + filename
|
|
}
|
|
|
|
func GetShardNumFilename(datetime time.Time) string {
|
|
return GetBlobFilename(config.ShardNumFilename, datetime)
|
|
}
|
|
|
|
func GetTransferStatusFilename(datetime time.Time) string {
|
|
return GetBlobFilename(config.TransferStatusFilename, datetime)
|
|
}
|
|
|
|
func ParseBlobFilename(key string) (time.Time, string, error) {
|
|
if len(key) < len(filePrefixFormat) {
|
|
return time.Now(), "", fmt.Errorf("%w: %s", errShortBlobName, key)
|
|
}
|
|
prefix := key[:len(filePrefixFormat)]
|
|
objectName := key[len(filePrefixFormat):]
|
|
t, err := time.Parse(filePrefixFormat, prefix)
|
|
if err != nil {
|
|
return t, "", fmt.Errorf("%w: %v", errParseBlobName, err)
|
|
}
|
|
return t, objectName, nil
|
|
}
|