diff --git a/api/storage.go b/api/storage.go index 7fca5ccf..d4518354 100644 --- a/api/storage.go +++ b/api/storage.go @@ -1,8 +1,16 @@ package api -type Storage struct { - ID int `json:"id"` - Name string `json:"name"` +type StorageType string + +const ( + StorageS3 StorageType = "S3" +) + +type StorageConfig struct { + S3Config *StorageS3Config `json:"s3Config"` +} + +type StorageS3Config struct { EndPoint string `json:"endPoint"` Region string `json:"region"` AccessKey string `json:"accessKey"` @@ -11,30 +19,28 @@ type Storage struct { URLPrefix string `json:"urlPrefix"` } +type Storage struct { + ID int `json:"id"` + Name string `json:"name"` + Type StorageType `json:"type"` + Config *StorageConfig `json:"config"` +} + type StorageCreate struct { - Name string `json:"name"` - EndPoint string `json:"endPoint"` - Region string `json:"region"` - AccessKey string `json:"accessKey"` - SecretKey string `json:"secretKey"` - Bucket string `json:"bucket"` - URLPrefix string `json:"urlPrefix"` + Name string `json:"name"` + Type StorageType `json:"type"` + Config *StorageConfig `json:"config"` } type StoragePatch struct { - ID int `json:"id"` - Name *string `json:"name"` - EndPoint *string `json:"endPoint"` - Region *string `json:"region"` - AccessKey *string `json:"accessKey"` - SecretKey *string `json:"secretKey"` - Bucket *string `json:"bucket"` - URLPrefix *string `json:"urlPrefix"` + ID int `json:"id"` + Type StorageType `json:"type"` + Name *string `json:"name"` + Config *StorageConfig `json:"config"` } type StorageFind struct { - ID *int `json:"id"` - Name *string `json:"name"` + ID *int `json:"id"` } type StorageDelete struct { diff --git a/plugin/storage/s3/s3.go b/plugin/storage/s3/s3.go index 3092a663..b9f5fd7e 100644 --- a/plugin/storage/s3/s3.go +++ b/plugin/storage/s3/s3.go @@ -6,31 +6,38 @@ import ( "io" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" + s3config "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/usememos/memos/api" ) -type Client struct { - Client *awss3.Client - BucketName string - URLPrefix string +type Config struct { + AccessKey string + SecretKey string + Bucket string + EndPoint string + Region string + URLPrefix string } -func NewClient(ctx context.Context, storage *api.Storage) (*Client, error) { +type Client struct { + Client *awss3.Client + Config *Config +} + +func NewClient(ctx context.Context, config *Config) (*Client, error) { resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { return aws.Endpoint{ - URL: storage.EndPoint, - SigningRegion: storage.Region, + URL: config.EndPoint, + SigningRegion: config.Region, }, nil }) - cfg, err := config.LoadDefaultConfig(ctx, - config.WithEndpointResolverWithOptions(resolver), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(storage.AccessKey, storage.SecretKey, "")), + cfg, err := s3config.LoadDefaultConfig(ctx, + s3config.WithEndpointResolverWithOptions(resolver), + s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKey, config.SecretKey, "")), ) if err != nil { return nil, err @@ -39,16 +46,15 @@ func NewClient(ctx context.Context, storage *api.Storage) (*Client, error) { client := awss3.NewFromConfig(cfg) return &Client{ - Client: client, - BucketName: storage.Bucket, - URLPrefix: storage.URLPrefix, + Client: client, + Config: config, }, nil } -func (client *Client) UploadFile(ctx context.Context, filename string, fileType string, src io.Reader, storage *api.Storage) (string, error) { +func (client *Client) UploadFile(ctx context.Context, filename string, fileType string, src io.Reader) (string, error) { uploader := manager.NewUploader(client.Client) resp, err := uploader.Upload(ctx, &awss3.PutObjectInput{ - Bucket: aws.String(client.BucketName), + Bucket: aws.String(client.Config.Bucket), Key: aws.String(filename), Body: src, ContentType: aws.String(fileType), @@ -58,10 +64,10 @@ func (client *Client) UploadFile(ctx context.Context, filename string, fileType return "", err } var link string - if storage.URLPrefix == "" { + if client.Config.URLPrefix == "" { link = resp.Location } else { - link = fmt.Sprintf("%s/%s", storage.URLPrefix, filename) + link = fmt.Sprintf("%s/%s", client.Config.URLPrefix, filename) } return link, nil } diff --git a/server/resource.go b/server/resource.go index fe44cb26..d816f4f8 100644 --- a/server/resource.go +++ b/server/resource.go @@ -111,20 +111,32 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) } - s3client, err := s3.NewClient(ctx, storage) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to new s3 client").SetInternal(err) - } + if storage.Type == api.StorageS3 { + s3Config := storage.Config.S3Config + s3client, err := s3.NewClient(ctx, &s3.Config{ + AccessKey: s3Config.AccessKey, + SecretKey: s3Config.SecretKey, + EndPoint: s3Config.EndPoint, + Region: s3Config.Region, + Bucket: s3Config.Bucket, + URLPrefix: s3Config.URLPrefix, + }) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to new s3 client").SetInternal(err) + } - link, err := s3client.UploadFile(ctx, filename, filetype, src, storage) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to upload via s3 client").SetInternal(err) - } - resourceCreate = &api.ResourceCreate{ - CreatorID: userID, - Filename: filename, - Type: filetype, - ExternalLink: link, + link, err := s3client.UploadFile(ctx, filename, filetype, src) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to upload via s3 client").SetInternal(err) + } + resourceCreate = &api.ResourceCreate{ + CreatorID: userID, + Filename: filename, + Type: filetype, + ExternalLink: link, + } + } else { + return echo.NewHTTPError(http.StatusInternalServerError, "Unsupported storage type") } } diff --git a/store/db/migration/dev/LATEST__SCHEMA.sql b/store/db/migration/dev/LATEST__SCHEMA.sql index d3dd8717..b2448a1e 100644 --- a/store/db/migration/dev/LATEST__SCHEMA.sql +++ b/store/db/migration/dev/LATEST__SCHEMA.sql @@ -108,19 +108,15 @@ CREATE TABLE activity ( -- storage CREATE TABLE storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL DEFAULT '' UNIQUE, - end_point TEXT NOT NULL DEFAULT '', - region TEXT NOT NULL DEFAULT '', - access_key TEXT NOT NULL DEFAULT '', - secret_key TEXT NOT NULL DEFAULT '', - bucket TEXT NOT NULL DEFAULT '', - url_prefix TEXT NOT NULL DEFAULT '' + name TEXT NOT NULL, + type TEXT NOT NULL, + config TEXT NOT NULL DEFAULT '{}' ); -- idp CREATE TABLE idp ( id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL, type TEXT NOT NULL, identifier_filter TEXT NOT NULL DEFAULT '', config TEXT NOT NULL DEFAULT '{}' diff --git a/store/storage.go b/store/storage.go index 6c9706bc..9bc777b4 100644 --- a/store/storage.go +++ b/store/storage.go @@ -3,6 +3,7 @@ package store import ( "context" "database/sql" + "encoding/json" "fmt" "strings" @@ -11,26 +12,18 @@ import ( ) type storageRaw struct { - ID int - Name string - EndPoint string - Region string - AccessKey string - SecretKey string - Bucket string - URLPrefix string + ID int + Name string + Type api.StorageType + Config *api.StorageConfig } func (raw *storageRaw) toStorage() *api.Storage { return &api.Storage{ - ID: raw.ID, - Name: raw.Name, - EndPoint: raw.EndPoint, - Region: raw.Region, - AccessKey: raw.AccessKey, - SecretKey: raw.SecretKey, - Bucket: raw.Bucket, - URLPrefix: raw.URLPrefix, + ID: raw.ID, + Name: raw.Name, + Type: raw.Type, + Config: raw.Config, } } @@ -131,27 +124,36 @@ func (s *Store) DeleteStorage(ctx context.Context, delete *api.StorageDelete) er } func createStorageRaw(ctx context.Context, tx *sql.Tx, create *api.StorageCreate) (*storageRaw, error) { - set := []string{"name", "end_point", "region", "access_key", "secret_key", "bucket", "url_prefix"} - args := []interface{}{create.Name, create.EndPoint, create.Region, create.AccessKey, create.SecretKey, create.Bucket, create.URLPrefix} - placeholder := []string{"?", "?", "?", "?", "?", "?", "?"} + set := []string{"name", "type", "config"} + args := []interface{}{create.Name, create.Type} + placeholder := []string{"?", "?", "?"} + + var configBytes []byte + var err error + if create.Type == api.StorageS3 { + configBytes, err = json.Marshal(create.Config.S3Config) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("unsupported storage type %s", string(create.Type)) + } + args = append(args, string(configBytes)) query := ` INSERT INTO storage ( ` + strings.Join(set, ", ") + ` ) VALUES (` + strings.Join(placeholder, ",") + `) - RETURNING id, name, end_point, region, access_key, secret_key, bucket, url_prefix + RETURNING id ` - var storageRaw storageRaw + storageRaw := storageRaw{ + Name: create.Name, + Type: create.Type, + Config: create.Config, + } if err := tx.QueryRowContext(ctx, query, args...).Scan( &storageRaw.ID, - &storageRaw.Name, - &storageRaw.EndPoint, - &storageRaw.Region, - &storageRaw.AccessKey, - &storageRaw.SecretKey, - &storageRaw.Bucket, - &storageRaw.URLPrefix, ); err != nil { return nil, FormatError(err) } @@ -164,47 +166,48 @@ func patchStorageRaw(ctx context.Context, tx *sql.Tx, patch *api.StoragePatch) ( if v := patch.Name; v != nil { set, args = append(set, "name = ?"), append(args, *v) } - if v := patch.EndPoint; v != nil { - set, args = append(set, "end_point = ?"), append(args, *v) + if v := patch.Config; v != nil { + var configBytes []byte + var err error + if patch.Type == api.StorageS3 { + configBytes, err = json.Marshal(patch.Config.S3Config) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("unsupported storage type %s", string(patch.Type)) + } + set, args = append(set, "config = ?"), append(args, string(configBytes)) } - if v := patch.Region; v != nil { - set, args = append(set, "region = ?"), append(args, *v) - } - if v := patch.AccessKey; v != nil { - set, args = append(set, "access_key = ?"), append(args, *v) - } - if v := patch.SecretKey; v != nil { - set, args = append(set, "secret_key = ?"), append(args, *v) - } - if v := patch.Bucket; v != nil { - set, args = append(set, "bucket = ?"), append(args, *v) - } - if v := patch.URLPrefix; v != nil { - set, args = append(set, "url_prefix = ?"), append(args, *v) - } - args = append(args, patch.ID) query := ` UPDATE storage SET ` + strings.Join(set, ", ") + ` WHERE id = ? - RETURNING id, name, end_point, region, access_key, secret_key, bucket, url_prefix + RETURNING id, name, type, config ` - var storageRaw storageRaw + var storageConfig string if err := tx.QueryRowContext(ctx, query, args...).Scan( &storageRaw.ID, &storageRaw.Name, - &storageRaw.EndPoint, - &storageRaw.Region, - &storageRaw.AccessKey, - &storageRaw.SecretKey, - &storageRaw.Bucket, - &storageRaw.URLPrefix, + &storageRaw.Type, + &storageConfig, ); err != nil { return nil, FormatError(err) } + if storageRaw.Type == api.StorageS3 { + s3Config := &api.StorageS3Config{} + if err := json.Unmarshal([]byte(storageConfig), s3Config); err != nil { + return nil, err + } + storageRaw.Config = &api.StorageConfig{ + S3Config: s3Config, + } + } else { + return nil, fmt.Errorf("unsupported storage type %s", string(storageRaw.Type)) + } return &storageRaw, nil } @@ -215,20 +218,13 @@ func findStorageRawList(ctx context.Context, tx *sql.Tx, find *api.StorageFind) if v := find.ID; v != nil { where, args = append(where, "id = ?"), append(args, *v) } - if v := find.Name; v != nil { - where, args = append(where, "name = ?"), append(args, *v) - } query := ` SELECT id, name, - end_point, - region, - access_key, - secret_key, - bucket, - url_prefix + type, + config FROM storage WHERE ` + strings.Join(where, " AND ") + ` ORDER BY id DESC @@ -242,19 +238,26 @@ func findStorageRawList(ctx context.Context, tx *sql.Tx, find *api.StorageFind) storageRawList := make([]*storageRaw, 0) for rows.Next() { var storageRaw storageRaw + var storageConfig string if err := rows.Scan( &storageRaw.ID, &storageRaw.Name, - &storageRaw.EndPoint, - &storageRaw.Region, - &storageRaw.AccessKey, - &storageRaw.SecretKey, - &storageRaw.Bucket, - &storageRaw.URLPrefix, + &storageRaw.Type, + &storageConfig, ); err != nil { return nil, FormatError(err) } - + if storageRaw.Type == api.StorageS3 { + s3Config := &api.StorageS3Config{} + if err := json.Unmarshal([]byte(storageConfig), s3Config); err != nil { + return nil, err + } + storageRaw.Config = &api.StorageConfig{ + S3Config: s3Config, + } + } else { + return nil, fmt.Errorf("unsupported storage type %s", string(storageRaw.Type)) + } storageRawList = append(storageRawList, &storageRaw) } diff --git a/web/src/components/CreateStorageServiceDialog.tsx b/web/src/components/CreateStorageServiceDialog.tsx index 7a0a0c01..6d078a55 100644 --- a/web/src/components/CreateStorageServiceDialog.tsx +++ b/web/src/components/CreateStorageServiceDialog.tsx @@ -7,15 +7,18 @@ import Icon from "./Icon"; import toastHelper from "./Toast"; interface Props extends DialogProps { - storage?: Storage; + storage?: ObjectStorage; confirmCallback?: () => void; } const CreateStorageServiceDialog: React.FC = (props: Props) => { const { destroy, storage, confirmCallback } = props; const { t } = useTranslation(); - const [storageCreate, setStorageCreate] = useState({ + const [basicInfo, setBasicInfo] = useState({ name: "", + }); + const [type, setType] = useState("S3"); + const [s3Config, setS3Config] = useState({ endPoint: "", region: "", accessKey: "", @@ -27,7 +30,13 @@ const CreateStorageServiceDialog: React.FC = (props: Props) => { useEffect(() => { if (storage) { - setStorageCreate({ ...storage }); + setBasicInfo({ + name: storage.name, + }); + setType(storage.type); + if (storage.type === "S3") { + setS3Config(storage.config.s3Config); + } } }, []); @@ -36,27 +45,35 @@ const CreateStorageServiceDialog: React.FC = (props: Props) => { }; const allowConfirmAction = () => { - if ( - storageCreate.name === "" || - storageCreate.endPoint === "" || - storageCreate.region === "" || - storageCreate.accessKey === "" || - storageCreate.bucket === "" || - storageCreate.bucket === "" - ) { + if (basicInfo.name === "") { return false; } + if (type === "S3") { + if (s3Config.endPoint === "" || s3Config.region === "" || s3Config.accessKey === "" || s3Config.bucket === "") { + return false; + } + } return true; }; const handleConfirmBtnClick = async () => { try { if (isCreating) { - await api.createStorage(storageCreate); + await api.createStorage({ + ...basicInfo, + type: type, + config: { + s3Config: s3Config, + }, + }); } else { await api.patchStorage({ id: storage.id, - ...storageCreate, + type: type, + ...basicInfo, + config: { + s3Config: s3Config, + }, }); } } catch (error: any) { @@ -69,59 +86,10 @@ const CreateStorageServiceDialog: React.FC = (props: Props) => { destroy(); }; - const handleNameChange = (event: React.ChangeEvent) => { - const name = event.target.value; - setStorageCreate({ - ...storageCreate, - name, - }); - }; - - const handleEndPointChange = (event: React.ChangeEvent) => { - const endPoint = event.target.value; - setStorageCreate({ - ...storageCreate, - endPoint, - }); - }; - - const handleRegionChange = (event: React.ChangeEvent) => { - const region = event.target.value; - setStorageCreate({ - ...storageCreate, - region, - }); - }; - - const handleAccessKeyChange = (event: React.ChangeEvent) => { - const accessKey = event.target.value; - setStorageCreate({ - ...storageCreate, - accessKey, - }); - }; - - const handleSecretKeyChange = (event: React.ChangeEvent) => { - const secretKey = event.target.value; - setStorageCreate({ - ...storageCreate, - secretKey, - }); - }; - - const handleBucketChange = (event: React.ChangeEvent) => { - const bucket = event.target.value; - setStorageCreate({ - ...storageCreate, - bucket, - }); - }; - - const handleURLPrefixChange = (event: React.ChangeEvent) => { - const urlPrefix = event.target.value; - setStorageCreate({ - ...storageCreate, - urlPrefix, + const setPartialS3Config = (state: Partial) => { + setS3Config({ + ...s3Config, + ...state, }); }; @@ -140,37 +108,84 @@ const CreateStorageServiceDialog: React.FC = (props: Props) => { Name (Unique identifier) - + + setBasicInfo({ + ...basicInfo, + name: e.target.value, + }) + } + fullWidth + /> EndPoint (S3-compatible server URL) - + setPartialS3Config({ endPoint: e.target.value })} + fullWidth + /> Region (Region name) - + setPartialS3Config({ region: e.target.value })} + fullWidth + /> AccessKey (Access Key / Access ID) - + setPartialS3Config({ accessKey: e.target.value })} + fullWidth + /> SecretKey (Secret Key / Secret Access Key) - + setPartialS3Config({ secretKey: e.target.value })} + fullWidth + /> Bucket (Bucket name) - + setPartialS3Config({ bucket: e.target.value })} + fullWidth + /> URLPrefix (Custom URL prefix; Optional) - + setPartialS3Config({ urlPrefix: e.target.value })} + fullWidth + />