diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 5549ce686d..031ecb6a0a 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -9,12 +9,12 @@ }, "scripts": { "build": "tsc", - "start": "node --loader ts-node/esm.mjs --es-module-specifier-resolution node ./src/index.ts", + "start": "node --loader ts-node/esm/transpile-only.mjs --es-module-specifier-resolution node ./src/index.ts", "dev": "nodemon ./src/index.ts", "test": "ava --concurrency 1 --serial", "test:coverage": "c8 ava --concurrency 1 --serial", "postinstall": "prisma generate", - "data-migration": "node --loader ts-node/esm.mjs --es-module-specifier-resolution node ./src/data/app.ts", + "data-migration": "node --loader ts-node/esm/transpile-only.mjs --es-module-specifier-resolution node ./src/data/app.ts", "predeploy": "yarn prisma migrate deploy && node --es-module-specifier-resolution node ./dist/data/app.js run" }, "dependencies": { diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma index d2518476b0..f19af17404 100644 --- a/packages/backend/server/schema.prisma +++ b/packages/backend/server/schema.prisma @@ -196,6 +196,7 @@ model VerificationToken { @@map("verificationtokens") } +// deprecated, use [ObjectStorage] model Blob { id Int @id @default(autoincrement()) @db.Integer hash String @db.VarChar @@ -210,6 +211,7 @@ model Blob { @@map("blobs") } +// deprecated, use [ObjectStorage] model OptimizedBlob { id Int @id @default(autoincrement()) @db.Integer hash String @db.VarChar diff --git a/packages/backend/server/src/affine.config.ts b/packages/backend/server/src/affine.config.ts index 394d8ed0ee..ae879a4132 100644 --- a/packages/backend/server/src/affine.config.ts +++ b/packages/backend/server/src/affine.config.ts @@ -15,6 +15,8 @@ if (node.prod && env.R2_OBJECT_STORAGE_ACCOUNT_ID) { }; AFFiNE.storage.storages.avatar.provider = 'r2'; AFFiNE.storage.storages.avatar.bucket = 'account-avatar'; + AFFiNE.storage.storages.avatar.publicLinkFactory = key => + `https://avatar.affineassets.com/${key}`; AFFiNE.storage.storages.blob.provider = 'r2'; AFFiNE.storage.storages.blob.bucket = 'workspace-blobs'; diff --git a/packages/backend/server/src/app.ts b/packages/backend/server/src/app.ts index a1f068f026..89a4019800 100644 --- a/packages/backend/server/src/app.ts +++ b/packages/backend/server/src/app.ts @@ -9,7 +9,6 @@ import { BusinessModules } from './modules'; import { AuthModule } from './modules/auth'; import { PrismaModule } from './prisma'; import { SessionModule } from './session'; -import { StorageModule } from './storage'; import { RateLimiterModule } from './throttler'; const BasicModules = [ @@ -17,7 +16,6 @@ const BasicModules = [ ConfigModule.forRoot(), CacheModule, EventModule, - StorageModule.forRoot(), SessionModule, RateLimiterModule, AuthModule, diff --git a/packages/backend/server/src/config/def.ts b/packages/backend/server/src/config/def.ts index e09d6db80c..a3f9c2288f 100644 --- a/packages/backend/server/src/config/def.ts +++ b/packages/backend/server/src/config/def.ts @@ -172,32 +172,6 @@ export interface AFFiNEConfig { */ storage: AFFiNEStorageConfig; - /** - * object storage Config - * - * all artifacts and logs will be stored on instance disk, - * and can not shared between instances if not configured - * @deprecated use `storage` instead - */ - objectStorage: { - /** - * whether use remote object storage - */ - r2: { - enabled: boolean; - accountId: string; - bucket: string; - accessKeyId: string; - secretAccessKey: string; - }; - /** - * Only used when `enable` is `false` - */ - fs: { - path: string; - }; - }; - /** * Rate limiter config */ diff --git a/packages/backend/server/src/config/default.ts b/packages/backend/server/src/config/default.ts index f0aa7e2926..ec3233cfbf 100644 --- a/packages/backend/server/src/config/default.ts +++ b/packages/backend/server/src/config/default.ts @@ -1,8 +1,6 @@ /// import { createPrivateKey, createPublicKey } from 'node:crypto'; -import { homedir } from 'node:os'; -import { join } from 'node:path'; import parse from 'parse-duration'; @@ -177,18 +175,6 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { }, }, storage: getDefaultAFFiNEStorageConfig(), - objectStorage: { - r2: { - enabled: false, - bucket: '', - accountId: '', - accessKeyId: '', - secretAccessKey: '', - }, - fs: { - path: join(homedir(), '.affine-storage'), - }, - }, rateLimiter: { ttl: 60, limit: 60, diff --git a/packages/backend/server/src/config/storage/index.ts b/packages/backend/server/src/config/storage/index.ts index 6d3e9c5ab2..aa90ab76fb 100644 --- a/packages/backend/server/src/config/storage/index.ts +++ b/packages/backend/server/src/config/storage/index.ts @@ -12,10 +12,10 @@ export type R2StorageConfig = S3ClientConfigType & { }; export type S3StorageConfig = S3ClientConfigType; -export type StorageTargetConfig = { +export type StorageTargetConfig = { provider: StorageProviderType; bucket: string; -}; +} & Ext; export interface AFFiNEStorageConfig { /** @@ -29,7 +29,7 @@ export interface AFFiNEStorageConfig { r2?: R2StorageConfig; }; storages: { - avatar: StorageTargetConfig; + avatar: StorageTargetConfig<{ publicLinkFactory: (key: string) => string }>; blob: StorageTargetConfig; }; } @@ -48,6 +48,7 @@ export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig { avatar: { provider: 'fs', bucket: 'avatars', + publicLinkFactory: key => `/api/avatars/${key}`, }, blob: { provider: 'fs', diff --git a/packages/backend/server/src/event/events.ts b/packages/backend/server/src/event/events.ts index 025d664663..f1e4f1e466 100644 --- a/packages/backend/server/src/event/events.ts +++ b/packages/backend/server/src/event/events.ts @@ -1,10 +1,16 @@ -import type { Snapshot, Workspace } from '@prisma/client'; +import type { Snapshot, User, Workspace } from '@prisma/client'; import { Flatten, Payload } from './types'; interface EventDefinitions { workspace: { deleted: Payload; + blob: { + deleted: Payload<{ + workspaceId: Workspace['id']; + name: string; + }>; + }; }; snapshot: { @@ -15,6 +21,10 @@ interface EventDefinitions { >; deleted: Payload>; }; + + user: { + deleted: Payload; + }; } export type EventKV = Flatten; diff --git a/packages/backend/server/src/index.ts b/packages/backend/server/src/index.ts index 1f42851a94..187dd1f920 100644 --- a/packages/backend/server/src/index.ts +++ b/packages/backend/server/src/index.ts @@ -5,7 +5,6 @@ startAutoMetrics(); import { NestFactory } from '@nestjs/core'; import type { NestExpressApplication } from '@nestjs/platform-express'; import cookieParser from 'cookie-parser'; -import { static as staticMiddleware } from 'express'; import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs'; import { AppModule } from './app'; @@ -43,10 +42,6 @@ const config = app.get(Config); const host = config.node.prod ? '0.0.0.0' : 'localhost'; const port = config.port ?? 3010; -if (!config.objectStorage.r2.enabled) { - app.use('/assets', staticMiddleware(config.objectStorage.fs.path)); -} - if (config.redis.enabled) { const redisIoAdapter = new RedisIoAdapter(app); await redisIoAdapter.connectToRedis( diff --git a/packages/backend/server/src/modules/index.ts b/packages/backend/server/src/modules/index.ts index 0b22eeb069..c2edffda85 100644 --- a/packages/backend/server/src/modules/index.ts +++ b/packages/backend/server/src/modules/index.ts @@ -8,6 +8,7 @@ import { DocModule } from './doc'; import { PaymentModule } from './payment'; import { QuotaModule } from './quota'; import { SelfHostedModule } from './self-hosted'; +import { StorageModule } from './storage'; import { SyncModule } from './sync'; import { UsersModule } from './users'; import { WorkspaceModule } from './workspaces'; @@ -27,7 +28,8 @@ switch (SERVER_FLAVOR) { WorkspaceModule, UsersModule, SyncModule, - DocModule + DocModule, + StorageModule ); break; case 'graphql': @@ -39,7 +41,8 @@ switch (SERVER_FLAVOR) { UsersModule, DocModule, PaymentModule, - QuotaModule + QuotaModule, + StorageModule ); break; case 'allinone': @@ -53,7 +56,8 @@ switch (SERVER_FLAVOR) { QuotaModule, SyncModule, DocModule, - PaymentModule + PaymentModule, + StorageModule ); break; } diff --git a/packages/backend/server/src/modules/quota/index.ts b/packages/backend/server/src/modules/quota/index.ts index 59ac207e37..2fc8c8edd5 100644 --- a/packages/backend/server/src/modules/quota/index.ts +++ b/packages/backend/server/src/modules/quota/index.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; +import { StorageModule } from '../storage'; import { PermissionService } from '../workspaces/permission'; import { QuotaService } from './service'; import { QuotaManagementService } from './storage'; @@ -11,6 +12,7 @@ import { QuotaManagementService } from './storage'; * - quota statistics */ @Module({ + imports: [StorageModule], providers: [PermissionService, QuotaService, QuotaManagementService], exports: [QuotaService, QuotaManagementService], }) diff --git a/packages/backend/server/src/modules/quota/storage.ts b/packages/backend/server/src/modules/quota/storage.ts index fff2dd50df..f754c6576c 100644 --- a/packages/backend/server/src/modules/quota/storage.ts +++ b/packages/backend/server/src/modules/quota/storage.ts @@ -1,7 +1,6 @@ -import type { Storage } from '@affine/storage'; -import { Inject, Injectable, NotFoundException } from '@nestjs/common'; +import { Injectable, NotFoundException } from '@nestjs/common'; -import { StorageProvide } from '../../storage'; +import { WorkspaceBlobStorage } from '../storage'; import { PermissionService } from '../workspaces/permission'; import { QuotaService } from './service'; @@ -10,7 +9,7 @@ export class QuotaManagementService { constructor( private readonly quota: QuotaService, private readonly permissions: PermissionService, - @Inject(StorageProvide) private readonly storage: Storage + private readonly storage: WorkspaceBlobStorage ) {} async getUserQuota(userId: string) { @@ -29,7 +28,12 @@ export class QuotaManagementService { // TODO: lazy calc, need to be optimized with cache async getUserUsage(userId: string) { const workspaces = await this.permissions.getOwnedWorkspaces(userId); - return this.storage.blobsSize(workspaces); + + const sizes = await Promise.all( + workspaces.map(workspace => this.storage.totalSize(workspace)) + ); + + return sizes.reduce((total, size) => total + size, 0); } // get workspace's owner quota and total size of used diff --git a/packages/backend/server/src/modules/storage/fs.ts b/packages/backend/server/src/modules/storage/fs.ts deleted file mode 100644 index af82527d8a..0000000000 --- a/packages/backend/server/src/modules/storage/fs.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { createWriteStream } from 'node:fs'; -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; -import { pipeline } from 'node:stream/promises'; - -import { Injectable } from '@nestjs/common'; - -import { Config } from '../../config'; -import { FileUpload } from '../../types'; - -@Injectable() -export class FSService { - constructor(private readonly config: Config) {} - - async writeFile(key: string, file: FileUpload) { - const dest = this.config.objectStorage.fs.path; - const fileName = `${key}-${randomUUID()}`; - const prefix = this.config.node.dev - ? `${this.config.https ? 'https' : 'http'}://${this.config.host}:${ - this.config.port - }` - : ''; - await mkdir(dest, { recursive: true }); - const destFile = join(dest, fileName); - await pipeline(file.createReadStream(), createWriteStream(destFile)); - - return `${prefix}/assets/${fileName}`; - } -} diff --git a/packages/backend/server/src/modules/storage/index.ts b/packages/backend/server/src/modules/storage/index.ts index 1a21758549..071931e24f 100644 --- a/packages/backend/server/src/modules/storage/index.ts +++ b/packages/backend/server/src/modules/storage/index.ts @@ -1,11 +1,11 @@ import { Module } from '@nestjs/common'; -import { FSService } from './fs'; -import { S3 } from './s3'; -import { StorageService } from './storage.service'; +import { AvatarStorage, WorkspaceBlobStorage } from './wrappers'; @Module({ - providers: [S3, StorageService, FSService], - exports: [StorageService], + providers: [WorkspaceBlobStorage, AvatarStorage], + exports: [WorkspaceBlobStorage, AvatarStorage], }) export class StorageModule {} + +export { AvatarStorage, WorkspaceBlobStorage }; diff --git a/packages/backend/server/src/modules/storage/providers/fs.ts b/packages/backend/server/src/modules/storage/providers/fs.ts index 6b4b70d3a3..3ad735d2e4 100644 --- a/packages/backend/server/src/modules/storage/providers/fs.ts +++ b/packages/backend/server/src/modules/storage/providers/fs.ts @@ -34,6 +34,8 @@ export class FsStorageProvider implements StorageProvider { private readonly path: string; private readonly logger: Logger; + readonly type = 'fs'; + constructor( config: FsStorageConfig, public readonly bucket: string diff --git a/packages/backend/server/src/modules/storage/providers/provider.ts b/packages/backend/server/src/modules/storage/providers/provider.ts index f6663843d6..95740d5a62 100644 --- a/packages/backend/server/src/modules/storage/providers/provider.ts +++ b/packages/backend/server/src/modules/storage/providers/provider.ts @@ -1,5 +1,7 @@ import type { Readable } from 'node:stream'; +import { StorageProviderType } from '../../../config'; + export interface GetObjectMetadata { /** * @default 'application/octet-stream' @@ -26,6 +28,7 @@ export type BlobInputType = Buffer | Readable | string; export type BlobOutputType = Readable; export interface StorageProvider { + readonly type: StorageProviderType; put( key: string, body: BlobInputType, diff --git a/packages/backend/server/src/modules/storage/providers/r2.ts b/packages/backend/server/src/modules/storage/providers/r2.ts index 670a59ba20..791e152002 100644 --- a/packages/backend/server/src/modules/storage/providers/r2.ts +++ b/packages/backend/server/src/modules/storage/providers/r2.ts @@ -2,6 +2,8 @@ import { R2StorageConfig } from '../../../config/storage'; import { S3StorageProvider } from './s3'; export class R2StorageProvider extends S3StorageProvider { + override readonly type = 'r2' as any /* cast 'r2' to 's3' */; + constructor(config: R2StorageConfig, bucket: string) { super( { diff --git a/packages/backend/server/src/modules/storage/providers/s3.ts b/packages/backend/server/src/modules/storage/providers/s3.ts index 4597ad0127..241d7272fc 100644 --- a/packages/backend/server/src/modules/storage/providers/s3.ts +++ b/packages/backend/server/src/modules/storage/providers/s3.ts @@ -21,8 +21,11 @@ import { import { autoMetadata, toBuffer } from './utils'; export class S3StorageProvider implements StorageProvider { - logger: Logger; - client: S3Client; + private readonly logger: Logger; + protected client: S3Client; + + readonly type = 's3'; + constructor( config: S3StorageConfig, public readonly bucket: string diff --git a/packages/backend/server/src/modules/storage/s3.ts b/packages/backend/server/src/modules/storage/s3.ts deleted file mode 100644 index c849c5aa26..0000000000 --- a/packages/backend/server/src/modules/storage/s3.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { S3Client } from '@aws-sdk/client-s3'; -import { FactoryProvider } from '@nestjs/common'; - -import { Config } from '../../config'; - -export const S3_SERVICE = Symbol('S3_SERVICE'); - -export const S3: FactoryProvider = { - provide: S3_SERVICE, - useFactory: (config: Config) => { - const s3 = new S3Client({ - region: 'auto', - endpoint: `https://${config.objectStorage.r2.accountId}.r2.cloudflarestorage.com`, - credentials: { - accessKeyId: config.objectStorage.r2.accessKeyId, - secretAccessKey: config.objectStorage.r2.secretAccessKey, - }, - }); - return s3; - }, - inject: [Config], -}; diff --git a/packages/backend/server/src/modules/storage/storage.service.ts b/packages/backend/server/src/modules/storage/storage.service.ts deleted file mode 100644 index bb714b00aa..0000000000 --- a/packages/backend/server/src/modules/storage/storage.service.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { Inject, Injectable } from '@nestjs/common'; -import { crc32 } from '@node-rs/crc32'; -import { fileTypeFromBuffer } from 'file-type'; -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore - no types -import { getStreamAsBuffer } from 'get-stream'; - -import { Config } from '../../config'; -import { FileUpload } from '../../types'; -import { FSService } from './fs'; -import { S3_SERVICE } from './s3'; - -@Injectable() -export class StorageService { - constructor( - @Inject(S3_SERVICE) private readonly s3: S3Client, - private readonly fs: FSService, - private readonly config: Config - ) {} - - async uploadFile(key: string, file: FileUpload) { - if (this.config.objectStorage.r2.enabled) { - const readableFile = file.createReadStream(); - const fileBuffer = await getStreamAsBuffer(readableFile); - const mime = (await fileTypeFromBuffer(fileBuffer))?.mime; - const crc32Value = crc32(fileBuffer); - const keyWithCrc32 = `${crc32Value}-${key}`; - await this.s3.send( - new PutObjectCommand({ - Body: fileBuffer, - Bucket: this.config.objectStorage.r2.bucket, - Key: keyWithCrc32, - ContentLength: fileBuffer.length, - ContentType: mime, - }) - ); - return `https://avatar.affineassets.com/${keyWithCrc32}`; - } else { - return this.fs.writeFile(key, file); - } - } -} diff --git a/packages/backend/server/src/modules/storage/wrappers/avatar.ts b/packages/backend/server/src/modules/storage/wrappers/avatar.ts index edc6e0a8c5..6622631bd8 100644 --- a/packages/backend/server/src/modules/storage/wrappers/avatar.ts +++ b/packages/backend/server/src/modules/storage/wrappers/avatar.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; -import { Config } from '../../../config'; +import { AFFiNEStorageConfig, Config } from '../../../config'; +import { type EventPayload, OnEvent } from '../../../event'; import { BlobInputType, createStorageProvider, @@ -11,20 +12,36 @@ import { @Injectable() export class AvatarStorage { public readonly provider: StorageProvider; + private readonly storageConfig: AFFiNEStorageConfig['storages']['avatar']; - constructor({ storage }: Config) { - this.provider = createStorageProvider(storage, 'avatar'); + constructor(private readonly config: Config) { + this.provider = createStorageProvider(this.config.storage, 'avatar'); + this.storageConfig = this.config.storage.storages.avatar; } - put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) { - return this.provider.put(key, blob, metadata); + async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) { + await this.provider.put(key, blob, metadata); + let link = this.storageConfig.publicLinkFactory(key); + + if (link.startsWith('/')) { + link = this.config.baseUrl + link; + } + + return link; } get(key: string) { return this.provider.get(key); } - async delete(key: string) { + delete(key: string) { return this.provider.delete(key); } + + @OnEvent('user.deleted') + async onUserDeleted(user: EventPayload<'user.deleted'>) { + if (user.avatarUrl) { + await this.delete(user.avatarUrl); + } + } } diff --git a/packages/backend/server/src/modules/storage/wrappers/blob.ts b/packages/backend/server/src/modules/storage/wrappers/blob.ts index 1d24526fe0..254a2801af 100644 --- a/packages/backend/server/src/modules/storage/wrappers/blob.ts +++ b/packages/backend/server/src/modules/storage/wrappers/blob.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { Config } from '../../../config'; +import { EventEmitter, type EventPayload, OnEvent } from '../../../event'; import { BlobInputType, createStorageProvider, @@ -10,7 +11,10 @@ import { @Injectable() export class WorkspaceBlobStorage { public readonly provider: StorageProvider; - constructor({ storage }: Config) { + constructor( + private readonly event: EventEmitter, + { storage }: Config + ) { this.provider = createStorageProvider(storage, 'blob'); } @@ -42,4 +46,25 @@ export class WorkspaceBlobStorage { // how could we ignore the ones get soft-deleted? return blobs.reduce((acc, item) => acc + item.size, 0); } + + @OnEvent('workspace.deleted') + async onWorkspaceDeleted(workspaceId: EventPayload<'workspace.deleted'>) { + const blobs = await this.list(workspaceId); + + // to reduce cpu time holding + blobs.forEach(blob => { + this.event.emit('workspace.blob.deleted', { + workspaceId: workspaceId, + name: blob.key, + }); + }); + } + + @OnEvent('workspace.blob.deleted') + async onDeleteWorkspaceBlob({ + workspaceId, + name, + }: EventPayload<'workspace.blob.deleted'>) { + await this.delete(workspaceId, name); + } } diff --git a/packages/backend/server/src/modules/users/controller.ts b/packages/backend/server/src/modules/users/controller.ts new file mode 100644 index 0000000000..388ced7f9e --- /dev/null +++ b/packages/backend/server/src/modules/users/controller.ts @@ -0,0 +1,40 @@ +import { + Controller, + ForbiddenException, + Get, + NotFoundException, + Param, + Res, +} from '@nestjs/common'; +import type { Response } from 'express'; + +import { AvatarStorage } from '../storage'; + +@Controller('/api/avatars') +export class UserAvatarController { + constructor(private readonly storage: AvatarStorage) {} + + @Get('/:id') + async getAvatar(@Res() res: Response, @Param('id') id: string) { + if (this.storage.provider.type !== 'fs') { + throw new ForbiddenException( + 'Only available when avatar storage provider set to fs.' + ); + } + + const { body, metadata } = await this.storage.get(id); + + if (!body) { + throw new NotFoundException(`Avatar ${id} not found.`); + } + + // metadata should always exists if body is not null + if (metadata) { + res.setHeader('content-type', metadata.contentType); + res.setHeader('last-modified', metadata.lastModified.toISOString()); + res.setHeader('content-length', metadata.contentLength); + } + + body.pipe(res); + } +} diff --git a/packages/backend/server/src/modules/users/index.ts b/packages/backend/server/src/modules/users/index.ts index e48f44e7b7..ee9dda42d8 100644 --- a/packages/backend/server/src/modules/users/index.ts +++ b/packages/backend/server/src/modules/users/index.ts @@ -3,12 +3,14 @@ import { Module } from '@nestjs/common'; import { FeatureModule } from '../features'; import { QuotaModule } from '../quota'; import { StorageModule } from '../storage'; +import { UserAvatarController } from './controller'; import { UserResolver } from './resolver'; import { UsersService } from './users'; @Module({ imports: [StorageModule, FeatureModule, QuotaModule], providers: [UserResolver, UsersService], + controllers: [UserAvatarController], exports: [UsersService], }) export class UsersModule {} diff --git a/packages/backend/server/src/modules/users/resolver.ts b/packages/backend/server/src/modules/users/resolver.ts index d816c111d2..22a54e44fc 100644 --- a/packages/backend/server/src/modules/users/resolver.ts +++ b/packages/backend/server/src/modules/users/resolver.ts @@ -17,6 +17,7 @@ import type { User } from '@prisma/client'; import { GraphQLError } from 'graphql'; import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; +import { EventEmitter } from '../../event'; import { PrismaService } from '../../prisma/service'; import { CloudThrottlerGuard, Throttle } from '../../throttler'; import type { FileUpload } from '../../types'; @@ -24,7 +25,7 @@ import { Auth, CurrentUser, Public, Publicable } from '../auth/guard'; import { AuthService } from '../auth/service'; import { FeatureManagementService } from '../features'; import { QuotaService } from '../quota'; -import { StorageService } from '../storage/storage.service'; +import { AvatarStorage } from '../storage'; import { DeleteAccount, RemoveAvatar, UserQuotaType, UserType } from './types'; import { UsersService } from './users'; @@ -39,10 +40,11 @@ export class UserResolver { constructor( private readonly auth: AuthService, private readonly prisma: PrismaService, - private readonly storage: StorageService, + private readonly storage: AvatarStorage, private readonly users: UsersService, private readonly feature: FeatureManagementService, - private readonly quota: QuotaService + private readonly quota: QuotaService, + private readonly event: EventEmitter ) {} @Throttle({ @@ -147,10 +149,20 @@ export class UserResolver { if (!user) { throw new BadRequestException(`User not found`); } - const url = await this.storage.uploadFile(`${user.id}-avatar`, avatar); + + const link = await this.storage.put( + `${user.id}-avatar`, + avatar.createReadStream(), + { + contentType: avatar.mimetype, + } + ); + return this.prisma.user.update({ where: { id: user.id }, - data: { avatarUrl: url }, + data: { + avatarUrl: link, + }, }); } @@ -183,7 +195,8 @@ export class UserResolver { }) @Mutation(() => DeleteAccount) async deleteAccount(@CurrentUser() user: UserType): Promise { - await this.users.deleteUser(user.id); + const deletedUser = await this.users.deleteUser(user.id); + this.event.emit('user.deleted', deletedUser); return { success: true }; } diff --git a/packages/backend/server/src/modules/workspaces/controller.ts b/packages/backend/server/src/modules/workspaces/controller.ts index 47b64cf4aa..02fbaea87f 100644 --- a/packages/backend/server/src/modules/workspaces/controller.ts +++ b/packages/backend/server/src/modules/workspaces/controller.ts @@ -1,9 +1,8 @@ -import type { Storage } from '@affine/storage'; import { Controller, ForbiddenException, Get, - Inject, + Logger, NotFoundException, Param, Res, @@ -12,18 +11,19 @@ import type { Response } from 'express'; import { CallTimer } from '../../metrics'; import { PrismaService } from '../../prisma'; -import { StorageProvide } from '../../storage'; import { DocID } from '../../utils/doc'; import { Auth, CurrentUser, Publicable } from '../auth'; import { DocHistoryManager, DocManager } from '../doc'; +import { WorkspaceBlobStorage } from '../storage'; import { UserType } from '../users'; import { PermissionService, PublicPageMode } from './permission'; import { Permission } from './types'; @Controller('/api/workspaces') export class WorkspacesController { + logger = new Logger(WorkspacesController.name); constructor( - @Inject(StorageProvide) private readonly storage: Storage, + private readonly storage: WorkspaceBlobStorage, private readonly permission: PermissionService, private readonly docManager: DocManager, private readonly historyManager: DocHistoryManager, @@ -40,19 +40,26 @@ export class WorkspacesController { @Param('name') name: string, @Res() res: Response ) { - const blob = await this.storage.getBlob(workspaceId, name); + const { body, metadata } = await this.storage.get(workspaceId, name); - if (!blob) { + if (!body) { throw new NotFoundException( `Blob not found in workspace ${workspaceId}: ${name}` ); } - res.setHeader('content-type', blob.contentType); - res.setHeader('last-modified', blob.lastModified); - res.setHeader('content-length', blob.size); + // metadata should always exists if body is not null + if (metadata) { + res.setHeader('content-type', metadata.contentType); + res.setHeader('last-modified', metadata.lastModified.toISOString()); + res.setHeader('content-length', metadata.contentLength); + res.setHeader('x-checksum-crc32', metadata.checksumCRC32); + } else { + this.logger.warn(`Blob ${workspaceId}/${name} has no metadata`); + } - res.send(blob.data); + res.setHeader('cache-control', 'public, max-age=31536000, immutable'); + body.pipe(res); } // get doc binary diff --git a/packages/backend/server/src/modules/workspaces/index.ts b/packages/backend/server/src/modules/workspaces/index.ts index 50259cd5d2..ecb09cb73a 100644 --- a/packages/backend/server/src/modules/workspaces/index.ts +++ b/packages/backend/server/src/modules/workspaces/index.ts @@ -2,14 +2,19 @@ import { Module } from '@nestjs/common'; import { DocModule } from '../doc'; import { QuotaModule } from '../quota'; +import { StorageModule } from '../storage'; import { UsersService } from '../users'; import { WorkspacesController } from './controller'; -import { DocHistoryResolver } from './history.resolver'; import { PermissionService } from './permission'; -import { PagePermissionResolver, WorkspaceResolver } from './resolver'; +import { + DocHistoryResolver, + PagePermissionResolver, + WorkspaceBlobResolver, + WorkspaceResolver, +} from './resolvers'; @Module({ - imports: [DocModule, QuotaModule], + imports: [DocModule, QuotaModule, StorageModule], controllers: [WorkspacesController], providers: [ WorkspaceResolver, @@ -17,8 +22,9 @@ import { PagePermissionResolver, WorkspaceResolver } from './resolver'; UsersService, PagePermissionResolver, DocHistoryResolver, + WorkspaceBlobResolver, ], exports: [PermissionService], }) export class WorkspaceModule {} -export { InvitationType, WorkspaceType } from './resolver'; +export { InvitationType, WorkspaceType } from './resolvers'; diff --git a/packages/backend/server/src/modules/workspaces/resolvers/blob.ts b/packages/backend/server/src/modules/workspaces/resolvers/blob.ts new file mode 100644 index 0000000000..6d86c3b5a9 --- /dev/null +++ b/packages/backend/server/src/modules/workspaces/resolvers/blob.ts @@ -0,0 +1,175 @@ +import { ForbiddenException, Logger, UseGuards } from '@nestjs/common'; +import { + Args, + Float, + Int, + Mutation, + Parent, + Query, + ResolveField, + Resolver, +} from '@nestjs/graphql'; +import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; + +import { MakeCache, PreventCache } from '../../../cache'; +import { CloudThrottlerGuard } from '../../../throttler'; +import type { FileUpload } from '../../../types'; +import { Auth, CurrentUser } from '../../auth'; +import { QuotaManagementService } from '../../quota'; +import { WorkspaceBlobStorage } from '../../storage'; +import { UserType } from '../../users'; +import { PermissionService } from '../permission'; +import { Permission } from '../types'; +import { WorkspaceBlobSizes, WorkspaceType } from './workspace'; + +@UseGuards(CloudThrottlerGuard) +@Auth() +@Resolver(() => WorkspaceType) +export class WorkspaceBlobResolver { + logger = new Logger(WorkspaceBlobResolver.name); + constructor( + private readonly permissions: PermissionService, + private readonly quota: QuotaManagementService, + private readonly storage: WorkspaceBlobStorage + ) {} + + @ResolveField(() => Int, { + description: 'Blobs size of workspace', + complexity: 2, + }) + async blobsSize(@Parent() workspace: WorkspaceType) { + return this.storage.totalSize(workspace.id); + } + + /** + * @deprecated use `workspace.blobs` instead + */ + @Query(() => [String], { + description: 'List blobs of workspace', + deprecationReason: 'use `workspace.blobs` instead', + }) + @MakeCache(['blobs'], ['workspaceId']) + async listBlobs( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string + ) { + await this.permissions.checkWorkspace(workspaceId, user.id); + + return this.storage + .list(workspaceId) + .then(list => list.map(item => item.key)); + } + + /** + * @deprecated use `user.storageUsage` instead + */ + @Query(() => WorkspaceBlobSizes, { + deprecationReason: 'use `user.storageUsage` instead', + }) + async collectAllBlobSizes(@CurrentUser() user: UserType) { + const size = await this.quota.getUserUsage(user.id); + return { size }; + } + + /** + * @deprecated mutation `setBlob` will check blob limit & quota usage + */ + @Query(() => WorkspaceBlobSizes, { + deprecationReason: 'no more needed', + }) + async checkBlobSize( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args('size', { type: () => Float }) blobSize: number + ) { + const canWrite = await this.permissions.tryCheckWorkspace( + workspaceId, + user.id, + Permission.Write + ); + if (canWrite) { + const size = await this.quota.checkBlobQuota(workspaceId, blobSize); + return { size }; + } + return false; + } + + @Mutation(() => String) + @PreventCache(['blobs'], ['workspaceId']) + async setBlob( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args({ name: 'blob', type: () => GraphQLUpload }) + blob: FileUpload + ) { + await this.permissions.checkWorkspace( + workspaceId, + user.id, + Permission.Write + ); + + const { quota, size } = await this.quota.getWorkspaceUsage(workspaceId); + + const checkExceeded = (recvSize: number) => { + if (!quota) { + throw new ForbiddenException('cannot find user quota'); + } + if (size + recvSize > quota) { + this.logger.log( + `storage size limit exceeded: ${size + recvSize} > ${quota}` + ); + return true; + } else { + return false; + } + }; + + if (checkExceeded(0)) { + throw new ForbiddenException('storage size limit exceeded'); + } + const buffer = await new Promise((resolve, reject) => { + const stream = blob.createReadStream(); + const chunks: Uint8Array[] = []; + stream.on('data', chunk => { + chunks.push(chunk); + + // check size after receive each chunk to avoid unnecessary memory usage + const bufferSize = chunks.reduce((acc, cur) => acc + cur.length, 0); + if (checkExceeded(bufferSize)) { + reject(new ForbiddenException('storage size limit exceeded')); + } + }); + stream.on('error', reject); + stream.on('end', () => { + const buffer = Buffer.concat(chunks); + + if (checkExceeded(buffer.length)) { + reject(new ForbiddenException('storage size limit exceeded')); + } else { + resolve(buffer); + } + }); + }); + + if (!(await this.quota.checkBlobQuota(workspaceId, buffer.length))) { + throw new ForbiddenException('blob size limit exceeded'); + } + + await this.storage.put(workspaceId, blob.filename, buffer); + return blob.filename; + } + + @Mutation(() => Boolean) + @PreventCache(['blobs'], ['workspaceId']) + async deleteBlob( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args('hash') name: string + ) { + await this.permissions.checkWorkspace(workspaceId, user.id); + + await this.storage.delete(workspaceId, name); + + return true; + } +} diff --git a/packages/backend/server/src/modules/workspaces/history.resolver.ts b/packages/backend/server/src/modules/workspaces/resolvers/history.ts similarity index 81% rename from packages/backend/server/src/modules/workspaces/history.resolver.ts rename to packages/backend/server/src/modules/workspaces/resolvers/history.ts index ad9483a542..77a3415bd7 100644 --- a/packages/backend/server/src/modules/workspaces/history.resolver.ts +++ b/packages/backend/server/src/modules/workspaces/resolvers/history.ts @@ -1,3 +1,4 @@ +import { UseGuards } from '@nestjs/common'; import { Args, Field, @@ -11,13 +12,14 @@ import { } from '@nestjs/graphql'; import type { SnapshotHistory } from '@prisma/client'; -import { DocID } from '../../utils/doc'; -import { Auth, CurrentUser } from '../auth'; -import { DocHistoryManager } from '../doc/history'; -import { UserType } from '../users'; -import { PermissionService } from './permission'; -import { WorkspaceType } from './resolver'; -import { Permission } from './types'; +import { CloudThrottlerGuard } from '../../../throttler'; +import { DocID } from '../../../utils/doc'; +import { Auth, CurrentUser } from '../../auth'; +import { DocHistoryManager } from '../../doc/history'; +import { UserType } from '../../users'; +import { PermissionService } from '../permission'; +import { Permission } from '../types'; +import { WorkspaceType } from './workspace'; @ObjectType() class DocHistoryType implements Partial { @@ -31,6 +33,7 @@ class DocHistoryType implements Partial { timestamp!: Date; } +@UseGuards(CloudThrottlerGuard) @Resolver(() => WorkspaceType) export class DocHistoryResolver { constructor( diff --git a/packages/backend/server/src/modules/workspaces/resolvers/index.ts b/packages/backend/server/src/modules/workspaces/resolvers/index.ts new file mode 100644 index 0000000000..d4e282afd0 --- /dev/null +++ b/packages/backend/server/src/modules/workspaces/resolvers/index.ts @@ -0,0 +1,4 @@ +export * from './blob'; +export * from './history'; +export * from './page'; +export * from './workspace'; diff --git a/packages/backend/server/src/modules/workspaces/resolvers/page.ts b/packages/backend/server/src/modules/workspaces/resolvers/page.ts new file mode 100644 index 0000000000..deb138c98c --- /dev/null +++ b/packages/backend/server/src/modules/workspaces/resolvers/page.ts @@ -0,0 +1,164 @@ +import { ForbiddenException, UseGuards } from '@nestjs/common'; +import { + Args, + Field, + Mutation, + ObjectType, + Parent, + registerEnumType, + ResolveField, + Resolver, +} from '@nestjs/graphql'; +import type { WorkspacePage as PrismaWorkspacePage } from '@prisma/client'; + +import { PrismaService } from '../../../prisma'; +import { CloudThrottlerGuard } from '../../../throttler'; +import { DocID } from '../../../utils/doc'; +import { Auth, CurrentUser } from '../../auth'; +import { UserType } from '../../users'; +import { PermissionService, PublicPageMode } from '../permission'; +import { Permission } from '../types'; +import { WorkspaceType } from './workspace'; + +registerEnumType(PublicPageMode, { + name: 'PublicPageMode', + description: 'The mode which the public page default in', +}); + +@ObjectType() +class WorkspacePage implements Partial { + @Field(() => String, { name: 'id' }) + pageId!: string; + + @Field() + workspaceId!: string; + + @Field(() => PublicPageMode) + mode!: PublicPageMode; + + @Field() + public!: boolean; +} + +@UseGuards(CloudThrottlerGuard) +@Auth() +@Resolver(() => WorkspaceType) +export class PagePermissionResolver { + constructor( + private readonly prisma: PrismaService, + private readonly permission: PermissionService + ) {} + + /** + * @deprecated + */ + @ResolveField(() => [String], { + description: 'Shared pages of workspace', + complexity: 2, + deprecationReason: 'use WorkspaceType.publicPages', + }) + async sharedPages(@Parent() workspace: WorkspaceType) { + const data = await this.prisma.workspacePage.findMany({ + where: { + workspaceId: workspace.id, + public: true, + }, + }); + + return data.map(row => row.pageId); + } + + @ResolveField(() => [WorkspacePage], { + description: 'Public pages of a workspace', + complexity: 2, + }) + async publicPages(@Parent() workspace: WorkspaceType) { + return this.prisma.workspacePage.findMany({ + where: { + workspaceId: workspace.id, + public: true, + }, + }); + } + + /** + * @deprecated + */ + @Mutation(() => Boolean, { + name: 'sharePage', + deprecationReason: 'renamed to publicPage', + }) + async deprecatedSharePage( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args('pageId') pageId: string + ) { + await this.publishPage(user, workspaceId, pageId, PublicPageMode.Page); + return true; + } + + @Mutation(() => WorkspacePage) + async publishPage( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args('pageId') pageId: string, + @Args({ + name: 'mode', + type: () => PublicPageMode, + nullable: true, + defaultValue: PublicPageMode.Page, + }) + mode: PublicPageMode + ) { + const docId = new DocID(pageId, workspaceId); + + if (docId.isWorkspace) { + throw new ForbiddenException('Expect page not to be workspace'); + } + + await this.permission.checkWorkspace( + docId.workspace, + user.id, + Permission.Read + ); + + return this.permission.publishPage(docId.workspace, docId.guid, mode); + } + + /** + * @deprecated + */ + @Mutation(() => Boolean, { + name: 'revokePage', + deprecationReason: 'use revokePublicPage', + }) + async deprecatedRevokePage( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args('pageId') pageId: string + ) { + await this.revokePublicPage(user, workspaceId, pageId); + return true; + } + + @Mutation(() => WorkspacePage) + async revokePublicPage( + @CurrentUser() user: UserType, + @Args('workspaceId') workspaceId: string, + @Args('pageId') pageId: string + ) { + const docId = new DocID(pageId, workspaceId); + + if (docId.isWorkspace) { + throw new ForbiddenException('Expect page not to be workspace'); + } + + await this.permission.checkWorkspace( + docId.workspace, + user.id, + Permission.Read + ); + + return this.permission.revokePublicPage(docId.workspace, docId.guid); + } +} diff --git a/packages/backend/server/src/modules/workspaces/resolver.ts b/packages/backend/server/src/modules/workspaces/resolvers/workspace.ts similarity index 65% rename from packages/backend/server/src/modules/workspaces/resolver.ts rename to packages/backend/server/src/modules/workspaces/resolvers/workspace.ts index 1880ea935a..71039e9857 100644 --- a/packages/backend/server/src/modules/workspaces/resolver.ts +++ b/packages/backend/server/src/modules/workspaces/resolvers/workspace.ts @@ -1,7 +1,5 @@ -import type { Storage } from '@affine/storage'; import { ForbiddenException, - Inject, InternalServerErrorException, Logger, NotFoundException, @@ -25,29 +23,23 @@ import { ResolveField, Resolver, } from '@nestjs/graphql'; -import type { - User, - Workspace, - WorkspacePage as PrismaWorkspacePage, -} from '@prisma/client'; +import type { User, Workspace } from '@prisma/client'; +import { getStreamAsBuffer } from 'get-stream'; import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; import { applyUpdate, Doc } from 'yjs'; -import { MakeCache, PreventCache } from '../../cache'; -import { EventEmitter } from '../../event'; -import { PrismaService } from '../../prisma'; -import { StorageProvide } from '../../storage'; -import { CloudThrottlerGuard, Throttle } from '../../throttler'; -import type { FileUpload } from '../../types'; -import { DocID } from '../../utils/doc'; -import { Auth, CurrentUser, Public } from '../auth'; -import { MailService } from '../auth/mailer'; -import { AuthService } from '../auth/service'; -import { QuotaManagementService } from '../quota'; -import { UsersService, UserType } from '../users'; -import { PermissionService, PublicPageMode } from './permission'; -import { Permission } from './types'; -import { defaultWorkspaceAvatar } from './utils'; +import { EventEmitter } from '../../../event'; +import { PrismaService } from '../../../prisma'; +import { CloudThrottlerGuard, Throttle } from '../../../throttler'; +import type { FileUpload } from '../../../types'; +import { Auth, CurrentUser, Public } from '../../auth'; +import { MailService } from '../../auth/mailer'; +import { AuthService } from '../../auth/service'; +import { WorkspaceBlobStorage } from '../../storage'; +import { UsersService, UserType } from '../../users'; +import { PermissionService } from '../permission'; +import { Permission } from '../types'; +import { defaultWorkspaceAvatar } from '../utils'; registerEnumType(Permission, { name: 'Permission', @@ -149,8 +141,7 @@ export class WorkspaceResolver { private readonly permissions: PermissionService, private readonly users: UsersService, private readonly event: EventEmitter, - private readonly quota: QuotaManagementService, - @Inject(StorageProvide) private readonly storage: Storage + private readonly blobStorage: WorkspaceBlobStorage ) {} @ResolveField(() => Permission, { @@ -235,14 +226,6 @@ export class WorkspaceResolver { })); } - @ResolveField(() => Int, { - description: 'Blobs size of workspace', - complexity: 2, - }) - async blobsSize(@Parent() workspace: WorkspaceType) { - return this.storage.blobsSize([workspace.id]); - } - @Query(() => Boolean, { description: 'Get is owner of workspace', complexity: 2, @@ -565,11 +548,14 @@ export class WorkspaceResolver { let avatar = ''; if (metaJSON.avatar) { - const avatarBlob = await this.storage.getBlob( + const avatarBlob = await this.blobStorage.get( workspaceId, metaJSON.avatar ); - avatar = avatarBlob?.data.toString('base64') || ''; + + if (avatarBlob.body) { + avatar = (await getStreamAsBuffer(avatarBlob.body)).toString('base64'); + } } return { @@ -653,256 +639,4 @@ export class WorkspaceResolver { return this.permissions.revokeWorkspace(workspaceId, user.id); } - - @Query(() => [String], { - description: 'List blobs of workspace', - }) - @MakeCache(['blobs'], ['workspaceId']) - async listBlobs( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string - ) { - await this.permissions.checkWorkspace(workspaceId, user.id); - - return this.storage.listBlobs(workspaceId); - } - - @Query(() => WorkspaceBlobSizes) - async collectAllBlobSizes(@CurrentUser() user: UserType) { - const size = await this.quota.getUserUsage(user.id); - return { size }; - } - - @Query(() => WorkspaceBlobSizes) - async checkBlobSize( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args('size', { type: () => Float }) blobSize: number - ) { - const canWrite = await this.permissions.tryCheckWorkspace( - workspaceId, - user.id, - Permission.Write - ); - if (canWrite) { - const size = await this.quota.checkBlobQuota(workspaceId, blobSize); - return { size }; - } - return false; - } - - @Mutation(() => String) - @PreventCache(['blobs'], ['workspaceId']) - async setBlob( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args({ name: 'blob', type: () => GraphQLUpload }) - blob: FileUpload - ) { - await this.permissions.checkWorkspace( - workspaceId, - user.id, - Permission.Write - ); - - const { quota, size } = await this.quota.getWorkspaceUsage(workspaceId); - - const checkExceeded = (recvSize: number) => { - if (!quota) { - throw new ForbiddenException('cannot find user quota'); - } - if (size + recvSize > quota) { - this.logger.log( - `storage size limit exceeded: ${size + recvSize} > ${quota}` - ); - return true; - } else { - return false; - } - }; - - if (checkExceeded(0)) { - throw new ForbiddenException('storage size limit exceeded'); - } - const buffer = await new Promise((resolve, reject) => { - const stream = blob.createReadStream(); - const chunks: Uint8Array[] = []; - stream.on('data', chunk => { - chunks.push(chunk); - - // check size after receive each chunk to avoid unnecessary memory usage - const bufferSize = chunks.reduce((acc, cur) => acc + cur.length, 0); - if (checkExceeded(bufferSize)) { - reject(new ForbiddenException('storage size limit exceeded')); - } - }); - stream.on('error', reject); - stream.on('end', () => { - const buffer = Buffer.concat(chunks); - - if (checkExceeded(buffer.length)) { - reject(new ForbiddenException('storage size limit exceeded')); - } else { - resolve(buffer); - } - }); - }); - - return this.storage.uploadBlob(workspaceId, buffer); - } - - @Mutation(() => Boolean) - @PreventCache(['blobs'], ['workspaceId']) - async deleteBlob( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args('hash') hash: string - ) { - await this.permissions.checkWorkspace(workspaceId, user.id); - - return this.storage.deleteBlob(workspaceId, hash); - } -} - -registerEnumType(PublicPageMode, { - name: 'PublicPageMode', - description: 'The mode which the public page default in', -}); - -@ObjectType() -class WorkspacePage implements Partial { - @Field(() => String, { name: 'id' }) - pageId!: string; - - @Field() - workspaceId!: string; - - @Field(() => PublicPageMode) - mode!: PublicPageMode; - - @Field() - public!: boolean; -} - -@UseGuards(CloudThrottlerGuard) -@Auth() -@Resolver(() => WorkspaceType) -export class PagePermissionResolver { - constructor( - private readonly prisma: PrismaService, - private readonly permission: PermissionService - ) {} - - /** - * @deprecated - */ - @ResolveField(() => [String], { - description: 'Shared pages of workspace', - complexity: 2, - deprecationReason: 'use WorkspaceType.publicPages', - }) - async sharedPages(@Parent() workspace: WorkspaceType) { - const data = await this.prisma.workspacePage.findMany({ - where: { - workspaceId: workspace.id, - public: true, - }, - }); - - return data.map(row => row.pageId); - } - - @ResolveField(() => [WorkspacePage], { - description: 'Public pages of a workspace', - complexity: 2, - }) - async publicPages(@Parent() workspace: WorkspaceType) { - return this.prisma.workspacePage.findMany({ - where: { - workspaceId: workspace.id, - public: true, - }, - }); - } - - /** - * @deprecated - */ - @Mutation(() => Boolean, { - name: 'sharePage', - deprecationReason: 'renamed to publicPage', - }) - async deprecatedSharePage( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args('pageId') pageId: string - ) { - await this.publishPage(user, workspaceId, pageId, PublicPageMode.Page); - return true; - } - - @Mutation(() => WorkspacePage) - async publishPage( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args('pageId') pageId: string, - @Args({ - name: 'mode', - type: () => PublicPageMode, - nullable: true, - defaultValue: PublicPageMode.Page, - }) - mode: PublicPageMode - ) { - const docId = new DocID(pageId, workspaceId); - - if (docId.isWorkspace) { - throw new ForbiddenException('Expect page not to be workspace'); - } - - await this.permission.checkWorkspace( - docId.workspace, - user.id, - Permission.Read - ); - - return this.permission.publishPage(docId.workspace, docId.guid, mode); - } - - /** - * @deprecated - */ - @Mutation(() => Boolean, { - name: 'revokePage', - deprecationReason: 'use revokePublicPage', - }) - async deprecatedRevokePage( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args('pageId') pageId: string - ) { - await this.revokePublicPage(user, workspaceId, pageId); - return true; - } - - @Mutation(() => WorkspacePage) - async revokePublicPage( - @CurrentUser() user: UserType, - @Args('workspaceId') workspaceId: string, - @Args('pageId') pageId: string - ) { - const docId = new DocID(pageId, workspaceId); - - if (docId.isWorkspace) { - throw new ForbiddenException('Expect page not to be workspace'); - } - - await this.permission.checkWorkspace( - docId.workspace, - user.id, - Permission.Read - ); - - return this.permission.revokePublicPage(docId.workspace, docId.guid); - } } diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index e7a348b317..32223cd389 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -206,15 +206,15 @@ type WorkspaceType { """Owner of workspace""" owner: UserType! - """Blobs size of workspace""" - blobsSize: Int! - """Shared pages of workspace""" sharedPages: [String!]! @deprecated(reason: "use WorkspaceType.publicPages") """Public pages of a workspace""" publicPages: [WorkspacePage!]! histories(guid: String!, before: DateTime, take: Int): [DocHistoryType!]! + + """Blobs size of workspace""" + blobsSize: Int! } type InvitationWorkspaceType { @@ -242,6 +242,12 @@ type InvitationType { invitee: UserType! } +type DocHistoryType { + workspaceId: String! + id: String! + timestamp: DateTime! +} + type WorkspacePage { id: String! workspaceId: String! @@ -255,12 +261,6 @@ enum PublicPageMode { Edgeless } -type DocHistoryType { - workspaceId: String! - id: String! - timestamp: DateTime! -} - type Query { """server config""" serverConfig: ServerConfigType! @@ -281,9 +281,9 @@ type Query { getInviteInfo(inviteId: String!): InvitationType! """List blobs of workspace""" - listBlobs(workspaceId: String!): [String!]! - collectAllBlobSizes: WorkspaceBlobSizes! - checkBlobSize(workspaceId: String!, size: Float!): WorkspaceBlobSizes! + listBlobs(workspaceId: String!): [String!]! @deprecated(reason: "use `workspace.blobs` instead") + collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.storageUsage` instead") + checkBlobSize(workspaceId: String!, size: Float!): WorkspaceBlobSizes! @deprecated(reason: "no more needed") """Get current user""" currentUser: UserType @@ -314,13 +314,13 @@ type Mutation { revoke(workspaceId: String!, userId: String!): Boolean! acceptInviteById(workspaceId: String!, inviteId: String!, sendAcceptMail: Boolean): Boolean! leaveWorkspace(workspaceId: String!, workspaceName: String!, sendLeaveMail: Boolean): Boolean! - setBlob(workspaceId: String!, blob: Upload!): String! - deleteBlob(workspaceId: String!, hash: String!): Boolean! sharePage(workspaceId: String!, pageId: String!): Boolean! @deprecated(reason: "renamed to publicPage") publishPage(workspaceId: String!, pageId: String!, mode: PublicPageMode = Page): WorkspacePage! revokePage(workspaceId: String!, pageId: String!): Boolean! @deprecated(reason: "use revokePublicPage") revokePublicPage(workspaceId: String!, pageId: String!): WorkspacePage! recoverDoc(workspaceId: String!, guid: String!, timestamp: DateTime!): DateTime! + setBlob(workspaceId: String!, blob: Upload!): String! + deleteBlob(workspaceId: String!, hash: String!): Boolean! """Upload user avatar""" uploadAvatar(avatar: Upload!): UserType! diff --git a/packages/backend/server/src/storage/index.ts b/packages/backend/server/src/storage/index.ts index 59299e27ca..0b8a8e5058 100644 --- a/packages/backend/server/src/storage/index.ts +++ b/packages/backend/server/src/storage/index.ts @@ -1,11 +1,8 @@ +// NODE: +// This file has been deprecated after blob storage moved to cloudflare r2 storage. +// It only exists for backward compatibility. import { createRequire } from 'node:module'; -import { type DynamicModule, type FactoryProvider } from '@nestjs/common'; - -import { Config } from '../config'; - -export const StorageProvide = Symbol('Storage'); - let storageModule: typeof import('@affine/storage'); try { storageModule = await import('@affine/storage'); @@ -17,25 +14,6 @@ try { : require('../../storage.node'); } -export class StorageModule { - static forRoot(): DynamicModule { - const storageProvider: FactoryProvider = { - provide: StorageProvide, - useFactory: async (config: Config) => { - return storageModule.Storage.connect(config.db.url); - }, - inject: [Config], - }; - - return { - global: true, - module: StorageModule, - providers: [storageProvider], - exports: [storageProvider], - }; - } -} - export const mergeUpdatesInApplyWay = storageModule.mergeUpdatesInApplyWay; export const verifyChallengeResponse = async ( diff --git a/packages/backend/server/tests/doc.spec.ts b/packages/backend/server/tests/doc.spec.ts index f2a8e5ac7e..e004d697ec 100644 --- a/packages/backend/server/tests/doc.spec.ts +++ b/packages/backend/server/tests/doc.spec.ts @@ -22,8 +22,9 @@ import { import { EventModule } from '../src/event'; import { DocManager, DocModule } from '../src/modules/doc'; import { QuotaModule } from '../src/modules/quota'; +import { StorageModule } from '../src/modules/storage'; import { PrismaModule, PrismaService } from '../src/prisma'; -import { FakeStorageModule, flushDB } from './utils'; +import { flushDB } from './utils'; const createModule = () => { return Test.createTestingModule({ @@ -32,7 +33,7 @@ const createModule = () => { CacheModule, EventModule, QuotaModule, - FakeStorageModule.forRoot(), + StorageModule, ConfigModule.forRoot(), DocModule, RevertCommand, diff --git a/packages/backend/server/tests/history.spec.ts b/packages/backend/server/tests/history.spec.ts index d496816461..050f30de56 100644 --- a/packages/backend/server/tests/history.spec.ts +++ b/packages/backend/server/tests/history.spec.ts @@ -6,11 +6,12 @@ import test from 'ava'; import * as Sinon from 'sinon'; import { ConfigModule } from '../src/config'; -import type { EventPayload } from '../src/event'; +import { EventModule, type EventPayload } from '../src/event'; import { DocHistoryManager } from '../src/modules/doc'; import { QuotaModule } from '../src/modules/quota'; +import { StorageModule } from '../src/modules/storage'; import { PrismaModule, PrismaService } from '../src/prisma'; -import { FakeStorageModule, flushDB } from './utils'; +import { flushDB } from './utils'; let app: INestApplication; let m: TestingModule; @@ -24,7 +25,8 @@ test.beforeEach(async () => { imports: [ PrismaModule, QuotaModule, - FakeStorageModule.forRoot(), + EventModule, + StorageModule, ScheduleModule.forRoot(), ConfigModule.forRoot(), ], diff --git a/packages/backend/server/tests/quota.spec.ts b/packages/backend/server/tests/quota.spec.ts index f9d974929e..d819c21752 100644 --- a/packages/backend/server/tests/quota.spec.ts +++ b/packages/backend/server/tests/quota.spec.ts @@ -6,6 +6,7 @@ import ava, { type TestFn } from 'ava'; import { ConfigModule } from '../src/config'; import { RevertCommand, RunCommand } from '../src/data/commands/run'; +import { EventModule } from '../src/event'; import { AuthModule } from '../src/modules/auth'; import { AuthService } from '../src/modules/auth/service'; import { @@ -15,9 +16,10 @@ import { QuotaService, QuotaType, } from '../src/modules/quota'; +import { StorageModule } from '../src/modules/storage'; import { PrismaModule } from '../src/prisma'; import { RateLimiterModule } from '../src/throttler'; -import { FakeStorageModule, initFeatureConfigs } from './utils'; +import { initFeatureConfigs } from './utils'; const test = ava as TestFn<{ auth: AuthService; @@ -48,8 +50,9 @@ test.beforeEach(async t => { }), PrismaModule, AuthModule, + EventModule, QuotaModule, - FakeStorageModule.forRoot(), + StorageModule, RateLimiterModule, RevertCommand, RunCommand, diff --git a/packages/backend/server/tests/utils/blobs.ts b/packages/backend/server/tests/utils/blobs.ts index c026a2922f..5d13797634 100644 --- a/packages/backend/server/tests/utils/blobs.ts +++ b/packages/backend/server/tests/utils/blobs.ts @@ -106,7 +106,11 @@ export async function setBlob( }) ) .field('map', JSON.stringify({ '0': ['variables.blob'] })) - .attach('0', buffer, 'blob.data') + .attach( + '0', + buffer, + `blob-${Math.random().toString(16).substring(2, 10)}.data` + ) .expect(200); return res.body.data.setBlob; } diff --git a/packages/backend/server/tests/utils/utils.ts b/packages/backend/server/tests/utils/utils.ts index 0274e63daf..042f718c93 100644 --- a/packages/backend/server/tests/utils/utils.ts +++ b/packages/backend/server/tests/utils/utils.ts @@ -1,12 +1,10 @@ import { randomUUID } from 'node:crypto'; -import type { DynamicModule, FactoryProvider } from '@nestjs/common'; import { TestingModule } from '@nestjs/testing'; import { hashSync } from '@node-rs/argon2'; import { PrismaClient, type User } from '@prisma/client'; import { RevertCommand, RunCommand } from '../../src/data/commands/run'; -import { StorageProvide } from '../../src/storage'; export async function flushDB() { const client = new PrismaClient(); @@ -56,24 +54,6 @@ export class FakePrisma { } } -export class FakeStorageModule { - static forRoot(): DynamicModule { - const storageProvider: FactoryProvider = { - provide: StorageProvide, - useFactory: async () => { - return null; - }, - }; - - return { - global: true, - module: FakeStorageModule, - providers: [storageProvider], - exports: [storageProvider], - }; - } -} - export async function initFeatureConfigs(module: TestingModule) { const run = module.get(RunCommand); const revert = module.get(RevertCommand); diff --git a/packages/backend/server/tests/workspace-blobs.spec.ts b/packages/backend/server/tests/workspace-blobs.spec.ts index af0f683adc..ed86723e8a 100644 --- a/packages/backend/server/tests/workspace-blobs.spec.ts +++ b/packages/backend/server/tests/workspace-blobs.spec.ts @@ -99,8 +99,8 @@ test('should list blobs', async t => { const ret = await listBlobs(app, u1.token.token, workspace.id); t.is(ret.length, 2, 'failed to list blobs'); - t.is(ret[0], hash1, 'failed to list blobs'); - t.is(ret[1], hash2, 'failed to list blobs'); + // list blob result is not ordered + t.deepEqual(ret.sort(), [hash1, hash2].sort()); }); test('should calc blobs size', async t => { @@ -189,3 +189,13 @@ test('should be able calc quota after switch plan', async t => { ); t.is(size2, 0, 'failed to check pro plan blob size'); }); + +test('should reject blob exceeded limit', t => { + // TODO + t.true(true); +}); + +test('should reject blob exceeded quota', t => { + // TODO + t.true(true); +}); diff --git a/packages/backend/server/tests/workspace-usage.spec.ts b/packages/backend/server/tests/workspace-usage.spec.ts deleted file mode 100644 index ad5623b393..0000000000 --- a/packages/backend/server/tests/workspace-usage.spec.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { INestApplication } from '@nestjs/common'; -import { Test } from '@nestjs/testing'; -import ava, { type TestFn } from 'ava'; -import { stub } from 'sinon'; - -import { AppModule } from '../src/app'; -import { FeatureManagementService } from '../src/modules/features'; -import { Quotas } from '../src/modules/quota'; -import { UsersService } from '../src/modules/users'; -import { PermissionService } from '../src/modules/workspaces/permission'; -import { WorkspaceResolver } from '../src/modules/workspaces/resolver'; -import { PrismaService } from '../src/prisma'; -import { StorageProvide } from '../src/storage'; -import { FakePrisma } from './utils'; - -class FakePermission { - async tryCheckWorkspace() { - return true; - } - async getWorkspaceOwner() { - return { - user: new FakePrisma().fakeUser, - }; - } - async getOwnedWorkspaces() { - return ['']; - } -} - -const fakeUserService = { - getStorageQuotaById: stub(), -}; - -const test = ava as TestFn<{ - app: INestApplication; - resolver: WorkspaceResolver; -}>; - -test.beforeEach(async t => { - const module = await Test.createTestingModule({ - imports: [AppModule], - }) - .overrideProvider(PrismaService) - .useValue({ - workspaceUserPermission: { - async findMany() { - return []; - }, - }, - userFeatures: { - async count() { - return 1; - }, - async findFirst() { - return { - createdAt: new Date(), - expiredAt: new Date(), - reason: '', - feature: Quotas[0], - }; - }, - }, - features: { - async findFirst() { - return { - id: 0, - feature: 'free_plan_v1', - version: 1, - type: 1, - configs: { - name: 'Free', - blobLimit: 1, - storageQuota: 1, - historyPeriod: 1, - memberLimit: 3, - }, - }; - }, - }, - }) - .overrideProvider(PermissionService) - .useClass(FakePermission) - .overrideProvider(UsersService) - .useValue(fakeUserService) - .overrideProvider(StorageProvide) - .useValue({ - blobsSize() { - return 1024 * 10; - }, - }) - .overrideProvider(FeatureManagementService) - .useValue({}) - .compile(); - t.context.app = module.createNestApplication(); - t.context.resolver = t.context.app.get(WorkspaceResolver); - await t.context.app.init(); -}); - -test.afterEach.always(async t => { - await t.context.app.close(); -}); - -test('should get blob size limit', async t => { - const { resolver } = t.context; - fakeUserService.getStorageQuotaById.resolves(100 * 1024 * 1024 * 1024); - const res = await resolver.checkBlobSize(new FakePrisma().fakeUser, '', 100); - t.not(res, false); - // @ts-expect-error - t.is(typeof res.size, 'number'); - fakeUserService.getStorageQuotaById.reset(); -}); diff --git a/packages/frontend/workspace-impl/src/cloud/blob.ts b/packages/frontend/workspace-impl/src/cloud/blob.ts index 62e690051b..5d6d54ec9a 100644 --- a/packages/frontend/workspace-impl/src/cloud/blob.ts +++ b/packages/frontend/workspace-impl/src/cloud/blob.ts @@ -1,5 +1,4 @@ import { - checkBlobSizesQuery, deleteBlobMutation, fetchWithTraceReport, getBaseUrl, @@ -31,20 +30,7 @@ export const createAffineCloudBlobStorage = ( }); }, set: async (key, value) => { - const { - checkBlobSize: { size }, - } = await fetcher({ - query: checkBlobSizesQuery, - variables: { - workspaceId, - size: value.size, - }, - }); - - if (size <= 0) { - throw new Error('Blob size limit exceeded'); - } - + // set blob will check blob size & quota const result = await fetcher({ query: setBlobMutation, variables: { @@ -52,7 +38,6 @@ export const createAffineCloudBlobStorage = ( blob: new File([value], key), }, }); - console.assert(result.setBlob === key, 'Blob hash mismatch'); return result.setBlob; }, list: async () => {