refactor(server): plugin modules (#5630)

- [x] separates modules into `fundamental`, `core`, `plugins`
- [x] optional modules with `@OptionalModule` decorator to install modules with requirements met(`requires`, `if`)
- [x] `module.contributesTo` defines optional features that will be enabled if module registered
- [x] `AFFiNE.plugins.use('payment', {})` to enable a optional/plugin module
- [x] `PaymentModule` is the first plugin module
- [x] GraphQLSchema will not be generated for non-included modules
- [x] Frontend can use `ServerConfigType` query to detect which features are enabled
- [x] override existing provider globally
This commit is contained in:
liuyi 2024-01-22 07:40:28 +00:00
parent ae8401b6f4
commit e516e0db23
No known key found for this signature in database
GPG Key ID: 56709255DC7EC728
130 changed files with 1297 additions and 974 deletions

View File

@ -19,6 +19,7 @@ env:
MACOSX_DEPLOYMENT_TARGET: '10.13'
NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
DISABLE_TELEMETRY: true
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}

View File

@ -1,39 +1,169 @@
import { DynamicModule, Module, Type } from '@nestjs/common';
import { join } from 'node:path';
import { Logger, Module } from '@nestjs/common';
import { APP_INTERCEPTOR } from '@nestjs/core';
import { ScheduleModule } from '@nestjs/schedule';
import { ServeStaticModule } from '@nestjs/serve-static';
import { get } from 'lodash-es';
import { AppController } from './app.controller';
import { AuthModule } from './core/auth';
import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config';
import { DocModule } from './core/doc';
import { FeatureModule } from './core/features';
import { QuotaModule } from './core/quota';
import { StorageModule } from './core/storage';
import { SyncModule } from './core/sync';
import { UsersModule } from './core/users';
import { WorkspaceModule } from './core/workspaces';
import { getOptionalModuleMetadata } from './fundamentals';
import { CacheInterceptor, CacheModule } from './fundamentals/cache';
import { ConfigModule } from './fundamentals/config';
import {
type AvailablePlugins,
Config,
ConfigModule,
} from './fundamentals/config';
import { EventModule } from './fundamentals/event';
import { GqlModule } from './fundamentals/graphql';
import { MailModule } from './fundamentals/mailer';
import { MetricsModule } from './fundamentals/metrics';
import { PrismaModule } from './fundamentals/prisma';
import { SessionModule } from './fundamentals/session';
import { RateLimiterModule } from './fundamentals/throttler';
import { BusinessModules } from './modules';
import { AuthModule } from './modules/auth';
import { WebSocketModule } from './fundamentals/websocket';
import { pluginsMap } from './plugins';
export const FunctionalityModules: Array<Type | DynamicModule> = [
export const FunctionalityModules = [
ConfigModule.forRoot(),
ScheduleModule.forRoot(),
EventModule,
CacheModule,
PrismaModule,
MetricsModule,
EventModule,
SessionModule,
RateLimiterModule,
AuthModule,
SessionModule,
MailModule,
];
@Module({
providers: [
{
provide: APP_INTERCEPTOR,
useClass: CacheInterceptor,
},
],
imports: [...FunctionalityModules, ...BusinessModules],
controllers:
process.env.SERVER_FLAVOR === 'selfhosted' ? [] : [AppController],
})
export class AppModule {}
export class AppModuleBuilder {
private readonly modules: AFFiNEModule[] = [];
constructor(private readonly config: Config) {}
use(...modules: AFFiNEModule[]): this {
modules.forEach(m => {
const requirements = getOptionalModuleMetadata(m, 'requires');
// if condition not set or condition met, include the module
if (requirements?.length) {
const nonMetRequirements = requirements.filter(c => {
const value = get(this.config, c);
return (
value === undefined ||
value === null ||
(typeof value === 'string' && value.trim().length === 0)
);
});
if (nonMetRequirements.length) {
const name = 'module' in m ? m.module.name : m.name;
new Logger(name).warn(
`${name} is not enabled because of the required configuration is not satisfied.`,
'Unsatisfied configuration:',
...nonMetRequirements.map(config => ` AFFiNE.${config}`)
);
return;
}
}
const predicator = getOptionalModuleMetadata(m, 'if');
if (predicator && !predicator(this.config)) {
return;
}
const contribution = getOptionalModuleMetadata(m, 'contributesTo');
if (contribution) {
ADD_ENABLED_FEATURES(contribution);
}
this.modules.push(m);
});
return this;
}
useIf(
predicator: (config: Config) => boolean,
...modules: AFFiNEModule[]
): this {
if (predicator(this.config)) {
this.use(...modules);
}
return this;
}
compile() {
@Module({
providers: [
{
provide: APP_INTERCEPTOR,
useClass: CacheInterceptor,
},
],
imports: this.modules,
controllers: this.config.flavor.selfhosted ? [] : [AppController],
})
class AppModule {}
return AppModule;
}
}
function buildAppModule() {
const factor = new AppModuleBuilder(AFFiNE);
factor
// common fundamental modules
.use(...FunctionalityModules)
// auth
.use(AuthModule)
// business modules
.use(DocModule)
// sync server only
.useIf(config => config.flavor.sync, SyncModule)
// main server only
.useIf(
config => config.flavor.main,
ServerConfigModule,
WebSocketModule,
GqlModule,
StorageModule,
UsersModule,
WorkspaceModule,
FeatureModule,
QuotaModule
)
// self hosted server only
.useIf(
config => config.flavor.selfhosted,
ServeStaticModule.forRoot({
rootPath: join('/app', 'static'),
})
);
// plugin modules
AFFiNE.plugins.enabled.forEach(name => {
const plugin = pluginsMap.get(name as AvailablePlugins);
if (!plugin) {
throw new Error(`Unknown plugin ${name}`);
}
factor.use(plugin);
});
return factor.compile();
}
export const AppModule = buildAppModule();

View File

@ -1,10 +1,11 @@
import { Type } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import type { NestExpressApplication } from '@nestjs/platform-express';
import cookieParser from 'cookie-parser';
import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
import { CacheRedis } from './fundamentals/cache/redis';
import { RedisIoAdapter } from './fundamentals/websocket';
import { SocketIoAdapter } from './fundamentals';
import { SocketIoAdapterImpl } from './fundamentals/websocket';
import { ExceptionLogger } from './middleware/exception-logger';
import { serverTimingAndCache } from './middleware/timing';
@ -31,11 +32,16 @@ export async function createApp() {
app.useGlobalFilters(new ExceptionLogger());
app.use(cookieParser());
if (AFFiNE.redis.enabled) {
const redis = app.get(CacheRedis, { strict: false });
const redisIoAdapter = new RedisIoAdapter(app);
await redisIoAdapter.connectToRedis(redis);
app.useWebSocketAdapter(redisIoAdapter);
if (AFFiNE.flavor.sync) {
const SocketIoAdapter = app.get<Type<SocketIoAdapter>>(
SocketIoAdapterImpl,
{
strict: false,
}
);
const adapter = new SocketIoAdapter(app);
app.useWebSocketAdapter(adapter);
}
return app;

View File

@ -21,20 +21,19 @@ AFFiNE.ENV_MAP = {
OAUTH_EMAIL_PASSWORD: 'auth.email.password',
THROTTLE_TTL: ['rateLimiter.ttl', 'int'],
THROTTLE_LIMIT: ['rateLimiter.limit', 'int'],
REDIS_SERVER_ENABLED: ['redis.enabled', 'boolean'],
REDIS_SERVER_HOST: 'redis.host',
REDIS_SERVER_PORT: ['redis.port', 'int'],
REDIS_SERVER_USER: 'redis.username',
REDIS_SERVER_PASSWORD: 'redis.password',
REDIS_SERVER_DATABASE: ['redis.database', 'int'],
REDIS_SERVER_HOST: 'plugins.redis.host',
REDIS_SERVER_PORT: ['plugins.redis.port', 'int'],
REDIS_SERVER_USER: 'plugins.redis.username',
REDIS_SERVER_PASSWORD: 'plugins.redis.password',
REDIS_SERVER_DATABASE: ['plugins.redis.db', 'int'],
DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
DOC_MERGE_USE_JWST_CODEC: [
'doc.manager.experimentalMergeWithJwstCodec',
'boolean',
],
ENABLE_LOCAL_EMAIL: ['auth.localEmail', 'boolean'],
STRIPE_API_KEY: 'payment.stripe.keys.APIKey',
STRIPE_WEBHOOK_KEY: 'payment.stripe.keys.webhookKey',
STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey',
STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey',
FEATURES_EARLY_ACCESS_PREVIEW: ['featureFlags.earlyAccessPreview', 'boolean'],
};

View File

@ -1,32 +1,38 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
// Custom configurations
const env = process.env;
const node = AFFiNE.node;
// TODO(@forehalo): detail explained
if (node.prod) {
// Storage
if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
AFFiNE.storage.providers.r2 = {
accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
credentials: {
accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
},
};
AFFiNE.storage.storages.avatar.provider = 'r2';
AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
`https://avatar.affineassets.com/${key}`;
// Storage
if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
AFFiNE.storage.providers.r2 = {
accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
credentials: {
accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
},
};
AFFiNE.storage.storages.avatar.provider = 'r2';
AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
`https://avatar.affineassets.com/${key}`;
AFFiNE.storage.storages.blob.provider = 'r2';
AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
AFFiNE.affine.canary ? 'canary' : 'prod'
}`;
}
// Metrics
AFFiNE.metrics.enabled = true;
AFFiNE.storage.storages.blob.provider = 'r2';
AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
AFFiNE.affine.canary ? 'canary' : 'prod'
}`;
}
// Metrics
AFFiNE.metrics.enabled = true;
// Plugins Section Start
AFFiNE.plugins.use('payment', {
stripe: {
apiVersion: '2023-10-16',
},
});
AFFiNE.plugins.use('redis');
// Plugins Section end
export default AFFiNE;

View File

@ -0,0 +1,58 @@
import { Module } from '@nestjs/common';
import { Field, ObjectType, Query, registerEnumType } from '@nestjs/graphql';
export enum ServerFeature {
Payment = 'payment',
}
registerEnumType(ServerFeature, {
name: 'ServerFeature',
});
const ENABLED_FEATURES: ServerFeature[] = [];
export function ADD_ENABLED_FEATURES(feature: ServerFeature) {
ENABLED_FEATURES.push(feature);
}
@ObjectType()
export class ServerConfigType {
@Field({
description:
'server identical name could be shown as badge on user interface',
})
name!: string;
@Field({ description: 'server version' })
version!: string;
@Field({ description: 'server base url' })
baseUrl!: string;
/**
* @deprecated
*/
@Field({ description: 'server flavor', deprecationReason: 'use `features`' })
flavor!: string;
@Field(() => [ServerFeature], { description: 'enabled server features' })
features!: ServerFeature[];
}
export class ServerConfigResolver {
@Query(() => ServerConfigType, {
description: 'server config',
})
serverConfig(): ServerConfigType {
return {
name: AFFiNE.serverName,
version: AFFiNE.version,
baseUrl: AFFiNE.baseUrl,
flavor: AFFiNE.flavor.type,
features: ENABLED_FEATURES,
};
}
}
@Module({
providers: [ServerConfigResolver],
})
export class ServerConfigModule {}

View File

@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { PrismaService } from '../../fundamentals';
import { type EventPayload, OnEvent, PrismaService } from '../../fundamentals';
import { FeatureKind } from '../features';
import { QuotaConfig } from './quota';
import { QuotaType } from './types';
@ -155,4 +155,26 @@ export class QuotaService {
})
.then(count => count > 0);
}
@OnEvent('user.subscription.activated')
async onSubscriptionUpdated({
userId,
}: EventPayload<'user.subscription.activated'>) {
await this.switchUserQuota(
userId,
QuotaType.ProPlanV1,
'subscription activated'
);
}
@OnEvent('user.subscription.canceled')
async onSubscriptionCanceled(
userId: EventPayload<'user.subscription.canceled'>
) {
await this.switchUserQuota(
userId,
QuotaType.FreePlanV1,
'subscription canceled'
);
}
}

View File

@ -13,6 +13,9 @@ import { RevertCommand, RunCommand } from './commands/run';
enableUpdateAutoMerging: false,
},
},
metrics: {
enabled: false,
},
}),
BusinessAppModule,
],

View File

@ -8,7 +8,7 @@ export class SelfHostAdmin1605053000403 {
// do the migration
static async up(db: PrismaClient, ref: ModuleRef) {
const config = ref.get(Config, { strict: false });
if (config.flavor === 'selfhosted') {
if (config.flavor.selfhosted) {
if (
!process.env.AFFINE_ADMIN_EMAIL ||
!process.env.AFFINE_ADMIN_PASSWORD

View File

@ -1,7 +1,7 @@
import { PrismaClient } from '@prisma/client';
import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs';
import { DocID } from '../../modules/utils/doc';
import { DocID } from '../../core/utils/doc';
export class Guid1698398506533 {
// do the migration

View File

@ -1,7 +1,7 @@
import { PrismaClient } from '@prisma/client';
import { Features } from '../../modules/features';
import { Quotas } from '../../modules/quota/schema';
import { Features } from '../../core/features';
import { Quotas } from '../../core/quota/schema';
import { migrateNewFeatureTable, upsertFeature } from './utils/user-features';
export class UserFeaturesInit1698652531198 {

View File

@ -1,6 +1,6 @@
import { PrismaClient } from '@prisma/client';
import { QuotaType } from '../../modules/quota/types';
import { QuotaType } from '../../core/quota/types';
export class OldUserFeature1702620653283 {
// do the migration
static async up(db: PrismaClient) {

View File

@ -1,6 +1,4 @@
import { PrismaClient } from '@prisma/client';
import type { UserType } from '../../modules/users';
import { PrismaClient, type User } from '@prisma/client';
export class UnamedAccount1703756315970 {
// do the migration
@ -8,7 +6,7 @@ export class UnamedAccount1703756315970 {
await db.$transaction(async tx => {
// only find users with empty names
const users = await db.$queryRaw<
UserType[]
User[]
>`SELECT * FROM users WHERE name ~ E'^[\\s\\u2000-\\u200F]*$';`;
console.log(
`renaming ${users.map(({ email }) => email).join('|')} users`

View File

@ -1,7 +1,7 @@
import { ModuleRef } from '@nestjs/core';
import { PrismaClient } from '@prisma/client';
import { WorkspaceBlobStorage } from '../../modules/storage';
import { WorkspaceBlobStorage } from '../../core/storage';
export class WorkspaceBlobs1703828796699 {
// do the migration

View File

@ -1,6 +1,6 @@
import { PrismaClient } from '@prisma/client';
import { Features } from '../../modules/features';
import { Features } from '../../core/features';
import { upsertFeature } from './utils/user-features';
export class RefreshUserFeatures1704352562369 {

View File

@ -1,7 +1,7 @@
import { PrismaClient } from '@prisma/client';
import { FeatureKind } from '../../modules/features';
import { Quotas } from '../../modules/quota';
import { FeatureKind } from '../../core/features';
import { Quotas } from '../../core/quota';
import { upsertFeature } from './utils/user-features';
export class NewFreePlan1705395933447 {

View File

@ -4,7 +4,7 @@ import {
CommonFeature,
FeatureKind,
FeatureType,
} from '../../../modules/features';
} from '../../../core/features';
// upgrade features from lower version to higher version
export async function upsertFeature(

View File

@ -0,0 +1,49 @@
export interface CacheSetOptions {
// in milliseconds
ttl?: number;
}
// extends if needed
export interface Cache {
// standard operation
get<T = unknown>(key: string): Promise<T | undefined>;
set<T = unknown>(
key: string,
value: T,
opts?: CacheSetOptions
): Promise<boolean>;
setnx<T = unknown>(
key: string,
value: T,
opts?: CacheSetOptions
): Promise<boolean>;
increase(key: string, count?: number): Promise<number>;
decrease(key: string, count?: number): Promise<number>;
delete(key: string): Promise<boolean>;
has(key: string): Promise<boolean>;
ttl(key: string): Promise<number>;
expire(key: string, ttl: number): Promise<boolean>;
// list operations
pushBack<T = unknown>(key: string, ...values: T[]): Promise<number>;
pushFront<T = unknown>(key: string, ...values: T[]): Promise<number>;
len(key: string): Promise<number>;
list<T = unknown>(key: string, start: number, end: number): Promise<T[]>;
popFront<T = unknown>(key: string, count?: number): Promise<T[]>;
popBack<T = unknown>(key: string, count?: number): Promise<T[]>;
// map operations
mapSet<T = unknown>(
map: string,
key: string,
value: T,
opts: CacheSetOptions
): Promise<boolean>;
mapIncrease(map: string, key: string, count?: number): Promise<number>;
mapDecrease(map: string, key: string, count?: number): Promise<number>;
mapGet<T = unknown>(map: string, key: string): Promise<T | undefined>;
mapDelete(map: string, key: string): Promise<boolean>;
mapKeys(map: string): Promise<string[]>;
mapRandomKey(map: string): Promise<string | undefined>;
mapLen(map: string): Promise<number>;
}

View File

@ -1,35 +1,13 @@
import { Global, Module, Provider, Type } from '@nestjs/common';
import { Redis } from 'ioredis';
import { Global, Module } from '@nestjs/common';
import { SessionCache, ThrottlerCache } from './instances';
import { LocalCache } from './providers/cache';
import { RedisCache } from './providers/redis';
import { CacheRedis, RedisModule, SessionRedis, ThrottlerRedis } from './redis';
function makeCacheProvider(CacheToken: Type, RedisToken: Type): Provider {
return {
provide: CacheToken,
useFactory: (redis?: Redis) => {
return redis ? new RedisCache(redis) : new LocalCache();
},
inject: [{ token: RedisToken, optional: true }],
};
}
const CacheProvider = makeCacheProvider(LocalCache, CacheRedis);
const SessionCacheProvider = makeCacheProvider(SessionCache, SessionRedis);
const ThrottlerCacheProvider = makeCacheProvider(
ThrottlerCache,
ThrottlerRedis
);
import { Cache, SessionCache } from './instances';
@Global()
@Module({
imports: AFFiNE.redis.enabled ? [RedisModule] : [],
providers: [CacheProvider, SessionCacheProvider, ThrottlerCacheProvider],
exports: [CacheProvider, SessionCacheProvider, ThrottlerCacheProvider],
providers: [Cache, SessionCache],
exports: [Cache, SessionCache],
})
export class CacheModule {}
export { LocalCache as Cache, SessionCache, ThrottlerCache };
export { Cache, SessionCache };
export { CacheInterceptor, MakeCache, PreventCache } from './interceptor';

View File

@ -1,4 +1,13 @@
import { LocalCache } from './providers/cache';
import { Injectable } from '@nestjs/common';
export class SessionCache extends LocalCache {}
export class ThrottlerCache extends LocalCache {}
import { LocalCache } from './local';
@Injectable()
export class Cache extends LocalCache {}
@Injectable()
export class SessionCache extends LocalCache {
constructor() {
super({ namespace: 'session' });
}
}

View File

@ -10,7 +10,7 @@ import { Reflector } from '@nestjs/core';
import { GqlContextType, GqlExecutionContext } from '@nestjs/graphql';
import { mergeMap, Observable, of } from 'rxjs';
import { LocalCache } from './providers/cache';
import { Cache } from './instances';
export const MakeCache = (key: string[], args?: string[]) =>
SetMetadata('cacheKey', [key, args]);
@ -24,7 +24,7 @@ export class CacheInterceptor implements NestInterceptor {
private readonly logger = new Logger(CacheInterceptor.name);
constructor(
private readonly reflector: Reflector,
private readonly cache: LocalCache
private readonly cache: Cache
) {}
async intercept(
ctx: ExecutionContext,

View File

@ -1,57 +1,7 @@
import { Injectable } from '@nestjs/common';
import Keyv from 'keyv';
export interface CacheSetOptions {
// in milliseconds
ttl?: number;
}
import type { Cache, CacheSetOptions } from './def';
// extends if needed
export interface Cache {
// standard operation
get<T = unknown>(key: string): Promise<T | undefined>;
set<T = unknown>(
key: string,
value: T,
opts?: CacheSetOptions
): Promise<boolean>;
setnx<T = unknown>(
key: string,
value: T,
opts?: CacheSetOptions
): Promise<boolean>;
increase(key: string, count?: number): Promise<number>;
decrease(key: string, count?: number): Promise<number>;
delete(key: string): Promise<boolean>;
has(key: string): Promise<boolean>;
ttl(key: string): Promise<number>;
expire(key: string, ttl: number): Promise<boolean>;
// list operations
pushBack<T = unknown>(key: string, ...values: T[]): Promise<number>;
pushFront<T = unknown>(key: string, ...values: T[]): Promise<number>;
len(key: string): Promise<number>;
list<T = unknown>(key: string, start: number, end: number): Promise<T[]>;
popFront<T = unknown>(key: string, count?: number): Promise<T[]>;
popBack<T = unknown>(key: string, count?: number): Promise<T[]>;
// map operations
mapSet<T = unknown>(
map: string,
key: string,
value: T,
opts: CacheSetOptions
): Promise<boolean>;
mapIncrease(map: string, key: string, count?: number): Promise<number>;
mapDecrease(map: string, key: string, count?: number): Promise<number>;
mapGet<T = unknown>(map: string, key: string): Promise<T | undefined>;
mapDelete(map: string, key: string): Promise<boolean>;
mapKeys(map: string): Promise<string[]>;
mapRandomKey(map: string): Promise<string | undefined>;
mapLen(map: string): Promise<number>;
}
@Injectable()
export class LocalCache implements Cache {
private readonly kv: Keyv;

View File

@ -1,38 +0,0 @@
import { Global, Injectable, Module, OnModuleDestroy } from '@nestjs/common';
import { Redis as IORedis } from 'ioredis';
import { Config } from '../../config';
class Redis extends IORedis implements OnModuleDestroy {
onModuleDestroy() {
this.disconnect();
}
}
@Injectable()
export class CacheRedis extends Redis {
constructor(config: Config) {
super({ ...config.redis, db: config.redis.database });
}
}
@Injectable()
export class ThrottlerRedis extends Redis {
constructor(config: Config) {
super({ ...config.redis, db: config.redis.database + 1 });
}
}
@Injectable()
export class SessionRedis extends Redis {
constructor(config: Config) {
super({ ...config.redis, db: config.redis.database + 2 });
}
}
@Global()
@Module({
providers: [CacheRedis, ThrottlerRedis, SessionRedis],
exports: [CacheRedis, ThrottlerRedis, SessionRedis],
})
export class RedisModule {}

View File

@ -10,13 +10,6 @@ declare global {
// eslint-disable-next-line no-var
var AFFiNE: AFFiNEConfig;
}
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace NodeJS {
interface ProcessEnv {
SERVER_FLAVOR: ServerFlavor | '';
}
}
}
export enum ExternalAccount {
@ -25,22 +18,28 @@ export enum ExternalAccount {
firebase = 'firebase',
}
export type ServerFlavor = 'allinone' | 'graphql' | 'sync' | 'selfhosted';
type ConfigPaths = LeafPaths<
export type ServerFlavor =
| 'allinone'
| 'main'
// @deprecated
| 'graphql'
| 'sync'
| 'selfhosted';
export type ConfigPaths = LeafPaths<
Omit<
AFFiNEConfig,
| 'ENV_MAP'
| 'version'
| 'baseUrl'
| 'origin'
| 'prod'
| 'dev'
| 'test'
| 'flavor'
| 'env'
| 'affine'
| 'deploy'
| 'node'
| 'baseUrl'
| 'origin'
>,
'',
'....'
'.....'
>;
/**
@ -52,15 +51,28 @@ export interface AFFiNEConfig {
/**
* Server Identity
*/
readonly serverId: string;
serverId: string;
/**
* Name may show on the UI
*/
serverName: string;
/**
* System version
*/
readonly version: string;
/**
* Server flavor
*/
readonly flavor: ServerFlavor;
get flavor(): {
type: string;
main: boolean;
sync: boolean;
selfhosted: boolean;
};
/**
* Deployment environment
*/
@ -172,38 +184,6 @@ export interface AFFiNEConfig {
limit: number;
};
/**
* Redis Config
*
* whether to use redis as Socket.IO adapter
*/
redis: {
/**
* if not enabled, use in-memory adapter by default
*/
enabled: boolean;
/**
* url of redis host
*/
host: string;
/**
* port of redis
*/
port: number;
username: string;
password: string;
/**
* redis database index
*
* Rate Limiter scope: database + 1
*
* Session scope: database + 2
*
* @default 0
*/
database: number;
};
/**
* authentication config
*/
@ -341,15 +321,6 @@ export interface AFFiNEConfig {
metrics: {
enabled: boolean;
};
payment: {
stripe: {
keys: {
APIKey: string;
webhookKey: string;
};
} & import('stripe').Stripe.StripeConfig;
};
}
export * from './storage';

View File

@ -2,6 +2,7 @@
import { createPrivateKey, createPublicKey } from 'node:crypto';
import { merge } from 'lodash-es';
import parse from 'parse-duration';
import pkg from '../../../package.json' assert { type: 'json' };
@ -46,11 +47,22 @@ const jwtKeyPair = (function () {
export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
let isHttps: boolean | null = null;
const flavor = (process.env.SERVER_FLAVOR ?? 'allinone') as ServerFlavor;
let flavor = (process.env.SERVER_FLAVOR ?? 'allinone') as ServerFlavor;
const defaultConfig = {
serverId: 'affine-nestjs-server',
serverName: flavor === 'selfhosted' ? 'Self-Host Cloud' : 'AFFiNE Cloud',
version: pkg.version,
flavor,
get flavor() {
if (flavor === 'graphql') {
flavor = 'main';
}
return {
type: flavor,
main: flavor === 'main' || flavor === 'allinone',
sync: flavor === 'sync' || flavor === 'allinone',
selfhosted: flavor === 'selfhosted',
};
},
ENV_MAP: {},
affineEnv: 'dev',
get affine() {
@ -142,15 +154,7 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
storage: getDefaultAFFiNEStorageConfig(),
rateLimiter: {
ttl: 60,
limit: 60,
},
redis: {
enabled: false,
host: '127.0.0.1',
port: 6379,
username: '',
password: '',
database: 0,
limit: 120,
},
doc: {
manager: {
@ -162,18 +166,16 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
interval: 1000 * 60 * 10 /* 10 mins */,
},
},
payment: {
stripe: {
keys: {
APIKey: '',
webhookKey: '',
},
apiVersion: '2023-10-16',
},
},
metrics: {
enabled: false,
},
plugins: {
enabled: [],
use(plugin, config) {
this[plugin] = merge(this[plugin], config || {});
this.enabled.push(plugin);
},
},
} satisfies AFFiNEConfig;
return defaultConfig;

View File

@ -1,7 +1,7 @@
import { DynamicModule, FactoryProvider } from '@nestjs/common';
import { merge } from 'lodash-es';
import { ApplyType, DeepPartial } from '../utils/types';
import { ApplyType } from '../utils/types';
import { AFFiNEConfig } from './def';
/**

View File

@ -0,0 +1,51 @@
import type { Snapshot, User, Workspace } from '@prisma/client';
import { Flatten, Payload } from './types';
export interface WorkspaceEvents {
deleted: Payload<Workspace['id']>;
blob: {
deleted: Payload<{
workspaceId: Workspace['id'];
name: string;
}>;
};
}
export interface DocEvents {
updated: Payload<
Pick<Snapshot, 'id' | 'workspaceId'> & {
previous: Pick<Snapshot, 'blob' | 'state' | 'updatedAt'>;
}
>;
deleted: Payload<Pick<Snapshot, 'id' | 'workspaceId'>>;
}
export interface UserEvents {
deleted: Payload<User>;
}
/**
* Event definitions can be extended by
*
* @example
*
* declare module './event/def' {
* interface UserEvents {
* created: Payload<User>;
* }
* }
*
* assert<Event, 'user.created'>()
*/
export interface EventDefinitions {
workspace: WorkspaceEvents;
snapshot: DocEvents;
user: UserEvents;
}
export type EventKV = Flatten<EventDefinitions>;
export type Event = keyof EventKV;
export type EventPayload<E extends Event> = EventKV[E];
export type { Payload };

View File

@ -1,33 +0,0 @@
import type { Snapshot, User, Workspace } from '@prisma/client';
import { Flatten, Payload } from './types';
interface EventDefinitions {
workspace: {
deleted: Payload<Workspace['id']>;
blob: {
deleted: Payload<{
workspaceId: Workspace['id'];
name: string;
}>;
};
};
snapshot: {
updated: Payload<
Pick<Snapshot, 'id' | 'workspaceId'> & {
previous: Pick<Snapshot, 'blob' | 'state' | 'updatedAt'>;
}
>;
deleted: Payload<Pick<Snapshot, 'id' | 'workspaceId'>>;
};
user: {
deleted: Payload<User>;
};
}
export type EventKV = Flatten<EventDefinitions>;
export type Event = keyof EventKV;
export type EventPayload<E extends Event> = EventKV[E];

View File

@ -5,7 +5,7 @@ import {
OnEvent as RawOnEvent,
} from '@nestjs/event-emitter';
import type { Event, EventPayload } from './events';
import type { Event, EventPayload } from './def';
@Injectable()
export class EventEmitter {
@ -40,4 +40,4 @@ export const OnEvent = RawOnEvent as (
exports: [EventEmitter],
})
export class EventModule {}
export { EventPayload };
export { Event, EventPayload };

View File

@ -28,6 +28,7 @@ import { GQLLoggerPlugin } from './logger-plugin';
? '../../../../node_modules/.cache/schema.gql'
: '../../../schema.gql'
),
sortSchema: true,
context: ({ req, res }: { req: Request; res: Response }) => ({
req,
res,

View File

@ -1,12 +1,20 @@
export { Cache, CacheInterceptor, MakeCache, PreventCache } from './cache';
export {
Cache,
CacheInterceptor,
MakeCache,
PreventCache,
SessionCache,
} from './cache';
export {
applyEnvToConfig,
Config,
type ConfigPaths,
getDefaultAFFiNEStorageConfig,
} from './config';
export { EventEmitter, type EventPayload, OnEvent } from './event';
export { MailService } from './mailer';
export { CallCounter, CallTimer, metrics } from './metrics';
export { getOptionalModuleMetadata, OptionalModule } from './nestjs';
export { PrismaService } from './prisma';
export { SessionService } from './session';
export * from './storage';
@ -17,4 +25,4 @@ export {
getRequestResponseFromHost,
} from './utils/request';
export type * from './utils/types';
export { RedisIoAdapter } from './websocket';
export { SocketIoAdapter } from './websocket';

View File

@ -0,0 +1 @@
export * from './optional-module';

View File

@ -0,0 +1,71 @@
import {
DynamicModule,
Module,
ModuleMetadata,
Provider,
Type,
} from '@nestjs/common';
import { omit } from 'lodash-es';
import { Config, ConfigPaths } from '../config';
interface OptionalModuleMetadata extends ModuleMetadata {
/**
* Only install module if given config paths are defined in AFFiNE config.
*/
requires?: ConfigPaths[];
/**
* Only install module if the predication returns true.
*/
if?: (config: Config) => boolean;
/**
* Defines which feature will be enabled if the module installed.
*/
contributesTo?: import('../../core/config').ServerFeature; // avoid circlar dependency
/**
* Defines which providers provided by other modules will be overridden if the module installed.
*/
overrides?: Provider[];
}
const additionalOptions = [
'contributesTo',
'requires',
'if',
'overrides',
] as const satisfies Array<keyof OptionalModuleMetadata>;
type OptionalDynamicModule = DynamicModule & OptionalModuleMetadata;
export function OptionalModule(metadata: OptionalModuleMetadata) {
return (target: Type) => {
additionalOptions.forEach(option => {
if (Object.hasOwn(metadata, option)) {
Reflect.defineMetadata(option, metadata[option], target);
}
});
if (metadata.overrides) {
metadata.providers = (metadata.providers ?? []).concat(
metadata.overrides
);
metadata.exports = (metadata.exports ?? []).concat(metadata.overrides);
}
const nestMetadata = omit(metadata, additionalOptions);
Module(nestMetadata)(target);
};
}
export function getOptionalModuleMetadata<
T extends keyof OptionalModuleMetadata,
>(target: Type | OptionalDynamicModule, key: T): OptionalModuleMetadata[T] {
if ('module' in target) {
return target[key];
} else {
return Reflect.getMetadata(key, target);
}
}

View File

@ -1,4 +1,4 @@
import { ExecutionContext, Injectable, Logger } from '@nestjs/common';
import { ExecutionContext, Injectable } from '@nestjs/common';
import { Global, Module } from '@nestjs/common';
import {
Throttle,
@ -6,40 +6,36 @@ import {
ThrottlerModule,
ThrottlerModuleOptions,
ThrottlerOptionsFactory,
ThrottlerStorageService,
} from '@nestjs/throttler';
import { ThrottlerStorageRedisService } from 'nestjs-throttler-storage-redis';
import { ThrottlerCache } from '../cache';
import { Config } from '../config';
import { getRequestResponseFromContext } from '../utils/request';
@Injectable()
export class ThrottlerStorage extends ThrottlerStorageService {}
@Injectable()
class CustomOptionsFactory implements ThrottlerOptionsFactory {
constructor(
private readonly config: Config,
private readonly cache: ThrottlerCache
private readonly storage: ThrottlerStorage
) {}
createThrottlerOptions() {
const options: ThrottlerModuleOptions = {
throttlers: [
{
ttl: this.config.rateLimiter.ttl,
ttl: this.config.rateLimiter.ttl * 1000,
limit: this.config.rateLimiter.limit,
},
],
skipIf: () => {
return !this.config.node.prod || this.config.affine.canary;
},
storage: this.storage,
};
if (this.config.redis.enabled) {
new Logger(RateLimiterModule.name).log('Use Redis');
options.storage = new ThrottlerStorageRedisService(
// @ts-expect-error hidden field
this.cache.redis
);
}
return options;
}
}
@ -51,6 +47,8 @@ class CustomOptionsFactory implements ThrottlerOptionsFactory {
useClass: CustomOptionsFactory,
}),
],
providers: [ThrottlerStorage],
exports: [ThrottlerStorage],
})
export class RateLimiterModule {}

View File

@ -1,9 +1,5 @@
import { Readable } from 'node:stream';
export type ConstructorOf<T> = {
new (): T;
};
export function ApplyType<T>(): ConstructorOf<T> {
// @ts-expect-error used to fake the type of config
return class Inner implements T {
@ -11,16 +7,6 @@ export function ApplyType<T>(): ConstructorOf<T> {
};
}
export type DeepPartial<T> = T extends Array<infer U>
? DeepPartial<U>[]
: T extends ReadonlyArray<infer U>
? ReadonlyArray<DeepPartial<U>>
: T extends object
? {
[K in keyof T]?: DeepPartial<T[K]>;
}
: T;
type Join<Prefix, Suffixes> = Prefix extends string | number
? Suffixes extends string | number
? Prefix extends ''
@ -29,14 +15,6 @@ type Join<Prefix, Suffixes> = Prefix extends string | number
: never
: never;
export type PrimitiveType =
| string
| number
| boolean
| symbol
| null
| undefined;
export type LeafPaths<
T,
Path extends string = '',

View File

@ -1 +1,17 @@
export { RedisIoAdapter } from './redis-adapter';
import { Module, Provider } from '@nestjs/common';
import { IoAdapter } from '@nestjs/platform-socket.io';
export const SocketIoAdapterImpl = Symbol('SocketIoAdapterImpl');
export class SocketIoAdapter extends IoAdapter {}
const SocketIoAdapterImplProvider: Provider = {
provide: SocketIoAdapterImpl,
useValue: SocketIoAdapter,
};
@Module({
providers: [SocketIoAdapterImplProvider],
exports: [SocketIoAdapterImplProvider],
})
export class WebSocketModule {}

View File

@ -1,27 +0,0 @@
import { IoAdapter } from '@nestjs/platform-socket.io';
import { createAdapter } from '@socket.io/redis-adapter';
import { Redis } from 'ioredis';
import { ServerOptions } from 'socket.io';
export class RedisIoAdapter extends IoAdapter {
private adapterConstructor: ReturnType<typeof createAdapter> | undefined;
async connectToRedis(redis: Redis): Promise<void> {
const pubClient = redis;
pubClient.on('error', err => {
console.error(err);
});
const subClient = pubClient.duplicate();
subClient.on('error', err => {
console.error(err);
});
this.adapterConstructor = createAdapter(pubClient, subClient);
}
override createIOServer(port: number, options?: ServerOptions): any {
const server = super.createIOServer(port, options);
server.adapter(this.adapterConstructor);
return server;
}
}

View File

@ -3,3 +3,29 @@ declare namespace Express {
user?: import('@prisma/client').User | null;
}
}
declare type PrimitiveType =
| string
| number
| boolean
| symbol
| null
| undefined;
declare type ConstructorOf<T> = {
new (): T;
};
declare type DeepPartial<T> = T extends Array<infer U>
? DeepPartial<U>[]
: T extends ReadonlyArray<infer U>
? ReadonlyArray<DeepPartial<U>>
: T extends object
? {
[K in keyof T]?: DeepPartial<T[K]>;
}
: T;
declare type AFFiNEModule =
| import('@nestjs/common').Type
| import('@nestjs/common').DynamicModule;

View File

@ -1,32 +0,0 @@
import { Module } from '@nestjs/common';
import { Field, ObjectType, Query } from '@nestjs/graphql';
@ObjectType()
export class ServerConfigType {
@Field({ description: 'server version' })
version!: string;
@Field({ description: 'server flavor' })
flavor!: string;
@Field({ description: 'server base url' })
baseUrl!: string;
}
export class ServerConfigResolver {
@Query(() => ServerConfigType, {
description: 'server config',
})
serverConfig(): ServerConfigType {
return {
version: AFFiNE.version,
flavor: AFFiNE.flavor,
baseUrl: AFFiNE.baseUrl,
};
}
}
@Module({
providers: [ServerConfigResolver],
})
export class ServerConfigModule {}

View File

@ -1,70 +0,0 @@
import { join } from 'node:path';
import { DynamicModule, Type } from '@nestjs/common';
import { ScheduleModule } from '@nestjs/schedule';
import { ServeStaticModule } from '@nestjs/serve-static';
import { GqlModule } from '../fundamentals/graphql';
import { ServerConfigModule } from './config';
import { DocModule } from './doc';
import { PaymentModule } from './payment';
import { QuotaModule } from './quota';
import { SelfHostedModule } from './self-hosted';
import { StorageModule } from './storage';
import { SyncModule } from './sync';
import { UsersModule } from './users';
import { WorkspaceModule } from './workspaces';
const BusinessModules: (Type | DynamicModule)[] = [];
switch (AFFiNE.flavor) {
case 'sync':
BusinessModules.push(SyncModule, DocModule);
break;
case 'selfhosted':
BusinessModules.push(
ServerConfigModule,
SelfHostedModule,
ScheduleModule.forRoot(),
GqlModule,
WorkspaceModule,
UsersModule,
SyncModule,
DocModule,
StorageModule,
ServeStaticModule.forRoot({
rootPath: join('/app', 'static'),
})
);
break;
case 'graphql':
BusinessModules.push(
ServerConfigModule,
ScheduleModule.forRoot(),
GqlModule,
WorkspaceModule,
UsersModule,
DocModule,
PaymentModule,
QuotaModule,
StorageModule
);
break;
case 'allinone':
default:
BusinessModules.push(
ServerConfigModule,
ScheduleModule.forRoot(),
GqlModule,
WorkspaceModule,
UsersModule,
QuotaModule,
SyncModule,
DocModule,
PaymentModule,
StorageModule
);
break;
}
export { BusinessModules };

View File

@ -1,38 +0,0 @@
import { Module } from '@nestjs/common';
import { ResolveField, Resolver } from '@nestjs/graphql';
import { UserSubscriptionType } from './payment/resolver';
import {
SubscriptionPlan,
SubscriptionRecurring,
SubscriptionStatus,
} from './payment/service';
import { UserType } from './users';
const YEAR = 1000 * 60 * 60 * 24 * 30 * 12;
@Resolver(() => UserType)
export class SelfHostedDummyResolver {
private readonly start = new Date();
private readonly end = new Date(Number(this.start) + YEAR);
constructor() {}
@ResolveField(() => UserSubscriptionType)
async subscription() {
return {
stripeSubscriptionId: 'dummy',
plan: SubscriptionPlan.SelfHosted,
recurring: SubscriptionRecurring.Yearly,
status: SubscriptionStatus.Active,
start: this.start,
end: this.end,
createdAt: this.start,
updatedAt: this.start,
};
}
}
@Module({
providers: [SelfHostedDummyResolver],
})
export class SelfHostedModule {}

View File

@ -0,0 +1,21 @@
import { PaymentConfig } from './payment';
import { RedisOptions } from './redis';
declare module '../fundamentals/config' {
interface PluginsConfig {
readonly payment: PaymentConfig;
readonly redis: RedisOptions;
}
export type AvailablePlugins = keyof PluginsConfig;
interface AFFiNEConfig {
readonly plugins: {
enabled: AvailablePlugins[];
use<Plugin extends AvailablePlugins>(
plugin: Plugin,
config?: DeepPartial<PluginsConfig[Plugin]>
): void;
} & Partial<PluginsConfig>;
}
}

View File

@ -0,0 +1,8 @@
import type { AvailablePlugins } from '../fundamentals/config';
import { PaymentModule } from './payment';
import { RedisModule } from './redis';
export const pluginsMap = new Map<AvailablePlugins, AFFiNEModule>([
['payment', PaymentModule],
['redis', RedisModule],
]);

View File

@ -1,15 +1,14 @@
import { Module } from '@nestjs/common';
import { FeatureModule } from '../features';
import { QuotaModule } from '../quota';
import { ServerFeature } from '../../core/config';
import { FeatureModule } from '../../core/features';
import { OptionalModule } from '../../fundamentals';
import { SubscriptionResolver, UserSubscriptionResolver } from './resolver';
import { ScheduleManager } from './schedule';
import { SubscriptionService } from './service';
import { StripeProvider } from './stripe';
import { StripeWebhook } from './webhook';
@Module({
imports: [FeatureModule, QuotaModule],
@OptionalModule({
imports: [FeatureModule],
providers: [
ScheduleManager,
StripeProvider,
@ -18,5 +17,12 @@ import { StripeWebhook } from './webhook';
UserSubscriptionResolver,
],
controllers: [StripeWebhook],
requires: [
'plugins.payment.stripe.keys.APIKey',
'plugins.payment.stripe.keys.webhookKey',
],
contributesTo: ServerFeature.Payment,
})
export class PaymentModule {}
export type { PaymentConfig } from './types';

View File

@ -16,17 +16,16 @@ import type { User, UserInvoice, UserSubscription } from '@prisma/client';
import { GraphQLError } from 'graphql';
import { groupBy } from 'lodash-es';
import { Auth, CurrentUser, Public } from '../../core/auth';
import { UserType } from '../../core/users';
import { Config, PrismaService } from '../../fundamentals';
import { Auth, CurrentUser, Public } from '../auth';
import { UserType } from '../users';
import { decodeLookupKey, SubscriptionService } from './service';
import {
decodeLookupKey,
InvoiceStatus,
SubscriptionPlan,
SubscriptionRecurring,
SubscriptionService,
SubscriptionStatus,
} from './service';
} from './types';
registerEnumType(SubscriptionStatus, { name: 'SubscriptionStatus' });
registerEnumType(SubscriptionRecurring, { name: 'SubscriptionRecurring' });
@ -251,7 +250,10 @@ export class SubscriptionResolver {
@Resolver(() => UserType)
export class UserSubscriptionResolver {
constructor(private readonly db: PrismaService) {}
constructor(
private readonly config: Config,
private readonly db: PrismaService
) {}
@ResolveField(() => UserSubscriptionType, { nullable: true })
async subscription(
@ -272,6 +274,25 @@ export class UserSubscriptionResolver {
);
}
// @FIXME(@forehalo): should not mock any api for selfhosted server
// the frontend should avoid calling such api if feature is not enabled
if (this.config.flavor.selfhosted) {
const start = new Date();
const end = new Date();
end.setFullYear(start.getFullYear() + 1);
return {
stripeSubscriptionId: 'dummy',
plan: SubscriptionPlan.SelfHosted,
recurring: SubscriptionRecurring.Yearly,
status: SubscriptionStatus.Active,
start,
end,
createdAt: start,
updatedAt: start,
};
}
return this.db.userSubscription.findUnique({
where: {
userId: user.id,

Some files were not shown because too many files have changed in this diff Show More