toeverything/AFFiNE
1
1
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2024-12-23 21:11:51 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): reorganize server configs (#5753)

Browse Source
This commit is contained in:
liuyi 2024-02-02 08:32:06 +00:00
parent ee3d195811
commit bef266ae3b
No known key found for this signature in database
GPG Key ID: 56709255DC7EC728
36 changed files with 423 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/deployment/self-host/compose.yaml vendored
View File

@ -27,9 +27,7 @@ services:
- AFFINE_CONFIG_PATH=/root/.affine/config
- REDIS_SERVER_HOST=redis
- DATABASE_URL=postgres://affine:affine@postgres:5432/affine
- DISABLE_TELEMETRY=true
- NODE_ENV=production
- SERVER_FLAVOR=selfhosted
- AFFINE_ADMIN_EMAIL=${AFFINE_ADMIN_EMAIL}
- AFFINE_ADMIN_PASSWORD=${AFFINE_ADMIN_PASSWORD}
redis:

2
.github/helm/affine/charts/graphql/templates/deployment.yaml vendored
View File

@ -39,6 +39,8 @@ spec:
value: "--max-old-space-size=4096"
- name: NO_COLOR
value: "1"
- name: DEPLOYMENT_TYPE
value: "affine"
- name: SERVER_FLAVOR
value: "graphql"
- name: AFFINE_ENV

2
.github/helm/affine/charts/sync/templates/deployment.yaml vendored
View File

@ -36,6 +36,8 @@ spec:
value: "{{ .Values.env }}"
- name: NO_COLOR
value: "1"
- name: DEPLOYMENT_TYPE
value: "affine"
- name: SERVER_FLAVOR
value: "sync"
- name: NEXTAUTH_URL

3
.github/workflows/build-test.yml vendored
View File

@ -19,7 +19,7 @@ env:
MACOSX_DEPLOYMENT_TARGET: '10.13'
NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
DISABLE_TELEMETRY: true
DEPLOYMENT_TYPE: affine
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -291,6 +291,7 @@ jobs:
runs-on: ubuntu-latest
needs: build-storage
env:
NODE_ENV: test
DISTRIBUTION: browser
services:
postgres:

1
packages/backend/server/package.json
View File

@ -162,7 +162,6 @@
"env": {
"TS_NODE_TRANSPILE_ONLY": true,
"TS_NODE_PROJECT": "./tsconfig.json",
"NODE_ENV": "development",
"DEBUG": "affine:*",
"FORCE_COLOR": true,
"DEBUG_COLORS": true

5
packages/backend/server/scripts/self-host-predeploy.js
View File

@ -13,7 +13,10 @@ const configFiles = [
];
function configCleaner(content) {
return content.replace(/(\/\/#.*$)|(\/\/\s+TODO.*$)/gm, '');
return content.replace(
/(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
''
);
}
function prepare() {

1
packages/backend/server/src/app.controller.ts
View File

@ -11,6 +11,7 @@ export class AppController {
return {
compatibility: this.config.version,
message: `AFFiNE ${this.config.version} Server`,
type: this.config.type,
flavor: this.config.flavor,
};
}

8
packages/backend/server/src/app.module.ts
View File

@ -109,7 +109,7 @@ export class AppModuleBuilder {
},
],
imports: this.modules,
controllers: this.config.flavor.selfhosted ? [] : [AppController],
controllers: this.config.isSelfhosted ? [] : [AppController],
})
class AppModule {}
@ -132,9 +132,9 @@ function buildAppModule() {
// sync server only
.useIf(config => config.flavor.sync, SyncModule)
// main server only
// graphql server only
.useIf(
config => config.flavor.main,
config => config.flavor.graphql,
ServerConfigModule,
WebSocketModule,
GqlModule,
@ -147,7 +147,7 @@ function buildAppModule() {
// self hosted server only
.useIf(
config => config.flavor.selfhosted,
config => config.isSelfhosted,
ServeStaticModule.forRoot({
rootPath: join('/app', 'static'),
})

7
packages/backend/server/src/config/affine.env.ts
View File

@ -3,8 +3,7 @@ AFFiNE.ENV_MAP = {
AFFINE_SERVER_PORT: ['port', 'int'],
AFFINE_SERVER_HOST: 'host',
AFFINE_SERVER_SUB_PATH: 'path',
AFFIHE_SERVER_HTTPS: ['https', 'boolean'],
AFFINE_ENV: 'affineEnv',
AFFINE_SERVER_HTTPS: ['https', 'boolean'],
DATABASE_URL: 'db.url',
ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
@ -28,7 +27,7 @@ AFFiNE.ENV_MAP = {
REDIS_SERVER_DATABASE: ['plugins.redis.db', 'int'],
DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
DOC_MERGE_USE_JWST_CODEC: [
'doc.manager.experimentalMergeWithJwstCodec',
'doc.manager.experimentalMergeWithYOcto',
'boolean',
],
ENABLE_LOCAL_EMAIL: ['auth.localEmail', 'boolean'],
@ -36,5 +35,3 @@ AFFiNE.ENV_MAP = {
STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey',
FEATURES_EARLY_ACCESS_PREVIEW: ['featureFlags.earlyAccessPreview', 'boolean'],
};
export default AFFiNE;

46
packages/backend/server/src/config/affine.self.ts Normal file
View File

@ -0,0 +1,46 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
// Custom configurations for AFFiNE Cloud
// ====================================================================================
// Q: WHY THIS FILE EXISTS?
// A: AFFiNE deployment environment may have a lot of custom environment variables,
// which are not suitable to be put in the `affine.ts` file.
// For example, AFFiNE Cloud Clusters are deployed on Google Cloud Platform.
// We need to enable the `gcloud` plugin to make sure the nodes working well,
// but the default selfhost version may not require it.
// So it's not a good idea to put such logic in the common `affine.ts` file.
//
// ```
// if (AFFiNE.deploy) {
// AFFiNE.plugins.use('gcloud');
// }
// ```
// ====================================================================================
const env = process.env;
AFFiNE.metrics.enabled = !AFFiNE.node.test;
if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
AFFiNE.storage.providers.r2 = {
accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
credentials: {
accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
},
};
AFFiNE.storage.storages.avatar.provider = 'r2';
AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
`https://avatar.affineassets.com/${key}`;
AFFiNE.storage.storages.blob.provider = 'r2';
AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
AFFiNE.affine.canary ? 'canary' : 'prod'
}`;
}
AFFiNE.plugins.use('redis');
AFFiNE.plugins.use('payment');
if (AFFiNE.deploy) {
AFFiNE.plugins.use('gcloud');
}

129
packages/backend/server/src/config/affine.ts
View File

@ -1,39 +1,94 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
// Custom configurations
const env = process.env;
// TODO(@forehalo): detail explained
// Storage
if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
AFFiNE.storage.providers.r2 = {
accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
credentials: {
accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
},
};
AFFiNE.storage.storages.avatar.provider = 'r2';
AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
`https://avatar.affineassets.com/${key}`;
AFFiNE.storage.storages.blob.provider = 'r2';
AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
AFFiNE.affine.canary ? 'canary' : 'prod'
}`;
}
// Metrics
AFFiNE.metrics.enabled = true;
// Plugins Section Start
AFFiNE.plugins.use('payment', {
stripe: {
keys: {},
apiVersion: '2023-10-16',
},
//
// ###############################################################
// ## AFFiNE Configuration System ##
// ###############################################################
// Here is the file of all AFFiNE configurations that will affect runtime behavior.
// Override any configuration here and it will be merged when starting the server.
// Any changes in this file won't take effect before server restarted.
//
//
// > Configurations merge order
// 1. load environment variables (`.env` if provided, and from system)
// 2. load `src/fundamentals/config/default.ts` for all default settings
// 3. apply `./affine.env.ts` patches
// 4. apply `./affine.ts` patches (this file)
//
//
// ###############################################################
// ## General settings ##
// ###############################################################
//
// /* The unique identity of the server */
// AFFiNE.serverId = 'some-randome-uuid';
//
// /* The name of AFFiNE Server, may show on the UI */
// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
//
// /* Whether the server is deployed behind a HTTPS proxied environment */
AFFiNE.https = false;
// /* Domain of your server that your server will be available at */
AFFiNE.host = 'localhost';
// /* The local port of your server that will listen on */
AFFiNE.port = 3010;
// /* The sub path of your server */
// /* For example, if you set `AFFiNE.path = '/affine'`, then the server will be available at `${domain}/affine` */
// AFFiNE.path = '/affine';
//
//
// ###############################################################
// ## Database settings ##
// ###############################################################
//
// /* The URL of the database where most of AFFiNE server data will be stored in */
// AFFiNE.db.url = 'postgres://user:passsword@localhost:5432/affine';
//
//
// ###############################################################
// ## Server Function settings ##
// ###############################################################
//
// /* Whether enable metrics and tracing while running the server */
// /* The metrics will be available at `http://localhost:9464/metrics` with [Prometheus] format exported */
// AFFiNE.metrics.enabled = true;
//
// /* GraphQL configurations that control the behavior of the Apollo Server behind */
// /* @see https://www.apollographql.com/docs/apollo-server/api/apollo-server */
// AFFiNE.graphql = {
// /* Path to mount GraphQL API */
// path: '/graphql',
// buildSchemaOptions: {
// numberScalarMode: 'integer',
// },
// /* Whether allow client to query the schema introspection */
// introspection: true,
// /* Whether enable GraphQL Playground UI */
// playground: true,
// }
//
// /* Doc Store & Collaberation */
// /* How long the buffer time of creating a new history snapshot when doc get updated */
// AFFiNE.doc.history.interval = 1000 * 60 * 10; // 10 minutes
//
// /* Use `y-octo` to merge updates at the same time when merging using Yjs */
// AFFiNE.doc.manager.experimentalMergeWithYOcto = true;
//
// /* How often the manager will start a new turn of merging pending updates into doc snapshot */
// AFFiNE.doc.manager.updatePollInterval = 1000 * 3;
//
//
// ###############################################################
// ## Plugins settings ##
// ###############################################################
//
// /* Redis Plugin */
// /* Provide caching and session storing backed by Redis. */
// /* Useful when you deploy AFFiNE server in a cluster. */
AFFiNE.plugins.use('redis', {
/* override options */
});
AFFiNE.plugins.use('redis');
// Plugins Section end
export default AFFiNE;
// /* Payment Plugin */
AFFiNE.plugins.use('payment', {
stripe: { keys: {}, apiVersion: '2023-10-16' },
});
//

2
packages/backend/server/src/core/auth/service.ts
View File

@ -136,7 +136,7 @@ export class AuthService {
return (
!!outcome.success &&
// skip hostname check in dev mode
(this.config.affineEnv === 'dev' || outcome.hostname === this.config.host)
(this.config.node.dev || outcome.hostname === this.config.host)
);
}

15
packages/backend/server/src/core/config.ts
View File

@ -1,6 +1,8 @@
import { Module } from '@nestjs/common';
import { Field, ObjectType, Query, registerEnumType } from '@nestjs/graphql';
import { DeploymentType } from '../fundamentals';
export enum ServerFeature {
Payment = 'payment',
}
@ -9,6 +11,10 @@ registerEnumType(ServerFeature, {
name: 'ServerFeature',
});
registerEnumType(DeploymentType, {
name: 'ServerDeploymentType',
});
const ENABLED_FEATURES: ServerFeature[] = [];
export function ADD_ENABLED_FEATURES(feature: ServerFeature) {
ENABLED_FEATURES.push(feature);
@ -28,6 +34,9 @@ export class ServerConfigType {
@Field({ description: 'server base url' })
baseUrl!: string;
@Field(() => DeploymentType, { description: 'server type' })
type!: DeploymentType;
/**
* @deprecated
*/
@ -46,7 +55,11 @@ export class ServerConfigResolver {
name: AFFiNE.serverName,
version: AFFiNE.version,
baseUrl: AFFiNE.baseUrl,
flavor: AFFiNE.flavor.type,
type: AFFiNE.type,
// BACKWARD COMPATIBILITY
// the old flavors contains `selfhosted` but it actually not flavor but deployment type
// this field should be removed after frontend feature flags implemented
flavor: AFFiNE.type,
features: ENABLED_FEATURES,
};
}

8
packages/backend/server/src/core/doc/manager.ts
View File

@ -125,11 +125,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
const doc = await this.recoverDoc(...updates);
// test jwst codec
if (
this.config.affine.canary &&
this.config.doc.manager.experimentalMergeWithJwstCodec &&
updates.length < 100 /* avoid overloading */
) {
if (this.config.doc.manager.experimentalMergeWithYOcto) {
metrics.jwst.counter('codec_merge_counter').add(1);
const yjsResult = Buffer.from(encodeStateAsUpdate(doc));
let log = false;
@ -180,7 +176,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
}, this.config.doc.manager.updatePollInterval);
this.logger.log('Automation started');
if (this.config.doc.manager.experimentalMergeWithJwstCodec) {
if (this.config.doc.manager.experimentalMergeWithYOcto) {
this.logger.warn(
'Experimental feature enabled: merge updates with jwst codec is enabled'
);

2
packages/backend/server/src/data/migrations/1605053000403-self-host-admin.ts
View File

@ -8,7 +8,7 @@ export class SelfHostAdmin1605053000403 {
// do the migration
static async up(db: PrismaClient, ref: ModuleRef) {
const config = ref.get(Config, { strict: false });
if (config.flavor.selfhosted) {
if (config.isSelfhosted) {
if (
!process.env.AFFINE_ADMIN_EMAIL ||
!process.env.AFFINE_ADMIN_PASSWORD

42
packages/backend/server/src/fundamentals/config/def.ts
View File

@ -18,18 +18,22 @@ export enum ExternalAccount {
firebase = 'firebase',
}
export type ServerFlavor =
| 'allinone'
| 'main'
// @deprecated
| 'graphql'
| 'sync'
| 'selfhosted';
export type ServerFlavor = 'allinone' | 'graphql' | 'sync';
export type AFFINE_ENV = 'dev' | 'beta' | 'production';
export type NODE_ENV = 'development' | 'test' | 'production';
export enum DeploymentType {
Affine = 'affine',
Selfhosted = 'selfhosted',
}
export type ConfigPaths = LeafPaths<
Omit<
AFFiNEConfig,
| 'ENV_MAP'
| 'version'
| 'type'
| 'isSelfhosted'
| 'flavor'
| 'env'
| 'affine'
@ -63,27 +67,36 @@ export interface AFFiNEConfig {
*/
readonly version: string;
/**
* Deployment type, AFFiNE Cloud, or Selfhosted
*/
get type(): DeploymentType;
/**
* Fast detect whether currently deployed in a selfhosted environment
*/
get isSelfhosted(): boolean;
/**
* Server flavor
*/
get flavor(): {
type: string;
main: boolean;
graphql: boolean;
sync: boolean;
selfhosted: boolean;
};
/**
* Deployment environment
*/
readonly affineEnv: 'dev' | 'beta' | 'production';
readonly AFFINE_ENV: AFFINE_ENV;
/**
* alias to `process.env.NODE_ENV`
*
* @default 'production'
* @default 'development'
* @env NODE_ENV
*/
readonly env: string;
readonly NODE_ENV: NODE_ENV;
/**
* fast AFFiNE environment judge
@ -101,6 +114,7 @@ export interface AFFiNEConfig {
dev: boolean;
test: boolean;
};
get deploy(): boolean;
/**
@ -302,11 +316,11 @@ export interface AFFiNEConfig {
updatePollInterval: number;
/**
* Use JwstCodec to merge updates at the same time when merging using Yjs.
* Use `y-octo` to merge updates at the same time when merging using Yjs.
*
* This is an experimental feature, and aimed to check the correctness of JwstCodec.
*/
experimentalMergeWithJwstCodec: boolean;
experimentalMergeWithYOcto: boolean;
};
history: {
/**

76
packages/backend/server/src/fundamentals/config/default.ts
View File

@ -6,7 +6,14 @@ import { merge } from 'lodash-es';
import parse from 'parse-duration';
import pkg from '../../../package.json' assert { type: 'json' };
import type { AFFiNEConfig, ServerFlavor } from './def';
import {
type AFFINE_ENV,
AFFiNEConfig,
DeploymentType,
type NODE_ENV,
type ServerFlavor,
} from './def';
import { readEnv } from './env';
import { getDefaultAFFiNEStorageConfig } from './storage';
// Don't use this in production
@ -46,40 +53,62 @@ const jwtKeyPair = (function () {
})();
export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
let isHttps: boolean | null = null;
let flavor = (process.env.SERVER_FLAVOR ?? 'allinone') as ServerFlavor;
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'development', [
'development',
'test',
'production',
]);
const AFFINE_ENV = readEnv<AFFINE_ENV>('AFFINE_ENV', 'dev', [
'dev',
'beta',
'production',
]);
const flavor = readEnv<ServerFlavor>('SERVER_FLAVOR', 'allinone', [
'allinone',
'graphql',
'sync',
]);
const deploymentType = readEnv<DeploymentType>(
'DEPLOYMENT_TYPE',
NODE_ENV === 'development'
? DeploymentType.Affine
: DeploymentType.Selfhosted,
Object.values(DeploymentType)
);
const isSelfhosted = deploymentType === DeploymentType.Selfhosted;
const defaultConfig = {
serverId: 'affine-nestjs-server',
serverName: flavor === 'selfhosted' ? 'Self-Host Cloud' : 'AFFiNE Cloud',
serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud',
version: pkg.version,
get type() {
return deploymentType;
},
get isSelfhosted() {
return isSelfhosted;
},
get flavor() {
if (flavor === 'graphql') {
flavor = 'main';
}
return {
type: flavor,
main: flavor === 'main' || flavor === 'allinone',
graphql: flavor === 'graphql' || flavor === 'allinone',
sync: flavor === 'sync' || flavor === 'allinone',
selfhosted: flavor === 'selfhosted',
};
},
ENV_MAP: {},
affineEnv: 'dev',
AFFINE_ENV,
get affine() {
const env = this.affineEnv;
return {
canary: env === 'dev',
beta: env === 'beta',
stable: env === 'production',
canary: AFFINE_ENV === 'dev',
beta: AFFINE_ENV === 'beta',
stable: AFFINE_ENV === 'production',
};
},
env: process.env.NODE_ENV ?? 'development',
NODE_ENV,
get node() {
const env = this.env;
return {
prod: env === 'production',
dev: env === 'development',
test: env === 'test',
prod: NODE_ENV === 'production',
dev: NODE_ENV === 'development',
test: NODE_ENV === 'test',
};
},
get deploy() {
@ -88,12 +117,7 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
featureFlags: {
earlyAccessPreview: false,
},
get https() {
return isHttps ?? !this.node.dev;
},
set https(value: boolean) {
isHttps = value;
},
https: false,
host: 'localhost',
port: 3010,
path: '',
@ -160,7 +184,7 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
manager: {
enableUpdateAutoMerging: flavor !== 'sync',
updatePollInterval: 3000,
experimentalMergeWithJwstCodec: false,
experimentalMergeWithYOcto: false,
},
history: {
interval: 1000 * 60 * 10 /* 10 mins */,

21
packages/backend/server/src/fundamentals/config/env.ts
View File

@ -48,3 +48,24 @@ export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
}
}
}
export function readEnv<T>(
env: string,
defaultValue: T,
availableValues?: T[]
) {
const value = process.env[env];
if (value === undefined) {
return defaultValue;
}
if (availableValues && !availableValues.includes(value as any)) {
throw new Error(
`Invalid value '${value}' for environment variable ${env}, expected one of [${availableValues.join(
', '
)}]`
);
}
return value as T;
}

1
packages/backend/server/src/fundamentals/index.ts
View File

@ -9,6 +9,7 @@ export {
applyEnvToConfig,
Config,
type ConfigPaths,
DeploymentType,
getDefaultAFFiNEStorageConfig,
} from './config';
export * from './error';

43
packages/backend/server/src/fundamentals/metrics/index.ts
View File

@ -1,28 +1,48 @@
import { Global, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import {
Global,
Module,
OnModuleDestroy,
OnModuleInit,
Provider,
} from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { Config, parseEnvValue } from '../config';
import { createSDK, registerCustomMetrics } from './opentelemetry';
import { Config } from '../config';
import {
LocalOpentelemetryFactory,
OpentelemetryFactory,
registerCustomMetrics,
} from './opentelemetry';
const factorProvider: Provider = {
provide: OpentelemetryFactory,
useFactory: (config: Config) => {
return config.metrics.enabled ? new LocalOpentelemetryFactory() : null;
},
inject: [Config],
};
@Global()
@Module({})
@Module({
providers: [factorProvider],
exports: [factorProvider],
})
export class MetricsModule implements OnModuleInit, OnModuleDestroy {
private sdk: NodeSDK | null = null;
constructor(private readonly config: Config) {}
constructor(private readonly ref: ModuleRef) {}
onModuleInit() {
if (
this.config.metrics.enabled &&
!parseEnvValue(process.env.DISABLE_TELEMETRY, 'boolean')
) {
this.sdk = createSDK();
const factor = this.ref.get(OpentelemetryFactory, { strict: false });
if (factor) {
this.sdk = factor.create();
this.sdk.start();
registerCustomMetrics();
}
}
async onModuleDestroy() {
if (this.config.metrics.enabled && this.sdk) {
if (this.sdk) {
await this.sdk.shutdown();
}
}
@ -30,3 +50,4 @@ export class MetricsModule implements OnModuleInit, OnModuleDestroy {
export * from './metrics';
export * from './utils';
export { OpentelemetryFactory };

72
packages/backend/server/src/fundamentals/metrics/opentelemetry.ts
View File

@ -1,6 +1,4 @@
import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
import { GcpDetectorSync } from '@google-cloud/opentelemetry-resource-util';
import { OnModuleDestroy } from '@nestjs/common';
import { metrics } from '@opentelemetry/api';
import {
CompositePropagator,
@ -18,16 +16,13 @@ import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'
import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io';
import { Resource } from '@opentelemetry/resources';
import {
ConsoleMetricExporter,
type MeterProvider,
MetricProducer,
MetricReader,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
import { NodeSDK } from '@opentelemetry/sdk-node';
import {
BatchSpanProcessor,
ConsoleSpanExporter,
SpanExporter,
TraceIdRatioBasedSampler,
} from '@opentelemetry/sdk-trace-node';
@ -38,7 +33,7 @@ import { PrismaMetricProducer } from './prisma';
const { PrismaInstrumentation } = prismaInstrument;
abstract class OpentelemetryFactor {
export abstract class OpentelemetryFactory {
abstract getMetricReader(): MetricReader;
abstract getSpanExporter(): SpanExporter;
@ -59,7 +54,7 @@ abstract class OpentelemetryFactor {
getResource() {
return new Resource({
[SemanticResourceAttributes.K8S_NAMESPACE_NAME]: AFFiNE.affineEnv,
[SemanticResourceAttributes.K8S_NAMESPACE_NAME]: AFFiNE.AFFINE_ENV,
[SemanticResourceAttributes.SERVICE_NAME]: AFFiNE.flavor.type,
[SemanticResourceAttributes.SERVICE_VERSION]: AFFiNE.version,
});
@ -85,32 +80,20 @@ abstract class OpentelemetryFactor {
}
}
class GCloudOpentelemetryFactor extends OpentelemetryFactor {
override getResource(): Resource {
return super.getResource().merge(new GcpDetectorSync().detect());
export class LocalOpentelemetryFactory
extends OpentelemetryFactory
implements OnModuleDestroy
{
private readonly metricsExporter = new PrometheusExporter({
metricProducers: this.getMetricsProducers(),
});
async onModuleDestroy() {
await this.metricsExporter.shutdown();
}
override getMetricReader(): MetricReader {
return new PeriodicExportingMetricReader({
exportIntervalMillis: 30000,
exportTimeoutMillis: 10000,
exporter: new MetricExporter({
prefix: 'custom.googleapis.com',
}),
metricProducers: this.getMetricsProducers(),
});
}
override getSpanExporter(): SpanExporter {
return new TraceExporter();
}
}
class LocalOpentelemetryFactor extends OpentelemetryFactor {
override getMetricReader(): MetricReader {
return new PrometheusExporter({
metricProducers: this.getMetricsProducers(),
});
return this.metricsExporter;
}
override getSpanExporter(): SpanExporter {
@ -118,33 +101,6 @@ class LocalOpentelemetryFactor extends OpentelemetryFactor {
}
}
class DebugOpentelemetryFactor extends OpentelemetryFactor {
override getMetricReader(): MetricReader {
return new PeriodicExportingMetricReader({
exporter: new ConsoleMetricExporter(),
metricProducers: this.getMetricsProducers(),
});
}
override getSpanExporter(): SpanExporter {
return new ConsoleSpanExporter();
}
}
// TODO(@forehalo): make it configurable
export function createSDK() {
let factor: OpentelemetryFactor | null = null;
if (process.env.NODE_ENV === 'production') {
factor = new GCloudOpentelemetryFactor();
} else if (process.env.DEBUG_METRICS) {
factor = new DebugOpentelemetryFactor();
} else {
factor = new LocalOpentelemetryFactor();
}
return factor?.create();
}
function getMeterProvider() {
return metrics.getMeterProvider();
}

14
packages/backend/server/src/index.ts
View File

@ -1,14 +1,16 @@
/// <reference types="./global.d.ts" />
// keep the config import at the top
// eslint-disable-next-line simple-import-sort/imports
import './prelude';
import { Logger } from '@nestjs/common';
import { createApp } from './app';
const app = await createApp();
const listeningHost = AFFiNE.deploy ? '0.0.0.0' : 'localhost';
await app.listen(AFFiNE.port, listeningHost);
console.log(
`AFFiNE Server has been started on http://${listeningHost}:${AFFiNE.port}.`
);
console.log(`And the public server should be recognized as ${AFFiNE.baseUrl}`);
const logger = new Logger('App');
logger.log(`AFFiNE Server is running in [${AFFiNE.type}] mode`);
logger.log(`Listening on http://${listeningHost}:${AFFiNE.port}`);
logger.log(`And the public server should be recognized as ${AFFiNE.baseUrl}`);

2
packages/backend/server/src/plugins/config.ts
View File

@ -1,3 +1,4 @@
import { GCloudConfig } from './gcloud/config';
import { PaymentConfig } from './payment';
import { RedisOptions } from './redis';
@ -5,6 +6,7 @@ declare module '../fundamentals/config' {
interface PluginsConfig {
readonly payment: PaymentConfig;
readonly redis: RedisOptions;
readonly gcloud: GCloudConfig;
}
export type AvailablePlugins = keyof PluginsConfig;

1
packages/backend/server/src/plugins/gcloud/config.ts Normal file
View File

@ -0,0 +1 @@
export interface GCloudConfig {}

10
packages/backend/server/src/plugins/gcloud/index.ts Normal file
View File

@ -0,0 +1,10 @@
import { Global } from '@nestjs/common';
import { OptionalModule } from '../../fundamentals';
import { GCloudMetrics } from './metrics';
@Global()
@OptionalModule({
imports: [GCloudMetrics],
})
export class GCloudModule {}

46
packages/backend/server/src/plugins/gcloud/metrics.ts Normal file
View File

@ -0,0 +1,46 @@
import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
import { GcpDetectorSync } from '@google-cloud/opentelemetry-resource-util';
import { Global, Provider } from '@nestjs/common';
import { Resource } from '@opentelemetry/resources';
import {
MetricReader,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
import { SpanExporter } from '@opentelemetry/sdk-trace-node';
import { OptionalModule } from '../../fundamentals';
import { OpentelemetryFactory } from '../../fundamentals/metrics';
export class GCloudOpentelemetryFactory extends OpentelemetryFactory {
override getResource(): Resource {
return super.getResource().merge(new GcpDetectorSync().detect());
}
override getMetricReader(): MetricReader {
return new PeriodicExportingMetricReader({
exportIntervalMillis: 30000,
exportTimeoutMillis: 10000,
exporter: new MetricExporter({
prefix: 'custom.googleapis.com',
}),
metricProducers: this.getMetricsProducers(),
});
}
override getSpanExporter(): SpanExporter {
return new TraceExporter();
}
}
const factorProvider: Provider = {
provide: OpentelemetryFactory,
useFactory: () => new GCloudOpentelemetryFactory(),
};
@Global()
@OptionalModule({
if: config => config.metrics.enabled,
overrides: [factorProvider],
})
export class GCloudMetrics {}

2
packages/backend/server/src/plugins/index.ts
View File

@ -1,8 +1,10 @@
import type { AvailablePlugins } from '../fundamentals/config';
import { GCloudModule } from './gcloud';
import { PaymentModule } from './payment';
import { RedisModule } from './redis';
export const pluginsMap = new Map<AvailablePlugins, AFFiNEModule>([
['payment', PaymentModule],
['redis', RedisModule],
['gcloud', GCloudModule],
]);

2
packages/backend/server/src/plugins/payment/resolver.ts
View File

@ -321,7 +321,7 @@ export class UserSubscriptionResolver {
// @FIXME(@forehalo): should not mock any api for selfhosted server
// the frontend should avoid calling such api if feature is not enabled
if (this.config.flavor.selfhosted) {
if (this.config.isSelfhosted) {
const start = new Date();
const end = new Date();
end.setFullYear(start.getFullYear() + 1);

14
packages/backend/server/src/prelude.ts
View File

@ -5,6 +5,7 @@ import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { config } from 'dotenv';
import { omit } from 'lodash-es';
import {
applyEnvToConfig,
@ -49,8 +50,17 @@ async function load() {
// 5. load `config/affine` to patch custom configs
await loadRemote(AFFiNE_CONFIG_PATH, 'affine.js');
if (process.env.NODE_ENV === 'development') {
console.log('AFFiNE Config:', JSON.stringify(globalThis.AFFiNE, null, 2));
// 6. load `config/affine.self` to patch custom configs
// This is the file only take effect in [AFFiNE Cloud]
if (!AFFiNE.isSelfhosted) {
await loadRemote(AFFiNE_CONFIG_PATH, 'affine.self.js');
}
if (AFFiNE.node.dev) {
console.log(
'AFFiNE Config:',
JSON.stringify(omit(globalThis.AFFiNE, 'ENV_MAP'), null, 2)
);
}
}

8
packages/backend/server/src/schema.gql
View File

@ -240,10 +240,18 @@ type ServerConfigType {
"""server identical name could be shown as badge on user interface"""
name: String!
"""server type"""
type: ServerDeploymentType!
"""server version"""
version: String!
}
enum ServerDeploymentType {
Affine
Selfhosted
}
enum ServerFeature {
Payment
}

2
packages/backend/server/tests/config.spec.ts
View File

@ -18,7 +18,7 @@ test.afterEach.always(async () => {
test('should be able to get config', t => {
t.true(typeof config.host === 'string');
t.is(config.env, 'test');
t.is(config.NODE_ENV, 'test');
});
test('should be able to override config', async t => {

10
packages/frontend/core/src/hooks/affine/use-server-config.ts
View File

@ -1,4 +1,4 @@
import { serverConfigQuery } from '@affine/graphql';
import { serverConfigQuery, ServerDeploymentType } from '@affine/graphql';
import type { BareFetcher, Middleware } from 'swr';
import { useQueryImmutable } from '../use-query';
@ -25,20 +25,20 @@ const useServerConfig = () => {
return config.serverConfig;
};
export const useServerFlavor = () => {
export const useServerType = () => {
const config = useServerConfig();
if (!config) {
return 'local';
}
return config.flavor;
return config.type;
};
export const useSelfHosted = () => {
const serverFlavor = useServerFlavor();
const serverType = useServerType();
return ['local', 'selfhosted'].includes(serverFlavor);
return ['local', ServerDeploymentType.Selfhosted].includes(serverType);
};
export const useServerBaseUrl = () => {

2
packages/frontend/graphql/src/graphql/index.ts
View File

@ -677,7 +677,7 @@ query serverConfig {
baseUrl
name
features
flavor
type
}
}`,
};

2
packages/frontend/graphql/src/graphql/server-config.gql
View File

@ -4,6 +4,6 @@ query serverConfig {
baseUrl
name
features
flavor
type
}
}

7
packages/frontend/graphql/src/schema.ts
View File

@ -71,6 +71,11 @@ export enum PublicPageMode {
Page = 'Page',
}
export enum ServerDeploymentType {
Affine = 'Affine',
Selfhosted = 'Selfhosted',
}
export enum ServerFeature {
Payment = 'Payment',
}
@ -673,7 +678,7 @@ export type ServerConfigQuery = {
baseUrl: string;
name: string;
features: Array<ServerFeature>;
flavor: string;
type: ServerDeploymentType;
};
};

2
tests/affine-cloud/playwright.config.ts
View File

@ -55,8 +55,6 @@ const config: PlaywrightTestConfig = {
OAUTH_EMAIL_SENDER: 'noreply@toeverything.info',
OAUTH_EMAIL_LOGIN: 'noreply@toeverything.info',
OAUTH_EMAIL_PASSWORD: 'affine',
STRIPE_API_KEY: '1',
STRIPE_WEBHOOK_KEY: '1',
},
},
],