fix: improve self-host convenience (#5582)

.github/deployment/self-host/compose.yaml

@@ -1,13 +1,9 @@
 services:
   affine:
     image: ghcr.io/toeverything/affine-graphql:beta
-    container_name: affine
+    container_name: affine_selfhosted
     command:
-      [
+      ['sh', '-c', 'node ./scripts/self-host-predeploy && node ./dist/index.js']
-        'sh',
-        '-c',
-        './node_modules/.bin/dotenv -e /root/.affine/.env -- npm run predeploy && node --es-module-specifier-resolution=node ./dist/index.js',
-      ]
     ports:
       - '3010:3010'
       - '5555:5555'
@@ -17,23 +13,31 @@ services:
       postgres:
         condition: service_healthy
     volumes:
-      - ~/.affine/storage:/root/.affine/storage
+      # custom configurations
-      - ~/.affine/.env:/root/.affine/.env
+      - ~/.affine/self-host/config:/root/.affine/config
+      # blob storage
+      - ~/.affine/self-host/storage:/root/.affine/storage
     logging:
       driver: 'json-file'
       options:
         max-size: '1000m'
     restart: unless-stopped
     environment:
+      - NODE_OPTIONS=--es-module-specifier-resolution node
+      - AFFINE_CONFIG_PATH=/root/.affine/config
+      - REDIS_SERVER_HOST=redis
+      - DATABASE_URL=postgres://affine:affine@postgres:5432/affine
       - DISABLE_TELEMETRY=true
       - NODE_ENV=production
       - SERVER_FLAVOR=selfhosted
+      - AFFINE_ADMIN_EMAIL=${AFFINE_ADMIN_EMAIL}
+      - AFFINE_ADMIN_PASSWORD=${AFFINE_ADMIN_PASSWORD}
   redis:
     image: redis
-    container_name: redis
+    container_name: affine_redis
     restart: unless-stopped
     volumes:
-      - ~/.affine/redis:/data
+      - ~/.affine/self-host/redis:/data
     healthcheck:
       test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
       interval: 10s
@@ -41,10 +45,10 @@ services:
       retries: 5
   postgres:
     image: postgres
-    container_name: postgres
+    container_name: affine_postgres
     restart: unless-stopped
     volumes:
-      - ~/.affine/postgres:/var/lib/postgresql/data
+      - ~/.affine/self-host/postgres:/var/lib/postgresql/data
     healthcheck:
       test: ['CMD-SHELL', 'pg_isready -U affine']
       interval: 10s

packages/backend/server/.env.example

@@ -1,8 +1,4 @@
-DATABASE_URL="postgresql://affine@localhost:5432/affine"
+# AFFINE_SERVER_PORT=3010
-NEXTAUTH_URL="http://localhost:8080"
+# AFFINE_SERVER_HOST=app.affine.pro
-OAUTH_EMAIL_SENDER="noreply@toeverything.info"
+# AFFINE_SERVER_HTTPS=true
-OAUTH_EMAIL_LOGIN=""
+# DATABASE_URL="postgres://affine@localhost:5432/affine"
-OAUTH_EMAIL_PASSWORD=""
-ENABLE_LOCAL_EMAIL="true"
-STRIPE_API_KEY=
-STRIPE_WEBHOOK_KEY=

packages/backend/server/scripts/self-host-predeploy.js

@@ -0,0 +1,51 @@
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
+/**
+ * @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
+ */
+const configFiles = [
+  { from: './.env.example', to: '.env' },
+  { from: './dist/config/affine.js', modifier: configCleaner },
+  { from: './dist/config/affine.env.js', modifier: configCleaner },
+];
+
+function configCleaner(content) {
+  return content.replace(/(\/\/#.*$)|(\/\/\s+TODO.*$)/gm, '');
+}
+
+function prepare() {
+  fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
+
+  for (const { from, to, modifier } of configFiles) {
+    const targetFileName = to ?? path.parse(from).base;
+    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
+    if (!fs.existsSync(targetFilePath)) {
+      console.log(`creating config file [${targetFilePath}].`);
+      if (modifier) {
+        const content = fs.readFileSync(from, 'utf-8');
+        fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
+      } else {
+        fs.cpSync(from, targetFilePath, {
+          force: false,
+        });
+      }
+    }
+  }
+}
+
+function runPredeployScript() {
+  console.log('running predeploy script.');
+  execSync('yarn predeploy', {
+    env: {
+      ...process.env,
+      NODE_OPTIONS:
+        (process.env.NODE_OPTIONS ?? '') + ' --import ./dist/prelude.js',
+    },
+  });
+}
+
+prepare();
+runPredeployScript();

packages/backend/server/src/app.controller.ts

@@ -11,6 +11,7 @@ export class AppController {
     return {
       compatibility: this.config.version,
       message: `AFFiNE ${this.config.version} Server`,
+      flavor: this.config.flavor,
     };
   }
 }

packages/backend/server/src/data/migrations/1605053000403-self-host-admin.ts

@@ -0,0 +1,39 @@
+import { ModuleRef } from '@nestjs/core';
+import { hash } from '@node-rs/argon2';
+import { PrismaClient } from '@prisma/client';
+
+import { Config } from '../../fundamentals';
+
+export class SelfHostAdmin1605053000403 {
+  // do the migration
+  static async up(db: PrismaClient, ref: ModuleRef) {
+    const config = ref.get(Config, { strict: false });
+    if (config.flavor === 'selfhosted') {
+      if (
+        !process.env.AFFINE_ADMIN_EMAIL ||
+        !process.env.AFFINE_ADMIN_PASSWORD
+      ) {
+        throw new Error(
+          'You have to set AFFINE_ADMIN_EMAIL and AFFINE_ADMIN_PASSWORD environment variables to generate the initial user for self-hosted AFFiNE Server.'
+        );
+      }
+      await db.user.create({
+        data: {
+          name: 'AFFINE First User',
+          email: process.env.AFFINE_ADMIN_EMAIL,
+          emailVerified: new Date(),
+          password: await hash(process.env.AFFINE_ADMIN_PASSWORD),
+        },
+      });
+    }
+  }
+
+  // revert the migration
+  static async down(db: PrismaClient) {
+    await db.user.deleteMany({
+      where: {
+        email: process.env.AFFINE_ADMIN_EMAIL ?? 'admin@example.com',
+      },
+    });
+  }
+}

packages/backend/server/src/data/migrations/1702620653283-old-user-feature.ts

@@ -16,7 +16,6 @@ export class OldUserFeature1702620653283 {
         where: { NOT: { features: { some: { NOT: { id: { gt: 0 } } } } } },
         select: { id: true },
       });
-      console.log(`migrating ${userIds.join('|')} users`);
 
       await tx.userFeatures.createMany({
         data: userIds.map(({ id: userId }) => ({

packages/backend/server/src/fundamentals/storage/index.ts

@@ -9,7 +9,7 @@ try {
   const require = createRequire(import.meta.url);
   storageModule =
     process.arch === 'arm64'
-      ? require('../.././storage.arm64.node')
+      ? require('../../../storage.arm64.node')
       : process.arch === 'arm'
         ? require('../../../storage.armv7.node')
         : require('../../../storage.node');

packages/backend/server/src/prelude.ts

@@ -1,5 +1,6 @@
 import 'reflect-metadata';
 
+import { cpSync } from 'node:fs';
 import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -10,7 +11,22 @@ import {
   getDefaultAFFiNEConfig,
 } from './fundamentals/config';
 
+const configDir = join(fileURLToPath(import.meta.url), '../config');
+async function loadRemote(remoteDir: string, file: string) {
+  console.log(remoteDir, configDir);
+  const filePath = join(configDir, file);
+  if (configDir !== remoteDir) {
+    console.log('cp remote file');
+    cpSync(join(remoteDir, file), filePath, {
+      force: true,
+    });
+  }
+
+  await import(filePath);
+}
+
 async function load() {
+  const AFFiNE_CONFIG_PATH = process.env.AFFINE_CONFIG_PATH ?? configDir;
   // Initializing AFFiNE config
   //
   // 1. load dotenv file to `process.env`
@@ -18,7 +34,7 @@ async function load() {
   config();
   // load `.env` under user config folder
   config({
-    path: join(fileURLToPath(import.meta.url), '../config/.env'),
+    path: join(AFFiNE_CONFIG_PATH, '.env'),
   });
 
   // 2. generate AFFiNE default config and assign to `globalThis.AFFiNE`
@@ -27,13 +43,13 @@ async function load() {
   // TODO(@forehalo):
   //   Modules may contribute to ENV_MAP, figure out a good way to involve them instead of hardcoding in `./config/affine.env`
   // 3. load env => config map to `globalThis.AFFiNE.ENV_MAP
-  await import('./config/affine.env');
+  await loadRemote(AFFiNE_CONFIG_PATH, 'affine.env.js');
 
   // 4. apply `process.env` map overriding to `globalThis.AFFiNE`
   applyEnvToConfig(globalThis.AFFiNE);
 
-  // 5. load `./config/affine` to patch custom configs
+  // 5. load `config/affine` to patch custom configs
-  await import('./config/affine');
+  await loadRemote(AFFiNE_CONFIG_PATH, 'affine.js');
 
   if (process.env.NODE_ENV === 'development') {
     console.log('AFFiNE Config:', JSON.stringify(globalThis.AFFiNE, null, 2));