diff --git a/.github/deployment/front/Dockerfile b/.github/deployment/front/Dockerfile index 2aa4d6913..33a267034 100644 --- a/.github/deployment/front/Dockerfile +++ b/.github/deployment/front/Dockerfile @@ -1,6 +1,6 @@ FROM openresty/openresty:1.21.4.3-0-buster WORKDIR /app -COPY ./packages/frontend/core/dist ./dist +COPY ./packages/frontend/core/dist/index.html ./dist/index.html COPY ./.github/deployment/front/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf COPY ./.github/deployment/front/affine.nginx.conf /etc/nginx/conf.d/affine.nginx.conf diff --git a/.github/deployment/node/Dockerfile b/.github/deployment/node/Dockerfile index a8f8a2c50..b2a0d442b 100644 --- a/.github/deployment/node/Dockerfile +++ b/.github/deployment/node/Dockerfile @@ -1,6 +1,7 @@ FROM node:18-bookworm-slim COPY ./packages/backend/server /app +COPY ./packages/frontend/core/dist /app/static WORKDIR /app RUN apt-get update && \ diff --git a/.github/deployment/self-host/compose.yaml b/.github/deployment/self-host/compose.yaml new file mode 100644 index 000000000..cf10de931 --- /dev/null +++ b/.github/deployment/self-host/compose.yaml @@ -0,0 +1,57 @@ +services: + affine: + image: ghcr.io/toeverything/affine-graphql:beta + container_name: affine + command: + [ + 'sh', + '-c', + './node_modules/.bin/dotenv -e /root/.affine/.env -- npm run predeploy && node --es-module-specifier-resolution=node ./dist/index.js', + ] + ports: + - '3010:3010' + - '5555:5555' + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + volumes: + - ~/.affine/storage:/root/.affine/storage + - ~/.affine/.env:/root/.affine/.env + logging: + driver: 'json-file' + options: + max-size: '1000m' + restart: unless-stopped + environment: + - DISABLE_TELEMETRY=true + - NODE_ENV=production + - SERVER_FLAVOR=selfhosted + redis: + image: redis + container_name: redis + restart: unless-stopped + volumes: + - ~/.affine/redis:/data + healthcheck: + test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] + interval: 10s + timeout: 5s + retries: 5 + postgres: + image: postgres + container_name: postgres + restart: unless-stopped + volumes: + - ~/.affine/postgres:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U affine'] + interval: 10s + timeout: 5s + retries: 5 + environment: + POSTGRES_USER: affine + POSTGRES_PASSWORD: affine + POSTGRES_DB: affine + PGDATA: /var/lib/postgresql/data/pgdata diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index 38e53c342..5452a8564 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -73,6 +73,8 @@ spec: value: "{{ .Values.app.path }}" - name: AFFINE_SERVER_HOST value: "{{ .Values.app.host }}" + - name: AFFINE_SERVER_HOST + value: "{{ .Values.app.https }}" - name: ENABLE_R2_OBJECT_STORAGE value: "{{ .Values.app.objectStorage.r2.enabled }}" - name: ENABLE_CAPTCHA diff --git a/.github/helm/affine/charts/graphql/values.yaml b/.github/helm/affine/charts/graphql/values.yaml index 629997382..b0e787d2b 100644 --- a/.github/helm/affine/charts/graphql/values.yaml +++ b/.github/helm/affine/charts/graphql/values.yaml @@ -16,6 +16,7 @@ app: path: '' # AFFINE_SERVER_HOST host: '0.0.0.0' + https: true doc: mergeInterval: "3000" jwt: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 39185531b..bc598c818 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,6 +29,7 @@ jobs: uses: ./.github/actions/setup-node with: electron-install: false + extra-flags: workspaces focus @affine/server - name: Build Server run: yarn workspace @affine/server build - name: Upload server dist @@ -62,6 +63,7 @@ jobs: SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} - name: Upload core artifact uses: actions/upload-artifact@v4 with: @@ -69,10 +71,10 @@ jobs: path: ./packages/frontend/core/dist if-no-files-found: error - build-storage: - name: Build Storage + build-core-selfhost: + name: Build @affine/core runs-on: ubuntu-latest - + environment: ${{ github.event.inputs.flavor }} steps: - uses: actions/checkout@v4 - name: Setup Version @@ -80,22 +82,33 @@ jobs: uses: ./.github/actions/setup-version - name: Setup Node.js uses: ./.github/actions/setup-node - - name: Build Rust - uses: ./.github/actions/build-rust - with: - target: 'x86_64-unknown-linux-gnu' - package: '@affine/storage' - nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} - - name: Upload storage.node + - name: Build Core + run: yarn nx build @affine/core --skip-nx-cache + env: + BUILD_TYPE: ${{ github.event.inputs.flavor }} + SHOULD_REPORT_TRACE: false + PUBLIC_PATH: '/' + - name: Download selfhost fonts + run: node ./scripts/download-blocksuite-fonts.mjs + - name: Upload core artifact uses: actions/upload-artifact@v4 with: - name: storage.node - path: ./packages/backend/storage/storage.node + name: selfhost-core + path: ./packages/frontend/core/dist if-no-files-found: error - build-storage-arm64: - name: Build Storage arm64 + build-storage: + name: Build Storage - ${{ matrix.targets.name }} runs-on: ubuntu-latest + strategy: + matrix: + targets: + - name: x86_64-unknown-linux-gnu + file: storage.node + - name: aarch64-unknown-linux-gnu + file: storage.arm64.node + - name: armv7-unknown-linux-gnueabihf + file: storage.armv7.node steps: - uses: actions/checkout@v4 @@ -104,16 +117,19 @@ jobs: uses: ./.github/actions/setup-version - name: Setup Node.js uses: ./.github/actions/setup-node + with: + electron-install: false + extra-flags: workspaces focus @affine/storage - name: Build Rust uses: ./.github/actions/build-rust with: - target: 'aarch64-unknown-linux-gnu' + target: ${{ matrix.targets.name }} package: '@affine/storage' nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} - - name: Upload storage.node + - name: Upload ${{ matrix.targets.file }} uses: actions/upload-artifact@v4 with: - name: storage.arm64.node + name: ${{ matrix.targets.file }} path: ./packages/backend/storage/storage.node if-no-files-found: error @@ -123,8 +139,8 @@ jobs: needs: - build-server - build-core + - build-core-selfhost - build-storage - - build-storage-arm64 steps: - uses: actions/checkout@v4 - name: Download core artifact @@ -147,8 +163,15 @@ jobs: with: name: storage.arm64.node path: ./packages/backend/storage - - name: move storage.arm64.node - run: mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node + - name: Download storage.node arm64 + uses: actions/download-artifact@v4 + with: + name: storage.armv7.node + path: . + - name: move storage files + run: | + mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node + mv storage.node ./packages/backend/server/storage.armv7.node - name: Setup env run: | echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" @@ -190,9 +213,19 @@ jobs: registry-url: https://npm.pkg.github.com scope: '@toeverything' + - name: Remove core dist + run: rm -rf ./packages/frontend/core/dist + + - name: Download selfhost core artifact + uses: actions/download-artifact@v4 + with: + name: selfhost-core + path: ./packages/frontend/core/dist + - name: Install Node.js dependencies run: | - yarn config set --json supportedArchitectures.cpu '["x64", "arm64"]' + yarn config set --json supportedArchitectures.cpu '["x64", "arm64", "arm"]' + yarn config set --json supportedArchitectures.libc '["glibc"]' yarn workspaces focus @affine/server --production - name: Generate Prisma client @@ -204,7 +237,7 @@ jobs: context: . push: true pull: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 provenance: true file: .github/deployment/node/Dockerfile tags: ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}} diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 7856cd5a7..bc479fa86 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -32,6 +32,7 @@ "@nestjs/platform-express": "^10.2.10", "@nestjs/platform-socket.io": "^10.2.10", "@nestjs/schedule": "^4.0.0", + "@nestjs/serve-static": "^4.0.0", "@nestjs/throttler": "^5.0.1", "@nestjs/websockets": "^10.2.10", "@node-rs/argon2": "^1.5.2", @@ -57,6 +58,7 @@ "@socket.io/redis-adapter": "^8.2.1", "cookie-parser": "^1.4.6", "dotenv": "^16.3.1", + "dotenv-cli": "^7.3.0", "express": "^4.18.2", "file-type": "^19.0.0", "get-stream": "^8.0.1", diff --git a/packages/backend/server/src/app.ts b/packages/backend/server/src/app.ts index 89a401980..ad97741aa 100644 --- a/packages/backend/server/src/app.ts +++ b/packages/backend/server/src/app.ts @@ -3,7 +3,7 @@ import { APP_INTERCEPTOR } from '@nestjs/core'; import { AppController } from './app.controller'; import { CacheInterceptor, CacheModule } from './cache'; -import { ConfigModule } from './config'; +import { ConfigModule, SERVER_FLAVOR } from './config'; import { EventModule } from './event'; import { BusinessModules } from './modules'; import { AuthModule } from './modules/auth'; @@ -29,6 +29,6 @@ const BasicModules = [ }, ], imports: [...BasicModules, ...BusinessModules], - controllers: [AppController], + controllers: SERVER_FLAVOR === 'selfhosted' ? [] : [AppController], }) export class AppModule {} diff --git a/packages/backend/server/src/config/def.ts b/packages/backend/server/src/config/def.ts index a3f9c2288..34221ff3d 100644 --- a/packages/backend/server/src/config/def.ts +++ b/packages/backend/server/src/config/def.ts @@ -91,6 +91,7 @@ export interface AFFiNEConfig { * @env NODE_ENV */ readonly env: string; + /** * fast AFFiNE environment judge */ diff --git a/packages/backend/server/src/config/default.ts b/packages/backend/server/src/config/default.ts index ec3233cfb..eca97146f 100644 --- a/packages/backend/server/src/config/default.ts +++ b/packages/backend/server/src/config/default.ts @@ -49,6 +49,7 @@ const jwtKeyPair = (function () { })(); export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { + let isHttps: boolean | null = null; const defaultConfig = { serverId: 'affine-nestjs-server', version: pkg.version, @@ -56,6 +57,7 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { AFFINE_SERVER_PORT: ['port', 'int'], AFFINE_SERVER_HOST: 'host', AFFINE_SERVER_SUB_PATH: 'path', + AFFIHE_SERVER_HTTPS: 'https', AFFINE_ENV: 'affineEnv', DATABASE_URL: 'db.url', ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'], @@ -117,7 +119,10 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { earlyAccessPreview: false, }, get https() { - return !this.node.dev; + return isHttps ?? !this.node.dev; + }, + set https(value: boolean) { + isHttps = value; }, host: 'localhost', port: 3010, diff --git a/packages/backend/server/src/metrics/opentelemetry.ts b/packages/backend/server/src/metrics/opentelemetry.ts index 38355a389..54e449ae1 100644 --- a/packages/backend/server/src/metrics/opentelemetry.ts +++ b/packages/backend/server/src/metrics/opentelemetry.ts @@ -33,6 +33,7 @@ import prismaInstrument from '@prisma/instrumentation'; const { PrismaInstrumentation } = prismaInstrument; +import { parseEnvValue } from '../config/def'; import { PrismaMetricProducer } from './prisma'; abstract class OpentelemetryFactor { @@ -155,6 +156,9 @@ export function getMeter(name = 'business') { } export function start() { + if (parseEnvValue(process.env.DISABLE_TELEMETRY, 'boolean')) { + return; + } const sdk = createSDK(); if (sdk) { diff --git a/packages/backend/server/src/modules/auth/next-auth-options.ts b/packages/backend/server/src/modules/auth/next-auth-options.ts index fa40774a2..841f24424 100644 --- a/packages/backend/server/src/modules/auth/next-auth-options.ts +++ b/packages/backend/server/src/modules/auth/next-auth-options.ts @@ -179,7 +179,7 @@ export const NextAuthOptionsProvider: FactoryProvider = { ); } - if (config.auth.oauthProviders.google) { + if (config.auth.oauthProviders.google?.enabled) { nextAuthOptions.providers.push( // @ts-expect-error esm interop issue Google.default({ diff --git a/packages/backend/server/src/modules/auth/next-auth.controller.ts b/packages/backend/server/src/modules/auth/next-auth.controller.ts index 9d717f012..fee533ece 100644 --- a/packages/backend/server/src/modules/auth/next-auth.controller.ts +++ b/packages/backend/server/src/modules/auth/next-auth.controller.ts @@ -186,15 +186,17 @@ export class NextAuthController { } let nextAuthTokenCookie: (CookieOption & { value: string }) | undefined; - const cookiePrefix = this.config.node.prod ? '__Secure-' : ''; - const sessionCookieName = `${cookiePrefix}next-auth.session-token`; + const secureCookiePrefix = '__Secure-'; + const sessionCookieName = `next-auth.session-token`; // next-auth credentials login only support JWT strategy // https://next-auth.js.org/configuration/providers/credentials // let's store the session token in the database if ( credentialsSignIn && (nextAuthTokenCookie = cookies?.find( - ({ name }) => name === sessionCookieName + ({ name }) => + name === sessionCookieName || + name === `${secureCookiePrefix}${sessionCookieName}` )) ) { const cookieExpires = new Date(); diff --git a/packages/backend/server/src/modules/index.ts b/packages/backend/server/src/modules/index.ts index c2edffda8..0522f3e2d 100644 --- a/packages/backend/server/src/modules/index.ts +++ b/packages/backend/server/src/modules/index.ts @@ -1,5 +1,8 @@ +import { join } from 'node:path'; + import { DynamicModule, Type } from '@nestjs/common'; import { ScheduleModule } from '@nestjs/schedule'; +import { ServeStaticModule } from '@nestjs/serve-static'; import { SERVER_FLAVOR } from '../config'; import { GqlModule } from '../graphql.module'; @@ -29,7 +32,10 @@ switch (SERVER_FLAVOR) { UsersModule, SyncModule, DocModule, - StorageModule + StorageModule, + ServeStaticModule.forRoot({ + rootPath: join('/app', 'static'), + }) ); break; case 'graphql': diff --git a/packages/backend/server/src/prelude.ts b/packages/backend/server/src/prelude.ts index bac73ad32..2ca504176 100644 --- a/packages/backend/server/src/prelude.ts +++ b/packages/backend/server/src/prelude.ts @@ -1,5 +1,17 @@ +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +import { config } from 'dotenv'; + +import { SERVER_FLAVOR } from './config/default'; + +if (SERVER_FLAVOR === 'selfhosted') { + config({ path: join(homedir(), '.affine', '.env') }); +} else { + config(); +} + import 'reflect-metadata'; -import 'dotenv/config'; import './affine'; import './affine.config'; diff --git a/packages/backend/server/src/storage/index.ts b/packages/backend/server/src/storage/index.ts index ad121b5ad..8d2fc87b1 100644 --- a/packages/backend/server/src/storage/index.ts +++ b/packages/backend/server/src/storage/index.ts @@ -10,7 +10,9 @@ try { storageModule = process.arch === 'arm64' ? require('../../storage.arm64.node') - : require('../../storage.node'); + : process.arch === 'arm' + ? require('../../storage.armv7.node') + : require('../../storage.node'); } export { storageModule as OctoBaseStorageModule }; diff --git a/packages/frontend/core/.webpack/config.ts b/packages/frontend/core/.webpack/config.ts index 83a3b1b5b..e9a92b200 100644 --- a/packages/frontend/core/.webpack/config.ts +++ b/packages/frontend/core/.webpack/config.ts @@ -72,7 +72,10 @@ const OptimizeOptionOptions: ( export const getPublicPath = (buildFlags: BuildFlags) => { const { BUILD_TYPE } = process.env; - const publicPath = process.env.PUBLIC_PATH ?? '/'; + if (typeof process.env.PUBLIC_PATH === 'string') { + return process.env.PUBLIC_PATH; + } + const publicPath = '/'; if (process.env.COVERAGE || buildFlags.distribution === 'desktop') { return publicPath; } diff --git a/packages/frontend/workspace-impl/src/cloud/sync/index.ts b/packages/frontend/workspace-impl/src/cloud/sync/index.ts index 14f458869..22225fcc3 100644 --- a/packages/frontend/workspace-impl/src/cloud/sync/index.ts +++ b/packages/frontend/workspace-impl/src/cloud/sync/index.ts @@ -172,8 +172,7 @@ export function createAffineStaticStorage(workspaceId: string): SyncStorage { name: 'affine-cloud-static', async pull(docId) { const response = await fetchWithTraceReport( - runtimeConfig.serverUrlPrefix + - `/api/workspaces/${workspaceId}/docs/${docId}`, + `/api/workspaces/${workspaceId}/docs/${docId}`, { priority: 'high', } diff --git a/scripts/download-blocksuite-fonts.mjs b/scripts/download-blocksuite-fonts.mjs new file mode 100644 index 000000000..38d5115f7 --- /dev/null +++ b/scripts/download-blocksuite-fonts.mjs @@ -0,0 +1,29 @@ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// eslint-disable-next-line @typescript-eslint/no-restricted-imports +import { DEFAULT_CANVAS_TEXT_FONT_CONFIG } from '@blocksuite/blocks/dist/surface-block/consts.js'; + +const fontPath = join( + fileURLToPath(import.meta.url), + '..', + '..', + 'packages', + 'frontend', + 'core', + 'dist', + 'assets' +); + +await Promise.all( + DEFAULT_CANVAS_TEXT_FONT_CONFIG.map(async ({ url }) => { + const buffer = await fetch(url).then(res => + res.arrayBuffer().then(res => Buffer.from(res)) + ); + const filename = url.split('/').pop(); + const distPath = join(fontPath, filename); + await writeFile(distPath, buffer); + console.info(`Downloaded ${distPath} successfully`); + }) +); diff --git a/yarn.lock b/yarn.lock index d4ba009b0..5ee1c3df1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -637,6 +637,7 @@ __metadata: "@nestjs/platform-express": "npm:^10.2.10" "@nestjs/platform-socket.io": "npm:^10.2.10" "@nestjs/schedule": "npm:^4.0.0" + "@nestjs/serve-static": "npm:^4.0.0" "@nestjs/testing": "npm:^10.2.10" "@nestjs/throttler": "npm:^5.0.1" "@nestjs/websockets": "npm:^10.2.10" @@ -678,6 +679,7 @@ __metadata: c8: "npm:^9.0.0" cookie-parser: "npm:^1.4.6" dotenv: "npm:^16.3.1" + dotenv-cli: "npm:^7.3.0" express: "npm:^4.18.2" file-type: "npm:^19.0.0" get-stream: "npm:^8.0.1" @@ -7445,6 +7447,28 @@ __metadata: languageName: node linkType: hard +"@nestjs/serve-static@npm:^4.0.0": + version: 4.0.0 + resolution: "@nestjs/serve-static@npm:4.0.0" + dependencies: + path-to-regexp: "npm:0.2.5" + peerDependencies: + "@fastify/static": ^6.5.0 + "@nestjs/common": ^9.0.0 || ^10.0.0 + "@nestjs/core": ^9.0.0 || ^10.0.0 + express: ^4.18.1 + fastify: ^4.7.0 + peerDependenciesMeta: + "@fastify/static": + optional: true + express: + optional: true + fastify: + optional: true + checksum: f9dde33701d05fe0309ecc4912c00b6ed81945dbeb17af13562b33406656ae6574b14397523e21c90b78292b073a510426409c08e5f6a2b88b46d73dc51faa9c + languageName: node + linkType: hard + "@nestjs/testing@npm:^10.2.10": version: 10.2.10 resolution: "@nestjs/testing@npm:10.2.10" @@ -18794,6 +18818,20 @@ __metadata: languageName: node linkType: hard +"dotenv-cli@npm:^7.3.0": + version: 7.3.0 + resolution: "dotenv-cli@npm:7.3.0" + dependencies: + cross-spawn: "npm:^7.0.3" + dotenv: "npm:^16.3.0" + dotenv-expand: "npm:^10.0.0" + minimist: "npm:^1.2.6" + bin: + dotenv: cli.js + checksum: bc48e9872ed451aa7633cfde0079f5e4b40837d49dca4eab947682c80f524bd1e63ec31ff69b7cf955ff969185a05a343dd5d754dd5569e4ae31f8e9a790ab1b + languageName: node + linkType: hard + "dotenv-expand@npm:^10.0.0, dotenv-expand@npm:~10.0.0": version: 10.0.0 resolution: "dotenv-expand@npm:10.0.0" @@ -18801,7 +18839,7 @@ __metadata: languageName: node linkType: hard -"dotenv@npm:^16.0.0, dotenv@npm:^16.3.1, dotenv@npm:~16.3.1": +"dotenv@npm:^16.0.0, dotenv@npm:^16.3.0, dotenv@npm:^16.3.1, dotenv@npm:~16.3.1": version: 16.3.1 resolution: "dotenv@npm:16.3.1" checksum: dbb778237ef8750e9e3cd1473d3c8eaa9cc3600e33a75c0e36415d0fa0848197f56c3800f77924c70e7828f0b03896818cd52f785b07b9ad4d88dba73fbba83f @@ -28540,6 +28578,13 @@ __metadata: languageName: node linkType: hard +"path-to-regexp@npm:0.2.5": + version: 0.2.5 + resolution: "path-to-regexp@npm:0.2.5" + checksum: 9652fd2b74ec932a0df8a868478478565da81e7445a8dde1e65ca80553ad03062336b1f79234068551ecc01f3b76ad774e34f784cc3a34a97c4646cb5cfcbea9 + languageName: node + linkType: hard + "path-to-regexp@npm:2.2.1": version: 2.2.1 resolution: "path-to-regexp@npm:2.2.1"