diff --git a/.prettierignore b/.prettierignore index 1c4786433e..bd5535a603 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1 @@ pnpm-lock.yaml -apps/electron/layers/preload/preload.d.ts diff --git a/apps/electron/README.md b/apps/electron/README.md index f4776ccbc3..2315112886 100644 --- a/apps/electron/README.md +++ b/apps/electron/README.md @@ -14,6 +14,24 @@ yarn generate-assets yarn dev # or yarn prod for production build ``` +## Troubleshooting + +### better-sqlite3 error + +When running tests or starting electron, you may encounter the following error: + +> Error: The module 'apps/electron/node_modules/better-sqlite3/build/Release/better_sqlite3.node' + +This is due to the fact that the `better-sqlite3` package is built for the Node.js version in Electron & in your machine. To fix this, run the following command based on different cases: + +```sh +# for running unit tests, we are not using Electron's node: +yarn rebuild better-sqlite3 + +# for running Electron, we are using Electron's node: +yarn postinstall +``` + ## Credits Most of the boilerplate code is generously borrowed from the following diff --git a/apps/electron/layers/main-events.ts b/apps/electron/layers/main-events.ts new file mode 100644 index 0000000000..48ce63e952 --- /dev/null +++ b/apps/electron/layers/main-events.ts @@ -0,0 +1,5 @@ +// This file contains the main process events +// It will guide preload and main process on the correct event types and payloads +export interface MainEventMap { + 'main:on-db-update': (workspaceId: string) => void; +} diff --git a/apps/electron/layers/main/src/__tests__/.gitignore b/apps/electron/layers/main/src/__tests__/.gitignore new file mode 100644 index 0000000000..a9a5aecf42 --- /dev/null +++ b/apps/electron/layers/main/src/__tests__/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/apps/electron/layers/main/src/__tests__/handlers.spec.ts b/apps/electron/layers/main/src/__tests__/handlers.spec.ts new file mode 100644 index 0000000000..e96e11684d --- /dev/null +++ b/apps/electron/layers/main/src/__tests__/handlers.spec.ts @@ -0,0 +1,226 @@ +import assert from 'node:assert'; +import path from 'node:path'; + +import fs from 'fs-extra'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import * as Y from 'yjs'; + +const registeredHandlers = new Map any>(); + +// common mock dispatcher for ipcMain.handle and app.on +async function dispatch(key: string, ...args: any[]) { + const handler = registeredHandlers.get(key); + assert(handler); + return await handler(null, ...args); +} + +const APP_PATH = path.join(__dirname, './tmp'); + +const browserWindow = { + isDestroyed: () => { + return false; + }, + setWindowButtonVisibility: (v: boolean) => { + // will be stubbed later + }, + webContents: { + send: (type: string, ...args: any[]) => { + // ... + }, + }, +}; + +const ipcMain = { + handle: (key: string, callback: (...args: any[]) => any) => { + registeredHandlers.set(key, callback); + }, +}; + +const nativeTheme = { + themeSource: 'light', +}; + +function compareBuffer(a: Uint8Array, b: Uint8Array) { + if (a.length !== b.length) { + return false; + } + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) { + return false; + } + } + return true; +} + +// dynamically import handlers so that we can inject local variables to mocks +vi.doMock('electron', () => { + return { + app: { + getPath: (name: string) => { + assert(name === 'appData'); + return APP_PATH; + }, + name: 'affine-test', + on: (name: string, callback: (...args: any[]) => any) => { + registeredHandlers.set(name, callback); + }, + }, + BrowserWindow: { + getAllWindows: () => { + return [browserWindow]; + }, + }, + nativeTheme: nativeTheme, + ipcMain, + }; +}); + +beforeEach(async () => { + // clean up tmp folder + const { registerHandlers } = await import('../handlers'); + registerHandlers(); +}); + +afterEach(async () => { + const { cleanupWorkspaceDBs } = await import('../handlers'); + cleanupWorkspaceDBs(); + await fs.remove(APP_PATH); +}); + +describe('ensureWorkspaceDB', () => { + test('should create db file on connection if it does not exist', async () => { + const id = 'test-workspace-id'; + const { ensureWorkspaceDB } = await import('../handlers'); + const workspaceDB = await ensureWorkspaceDB(id); + const file = workspaceDB.path; + const fileExists = await fs.pathExists(file); + expect(fileExists).toBe(true); + }); +}); + +describe('workspace handlers', () => { + test('list all workspace ids', async () => { + const ids = ['test-workspace-id', 'test-workspace-id-2']; + const { ensureWorkspaceDB } = await import('../handlers'); + await Promise.all(ids.map(id => ensureWorkspaceDB(id))); + const list = await dispatch('workspace:list'); + expect(list).toEqual(ids); + }); + + test('delete workspace', async () => { + const ids = ['test-workspace-id', 'test-workspace-id-2']; + const { ensureWorkspaceDB } = await import('../handlers'); + await Promise.all(ids.map(id => ensureWorkspaceDB(id))); + await dispatch('workspace:delete', 'test-workspace-id-2'); + const list = await dispatch('workspace:list'); + expect(list).toEqual(['test-workspace-id']); + }); +}); + +describe('UI handlers', () => { + test('theme-change', async () => { + await dispatch('ui:theme-change', 'dark'); + expect(nativeTheme.themeSource).toBe('dark'); + await dispatch('ui:theme-change', 'light'); + expect(nativeTheme.themeSource).toBe('light'); + }); + + test('sidebar-visibility-change (macOS)', async () => { + vi.stubGlobal('process', { platform: 'darwin' }); + const setWindowButtonVisibility = vi.fn(); + browserWindow.setWindowButtonVisibility = setWindowButtonVisibility; + await dispatch('ui:sidebar-visibility-change', true); + expect(setWindowButtonVisibility).toBeCalledWith(true); + await dispatch('ui:sidebar-visibility-change', false); + expect(setWindowButtonVisibility).toBeCalledWith(false); + vi.unstubAllGlobals(); + }); + + test('sidebar-visibility-change (non-macOS)', async () => { + vi.stubGlobal('process', { platform: 'linux' }); + const setWindowButtonVisibility = vi.fn(); + browserWindow.setWindowButtonVisibility = setWindowButtonVisibility; + await dispatch('ui:sidebar-visibility-change', true); + expect(setWindowButtonVisibility).not.toBeCalled(); + vi.unstubAllGlobals(); + }); +}); + +describe('db handlers', () => { + test('will reconnect on activate', async () => { + const { ensureWorkspaceDB } = await import('../handlers'); + const workspaceDB = await ensureWorkspaceDB('test-workspace-id'); + const instance = vi.spyOn(workspaceDB, 'reconnectDB'); + await dispatch('activate'); + expect(instance).toBeCalled(); + }); + + test('apply doc and get doc updates', async () => { + const workspaceId = 'test-workspace-id'; + const bin = await dispatch('db:get-doc', workspaceId); + // ? is this a good test? + expect(bin.every((byte: number) => byte === 0)).toBe(true); + + const ydoc = new Y.Doc(); + const ytext = ydoc.getText('test'); + ytext.insert(0, 'hello world'); + const bin2 = Y.encodeStateAsUpdate(ydoc); + + await dispatch('db:apply-doc-update', workspaceId, bin2); + + const bin3 = await dispatch('db:get-doc', workspaceId); + const ydoc2 = new Y.Doc(); + Y.applyUpdate(ydoc2, bin3); + const ytext2 = ydoc2.getText('test'); + expect(ytext2.toString()).toBe('hello world'); + }); + + test('get non existent doc', async () => { + const workspaceId = 'test-workspace-id'; + const bin = await dispatch('db:get-blob', workspaceId, 'non-existent-id'); + expect(bin).toBeNull(); + }); + + test('list blobs (empty)', async () => { + const workspaceId = 'test-workspace-id'; + const list = await dispatch('db:get-persisted-blobs', workspaceId); + expect(list).toEqual([]); + }); + + test('CRUD blobs', async () => { + const testBin = new Uint8Array([1, 2, 3, 4, 5]); + const testBin2 = new Uint8Array([6, 7, 8, 9, 10]); + const workspaceId = 'test-workspace-id'; + + // add blob + await dispatch('db:add-blob', workspaceId, 'testBin', testBin); + + // get blob + expect( + compareBuffer( + await dispatch('db:get-blob', workspaceId, 'testBin'), + testBin + ) + ).toBe(true); + + // add another blob + await dispatch('db:add-blob', workspaceId, 'testBin2', testBin2); + expect( + compareBuffer( + await dispatch('db:get-blob', workspaceId, 'testBin2'), + testBin2 + ) + ).toBe(true); + + // list blobs + let lists = await dispatch('db:get-persisted-blobs', workspaceId); + expect(lists).toHaveLength(2); + expect(lists).toContain('testBin'); + expect(lists).toContain('testBin2'); + + // delete blob + await dispatch('db:delete-blob', workspaceId, 'testBin'); + lists = await dispatch('db:get-persisted-blobs', workspaceId); + expect(lists).toEqual(['testBin2']); + }); +}); diff --git a/apps/electron/layers/main/src/data/fs-watch.ts b/apps/electron/layers/main/src/data/fs-watch.ts new file mode 100644 index 0000000000..3545a6b1ce --- /dev/null +++ b/apps/electron/layers/main/src/data/fs-watch.ts @@ -0,0 +1,7 @@ +import type { WatchListener } from 'fs-extra'; +import fs from 'fs-extra'; + +export function watchFile(path: string, callback: WatchListener) { + const watcher = fs.watch(path, callback); + return () => watcher.close(); +} diff --git a/apps/electron/layers/main/src/data/sqlite.ts b/apps/electron/layers/main/src/data/sqlite.ts index 19b1025feb..674d6246eb 100644 --- a/apps/electron/layers/main/src/data/sqlite.ts +++ b/apps/electron/layers/main/src/data/sqlite.ts @@ -36,24 +36,10 @@ interface BlobRow { export class WorkspaceDatabase { sqliteDB: Database; ydoc = new Y.Doc(); - - ready: Promise; + firstConnect = false; constructor(public path: string) { this.sqliteDB = this.reconnectDB(); - logger.log('open db', path); - - this.ydoc.on('update', update => { - this.addUpdateToSQLite(update); - }); - - this.ready = (async () => { - const updates = await this.getUpdates(); - updates.forEach(update => { - Y.applyUpdate(this.ydoc, update.data); - }); - return this.getEncodedDocUpdates(); - })(); } // release resources @@ -67,15 +53,22 @@ export class WorkspaceDatabase { if (this.sqliteDB) { this.sqliteDB.close(); } + // use cached version? - const db = sqlite(this.path); - // const db = new sqlite.Database(this.path, error => { - // if (error) { - // logger.error('open db error', error); - // } - // }); - this.sqliteDB = db; + const db = (this.sqliteDB = sqlite(this.path)); db.exec(schemas.join(';')); + + if (!this.firstConnect) { + this.ydoc.on('update', this.addUpdateToSQLite); + } + + const updates = this.getUpdates(); + updates.forEach(update => { + Y.applyUpdate(this.ydoc, update.data); + }); + + this.firstConnect = true; + return db; }; @@ -153,12 +146,15 @@ export class WorkspaceDatabase { } }; + // batch write instead write per key stroke? private addUpdateToSQLite = (data: Uint8Array) => { try { + const start = performance.now(); const statement = this.sqliteDB.prepare( 'INSERT INTO updates (data) VALUES (?)' ); statement.run(data); + logger.debug('addUpdateToSQLite', performance.now() - start, 'ms'); } catch (error) { logger.error('addUpdateToSQLite', error); } diff --git a/apps/electron/layers/main/src/handlers.ts b/apps/electron/layers/main/src/handlers.ts index 1a4c4ed028..ec8d477550 100644 --- a/apps/electron/layers/main/src/handlers.ts +++ b/apps/electron/layers/main/src/handlers.ts @@ -12,26 +12,69 @@ import { logger } from '../../logger'; import { isMacOS } from '../../utils'; import { appContext } from './context'; import { exportDatabase } from './data/export'; +import { watchFile } from './data/fs-watch'; import type { WorkspaceDatabase } from './data/sqlite'; import { openWorkspaceDatabase } from './data/sqlite'; import { deleteWorkspace, listWorkspaces } from './data/workspace'; import { getExchangeTokenParams, oauthEndpoint } from './google-auth'; +import { sendMainEvent } from './send-main-event'; let currentWorkspaceId = ''; const dbMapping = new Map(); +const dbWatchers = new Map void>(); +const dBLastUse = new Map(); -async function ensureWorkspaceDB(id: string) { +export async function ensureWorkspaceDB(id: string) { let workspaceDB = dbMapping.get(id); if (!workspaceDB) { // hmm... potential race condition? workspaceDB = await openWorkspaceDatabase(appContext, id); dbMapping.set(id, workspaceDB); + + logger.info('watch db file', workspaceDB.path); + + dbWatchers.set( + id, + watchFile(workspaceDB.path, (event, filename) => { + const minTime = 1000; + logger.debug( + 'db file changed', + event, + filename, + Date.now() - dBLastUse.get(id)! + ); + + if (Date.now() - dBLastUse.get(id)! < minTime || !filename) { + logger.debug('skip db update'); + return; + } + + sendMainEvent('main:on-db-update', id); + + // handle DB file update by other process + dbWatchers.get(id)?.(); + dbMapping.delete(id); + dbWatchers.delete(id); + ensureWorkspaceDB(id); + }) + ); } - await workspaceDB.ready; + dBLastUse.set(id, Date.now()); return workspaceDB; } +export async function cleanupWorkspaceDBs() { + for (const [id, db] of dbMapping) { + logger.info('close db connection', id); + db.destroy(); + dbWatchers.get(id)?.(); + } + dbMapping.clear(); + dbWatchers.clear(); + dBLastUse.clear(); +} + function registerWorkspaceHandlers() { ipcMain.handle('workspace:list', async _ => { logger.info('list workspaces'); diff --git a/apps/electron/layers/main/src/send-main-event.ts b/apps/electron/layers/main/src/send-main-event.ts new file mode 100644 index 0000000000..b1849168cc --- /dev/null +++ b/apps/electron/layers/main/src/send-main-event.ts @@ -0,0 +1,14 @@ +import { BrowserWindow } from 'electron'; + +import type { MainEventMap } from '../../main-events'; + +function getActiveWindows() { + return BrowserWindow.getAllWindows().filter(win => !win.isDestroyed()); +} + +export function sendMainEvent( + type: T, + ...args: Parameters +) { + getActiveWindows().forEach(win => win.webContents.send(type, ...args)); +} diff --git a/apps/electron/layers/preload/preload.d.ts b/apps/electron/layers/preload/preload.d.ts index 3fa7cba22d..1c259896b3 100644 --- a/apps/electron/layers/preload/preload.d.ts +++ b/apps/electron/layers/preload/preload.d.ts @@ -1,12 +1,6 @@ +/* eslint-disable @typescript-eslint/consistent-type-imports */ + interface Window { - /** - * After analyzing the `exposeInMainWorld` calls, - * `packages/preload/exposedInMainWorld.d.ts` file will be generated. - * It contains all interfaces. - * `packages/preload/exposedInMainWorld.d.ts` file is required for TS is `renderer` - * - * @see https://github.com/cawa-93/dts-for-context-bridge - */ - readonly apis: { db: { getDoc: (id: string) => Promise; applyDocUpdate: (id: string, update: Uint8Array) => Promise; addBlob: (workspaceId: string, key: string, data: Uint8Array) => Promise; getBlob: (workspaceId: string, key: string) => Promise; deleteBlob: (workspaceId: string, key: string) => Promise; getPersistedBlobs: (workspaceId: string) => Promise; }; workspace: { list: () => Promise; delete: (id: string) => Promise; }; openLoadDBFileDialog: () => Promise; openSaveDBFileDialog: () => Promise; onThemeChange: (theme: string) => Promise; onSidebarVisibilityChange: (visible: boolean) => Promise; onWorkspaceChange: (workspaceId: string) => Promise; openDBFolder: () => Promise; getGoogleOauthCode: () => Promise<{ requestInit: RequestInit; url: string; }>; updateEnv: (env: string, value: string) => void; }; - readonly appInfo: { electron: boolean; isMacOS: boolean; }; + apis: typeof import('./src/affine-apis').apis; + appInfo: typeof import('./src/affine-apis').appInfo; } diff --git a/apps/electron/layers/preload/src/affine-apis.ts b/apps/electron/layers/preload/src/affine-apis.ts new file mode 100644 index 0000000000..9b7fa21bfd --- /dev/null +++ b/apps/electron/layers/preload/src/affine-apis.ts @@ -0,0 +1,81 @@ +// NOTE: we will generate preload types from this file + +import { ipcRenderer } from 'electron'; + +import type { MainEventMap } from '../../main-events'; + +// main -> renderer +function onMainEvent( + eventName: T, + callback: MainEventMap[T] +): () => void { + // @ts-expect-error fix me later + const fn = (_, ...args) => callback(...args); + ipcRenderer.on(eventName, fn); + return () => ipcRenderer.off(eventName, fn); +} + +const apis = { + db: { + // workspace providers + getDoc: (id: string): Promise => + ipcRenderer.invoke('db:get-doc', id), + applyDocUpdate: (id: string, update: Uint8Array) => + ipcRenderer.invoke('db:apply-doc-update', id, update), + addBlob: (workspaceId: string, key: string, data: Uint8Array) => + ipcRenderer.invoke('db:add-blob', workspaceId, key, data), + getBlob: (workspaceId: string, key: string): Promise => + ipcRenderer.invoke('db:get-blob', workspaceId, key), + deleteBlob: (workspaceId: string, key: string) => + ipcRenderer.invoke('db:delete-blob', workspaceId, key), + getPersistedBlobs: (workspaceId: string): Promise => + ipcRenderer.invoke('db:get-persisted-blobs', workspaceId), + + // listeners + onDBUpdate: (callback: (workspaceId: string) => void) => { + return onMainEvent('main:on-db-update', callback); + }, + }, + + workspace: { + list: (): Promise => ipcRenderer.invoke('workspace:list'), + delete: (id: string): Promise => + ipcRenderer.invoke('workspace:delete', id), + // create will be implicitly called by db functions + }, + + openLoadDBFileDialog: () => ipcRenderer.invoke('ui:open-load-db-file-dialog'), + openSaveDBFileDialog: () => ipcRenderer.invoke('ui:open-save-db-file-dialog'), + + // ui + onThemeChange: (theme: string) => + ipcRenderer.invoke('ui:theme-change', theme), + + onSidebarVisibilityChange: (visible: boolean) => + ipcRenderer.invoke('ui:sidebar-visibility-change', visible), + + onWorkspaceChange: (workspaceId: string) => + ipcRenderer.invoke('ui:workspace-change', workspaceId), + + openDBFolder: () => ipcRenderer.invoke('ui:open-db-folder'), + + /** + * Try sign in using Google and return a request object to exchange the code for a token + * Not exchange in Node side because it is easier to do it in the renderer with VPN + */ + getGoogleOauthCode: (): Promise<{ requestInit: RequestInit; url: string }> => + ipcRenderer.invoke('ui:get-google-oauth-code'), + + /** + * Secret backdoor to update environment variables in main process + */ + updateEnv: (env: string, value: string) => { + ipcRenderer.invoke('main:env-update', env, value); + }, +}; + +const appInfo = { + electron: true, +}; + +export { apis, appInfo }; diff --git a/apps/electron/layers/preload/src/index.ts b/apps/electron/layers/preload/src/index.ts index 7d650ceef1..c187f096d6 100644 --- a/apps/electron/layers/preload/src/index.ts +++ b/apps/electron/layers/preload/src/index.ts @@ -2,9 +2,9 @@ * @module preload */ -import { contextBridge, ipcRenderer } from 'electron'; +import { contextBridge } from 'electron'; -import { isMacOS } from '../../utils'; +import * as affineApis from './affine-apis'; /** * The "Main World" is the JavaScript context that your main renderer code runs in. @@ -13,69 +13,5 @@ import { isMacOS } from '../../utils'; * @see https://www.electronjs.org/docs/api/context-bridge */ -/** - * After analyzing the `exposeInMainWorld` calls, - * `packages/preload/exposedInMainWorld.d.ts` file will be generated. - * It contains all interfaces. - * `packages/preload/exposedInMainWorld.d.ts` file is required for TS is `renderer` - * - * @see https://github.com/cawa-93/dts-for-context-bridge - */ -contextBridge.exposeInMainWorld('apis', { - db: { - // workspace providers - getDoc: (id: string): Promise => - ipcRenderer.invoke('db:get-doc', id), - applyDocUpdate: (id: string, update: Uint8Array) => - ipcRenderer.invoke('db:apply-doc-update', id, update), - addBlob: (workspaceId: string, key: string, data: Uint8Array) => - ipcRenderer.invoke('db:add-blob', workspaceId, key, data), - getBlob: (workspaceId: string, key: string): Promise => - ipcRenderer.invoke('db:get-blob', workspaceId, key), - deleteBlob: (workspaceId: string, key: string) => - ipcRenderer.invoke('db:delete-blob', workspaceId, key), - getPersistedBlobs: (workspaceId: string): Promise => - ipcRenderer.invoke('db:get-persisted-blobs', workspaceId), - }, - - workspace: { - list: (): Promise => ipcRenderer.invoke('workspace:list'), - delete: (id: string): Promise => - ipcRenderer.invoke('workspace:delete', id), - // create will be implicitly called by db functions - }, - - openLoadDBFileDialog: () => ipcRenderer.invoke('ui:open-load-db-file-dialog'), - openSaveDBFileDialog: () => ipcRenderer.invoke('ui:open-save-db-file-dialog'), - - // ui - onThemeChange: (theme: string) => - ipcRenderer.invoke('ui:theme-change', theme), - - onSidebarVisibilityChange: (visible: boolean) => - ipcRenderer.invoke('ui:sidebar-visibility-change', visible), - - onWorkspaceChange: (workspaceId: string) => - ipcRenderer.invoke('ui:workspace-change', workspaceId), - - openDBFolder: () => ipcRenderer.invoke('ui:open-db-folder'), - - /** - * Try sign in using Google and return a request object to exchange the code for a token - * Not exchange in Node side because it is easier to do it in the renderer with VPN - */ - getGoogleOauthCode: (): Promise<{ requestInit: RequestInit; url: string }> => - ipcRenderer.invoke('ui:get-google-oauth-code'), - - /** - * Secret backdoor to update environment variables in main process - */ - updateEnv: (env: string, value: string) => { - ipcRenderer.invoke('main:env-update', env, value); - }, -}); - -contextBridge.exposeInMainWorld('appInfo', { - electron: true, - isMacOS: isMacOS(), -}); +contextBridge.exposeInMainWorld('apis', affineApis.apis); +contextBridge.exposeInMainWorld('appInfo', affineApis.appInfo); diff --git a/apps/electron/package.json b/apps/electron/package.json index 48ed32335b..70664c4b81 100644 --- a/apps/electron/package.json +++ b/apps/electron/package.json @@ -6,8 +6,8 @@ "description": "AFFiNE App", "homepage": "https://github.com/toeverything/AFFiNE", "scripts": { - "dev": "cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs", - "prod": "node scripts/dev.mjs", + "dev": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs", + "prod": "yarn electron-rebuild && yarn node scripts/dev.mjs", "generate-assets": "zx scripts/generate-assets.mjs", "package": "electron-forge package", "make": "electron-forge make", @@ -15,7 +15,8 @@ "make-macos-x64": "electron-forge make --platform=darwin --arch=x64", "make-windows-x64": "electron-forge make --platform=win32 --arch=x64", "make-linux-x64": "electron-forge make --platform=linux --arch=x64", - "electron-rebuild": "electron-rebuild" + "rebuild:for-test": "yarn rebuild better-sqlite3", + "rebuild:for-electron": "yarn electron-rebuild" }, "config": { "forge": "./forge.config.js" @@ -35,7 +36,6 @@ "@types/better-sqlite3": "^7.6.4", "@types/fs-extra": "^11.0.1", "cross-env": "7.0.3", - "dts-for-context-bridge": "^0.7.1", "electron": "24.1.2", "electron-log": "^5.0.0-beta.23", "electron-squirrel-startup": "1.0.0", diff --git a/apps/electron/scripts/dev.mjs b/apps/electron/scripts/dev.mjs index 7d9ca38728..ffca8e4274 100644 --- a/apps/electron/scripts/dev.mjs +++ b/apps/electron/scripts/dev.mjs @@ -3,7 +3,6 @@ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { generateAsync } from 'dts-for-context-bridge'; import electronPath from 'electron'; import * as esbuild from 'esbuild'; @@ -81,10 +80,6 @@ async function main() { setup(build) { let initialBuild = false; build.onEnd(() => { - generateAsync({ - input: 'layers/preload/src/**/*.ts', - output: 'layers/preload/preload.d.ts', - }); if (initialBuild) { console.log(`[preload] has changed`); spawnOrReloadElectron(); diff --git a/packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts b/packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts index c8f2a8d4f0..0db6874752 100644 --- a/packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts +++ b/packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts @@ -14,6 +14,10 @@ let provider: SQLiteProvider; let offlineYdoc: YType.Doc; +let triggerDBUpdate: ((_: string) => void) | null = null; + +const mockedAddBlob = vi.fn(); + vi.stubGlobal('window', { apis: { db: { @@ -24,8 +28,16 @@ vi.stubGlobal('window', { Y.applyUpdate(offlineYdoc, update, 'sqlite'); }, getPersistedBlobs: async (id: string) => { + // todo: may need to hack the way to get hash keys of blobs return []; }, + onDBUpdate: (fn: (id: string) => void) => { + triggerDBUpdate = fn; + return () => { + triggerDBUpdate = null; + }; + }, + addBlob: mockedAddBlob, } satisfies Partial, }, }); @@ -43,7 +55,7 @@ beforeEach(() => { workspace.register(AffineSchemas).register(__unstableSchemas); provider = createSQLiteProvider(workspace); offlineYdoc = new Y.Doc(); - offlineYdoc.getText('text').insert(0, ''); + offlineYdoc.getText('text').insert(0, 'sqlite-hello'); }); describe('SQLite provider', () => { @@ -53,19 +65,71 @@ describe('SQLite provider', () => { // Workspace.Y.applyUpdate(workspace.doc); workspace.doc.getText('text').insert(0, 'mem-hello'); - expect(offlineYdoc.getText('text').toString()).toBe(''); + expect(offlineYdoc.getText('text').toString()).toBe('sqlite-hello'); await provider.connect(); - expect(offlineYdoc.getText('text').toString()).toBe('mem-hello'); - expect(workspace.doc.getText('text').toString()).toBe('mem-hello'); + // depending on the nature of the sync, the data can be sync'ed in either direction + const options = ['mem-hellosqlite-hello', 'sqlite-hellomem-hello']; + const synced = options.filter( + o => o === offlineYdoc.getText('text').toString() + ); + expect(synced.length).toBe(1); + expect(workspace.doc.getText('text').toString()).toBe(synced[0]); workspace.doc.getText('text').insert(0, 'world'); // check if the data are sync'ed - expect(offlineYdoc.getText('text').toString()).toBe('worldmem-hello'); + expect(offlineYdoc.getText('text').toString()).toBe('world' + synced[0]); }); - // todo: test disconnect - // todo: test blob sync + test('blobs will be synced to sqlite on connect', async () => { + // mock bs.list + const bin = new Uint8Array([1, 2, 3]); + const blob = new Blob([bin]); + workspace.blobs.list = vi.fn(async () => ['blob1']); + workspace.blobs.get = vi.fn(async (key: string) => { + return blob; + }); + + await provider.connect(); + await new Promise(resolve => setTimeout(resolve, 100)); + + expect(mockedAddBlob).toBeCalledWith(id, 'blob1', bin); + }); + + test('on db update', async () => { + vi.useFakeTimers(); + await provider.connect(); + + offlineYdoc.getText('text').insert(0, 'sqlite-world'); + + triggerDBUpdate?.(id); + + // not yet updated + expect(workspace.doc.getText('text').toString()).toBe('sqlite-hello'); + + // wait for the update to be sync'ed + await vi.advanceTimersByTimeAsync(1000); + + expect(workspace.doc.getText('text').toString()).toBe( + 'sqlite-worldsqlite-hello' + ); + vi.useRealTimers(); + }); + + test('disconnect handlers', async () => { + const offHandler = vi.fn(); + let handleUpdate = () => {}; + workspace.doc.on = (_: string, fn: () => void) => { + handleUpdate = fn; + }; + workspace.doc.off = offHandler; + await provider.connect(); + + provider.disconnect(); + + expect(triggerDBUpdate).toBe(null); + expect(offHandler).toBeCalledWith('update', handleUpdate); + }); }); diff --git a/packages/workspace/src/providers/index.ts b/packages/workspace/src/providers/index.ts index be3e3ed9e7..33da1428b5 100644 --- a/packages/workspace/src/providers/index.ts +++ b/packages/workspace/src/providers/index.ts @@ -169,7 +169,7 @@ const createSQLiteProvider = ( keysToPersist.forEach(async k => { const blob = await bs.get(k); if (!blob) { - logger.warn('blob url not found', k); + logger.warn('blob not found for', k); return; } window.apis.db.addBlob( @@ -180,6 +180,29 @@ const createSQLiteProvider = ( }); } + async function syncUpdates() { + logger.info('syncing updates from sqlite', blockSuiteWorkspace.id); + const updates = await window.apis.db.getDoc(blockSuiteWorkspace.id); + + if (updates) { + Y.applyUpdate(blockSuiteWorkspace.doc, updates, sqliteOrigin); + } + + const mergeUpdates = Y.encodeStateAsUpdate(blockSuiteWorkspace.doc); + + // also apply updates to sqlite + window.apis.db.applyDocUpdate(blockSuiteWorkspace.id, mergeUpdates); + + const bs = blockSuiteWorkspace.blobs; + + if (bs) { + // this can be non-blocking + syncBlobIntoSQLite(bs); + } + } + + let unsubscribe = () => {}; + const provider = { flavour: 'sqlite', background: true, @@ -188,31 +211,32 @@ const createSQLiteProvider = ( }, connect: async () => { logger.info('connecting sqlite provider', blockSuiteWorkspace.id); - const updates = await window.apis.db.getDoc(blockSuiteWorkspace.id); - - if (updates) { - Y.applyUpdate(blockSuiteWorkspace.doc, updates, sqliteOrigin); - } - - const mergeUpdates = Y.encodeStateAsUpdate(blockSuiteWorkspace.doc); - - // also apply updates to sqlite - window.apis.db.applyDocUpdate(blockSuiteWorkspace.id, mergeUpdates); + await syncUpdates(); blockSuiteWorkspace.doc.on('update', handleUpdate); - const bs = blockSuiteWorkspace.blobs; - - if (bs) { - // this can be non-blocking - syncBlobIntoSQLite(bs); - } + let timer = 0; + unsubscribe = window.apis.db.onDBUpdate(workspaceId => { + if (workspaceId === blockSuiteWorkspace.id) { + // throttle + logger.debug('on db update', workspaceId); + if (timer) { + clearTimeout(timer); + } + // @ts-expect-error ignore the type + timer = setTimeout(() => { + syncUpdates(); + timer = 0; + }, 1000); + } + }); // blockSuiteWorkspace.doc.on('destroy', ...); logger.info('connecting sqlite done', blockSuiteWorkspace.id); }, disconnect: () => { - // todo: not implemented + unsubscribe(); + blockSuiteWorkspace.doc.off('update', handleUpdate); }, } satisfies SQLiteProvider; diff --git a/vitest.config.ts b/vitest.config.ts index 7486bb2c35..7a3b5478f2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -26,6 +26,7 @@ export default defineConfig({ 'packages/**/*.spec.tsx', 'apps/web/**/*.spec.ts', 'apps/web/**/*.spec.tsx', + 'apps/electron/**/*.spec.ts', 'tests/unit/**/*.spec.ts', 'tests/unit/**/*.spec.tsx', ], diff --git a/yarn.lock b/yarn.lock index 907114068b..b1ea409bc6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -126,7 +126,6 @@ __metadata: "@types/fs-extra": ^11.0.1 better-sqlite3: ^8.3.0 cross-env: 7.0.3 - dts-for-context-bridge: ^0.7.1 electron: 24.1.2 electron-log: ^5.0.0-beta.23 electron-squirrel-startup: 1.0.0 @@ -7398,18 +7397,6 @@ __metadata: languageName: node linkType: hard -"@ts-morph/common@npm:~0.12.2": - version: 0.12.3 - resolution: "@ts-morph/common@npm:0.12.3" - dependencies: - fast-glob: ^3.2.7 - minimatch: ^3.0.4 - mkdirp: ^1.0.4 - path-browserify: ^1.0.1 - checksum: d96ea9805d4f0300cc05c47daa9454438903b86ffb7116f5181a1eba71e881012a1adc2a867b3afbe4429ef29e3e0d6204175cbaf33ecdd7a7d09b5d8a37f12d - languageName: node - linkType: hard - "@ts-morph/common@npm:~0.19.0": version: 0.19.0 resolution: "@ts-morph/common@npm:0.19.0" @@ -7959,13 +7946,6 @@ __metadata: languageName: node linkType: hard -"@types/q@npm:^1.5.1": - version: 1.5.5 - resolution: "@types/q@npm:1.5.5" - checksum: 3bd386fb97a0e5f1ce1ed7a14e39b60e469b5ca9d920a7f69e0cdb58d22c0f5bdd16637d8c3a5bfeda76663c023564dd47a65389ee9aaabd65aee54803d5ba45 - languageName: node - linkType: hard - "@types/qs@npm:*, @types/qs@npm:^6.9.5": version: 6.9.7 resolution: "@types/qs@npm:6.9.7" @@ -10155,7 +10135,7 @@ __metadata: languageName: node linkType: hard -"chalk@npm:^2.0.0, chalk@npm:^2.4.1, chalk@npm:^2.4.2": +"chalk@npm:^2.0.0, chalk@npm:^2.4.2": version: 2.4.2 resolution: "chalk@npm:2.4.2" dependencies: @@ -10486,24 +10466,6 @@ __metadata: languageName: node linkType: hard -"coa@npm:2.0.2": - version: 2.0.2 - resolution: "coa@npm:2.0.2" - dependencies: - "@types/q": ^1.5.1 - chalk: ^2.4.1 - q: ^1.1.2 - checksum: 44736914aac2160d3d840ed64432a90a3bb72285a0cd6a688eb5cabdf15d15a85eee0915b3f6f2a4659d5075817b1cb577340d3c9cbb47d636d59ab69f819552 - languageName: node - linkType: hard - -"code-block-writer@npm:^11.0.0": - version: 11.0.3 - resolution: "code-block-writer@npm:11.0.3" - checksum: f0a2605f19963d7087267c9b0fd0b05a6638a50e7b29b70f97aa01a514f59475b0626f8aa092188df853ee6d96745426dfa132d6a677795df462c6ce32c21639 - languageName: node - linkType: hard - "code-block-writer@npm:^12.0.0": version: 12.0.0 resolution: "code-block-writer@npm:12.0.0" @@ -11678,18 +11640,6 @@ __metadata: languageName: node linkType: hard -"dts-for-context-bridge@npm:^0.7.1": - version: 0.7.1 - resolution: "dts-for-context-bridge@npm:0.7.1" - dependencies: - coa: 2.0.2 - ts-morph: 13.0.2 - bin: - dts-cb: bin/index.js - checksum: cb910eecb82ac1b3d506990a244083389c74c90ab599382e174a980d247e905a1d49500e85806ede75f00b3848c14197a129ecaad436e7a5bfe0f3dcb8cd1859 - languageName: node - linkType: hard - "duplexer2@npm:^0.1.2": version: 0.1.4 resolution: "duplexer2@npm:0.1.4" @@ -19390,7 +19340,7 @@ __metadata: languageName: node linkType: hard -"q@npm:^1.1.2, q@npm:^1.5.1": +"q@npm:^1.5.1": version: 1.5.1 resolution: "q@npm:1.5.1" checksum: 147baa93c805bc1200ed698bdf9c72e9e42c05f96d007e33a558b5fdfd63e5ea130e99313f28efc1783e90e6bdb4e48b67a36fcc026b7b09202437ae88a1fb12 @@ -22141,16 +22091,6 @@ __metadata: languageName: node linkType: hard -"ts-morph@npm:13.0.2": - version: 13.0.2 - resolution: "ts-morph@npm:13.0.2" - dependencies: - "@ts-morph/common": ~0.12.2 - code-block-writer: ^11.0.0 - checksum: 325e850b99f96a71b57ced55e2bf687bdc6dc477070a946a47ca925633c62f201d2a706c6972e7bb6a37d7d9e0414b040fd46800d2fd86423185818fefe284a6 - languageName: node - linkType: hard - "ts-morph@npm:18.0.0": version: 18.0.0 resolution: "ts-morph@npm:18.0.0"