feat: sqlite subdocument (#2816)

Co-authored-by: Alex Yang <himself65@outlook.com>
This commit is contained in:
Peng Xiao 2023-06-27 15:40:37 +08:00 committed by GitHub
parent 4307e1eb6b
commit 05452bb297
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 842 additions and 426 deletions

View File

@ -12,6 +12,7 @@ import type { PlaywrightTestConfig } from '@playwright/test';
*/ */
const config: PlaywrightTestConfig = { const config: PlaywrightTestConfig = {
testDir: './tests', testDir: './tests',
testIgnore: '**/lib/**',
fullyParallel: true, fullyParallel: true,
timeout: process.env.CI ? 50_000 : 30_000, timeout: process.env.CI ? 50_000 : 30_000,
use: { use: {

View File

@ -20,14 +20,31 @@ afterEach(async () => {
await fs.remove(tmpDir); await fs.remove(tmpDir);
}); });
let testYDoc: Y.Doc;
let testYSubDoc: Y.Doc;
function getTestUpdates() { function getTestUpdates() {
const testYDoc = new Y.Doc(); testYDoc = new Y.Doc();
const yText = testYDoc.getText('test'); const yText = testYDoc.getText('test');
yText.insert(0, 'hello'); yText.insert(0, 'hello');
testYSubDoc = new Y.Doc();
testYDoc.getMap('subdocs').set('test-subdoc', testYSubDoc);
const updates = Y.encodeStateAsUpdate(testYDoc); const updates = Y.encodeStateAsUpdate(testYDoc);
return updates; return updates;
} }
function getTestSubDocUpdates() {
const yText = testYSubDoc.getText('test');
yText.insert(0, 'hello');
const updates = Y.encodeStateAsUpdate(testYSubDoc);
return updates;
}
test('can create new db file if not exists', async () => { test('can create new db file if not exists', async () => {
const { openWorkspaceDatabase } = await import('../workspace-db-adapter'); const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
const workspaceId = v4(); const workspaceId = v4();
@ -68,6 +85,31 @@ test('on applyUpdate (from renderer), will trigger update', async () => {
await db.destroy(); await db.destroy();
}); });
test('on applyUpdate (from renderer, subdoc), will trigger update', async () => {
const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
const workspaceId = v4();
const onUpdate = vi.fn();
const insertUpdates = vi.fn();
const db = await openWorkspaceDatabase(workspaceId);
db.applyUpdate(getTestUpdates(), 'renderer');
db.db!.insertUpdates = insertUpdates;
db.update$.subscribe(onUpdate);
const subdocUpdates = getTestSubDocUpdates();
db.applyUpdate(subdocUpdates, 'renderer', testYSubDoc.guid);
expect(onUpdate).toHaveBeenCalled();
expect(insertUpdates).toHaveBeenCalledWith([
{
docId: testYSubDoc.guid,
data: subdocUpdates,
},
]);
await db.destroy();
});
test('on applyUpdate (from external), will trigger update & send external update event', async () => { test('on applyUpdate (from external), will trigger update & send external update event', async () => {
const { openWorkspaceDatabase } = await import('../workspace-db-adapter'); const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
const workspaceId = v4(); const workspaceId = v4();

View File

@ -1,4 +1,4 @@
import { SqliteConnection } from '@affine/native'; import { type InsertRow, SqliteConnection } from '@affine/native';
import { logger } from '../logger'; import { logger } from '../logger';
@ -79,21 +79,34 @@ export abstract class BaseSQLiteAdapter {
} }
} }
async getUpdates() { async getUpdates(docId?: string) {
try { try {
if (!this.db) { if (!this.db) {
logger.warn(`${this.path} is not connected`); logger.warn(`${this.path} is not connected`);
return []; return [];
} }
return await this.db.getUpdates(); return await this.db.getUpdates(docId);
} catch (error) { } catch (error) {
logger.error('getUpdates', error); logger.error('getUpdates', error);
return []; return [];
} }
} }
async getAllUpdates() {
try {
if (!this.db) {
logger.warn(`${this.path} is not connected`);
return [];
}
return await this.db.getAllUpdates();
} catch (error) {
logger.error('getAllUpdates', error);
return [];
}
}
// add a single update to SQLite // add a single update to SQLite
async addUpdateToSQLite(updates: Uint8Array[]) { async addUpdateToSQLite(updates: InsertRow[]) {
// batch write instead write per key stroke? // batch write instead write per key stroke?
try { try {
if (!this.db) { if (!this.db) {

View File

@ -7,13 +7,17 @@ export * from './ensure-db';
export * from './subjects'; export * from './subjects';
export const dbHandlers = { export const dbHandlers = {
getDocAsUpdates: async (id: string) => { getDocAsUpdates: async (workspaceId: string, subdocId?: string) => {
const workspaceDB = await ensureSQLiteDB(id); const workspaceDB = await ensureSQLiteDB(workspaceId);
return workspaceDB.getDocAsUpdates(); return workspaceDB.getDocAsUpdates(subdocId);
}, },
applyDocUpdate: async (id: string, update: Uint8Array) => { applyDocUpdate: async (
const workspaceDB = await ensureSQLiteDB(id); workspaceId: string,
return workspaceDB.applyUpdate(update); update: Uint8Array,
subdocId?: string
) => {
const workspaceDB = await ensureSQLiteDB(workspaceId);
return workspaceDB.applyUpdate(update, 'renderer', subdocId);
}, },
addBlob: async (workspaceId: string, key: string, data: Uint8Array) => { addBlob: async (workspaceId: string, key: string, data: Uint8Array) => {
const workspaceDB = await ensureSQLiteDB(workspaceId); const workspaceDB = await ensureSQLiteDB(workspaceId);
@ -38,7 +42,11 @@ export const dbHandlers = {
export const dbEvents = { export const dbEvents = {
onExternalUpdate: ( onExternalUpdate: (
fn: (update: { workspaceId: string; update: Uint8Array }) => void fn: (update: {
workspaceId: string;
update: Uint8Array;
docId?: string;
}) => void
) => { ) => {
const sub = dbSubjects.externalUpdate.subscribe(fn); const sub = dbSubjects.externalUpdate.subscribe(fn);
return () => { return () => {

View File

@ -1,6 +1,6 @@
import assert from 'node:assert'; import assert from 'node:assert';
import type { SqliteConnection } from '@affine/native'; import type { InsertRow } from '@affine/native';
import { debounce } from 'lodash-es'; import { debounce } from 'lodash-es';
import * as Y from 'yjs'; import * as Y from 'yjs';
@ -8,19 +8,19 @@ import { logger } from '../logger';
import type { YOrigin } from '../type'; import type { YOrigin } from '../type';
import { getWorkspaceMeta } from '../workspace'; import { getWorkspaceMeta } from '../workspace';
import { BaseSQLiteAdapter } from './base-db-adapter'; import { BaseSQLiteAdapter } from './base-db-adapter';
import { mergeUpdate } from './merge-update';
import type { WorkspaceSQLiteDB } from './workspace-db-adapter'; import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
const FLUSH_WAIT_TIME = 5000; const FLUSH_WAIT_TIME = 5000;
const FLUSH_MAX_WAIT_TIME = 10000; const FLUSH_MAX_WAIT_TIME = 10000;
// todo: trim db when it is too big
export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
role = 'secondary'; role = 'secondary';
yDoc = new Y.Doc(); yDoc = new Y.Doc();
firstConnected = false; firstConnected = false;
destroyed = false; destroyed = false;
updateQueue: Uint8Array[] = []; updateQueue: { data: Uint8Array; docId?: string }[] = [];
unsubscribers = new Set<() => void>(); unsubscribers = new Set<() => void>();
@ -29,10 +29,23 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
public upstream: WorkspaceSQLiteDB public upstream: WorkspaceSQLiteDB
) { ) {
super(path); super(path);
this.setupAndListen(); this.init();
logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId); logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId);
} }
getDoc(docId?: string) {
if (!docId) {
return this.yDoc;
}
// this should be pretty fast and we don't need to cache it
for (const subdoc of this.yDoc.subdocs) {
if (subdoc.guid === docId) {
return subdoc;
}
}
return null;
}
override async destroy() { override async destroy() {
await this.flushUpdateQueue(); await this.flushUpdateQueue();
this.unsubscribers.forEach(unsub => unsub()); this.unsubscribers.forEach(unsub => unsub());
@ -47,7 +60,7 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
// do not update db immediately, instead, push to a queue // do not update db immediately, instead, push to a queue
// and flush the queue in a future time // and flush the queue in a future time
async addUpdateToUpdateQueue(db: SqliteConnection, update: Uint8Array) { async addUpdateToUpdateQueue(update: InsertRow) {
this.updateQueue.push(update); this.updateQueue.push(update);
await this.debouncedFlush(); await this.debouncedFlush();
} }
@ -101,55 +114,82 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
} }
} }
setupAndListen() { setupListener(docId?: string) {
if (this.firstConnected) { const doc = this.getDoc(docId);
if (!doc) {
return; return;
} }
this.firstConnected = true;
const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => { const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
if (origin === 'renderer') { if (origin === 'renderer') {
// update to upstream yDoc should be replicated to self yDoc // update to upstream yDoc should be replicated to self yDoc
this.applyUpdate(update, 'upstream'); this.applyUpdate(update, 'upstream', docId);
} }
}; };
const onSelfUpdate = async (update: Uint8Array, origin: YOrigin) => { const onSelfUpdate = async (update: Uint8Array, origin: YOrigin) => {
// for self update from upstream, we need to push it to external DB // for self update from upstream, we need to push it to external DB
if (origin === 'upstream' && this.db) { if (origin === 'upstream') {
await this.addUpdateToUpdateQueue(this.db, update); await this.addUpdateToUpdateQueue({
data: update,
docId,
});
} }
if (origin === 'self') { if (origin === 'self') {
this.upstream.applyUpdate(update, 'external'); this.upstream.applyUpdate(update, 'external', docId);
} }
}; };
const onSubdocs = ({ added }: { added: Set<Y.Doc> }) => {
added.forEach(subdoc => {
this.setupListener(subdoc.guid);
});
};
// listen to upstream update // listen to upstream update
this.upstream.yDoc.on('update', onUpstreamUpdate); this.upstream.yDoc.on('update', onUpstreamUpdate);
this.yDoc.on('update', onSelfUpdate); this.yDoc.on('update', onSelfUpdate);
this.yDoc.on('subdocs', onSubdocs);
this.unsubscribers.add(() => { this.unsubscribers.add(() => {
this.upstream.yDoc.off('update', onUpstreamUpdate); this.upstream.yDoc.off('update', onUpstreamUpdate);
this.yDoc.off('update', onSelfUpdate); this.yDoc.off('update', onSelfUpdate);
}); this.yDoc.off('subdocs', onSubdocs);
this.run(() => {
// apply all updates from upstream
const upstreamUpdate = this.upstream.getDocAsUpdates();
// to initialize the yDoc, we need to apply all updates from the db
this.applyUpdate(upstreamUpdate, 'upstream');
})
.then(() => {
logger.debug('run success');
})
.catch(err => {
logger.error('run error', err);
}); });
} }
applyUpdate = (data: Uint8Array, origin: YOrigin = 'upstream') => { init() {
if (this.firstConnected) {
return;
}
this.firstConnected = true;
this.setupListener();
// apply all updates from upstream
// we assume here that the upstream ydoc is already sync'ed
const syncUpstreamDoc = (docId?: string) => {
const update = this.upstream.getDocAsUpdates(docId);
if (update) {
this.applyUpdate(update, 'upstream');
}
};
syncUpstreamDoc();
this.upstream.yDoc.subdocs.forEach(subdoc => {
syncUpstreamDoc(subdoc.guid);
});
}
applyUpdate = (
data: Uint8Array,
origin: YOrigin = 'upstream',
docId?: string
) => {
const doc = this.getDoc(docId);
if (doc) {
Y.applyUpdate(this.yDoc, data, origin); Y.applyUpdate(this.yDoc, data, origin);
} else {
logger.warn('applyUpdate: doc not found', docId);
}
}; };
// TODO: have a better solution to handle blobs // TODO: have a better solution to handle blobs
@ -186,23 +226,33 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
async pull() { async pull() {
const start = performance.now(); const start = performance.now();
assert(this.upstream.db, 'upstream db should be connected'); assert(this.upstream.db, 'upstream db should be connected');
const updates = await this.run(async () => { const rows = await this.run(async () => {
// TODO: no need to get all updates, just get the latest ones (using a cursor, etc)? // TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
await this.syncBlobs(); await this.syncBlobs();
return (await this.getUpdates()).map(update => update.data); return await this.getAllUpdates();
}); });
if (!updates || this.destroyed) { if (!rows || this.destroyed) {
return; return;
} }
const merged = mergeUpdate(updates); // apply root doc first
this.applyUpdate(merged, 'self'); rows.forEach(row => {
if (!row.docId) {
this.applyUpdate(row.data, 'self');
}
});
rows.forEach(row => {
if (row.docId) {
this.applyUpdate(row.data, 'self', row.docId);
}
});
logger.debug( logger.debug(
'pull external updates', 'pull external updates',
this.path, this.path,
updates.length, rows.length,
(performance.now() - start).toFixed(2), (performance.now() - start).toFixed(2),
'ms' 'ms'
); );

View File

@ -1,5 +1,9 @@
import { Subject } from 'rxjs'; import { Subject } from 'rxjs';
export const dbSubjects = { export const dbSubjects = {
externalUpdate: new Subject<{ workspaceId: string; update: Uint8Array }>(), externalUpdate: new Subject<{
workspaceId: string;
update: Uint8Array;
docId?: string;
}>(),
}; };

View File

@ -1,3 +1,5 @@
import type { InsertRow } from '@affine/native';
import { debounce } from 'lodash-es';
import { Subject } from 'rxjs'; import { Subject } from 'rxjs';
import * as Y from 'yjs'; import * as Y from 'yjs';
@ -5,9 +7,10 @@ import { logger } from '../logger';
import type { YOrigin } from '../type'; import type { YOrigin } from '../type';
import { getWorkspaceMeta } from '../workspace'; import { getWorkspaceMeta } from '../workspace';
import { BaseSQLiteAdapter } from './base-db-adapter'; import { BaseSQLiteAdapter } from './base-db-adapter';
import { mergeUpdate } from './merge-update';
import { dbSubjects } from './subjects'; import { dbSubjects } from './subjects';
const TRIM_SIZE = 500;
export class WorkspaceSQLiteDB extends BaseSQLiteAdapter { export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
role = 'primary'; role = 'primary';
yDoc = new Y.Doc(); yDoc = new Y.Doc();
@ -28,33 +31,76 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
this.firstConnected = false; this.firstConnected = false;
} }
getDoc(docId?: string) {
if (!docId) {
return this.yDoc;
}
// this should be pretty fast and we don't need to cache it
for (const subdoc of this.yDoc.subdocs) {
if (subdoc.guid === docId) {
return subdoc;
}
}
return null;
}
getWorkspaceName = () => { getWorkspaceName = () => {
return this.yDoc.getMap('space:meta').get('name') as string; return this.yDoc.getMap('meta').get('name') as string;
}; };
setupListener(docId?: string) {
const doc = this.getDoc(docId);
if (doc) {
const onUpdate = async (update: Uint8Array, origin: YOrigin) => {
const insertRows = [{ data: update, docId }];
if (origin === 'renderer') {
await this.addUpdateToSQLite(insertRows);
} else if (origin === 'external') {
dbSubjects.externalUpdate.next({
workspaceId: this.workspaceId,
update,
docId,
});
await this.addUpdateToSQLite(insertRows);
logger.debug('external update', this.workspaceId);
}
};
const onSubdocs = ({ added }: { added: Set<Y.Doc> }) => {
added.forEach(subdoc => {
this.setupListener(subdoc.guid);
});
};
doc.on('update', onUpdate);
doc.on('subdocs', onSubdocs);
} else {
logger.error('setupListener: doc not found', docId);
}
}
async init() { async init() {
const db = await super.connectIfNeeded(); const db = await super.connectIfNeeded();
if (!this.firstConnected) { if (!this.firstConnected) {
this.yDoc.on('update', async (update: Uint8Array, origin: YOrigin) => { this.setupListener();
if (origin === 'renderer') {
await this.addUpdateToSQLite([update]);
} else if (origin === 'external') {
dbSubjects.externalUpdate.next({
workspaceId: this.workspaceId,
update,
});
await this.addUpdateToSQLite([update]);
logger.debug('external update', this.workspaceId);
}
});
} }
const updates = await this.getUpdates(); const updates = await this.getAllUpdates();
const merged = mergeUpdate(updates.map(update => update.data));
// to initialize the yDoc, we need to apply all updates from the db // apply root first (without ID).
this.applyUpdate(merged, 'self'); // subdoc will be available after root is applied
updates.forEach(update => {
if (!update.docId) {
this.applyUpdate(update.data, 'self');
}
});
// then, for all subdocs, apply the updates
updates.forEach(update => {
if (update.docId) {
this.applyUpdate(update.data, 'self', update.docId);
}
});
this.firstConnected = true; this.firstConnected = true;
this.update$.next(); this.update$.next();
@ -62,18 +108,32 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
return db; return db;
} }
getDocAsUpdates = () => { // unlike getUpdates, this will return updates in yDoc
return Y.encodeStateAsUpdate(this.yDoc); getDocAsUpdates = (docId?: string) => {
const doc = docId ? this.getDoc(docId) : this.yDoc;
if (doc) {
return Y.encodeStateAsUpdate(doc);
}
return null;
}; };
// non-blocking and use yDoc to validate the update // non-blocking and use yDoc to validate the update
// after that, the update is added to the db // after that, the update is added to the db
applyUpdate = (data: Uint8Array, origin: YOrigin = 'renderer') => { applyUpdate = (
data: Uint8Array,
origin: YOrigin = 'renderer',
docId?: string
) => {
// todo: trim the updates when the number of records is too large // todo: trim the updates when the number of records is too large
// 1. store the current ydoc state in the db // 1. store the current ydoc state in the db
// 2. then delete the old updates // 2. then delete the old updates
// yjs-idb will always trim the db for the first time after DB is loaded // yjs-idb will always trim the db for the first time after DB is loaded
Y.applyUpdate(this.yDoc, data, origin); const doc = this.getDoc(docId);
if (doc) {
Y.applyUpdate(doc, data, origin);
} else {
logger.warn('applyUpdate: doc not found', docId);
}
}; };
override async addBlob(key: string, value: Uint8Array) { override async addBlob(key: string, value: Uint8Array) {
@ -87,10 +147,30 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
await super.deleteBlob(key); await super.deleteBlob(key);
} }
override async addUpdateToSQLite(data: Uint8Array[]) { override async addUpdateToSQLite(data: InsertRow[]) {
this.update$.next(); this.update$.next();
data.forEach(row => {
this.trimWhenNecessary(row.docId)?.catch(err => {
logger.error('trimWhenNecessary failed', err);
});
});
await super.addUpdateToSQLite(data); await super.addUpdateToSQLite(data);
} }
trimWhenNecessary = debounce(async (docId?: string) => {
if (this.firstConnected) {
const count = (await this.db?.getUpdatesCount(docId)) ?? 0;
if (count > TRIM_SIZE) {
logger.debug(`trim ${this.workspaceId}:${docId} ${count}`);
const update = this.getDocAsUpdates(docId);
if (update) {
const insertRows = [{ data: update, docId }];
await this.db?.replaceUpdates(docId, insertRows);
logger.debug(`trim ${this.workspaceId}:${docId} successfully`);
}
}
}
}, 1000);
} }
export async function openWorkspaceDatabase(workspaceId: string) { export async function openWorkspaceDatabase(workspaceId: string) {

View File

@ -34,7 +34,6 @@ export function registerProtocol() {
const url = request.url.replace(/^file:\/\//, ''); const url = request.url.replace(/^file:\/\//, '');
const realpath = toAbsolutePath(url); const realpath = toAbsolutePath(url);
callback(realpath); callback(realpath);
console.log('interceptFileProtocol realpath', request.url, realpath);
return true; return true;
}); });

View File

@ -71,7 +71,7 @@ beforeEach(async () => {
.register(AffineSchemas) .register(AffineSchemas)
.register(__unstableSchemas); .register(__unstableSchemas);
const initPage = async (page: Page) => { const initPage = async (page: Page) => {
await page.waitForLoaded() await page.waitForLoaded();
expect(page).not.toBeNull(); expect(page).not.toBeNull();
assertExists(page); assertExists(page);
const pageBlockId = page.addBlock('affine:page', { const pageBlockId = page.addBlock('affine:page', {

View File

@ -9,7 +9,7 @@ import { useAtom, useAtomValue } from 'jotai';
import Link from 'next/link'; import Link from 'next/link';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import type { ReactElement } from 'react'; import type { ReactElement } from 'react';
import { Suspense, useEffect } from 'react'; import { Suspense, useCallback, useEffect } from 'react';
import { import {
publicPageBlockSuiteAtom, publicPageBlockSuiteAtom,
@ -17,7 +17,10 @@ import {
publicWorkspacePageIdAtom, publicWorkspacePageIdAtom,
} from '../../../atoms/public-workspace'; } from '../../../atoms/public-workspace';
import { BlockSuiteEditorHeader } from '../../../components/blocksuite/workspace-header'; import { BlockSuiteEditorHeader } from '../../../components/blocksuite/workspace-header';
import { PageDetailEditor } from '../../../components/page-detail-editor'; import {
PageDetailEditor,
type PageDetailEditorProps,
} from '../../../components/page-detail-editor';
import { WorkspaceAvatar } from '../../../components/pure/footer'; import { WorkspaceAvatar } from '../../../components/pure/footer';
import { PageLoading } from '../../../components/pure/loading'; import { PageLoading } from '../../../components/pure/loading';
import { useRouterHelper } from '../../../hooks/use-router-helper'; import { useRouterHelper } from '../../../hooks/use-router-helper';
@ -68,6 +71,19 @@ const PublicWorkspaceDetailPageInner = (): ReactElement => {
const [name] = useBlockSuiteWorkspaceName(blockSuiteWorkspace); const [name] = useBlockSuiteWorkspaceName(blockSuiteWorkspace);
const [avatar] = useBlockSuiteWorkspaceAvatarUrl(blockSuiteWorkspace); const [avatar] = useBlockSuiteWorkspaceAvatarUrl(blockSuiteWorkspace);
const pageTitle = blockSuiteWorkspace.meta.getPageMeta(pageId)?.title; const pageTitle = blockSuiteWorkspace.meta.getPageMeta(pageId)?.title;
const onLoad = useCallback<NonNullable<PageDetailEditorProps['onLoad']>>(
(_, editor) => {
const { page } = editor;
page.awarenessStore.setReadonly(page, true);
const dispose = editor.slots.pageLinkClicked.on(({ pageId }) => {
return openPage(blockSuiteWorkspace.id, pageId);
});
return () => {
dispose.dispose();
};
},
[blockSuiteWorkspace.id, openPage]
);
return ( return (
<> <>
<PublicQuickSearch workspace={publicWorkspace} /> <PublicQuickSearch workspace={publicWorkspace} />
@ -97,16 +113,7 @@ const PublicWorkspaceDetailPageInner = (): ReactElement => {
isPublic={true} isPublic={true}
pageId={pageId} pageId={pageId}
workspace={publicWorkspace} workspace={publicWorkspace}
onLoad={(_, editor) => { onLoad={onLoad}
const { page } = editor;
page.awarenessStore.setReadonly(page, true);
const dispose = editor.slots.pageLinkClicked.on(({ pageId }) => {
return openPage(blockSuiteWorkspace.id, pageId);
});
return () => {
dispose.dispose();
};
}}
onInit={initEmptyPage} onInit={initEmptyPage}
/> />
</> </>

View File

@ -33,7 +33,8 @@
"{workspaceRoot}/apps/web/.next", "{workspaceRoot}/apps/web/.next",
"{workspaceRoot}/packages/storybook/storybook-static", "{workspaceRoot}/packages/storybook/storybook-static",
"{workspaceRoot}/packages/native/affine.*.node", "{workspaceRoot}/packages/native/affine.*.node",
"{workspaceRoot}/affine.db" "{workspaceRoot}/affine.db",
"{workspaceRoot/apps/electron/dist"
], ],
"inputs": [ "inputs": [
{ {

View File

@ -181,7 +181,6 @@ export const useZoomControls = ({
} }
}, [imageRef]); }, [imageRef]);
useEffect(() => { useEffect(() => {
const handleScroll = (event: WheelEvent) => { const handleScroll = (event: WheelEvent) => {
const { deltaY } = event; const { deltaY } = event;

View File

@ -62,7 +62,6 @@ export const imagePreviewModalCloseButtonStyle = style({
zIndex: 1, zIndex: 1,
marginTop: '38px', marginTop: '38px',
marginRight: '38px', marginRight: '38px',
}); });
export const imagePreviewModalGoStyle = style({ export const imagePreviewModalGoStyle = style({

View File

@ -5,7 +5,14 @@ import { useMediaQuery, useTheme } from '@mui/material';
import type React from 'react'; import type React from 'react';
import { type CSSProperties } from 'react'; import { type CSSProperties } from 'react';
import { ScrollableContainer, Table, TableBody, TableCell, TableHead, TableHeadRow } from '../..'; import {
ScrollableContainer,
Table,
TableBody,
TableCell,
TableHead,
TableHeadRow,
} from '../..';
import { TableBodyRow } from '../../ui/table'; import { TableBodyRow } from '../../ui/table';
import { useHasScrollTop } from '../app-sidebar/sidebar-containers/use-has-scroll-top'; import { useHasScrollTop } from '../app-sidebar/sidebar-containers/use-has-scroll-top';
import { AllPagesBody } from './all-pages-body'; import { AllPagesBody } from './all-pages-body';
@ -141,12 +148,9 @@ export const PageList = ({
? DEFAULT_SORT_KEY ? DEFAULT_SORT_KEY
: undefined; : undefined;
return ( return sorter.data.length === 0 && fallback ? (
sorter.data.length === 0 && fallback ? <StyledTableContainer>{fallback}</StyledTableContainer>
<StyledTableContainer> ) : (
{fallback}
</StyledTableContainer>
:
<ScrollableContainer inTableView> <ScrollableContainer inTableView>
<StyledTableContainer ref={ref}> <StyledTableContainer ref={ref}>
<Table showBorder={hasScrollTop} style={{ maxHeight: '100%' }}> <Table showBorder={hasScrollTop} style={{ maxHeight: '100%' }}>
@ -158,7 +162,6 @@ export const PageList = ({
importFile={onImportFile} importFile={onImportFile}
/> />
<AllPagesBody <AllPagesBody
isPublicWorkspace={isPublicWorkspace} isPublicWorkspace={isPublicWorkspace}
groupKey={groupKey} groupKey={groupKey}
data={sorter.data} data={sorter.data}
@ -246,12 +249,9 @@ export const PageListTrashView: React.FC<{
} }
); );
return ( return list.length === 0 && fallback ? (
list.length === 0 && fallback ? <StyledTableContainer>{fallback}</StyledTableContainer>
<StyledTableContainer> ) : (
{fallback}
</StyledTableContainer>
:
<ScrollableContainer inTableView> <ScrollableContainer inTableView>
<StyledTableContainer ref={ref}> <StyledTableContainer ref={ref}>
<Table showBorder={hasScrollTop}> <Table showBorder={hasScrollTop}>

View File

@ -26,18 +26,16 @@ export const ScrollableContainer = ({
className={clsx([styles.scrollableViewport])} className={clsx([styles.scrollableViewport])}
ref={ref} ref={ref}
> >
<div className={styles.scrollableContainer}> <div className={styles.scrollableContainer}>{children}</div>
{children}
</div>
</ScrollArea.Viewport> </ScrollArea.Viewport>
<ScrollArea.Scrollbar <ScrollArea.Scrollbar
orientation="vertical" orientation="vertical"
className={clsx(styles.scrollbar,{[styles.TableScrollbar]:inTableView})} className={clsx(styles.scrollbar, {
[styles.TableScrollbar]: inTableView,
})}
> >
<ScrollArea.Thumb className={styles.scrollbarThumb} /> <ScrollArea.Thumb className={styles.scrollbarThumb} />
</ScrollArea.Scrollbar> </ScrollArea.Scrollbar>
</ScrollArea.Root> </ScrollArea.Root>
); );
} };

View File

@ -25,7 +25,7 @@ beforeEach(async () => {
.register(AffineSchemas) .register(AffineSchemas)
.register(__unstableSchemas); .register(__unstableSchemas);
const initPage = async (page: Page) => { const initPage = async (page: Page) => {
await page.waitForLoaded() await page.waitForLoaded();
expect(page).not.toBeNull(); expect(page).not.toBeNull();
assertExists(page); assertExists(page);
const pageBlockId = page.addBlock('affine:page', { const pageBlockId = page.addBlock('affine:page', {

View File

@ -51,5 +51,6 @@ export function useBlockSuiteWorkspacePage(
): Page | null { ): Page | null {
const pageAtom = getAtom(blockSuiteWorkspace, pageId); const pageAtom = getAtom(blockSuiteWorkspace, pageId);
assertExists(pageAtom); assertExists(pageAtom);
return useAtomValue(pageAtom); const page = useAtomValue(pageAtom);
return page;
} }

View File

@ -17,8 +17,15 @@ export abstract class HandlerManager<
} }
type DBHandlers = { type DBHandlers = {
getDocAsUpdates: (id: string) => Promise<Uint8Array>; getDocAsUpdates: (
applyDocUpdate: (id: string, update: Uint8Array) => Promise<void>; workspaceId: string,
subdocId?: string
) => Promise<Uint8Array>;
applyDocUpdate: (
id: string,
update: Uint8Array,
subdocId?: string
) => Promise<void>;
addBlob: ( addBlob: (
workspaceId: string, workspaceId: string,
key: string, key: string,

View File

@ -1,12 +1,22 @@
use sqlx::sqlite::SqliteConnectOptions; use sqlx::sqlite::SqliteConnectOptions;
use std::fs;
#[tokio::main] #[tokio::main]
async fn main() -> Result<(), std::io::Error> { async fn main() -> Result<(), std::io::Error> {
dotenv::dotenv().ok(); dotenv::dotenv().ok();
// always start with a fresh database to have
// latest db schema
let db_path = "../../affine.db";
// check if db exists and then remove file
if fs::metadata(db_path).is_ok() {
fs::remove_file(db_path)?;
}
napi_build::setup(); napi_build::setup();
let options = SqliteConnectOptions::new() let options = SqliteConnectOptions::new()
.filename("../../affine.db") .filename(db_path)
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off) .journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
.locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
.create_if_missing(true); .create_if_missing(true);

View File

@ -32,6 +32,11 @@ export interface UpdateRow {
id: number; id: number;
timestamp: Date; timestamp: Date;
data: Buffer; data: Buffer;
docId?: string;
}
export interface InsertRow {
docId?: string;
data: Uint8Array;
} }
export class Subscription { export class Subscription {
toString(): string; toString(): string;
@ -56,8 +61,14 @@ export class SqliteConnection {
getBlob(key: string): Promise<BlobRow | null>; getBlob(key: string): Promise<BlobRow | null>;
deleteBlob(key: string): Promise<void>; deleteBlob(key: string): Promise<void>;
getBlobKeys(): Promise<Array<string>>; getBlobKeys(): Promise<Array<string>>;
getUpdates(): Promise<Array<UpdateRow>>; getUpdates(docId?: string | undefined | null): Promise<Array<UpdateRow>>;
insertUpdates(updates: Array<Uint8Array>): Promise<void>; getUpdatesCount(docId?: string | undefined | null): Promise<number>;
getAllUpdates(): Promise<Array<UpdateRow>>;
insertUpdates(updates: Array<InsertRow>): Promise<void>;
replaceUpdates(
docId: string | undefined | null,
updates: Array<InsertRow>
): Promise<void>;
close(): Promise<void>; close(): Promise<void>;
get isClose(): boolean; get isClose(): boolean;
static validate(path: string): Promise<boolean>; static validate(path: string): Promise<boolean>;

View File

@ -4,7 +4,8 @@
pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" ( pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
data BLOB NOT NULL, data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
doc_id TEXT
); );
CREATE TABLE IF NOT EXISTS "blobs" ( CREATE TABLE IF NOT EXISTS "blobs" (
key TEXT PRIMARY KEY NOT NULL, key TEXT PRIMARY KEY NOT NULL,

View File

@ -19,6 +19,13 @@ pub struct UpdateRow {
pub id: i64, pub id: i64,
pub timestamp: NaiveDateTime, pub timestamp: NaiveDateTime,
pub data: Buffer, pub data: Buffer,
pub doc_id: Option<String>,
}
#[napi(object)]
pub struct InsertRow {
pub doc_id: Option<String>,
pub data: Uint8Array,
} }
#[napi] #[napi]
@ -53,6 +60,7 @@ impl SqliteConnection {
.execute(connection.as_mut()) .execute(connection.as_mut())
.await .await
.map_err(anyhow::Error::from)?; .map_err(anyhow::Error::from)?;
self.migrate_add_doc_id().await?;
connection.detach(); connection.detach();
Ok(()) Ok(())
} }
@ -74,7 +82,11 @@ impl SqliteConnection {
#[napi] #[napi]
pub async fn get_blob(&self, key: String) -> Option<BlobRow> { pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
sqlx::query_as!(BlobRow, "SELECT * FROM blobs WHERE key = ?", key) sqlx::query_as!(
BlobRow,
"SELECT key, data, timestamp FROM blobs WHERE key = ?",
key
)
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
.ok() .ok()
@ -100,8 +112,54 @@ impl SqliteConnection {
} }
#[napi] #[napi]
pub async fn get_updates(&self) -> napi::Result<Vec<UpdateRow>> { pub async fn get_updates(&self, doc_id: Option<String>) -> napi::Result<Vec<UpdateRow>> {
let updates = sqlx::query_as!(UpdateRow, "SELECT * FROM updates") let updates = match doc_id {
Some(doc_id) => sqlx::query_as!(
UpdateRow,
"SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id = ?",
doc_id
)
.fetch_all(&self.pool)
.await
.map_err(anyhow::Error::from)?,
None => sqlx::query_as!(
UpdateRow,
"SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id is NULL",
)
.fetch_all(&self.pool)
.await
.map_err(anyhow::Error::from)?,
};
Ok(updates)
}
#[napi]
pub async fn get_updates_count(&self, doc_id: Option<String>) -> napi::Result<i32> {
let count = match doc_id {
Some(doc_id) => {
sqlx::query!(
"SELECT COUNT(*) as count FROM updates WHERE doc_id = ?",
doc_id
)
.fetch_one(&self.pool)
.await
.map_err(anyhow::Error::from)?
.count
}
None => {
sqlx::query!("SELECT COUNT(*) as count FROM updates WHERE doc_id is NULL")
.fetch_one(&self.pool)
.await
.map_err(anyhow::Error::from)?
.count
}
};
Ok(count)
}
#[napi]
pub async fn get_all_updates(&self) -> napi::Result<Vec<UpdateRow>> {
let updates = sqlx::query_as!(UpdateRow, "SELECT id, timestamp, data, doc_id FROM updates")
.fetch_all(&self.pool) .fetch_all(&self.pool)
.await .await
.map_err(anyhow::Error::from)?; .map_err(anyhow::Error::from)?;
@ -109,11 +167,51 @@ impl SqliteConnection {
} }
#[napi] #[napi]
pub async fn insert_updates(&self, updates: Vec<Uint8Array>) -> napi::Result<()> { pub async fn insert_updates(&self, updates: Vec<InsertRow>) -> napi::Result<()> {
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?; let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
for update in updates.into_iter() { for InsertRow { data, doc_id } in updates {
let update = update.as_ref(); let update = data.as_ref();
sqlx::query_as!(UpdateRow, "INSERT INTO updates (data) VALUES ($1)", update) sqlx::query_as!(
UpdateRow,
"INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
update,
doc_id
)
.execute(&mut *transaction)
.await
.map_err(anyhow::Error::from)?;
}
transaction.commit().await.map_err(anyhow::Error::from)?;
Ok(())
}
#[napi]
pub async fn replace_updates(
&self,
doc_id: Option<String>,
updates: Vec<InsertRow>,
) -> napi::Result<()> {
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
match doc_id {
Some(doc_id) => sqlx::query!("DELETE FROM updates where doc_id = ?", doc_id)
.execute(&mut *transaction)
.await
.map_err(anyhow::Error::from)?,
None => sqlx::query!("DELETE FROM updates where doc_id is NULL",)
.execute(&mut *transaction)
.await
.map_err(anyhow::Error::from)?,
};
for InsertRow { data, doc_id } in updates {
let update = data.as_ref();
sqlx::query_as!(
UpdateRow,
"INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
update,
doc_id
)
.execute(&mut *transaction) .execute(&mut *transaction)
.await .await
.map_err(anyhow::Error::from)?; .map_err(anyhow::Error::from)?;
@ -158,4 +256,22 @@ impl SqliteConnection {
false false
} }
} }
// todo: have a better way to handle migration
async fn migrate_add_doc_id(&self) -> Result<(), anyhow::Error> {
// ignore errors
match sqlx::query("ALTER TABLE updates ADD COLUMN doc_id TEXT")
.execute(&self.pool)
.await
{
Ok(_) => Ok(()),
Err(err) => {
if err.to_string().contains("duplicate column name") {
Ok(()) // Ignore error if it's due to duplicate column
} else {
Err(anyhow::Error::from(err)) // Propagate other errors
}
}
}
}
} }

View File

@ -140,7 +140,7 @@ describe('ydoc sync', () => {
const pageId = uuidv4(); const pageId = uuidv4();
const page1 = workspace1.createPage({ id: pageId }); const page1 = workspace1.createPage({ id: pageId });
await page1.waitForLoaded() await page1.waitForLoaded();
const pageBlockId = page1.addBlock('affine:page', { const pageBlockId = page1.addBlock('affine:page', {
title: new page1.Text(''), title: new page1.Text(''),
}); });
@ -153,7 +153,7 @@ describe('ydoc sync', () => {
workspace1.doc.getMap(`space:${pageId}`).toJSON() workspace1.doc.getMap(`space:${pageId}`).toJSON()
); );
const page2 = workspace2.getPage(pageId) as Page; const page2 = workspace2.getPage(pageId) as Page;
await page2.waitForLoaded() await page2.waitForLoaded();
page1.updateBlock( page1.updateBlock(
page1.getBlockById(paragraphId) as ParagraphBlockModel, page1.getBlockById(paragraphId) as ParagraphBlockModel,
{ {

View File

@ -7,7 +7,10 @@ import type { Y as YType } from '@blocksuite/store';
import { uuidv4, Workspace } from '@blocksuite/store'; import { uuidv4, Workspace } from '@blocksuite/store';
import { beforeEach, describe, expect, test, vi } from 'vitest'; import { beforeEach, describe, expect, test, vi } from 'vitest';
import { createSQLiteDBDownloadProvider, createSQLiteProvider } from '../index'; import {
createSQLiteDBDownloadProvider,
createSQLiteProvider,
} from '../sqlite-providers';
const Y = Workspace.Y; const Y = Workspace.Y;
@ -148,15 +151,21 @@ describe('SQLite download provider', () => {
test('disconnect handlers', async () => { test('disconnect handlers', async () => {
const offHandler = vi.fn(); const offHandler = vi.fn();
let handleUpdate = () => {}; let handleUpdate = () => {};
workspace.doc.on = (_: string, fn: () => void) => { let handleSubdocs = () => {};
workspace.doc.on = (event: string, fn: () => void) => {
if (event === 'update') {
handleUpdate = fn; handleUpdate = fn;
} else if (event === 'subdocs') {
handleSubdocs = fn;
}
}; };
workspace.doc.off = offHandler; workspace.doc.off = offHandler;
await provider.connect(); provider.connect();
provider.disconnect(); provider.disconnect();
expect(triggerDBUpdate).toBe(null); expect(triggerDBUpdate).toBe(null);
expect(offHandler).toBeCalledWith('update', handleUpdate); expect(offHandler).toBeCalledWith('update', handleUpdate);
expect(offHandler).toBeCalledWith('subdocs', handleSubdocs);
}); });
}); });

View File

@ -3,8 +3,6 @@ import type {
AffineWebSocketProvider, AffineWebSocketProvider,
LocalIndexedDBBackgroundProvider, LocalIndexedDBBackgroundProvider,
LocalIndexedDBDownloadProvider, LocalIndexedDBDownloadProvider,
SQLiteDBDownloadProvider,
SQLiteProvider,
} from '@affine/env/workspace'; } from '@affine/env/workspace';
import type { Disposable, DocProviderCreator } from '@blocksuite/store'; import type { Disposable, DocProviderCreator } from '@blocksuite/store';
import { assertExists, Workspace } from '@blocksuite/store'; import { assertExists, Workspace } from '@blocksuite/store';
@ -21,6 +19,10 @@ import { getLoginStorage, storageChangeSlot } from '../affine/login';
import { CallbackSet } from '../utils'; import { CallbackSet } from '../utils';
import { createAffineDownloadProvider } from './affine-download'; import { createAffineDownloadProvider } from './affine-download';
import { localProviderLogger as logger } from './logger'; import { localProviderLogger as logger } from './logger';
import {
createSQLiteDBDownloadProvider,
createSQLiteProvider,
} from './sqlite-providers';
const Y = Workspace.Y; const Y = Workspace.Y;
@ -151,151 +153,6 @@ const createIndexedDBDownloadProvider: DocProviderCreator = (
}; };
}; };
const sqliteOrigin = Symbol('sqlite-provider-origin');
const createSQLiteProvider: DocProviderCreator = (id, doc): SQLiteProvider => {
const { apis, events } = window;
// make sure it is being used in Electron with APIs
assertExists(apis);
assertExists(events);
function handleUpdate(update: Uint8Array, origin: unknown) {
if (origin === sqliteOrigin) {
return;
}
apis.db.applyDocUpdate(id, update).catch(err => {
console.error(err);
});
}
let unsubscribe = () => {};
let connected = false;
const connect = () => {
logger.info('connecting sqlite provider', id);
doc.on('update', handleUpdate);
unsubscribe = events.db.onExternalUpdate(
({
update,
workspaceId,
}: {
workspaceId: string;
update: Uint8Array;
}) => {
if (workspaceId === id) {
Y.applyUpdate(doc, update, sqliteOrigin);
}
}
);
connected = true;
logger.info('connecting sqlite done', id);
};
const cleanup = () => {
logger.info('disconnecting sqlite provider', id);
unsubscribe();
doc.off('update', handleUpdate);
connected = false;
};
return {
flavour: 'sqlite',
passive: true,
get connected(): boolean {
return connected;
},
cleanup,
connect,
disconnect: cleanup,
};
};
const createSQLiteDBDownloadProvider: DocProviderCreator = (
id,
doc
): SQLiteDBDownloadProvider => {
const { apis } = window;
let disconnected = false;
let _resolve: () => void;
let _reject: (error: unknown) => void;
const promise = new Promise<void>((resolve, reject) => {
_resolve = resolve;
_reject = reject;
});
async function syncUpdates() {
logger.info('syncing updates from sqlite', id);
const updates = await apis.db.getDocAsUpdates(id);
if (disconnected) {
return;
}
if (updates) {
Y.applyUpdate(doc, updates, sqliteOrigin);
}
const diff = Y.encodeStateAsUpdate(doc, updates);
// also apply updates to sqlite
await apis.db.applyDocUpdate(id, diff);
}
// fixme(pengx17): should n't sync blob in doc provider
// async function _syncBlobIntoSQLite(bs: BlobManager) {
// const persistedKeys = await apis.db.getBlobKeys(id);
//
// if (disconnected) {
// return;
// }
//
// const allKeys = await bs.list().catch(() => []);
// const keysToPersist = allKeys.filter(k => !persistedKeys.includes(k));
//
// logger.info('persisting blobs', keysToPersist, 'to sqlite');
// return Promise.all(
// keysToPersist.map(async k => {
// const blob = await bs.get(k);
// if (!blob) {
// logger.warn('blob not found for', k);
// return;
// }
//
// if (disconnected) {
// return;
// }
//
// return apis?.db.addBlob(
// id,
// k,
// new Uint8Array(await blob.arrayBuffer())
// );
// })
// );
// }
return {
flavour: 'sqlite-download',
active: true,
get whenReady() {
return promise;
},
cleanup: () => {
disconnected = true;
},
sync: async () => {
logger.info('connect indexeddb provider', id);
try {
await syncUpdates();
_resolve();
} catch (error) {
_reject(error);
}
},
};
};
export { export {
createAffineDownloadProvider, createAffineDownloadProvider,
createAffineWebSocketProvider, createAffineWebSocketProvider,

View File

@ -0,0 +1,212 @@
import type {
SQLiteDBDownloadProvider,
SQLiteProvider,
} from '@affine/env/workspace';
import type { DocProviderCreator } from '@blocksuite/store';
import {
assertExists,
Workspace as BlockSuiteWorkspace,
} from '@blocksuite/store';
import type { Doc } from 'yjs';
import { localProviderLogger as logger } from './logger';
const Y = BlockSuiteWorkspace.Y;
const sqliteOrigin = Symbol('sqlite-provider-origin');
type SubDocsEvent = {
added: Set<Doc>;
removed: Set<Doc>;
loaded: Set<Doc>;
};
/**
* A provider that is responsible for syncing updates the workspace with the local SQLite database.
*/
export const createSQLiteProvider: DocProviderCreator = (
id,
rootDoc
): SQLiteProvider => {
const { apis, events } = window;
// make sure it is being used in Electron with APIs
assertExists(apis);
assertExists(events);
const updateHandlerMap = new WeakMap<
Doc,
(update: Uint8Array, origin: unknown) => void
>();
const subDocsHandlerMap = new WeakMap<Doc, (event: SubDocsEvent) => void>();
const createOrHandleUpdate = (doc: Doc) => {
if (updateHandlerMap.has(doc)) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return updateHandlerMap.get(doc)!;
}
function handleUpdate(update: Uint8Array, origin: unknown) {
if (origin === sqliteOrigin) {
return;
}
const subdocId = doc.guid === id ? undefined : doc.guid;
apis.db.applyDocUpdate(id, update, subdocId).catch(err => {
logger.error(err);
});
}
updateHandlerMap.set(doc, handleUpdate);
return handleUpdate;
};
const createOrGetHandleSubDocs = (doc: Doc) => {
if (subDocsHandlerMap.has(doc)) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return subDocsHandlerMap.get(doc)!;
}
function handleSubdocs(event: SubDocsEvent) {
event.removed.forEach(doc => {
untrackDoc(doc);
});
event.loaded.forEach(doc => {
trackDoc(doc);
});
}
subDocsHandlerMap.set(doc, handleSubdocs);
return handleSubdocs;
};
function trackDoc(doc: Doc) {
doc.on('update', createOrHandleUpdate(doc));
doc.on('subdocs', createOrGetHandleSubDocs(doc));
doc.subdocs.forEach(doc => {
trackDoc(doc);
});
}
function untrackDoc(doc: Doc) {
doc.subdocs.forEach(doc => {
untrackDoc(doc);
});
doc.off('update', createOrHandleUpdate(doc));
doc.off('subdocs', createOrGetHandleSubDocs(doc));
}
let unsubscribe = () => {};
let connected = false;
const connect = () => {
logger.info('connecting sqlite provider', id);
trackDoc(rootDoc);
unsubscribe = events.db.onExternalUpdate(
({
update,
workspaceId,
docId,
}: {
workspaceId: string;
update: Uint8Array;
docId?: string;
}) => {
if (workspaceId === id) {
if (docId) {
for (const doc of rootDoc.subdocs) {
if (doc.guid === docId) {
Y.applyUpdate(doc, update, sqliteOrigin);
return;
}
}
} else {
Y.applyUpdate(rootDoc, update, sqliteOrigin);
}
}
}
);
connected = true;
logger.info('connecting sqlite done', id);
};
const cleanup = () => {
logger.info('disconnecting sqlite provider', id);
unsubscribe();
untrackDoc(rootDoc);
connected = false;
};
return {
flavour: 'sqlite',
passive: true,
get connected(): boolean {
return connected;
},
cleanup,
connect,
disconnect: cleanup,
};
};
/**
* A provider that is responsible for DOWNLOADING updates from the local SQLite database.
*/
export const createSQLiteDBDownloadProvider: DocProviderCreator = (
id,
rootDoc
): SQLiteDBDownloadProvider => {
const { apis } = window;
let disconnected = false;
let _resolve: () => void;
let _reject: (error: unknown) => void;
const promise = new Promise<void>((resolve, reject) => {
_resolve = resolve;
_reject = reject;
});
async function syncUpdates(doc: Doc) {
logger.info('syncing updates from sqlite', id);
const subdocId = doc.guid === id ? undefined : doc.guid;
const updates = await apis.db.getDocAsUpdates(id, subdocId);
if (disconnected) {
return false;
}
if (updates) {
Y.applyUpdate(doc, updates, sqliteOrigin);
}
const diff = Y.encodeStateAsUpdate(doc, updates);
// also apply updates to sqlite
await apis.db.applyDocUpdate(id, diff, subdocId);
return true;
}
async function syncAllUpdates(doc: Doc) {
if (await syncUpdates(doc)) {
const subdocs = Array.from(doc.subdocs).filter(d => d.shouldLoad);
await Promise.all(subdocs.map(syncAllUpdates));
}
}
return {
flavour: 'sqlite-download',
active: true,
get whenReady() {
return promise;
},
cleanup: () => {
disconnected = true;
},
sync: async () => {
logger.info('connect indexeddb provider', id);
try {
await syncAllUpdates(rootDoc);
_resolve();
} catch (error) {
_reject(error);
}
},
};
};

View File

@ -107,6 +107,7 @@ describe('indexeddb provider', () => {
}) })
.register(AffineSchemas) .register(AffineSchemas)
.register(__unstableSchemas); .register(__unstableSchemas);
// data should only contain updates for the root doc
data.updates.forEach(({ update }) => { data.updates.forEach(({ update }) => {
Workspace.Y.applyUpdate(testWorkspace.doc, update); Workspace.Y.applyUpdate(testWorkspace.doc, update);
}); });
@ -125,19 +126,6 @@ describe('indexeddb provider', () => {
} }
expect(workspace.doc.toJSON()).toEqual(testWorkspace.doc.toJSON()); expect(workspace.doc.toJSON()).toEqual(testWorkspace.doc.toJSON());
} }
const secondWorkspace = new Workspace({
id,
})
.register(AffineSchemas)
.register(__unstableSchemas);
const provider2 = createIndexedDBProvider(secondWorkspace.doc, rootDBName);
provider2.connect();
await provider2.whenSynced;
const page = secondWorkspace.getPage('page0');
assertExists(page);
await page.waitForLoaded();
expect(workspace.doc.toJSON()).toEqual(secondWorkspace.doc.toJSON());
}); });
test('disconnect suddenly', async () => { test('disconnect suddenly', async () => {
@ -423,6 +411,7 @@ describe('subDoc', () => {
provider.disconnect(); provider.disconnect();
json2 = doc.toJSON(); json2 = doc.toJSON();
} }
// the following line compares {} with {}
expect(json1['']['1'].toJSON()).toEqual(json2['']['1'].toJSON()); expect(json1['']['1'].toJSON()).toEqual(json2['']['1'].toJSON());
expect(json1['']['2']).toEqual(json2['']['2']); expect(json1['']['2']).toEqual(json2['']['2']);
}); });

View File

@ -195,10 +195,12 @@ const selectDateFromDatePicker = async (page: Page, date: Date) => {
); );
await nextMonthButton.click(); await nextMonthButton.click();
} }
const map = ['th', 'st', 'nd', 'rd'] const map = ['th', 'st', 'nd', 'rd'];
// Click on the day cell // Click on the day cell
const dateCell = page.locator( const dateCell = page.locator(
`[aria-disabled="false"][aria-label="Choose ${weekday}, ${month} ${day}${map[Number.parseInt(day) % 10] ?? 'th'}, ${year}"]` `[aria-disabled="false"][aria-label="Choose ${weekday}, ${month} ${day}${
map[Number.parseInt(day) % 10] ?? 'th'
}, ${year}"]`
); );
await dateCell.click(); await dateCell.click();
}; };