mirror of
https://github.com/toeverything/AFFiNE.git
synced 2024-12-23 02:21:49 +03:00
feat: sqlite subdocument (#2816)
Co-authored-by: Alex Yang <himself65@outlook.com>
This commit is contained in:
parent
4307e1eb6b
commit
05452bb297
@ -12,6 +12,7 @@ import type { PlaywrightTestConfig } from '@playwright/test';
|
|||||||
*/
|
*/
|
||||||
const config: PlaywrightTestConfig = {
|
const config: PlaywrightTestConfig = {
|
||||||
testDir: './tests',
|
testDir: './tests',
|
||||||
|
testIgnore: '**/lib/**',
|
||||||
fullyParallel: true,
|
fullyParallel: true,
|
||||||
timeout: process.env.CI ? 50_000 : 30_000,
|
timeout: process.env.CI ? 50_000 : 30_000,
|
||||||
use: {
|
use: {
|
||||||
|
@ -20,14 +20,31 @@ afterEach(async () => {
|
|||||||
await fs.remove(tmpDir);
|
await fs.remove(tmpDir);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let testYDoc: Y.Doc;
|
||||||
|
let testYSubDoc: Y.Doc;
|
||||||
|
|
||||||
function getTestUpdates() {
|
function getTestUpdates() {
|
||||||
const testYDoc = new Y.Doc();
|
testYDoc = new Y.Doc();
|
||||||
const yText = testYDoc.getText('test');
|
const yText = testYDoc.getText('test');
|
||||||
yText.insert(0, 'hello');
|
yText.insert(0, 'hello');
|
||||||
|
|
||||||
|
testYSubDoc = new Y.Doc();
|
||||||
|
testYDoc.getMap('subdocs').set('test-subdoc', testYSubDoc);
|
||||||
|
|
||||||
const updates = Y.encodeStateAsUpdate(testYDoc);
|
const updates = Y.encodeStateAsUpdate(testYDoc);
|
||||||
|
|
||||||
return updates;
|
return updates;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getTestSubDocUpdates() {
|
||||||
|
const yText = testYSubDoc.getText('test');
|
||||||
|
yText.insert(0, 'hello');
|
||||||
|
|
||||||
|
const updates = Y.encodeStateAsUpdate(testYSubDoc);
|
||||||
|
|
||||||
|
return updates;
|
||||||
|
}
|
||||||
|
|
||||||
test('can create new db file if not exists', async () => {
|
test('can create new db file if not exists', async () => {
|
||||||
const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
|
const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
|
||||||
const workspaceId = v4();
|
const workspaceId = v4();
|
||||||
@ -68,6 +85,31 @@ test('on applyUpdate (from renderer), will trigger update', async () => {
|
|||||||
await db.destroy();
|
await db.destroy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('on applyUpdate (from renderer, subdoc), will trigger update', async () => {
|
||||||
|
const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
|
||||||
|
const workspaceId = v4();
|
||||||
|
const onUpdate = vi.fn();
|
||||||
|
const insertUpdates = vi.fn();
|
||||||
|
|
||||||
|
const db = await openWorkspaceDatabase(workspaceId);
|
||||||
|
db.applyUpdate(getTestUpdates(), 'renderer');
|
||||||
|
|
||||||
|
db.db!.insertUpdates = insertUpdates;
|
||||||
|
db.update$.subscribe(onUpdate);
|
||||||
|
|
||||||
|
const subdocUpdates = getTestSubDocUpdates();
|
||||||
|
db.applyUpdate(subdocUpdates, 'renderer', testYSubDoc.guid);
|
||||||
|
|
||||||
|
expect(onUpdate).toHaveBeenCalled();
|
||||||
|
expect(insertUpdates).toHaveBeenCalledWith([
|
||||||
|
{
|
||||||
|
docId: testYSubDoc.guid,
|
||||||
|
data: subdocUpdates,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
await db.destroy();
|
||||||
|
});
|
||||||
|
|
||||||
test('on applyUpdate (from external), will trigger update & send external update event', async () => {
|
test('on applyUpdate (from external), will trigger update & send external update event', async () => {
|
||||||
const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
|
const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
|
||||||
const workspaceId = v4();
|
const workspaceId = v4();
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
import { SqliteConnection } from '@affine/native';
|
import { type InsertRow, SqliteConnection } from '@affine/native';
|
||||||
|
|
||||||
import { logger } from '../logger';
|
import { logger } from '../logger';
|
||||||
|
|
||||||
@ -79,21 +79,34 @@ export abstract class BaseSQLiteAdapter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async getUpdates() {
|
async getUpdates(docId?: string) {
|
||||||
try {
|
try {
|
||||||
if (!this.db) {
|
if (!this.db) {
|
||||||
logger.warn(`${this.path} is not connected`);
|
logger.warn(`${this.path} is not connected`);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
return await this.db.getUpdates();
|
return await this.db.getUpdates(docId);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('getUpdates', error);
|
logger.error('getUpdates', error);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getAllUpdates() {
|
||||||
|
try {
|
||||||
|
if (!this.db) {
|
||||||
|
logger.warn(`${this.path} is not connected`);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return await this.db.getAllUpdates();
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('getAllUpdates', error);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// add a single update to SQLite
|
// add a single update to SQLite
|
||||||
async addUpdateToSQLite(updates: Uint8Array[]) {
|
async addUpdateToSQLite(updates: InsertRow[]) {
|
||||||
// batch write instead write per key stroke?
|
// batch write instead write per key stroke?
|
||||||
try {
|
try {
|
||||||
if (!this.db) {
|
if (!this.db) {
|
||||||
|
@ -7,13 +7,17 @@ export * from './ensure-db';
|
|||||||
export * from './subjects';
|
export * from './subjects';
|
||||||
|
|
||||||
export const dbHandlers = {
|
export const dbHandlers = {
|
||||||
getDocAsUpdates: async (id: string) => {
|
getDocAsUpdates: async (workspaceId: string, subdocId?: string) => {
|
||||||
const workspaceDB = await ensureSQLiteDB(id);
|
const workspaceDB = await ensureSQLiteDB(workspaceId);
|
||||||
return workspaceDB.getDocAsUpdates();
|
return workspaceDB.getDocAsUpdates(subdocId);
|
||||||
},
|
},
|
||||||
applyDocUpdate: async (id: string, update: Uint8Array) => {
|
applyDocUpdate: async (
|
||||||
const workspaceDB = await ensureSQLiteDB(id);
|
workspaceId: string,
|
||||||
return workspaceDB.applyUpdate(update);
|
update: Uint8Array,
|
||||||
|
subdocId?: string
|
||||||
|
) => {
|
||||||
|
const workspaceDB = await ensureSQLiteDB(workspaceId);
|
||||||
|
return workspaceDB.applyUpdate(update, 'renderer', subdocId);
|
||||||
},
|
},
|
||||||
addBlob: async (workspaceId: string, key: string, data: Uint8Array) => {
|
addBlob: async (workspaceId: string, key: string, data: Uint8Array) => {
|
||||||
const workspaceDB = await ensureSQLiteDB(workspaceId);
|
const workspaceDB = await ensureSQLiteDB(workspaceId);
|
||||||
@ -38,7 +42,11 @@ export const dbHandlers = {
|
|||||||
|
|
||||||
export const dbEvents = {
|
export const dbEvents = {
|
||||||
onExternalUpdate: (
|
onExternalUpdate: (
|
||||||
fn: (update: { workspaceId: string; update: Uint8Array }) => void
|
fn: (update: {
|
||||||
|
workspaceId: string;
|
||||||
|
update: Uint8Array;
|
||||||
|
docId?: string;
|
||||||
|
}) => void
|
||||||
) => {
|
) => {
|
||||||
const sub = dbSubjects.externalUpdate.subscribe(fn);
|
const sub = dbSubjects.externalUpdate.subscribe(fn);
|
||||||
return () => {
|
return () => {
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import assert from 'node:assert';
|
import assert from 'node:assert';
|
||||||
|
|
||||||
import type { SqliteConnection } from '@affine/native';
|
import type { InsertRow } from '@affine/native';
|
||||||
import { debounce } from 'lodash-es';
|
import { debounce } from 'lodash-es';
|
||||||
import * as Y from 'yjs';
|
import * as Y from 'yjs';
|
||||||
|
|
||||||
@ -8,19 +8,19 @@ import { logger } from '../logger';
|
|||||||
import type { YOrigin } from '../type';
|
import type { YOrigin } from '../type';
|
||||||
import { getWorkspaceMeta } from '../workspace';
|
import { getWorkspaceMeta } from '../workspace';
|
||||||
import { BaseSQLiteAdapter } from './base-db-adapter';
|
import { BaseSQLiteAdapter } from './base-db-adapter';
|
||||||
import { mergeUpdate } from './merge-update';
|
|
||||||
import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
|
import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
|
||||||
|
|
||||||
const FLUSH_WAIT_TIME = 5000;
|
const FLUSH_WAIT_TIME = 5000;
|
||||||
const FLUSH_MAX_WAIT_TIME = 10000;
|
const FLUSH_MAX_WAIT_TIME = 10000;
|
||||||
|
|
||||||
|
// todo: trim db when it is too big
|
||||||
export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
||||||
role = 'secondary';
|
role = 'secondary';
|
||||||
yDoc = new Y.Doc();
|
yDoc = new Y.Doc();
|
||||||
firstConnected = false;
|
firstConnected = false;
|
||||||
destroyed = false;
|
destroyed = false;
|
||||||
|
|
||||||
updateQueue: Uint8Array[] = [];
|
updateQueue: { data: Uint8Array; docId?: string }[] = [];
|
||||||
|
|
||||||
unsubscribers = new Set<() => void>();
|
unsubscribers = new Set<() => void>();
|
||||||
|
|
||||||
@ -29,10 +29,23 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
public upstream: WorkspaceSQLiteDB
|
public upstream: WorkspaceSQLiteDB
|
||||||
) {
|
) {
|
||||||
super(path);
|
super(path);
|
||||||
this.setupAndListen();
|
this.init();
|
||||||
logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId);
|
logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getDoc(docId?: string) {
|
||||||
|
if (!docId) {
|
||||||
|
return this.yDoc;
|
||||||
|
}
|
||||||
|
// this should be pretty fast and we don't need to cache it
|
||||||
|
for (const subdoc of this.yDoc.subdocs) {
|
||||||
|
if (subdoc.guid === docId) {
|
||||||
|
return subdoc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
override async destroy() {
|
override async destroy() {
|
||||||
await this.flushUpdateQueue();
|
await this.flushUpdateQueue();
|
||||||
this.unsubscribers.forEach(unsub => unsub());
|
this.unsubscribers.forEach(unsub => unsub());
|
||||||
@ -47,7 +60,7 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
|
|
||||||
// do not update db immediately, instead, push to a queue
|
// do not update db immediately, instead, push to a queue
|
||||||
// and flush the queue in a future time
|
// and flush the queue in a future time
|
||||||
async addUpdateToUpdateQueue(db: SqliteConnection, update: Uint8Array) {
|
async addUpdateToUpdateQueue(update: InsertRow) {
|
||||||
this.updateQueue.push(update);
|
this.updateQueue.push(update);
|
||||||
await this.debouncedFlush();
|
await this.debouncedFlush();
|
||||||
}
|
}
|
||||||
@ -101,55 +114,82 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
setupAndListen() {
|
setupListener(docId?: string) {
|
||||||
if (this.firstConnected) {
|
const doc = this.getDoc(docId);
|
||||||
|
if (!doc) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.firstConnected = true;
|
|
||||||
|
|
||||||
const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
|
const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
|
||||||
if (origin === 'renderer') {
|
if (origin === 'renderer') {
|
||||||
// update to upstream yDoc should be replicated to self yDoc
|
// update to upstream yDoc should be replicated to self yDoc
|
||||||
this.applyUpdate(update, 'upstream');
|
this.applyUpdate(update, 'upstream', docId);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const onSelfUpdate = async (update: Uint8Array, origin: YOrigin) => {
|
const onSelfUpdate = async (update: Uint8Array, origin: YOrigin) => {
|
||||||
// for self update from upstream, we need to push it to external DB
|
// for self update from upstream, we need to push it to external DB
|
||||||
if (origin === 'upstream' && this.db) {
|
if (origin === 'upstream') {
|
||||||
await this.addUpdateToUpdateQueue(this.db, update);
|
await this.addUpdateToUpdateQueue({
|
||||||
|
data: update,
|
||||||
|
docId,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (origin === 'self') {
|
if (origin === 'self') {
|
||||||
this.upstream.applyUpdate(update, 'external');
|
this.upstream.applyUpdate(update, 'external', docId);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const onSubdocs = ({ added }: { added: Set<Y.Doc> }) => {
|
||||||
|
added.forEach(subdoc => {
|
||||||
|
this.setupListener(subdoc.guid);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
// listen to upstream update
|
// listen to upstream update
|
||||||
this.upstream.yDoc.on('update', onUpstreamUpdate);
|
this.upstream.yDoc.on('update', onUpstreamUpdate);
|
||||||
this.yDoc.on('update', onSelfUpdate);
|
this.yDoc.on('update', onSelfUpdate);
|
||||||
|
this.yDoc.on('subdocs', onSubdocs);
|
||||||
|
|
||||||
this.unsubscribers.add(() => {
|
this.unsubscribers.add(() => {
|
||||||
this.upstream.yDoc.off('update', onUpstreamUpdate);
|
this.upstream.yDoc.off('update', onUpstreamUpdate);
|
||||||
this.yDoc.off('update', onSelfUpdate);
|
this.yDoc.off('update', onSelfUpdate);
|
||||||
});
|
this.yDoc.off('subdocs', onSubdocs);
|
||||||
|
|
||||||
this.run(() => {
|
|
||||||
// apply all updates from upstream
|
|
||||||
const upstreamUpdate = this.upstream.getDocAsUpdates();
|
|
||||||
// to initialize the yDoc, we need to apply all updates from the db
|
|
||||||
this.applyUpdate(upstreamUpdate, 'upstream');
|
|
||||||
})
|
|
||||||
.then(() => {
|
|
||||||
logger.debug('run success');
|
|
||||||
})
|
|
||||||
.catch(err => {
|
|
||||||
logger.error('run error', err);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
applyUpdate = (data: Uint8Array, origin: YOrigin = 'upstream') => {
|
init() {
|
||||||
|
if (this.firstConnected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.firstConnected = true;
|
||||||
|
this.setupListener();
|
||||||
|
// apply all updates from upstream
|
||||||
|
// we assume here that the upstream ydoc is already sync'ed
|
||||||
|
const syncUpstreamDoc = (docId?: string) => {
|
||||||
|
const update = this.upstream.getDocAsUpdates(docId);
|
||||||
|
if (update) {
|
||||||
|
this.applyUpdate(update, 'upstream');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
syncUpstreamDoc();
|
||||||
|
this.upstream.yDoc.subdocs.forEach(subdoc => {
|
||||||
|
syncUpstreamDoc(subdoc.guid);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
applyUpdate = (
|
||||||
|
data: Uint8Array,
|
||||||
|
origin: YOrigin = 'upstream',
|
||||||
|
docId?: string
|
||||||
|
) => {
|
||||||
|
const doc = this.getDoc(docId);
|
||||||
|
if (doc) {
|
||||||
Y.applyUpdate(this.yDoc, data, origin);
|
Y.applyUpdate(this.yDoc, data, origin);
|
||||||
|
} else {
|
||||||
|
logger.warn('applyUpdate: doc not found', docId);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// TODO: have a better solution to handle blobs
|
// TODO: have a better solution to handle blobs
|
||||||
@ -186,23 +226,33 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
async pull() {
|
async pull() {
|
||||||
const start = performance.now();
|
const start = performance.now();
|
||||||
assert(this.upstream.db, 'upstream db should be connected');
|
assert(this.upstream.db, 'upstream db should be connected');
|
||||||
const updates = await this.run(async () => {
|
const rows = await this.run(async () => {
|
||||||
// TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
|
// TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
|
||||||
await this.syncBlobs();
|
await this.syncBlobs();
|
||||||
return (await this.getUpdates()).map(update => update.data);
|
return await this.getAllUpdates();
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!updates || this.destroyed) {
|
if (!rows || this.destroyed) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const merged = mergeUpdate(updates);
|
// apply root doc first
|
||||||
this.applyUpdate(merged, 'self');
|
rows.forEach(row => {
|
||||||
|
if (!row.docId) {
|
||||||
|
this.applyUpdate(row.data, 'self');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
rows.forEach(row => {
|
||||||
|
if (row.docId) {
|
||||||
|
this.applyUpdate(row.data, 'self', row.docId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
'pull external updates',
|
'pull external updates',
|
||||||
this.path,
|
this.path,
|
||||||
updates.length,
|
rows.length,
|
||||||
(performance.now() - start).toFixed(2),
|
(performance.now() - start).toFixed(2),
|
||||||
'ms'
|
'ms'
|
||||||
);
|
);
|
||||||
|
@ -1,5 +1,9 @@
|
|||||||
import { Subject } from 'rxjs';
|
import { Subject } from 'rxjs';
|
||||||
|
|
||||||
export const dbSubjects = {
|
export const dbSubjects = {
|
||||||
externalUpdate: new Subject<{ workspaceId: string; update: Uint8Array }>(),
|
externalUpdate: new Subject<{
|
||||||
|
workspaceId: string;
|
||||||
|
update: Uint8Array;
|
||||||
|
docId?: string;
|
||||||
|
}>(),
|
||||||
};
|
};
|
||||||
|
@ -1,3 +1,5 @@
|
|||||||
|
import type { InsertRow } from '@affine/native';
|
||||||
|
import { debounce } from 'lodash-es';
|
||||||
import { Subject } from 'rxjs';
|
import { Subject } from 'rxjs';
|
||||||
import * as Y from 'yjs';
|
import * as Y from 'yjs';
|
||||||
|
|
||||||
@ -5,9 +7,10 @@ import { logger } from '../logger';
|
|||||||
import type { YOrigin } from '../type';
|
import type { YOrigin } from '../type';
|
||||||
import { getWorkspaceMeta } from '../workspace';
|
import { getWorkspaceMeta } from '../workspace';
|
||||||
import { BaseSQLiteAdapter } from './base-db-adapter';
|
import { BaseSQLiteAdapter } from './base-db-adapter';
|
||||||
import { mergeUpdate } from './merge-update';
|
|
||||||
import { dbSubjects } from './subjects';
|
import { dbSubjects } from './subjects';
|
||||||
|
|
||||||
|
const TRIM_SIZE = 500;
|
||||||
|
|
||||||
export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
||||||
role = 'primary';
|
role = 'primary';
|
||||||
yDoc = new Y.Doc();
|
yDoc = new Y.Doc();
|
||||||
@ -28,33 +31,76 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
this.firstConnected = false;
|
this.firstConnected = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getDoc(docId?: string) {
|
||||||
|
if (!docId) {
|
||||||
|
return this.yDoc;
|
||||||
|
}
|
||||||
|
// this should be pretty fast and we don't need to cache it
|
||||||
|
for (const subdoc of this.yDoc.subdocs) {
|
||||||
|
if (subdoc.guid === docId) {
|
||||||
|
return subdoc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
getWorkspaceName = () => {
|
getWorkspaceName = () => {
|
||||||
return this.yDoc.getMap('space:meta').get('name') as string;
|
return this.yDoc.getMap('meta').get('name') as string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
setupListener(docId?: string) {
|
||||||
|
const doc = this.getDoc(docId);
|
||||||
|
if (doc) {
|
||||||
|
const onUpdate = async (update: Uint8Array, origin: YOrigin) => {
|
||||||
|
const insertRows = [{ data: update, docId }];
|
||||||
|
if (origin === 'renderer') {
|
||||||
|
await this.addUpdateToSQLite(insertRows);
|
||||||
|
} else if (origin === 'external') {
|
||||||
|
dbSubjects.externalUpdate.next({
|
||||||
|
workspaceId: this.workspaceId,
|
||||||
|
update,
|
||||||
|
docId,
|
||||||
|
});
|
||||||
|
await this.addUpdateToSQLite(insertRows);
|
||||||
|
logger.debug('external update', this.workspaceId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const onSubdocs = ({ added }: { added: Set<Y.Doc> }) => {
|
||||||
|
added.forEach(subdoc => {
|
||||||
|
this.setupListener(subdoc.guid);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
doc.on('update', onUpdate);
|
||||||
|
doc.on('subdocs', onSubdocs);
|
||||||
|
} else {
|
||||||
|
logger.error('setupListener: doc not found', docId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async init() {
|
async init() {
|
||||||
const db = await super.connectIfNeeded();
|
const db = await super.connectIfNeeded();
|
||||||
|
|
||||||
if (!this.firstConnected) {
|
if (!this.firstConnected) {
|
||||||
this.yDoc.on('update', async (update: Uint8Array, origin: YOrigin) => {
|
this.setupListener();
|
||||||
if (origin === 'renderer') {
|
|
||||||
await this.addUpdateToSQLite([update]);
|
|
||||||
} else if (origin === 'external') {
|
|
||||||
dbSubjects.externalUpdate.next({
|
|
||||||
workspaceId: this.workspaceId,
|
|
||||||
update,
|
|
||||||
});
|
|
||||||
await this.addUpdateToSQLite([update]);
|
|
||||||
logger.debug('external update', this.workspaceId);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const updates = await this.getUpdates();
|
const updates = await this.getAllUpdates();
|
||||||
const merged = mergeUpdate(updates.map(update => update.data));
|
|
||||||
|
|
||||||
// to initialize the yDoc, we need to apply all updates from the db
|
// apply root first (without ID).
|
||||||
this.applyUpdate(merged, 'self');
|
// subdoc will be available after root is applied
|
||||||
|
updates.forEach(update => {
|
||||||
|
if (!update.docId) {
|
||||||
|
this.applyUpdate(update.data, 'self');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// then, for all subdocs, apply the updates
|
||||||
|
updates.forEach(update => {
|
||||||
|
if (update.docId) {
|
||||||
|
this.applyUpdate(update.data, 'self', update.docId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
this.firstConnected = true;
|
this.firstConnected = true;
|
||||||
this.update$.next();
|
this.update$.next();
|
||||||
@ -62,18 +108,32 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
return db;
|
return db;
|
||||||
}
|
}
|
||||||
|
|
||||||
getDocAsUpdates = () => {
|
// unlike getUpdates, this will return updates in yDoc
|
||||||
return Y.encodeStateAsUpdate(this.yDoc);
|
getDocAsUpdates = (docId?: string) => {
|
||||||
|
const doc = docId ? this.getDoc(docId) : this.yDoc;
|
||||||
|
if (doc) {
|
||||||
|
return Y.encodeStateAsUpdate(doc);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
// non-blocking and use yDoc to validate the update
|
// non-blocking and use yDoc to validate the update
|
||||||
// after that, the update is added to the db
|
// after that, the update is added to the db
|
||||||
applyUpdate = (data: Uint8Array, origin: YOrigin = 'renderer') => {
|
applyUpdate = (
|
||||||
|
data: Uint8Array,
|
||||||
|
origin: YOrigin = 'renderer',
|
||||||
|
docId?: string
|
||||||
|
) => {
|
||||||
// todo: trim the updates when the number of records is too large
|
// todo: trim the updates when the number of records is too large
|
||||||
// 1. store the current ydoc state in the db
|
// 1. store the current ydoc state in the db
|
||||||
// 2. then delete the old updates
|
// 2. then delete the old updates
|
||||||
// yjs-idb will always trim the db for the first time after DB is loaded
|
// yjs-idb will always trim the db for the first time after DB is loaded
|
||||||
Y.applyUpdate(this.yDoc, data, origin);
|
const doc = this.getDoc(docId);
|
||||||
|
if (doc) {
|
||||||
|
Y.applyUpdate(doc, data, origin);
|
||||||
|
} else {
|
||||||
|
logger.warn('applyUpdate: doc not found', docId);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
override async addBlob(key: string, value: Uint8Array) {
|
override async addBlob(key: string, value: Uint8Array) {
|
||||||
@ -87,10 +147,30 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
|
|||||||
await super.deleteBlob(key);
|
await super.deleteBlob(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
override async addUpdateToSQLite(data: Uint8Array[]) {
|
override async addUpdateToSQLite(data: InsertRow[]) {
|
||||||
this.update$.next();
|
this.update$.next();
|
||||||
|
data.forEach(row => {
|
||||||
|
this.trimWhenNecessary(row.docId)?.catch(err => {
|
||||||
|
logger.error('trimWhenNecessary failed', err);
|
||||||
|
});
|
||||||
|
});
|
||||||
await super.addUpdateToSQLite(data);
|
await super.addUpdateToSQLite(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
trimWhenNecessary = debounce(async (docId?: string) => {
|
||||||
|
if (this.firstConnected) {
|
||||||
|
const count = (await this.db?.getUpdatesCount(docId)) ?? 0;
|
||||||
|
if (count > TRIM_SIZE) {
|
||||||
|
logger.debug(`trim ${this.workspaceId}:${docId} ${count}`);
|
||||||
|
const update = this.getDocAsUpdates(docId);
|
||||||
|
if (update) {
|
||||||
|
const insertRows = [{ data: update, docId }];
|
||||||
|
await this.db?.replaceUpdates(docId, insertRows);
|
||||||
|
logger.debug(`trim ${this.workspaceId}:${docId} successfully`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function openWorkspaceDatabase(workspaceId: string) {
|
export async function openWorkspaceDatabase(workspaceId: string) {
|
||||||
|
@ -34,7 +34,6 @@ export function registerProtocol() {
|
|||||||
const url = request.url.replace(/^file:\/\//, '');
|
const url = request.url.replace(/^file:\/\//, '');
|
||||||
const realpath = toAbsolutePath(url);
|
const realpath = toAbsolutePath(url);
|
||||||
callback(realpath);
|
callback(realpath);
|
||||||
console.log('interceptFileProtocol realpath', request.url, realpath);
|
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -71,7 +71,7 @@ beforeEach(async () => {
|
|||||||
.register(AffineSchemas)
|
.register(AffineSchemas)
|
||||||
.register(__unstableSchemas);
|
.register(__unstableSchemas);
|
||||||
const initPage = async (page: Page) => {
|
const initPage = async (page: Page) => {
|
||||||
await page.waitForLoaded()
|
await page.waitForLoaded();
|
||||||
expect(page).not.toBeNull();
|
expect(page).not.toBeNull();
|
||||||
assertExists(page);
|
assertExists(page);
|
||||||
const pageBlockId = page.addBlock('affine:page', {
|
const pageBlockId = page.addBlock('affine:page', {
|
||||||
|
@ -9,7 +9,7 @@ import { useAtom, useAtomValue } from 'jotai';
|
|||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { useRouter } from 'next/router';
|
import { useRouter } from 'next/router';
|
||||||
import type { ReactElement } from 'react';
|
import type { ReactElement } from 'react';
|
||||||
import { Suspense, useEffect } from 'react';
|
import { Suspense, useCallback, useEffect } from 'react';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
publicPageBlockSuiteAtom,
|
publicPageBlockSuiteAtom,
|
||||||
@ -17,7 +17,10 @@ import {
|
|||||||
publicWorkspacePageIdAtom,
|
publicWorkspacePageIdAtom,
|
||||||
} from '../../../atoms/public-workspace';
|
} from '../../../atoms/public-workspace';
|
||||||
import { BlockSuiteEditorHeader } from '../../../components/blocksuite/workspace-header';
|
import { BlockSuiteEditorHeader } from '../../../components/blocksuite/workspace-header';
|
||||||
import { PageDetailEditor } from '../../../components/page-detail-editor';
|
import {
|
||||||
|
PageDetailEditor,
|
||||||
|
type PageDetailEditorProps,
|
||||||
|
} from '../../../components/page-detail-editor';
|
||||||
import { WorkspaceAvatar } from '../../../components/pure/footer';
|
import { WorkspaceAvatar } from '../../../components/pure/footer';
|
||||||
import { PageLoading } from '../../../components/pure/loading';
|
import { PageLoading } from '../../../components/pure/loading';
|
||||||
import { useRouterHelper } from '../../../hooks/use-router-helper';
|
import { useRouterHelper } from '../../../hooks/use-router-helper';
|
||||||
@ -68,6 +71,19 @@ const PublicWorkspaceDetailPageInner = (): ReactElement => {
|
|||||||
const [name] = useBlockSuiteWorkspaceName(blockSuiteWorkspace);
|
const [name] = useBlockSuiteWorkspaceName(blockSuiteWorkspace);
|
||||||
const [avatar] = useBlockSuiteWorkspaceAvatarUrl(blockSuiteWorkspace);
|
const [avatar] = useBlockSuiteWorkspaceAvatarUrl(blockSuiteWorkspace);
|
||||||
const pageTitle = blockSuiteWorkspace.meta.getPageMeta(pageId)?.title;
|
const pageTitle = blockSuiteWorkspace.meta.getPageMeta(pageId)?.title;
|
||||||
|
const onLoad = useCallback<NonNullable<PageDetailEditorProps['onLoad']>>(
|
||||||
|
(_, editor) => {
|
||||||
|
const { page } = editor;
|
||||||
|
page.awarenessStore.setReadonly(page, true);
|
||||||
|
const dispose = editor.slots.pageLinkClicked.on(({ pageId }) => {
|
||||||
|
return openPage(blockSuiteWorkspace.id, pageId);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
dispose.dispose();
|
||||||
|
};
|
||||||
|
},
|
||||||
|
[blockSuiteWorkspace.id, openPage]
|
||||||
|
);
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<PublicQuickSearch workspace={publicWorkspace} />
|
<PublicQuickSearch workspace={publicWorkspace} />
|
||||||
@ -97,16 +113,7 @@ const PublicWorkspaceDetailPageInner = (): ReactElement => {
|
|||||||
isPublic={true}
|
isPublic={true}
|
||||||
pageId={pageId}
|
pageId={pageId}
|
||||||
workspace={publicWorkspace}
|
workspace={publicWorkspace}
|
||||||
onLoad={(_, editor) => {
|
onLoad={onLoad}
|
||||||
const { page } = editor;
|
|
||||||
page.awarenessStore.setReadonly(page, true);
|
|
||||||
const dispose = editor.slots.pageLinkClicked.on(({ pageId }) => {
|
|
||||||
return openPage(blockSuiteWorkspace.id, pageId);
|
|
||||||
});
|
|
||||||
return () => {
|
|
||||||
dispose.dispose();
|
|
||||||
};
|
|
||||||
}}
|
|
||||||
onInit={initEmptyPage}
|
onInit={initEmptyPage}
|
||||||
/>
|
/>
|
||||||
</>
|
</>
|
||||||
|
3
nx.json
3
nx.json
@ -33,7 +33,8 @@
|
|||||||
"{workspaceRoot}/apps/web/.next",
|
"{workspaceRoot}/apps/web/.next",
|
||||||
"{workspaceRoot}/packages/storybook/storybook-static",
|
"{workspaceRoot}/packages/storybook/storybook-static",
|
||||||
"{workspaceRoot}/packages/native/affine.*.node",
|
"{workspaceRoot}/packages/native/affine.*.node",
|
||||||
"{workspaceRoot}/affine.db"
|
"{workspaceRoot}/affine.db",
|
||||||
|
"{workspaceRoot/apps/electron/dist"
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
{
|
||||||
|
@ -181,7 +181,6 @@ export const useZoomControls = ({
|
|||||||
}
|
}
|
||||||
}, [imageRef]);
|
}, [imageRef]);
|
||||||
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const handleScroll = (event: WheelEvent) => {
|
const handleScroll = (event: WheelEvent) => {
|
||||||
const { deltaY } = event;
|
const { deltaY } = event;
|
||||||
|
@ -62,7 +62,6 @@ export const imagePreviewModalCloseButtonStyle = style({
|
|||||||
zIndex: 1,
|
zIndex: 1,
|
||||||
marginTop: '38px',
|
marginTop: '38px',
|
||||||
marginRight: '38px',
|
marginRight: '38px',
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
export const imagePreviewModalGoStyle = style({
|
export const imagePreviewModalGoStyle = style({
|
||||||
|
@ -5,7 +5,14 @@ import { useMediaQuery, useTheme } from '@mui/material';
|
|||||||
import type React from 'react';
|
import type React from 'react';
|
||||||
import { type CSSProperties } from 'react';
|
import { type CSSProperties } from 'react';
|
||||||
|
|
||||||
import { ScrollableContainer, Table, TableBody, TableCell, TableHead, TableHeadRow } from '../..';
|
import {
|
||||||
|
ScrollableContainer,
|
||||||
|
Table,
|
||||||
|
TableBody,
|
||||||
|
TableCell,
|
||||||
|
TableHead,
|
||||||
|
TableHeadRow,
|
||||||
|
} from '../..';
|
||||||
import { TableBodyRow } from '../../ui/table';
|
import { TableBodyRow } from '../../ui/table';
|
||||||
import { useHasScrollTop } from '../app-sidebar/sidebar-containers/use-has-scroll-top';
|
import { useHasScrollTop } from '../app-sidebar/sidebar-containers/use-has-scroll-top';
|
||||||
import { AllPagesBody } from './all-pages-body';
|
import { AllPagesBody } from './all-pages-body';
|
||||||
@ -141,12 +148,9 @@ export const PageList = ({
|
|||||||
? DEFAULT_SORT_KEY
|
? DEFAULT_SORT_KEY
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
return (
|
return sorter.data.length === 0 && fallback ? (
|
||||||
sorter.data.length === 0 && fallback ?
|
<StyledTableContainer>{fallback}</StyledTableContainer>
|
||||||
<StyledTableContainer>
|
) : (
|
||||||
{fallback}
|
|
||||||
</StyledTableContainer>
|
|
||||||
:
|
|
||||||
<ScrollableContainer inTableView>
|
<ScrollableContainer inTableView>
|
||||||
<StyledTableContainer ref={ref}>
|
<StyledTableContainer ref={ref}>
|
||||||
<Table showBorder={hasScrollTop} style={{ maxHeight: '100%' }}>
|
<Table showBorder={hasScrollTop} style={{ maxHeight: '100%' }}>
|
||||||
@ -158,7 +162,6 @@ export const PageList = ({
|
|||||||
importFile={onImportFile}
|
importFile={onImportFile}
|
||||||
/>
|
/>
|
||||||
<AllPagesBody
|
<AllPagesBody
|
||||||
|
|
||||||
isPublicWorkspace={isPublicWorkspace}
|
isPublicWorkspace={isPublicWorkspace}
|
||||||
groupKey={groupKey}
|
groupKey={groupKey}
|
||||||
data={sorter.data}
|
data={sorter.data}
|
||||||
@ -246,12 +249,9 @@ export const PageListTrashView: React.FC<{
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
return (
|
return list.length === 0 && fallback ? (
|
||||||
list.length === 0 && fallback ?
|
<StyledTableContainer>{fallback}</StyledTableContainer>
|
||||||
<StyledTableContainer>
|
) : (
|
||||||
{fallback}
|
|
||||||
</StyledTableContainer>
|
|
||||||
:
|
|
||||||
<ScrollableContainer inTableView>
|
<ScrollableContainer inTableView>
|
||||||
<StyledTableContainer ref={ref}>
|
<StyledTableContainer ref={ref}>
|
||||||
<Table showBorder={hasScrollTop}>
|
<Table showBorder={hasScrollTop}>
|
||||||
|
@ -26,18 +26,16 @@ export const ScrollableContainer = ({
|
|||||||
className={clsx([styles.scrollableViewport])}
|
className={clsx([styles.scrollableViewport])}
|
||||||
ref={ref}
|
ref={ref}
|
||||||
>
|
>
|
||||||
<div className={styles.scrollableContainer}>
|
<div className={styles.scrollableContainer}>{children}</div>
|
||||||
{children}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</ScrollArea.Viewport>
|
</ScrollArea.Viewport>
|
||||||
<ScrollArea.Scrollbar
|
<ScrollArea.Scrollbar
|
||||||
|
|
||||||
orientation="vertical"
|
orientation="vertical"
|
||||||
className={clsx(styles.scrollbar,{[styles.TableScrollbar]:inTableView})}
|
className={clsx(styles.scrollbar, {
|
||||||
|
[styles.TableScrollbar]: inTableView,
|
||||||
|
})}
|
||||||
>
|
>
|
||||||
<ScrollArea.Thumb className={styles.scrollbarThumb} />
|
<ScrollArea.Thumb className={styles.scrollbarThumb} />
|
||||||
</ScrollArea.Scrollbar>
|
</ScrollArea.Scrollbar>
|
||||||
</ScrollArea.Root>
|
</ScrollArea.Root>
|
||||||
);
|
);
|
||||||
}
|
};
|
||||||
|
@ -25,7 +25,7 @@ beforeEach(async () => {
|
|||||||
.register(AffineSchemas)
|
.register(AffineSchemas)
|
||||||
.register(__unstableSchemas);
|
.register(__unstableSchemas);
|
||||||
const initPage = async (page: Page) => {
|
const initPage = async (page: Page) => {
|
||||||
await page.waitForLoaded()
|
await page.waitForLoaded();
|
||||||
expect(page).not.toBeNull();
|
expect(page).not.toBeNull();
|
||||||
assertExists(page);
|
assertExists(page);
|
||||||
const pageBlockId = page.addBlock('affine:page', {
|
const pageBlockId = page.addBlock('affine:page', {
|
||||||
|
@ -51,5 +51,6 @@ export function useBlockSuiteWorkspacePage(
|
|||||||
): Page | null {
|
): Page | null {
|
||||||
const pageAtom = getAtom(blockSuiteWorkspace, pageId);
|
const pageAtom = getAtom(blockSuiteWorkspace, pageId);
|
||||||
assertExists(pageAtom);
|
assertExists(pageAtom);
|
||||||
return useAtomValue(pageAtom);
|
const page = useAtomValue(pageAtom);
|
||||||
|
return page;
|
||||||
}
|
}
|
||||||
|
@ -17,8 +17,15 @@ export abstract class HandlerManager<
|
|||||||
}
|
}
|
||||||
|
|
||||||
type DBHandlers = {
|
type DBHandlers = {
|
||||||
getDocAsUpdates: (id: string) => Promise<Uint8Array>;
|
getDocAsUpdates: (
|
||||||
applyDocUpdate: (id: string, update: Uint8Array) => Promise<void>;
|
workspaceId: string,
|
||||||
|
subdocId?: string
|
||||||
|
) => Promise<Uint8Array>;
|
||||||
|
applyDocUpdate: (
|
||||||
|
id: string,
|
||||||
|
update: Uint8Array,
|
||||||
|
subdocId?: string
|
||||||
|
) => Promise<void>;
|
||||||
addBlob: (
|
addBlob: (
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
key: string,
|
key: string,
|
||||||
|
@ -1,12 +1,22 @@
|
|||||||
use sqlx::sqlite::SqliteConnectOptions;
|
use sqlx::sqlite::SqliteConnectOptions;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), std::io::Error> {
|
async fn main() -> Result<(), std::io::Error> {
|
||||||
dotenv::dotenv().ok();
|
dotenv::dotenv().ok();
|
||||||
|
|
||||||
|
// always start with a fresh database to have
|
||||||
|
// latest db schema
|
||||||
|
let db_path = "../../affine.db";
|
||||||
|
|
||||||
|
// check if db exists and then remove file
|
||||||
|
if fs::metadata(db_path).is_ok() {
|
||||||
|
fs::remove_file(db_path)?;
|
||||||
|
}
|
||||||
|
|
||||||
napi_build::setup();
|
napi_build::setup();
|
||||||
let options = SqliteConnectOptions::new()
|
let options = SqliteConnectOptions::new()
|
||||||
.filename("../../affine.db")
|
.filename(db_path)
|
||||||
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
|
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
|
||||||
.locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
|
.locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
|
||||||
.create_if_missing(true);
|
.create_if_missing(true);
|
||||||
|
15
packages/native/index.d.ts
vendored
15
packages/native/index.d.ts
vendored
@ -32,6 +32,11 @@ export interface UpdateRow {
|
|||||||
id: number;
|
id: number;
|
||||||
timestamp: Date;
|
timestamp: Date;
|
||||||
data: Buffer;
|
data: Buffer;
|
||||||
|
docId?: string;
|
||||||
|
}
|
||||||
|
export interface InsertRow {
|
||||||
|
docId?: string;
|
||||||
|
data: Uint8Array;
|
||||||
}
|
}
|
||||||
export class Subscription {
|
export class Subscription {
|
||||||
toString(): string;
|
toString(): string;
|
||||||
@ -56,8 +61,14 @@ export class SqliteConnection {
|
|||||||
getBlob(key: string): Promise<BlobRow | null>;
|
getBlob(key: string): Promise<BlobRow | null>;
|
||||||
deleteBlob(key: string): Promise<void>;
|
deleteBlob(key: string): Promise<void>;
|
||||||
getBlobKeys(): Promise<Array<string>>;
|
getBlobKeys(): Promise<Array<string>>;
|
||||||
getUpdates(): Promise<Array<UpdateRow>>;
|
getUpdates(docId?: string | undefined | null): Promise<Array<UpdateRow>>;
|
||||||
insertUpdates(updates: Array<Uint8Array>): Promise<void>;
|
getUpdatesCount(docId?: string | undefined | null): Promise<number>;
|
||||||
|
getAllUpdates(): Promise<Array<UpdateRow>>;
|
||||||
|
insertUpdates(updates: Array<InsertRow>): Promise<void>;
|
||||||
|
replaceUpdates(
|
||||||
|
docId: string | undefined | null,
|
||||||
|
updates: Array<InsertRow>
|
||||||
|
): Promise<void>;
|
||||||
close(): Promise<void>;
|
close(): Promise<void>;
|
||||||
get isClose(): boolean;
|
get isClose(): boolean;
|
||||||
static validate(path: string): Promise<boolean>;
|
static validate(path: string): Promise<boolean>;
|
||||||
|
@ -4,7 +4,8 @@
|
|||||||
pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
|
pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
data BLOB NOT NULL,
|
data BLOB NOT NULL,
|
||||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||||
|
doc_id TEXT
|
||||||
);
|
);
|
||||||
CREATE TABLE IF NOT EXISTS "blobs" (
|
CREATE TABLE IF NOT EXISTS "blobs" (
|
||||||
key TEXT PRIMARY KEY NOT NULL,
|
key TEXT PRIMARY KEY NOT NULL,
|
||||||
|
@ -19,6 +19,13 @@ pub struct UpdateRow {
|
|||||||
pub id: i64,
|
pub id: i64,
|
||||||
pub timestamp: NaiveDateTime,
|
pub timestamp: NaiveDateTime,
|
||||||
pub data: Buffer,
|
pub data: Buffer,
|
||||||
|
pub doc_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(object)]
|
||||||
|
pub struct InsertRow {
|
||||||
|
pub doc_id: Option<String>,
|
||||||
|
pub data: Uint8Array,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[napi]
|
#[napi]
|
||||||
@ -53,6 +60,7 @@ impl SqliteConnection {
|
|||||||
.execute(connection.as_mut())
|
.execute(connection.as_mut())
|
||||||
.await
|
.await
|
||||||
.map_err(anyhow::Error::from)?;
|
.map_err(anyhow::Error::from)?;
|
||||||
|
self.migrate_add_doc_id().await?;
|
||||||
connection.detach();
|
connection.detach();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -74,7 +82,11 @@ impl SqliteConnection {
|
|||||||
|
|
||||||
#[napi]
|
#[napi]
|
||||||
pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
|
pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
|
||||||
sqlx::query_as!(BlobRow, "SELECT * FROM blobs WHERE key = ?", key)
|
sqlx::query_as!(
|
||||||
|
BlobRow,
|
||||||
|
"SELECT key, data, timestamp FROM blobs WHERE key = ?",
|
||||||
|
key
|
||||||
|
)
|
||||||
.fetch_one(&self.pool)
|
.fetch_one(&self.pool)
|
||||||
.await
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
@ -100,8 +112,54 @@ impl SqliteConnection {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[napi]
|
#[napi]
|
||||||
pub async fn get_updates(&self) -> napi::Result<Vec<UpdateRow>> {
|
pub async fn get_updates(&self, doc_id: Option<String>) -> napi::Result<Vec<UpdateRow>> {
|
||||||
let updates = sqlx::query_as!(UpdateRow, "SELECT * FROM updates")
|
let updates = match doc_id {
|
||||||
|
Some(doc_id) => sqlx::query_as!(
|
||||||
|
UpdateRow,
|
||||||
|
"SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id = ?",
|
||||||
|
doc_id
|
||||||
|
)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?,
|
||||||
|
None => sqlx::query_as!(
|
||||||
|
UpdateRow,
|
||||||
|
"SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id is NULL",
|
||||||
|
)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?,
|
||||||
|
};
|
||||||
|
Ok(updates)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi]
|
||||||
|
pub async fn get_updates_count(&self, doc_id: Option<String>) -> napi::Result<i32> {
|
||||||
|
let count = match doc_id {
|
||||||
|
Some(doc_id) => {
|
||||||
|
sqlx::query!(
|
||||||
|
"SELECT COUNT(*) as count FROM updates WHERE doc_id = ?",
|
||||||
|
doc_id
|
||||||
|
)
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?
|
||||||
|
.count
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
sqlx::query!("SELECT COUNT(*) as count FROM updates WHERE doc_id is NULL")
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?
|
||||||
|
.count
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi]
|
||||||
|
pub async fn get_all_updates(&self) -> napi::Result<Vec<UpdateRow>> {
|
||||||
|
let updates = sqlx::query_as!(UpdateRow, "SELECT id, timestamp, data, doc_id FROM updates")
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await
|
.await
|
||||||
.map_err(anyhow::Error::from)?;
|
.map_err(anyhow::Error::from)?;
|
||||||
@ -109,11 +167,51 @@ impl SqliteConnection {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[napi]
|
#[napi]
|
||||||
pub async fn insert_updates(&self, updates: Vec<Uint8Array>) -> napi::Result<()> {
|
pub async fn insert_updates(&self, updates: Vec<InsertRow>) -> napi::Result<()> {
|
||||||
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
|
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
|
||||||
for update in updates.into_iter() {
|
for InsertRow { data, doc_id } in updates {
|
||||||
let update = update.as_ref();
|
let update = data.as_ref();
|
||||||
sqlx::query_as!(UpdateRow, "INSERT INTO updates (data) VALUES ($1)", update)
|
sqlx::query_as!(
|
||||||
|
UpdateRow,
|
||||||
|
"INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
|
||||||
|
update,
|
||||||
|
doc_id
|
||||||
|
)
|
||||||
|
.execute(&mut *transaction)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?;
|
||||||
|
}
|
||||||
|
transaction.commit().await.map_err(anyhow::Error::from)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi]
|
||||||
|
pub async fn replace_updates(
|
||||||
|
&self,
|
||||||
|
doc_id: Option<String>,
|
||||||
|
updates: Vec<InsertRow>,
|
||||||
|
) -> napi::Result<()> {
|
||||||
|
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
|
||||||
|
|
||||||
|
match doc_id {
|
||||||
|
Some(doc_id) => sqlx::query!("DELETE FROM updates where doc_id = ?", doc_id)
|
||||||
|
.execute(&mut *transaction)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?,
|
||||||
|
None => sqlx::query!("DELETE FROM updates where doc_id is NULL",)
|
||||||
|
.execute(&mut *transaction)
|
||||||
|
.await
|
||||||
|
.map_err(anyhow::Error::from)?,
|
||||||
|
};
|
||||||
|
|
||||||
|
for InsertRow { data, doc_id } in updates {
|
||||||
|
let update = data.as_ref();
|
||||||
|
sqlx::query_as!(
|
||||||
|
UpdateRow,
|
||||||
|
"INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
|
||||||
|
update,
|
||||||
|
doc_id
|
||||||
|
)
|
||||||
.execute(&mut *transaction)
|
.execute(&mut *transaction)
|
||||||
.await
|
.await
|
||||||
.map_err(anyhow::Error::from)?;
|
.map_err(anyhow::Error::from)?;
|
||||||
@ -158,4 +256,22 @@ impl SqliteConnection {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// todo: have a better way to handle migration
|
||||||
|
async fn migrate_add_doc_id(&self) -> Result<(), anyhow::Error> {
|
||||||
|
// ignore errors
|
||||||
|
match sqlx::query("ALTER TABLE updates ADD COLUMN doc_id TEXT")
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(_) => Ok(()),
|
||||||
|
Err(err) => {
|
||||||
|
if err.to_string().contains("duplicate column name") {
|
||||||
|
Ok(()) // Ignore error if it's due to duplicate column
|
||||||
|
} else {
|
||||||
|
Err(anyhow::Error::from(err)) // Propagate other errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -140,7 +140,7 @@ describe('ydoc sync', () => {
|
|||||||
|
|
||||||
const pageId = uuidv4();
|
const pageId = uuidv4();
|
||||||
const page1 = workspace1.createPage({ id: pageId });
|
const page1 = workspace1.createPage({ id: pageId });
|
||||||
await page1.waitForLoaded()
|
await page1.waitForLoaded();
|
||||||
const pageBlockId = page1.addBlock('affine:page', {
|
const pageBlockId = page1.addBlock('affine:page', {
|
||||||
title: new page1.Text(''),
|
title: new page1.Text(''),
|
||||||
});
|
});
|
||||||
@ -153,7 +153,7 @@ describe('ydoc sync', () => {
|
|||||||
workspace1.doc.getMap(`space:${pageId}`).toJSON()
|
workspace1.doc.getMap(`space:${pageId}`).toJSON()
|
||||||
);
|
);
|
||||||
const page2 = workspace2.getPage(pageId) as Page;
|
const page2 = workspace2.getPage(pageId) as Page;
|
||||||
await page2.waitForLoaded()
|
await page2.waitForLoaded();
|
||||||
page1.updateBlock(
|
page1.updateBlock(
|
||||||
page1.getBlockById(paragraphId) as ParagraphBlockModel,
|
page1.getBlockById(paragraphId) as ParagraphBlockModel,
|
||||||
{
|
{
|
||||||
|
@ -7,7 +7,10 @@ import type { Y as YType } from '@blocksuite/store';
|
|||||||
import { uuidv4, Workspace } from '@blocksuite/store';
|
import { uuidv4, Workspace } from '@blocksuite/store';
|
||||||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||||
|
|
||||||
import { createSQLiteDBDownloadProvider, createSQLiteProvider } from '../index';
|
import {
|
||||||
|
createSQLiteDBDownloadProvider,
|
||||||
|
createSQLiteProvider,
|
||||||
|
} from '../sqlite-providers';
|
||||||
|
|
||||||
const Y = Workspace.Y;
|
const Y = Workspace.Y;
|
||||||
|
|
||||||
@ -148,15 +151,21 @@ describe('SQLite download provider', () => {
|
|||||||
test('disconnect handlers', async () => {
|
test('disconnect handlers', async () => {
|
||||||
const offHandler = vi.fn();
|
const offHandler = vi.fn();
|
||||||
let handleUpdate = () => {};
|
let handleUpdate = () => {};
|
||||||
workspace.doc.on = (_: string, fn: () => void) => {
|
let handleSubdocs = () => {};
|
||||||
|
workspace.doc.on = (event: string, fn: () => void) => {
|
||||||
|
if (event === 'update') {
|
||||||
handleUpdate = fn;
|
handleUpdate = fn;
|
||||||
|
} else if (event === 'subdocs') {
|
||||||
|
handleSubdocs = fn;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
workspace.doc.off = offHandler;
|
workspace.doc.off = offHandler;
|
||||||
await provider.connect();
|
provider.connect();
|
||||||
|
|
||||||
provider.disconnect();
|
provider.disconnect();
|
||||||
|
|
||||||
expect(triggerDBUpdate).toBe(null);
|
expect(triggerDBUpdate).toBe(null);
|
||||||
expect(offHandler).toBeCalledWith('update', handleUpdate);
|
expect(offHandler).toBeCalledWith('update', handleUpdate);
|
||||||
|
expect(offHandler).toBeCalledWith('subdocs', handleSubdocs);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -3,8 +3,6 @@ import type {
|
|||||||
AffineWebSocketProvider,
|
AffineWebSocketProvider,
|
||||||
LocalIndexedDBBackgroundProvider,
|
LocalIndexedDBBackgroundProvider,
|
||||||
LocalIndexedDBDownloadProvider,
|
LocalIndexedDBDownloadProvider,
|
||||||
SQLiteDBDownloadProvider,
|
|
||||||
SQLiteProvider,
|
|
||||||
} from '@affine/env/workspace';
|
} from '@affine/env/workspace';
|
||||||
import type { Disposable, DocProviderCreator } from '@blocksuite/store';
|
import type { Disposable, DocProviderCreator } from '@blocksuite/store';
|
||||||
import { assertExists, Workspace } from '@blocksuite/store';
|
import { assertExists, Workspace } from '@blocksuite/store';
|
||||||
@ -21,6 +19,10 @@ import { getLoginStorage, storageChangeSlot } from '../affine/login';
|
|||||||
import { CallbackSet } from '../utils';
|
import { CallbackSet } from '../utils';
|
||||||
import { createAffineDownloadProvider } from './affine-download';
|
import { createAffineDownloadProvider } from './affine-download';
|
||||||
import { localProviderLogger as logger } from './logger';
|
import { localProviderLogger as logger } from './logger';
|
||||||
|
import {
|
||||||
|
createSQLiteDBDownloadProvider,
|
||||||
|
createSQLiteProvider,
|
||||||
|
} from './sqlite-providers';
|
||||||
|
|
||||||
const Y = Workspace.Y;
|
const Y = Workspace.Y;
|
||||||
|
|
||||||
@ -151,151 +153,6 @@ const createIndexedDBDownloadProvider: DocProviderCreator = (
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const sqliteOrigin = Symbol('sqlite-provider-origin');
|
|
||||||
|
|
||||||
const createSQLiteProvider: DocProviderCreator = (id, doc): SQLiteProvider => {
|
|
||||||
const { apis, events } = window;
|
|
||||||
// make sure it is being used in Electron with APIs
|
|
||||||
assertExists(apis);
|
|
||||||
assertExists(events);
|
|
||||||
|
|
||||||
function handleUpdate(update: Uint8Array, origin: unknown) {
|
|
||||||
if (origin === sqliteOrigin) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
apis.db.applyDocUpdate(id, update).catch(err => {
|
|
||||||
console.error(err);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let unsubscribe = () => {};
|
|
||||||
let connected = false;
|
|
||||||
|
|
||||||
const connect = () => {
|
|
||||||
logger.info('connecting sqlite provider', id);
|
|
||||||
doc.on('update', handleUpdate);
|
|
||||||
unsubscribe = events.db.onExternalUpdate(
|
|
||||||
({
|
|
||||||
update,
|
|
||||||
workspaceId,
|
|
||||||
}: {
|
|
||||||
workspaceId: string;
|
|
||||||
update: Uint8Array;
|
|
||||||
}) => {
|
|
||||||
if (workspaceId === id) {
|
|
||||||
Y.applyUpdate(doc, update, sqliteOrigin);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
connected = true;
|
|
||||||
logger.info('connecting sqlite done', id);
|
|
||||||
};
|
|
||||||
|
|
||||||
const cleanup = () => {
|
|
||||||
logger.info('disconnecting sqlite provider', id);
|
|
||||||
unsubscribe();
|
|
||||||
doc.off('update', handleUpdate);
|
|
||||||
connected = false;
|
|
||||||
};
|
|
||||||
|
|
||||||
return {
|
|
||||||
flavour: 'sqlite',
|
|
||||||
passive: true,
|
|
||||||
get connected(): boolean {
|
|
||||||
return connected;
|
|
||||||
},
|
|
||||||
cleanup,
|
|
||||||
connect,
|
|
||||||
disconnect: cleanup,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const createSQLiteDBDownloadProvider: DocProviderCreator = (
|
|
||||||
id,
|
|
||||||
doc
|
|
||||||
): SQLiteDBDownloadProvider => {
|
|
||||||
const { apis } = window;
|
|
||||||
let disconnected = false;
|
|
||||||
|
|
||||||
let _resolve: () => void;
|
|
||||||
let _reject: (error: unknown) => void;
|
|
||||||
const promise = new Promise<void>((resolve, reject) => {
|
|
||||||
_resolve = resolve;
|
|
||||||
_reject = reject;
|
|
||||||
});
|
|
||||||
|
|
||||||
async function syncUpdates() {
|
|
||||||
logger.info('syncing updates from sqlite', id);
|
|
||||||
const updates = await apis.db.getDocAsUpdates(id);
|
|
||||||
|
|
||||||
if (disconnected) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (updates) {
|
|
||||||
Y.applyUpdate(doc, updates, sqliteOrigin);
|
|
||||||
}
|
|
||||||
|
|
||||||
const diff = Y.encodeStateAsUpdate(doc, updates);
|
|
||||||
|
|
||||||
// also apply updates to sqlite
|
|
||||||
await apis.db.applyDocUpdate(id, diff);
|
|
||||||
}
|
|
||||||
|
|
||||||
// fixme(pengx17): should n't sync blob in doc provider
|
|
||||||
// async function _syncBlobIntoSQLite(bs: BlobManager) {
|
|
||||||
// const persistedKeys = await apis.db.getBlobKeys(id);
|
|
||||||
//
|
|
||||||
// if (disconnected) {
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// const allKeys = await bs.list().catch(() => []);
|
|
||||||
// const keysToPersist = allKeys.filter(k => !persistedKeys.includes(k));
|
|
||||||
//
|
|
||||||
// logger.info('persisting blobs', keysToPersist, 'to sqlite');
|
|
||||||
// return Promise.all(
|
|
||||||
// keysToPersist.map(async k => {
|
|
||||||
// const blob = await bs.get(k);
|
|
||||||
// if (!blob) {
|
|
||||||
// logger.warn('blob not found for', k);
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (disconnected) {
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// return apis?.db.addBlob(
|
|
||||||
// id,
|
|
||||||
// k,
|
|
||||||
// new Uint8Array(await blob.arrayBuffer())
|
|
||||||
// );
|
|
||||||
// })
|
|
||||||
// );
|
|
||||||
// }
|
|
||||||
|
|
||||||
return {
|
|
||||||
flavour: 'sqlite-download',
|
|
||||||
active: true,
|
|
||||||
get whenReady() {
|
|
||||||
return promise;
|
|
||||||
},
|
|
||||||
cleanup: () => {
|
|
||||||
disconnected = true;
|
|
||||||
},
|
|
||||||
sync: async () => {
|
|
||||||
logger.info('connect indexeddb provider', id);
|
|
||||||
try {
|
|
||||||
await syncUpdates();
|
|
||||||
_resolve();
|
|
||||||
} catch (error) {
|
|
||||||
_reject(error);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export {
|
export {
|
||||||
createAffineDownloadProvider,
|
createAffineDownloadProvider,
|
||||||
createAffineWebSocketProvider,
|
createAffineWebSocketProvider,
|
||||||
|
212
packages/workspace/src/providers/sqlite-providers.ts
Normal file
212
packages/workspace/src/providers/sqlite-providers.ts
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
import type {
|
||||||
|
SQLiteDBDownloadProvider,
|
||||||
|
SQLiteProvider,
|
||||||
|
} from '@affine/env/workspace';
|
||||||
|
import type { DocProviderCreator } from '@blocksuite/store';
|
||||||
|
import {
|
||||||
|
assertExists,
|
||||||
|
Workspace as BlockSuiteWorkspace,
|
||||||
|
} from '@blocksuite/store';
|
||||||
|
import type { Doc } from 'yjs';
|
||||||
|
|
||||||
|
import { localProviderLogger as logger } from './logger';
|
||||||
|
|
||||||
|
const Y = BlockSuiteWorkspace.Y;
|
||||||
|
|
||||||
|
const sqliteOrigin = Symbol('sqlite-provider-origin');
|
||||||
|
|
||||||
|
type SubDocsEvent = {
|
||||||
|
added: Set<Doc>;
|
||||||
|
removed: Set<Doc>;
|
||||||
|
loaded: Set<Doc>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A provider that is responsible for syncing updates the workspace with the local SQLite database.
|
||||||
|
*/
|
||||||
|
export const createSQLiteProvider: DocProviderCreator = (
|
||||||
|
id,
|
||||||
|
rootDoc
|
||||||
|
): SQLiteProvider => {
|
||||||
|
const { apis, events } = window;
|
||||||
|
// make sure it is being used in Electron with APIs
|
||||||
|
assertExists(apis);
|
||||||
|
assertExists(events);
|
||||||
|
|
||||||
|
const updateHandlerMap = new WeakMap<
|
||||||
|
Doc,
|
||||||
|
(update: Uint8Array, origin: unknown) => void
|
||||||
|
>();
|
||||||
|
const subDocsHandlerMap = new WeakMap<Doc, (event: SubDocsEvent) => void>();
|
||||||
|
|
||||||
|
const createOrHandleUpdate = (doc: Doc) => {
|
||||||
|
if (updateHandlerMap.has(doc)) {
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||||
|
return updateHandlerMap.get(doc)!;
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleUpdate(update: Uint8Array, origin: unknown) {
|
||||||
|
if (origin === sqliteOrigin) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const subdocId = doc.guid === id ? undefined : doc.guid;
|
||||||
|
apis.db.applyDocUpdate(id, update, subdocId).catch(err => {
|
||||||
|
logger.error(err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
updateHandlerMap.set(doc, handleUpdate);
|
||||||
|
return handleUpdate;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createOrGetHandleSubDocs = (doc: Doc) => {
|
||||||
|
if (subDocsHandlerMap.has(doc)) {
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||||
|
return subDocsHandlerMap.get(doc)!;
|
||||||
|
}
|
||||||
|
function handleSubdocs(event: SubDocsEvent) {
|
||||||
|
event.removed.forEach(doc => {
|
||||||
|
untrackDoc(doc);
|
||||||
|
});
|
||||||
|
event.loaded.forEach(doc => {
|
||||||
|
trackDoc(doc);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
subDocsHandlerMap.set(doc, handleSubdocs);
|
||||||
|
return handleSubdocs;
|
||||||
|
};
|
||||||
|
|
||||||
|
function trackDoc(doc: Doc) {
|
||||||
|
doc.on('update', createOrHandleUpdate(doc));
|
||||||
|
doc.on('subdocs', createOrGetHandleSubDocs(doc));
|
||||||
|
doc.subdocs.forEach(doc => {
|
||||||
|
trackDoc(doc);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function untrackDoc(doc: Doc) {
|
||||||
|
doc.subdocs.forEach(doc => {
|
||||||
|
untrackDoc(doc);
|
||||||
|
});
|
||||||
|
doc.off('update', createOrHandleUpdate(doc));
|
||||||
|
doc.off('subdocs', createOrGetHandleSubDocs(doc));
|
||||||
|
}
|
||||||
|
|
||||||
|
let unsubscribe = () => {};
|
||||||
|
let connected = false;
|
||||||
|
|
||||||
|
const connect = () => {
|
||||||
|
logger.info('connecting sqlite provider', id);
|
||||||
|
trackDoc(rootDoc);
|
||||||
|
|
||||||
|
unsubscribe = events.db.onExternalUpdate(
|
||||||
|
({
|
||||||
|
update,
|
||||||
|
workspaceId,
|
||||||
|
docId,
|
||||||
|
}: {
|
||||||
|
workspaceId: string;
|
||||||
|
update: Uint8Array;
|
||||||
|
docId?: string;
|
||||||
|
}) => {
|
||||||
|
if (workspaceId === id) {
|
||||||
|
if (docId) {
|
||||||
|
for (const doc of rootDoc.subdocs) {
|
||||||
|
if (doc.guid === docId) {
|
||||||
|
Y.applyUpdate(doc, update, sqliteOrigin);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Y.applyUpdate(rootDoc, update, sqliteOrigin);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
connected = true;
|
||||||
|
logger.info('connecting sqlite done', id);
|
||||||
|
};
|
||||||
|
|
||||||
|
const cleanup = () => {
|
||||||
|
logger.info('disconnecting sqlite provider', id);
|
||||||
|
unsubscribe();
|
||||||
|
untrackDoc(rootDoc);
|
||||||
|
connected = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
flavour: 'sqlite',
|
||||||
|
passive: true,
|
||||||
|
get connected(): boolean {
|
||||||
|
return connected;
|
||||||
|
},
|
||||||
|
cleanup,
|
||||||
|
connect,
|
||||||
|
disconnect: cleanup,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A provider that is responsible for DOWNLOADING updates from the local SQLite database.
|
||||||
|
*/
|
||||||
|
export const createSQLiteDBDownloadProvider: DocProviderCreator = (
|
||||||
|
id,
|
||||||
|
rootDoc
|
||||||
|
): SQLiteDBDownloadProvider => {
|
||||||
|
const { apis } = window;
|
||||||
|
let disconnected = false;
|
||||||
|
|
||||||
|
let _resolve: () => void;
|
||||||
|
let _reject: (error: unknown) => void;
|
||||||
|
const promise = new Promise<void>((resolve, reject) => {
|
||||||
|
_resolve = resolve;
|
||||||
|
_reject = reject;
|
||||||
|
});
|
||||||
|
|
||||||
|
async function syncUpdates(doc: Doc) {
|
||||||
|
logger.info('syncing updates from sqlite', id);
|
||||||
|
const subdocId = doc.guid === id ? undefined : doc.guid;
|
||||||
|
const updates = await apis.db.getDocAsUpdates(id, subdocId);
|
||||||
|
|
||||||
|
if (disconnected) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (updates) {
|
||||||
|
Y.applyUpdate(doc, updates, sqliteOrigin);
|
||||||
|
}
|
||||||
|
|
||||||
|
const diff = Y.encodeStateAsUpdate(doc, updates);
|
||||||
|
|
||||||
|
// also apply updates to sqlite
|
||||||
|
await apis.db.applyDocUpdate(id, diff, subdocId);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncAllUpdates(doc: Doc) {
|
||||||
|
if (await syncUpdates(doc)) {
|
||||||
|
const subdocs = Array.from(doc.subdocs).filter(d => d.shouldLoad);
|
||||||
|
await Promise.all(subdocs.map(syncAllUpdates));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
flavour: 'sqlite-download',
|
||||||
|
active: true,
|
||||||
|
get whenReady() {
|
||||||
|
return promise;
|
||||||
|
},
|
||||||
|
cleanup: () => {
|
||||||
|
disconnected = true;
|
||||||
|
},
|
||||||
|
sync: async () => {
|
||||||
|
logger.info('connect indexeddb provider', id);
|
||||||
|
try {
|
||||||
|
await syncAllUpdates(rootDoc);
|
||||||
|
_resolve();
|
||||||
|
} catch (error) {
|
||||||
|
_reject(error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
@ -107,6 +107,7 @@ describe('indexeddb provider', () => {
|
|||||||
})
|
})
|
||||||
.register(AffineSchemas)
|
.register(AffineSchemas)
|
||||||
.register(__unstableSchemas);
|
.register(__unstableSchemas);
|
||||||
|
// data should only contain updates for the root doc
|
||||||
data.updates.forEach(({ update }) => {
|
data.updates.forEach(({ update }) => {
|
||||||
Workspace.Y.applyUpdate(testWorkspace.doc, update);
|
Workspace.Y.applyUpdate(testWorkspace.doc, update);
|
||||||
});
|
});
|
||||||
@ -125,19 +126,6 @@ describe('indexeddb provider', () => {
|
|||||||
}
|
}
|
||||||
expect(workspace.doc.toJSON()).toEqual(testWorkspace.doc.toJSON());
|
expect(workspace.doc.toJSON()).toEqual(testWorkspace.doc.toJSON());
|
||||||
}
|
}
|
||||||
|
|
||||||
const secondWorkspace = new Workspace({
|
|
||||||
id,
|
|
||||||
})
|
|
||||||
.register(AffineSchemas)
|
|
||||||
.register(__unstableSchemas);
|
|
||||||
const provider2 = createIndexedDBProvider(secondWorkspace.doc, rootDBName);
|
|
||||||
provider2.connect();
|
|
||||||
await provider2.whenSynced;
|
|
||||||
const page = secondWorkspace.getPage('page0');
|
|
||||||
assertExists(page);
|
|
||||||
await page.waitForLoaded();
|
|
||||||
expect(workspace.doc.toJSON()).toEqual(secondWorkspace.doc.toJSON());
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test('disconnect suddenly', async () => {
|
test('disconnect suddenly', async () => {
|
||||||
@ -423,6 +411,7 @@ describe('subDoc', () => {
|
|||||||
provider.disconnect();
|
provider.disconnect();
|
||||||
json2 = doc.toJSON();
|
json2 = doc.toJSON();
|
||||||
}
|
}
|
||||||
|
// the following line compares {} with {}
|
||||||
expect(json1['']['1'].toJSON()).toEqual(json2['']['1'].toJSON());
|
expect(json1['']['1'].toJSON()).toEqual(json2['']['1'].toJSON());
|
||||||
expect(json1['']['2']).toEqual(json2['']['2']);
|
expect(json1['']['2']).toEqual(json2['']['2']);
|
||||||
});
|
});
|
||||||
|
@ -195,10 +195,12 @@ const selectDateFromDatePicker = async (page: Page, date: Date) => {
|
|||||||
);
|
);
|
||||||
await nextMonthButton.click();
|
await nextMonthButton.click();
|
||||||
}
|
}
|
||||||
const map = ['th', 'st', 'nd', 'rd']
|
const map = ['th', 'st', 'nd', 'rd'];
|
||||||
// Click on the day cell
|
// Click on the day cell
|
||||||
const dateCell = page.locator(
|
const dateCell = page.locator(
|
||||||
`[aria-disabled="false"][aria-label="Choose ${weekday}, ${month} ${day}${map[Number.parseInt(day) % 10] ?? 'th'}, ${year}"]`
|
`[aria-disabled="false"][aria-label="Choose ${weekday}, ${month} ${day}${
|
||||||
|
map[Number.parseInt(day) % 10] ?? 'th'
|
||||||
|
}, ${year}"]`
|
||||||
);
|
);
|
||||||
await dateCell.click();
|
await dateCell.click();
|
||||||
};
|
};
|
||||||
|
Loading…
Reference in New Issue
Block a user