feat(nbstore): add nbstore worker (#9185)

This commit is contained in:
EYHN 2024-12-20 08:01:23 +00:00
parent 30200ff86d
commit cbaf35df0b
No known key found for this signature in database
GPG Key ID: 46C9E26A75AB276C
51 changed files with 1144 additions and 501 deletions

View File

@ -126,6 +126,16 @@ export class OpConsumer<Ops extends OpSchema> extends AutoMessageHandler {
this.registeredOpHandlers.set(op, handler); this.registeredOpHandlers.set(op, handler);
} }
registerAll(
handlers: OpNames<Ops> extends string
? { [K in OpNames<Ops>]: OpHandler<Ops, K> }
: never
) {
for (const [op, handler] of Object.entries(handlers)) {
this.register(op as any, handler as any);
}
}
before<Op extends OpNames<Ops>>( before<Op extends OpNames<Ops>>(
op: Op, op: Op,
handler: (...input: OpInput<Ops, Op>) => void handler: (...input: OpInput<Ops, Op>) => void

View File

@ -6,7 +6,7 @@
"sideEffects": false, "sideEffects": false,
"exports": { "exports": {
".": "./src/index.ts", ".": "./src/index.ts",
"./op": "./src/op/index.ts", "./worker": "./src/worker/index.ts",
"./idb": "./src/impls/idb/index.ts", "./idb": "./src/impls/idb/index.ts",
"./idb/v1": "./src/impls/idb/v1/index.ts", "./idb/v1": "./src/impls/idb/v1/index.ts",
"./cloud": "./src/impls/cloud/index.ts", "./cloud": "./src/impls/cloud/index.ts",

View File

@ -8,7 +8,7 @@ import { AwarenessFrontend } from '../frontend/awareness';
import { DocFrontend } from '../frontend/doc'; import { DocFrontend } from '../frontend/doc';
import { BroadcastChannelAwarenessStorage } from '../impls/broadcast-channel/awareness'; import { BroadcastChannelAwarenessStorage } from '../impls/broadcast-channel/awareness';
import { IndexedDBDocStorage } from '../impls/idb'; import { IndexedDBDocStorage } from '../impls/idb';
import { AwarenessSync } from '../sync/awareness'; import { AwarenessSyncImpl } from '../sync/awareness';
import { expectYjsEqual } from './utils'; import { expectYjsEqual } from './utils';
test('doc', async () => { test('doc', async () => {
@ -23,9 +23,9 @@ test('doc', async () => {
type: 'workspace', type: 'workspace',
}); });
docStorage.connect(); docStorage.connection.connect();
await docStorage.waitForConnected(); await docStorage.connection.waitForConnected();
const frontend1 = new DocFrontend(docStorage, null); const frontend1 = new DocFrontend(docStorage, null);
frontend1.start(); frontend1.start();
@ -68,11 +68,11 @@ test('awareness', async () => {
type: 'workspace', type: 'workspace',
}); });
storage1.connect(); storage1.connection.connect();
storage2.connect(); storage2.connection.connect();
await storage1.waitForConnected(); await storage1.connection.waitForConnected();
await storage2.waitForConnected(); await storage2.connection.waitForConnected();
// peer a // peer a
const docA = new YDoc({ guid: 'test-doc' }); const docA = new YDoc({ guid: 'test-doc' });
@ -90,13 +90,13 @@ test('awareness', async () => {
const awarenessC = new Awareness(docC); const awarenessC = new Awareness(docC);
{ {
const sync = new AwarenessSync(storage1, [storage2]); const sync = new AwarenessSyncImpl(storage1, [storage2]);
const frontend = new AwarenessFrontend(sync); const frontend = new AwarenessFrontend(sync);
frontend.connect(awarenessA); frontend.connect(awarenessA);
frontend.connect(awarenessB); frontend.connect(awarenessB);
} }
{ {
const sync = new AwarenessSync(storage2, [storage1]); const sync = new AwarenessSyncImpl(storage2, [storage1]);
const frontend = new AwarenessFrontend(sync); const frontend = new AwarenessFrontend(sync);
frontend.connect(awarenessC); frontend.connect(awarenessC);
} }

View File

@ -8,7 +8,20 @@ export type ConnectionStatus =
| 'error' | 'error'
| 'closed'; | 'closed';
export abstract class Connection<T = any> { export interface Connection<T = any> {
readonly status: ConnectionStatus;
readonly inner: T;
connect(): void;
disconnect(): void;
waitForConnected(signal?: AbortSignal): Promise<void>;
onStatusChanged(
cb: (status: ConnectionStatus, error?: Error) => void
): () => void;
}
export abstract class AutoReconnectConnection<T = any>
implements Connection<T>
{
private readonly event = new EventEmitter2(); private readonly event = new EventEmitter2();
private _inner: T | null = null; private _inner: T | null = null;
private _status: ConnectionStatus = 'idle'; private _status: ConnectionStatus = 'idle';
@ -160,12 +173,22 @@ export abstract class Connection<T = any> {
}; };
} }
export class DummyConnection extends Connection<undefined> { export class DummyConnection implements Connection<undefined> {
doConnect() { readonly status: ConnectionStatus = 'connected';
return Promise.resolve(undefined); readonly inner: undefined;
}
doDisconnect() { connect(): void {
return; return;
} }
disconnect(): void {
return;
}
waitForConnected(_signal?: AbortSignal): Promise<void> {
return Promise.resolve();
}
onStatusChanged(
_cb: (status: ConnectionStatus, error?: Error) => void
): () => void {
return () => {};
}
} }

View File

@ -1,7 +1,7 @@
import type { Connection } from './connection'; import type { AutoReconnectConnection } from './connection';
const CONNECTIONS: Map<string, Connection<any>> = new Map(); const CONNECTIONS: Map<string, AutoReconnectConnection<any>> = new Map();
export function share<T extends Connection<any>>(conn: T): T { export function share<T extends AutoReconnectConnection<any>>(conn: T): T {
if (!conn.shareId) { if (!conn.shareId) {
throw new Error( throw new Error(
`Connection ${conn.constructor.name} is not shareable.\nIf you want to make it shareable, please override [shareId].` `Connection ${conn.constructor.name} is not shareable.\nIf you want to make it shareable, please override [shareId].`

View File

@ -51,10 +51,10 @@ export class AwarenessFrontend {
applyAwarenessUpdate(awareness, update.bin, origin); applyAwarenessUpdate(awareness, update.bin, origin);
}; };
const handleSyncCollect = () => { const handleSyncCollect = () => {
return { return Promise.resolve({
docId: awareness.doc.guid, docId: awareness.doc.guid,
bin: encodeAwarenessUpdate(awareness, [awareness.clientID]), bin: encodeAwarenessUpdate(awareness, [awareness.clientID]),
}; });
}; };
const unsubscribe = this.sync.subscribeUpdate( const unsubscribe = this.sync.subscribeUpdate(
awareness.doc.guid, awareness.doc.guid,

View File

@ -17,7 +17,7 @@ export class BlobFrontend {
return this.sync ? this.sync.uploadBlob(blob) : this.storage.set(blob); return this.sync ? this.sync.uploadBlob(blob) : this.storage.set(blob);
} }
addPriority(id: string, priority: number) { addPriority(_id: string, _priority: number) {
return this.sync?.addPriority(id, priority); // not support yet
} }
} }

View File

@ -37,7 +37,7 @@ interface DocFrontendOptions {
} }
export class DocFrontend { export class DocFrontend {
private readonly uniqueId = `frontend:${this.storage.peer}:${nanoid()}`; private readonly uniqueId = `frontend:${nanoid()}`;
private readonly prioritySettings = new Map<string, number>(); private readonly prioritySettings = new Map<string, number>();
@ -88,7 +88,6 @@ export class DocFrontend {
}), }),
]); ]);
// eslint-disable-next-line no-constant-condition
while (true) { while (true) {
throwIfAborted(signal); throwIfAborted(signal);
const docId = await this.status.jobDocQueue.asyncPop(signal); const docId = await this.status.jobDocQueue.asyncPop(signal);

View File

@ -1,9 +1,6 @@
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import { import { type AwarenessRecord, AwarenessStorageBase } from '../../storage';
type AwarenessRecord,
AwarenessStorage,
} from '../../storage/awareness';
import { BroadcastChannelConnection } from './channel'; import { BroadcastChannelConnection } from './channel';
type ChannelMessage = type ChannelMessage =
@ -19,13 +16,13 @@ type ChannelMessage =
collectId: string; collectId: string;
} }
| { | {
type: 'awareness-collect-fallback'; type: 'awareness-collect-feedback';
docId: string; docId: string;
bin: Uint8Array; bin: Uint8Array;
collectId: string; collectId: string;
}; };
export class BroadcastChannelAwarenessStorage extends AwarenessStorage { export class BroadcastChannelAwarenessStorage extends AwarenessStorageBase {
override readonly storageType = 'awareness'; override readonly storageType = 'awareness';
override readonly connection = new BroadcastChannelConnection(this.options); override readonly connection = new BroadcastChannelConnection(this.options);
get channel() { get channel() {
@ -36,7 +33,7 @@ export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
string, string,
Set<{ Set<{
onUpdate: (update: AwarenessRecord, origin?: string) => void; onUpdate: (update: AwarenessRecord, origin?: string) => void;
onCollect: () => AwarenessRecord; onCollect: () => Promise<AwarenessRecord | null>;
}> }>
>(); >();
@ -57,12 +54,20 @@ export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
override subscribeUpdate( override subscribeUpdate(
id: string, id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void, onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => AwarenessRecord onCollect: () => Promise<AwarenessRecord | null>
): () => void { ): () => void {
const subscribers = this.subscriptions.get(id) ?? new Set(); const subscribers = this.subscriptions.get(id) ?? new Set();
subscribers.forEach(subscriber => { subscribers.forEach(subscriber => {
const fallback = subscriber.onCollect(); subscriber
onUpdate(fallback); .onCollect()
.then(awareness => {
if (awareness) {
onUpdate(awareness);
}
})
.catch(error => {
console.error('error in on collect awareness', error);
});
}); });
const collectUniqueId = nanoid(); const collectUniqueId = nanoid();
@ -84,18 +89,23 @@ export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
message.data.type === 'awareness-collect' && message.data.type === 'awareness-collect' &&
message.data.docId === id message.data.docId === id
) { ) {
const fallback = onCollect(); onCollect()
if (fallback) { .then(awareness => {
if (awareness) {
this.channel.postMessage({ this.channel.postMessage({
type: 'awareness-collect-fallback', type: 'awareness-collect-feedback',
docId: message.data.docId, docId: message.data.docId,
bin: fallback.bin, bin: awareness.bin,
collectId: collectUniqueId, collectId: collectUniqueId,
} satisfies ChannelMessage); } satisfies ChannelMessage);
} }
})
.catch(error => {
console.error('error in on collect awareness', error);
});
} }
if ( if (
message.data.type === 'awareness-collect-fallback' && message.data.type === 'awareness-collect-feedback' &&
message.data.docId === id && message.data.docId === id &&
message.data.collectId === collectUniqueId message.data.collectId === collectUniqueId
) { ) {

View File

@ -1,7 +1,7 @@
import { Connection } from '../../connection'; import { AutoReconnectConnection } from '../../connection';
import type { StorageOptions } from '../../storage'; import type { StorageOptions } from '../../storage';
export class BroadcastChannelConnection extends Connection<BroadcastChannel> { export class BroadcastChannelConnection extends AutoReconnectConnection<BroadcastChannel> {
readonly channelName = `channel:${this.opts.peer}:${this.opts.type}:${this.opts.id}`; readonly channelName = `channel:${this.opts.peer}:${this.opts.type}:${this.opts.id}`;
constructor(private readonly opts: StorageOptions) { constructor(private readonly opts: StorageOptions) {

View File

@ -3,7 +3,7 @@ import type { SocketOptions } from 'socket.io-client';
import { share } from '../../connection'; import { share } from '../../connection';
import { import {
type AwarenessRecord, type AwarenessRecord,
AwarenessStorage, AwarenessStorageBase,
type AwarenessStorageOptions, type AwarenessStorageOptions,
} from '../../storage/awareness'; } from '../../storage/awareness';
import { import {
@ -16,7 +16,7 @@ interface CloudAwarenessStorageOptions extends AwarenessStorageOptions {
socketOptions: SocketOptions; socketOptions: SocketOptions;
} }
export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorageOptions> { export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessStorageOptions> {
connection = share( connection = share(
new SocketConnection(this.peer, this.options.socketOptions) new SocketConnection(this.peer, this.options.socketOptions)
); );
@ -38,7 +38,7 @@ export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorag
override subscribeUpdate( override subscribeUpdate(
id: string, id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void, onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => AwarenessRecord onCollect: () => Promise<AwarenessRecord | null>
): () => void { ): () => void {
// TODO: handle disconnect // TODO: handle disconnect
// leave awareness // leave awareness
@ -92,7 +92,8 @@ export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorag
docId === id docId === id
) { ) {
(async () => { (async () => {
const record = onCollect(); const record = await onCollect();
if (record) {
const encodedUpdate = await uint8ArrayToBase64(record.bin); const encodedUpdate = await uint8ArrayToBase64(record.bin);
this.socket.emit('space:update-awareness', { this.socket.emit('space:update-awareness', {
spaceType: this.spaceType, spaceType: this.spaceType,
@ -100,6 +101,7 @@ export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorag
docId: record.docId, docId: record.docId,
awarenessUpdate: encodedUpdate, awarenessUpdate: encodedUpdate,
}); });
}
})().catch(err => console.error('awareness upload failed', err)); })().catch(err => console.error('awareness upload failed', err));
} }
}; };

View File

@ -7,16 +7,31 @@ import {
} from '@affine/graphql'; } from '@affine/graphql';
import { DummyConnection } from '../../connection'; import { DummyConnection } from '../../connection';
import { type BlobRecord, BlobStorage } from '../../storage'; import {
type BlobRecord,
BlobStorageBase,
type BlobStorageOptions,
} from '../../storage';
export class CloudBlobStorage extends BlobStorage { interface CloudBlobStorageOptions extends BlobStorageOptions {
private readonly gql = gqlFetcherFactory(this.options.peer + '/graphql'); apiBaseUrl: string;
}
export class CloudBlobStorage extends BlobStorageBase<CloudBlobStorageOptions> {
private readonly gql = gqlFetcherFactory(
this.options.apiBaseUrl + '/graphql'
);
override connection = new DummyConnection(); override connection = new DummyConnection();
override async get(key: string) { override async get(key: string) {
const res = await fetch( const res = await fetch(
this.options.peer + '/api/workspaces/' + this.spaceId + '/blobs/' + key, this.options.apiBaseUrl +
'/api/workspaces/' +
this.spaceId +
'/blobs/' +
key,
{ {
cache: 'default',
headers: { headers: {
'x-affine-version': BUILD_CONFIG.appVersion, 'x-affine-version': BUILD_CONFIG.appVersion,
}, },

View File

@ -1,10 +1,14 @@
import type { SocketOptions } from 'socket.io-client'; import type { Socket, SocketOptions } from 'socket.io-client';
import { share } from '../../connection'; import {
type Connection,
type ConnectionStatus,
share,
} from '../../connection';
import { import {
type DocClock, type DocClock,
type DocClocks, type DocClocks,
DocStorage, DocStorageBase,
type DocStorageOptions, type DocStorageOptions,
type DocUpdate, type DocUpdate,
} from '../../storage'; } from '../../storage';
@ -17,63 +21,14 @@ import {
interface CloudDocStorageOptions extends DocStorageOptions { interface CloudDocStorageOptions extends DocStorageOptions {
socketOptions: SocketOptions; socketOptions: SocketOptions;
serverBaseUrl: string;
} }
export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> { export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
connection = share( get socket() {
new SocketConnection(this.peer, this.options.socketOptions)
);
private disposeConnectionStatusListener?: () => void;
private get socket() {
return this.connection.inner; return this.connection.inner;
} }
override connect() {
if (!this.disposeConnectionStatusListener) {
this.disposeConnectionStatusListener = this.connection.onStatusChanged(
status => {
if (status === 'connected') {
this.join().catch(err => {
console.error('doc storage join failed', err);
});
this.socket.on('space:broadcast-doc-update', this.onServerUpdate);
}
}
);
}
super.connect();
}
override disconnect() {
if (this.disposeConnectionStatusListener) {
this.disposeConnectionStatusListener();
}
this.socket.emit('space:leave', {
spaceType: this.spaceType,
spaceId: this.spaceId,
});
this.socket.off('space:broadcast-doc-update', this.onServerUpdate);
super.disconnect();
}
async join() {
try {
const res = await this.socket.emitWithAck('space:join', {
spaceType: this.spaceType,
spaceId: this.spaceId,
clientVersion: BUILD_CONFIG.appVersion,
});
if ('error' in res) {
this.connection.setStatus('closed', new Error(res.error.message));
}
} catch (e) {
this.connection.setStatus('error', e as Error);
}
}
onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => { onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {
if ( if (
this.spaceType === message.spaceType && this.spaceType === message.spaceType &&
@ -88,6 +43,11 @@ export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> {
} }
}; };
readonly connection = new CloudDocStorageConnection(
this.options,
this.onServerUpdate
);
override async getDocSnapshot(docId: string) { override async getDocSnapshot(docId: string) {
const response = await this.socket.emitWithAck('space:load-doc', { const response = await this.socket.emitWithAck('space:load-doc', {
spaceType: this.spaceType, spaceType: this.spaceType,
@ -207,3 +167,84 @@ export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> {
return 0; return 0;
} }
} }
class CloudDocStorageConnection implements Connection<Socket> {
connection = share(
new SocketConnection(
`${this.options.serverBaseUrl}/`,
this.options.socketOptions
)
);
private disposeConnectionStatusListener?: () => void;
private get socket() {
return this.connection.inner;
}
constructor(
private readonly options: CloudDocStorageOptions,
private readonly onServerUpdate: ServerEventsMap['space:broadcast-doc-update']
) {}
get status() {
return this.connection.status;
}
get inner() {
return this.connection.inner;
}
connect(): void {
if (!this.disposeConnectionStatusListener) {
this.disposeConnectionStatusListener = this.connection.onStatusChanged(
status => {
if (status === 'connected') {
this.join().catch(err => {
console.error('doc storage join failed', err);
});
this.socket.on('space:broadcast-doc-update', this.onServerUpdate);
}
}
);
}
return this.connection.connect();
}
async join() {
try {
const res = await this.socket.emitWithAck('space:join', {
spaceType: this.options.type,
spaceId: this.options.id,
clientVersion: BUILD_CONFIG.appVersion,
});
if ('error' in res) {
this.connection.setStatus('closed', new Error(res.error.message));
}
} catch (e) {
this.connection.setStatus('error', e as Error);
}
}
disconnect() {
if (this.disposeConnectionStatusListener) {
this.disposeConnectionStatusListener();
}
this.socket.emit('space:leave', {
spaceType: this.options.type,
spaceId: this.options.id,
});
this.socket.off('space:broadcast-doc-update', this.onServerUpdate);
this.connection.disconnect();
}
waitForConnected(signal?: AbortSignal): Promise<void> {
return this.connection.waitForConnected(signal);
}
onStatusChanged(
cb: (status: ConnectionStatus, error?: Error) => void
): () => void {
return this.connection.onStatusChanged(cb);
}
}

View File

@ -4,7 +4,10 @@ import {
type SocketOptions, type SocketOptions,
} from 'socket.io-client'; } from 'socket.io-client';
import { Connection, type ConnectionStatus } from '../../connection'; import {
AutoReconnectConnection,
type ConnectionStatus,
} from '../../connection';
// TODO(@forehalo): use [UserFriendlyError] // TODO(@forehalo): use [UserFriendlyError]
interface EventError { interface EventError {
@ -150,7 +153,7 @@ export function base64ToUint8Array(base64: string) {
return new Uint8Array(binaryArray); return new Uint8Array(binaryArray);
} }
export class SocketConnection extends Connection<Socket> { export class SocketConnection extends AutoReconnectConnection<Socket> {
manager = new SocketIOManager(this.endpoint, { manager = new SocketIOManager(this.endpoint, {
autoConnect: false, autoConnect: false,
transports: ['websocket'], transports: ['websocket'],

View File

@ -1,12 +1,12 @@
import { share } from '../../connection'; import { share } from '../../connection';
import { import {
type BlobRecord, type BlobRecord,
BlobStorage, BlobStorageBase,
type ListedBlobRecord, type ListedBlobRecord,
} from '../../storage'; } from '../../storage';
import { IDBConnection } from './db'; import { IDBConnection } from './db';
export class IndexedDBBlobStorage extends BlobStorage { export class IndexedDBBlobStorage extends BlobStorageBase {
readonly connection = share(new IDBConnection(this.options)); readonly connection = share(new IDBConnection(this.options));
get db() { get db() {

View File

@ -1,10 +1,10 @@
import { type IDBPDatabase, openDB } from 'idb'; import { type IDBPDatabase, openDB } from 'idb';
import { Connection } from '../../connection'; import { AutoReconnectConnection } from '../../connection';
import type { StorageOptions } from '../../storage'; import type { StorageOptions } from '../../storage';
import { type DocStorageSchema, migrator } from './schema'; import { type DocStorageSchema, migrator } from './schema';
export class IDBConnection extends Connection<{ export class IDBConnection extends AutoReconnectConnection<{
db: IDBPDatabase<DocStorageSchema>; db: IDBPDatabase<DocStorageSchema>;
channel: BroadcastChannel; channel: BroadcastChannel;
}> { }> {

View File

@ -2,7 +2,7 @@ import {
type DocClock, type DocClock,
type DocClocks, type DocClocks,
type DocRecord, type DocRecord,
DocStorage, DocStorageBase,
type DocStorageOptions, type DocStorageOptions,
type DocUpdate, type DocUpdate,
} from '../../storage'; } from '../../storage';
@ -15,7 +15,7 @@ interface ChannelMessage {
origin?: string; origin?: string;
} }
export class IndexedDBDocStorage extends DocStorage { export class IndexedDBDocStorage extends DocStorageBase {
readonly connection = new IDBConnection(this.options); readonly connection = new IDBConnection(this.options);
get db() { get db() {

View File

@ -1,7 +1,7 @@
import { share } from '../../connection'; import { share } from '../../connection';
import { type DocClock, type DocClocks, SyncStorage } from '../../storage'; import { BasicSyncStorage, type DocClock, type DocClocks } from '../../storage';
import { IDBConnection } from './db'; import { IDBConnection } from './db';
export class IndexedDBSyncStorage extends SyncStorage { export class IndexedDBSyncStorage extends BasicSyncStorage {
readonly connection = share(new IDBConnection(this.options)); readonly connection = share(new IDBConnection(this.options));
get db() { get db() {

View File

@ -1,11 +1,11 @@
import { share } from '../../../connection'; import { share } from '../../../connection';
import { BlobStorage, type ListedBlobRecord } from '../../../storage'; import { BlobStorageBase, type ListedBlobRecord } from '../../../storage';
import { BlobIDBConnection } from './db'; import { BlobIDBConnection } from './db';
/** /**
* @deprecated readonly * @deprecated readonly
*/ */
export class IndexedDBV1BlobStorage extends BlobStorage { export class IndexedDBV1BlobStorage extends BlobStorageBase {
readonly connection = share(new BlobIDBConnection(this.spaceId)); readonly connection = share(new BlobIDBConnection(this.spaceId));
get db() { get db() {

View File

@ -1,6 +1,6 @@
import { type DBSchema, type IDBPDatabase, openDB } from 'idb'; import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
import { Connection } from '../../../connection'; import { AutoReconnectConnection } from '../../../connection';
export interface DocDBSchema extends DBSchema { export interface DocDBSchema extends DBSchema {
workspace: { workspace: {
@ -15,7 +15,9 @@ export interface DocDBSchema extends DBSchema {
}; };
} }
export class DocIDBConnection extends Connection<IDBPDatabase<DocDBSchema>> { export class DocIDBConnection extends AutoReconnectConnection<
IDBPDatabase<DocDBSchema>
> {
override get shareId() { override get shareId() {
return 'idb(old):affine-local'; return 'idb(old):affine-local';
} }
@ -40,7 +42,9 @@ export interface BlobDBSchema extends DBSchema {
}; };
} }
export class BlobIDBConnection extends Connection<IDBPDatabase<BlobDBSchema>> { export class BlobIDBConnection extends AutoReconnectConnection<
IDBPDatabase<BlobDBSchema>
> {
constructor(private readonly workspaceId: string) { constructor(private readonly workspaceId: string) {
super(); super();
} }

View File

@ -1,11 +1,15 @@
import { share } from '../../../connection'; import { share } from '../../../connection';
import { type DocRecord, DocStorage, type DocUpdate } from '../../../storage'; import {
type DocRecord,
DocStorageBase,
type DocUpdate,
} from '../../../storage';
import { DocIDBConnection } from './db'; import { DocIDBConnection } from './db';
/** /**
* @deprecated readonly * @deprecated readonly
*/ */
export class IndexedDBV1DocStorage extends DocStorage { export class IndexedDBV1DocStorage extends DocStorageBase {
readonly connection = share(new DocIDBConnection()); readonly connection = share(new DocIDBConnection());
get db() { get db() {

View File

@ -1,5 +1,10 @@
import type { Storage } from '../storage'; import type { Storage } from '../storage';
import { CloudBlobStorage, CloudDocStorage } from './cloud'; import { BroadcastChannelAwarenessStorage } from './broadcast-channel/awareness';
import {
CloudAwarenessStorage,
CloudBlobStorage,
CloudDocStorage,
} from './cloud';
import { import {
IndexedDBBlobStorage, IndexedDBBlobStorage,
IndexedDBDocStorage, IndexedDBDocStorage,
@ -13,6 +18,7 @@ const idb: StorageConstructor[] = [
IndexedDBDocStorage, IndexedDBDocStorage,
IndexedDBBlobStorage, IndexedDBBlobStorage,
IndexedDBSyncStorage, IndexedDBSyncStorage,
BroadcastChannelAwarenessStorage,
]; ];
const idbv1: StorageConstructor[] = [ const idbv1: StorageConstructor[] = [
@ -20,7 +26,11 @@ const idbv1: StorageConstructor[] = [
IndexedDBV1BlobStorage, IndexedDBV1BlobStorage,
]; ];
const cloud: StorageConstructor[] = [CloudDocStorage, CloudBlobStorage]; const cloud: StorageConstructor[] = [
CloudDocStorage,
CloudBlobStorage,
CloudAwarenessStorage,
];
export const storages: StorageConstructor[] = cloud.concat(idbv1, idb); export const storages: StorageConstructor[] = cloud.concat(idbv1, idb);

View File

@ -1,8 +1,8 @@
import { share } from '../../connection'; import { share } from '../../connection';
import { type BlobRecord, BlobStorage } from '../../storage'; import { type BlobRecord, BlobStorageBase } from '../../storage';
import { NativeDBConnection } from './db'; import { NativeDBConnection } from './db';
export class SqliteBlobStorage extends BlobStorage { export class SqliteBlobStorage extends BlobStorageBase {
override connection = share( override connection = share(
new NativeDBConnection(this.peer, this.spaceType, this.spaceId) new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
); );

View File

@ -1,6 +1,6 @@
import { apis } from '@affine/electron-api'; import { apis } from '@affine/electron-api';
import { Connection } from '../../connection'; import { AutoReconnectConnection } from '../../connection';
import { type SpaceType, universalId } from '../../storage'; import { type SpaceType, universalId } from '../../storage';
type NativeDBApis = NonNullable<typeof apis>['nbstore'] extends infer APIs type NativeDBApis = NonNullable<typeof apis>['nbstore'] extends infer APIs
@ -13,7 +13,7 @@ type NativeDBApis = NonNullable<typeof apis>['nbstore'] extends infer APIs
} }
: never; : never;
export class NativeDBConnection extends Connection<void> { export class NativeDBConnection extends AutoReconnectConnection<void> {
readonly apis: NativeDBApis; readonly apis: NativeDBApis;
constructor( constructor(

View File

@ -1,8 +1,8 @@
import { share } from '../../connection'; import { share } from '../../connection';
import { type DocClock, DocStorage, type DocUpdate } from '../../storage'; import { type DocClock, DocStorageBase, type DocUpdate } from '../../storage';
import { NativeDBConnection } from './db'; import { NativeDBConnection } from './db';
export class SqliteDocStorage extends DocStorage { export class SqliteDocStorage extends DocStorageBase {
override connection = share( override connection = share(
new NativeDBConnection(this.peer, this.spaceType, this.spaceId) new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
); );

View File

@ -1,8 +1,8 @@
import { share } from '../../connection'; import { share } from '../../connection';
import { type DocClock, SyncStorage } from '../../storage'; import { BasicSyncStorage, type DocClock } from '../../storage';
import { NativeDBConnection } from './db'; import { NativeDBConnection } from './db';
export class SqliteSyncStorage extends SyncStorage { export class SqliteSyncStorage extends BasicSyncStorage {
override connection = share( override connection = share(
new NativeDBConnection(this.peer, this.spaceType, this.spaceId) new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
); );

View File

@ -1,13 +1,13 @@
import { apis } from '@affine/electron-api'; import { apis } from '@affine/electron-api';
import { DummyConnection, share } from '../../../connection'; import { DummyConnection } from '../../../connection';
import { BlobStorage } from '../../../storage'; import { BlobStorageBase } from '../../../storage';
/** /**
* @deprecated readonly * @deprecated readonly
*/ */
export class SqliteV1BlobStorage extends BlobStorage { export class SqliteV1BlobStorage extends BlobStorageBase {
override connection = share(new DummyConnection()); override connection = new DummyConnection();
get db() { get db() {
if (!apis) { if (!apis) {

View File

@ -1,13 +1,17 @@
import { apis } from '@affine/electron-api'; import { apis } from '@affine/electron-api';
import { DummyConnection, share } from '../../../connection'; import { DummyConnection } from '../../../connection';
import { type DocRecord, DocStorage, type DocUpdate } from '../../../storage'; import {
type DocRecord,
DocStorageBase,
type DocUpdate,
} from '../../../storage';
/** /**
* @deprecated readonly * @deprecated readonly
*/ */
export class SqliteV1DocStorage extends DocStorage { export class SqliteV1DocStorage extends DocStorageBase {
override connection = share(new DummyConnection()); override connection = new DummyConnection();
get db() { get db() {
if (!apis) { if (!apis) {

View File

@ -1,128 +0,0 @@
import type { OpConsumer } from '@toeverything/infra/op';
import { Observable } from 'rxjs';
import { getAvailableStorageImplementations } from '../impls';
import {
BlobStorage,
DocStorage,
HistoricalDocStorage,
SpaceStorage,
type Storage,
type StorageOptions,
SyncStorage,
} from '../storage';
import type { SpaceStorageOps } from './ops';
export class SpaceStorageConsumer extends SpaceStorage {
constructor(private readonly consumer: OpConsumer<SpaceStorageOps>) {
super([]);
this.registerConnectionHandlers();
this.listen();
}
listen() {
this.consumer.listen();
}
add(name: string, options: StorageOptions) {
const Storage = getAvailableStorageImplementations(name);
const storage = new Storage(options);
this.storages.set(storage.storageType, storage);
this.registerStorageHandlers(storage);
}
override async destroy() {
await super.destroy();
this.consumer.destroy();
}
private registerConnectionHandlers() {
this.consumer.register('addStorage', ({ name, opts }) => {
this.add(name, opts);
});
this.consumer.register('connect', this.connect.bind(this));
this.consumer.register('disconnect', this.disconnect.bind(this));
this.consumer.register('destroy', this.destroy.bind(this));
}
private registerStorageHandlers(storage: Storage) {
if (storage instanceof DocStorage) {
this.registerDocHandlers(storage);
} else if (storage instanceof BlobStorage) {
this.registerBlobHandlers(storage);
} else if (storage instanceof SyncStorage) {
this.registerSyncHandlers(storage);
}
}
private registerDocHandlers(storage: DocStorage) {
this.consumer.register('getDoc', storage.getDoc.bind(storage));
this.consumer.register('getDocDiff', ({ docId, state }) => {
return storage.getDocDiff(docId, state);
});
this.consumer.register('pushDocUpdate', ({ update, origin }) => {
return storage.pushDocUpdate(update, origin);
});
this.consumer.register(
'getDocTimestamps',
storage.getDocTimestamps.bind(storage)
);
this.consumer.register('deleteDoc', storage.deleteDoc.bind(storage));
this.consumer.register('subscribeDocUpdate', () => {
return new Observable(subscriber => {
subscriber.add(
storage.subscribeDocUpdate((update, origin) => {
subscriber.next({ update, origin });
})
);
});
});
if (storage instanceof HistoricalDocStorage) {
this.consumer.register('listHistory', ({ docId, filter }) => {
return storage.listHistories(docId, filter);
});
this.consumer.register('getHistory', ({ docId, timestamp }) => {
return storage.getHistory(docId, timestamp);
});
this.consumer.register('deleteHistory', ({ docId, timestamp }) => {
return storage.deleteHistory(docId, timestamp);
});
this.consumer.register('rollbackDoc', ({ docId, timestamp }) => {
return storage.rollbackDoc(docId, timestamp);
});
}
}
private registerBlobHandlers(storage: BlobStorage) {
this.consumer.register('getBlob', key => {
return storage.get(key);
});
this.consumer.register('setBlob', blob => {
return storage.set(blob);
});
this.consumer.register('deleteBlob', ({ key, permanently }) => {
return storage.delete(key, permanently);
});
this.consumer.register('listBlobs', storage.list.bind(storage));
this.consumer.register('releaseBlobs', storage.release.bind(storage));
}
private registerSyncHandlers(storage: SyncStorage) {
this.consumer.register(
'getPeerClocks',
storage.getPeerRemoteClocks.bind(storage)
);
this.consumer.register('setPeerClock', ({ peer, ...clock }) => {
return storage.setPeerRemoteClock(peer, clock);
});
this.consumer.register(
'getPeerPushedClocks',
storage.getPeerPushedClocks.bind(storage)
);
this.consumer.register('setPeerPushedClock', ({ peer, ...clock }) => {
return storage.setPeerPushedClock(peer, clock);
});
this.consumer.register('clearClocks', storage.clearClocks.bind(storage));
}
}

View File

@ -1,53 +0,0 @@
import { OpClient } from '@toeverything/infra/op';
import type { Storage } from '../storage';
import type { SpaceStorageOps } from './ops';
export { SpaceStorageConsumer } from './consumer';
export class SpaceStorageClient extends OpClient<SpaceStorageOps> {
/**
* Adding a storage implementation to the backend.
*
* NOTE:
* Because the storage beckend might be put behind a worker, we cant pass the instance but only
* the constructor name and its options to let the backend construct the instance.
*/
async addStorage<T extends new (...args: any) => Storage>(
Impl: T,
...opts: ConstructorParameters<T>
) {
await this.call('addStorage', { name: Impl.name, opts: opts[0] });
}
async connect() {
await this.call('connect');
}
async disconnect() {
await this.call('disconnect');
}
override destroy() {
this.call('destroy').catch(console.error);
super.destroy();
}
connection$() {
return this.ob$('connection');
}
}
export class SpaceStorageWorkerClient extends SpaceStorageClient {
private readonly worker: Worker;
constructor() {
const worker = new Worker(new URL('./worker.ts', import.meta.url));
super(worker);
this.worker = worker;
}
override destroy() {
super.destroy();
this.worker.terminate();
}
}

View File

@ -1,58 +0,0 @@
import { type OpSchema } from '@toeverything/infra/op';
import type { ConnectionStatus } from '../connection';
import type {
BlobRecord,
DocClock,
DocClocks,
DocDiff,
DocRecord,
DocUpdate,
HistoryFilter,
ListedBlobRecord,
ListedHistory,
StorageOptions,
StorageType,
} from '../storage';
export interface SpaceStorageOps extends OpSchema {
// init
addStorage: [{ name: string; opts: StorageOptions }, void];
// connection
connect: [void, void];
disconnect: [void, void];
connection: [
void,
{ storage: StorageType; status: ConnectionStatus; error?: Error },
];
destroy: [void, void];
// doc
getDoc: [string, DocRecord | null];
getDocDiff: [{ docId: string; state?: Uint8Array }, DocDiff | null];
pushDocUpdate: [{ update: DocUpdate; origin?: string }, DocClock];
getDocTimestamps: [Date, DocClocks];
deleteDoc: [string, void];
subscribeDocUpdate: [void, { update: DocRecord; origin?: string }];
// history
listHistory: [{ docId: string; filter?: HistoryFilter }, ListedHistory[]];
getHistory: [DocClock, DocRecord | null];
deleteHistory: [DocClock, void];
rollbackDoc: [DocClock & { editor?: string }, void];
// blob
getBlob: [string, BlobRecord | null];
setBlob: [BlobRecord, void];
deleteBlob: [{ key: string; permanently: boolean }, void];
releaseBlobs: [void, void];
listBlobs: [void, ListedBlobRecord[]];
// sync
getPeerClocks: [string, DocClocks];
setPeerClock: [{ peer: string } & DocClock, void];
getPeerPushedClocks: [string, DocClocks];
setPeerPushedClock: [{ peer: string } & DocClock, void];
clearClocks: [void, void];
}

View File

@ -1,11 +0,0 @@
import { OpConsumer } from '@toeverything/infra/op';
import { SpaceStorageConsumer } from './consumer';
import type { SpaceStorageOps } from './ops';
const consumer = new SpaceStorageConsumer(
// @ts-expect-error safe
new OpConsumer<SpaceStorageOps>(self)
);
consumer.listen();

View File

@ -1,4 +1,4 @@
import { Storage, type StorageOptions } from './storage'; import { type Storage, StorageBase, type StorageOptions } from './storage';
export interface AwarenessStorageOptions extends StorageOptions {} export interface AwarenessStorageOptions extends StorageOptions {}
@ -7,21 +7,35 @@ export type AwarenessRecord = {
bin: Uint8Array; bin: Uint8Array;
}; };
export abstract class AwarenessStorage< export interface AwarenessStorage extends Storage {
Options extends AwarenessStorageOptions = AwarenessStorageOptions, readonly storageType: 'awareness';
> extends Storage<Options> {
override readonly storageType = 'awareness';
/** /**
* Update the awareness record. * Update the awareness record.
* *
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred. * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
*/ */
update(record: AwarenessRecord, origin?: string): Promise<void>;
subscribeUpdate(
id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => Promise<AwarenessRecord | null>
): () => void;
}
export abstract class AwarenessStorageBase<
Options extends AwarenessStorageOptions = AwarenessStorageOptions,
>
extends StorageBase<Options>
implements AwarenessStorage
{
override readonly storageType = 'awareness';
abstract update(record: AwarenessRecord, origin?: string): Promise<void>; abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
abstract subscribeUpdate( abstract subscribeUpdate(
id: string, id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void, onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => AwarenessRecord onCollect: () => Promise<AwarenessRecord | null>
): () => void; ): () => void;
} }

View File

@ -1,4 +1,4 @@
import { Storage, type StorageOptions } from './storage'; import { type Storage, StorageBase, type StorageOptions } from './storage';
export interface BlobStorageOptions extends StorageOptions {} export interface BlobStorageOptions extends StorageOptions {}
@ -16,9 +16,25 @@ export interface ListedBlobRecord {
createdAt?: Date; createdAt?: Date;
} }
export abstract class BlobStorage< export interface BlobStorage extends Storage {
readonly storageType: 'blob';
get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;
set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
delete(
key: string,
permanently: boolean,
signal?: AbortSignal
): Promise<void>;
release(signal?: AbortSignal): Promise<void>;
list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
}
export abstract class BlobStorageBase<
Options extends BlobStorageOptions = BlobStorageOptions, Options extends BlobStorageOptions = BlobStorageOptions,
> extends Storage<Options> { >
extends StorageBase<Options>
implements BlobStorage
{
override readonly storageType = 'blob'; override readonly storageType = 'blob';
abstract get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>; abstract get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;

View File

@ -4,7 +4,7 @@ import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs';
import { isEmptyUpdate } from '../utils/is-empty-update'; import { isEmptyUpdate } from '../utils/is-empty-update';
import type { Locker } from './lock'; import type { Locker } from './lock';
import { SingletonLocker } from './lock'; import { SingletonLocker } from './lock';
import { Storage, type StorageOptions } from './storage'; import { type Storage, StorageBase, type StorageOptions } from './storage';
export interface DocClock { export interface DocClock {
docId: string; docId: string;
@ -37,17 +37,67 @@ export interface DocStorageOptions extends StorageOptions {
mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array; mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
} }
export abstract class DocStorage< export interface DocStorage extends Storage {
readonly storageType: 'doc';
/**
* Get a doc record with latest binary.
*/
getDoc(docId: string): Promise<DocRecord | null>;
/**
* Get a yjs binary diff with the given state vector.
*/
getDocDiff(docId: string, state?: Uint8Array): Promise<DocDiff | null>;
/**
* Push updates into storage
*
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
*/
pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
/**
* Get the timestamp of the latest update of a doc.
*/
getDocTimestamp(docId: string): Promise<DocClock | null>;
/**
* Get all docs timestamps info. especially for useful in sync process.
*/
getDocTimestamps(after?: Date): Promise<DocClocks>;
/**
* Delete a specific doc data with all snapshots and updates
*/
deleteDoc(docId: string): Promise<void>;
/**
* Subscribe on doc updates emitted from storage itself.
*
* NOTE:
*
* There is not always update emitted from storage itself.
*
* For example, in Sqlite storage, the update will only come from user's updating on docs,
* in other words, the update will never somehow auto generated in storage internally.
*
* But for Cloud storage, there will be updates broadcasted from other clients,
* so the storage will emit updates to notify the client to integrate them.
*/
subscribeDocUpdate(
callback: (update: DocRecord, origin?: string) => void
): () => void;
}
export abstract class DocStorageBase<
Opts extends DocStorageOptions = DocStorageOptions, Opts extends DocStorageOptions = DocStorageOptions,
> extends Storage<Opts> { >
extends StorageBase<Opts>
implements DocStorage
{
private readonly event = new EventEmitter2(); private readonly event = new EventEmitter2();
override readonly storageType = 'doc'; override readonly storageType = 'doc';
protected readonly locker: Locker = new SingletonLocker(); protected readonly locker: Locker = new SingletonLocker();
// REGION: open apis by Op system
/**
* Get a doc record with latest binary.
*/
async getDoc(docId: string) { async getDoc(docId: string) {
await using _lock = await this.lockDocForUpdate(docId); await using _lock = await this.lockDocForUpdate(docId);
@ -78,9 +128,6 @@ export abstract class DocStorage<
return snapshot; return snapshot;
} }
/**
* Get a yjs binary diff with the given state vector.
*/
async getDocDiff(docId: string, state?: Uint8Array) { async getDocDiff(docId: string, state?: Uint8Array) {
const doc = await this.getDoc(docId); const doc = await this.getDoc(docId);
@ -96,41 +143,14 @@ export abstract class DocStorage<
}; };
} }
/**
* Push updates into storage
*
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
*/
abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>; abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
/**
* Get the timestamp of the latest update of a doc.
*/
abstract getDocTimestamp(docId: string): Promise<DocClock | null>; abstract getDocTimestamp(docId: string): Promise<DocClock | null>;
/**
* Get all docs timestamps info. especially for useful in sync process.
*/
abstract getDocTimestamps(after?: Date): Promise<DocClocks>; abstract getDocTimestamps(after?: Date): Promise<DocClocks>;
/**
* Delete a specific doc data with all snapshots and updates
*/
abstract deleteDoc(docId: string): Promise<void>; abstract deleteDoc(docId: string): Promise<void>;
/**
* Subscribe on doc updates emitted from storage itself.
*
* NOTE:
*
* There is not always update emitted from storage itself.
*
* For example, in Sqlite storage, the update will only come from user's updating on docs,
* in other words, the update will never somehow auto generated in storage internally.
*
* But for Cloud storage, there will be updates broadcasted from other clients,
* so the storage will emit updates to notify the client to integrate them.
*/
subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) { subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
this.event.on('update', callback); this.event.on('update', callback);
@ -138,7 +158,6 @@ export abstract class DocStorage<
this.event.off('update', callback); this.event.off('update', callback);
}; };
} }
// ENDREGION
// REGION: api for internal usage // REGION: api for internal usage
protected on( protected on(

View File

@ -7,7 +7,7 @@ import {
UndoManager, UndoManager,
} from 'yjs'; } from 'yjs';
import { type DocRecord, DocStorage, type DocStorageOptions } from './doc'; import { type DocRecord, DocStorageBase, type DocStorageOptions } from './doc';
export interface HistoryFilter { export interface HistoryFilter {
before?: Date; before?: Date;
@ -21,7 +21,7 @@ export interface ListedHistory {
export abstract class HistoricalDocStorage< export abstract class HistoricalDocStorage<
Options extends DocStorageOptions = DocStorageOptions, Options extends DocStorageOptions = DocStorageOptions,
> extends DocStorage<Options> { > extends DocStorageBase<Options> {
constructor(opts: Options) { constructor(opts: Options) {
super(opts); super(opts);

View File

@ -40,20 +40,20 @@ export class SpaceStorage {
connect() { connect() {
Array.from(this.storages.values()).forEach(storage => { Array.from(this.storages.values()).forEach(storage => {
storage.connect(); storage.connection.connect();
}); });
} }
disconnect() { disconnect() {
Array.from(this.storages.values()).forEach(storage => { Array.from(this.storages.values()).forEach(storage => {
storage.disconnect(); storage.connection.disconnect();
}); });
} }
async waitForConnected() { async waitForConnected(signal?: AbortSignal) {
await Promise.all( await Promise.all(
Array.from(this.storages.values()).map(storage => Array.from(this.storages.values()).map(storage =>
storage.waitForConnected() storage.connection.waitForConnected(signal)
) )
); );
} }
@ -65,6 +65,7 @@ export class SpaceStorage {
} }
} }
export * from './awareness';
export * from './blob'; export * from './blob';
export * from './doc'; export * from './doc';
export * from './history'; export * from './history';

View File

@ -80,7 +80,18 @@ export function parseUniversalId(id: string) {
return result as any; return result as any;
} }
export abstract class Storage<Opts extends StorageOptions = StorageOptions> { export interface Storage {
readonly storageType: StorageType;
readonly connection: Connection;
readonly peer: string;
readonly spaceType: string;
readonly spaceId: string;
readonly universalId: string;
}
export abstract class StorageBase<Opts extends StorageOptions = StorageOptions>
implements Storage
{
abstract readonly storageType: StorageType; abstract readonly storageType: StorageType;
abstract readonly connection: Connection; abstract readonly connection: Connection;
@ -101,16 +112,4 @@ export abstract class Storage<Opts extends StorageOptions = StorageOptions> {
} }
constructor(public readonly options: Opts) {} constructor(public readonly options: Opts) {}
connect() {
this.connection.connect();
}
disconnect() {
this.connection.disconnect();
}
async waitForConnected() {
await this.connection.waitForConnected();
}
} }

View File

@ -1,11 +1,32 @@
import type { DocClock, DocClocks } from './doc'; import type { DocClock, DocClocks } from './doc';
import { Storage, type StorageOptions } from './storage'; import { type Storage, StorageBase, type StorageOptions } from './storage';
export interface SyncStorageOptions extends StorageOptions {} export interface SyncStorageOptions extends StorageOptions {}
export abstract class SyncStorage< export interface SyncStorage extends Storage {
readonly storageType: 'sync';
getPeerRemoteClock(peer: string, docId: string): Promise<DocClock | null>;
getPeerRemoteClocks(peer: string): Promise<DocClocks>;
setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
getPeerPulledRemoteClock(
peer: string,
docId: string
): Promise<DocClock | null>;
getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
setPeerPulledRemoteClock(peer: string, clock: DocClock): Promise<void>;
getPeerPushedClock(peer: string, docId: string): Promise<DocClock | null>;
getPeerPushedClocks(peer: string): Promise<DocClocks>;
setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
clearClocks(): Promise<void>;
}
export abstract class BasicSyncStorage<
Opts extends SyncStorageOptions = SyncStorageOptions, Opts extends SyncStorageOptions = SyncStorageOptions,
> extends Storage<Opts> { >
extends StorageBase<Opts>
implements SyncStorage
{
override readonly storageType = 'sync'; override readonly storageType = 'sync';
abstract getPeerRemoteClock( abstract getPeerRemoteClock(

View File

@ -3,7 +3,16 @@ import type {
AwarenessStorage, AwarenessStorage,
} from '../../storage/awareness'; } from '../../storage/awareness';
export class AwarenessSync { export interface AwarenessSync {
update(record: AwarenessRecord, origin?: string): Promise<void>;
subscribeUpdate(
id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => Promise<AwarenessRecord | null>
): () => void;
}
export class AwarenessSyncImpl implements AwarenessSync {
constructor( constructor(
readonly local: AwarenessStorage, readonly local: AwarenessStorage,
readonly remotes: AwarenessStorage[] readonly remotes: AwarenessStorage[]
@ -18,7 +27,7 @@ export class AwarenessSync {
subscribeUpdate( subscribeUpdate(
id: string, id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void, onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => AwarenessRecord onCollect: () => Promise<AwarenessRecord | null>
): () => void { ): () => void {
const unsubscribes = [this.local, ...this.remotes].map(peer => const unsubscribes = [this.local, ...this.remotes].map(peer =>
peer.subscribeUpdate(id, onUpdate, onCollect) peer.subscribeUpdate(id, onUpdate, onCollect)

View File

@ -3,7 +3,15 @@ import { difference } from 'lodash-es';
import type { BlobRecord, BlobStorage } from '../../storage'; import type { BlobRecord, BlobStorage } from '../../storage';
import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted'; import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
export class BlobSync { export interface BlobSync {
downloadBlob(
blobId: string,
signal?: AbortSignal
): Promise<BlobRecord | null>;
uploadBlob(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
}
export class BlobSyncImpl implements BlobSync {
private abort: AbortController | null = null; private abort: AbortController | null = null;
constructor( constructor(

View File

@ -17,7 +17,13 @@ export interface DocSyncDocState {
errorMessage: string | null; errorMessage: string | null;
} }
export class DocSync { export interface DocSync {
readonly state$: Observable<DocSyncState>;
docState$(docId: string): Observable<DocSyncDocState>;
addPriority(id: string, priority: number): () => void;
}
export class DocSyncImpl implements DocSync {
private readonly peers: DocSyncPeer[] = this.remotes.map( private readonly peers: DocSyncPeer[] = this.remotes.map(
remote => new DocSyncPeer(this.local, this.sync, remote) remote => new DocSyncPeer(this.local, this.sync, remote)
); );

View File

@ -92,7 +92,7 @@ export class DocSyncPeer {
/** /**
* random unique id for recognize self in "update" event * random unique id for recognize self in "update" event
*/ */
private readonly uniqueId = `sync:${this.local.peer}:${this.remote.peer}:${nanoid()}`; private readonly uniqueId = `sync:${this.local.universalId}:${this.remote.universalId}:${nanoid()}`;
private readonly prioritySettings = new Map<string, number>(); private readonly prioritySettings = new Map<string, number>();
constructor( constructor(
@ -435,7 +435,6 @@ export class DocSyncPeer {
}; };
async mainLoop(signal?: AbortSignal) { async mainLoop(signal?: AbortSignal) {
// eslint-disable-next-line no-constant-condition
while (true) { while (true) {
try { try {
await this.retryLoop(signal); await this.retryLoop(signal);
@ -594,12 +593,12 @@ export class DocSyncPeer {
} }
// begin to process jobs // begin to process jobs
// eslint-disable-next-line no-constant-condition
while (true) { while (true) {
throwIfAborted(signal); throwIfAborted(signal);
const docId = await this.status.jobDocQueue.asyncPop(signal); const docId = await this.status.jobDocQueue.asyncPop(signal);
// eslint-disable-next-line no-constant-condition
while (true) { while (true) {
// batch process jobs for the same doc // batch process jobs for the same doc
const jobs = this.status.jobMap.get(docId); const jobs = this.status.jobMap.get(docId);

View File

@ -1,19 +1,23 @@
import { combineLatest, map, type Observable, of } from 'rxjs'; import { combineLatest, map, type Observable, of } from 'rxjs';
import type { BlobStorage, DocStorage, SpaceStorage } from '../storage'; import type {
import type { AwarenessStorage } from '../storage/awareness'; AwarenessStorage,
import { AwarenessSync } from './awareness'; BlobStorage,
import { BlobSync } from './blob'; DocStorage,
import { DocSync, type DocSyncState } from './doc'; SpaceStorage,
} from '../storage';
import { AwarenessSyncImpl } from './awareness';
import { BlobSyncImpl } from './blob';
import { DocSyncImpl, type DocSyncState } from './doc';
export interface SyncState { export interface SyncState {
doc?: DocSyncState; doc?: DocSyncState;
} }
export class Sync { export class Sync {
readonly doc: DocSync | null; readonly doc: DocSyncImpl | null;
readonly blob: BlobSync | null; readonly blob: BlobSyncImpl | null;
readonly awareness: AwarenessSync | null; readonly awareness: AwarenessSyncImpl | null;
readonly state$: Observable<SyncState>; readonly state$: Observable<SyncState>;
@ -28,7 +32,7 @@ export class Sync {
this.doc = this.doc =
doc && sync doc && sync
? new DocSync( ? new DocSyncImpl(
doc, doc,
sync, sync,
peers peers
@ -37,7 +41,7 @@ export class Sync {
) )
: null; : null;
this.blob = blob this.blob = blob
? new BlobSync( ? new BlobSyncImpl(
blob, blob,
peers peers
.map(peer => peer.tryGet('blob')) .map(peer => peer.tryGet('blob'))
@ -45,7 +49,7 @@ export class Sync {
) )
: null; : null;
this.awareness = awareness this.awareness = awareness
? new AwarenessSync( ? new AwarenessSyncImpl(
awareness, awareness,
peers peers
.map(peer => peer.tryGet('awareness')) .map(peer => peer.tryGet('awareness'))

View File

@ -0,0 +1,294 @@
import type { OpClient } from '@toeverything/infra/op';
import { DummyConnection } from '../connection';
import { DocFrontend } from '../frontend/doc';
import {
type AwarenessRecord,
type AwarenessStorage,
type BlobRecord,
type BlobStorage,
type DocRecord,
type DocStorage,
type DocUpdate,
type ListedBlobRecord,
type StorageOptions,
universalId,
} from '../storage';
import type { AwarenessSync } from '../sync/awareness';
import type { BlobSync } from '../sync/blob';
import type { DocSync } from '../sync/doc';
import type { WorkerOps } from './ops';
export class WorkerClient {
constructor(
private readonly client: OpClient<WorkerOps>,
private readonly options: StorageOptions
) {}
readonly docStorage = new WorkerDocStorage(this.client, this.options);
readonly blobStorage = new WorkerBlobStorage(this.client, this.options);
readonly awarenessStorage = new WorkerAwarenessStorage(
this.client,
this.options
);
readonly docSync = new WorkerDocSync(this.client);
readonly blobSync = new WorkerBlobSync(this.client);
readonly awarenessSync = new WorkerAwarenessSync(this.client);
readonly docFrontend = new DocFrontend(this.docStorage, this.docSync);
}
class WorkerDocStorage implements DocStorage {
constructor(
private readonly client: OpClient<WorkerOps>,
private readonly options: StorageOptions
) {}
readonly peer = this.options.peer;
readonly spaceType = this.options.type;
readonly spaceId = this.options.id;
readonly universalId = universalId(this.options);
readonly storageType = 'doc';
async getDoc(docId: string) {
return this.client.call('docStorage.getDoc', docId);
}
async getDocDiff(docId: string, state?: Uint8Array) {
return this.client.call('docStorage.getDocDiff', { docId, state });
}
async pushDocUpdate(update: DocUpdate, origin?: string) {
return this.client.call('docStorage.pushDocUpdate', { update, origin });
}
async getDocTimestamp(docId: string) {
return this.client.call('docStorage.getDocTimestamp', docId);
}
async getDocTimestamps(after?: Date) {
return this.client.call('docStorage.getDocTimestamps', after ?? null);
}
async deleteDoc(docId: string) {
return this.client.call('docStorage.deleteDoc', docId);
}
subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
const subscription = this.client
.ob$('docStorage.subscribeDocUpdate')
.subscribe(value => {
callback(value.update, value.origin);
});
return () => {
subscription.unsubscribe();
};
}
connection = new WorkerDocConnection(this.client);
}
class WorkerDocConnection extends DummyConnection {
constructor(private readonly client: OpClient<WorkerOps>) {
super();
}
override waitForConnected(signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const abortListener = () => {
reject(signal?.reason);
subscription.unsubscribe();
};
signal?.addEventListener('abort', abortListener);
const subscription = this.client
.ob$('docStorage.waitForConnected')
.subscribe({
next() {
signal?.removeEventListener('abort', abortListener);
resolve();
},
error(err) {
signal?.removeEventListener('abort', abortListener);
reject(err);
},
});
});
}
}
class WorkerBlobStorage implements BlobStorage {
constructor(
private readonly client: OpClient<WorkerOps>,
private readonly options: StorageOptions
) {}
readonly storageType = 'blob';
readonly peer = this.options.peer;
readonly spaceType = this.options.type;
readonly spaceId = this.options.id;
readonly universalId = universalId(this.options);
get(key: string, _signal?: AbortSignal): Promise<BlobRecord | null> {
return this.client.call('blobStorage.getBlob', key);
}
set(blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
return this.client.call('blobStorage.setBlob', blob);
}
delete(
key: string,
permanently: boolean,
_signal?: AbortSignal
): Promise<void> {
return this.client.call('blobStorage.deleteBlob', { key, permanently });
}
release(_signal?: AbortSignal): Promise<void> {
return this.client.call('blobStorage.releaseBlobs');
}
list(_signal?: AbortSignal): Promise<ListedBlobRecord[]> {
return this.client.call('blobStorage.listBlobs');
}
connection = new DummyConnection();
}
class WorkerAwarenessStorage implements AwarenessStorage {
constructor(
private readonly client: OpClient<WorkerOps>,
private readonly options: StorageOptions
) {}
readonly storageType = 'awareness';
readonly peer = this.options.peer;
readonly spaceType = this.options.type;
readonly spaceId = this.options.id;
readonly universalId = universalId(this.options);
update(record: AwarenessRecord, origin?: string): Promise<void> {
return this.client.call('awarenessStorage.update', {
awareness: record,
origin,
});
}
subscribeUpdate(
id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => Promise<AwarenessRecord | null>
): () => void {
const subscription = this.client
.ob$('awarenessStorage.subscribeUpdate', id)
.subscribe({
next: update => {
if (update.type === 'awareness-update') {
onUpdate(update.awareness, update.origin);
}
if (update.type === 'awareness-collect') {
onCollect()
.then(record => {
if (record) {
this.client
.call('awarenessStorage.collect', {
awareness: record,
collectId: update.collectId,
})
.catch(err => {
console.error('error feedback collected awareness', err);
});
}
})
.catch(err => {
console.error('error collecting awareness', err);
});
}
},
});
return () => {
subscription.unsubscribe();
};
}
connection = new DummyConnection();
}
class WorkerDocSync implements DocSync {
constructor(private readonly client: OpClient<WorkerOps>) {}
readonly state$ = this.client.ob$('docSync.state');
docState$(docId: string) {
return this.client.ob$('docSync.docState', docId);
}
addPriority(docId: string, priority: number) {
const subscription = this.client
.ob$('docSync.addPriority', { docId, priority })
.subscribe();
return () => {
subscription.unsubscribe();
};
}
}
class WorkerBlobSync implements BlobSync {
constructor(private readonly client: OpClient<WorkerOps>) {}
downloadBlob(
blobId: string,
_signal?: AbortSignal
): Promise<BlobRecord | null> {
return this.client.call('blobSync.downloadBlob', blobId);
}
uploadBlob(blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
return this.client.call('blobSync.uploadBlob', blob);
}
}
class WorkerAwarenessSync implements AwarenessSync {
constructor(private readonly client: OpClient<WorkerOps>) {}
update(record: AwarenessRecord, origin?: string): Promise<void> {
return this.client.call('awarenessSync.update', {
awareness: record,
origin,
});
}
subscribeUpdate(
id: string,
onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => Promise<AwarenessRecord | null>
): () => void {
const subscription = this.client
.ob$('awarenessSync.subscribeUpdate', id)
.subscribe({
next: update => {
if (update.type === 'awareness-update') {
onUpdate(update.awareness, update.origin);
}
if (update.type === 'awareness-collect') {
onCollect()
.then(record => {
if (record) {
this.client
.call('awarenessSync.collect', {
awareness: record,
collectId: update.collectId,
})
.catch(err => {
console.error('error feedback collected awareness', err);
});
}
})
.catch(err => {
console.error('error collecting awareness', err);
});
}
},
});
return () => {
subscription.unsubscribe();
};
}
}

View File

@ -0,0 +1,256 @@
import type { OpConsumer } from '@toeverything/infra/op';
import { Observable } from 'rxjs';
import { getAvailableStorageImplementations } from '../impls';
import { SpaceStorage, type StorageOptions } from '../storage';
import type { AwarenessRecord } from '../storage/awareness';
import { Sync } from '../sync';
import type { WorkerOps } from './ops';
export class WorkerConsumer {
private remotes: SpaceStorage[] = [];
private local: SpaceStorage | null = null;
private sync: Sync | null = null;
get ensureLocal() {
if (!this.local) {
throw new Error('Not initialized');
}
return this.local;
}
get ensureSync() {
if (!this.sync) {
throw new Error('Not initialized');
}
return this.sync;
}
get docStorage() {
return this.ensureLocal.get('doc');
}
get docSync() {
const docSync = this.ensureSync.doc;
if (!docSync) {
throw new Error('Doc sync not initialized');
}
return docSync;
}
get blobStorage() {
return this.ensureLocal.get('blob');
}
get blobSync() {
const blobSync = this.ensureSync.blob;
if (!blobSync) {
throw new Error('Blob sync not initialized');
}
return blobSync;
}
get syncStorage() {
return this.ensureLocal.get('sync');
}
get awarenessStorage() {
return this.ensureLocal.get('awareness');
}
get awarenessSync() {
const awarenessSync = this.ensureSync.awareness;
if (!awarenessSync) {
throw new Error('Awareness sync not initialized');
}
return awarenessSync;
}
constructor(private readonly consumer: OpConsumer<WorkerOps>) {}
listen() {
this.registerHandlers();
this.consumer.listen();
}
async init(init: {
local: { name: string; opts: StorageOptions }[];
remotes: { name: string; opts: StorageOptions }[][];
}) {
this.local = new SpaceStorage(
init.local.map(opt => {
const Storage = getAvailableStorageImplementations(opt.name);
return new Storage(opt.opts);
})
);
this.remotes = init.remotes.map(opts => {
return new SpaceStorage(
opts.map(opt => {
const Storage = getAvailableStorageImplementations(opt.name);
return new Storage(opt.opts);
})
);
});
this.sync = new Sync(this.local, this.remotes);
this.local.connect();
for (const remote of this.remotes) {
remote.connect();
}
this.sync.start();
}
async destroy() {
this.sync?.stop();
this.local?.disconnect();
await this.local?.destroy();
for (const remote of this.remotes) {
remote.disconnect();
await remote.destroy();
}
}
private registerHandlers() {
const collectJobs = new Map<
string,
(awareness: AwarenessRecord | null) => void
>();
let collectId = 0;
this.consumer.registerAll({
'worker.init': this.init.bind(this),
'worker.destroy': this.destroy.bind(this),
'docStorage.getDoc': (docId: string) => this.docStorage.getDoc(docId),
'docStorage.getDocDiff': ({ docId, state }) =>
this.docStorage.getDocDiff(docId, state),
'docStorage.pushDocUpdate': ({ update, origin }) =>
this.docStorage.pushDocUpdate(update, origin),
'docStorage.getDocTimestamps': after =>
this.docStorage.getDocTimestamps(after ?? undefined),
'docStorage.getDocTimestamp': docId =>
this.docStorage.getDocTimestamp(docId),
'docStorage.deleteDoc': (docId: string) =>
this.docStorage.deleteDoc(docId),
'docStorage.subscribeDocUpdate': () =>
new Observable(subscriber => {
return this.docStorage.subscribeDocUpdate((update, origin) => {
subscriber.next({ update, origin });
});
}),
'docStorage.waitForConnected': () =>
new Observable(subscriber => {
const abortController = new AbortController();
this.docStorage.connection
.waitForConnected(abortController.signal)
.then(() => {
subscriber.next(true);
subscriber.complete();
})
.catch(error => {
subscriber.error(error);
});
return () => abortController.abort();
}),
'blobStorage.getBlob': key => this.blobStorage.get(key),
'blobStorage.setBlob': blob => this.blobStorage.set(blob),
'blobStorage.deleteBlob': ({ key, permanently }) =>
this.blobStorage.delete(key, permanently),
'blobStorage.releaseBlobs': () => this.blobStorage.release(),
'blobStorage.listBlobs': () => this.blobStorage.list(),
'syncStorage.clearClocks': () => this.syncStorage.clearClocks(),
'syncStorage.getPeerPulledRemoteClock': ({ peer, docId }) =>
this.syncStorage.getPeerPulledRemoteClock(peer, docId),
'syncStorage.getPeerPulledRemoteClocks': ({ peer }) =>
this.syncStorage.getPeerPulledRemoteClocks(peer),
'syncStorage.setPeerPulledRemoteClock': ({ peer, clock }) =>
this.syncStorage.setPeerPulledRemoteClock(peer, clock),
'syncStorage.getPeerRemoteClock': ({ peer, docId }) =>
this.syncStorage.getPeerRemoteClock(peer, docId),
'syncStorage.getPeerRemoteClocks': ({ peer }) =>
this.syncStorage.getPeerRemoteClocks(peer),
'syncStorage.setPeerRemoteClock': ({ peer, clock }) =>
this.syncStorage.setPeerRemoteClock(peer, clock),
'syncStorage.getPeerPushedClock': ({ peer, docId }) =>
this.syncStorage.getPeerPushedClock(peer, docId),
'syncStorage.getPeerPushedClocks': ({ peer }) =>
this.syncStorage.getPeerPushedClocks(peer),
'syncStorage.setPeerPushedClock': ({ peer, clock }) =>
this.syncStorage.setPeerPushedClock(peer, clock),
'awarenessStorage.update': ({ awareness, origin }) =>
this.awarenessStorage.update(awareness, origin),
'awarenessStorage.subscribeUpdate': docId =>
new Observable(subscriber => {
return this.awarenessStorage.subscribeUpdate(
docId,
(update, origin) => {
subscriber.next({
type: 'awareness-update',
awareness: update,
origin,
});
},
() => {
const currentCollectId = collectId++;
const promise = new Promise<AwarenessRecord | null>(resolve => {
collectJobs.set(currentCollectId.toString(), awareness => {
resolve(awareness);
collectJobs.delete(currentCollectId.toString());
});
});
return promise;
}
);
}),
'awarenessStorage.collect': ({ collectId, awareness }) =>
collectJobs.get(collectId)?.(awareness),
'docSync.state': () =>
new Observable(subscriber => {
const subscription = this.docSync.state$.subscribe(state => {
subscriber.next(state);
});
return () => subscription.unsubscribe();
}),
'docSync.docState': docId =>
new Observable(subscriber => {
const subscription = this.docSync
.docState$(docId)
.subscribe(state => {
subscriber.next(state);
});
return () => subscription.unsubscribe();
}),
'docSync.addPriority': ({ docId, priority }) =>
new Observable(() => {
const undo = this.docSync.addPriority(docId, priority);
return () => undo();
}),
'blobSync.downloadBlob': key => this.blobSync.downloadBlob(key),
'blobSync.uploadBlob': blob => this.blobSync.uploadBlob(blob),
'awarenessSync.update': ({ awareness, origin }) =>
this.awarenessSync.update(awareness, origin),
'awarenessSync.subscribeUpdate': docId =>
new Observable(subscriber => {
return this.awarenessStorage.subscribeUpdate(
docId,
(update, origin) => {
subscriber.next({
type: 'awareness-update',
awareness: update,
origin,
});
},
() => {
const currentCollectId = collectId++;
const promise = new Promise<AwarenessRecord | null>(resolve => {
collectJobs.set(currentCollectId.toString(), awareness => {
resolve(awareness);
collectJobs.delete(currentCollectId.toString());
});
});
return promise;
}
);
}),
'awarenessSync.collect': ({ collectId, awareness }) =>
collectJobs.get(collectId)?.(awareness),
});
}
}

View File

@ -0,0 +1,122 @@
import type {
BlobRecord,
DocClock,
DocClocks,
DocDiff,
DocRecord,
DocUpdate,
ListedBlobRecord,
StorageOptions,
} from '../storage';
import type { AwarenessRecord } from '../storage/awareness';
import type { DocSyncDocState, DocSyncState } from '../sync/doc';
interface GroupedWorkerOps {
worker: {
init: [
{
local: { name: string; opts: StorageOptions }[];
remotes: { name: string; opts: StorageOptions }[][];
},
void,
];
destroy: [void, void];
};
docStorage: {
getDoc: [string, DocRecord | null];
getDocDiff: [{ docId: string; state?: Uint8Array }, DocDiff | null];
pushDocUpdate: [{ update: DocUpdate; origin?: string }, DocClock];
getDocTimestamps: [Date | null, DocClocks];
getDocTimestamp: [string, DocClock | null];
deleteDoc: [string, void];
subscribeDocUpdate: [void, { update: DocRecord; origin?: string }];
waitForConnected: [void, boolean];
};
blobStorage: {
getBlob: [string, BlobRecord | null];
setBlob: [BlobRecord, void];
deleteBlob: [{ key: string; permanently: boolean }, void];
releaseBlobs: [void, void];
listBlobs: [void, ListedBlobRecord[]];
};
syncStorage: {
getPeerPulledRemoteClocks: [{ peer: string }, DocClocks];
getPeerPulledRemoteClock: [
{ peer: string; docId: string },
DocClock | null,
];
setPeerPulledRemoteClock: [{ peer: string; clock: DocClock }, void];
getPeerRemoteClocks: [{ peer: string }, DocClocks];
getPeerRemoteClock: [{ peer: string; docId: string }, DocClock | null];
setPeerRemoteClock: [{ peer: string; clock: DocClock }, void];
getPeerPushedClocks: [{ peer: string }, DocClocks];
getPeerPushedClock: [{ peer: string; docId: string }, DocClock | null];
setPeerPushedClock: [{ peer: string; clock: DocClock }, void];
clearClocks: [void, void];
};
awarenessStorage: {
update: [{ awareness: AwarenessRecord; origin?: string }, void];
subscribeUpdate: [
string,
(
| {
type: 'awareness-update';
awareness: AwarenessRecord;
origin?: string;
}
| { type: 'awareness-collect'; collectId: string }
),
];
collect: [{ collectId: string; awareness: AwarenessRecord }, void];
};
docSync: {
state: [void, DocSyncState];
docState: [string, DocSyncDocState];
addPriority: [{ docId: string; priority: number }, boolean];
};
blobSync: {
downloadBlob: [string, BlobRecord | null];
uploadBlob: [BlobRecord, void];
};
awarenessSync: {
update: [{ awareness: AwarenessRecord; origin?: string }, void];
subscribeUpdate: [
string,
(
| {
type: 'awareness-update';
awareness: AwarenessRecord;
origin?: string;
}
| { type: 'awareness-collect'; collectId: string }
),
];
collect: [{ collectId: string; awareness: AwarenessRecord }, void];
};
}
type Values<T> = T extends { [k in keyof T]: any } ? T[keyof T] : never;
type UnionToIntersection<U> = (U extends any ? (x: U) => void : never) extends (
x: infer I
) => void
? I
: never;
export type WorkerOps = UnionToIntersection<
Values<
Values<{
[k in keyof GroupedWorkerOps]: {
[k2 in keyof GroupedWorkerOps[k]]: k2 extends string
? Record<`${k}.${k2}`, GroupedWorkerOps[k][k2]>
: never;
};
}>
>
>;

View File

@ -1,8 +1,8 @@
import { type BlobRecord, BlobStorage, share } from '@affine/nbstore'; import { type BlobRecord, BlobStorageBase, share } from '@affine/nbstore';
import { NativeDBConnection } from './db'; import { NativeDBConnection } from './db';
export class SqliteBlobStorage extends BlobStorage { export class SqliteBlobStorage extends BlobStorageBase {
override connection = share( override connection = share(
new NativeDBConnection(this.peer, this.spaceType, this.spaceId) new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
); );

View File

@ -1,13 +1,13 @@
import path from 'node:path'; import path from 'node:path';
import { DocStorage as NativeDocStorage } from '@affine/native'; import { DocStorage as NativeDocStorage } from '@affine/native';
import { Connection, type SpaceType } from '@affine/nbstore'; import { AutoReconnectConnection, type SpaceType } from '@affine/nbstore';
import fs from 'fs-extra'; import fs from 'fs-extra';
import { logger } from '../logger'; import { logger } from '../logger';
import { getSpaceDBPath } from '../workspace/meta'; import { getSpaceDBPath } from '../workspace/meta';
export class NativeDBConnection extends Connection<NativeDocStorage> { export class NativeDBConnection extends AutoReconnectConnection<NativeDocStorage> {
constructor( constructor(
private readonly peer: string, private readonly peer: string,
private readonly type: SpaceType, private readonly type: SpaceType,

View File

@ -1,14 +1,14 @@
import { import {
type DocClocks, type DocClocks,
type DocRecord, type DocRecord,
DocStorage, DocStorageBase,
type DocUpdate, type DocUpdate,
share, share,
} from '@affine/nbstore'; } from '@affine/nbstore';
import { NativeDBConnection } from './db'; import { NativeDBConnection } from './db';
export class SqliteDocStorage extends DocStorage { export class SqliteDocStorage extends DocStorageBase {
override connection = share( override connection = share(
new NativeDBConnection(this.peer, this.spaceType, this.spaceId) new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
); );

View File

@ -1,13 +1,13 @@
import { import {
BasicSyncStorage,
type DocClock, type DocClock,
type DocClocks, type DocClocks,
share, share,
SyncStorage,
} from '@affine/nbstore'; } from '@affine/nbstore';
import { NativeDBConnection } from './db'; import { NativeDBConnection } from './db';
export class SqliteSyncStorage extends SyncStorage { export class SqliteSyncStorage extends BasicSyncStorage {
override connection = share( override connection = share(
new NativeDBConnection(this.peer, this.spaceType, this.spaceId) new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
); );