mirror of
https://github.com/toeverything/AFFiNE.git
synced 2024-12-23 05:42:30 +03:00
feat(nbstore): add nbstore worker (#9185)
This commit is contained in:
parent
30200ff86d
commit
cbaf35df0b
@ -126,6 +126,16 @@ export class OpConsumer<Ops extends OpSchema> extends AutoMessageHandler {
|
|||||||
this.registeredOpHandlers.set(op, handler);
|
this.registeredOpHandlers.set(op, handler);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
registerAll(
|
||||||
|
handlers: OpNames<Ops> extends string
|
||||||
|
? { [K in OpNames<Ops>]: OpHandler<Ops, K> }
|
||||||
|
: never
|
||||||
|
) {
|
||||||
|
for (const [op, handler] of Object.entries(handlers)) {
|
||||||
|
this.register(op as any, handler as any);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
before<Op extends OpNames<Ops>>(
|
before<Op extends OpNames<Ops>>(
|
||||||
op: Op,
|
op: Op,
|
||||||
handler: (...input: OpInput<Ops, Op>) => void
|
handler: (...input: OpInput<Ops, Op>) => void
|
||||||
|
@ -6,7 +6,7 @@
|
|||||||
"sideEffects": false,
|
"sideEffects": false,
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./src/index.ts",
|
".": "./src/index.ts",
|
||||||
"./op": "./src/op/index.ts",
|
"./worker": "./src/worker/index.ts",
|
||||||
"./idb": "./src/impls/idb/index.ts",
|
"./idb": "./src/impls/idb/index.ts",
|
||||||
"./idb/v1": "./src/impls/idb/v1/index.ts",
|
"./idb/v1": "./src/impls/idb/v1/index.ts",
|
||||||
"./cloud": "./src/impls/cloud/index.ts",
|
"./cloud": "./src/impls/cloud/index.ts",
|
||||||
|
@ -8,7 +8,7 @@ import { AwarenessFrontend } from '../frontend/awareness';
|
|||||||
import { DocFrontend } from '../frontend/doc';
|
import { DocFrontend } from '../frontend/doc';
|
||||||
import { BroadcastChannelAwarenessStorage } from '../impls/broadcast-channel/awareness';
|
import { BroadcastChannelAwarenessStorage } from '../impls/broadcast-channel/awareness';
|
||||||
import { IndexedDBDocStorage } from '../impls/idb';
|
import { IndexedDBDocStorage } from '../impls/idb';
|
||||||
import { AwarenessSync } from '../sync/awareness';
|
import { AwarenessSyncImpl } from '../sync/awareness';
|
||||||
import { expectYjsEqual } from './utils';
|
import { expectYjsEqual } from './utils';
|
||||||
|
|
||||||
test('doc', async () => {
|
test('doc', async () => {
|
||||||
@ -23,9 +23,9 @@ test('doc', async () => {
|
|||||||
type: 'workspace',
|
type: 'workspace',
|
||||||
});
|
});
|
||||||
|
|
||||||
docStorage.connect();
|
docStorage.connection.connect();
|
||||||
|
|
||||||
await docStorage.waitForConnected();
|
await docStorage.connection.waitForConnected();
|
||||||
|
|
||||||
const frontend1 = new DocFrontend(docStorage, null);
|
const frontend1 = new DocFrontend(docStorage, null);
|
||||||
frontend1.start();
|
frontend1.start();
|
||||||
@ -68,11 +68,11 @@ test('awareness', async () => {
|
|||||||
type: 'workspace',
|
type: 'workspace',
|
||||||
});
|
});
|
||||||
|
|
||||||
storage1.connect();
|
storage1.connection.connect();
|
||||||
storage2.connect();
|
storage2.connection.connect();
|
||||||
|
|
||||||
await storage1.waitForConnected();
|
await storage1.connection.waitForConnected();
|
||||||
await storage2.waitForConnected();
|
await storage2.connection.waitForConnected();
|
||||||
|
|
||||||
// peer a
|
// peer a
|
||||||
const docA = new YDoc({ guid: 'test-doc' });
|
const docA = new YDoc({ guid: 'test-doc' });
|
||||||
@ -90,13 +90,13 @@ test('awareness', async () => {
|
|||||||
const awarenessC = new Awareness(docC);
|
const awarenessC = new Awareness(docC);
|
||||||
|
|
||||||
{
|
{
|
||||||
const sync = new AwarenessSync(storage1, [storage2]);
|
const sync = new AwarenessSyncImpl(storage1, [storage2]);
|
||||||
const frontend = new AwarenessFrontend(sync);
|
const frontend = new AwarenessFrontend(sync);
|
||||||
frontend.connect(awarenessA);
|
frontend.connect(awarenessA);
|
||||||
frontend.connect(awarenessB);
|
frontend.connect(awarenessB);
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
const sync = new AwarenessSync(storage2, [storage1]);
|
const sync = new AwarenessSyncImpl(storage2, [storage1]);
|
||||||
const frontend = new AwarenessFrontend(sync);
|
const frontend = new AwarenessFrontend(sync);
|
||||||
frontend.connect(awarenessC);
|
frontend.connect(awarenessC);
|
||||||
}
|
}
|
||||||
|
@ -8,7 +8,20 @@ export type ConnectionStatus =
|
|||||||
| 'error'
|
| 'error'
|
||||||
| 'closed';
|
| 'closed';
|
||||||
|
|
||||||
export abstract class Connection<T = any> {
|
export interface Connection<T = any> {
|
||||||
|
readonly status: ConnectionStatus;
|
||||||
|
readonly inner: T;
|
||||||
|
connect(): void;
|
||||||
|
disconnect(): void;
|
||||||
|
waitForConnected(signal?: AbortSignal): Promise<void>;
|
||||||
|
onStatusChanged(
|
||||||
|
cb: (status: ConnectionStatus, error?: Error) => void
|
||||||
|
): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export abstract class AutoReconnectConnection<T = any>
|
||||||
|
implements Connection<T>
|
||||||
|
{
|
||||||
private readonly event = new EventEmitter2();
|
private readonly event = new EventEmitter2();
|
||||||
private _inner: T | null = null;
|
private _inner: T | null = null;
|
||||||
private _status: ConnectionStatus = 'idle';
|
private _status: ConnectionStatus = 'idle';
|
||||||
@ -160,12 +173,22 @@ export abstract class Connection<T = any> {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export class DummyConnection extends Connection<undefined> {
|
export class DummyConnection implements Connection<undefined> {
|
||||||
doConnect() {
|
readonly status: ConnectionStatus = 'connected';
|
||||||
return Promise.resolve(undefined);
|
readonly inner: undefined;
|
||||||
}
|
|
||||||
|
|
||||||
doDisconnect() {
|
connect(): void {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
disconnect(): void {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
waitForConnected(_signal?: AbortSignal): Promise<void> {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
onStatusChanged(
|
||||||
|
_cb: (status: ConnectionStatus, error?: Error) => void
|
||||||
|
): () => void {
|
||||||
|
return () => {};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import type { Connection } from './connection';
|
import type { AutoReconnectConnection } from './connection';
|
||||||
|
|
||||||
const CONNECTIONS: Map<string, Connection<any>> = new Map();
|
const CONNECTIONS: Map<string, AutoReconnectConnection<any>> = new Map();
|
||||||
export function share<T extends Connection<any>>(conn: T): T {
|
export function share<T extends AutoReconnectConnection<any>>(conn: T): T {
|
||||||
if (!conn.shareId) {
|
if (!conn.shareId) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Connection ${conn.constructor.name} is not shareable.\nIf you want to make it shareable, please override [shareId].`
|
`Connection ${conn.constructor.name} is not shareable.\nIf you want to make it shareable, please override [shareId].`
|
||||||
|
@ -51,10 +51,10 @@ export class AwarenessFrontend {
|
|||||||
applyAwarenessUpdate(awareness, update.bin, origin);
|
applyAwarenessUpdate(awareness, update.bin, origin);
|
||||||
};
|
};
|
||||||
const handleSyncCollect = () => {
|
const handleSyncCollect = () => {
|
||||||
return {
|
return Promise.resolve({
|
||||||
docId: awareness.doc.guid,
|
docId: awareness.doc.guid,
|
||||||
bin: encodeAwarenessUpdate(awareness, [awareness.clientID]),
|
bin: encodeAwarenessUpdate(awareness, [awareness.clientID]),
|
||||||
};
|
});
|
||||||
};
|
};
|
||||||
const unsubscribe = this.sync.subscribeUpdate(
|
const unsubscribe = this.sync.subscribeUpdate(
|
||||||
awareness.doc.guid,
|
awareness.doc.guid,
|
||||||
|
@ -17,7 +17,7 @@ export class BlobFrontend {
|
|||||||
return this.sync ? this.sync.uploadBlob(blob) : this.storage.set(blob);
|
return this.sync ? this.sync.uploadBlob(blob) : this.storage.set(blob);
|
||||||
}
|
}
|
||||||
|
|
||||||
addPriority(id: string, priority: number) {
|
addPriority(_id: string, _priority: number) {
|
||||||
return this.sync?.addPriority(id, priority);
|
// not support yet
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -37,7 +37,7 @@ interface DocFrontendOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export class DocFrontend {
|
export class DocFrontend {
|
||||||
private readonly uniqueId = `frontend:${this.storage.peer}:${nanoid()}`;
|
private readonly uniqueId = `frontend:${nanoid()}`;
|
||||||
|
|
||||||
private readonly prioritySettings = new Map<string, number>();
|
private readonly prioritySettings = new Map<string, number>();
|
||||||
|
|
||||||
@ -88,7 +88,6 @@ export class DocFrontend {
|
|||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// eslint-disable-next-line no-constant-condition
|
|
||||||
while (true) {
|
while (true) {
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
const docId = await this.status.jobDocQueue.asyncPop(signal);
|
const docId = await this.status.jobDocQueue.asyncPop(signal);
|
||||||
|
@ -1,9 +1,6 @@
|
|||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
|
|
||||||
import {
|
import { type AwarenessRecord, AwarenessStorageBase } from '../../storage';
|
||||||
type AwarenessRecord,
|
|
||||||
AwarenessStorage,
|
|
||||||
} from '../../storage/awareness';
|
|
||||||
import { BroadcastChannelConnection } from './channel';
|
import { BroadcastChannelConnection } from './channel';
|
||||||
|
|
||||||
type ChannelMessage =
|
type ChannelMessage =
|
||||||
@ -19,13 +16,13 @@ type ChannelMessage =
|
|||||||
collectId: string;
|
collectId: string;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
type: 'awareness-collect-fallback';
|
type: 'awareness-collect-feedback';
|
||||||
docId: string;
|
docId: string;
|
||||||
bin: Uint8Array;
|
bin: Uint8Array;
|
||||||
collectId: string;
|
collectId: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
|
export class BroadcastChannelAwarenessStorage extends AwarenessStorageBase {
|
||||||
override readonly storageType = 'awareness';
|
override readonly storageType = 'awareness';
|
||||||
override readonly connection = new BroadcastChannelConnection(this.options);
|
override readonly connection = new BroadcastChannelConnection(this.options);
|
||||||
get channel() {
|
get channel() {
|
||||||
@ -36,7 +33,7 @@ export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
|
|||||||
string,
|
string,
|
||||||
Set<{
|
Set<{
|
||||||
onUpdate: (update: AwarenessRecord, origin?: string) => void;
|
onUpdate: (update: AwarenessRecord, origin?: string) => void;
|
||||||
onCollect: () => AwarenessRecord;
|
onCollect: () => Promise<AwarenessRecord | null>;
|
||||||
}>
|
}>
|
||||||
>();
|
>();
|
||||||
|
|
||||||
@ -57,12 +54,20 @@ export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
|
|||||||
override subscribeUpdate(
|
override subscribeUpdate(
|
||||||
id: string,
|
id: string,
|
||||||
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
onCollect: () => AwarenessRecord
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
): () => void {
|
): () => void {
|
||||||
const subscribers = this.subscriptions.get(id) ?? new Set();
|
const subscribers = this.subscriptions.get(id) ?? new Set();
|
||||||
subscribers.forEach(subscriber => {
|
subscribers.forEach(subscriber => {
|
||||||
const fallback = subscriber.onCollect();
|
subscriber
|
||||||
onUpdate(fallback);
|
.onCollect()
|
||||||
|
.then(awareness => {
|
||||||
|
if (awareness) {
|
||||||
|
onUpdate(awareness);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('error in on collect awareness', error);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
const collectUniqueId = nanoid();
|
const collectUniqueId = nanoid();
|
||||||
@ -84,18 +89,23 @@ export class BroadcastChannelAwarenessStorage extends AwarenessStorage {
|
|||||||
message.data.type === 'awareness-collect' &&
|
message.data.type === 'awareness-collect' &&
|
||||||
message.data.docId === id
|
message.data.docId === id
|
||||||
) {
|
) {
|
||||||
const fallback = onCollect();
|
onCollect()
|
||||||
if (fallback) {
|
.then(awareness => {
|
||||||
|
if (awareness) {
|
||||||
this.channel.postMessage({
|
this.channel.postMessage({
|
||||||
type: 'awareness-collect-fallback',
|
type: 'awareness-collect-feedback',
|
||||||
docId: message.data.docId,
|
docId: message.data.docId,
|
||||||
bin: fallback.bin,
|
bin: awareness.bin,
|
||||||
collectId: collectUniqueId,
|
collectId: collectUniqueId,
|
||||||
} satisfies ChannelMessage);
|
} satisfies ChannelMessage);
|
||||||
}
|
}
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('error in on collect awareness', error);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
message.data.type === 'awareness-collect-fallback' &&
|
message.data.type === 'awareness-collect-feedback' &&
|
||||||
message.data.docId === id &&
|
message.data.docId === id &&
|
||||||
message.data.collectId === collectUniqueId
|
message.data.collectId === collectUniqueId
|
||||||
) {
|
) {
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import { Connection } from '../../connection';
|
import { AutoReconnectConnection } from '../../connection';
|
||||||
import type { StorageOptions } from '../../storage';
|
import type { StorageOptions } from '../../storage';
|
||||||
|
|
||||||
export class BroadcastChannelConnection extends Connection<BroadcastChannel> {
|
export class BroadcastChannelConnection extends AutoReconnectConnection<BroadcastChannel> {
|
||||||
readonly channelName = `channel:${this.opts.peer}:${this.opts.type}:${this.opts.id}`;
|
readonly channelName = `channel:${this.opts.peer}:${this.opts.type}:${this.opts.id}`;
|
||||||
|
|
||||||
constructor(private readonly opts: StorageOptions) {
|
constructor(private readonly opts: StorageOptions) {
|
||||||
|
@ -3,7 +3,7 @@ import type { SocketOptions } from 'socket.io-client';
|
|||||||
import { share } from '../../connection';
|
import { share } from '../../connection';
|
||||||
import {
|
import {
|
||||||
type AwarenessRecord,
|
type AwarenessRecord,
|
||||||
AwarenessStorage,
|
AwarenessStorageBase,
|
||||||
type AwarenessStorageOptions,
|
type AwarenessStorageOptions,
|
||||||
} from '../../storage/awareness';
|
} from '../../storage/awareness';
|
||||||
import {
|
import {
|
||||||
@ -16,7 +16,7 @@ interface CloudAwarenessStorageOptions extends AwarenessStorageOptions {
|
|||||||
socketOptions: SocketOptions;
|
socketOptions: SocketOptions;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorageOptions> {
|
export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessStorageOptions> {
|
||||||
connection = share(
|
connection = share(
|
||||||
new SocketConnection(this.peer, this.options.socketOptions)
|
new SocketConnection(this.peer, this.options.socketOptions)
|
||||||
);
|
);
|
||||||
@ -38,7 +38,7 @@ export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorag
|
|||||||
override subscribeUpdate(
|
override subscribeUpdate(
|
||||||
id: string,
|
id: string,
|
||||||
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
onCollect: () => AwarenessRecord
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
): () => void {
|
): () => void {
|
||||||
// TODO: handle disconnect
|
// TODO: handle disconnect
|
||||||
// leave awareness
|
// leave awareness
|
||||||
@ -92,7 +92,8 @@ export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorag
|
|||||||
docId === id
|
docId === id
|
||||||
) {
|
) {
|
||||||
(async () => {
|
(async () => {
|
||||||
const record = onCollect();
|
const record = await onCollect();
|
||||||
|
if (record) {
|
||||||
const encodedUpdate = await uint8ArrayToBase64(record.bin);
|
const encodedUpdate = await uint8ArrayToBase64(record.bin);
|
||||||
this.socket.emit('space:update-awareness', {
|
this.socket.emit('space:update-awareness', {
|
||||||
spaceType: this.spaceType,
|
spaceType: this.spaceType,
|
||||||
@ -100,6 +101,7 @@ export class CloudAwarenessStorage extends AwarenessStorage<CloudAwarenessStorag
|
|||||||
docId: record.docId,
|
docId: record.docId,
|
||||||
awarenessUpdate: encodedUpdate,
|
awarenessUpdate: encodedUpdate,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
})().catch(err => console.error('awareness upload failed', err));
|
})().catch(err => console.error('awareness upload failed', err));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -7,16 +7,31 @@ import {
|
|||||||
} from '@affine/graphql';
|
} from '@affine/graphql';
|
||||||
|
|
||||||
import { DummyConnection } from '../../connection';
|
import { DummyConnection } from '../../connection';
|
||||||
import { type BlobRecord, BlobStorage } from '../../storage';
|
import {
|
||||||
|
type BlobRecord,
|
||||||
|
BlobStorageBase,
|
||||||
|
type BlobStorageOptions,
|
||||||
|
} from '../../storage';
|
||||||
|
|
||||||
export class CloudBlobStorage extends BlobStorage {
|
interface CloudBlobStorageOptions extends BlobStorageOptions {
|
||||||
private readonly gql = gqlFetcherFactory(this.options.peer + '/graphql');
|
apiBaseUrl: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CloudBlobStorage extends BlobStorageBase<CloudBlobStorageOptions> {
|
||||||
|
private readonly gql = gqlFetcherFactory(
|
||||||
|
this.options.apiBaseUrl + '/graphql'
|
||||||
|
);
|
||||||
override connection = new DummyConnection();
|
override connection = new DummyConnection();
|
||||||
|
|
||||||
override async get(key: string) {
|
override async get(key: string) {
|
||||||
const res = await fetch(
|
const res = await fetch(
|
||||||
this.options.peer + '/api/workspaces/' + this.spaceId + '/blobs/' + key,
|
this.options.apiBaseUrl +
|
||||||
|
'/api/workspaces/' +
|
||||||
|
this.spaceId +
|
||||||
|
'/blobs/' +
|
||||||
|
key,
|
||||||
{
|
{
|
||||||
|
cache: 'default',
|
||||||
headers: {
|
headers: {
|
||||||
'x-affine-version': BUILD_CONFIG.appVersion,
|
'x-affine-version': BUILD_CONFIG.appVersion,
|
||||||
},
|
},
|
||||||
|
@ -1,10 +1,14 @@
|
|||||||
import type { SocketOptions } from 'socket.io-client';
|
import type { Socket, SocketOptions } from 'socket.io-client';
|
||||||
|
|
||||||
import { share } from '../../connection';
|
import {
|
||||||
|
type Connection,
|
||||||
|
type ConnectionStatus,
|
||||||
|
share,
|
||||||
|
} from '../../connection';
|
||||||
import {
|
import {
|
||||||
type DocClock,
|
type DocClock,
|
||||||
type DocClocks,
|
type DocClocks,
|
||||||
DocStorage,
|
DocStorageBase,
|
||||||
type DocStorageOptions,
|
type DocStorageOptions,
|
||||||
type DocUpdate,
|
type DocUpdate,
|
||||||
} from '../../storage';
|
} from '../../storage';
|
||||||
@ -17,63 +21,14 @@ import {
|
|||||||
|
|
||||||
interface CloudDocStorageOptions extends DocStorageOptions {
|
interface CloudDocStorageOptions extends DocStorageOptions {
|
||||||
socketOptions: SocketOptions;
|
socketOptions: SocketOptions;
|
||||||
|
serverBaseUrl: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> {
|
export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
|
||||||
connection = share(
|
get socket() {
|
||||||
new SocketConnection(this.peer, this.options.socketOptions)
|
|
||||||
);
|
|
||||||
|
|
||||||
private disposeConnectionStatusListener?: () => void;
|
|
||||||
|
|
||||||
private get socket() {
|
|
||||||
return this.connection.inner;
|
return this.connection.inner;
|
||||||
}
|
}
|
||||||
|
|
||||||
override connect() {
|
|
||||||
if (!this.disposeConnectionStatusListener) {
|
|
||||||
this.disposeConnectionStatusListener = this.connection.onStatusChanged(
|
|
||||||
status => {
|
|
||||||
if (status === 'connected') {
|
|
||||||
this.join().catch(err => {
|
|
||||||
console.error('doc storage join failed', err);
|
|
||||||
});
|
|
||||||
this.socket.on('space:broadcast-doc-update', this.onServerUpdate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
super.connect();
|
|
||||||
}
|
|
||||||
|
|
||||||
override disconnect() {
|
|
||||||
if (this.disposeConnectionStatusListener) {
|
|
||||||
this.disposeConnectionStatusListener();
|
|
||||||
}
|
|
||||||
this.socket.emit('space:leave', {
|
|
||||||
spaceType: this.spaceType,
|
|
||||||
spaceId: this.spaceId,
|
|
||||||
});
|
|
||||||
this.socket.off('space:broadcast-doc-update', this.onServerUpdate);
|
|
||||||
super.disconnect();
|
|
||||||
}
|
|
||||||
|
|
||||||
async join() {
|
|
||||||
try {
|
|
||||||
const res = await this.socket.emitWithAck('space:join', {
|
|
||||||
spaceType: this.spaceType,
|
|
||||||
spaceId: this.spaceId,
|
|
||||||
clientVersion: BUILD_CONFIG.appVersion,
|
|
||||||
});
|
|
||||||
|
|
||||||
if ('error' in res) {
|
|
||||||
this.connection.setStatus('closed', new Error(res.error.message));
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
this.connection.setStatus('error', e as Error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {
|
onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {
|
||||||
if (
|
if (
|
||||||
this.spaceType === message.spaceType &&
|
this.spaceType === message.spaceType &&
|
||||||
@ -88,6 +43,11 @@ export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
readonly connection = new CloudDocStorageConnection(
|
||||||
|
this.options,
|
||||||
|
this.onServerUpdate
|
||||||
|
);
|
||||||
|
|
||||||
override async getDocSnapshot(docId: string) {
|
override async getDocSnapshot(docId: string) {
|
||||||
const response = await this.socket.emitWithAck('space:load-doc', {
|
const response = await this.socket.emitWithAck('space:load-doc', {
|
||||||
spaceType: this.spaceType,
|
spaceType: this.spaceType,
|
||||||
@ -207,3 +167,84 @@ export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class CloudDocStorageConnection implements Connection<Socket> {
|
||||||
|
connection = share(
|
||||||
|
new SocketConnection(
|
||||||
|
`${this.options.serverBaseUrl}/`,
|
||||||
|
this.options.socketOptions
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
private disposeConnectionStatusListener?: () => void;
|
||||||
|
|
||||||
|
private get socket() {
|
||||||
|
return this.connection.inner;
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly options: CloudDocStorageOptions,
|
||||||
|
private readonly onServerUpdate: ServerEventsMap['space:broadcast-doc-update']
|
||||||
|
) {}
|
||||||
|
|
||||||
|
get status() {
|
||||||
|
return this.connection.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
get inner() {
|
||||||
|
return this.connection.inner;
|
||||||
|
}
|
||||||
|
|
||||||
|
connect(): void {
|
||||||
|
if (!this.disposeConnectionStatusListener) {
|
||||||
|
this.disposeConnectionStatusListener = this.connection.onStatusChanged(
|
||||||
|
status => {
|
||||||
|
if (status === 'connected') {
|
||||||
|
this.join().catch(err => {
|
||||||
|
console.error('doc storage join failed', err);
|
||||||
|
});
|
||||||
|
this.socket.on('space:broadcast-doc-update', this.onServerUpdate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return this.connection.connect();
|
||||||
|
}
|
||||||
|
|
||||||
|
async join() {
|
||||||
|
try {
|
||||||
|
const res = await this.socket.emitWithAck('space:join', {
|
||||||
|
spaceType: this.options.type,
|
||||||
|
spaceId: this.options.id,
|
||||||
|
clientVersion: BUILD_CONFIG.appVersion,
|
||||||
|
});
|
||||||
|
|
||||||
|
if ('error' in res) {
|
||||||
|
this.connection.setStatus('closed', new Error(res.error.message));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
this.connection.setStatus('error', e as Error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
disconnect() {
|
||||||
|
if (this.disposeConnectionStatusListener) {
|
||||||
|
this.disposeConnectionStatusListener();
|
||||||
|
}
|
||||||
|
this.socket.emit('space:leave', {
|
||||||
|
spaceType: this.options.type,
|
||||||
|
spaceId: this.options.id,
|
||||||
|
});
|
||||||
|
this.socket.off('space:broadcast-doc-update', this.onServerUpdate);
|
||||||
|
this.connection.disconnect();
|
||||||
|
}
|
||||||
|
|
||||||
|
waitForConnected(signal?: AbortSignal): Promise<void> {
|
||||||
|
return this.connection.waitForConnected(signal);
|
||||||
|
}
|
||||||
|
onStatusChanged(
|
||||||
|
cb: (status: ConnectionStatus, error?: Error) => void
|
||||||
|
): () => void {
|
||||||
|
return this.connection.onStatusChanged(cb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -4,7 +4,10 @@ import {
|
|||||||
type SocketOptions,
|
type SocketOptions,
|
||||||
} from 'socket.io-client';
|
} from 'socket.io-client';
|
||||||
|
|
||||||
import { Connection, type ConnectionStatus } from '../../connection';
|
import {
|
||||||
|
AutoReconnectConnection,
|
||||||
|
type ConnectionStatus,
|
||||||
|
} from '../../connection';
|
||||||
|
|
||||||
// TODO(@forehalo): use [UserFriendlyError]
|
// TODO(@forehalo): use [UserFriendlyError]
|
||||||
interface EventError {
|
interface EventError {
|
||||||
@ -150,7 +153,7 @@ export function base64ToUint8Array(base64: string) {
|
|||||||
return new Uint8Array(binaryArray);
|
return new Uint8Array(binaryArray);
|
||||||
}
|
}
|
||||||
|
|
||||||
export class SocketConnection extends Connection<Socket> {
|
export class SocketConnection extends AutoReconnectConnection<Socket> {
|
||||||
manager = new SocketIOManager(this.endpoint, {
|
manager = new SocketIOManager(this.endpoint, {
|
||||||
autoConnect: false,
|
autoConnect: false,
|
||||||
transports: ['websocket'],
|
transports: ['websocket'],
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
import { share } from '../../connection';
|
import { share } from '../../connection';
|
||||||
import {
|
import {
|
||||||
type BlobRecord,
|
type BlobRecord,
|
||||||
BlobStorage,
|
BlobStorageBase,
|
||||||
type ListedBlobRecord,
|
type ListedBlobRecord,
|
||||||
} from '../../storage';
|
} from '../../storage';
|
||||||
import { IDBConnection } from './db';
|
import { IDBConnection } from './db';
|
||||||
|
|
||||||
export class IndexedDBBlobStorage extends BlobStorage {
|
export class IndexedDBBlobStorage extends BlobStorageBase {
|
||||||
readonly connection = share(new IDBConnection(this.options));
|
readonly connection = share(new IDBConnection(this.options));
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
import { type IDBPDatabase, openDB } from 'idb';
|
import { type IDBPDatabase, openDB } from 'idb';
|
||||||
|
|
||||||
import { Connection } from '../../connection';
|
import { AutoReconnectConnection } from '../../connection';
|
||||||
import type { StorageOptions } from '../../storage';
|
import type { StorageOptions } from '../../storage';
|
||||||
import { type DocStorageSchema, migrator } from './schema';
|
import { type DocStorageSchema, migrator } from './schema';
|
||||||
|
|
||||||
export class IDBConnection extends Connection<{
|
export class IDBConnection extends AutoReconnectConnection<{
|
||||||
db: IDBPDatabase<DocStorageSchema>;
|
db: IDBPDatabase<DocStorageSchema>;
|
||||||
channel: BroadcastChannel;
|
channel: BroadcastChannel;
|
||||||
}> {
|
}> {
|
||||||
|
@ -2,7 +2,7 @@ import {
|
|||||||
type DocClock,
|
type DocClock,
|
||||||
type DocClocks,
|
type DocClocks,
|
||||||
type DocRecord,
|
type DocRecord,
|
||||||
DocStorage,
|
DocStorageBase,
|
||||||
type DocStorageOptions,
|
type DocStorageOptions,
|
||||||
type DocUpdate,
|
type DocUpdate,
|
||||||
} from '../../storage';
|
} from '../../storage';
|
||||||
@ -15,7 +15,7 @@ interface ChannelMessage {
|
|||||||
origin?: string;
|
origin?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class IndexedDBDocStorage extends DocStorage {
|
export class IndexedDBDocStorage extends DocStorageBase {
|
||||||
readonly connection = new IDBConnection(this.options);
|
readonly connection = new IDBConnection(this.options);
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import { share } from '../../connection';
|
import { share } from '../../connection';
|
||||||
import { type DocClock, type DocClocks, SyncStorage } from '../../storage';
|
import { BasicSyncStorage, type DocClock, type DocClocks } from '../../storage';
|
||||||
import { IDBConnection } from './db';
|
import { IDBConnection } from './db';
|
||||||
export class IndexedDBSyncStorage extends SyncStorage {
|
export class IndexedDBSyncStorage extends BasicSyncStorage {
|
||||||
readonly connection = share(new IDBConnection(this.options));
|
readonly connection = share(new IDBConnection(this.options));
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
import { share } from '../../../connection';
|
import { share } from '../../../connection';
|
||||||
import { BlobStorage, type ListedBlobRecord } from '../../../storage';
|
import { BlobStorageBase, type ListedBlobRecord } from '../../../storage';
|
||||||
import { BlobIDBConnection } from './db';
|
import { BlobIDBConnection } from './db';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated readonly
|
* @deprecated readonly
|
||||||
*/
|
*/
|
||||||
export class IndexedDBV1BlobStorage extends BlobStorage {
|
export class IndexedDBV1BlobStorage extends BlobStorageBase {
|
||||||
readonly connection = share(new BlobIDBConnection(this.spaceId));
|
readonly connection = share(new BlobIDBConnection(this.spaceId));
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
|
import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
|
||||||
|
|
||||||
import { Connection } from '../../../connection';
|
import { AutoReconnectConnection } from '../../../connection';
|
||||||
|
|
||||||
export interface DocDBSchema extends DBSchema {
|
export interface DocDBSchema extends DBSchema {
|
||||||
workspace: {
|
workspace: {
|
||||||
@ -15,7 +15,9 @@ export interface DocDBSchema extends DBSchema {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export class DocIDBConnection extends Connection<IDBPDatabase<DocDBSchema>> {
|
export class DocIDBConnection extends AutoReconnectConnection<
|
||||||
|
IDBPDatabase<DocDBSchema>
|
||||||
|
> {
|
||||||
override get shareId() {
|
override get shareId() {
|
||||||
return 'idb(old):affine-local';
|
return 'idb(old):affine-local';
|
||||||
}
|
}
|
||||||
@ -40,7 +42,9 @@ export interface BlobDBSchema extends DBSchema {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export class BlobIDBConnection extends Connection<IDBPDatabase<BlobDBSchema>> {
|
export class BlobIDBConnection extends AutoReconnectConnection<
|
||||||
|
IDBPDatabase<BlobDBSchema>
|
||||||
|
> {
|
||||||
constructor(private readonly workspaceId: string) {
|
constructor(private readonly workspaceId: string) {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
@ -1,11 +1,15 @@
|
|||||||
import { share } from '../../../connection';
|
import { share } from '../../../connection';
|
||||||
import { type DocRecord, DocStorage, type DocUpdate } from '../../../storage';
|
import {
|
||||||
|
type DocRecord,
|
||||||
|
DocStorageBase,
|
||||||
|
type DocUpdate,
|
||||||
|
} from '../../../storage';
|
||||||
import { DocIDBConnection } from './db';
|
import { DocIDBConnection } from './db';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated readonly
|
* @deprecated readonly
|
||||||
*/
|
*/
|
||||||
export class IndexedDBV1DocStorage extends DocStorage {
|
export class IndexedDBV1DocStorage extends DocStorageBase {
|
||||||
readonly connection = share(new DocIDBConnection());
|
readonly connection = share(new DocIDBConnection());
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
|
@ -1,5 +1,10 @@
|
|||||||
import type { Storage } from '../storage';
|
import type { Storage } from '../storage';
|
||||||
import { CloudBlobStorage, CloudDocStorage } from './cloud';
|
import { BroadcastChannelAwarenessStorage } from './broadcast-channel/awareness';
|
||||||
|
import {
|
||||||
|
CloudAwarenessStorage,
|
||||||
|
CloudBlobStorage,
|
||||||
|
CloudDocStorage,
|
||||||
|
} from './cloud';
|
||||||
import {
|
import {
|
||||||
IndexedDBBlobStorage,
|
IndexedDBBlobStorage,
|
||||||
IndexedDBDocStorage,
|
IndexedDBDocStorage,
|
||||||
@ -13,6 +18,7 @@ const idb: StorageConstructor[] = [
|
|||||||
IndexedDBDocStorage,
|
IndexedDBDocStorage,
|
||||||
IndexedDBBlobStorage,
|
IndexedDBBlobStorage,
|
||||||
IndexedDBSyncStorage,
|
IndexedDBSyncStorage,
|
||||||
|
BroadcastChannelAwarenessStorage,
|
||||||
];
|
];
|
||||||
|
|
||||||
const idbv1: StorageConstructor[] = [
|
const idbv1: StorageConstructor[] = [
|
||||||
@ -20,7 +26,11 @@ const idbv1: StorageConstructor[] = [
|
|||||||
IndexedDBV1BlobStorage,
|
IndexedDBV1BlobStorage,
|
||||||
];
|
];
|
||||||
|
|
||||||
const cloud: StorageConstructor[] = [CloudDocStorage, CloudBlobStorage];
|
const cloud: StorageConstructor[] = [
|
||||||
|
CloudDocStorage,
|
||||||
|
CloudBlobStorage,
|
||||||
|
CloudAwarenessStorage,
|
||||||
|
];
|
||||||
|
|
||||||
export const storages: StorageConstructor[] = cloud.concat(idbv1, idb);
|
export const storages: StorageConstructor[] = cloud.concat(idbv1, idb);
|
||||||
|
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
import { share } from '../../connection';
|
import { share } from '../../connection';
|
||||||
import { type BlobRecord, BlobStorage } from '../../storage';
|
import { type BlobRecord, BlobStorageBase } from '../../storage';
|
||||||
import { NativeDBConnection } from './db';
|
import { NativeDBConnection } from './db';
|
||||||
|
|
||||||
export class SqliteBlobStorage extends BlobStorage {
|
export class SqliteBlobStorage extends BlobStorageBase {
|
||||||
override connection = share(
|
override connection = share(
|
||||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||||
);
|
);
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import { apis } from '@affine/electron-api';
|
import { apis } from '@affine/electron-api';
|
||||||
|
|
||||||
import { Connection } from '../../connection';
|
import { AutoReconnectConnection } from '../../connection';
|
||||||
import { type SpaceType, universalId } from '../../storage';
|
import { type SpaceType, universalId } from '../../storage';
|
||||||
|
|
||||||
type NativeDBApis = NonNullable<typeof apis>['nbstore'] extends infer APIs
|
type NativeDBApis = NonNullable<typeof apis>['nbstore'] extends infer APIs
|
||||||
@ -13,7 +13,7 @@ type NativeDBApis = NonNullable<typeof apis>['nbstore'] extends infer APIs
|
|||||||
}
|
}
|
||||||
: never;
|
: never;
|
||||||
|
|
||||||
export class NativeDBConnection extends Connection<void> {
|
export class NativeDBConnection extends AutoReconnectConnection<void> {
|
||||||
readonly apis: NativeDBApis;
|
readonly apis: NativeDBApis;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
import { share } from '../../connection';
|
import { share } from '../../connection';
|
||||||
import { type DocClock, DocStorage, type DocUpdate } from '../../storage';
|
import { type DocClock, DocStorageBase, type DocUpdate } from '../../storage';
|
||||||
import { NativeDBConnection } from './db';
|
import { NativeDBConnection } from './db';
|
||||||
|
|
||||||
export class SqliteDocStorage extends DocStorage {
|
export class SqliteDocStorage extends DocStorageBase {
|
||||||
override connection = share(
|
override connection = share(
|
||||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||||
);
|
);
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
import { share } from '../../connection';
|
import { share } from '../../connection';
|
||||||
import { type DocClock, SyncStorage } from '../../storage';
|
import { BasicSyncStorage, type DocClock } from '../../storage';
|
||||||
import { NativeDBConnection } from './db';
|
import { NativeDBConnection } from './db';
|
||||||
|
|
||||||
export class SqliteSyncStorage extends SyncStorage {
|
export class SqliteSyncStorage extends BasicSyncStorage {
|
||||||
override connection = share(
|
override connection = share(
|
||||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||||
);
|
);
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
import { apis } from '@affine/electron-api';
|
import { apis } from '@affine/electron-api';
|
||||||
|
|
||||||
import { DummyConnection, share } from '../../../connection';
|
import { DummyConnection } from '../../../connection';
|
||||||
import { BlobStorage } from '../../../storage';
|
import { BlobStorageBase } from '../../../storage';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated readonly
|
* @deprecated readonly
|
||||||
*/
|
*/
|
||||||
export class SqliteV1BlobStorage extends BlobStorage {
|
export class SqliteV1BlobStorage extends BlobStorageBase {
|
||||||
override connection = share(new DummyConnection());
|
override connection = new DummyConnection();
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
if (!apis) {
|
if (!apis) {
|
||||||
|
@ -1,13 +1,17 @@
|
|||||||
import { apis } from '@affine/electron-api';
|
import { apis } from '@affine/electron-api';
|
||||||
|
|
||||||
import { DummyConnection, share } from '../../../connection';
|
import { DummyConnection } from '../../../connection';
|
||||||
import { type DocRecord, DocStorage, type DocUpdate } from '../../../storage';
|
import {
|
||||||
|
type DocRecord,
|
||||||
|
DocStorageBase,
|
||||||
|
type DocUpdate,
|
||||||
|
} from '../../../storage';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated readonly
|
* @deprecated readonly
|
||||||
*/
|
*/
|
||||||
export class SqliteV1DocStorage extends DocStorage {
|
export class SqliteV1DocStorage extends DocStorageBase {
|
||||||
override connection = share(new DummyConnection());
|
override connection = new DummyConnection();
|
||||||
|
|
||||||
get db() {
|
get db() {
|
||||||
if (!apis) {
|
if (!apis) {
|
||||||
|
@ -1,128 +0,0 @@
|
|||||||
import type { OpConsumer } from '@toeverything/infra/op';
|
|
||||||
import { Observable } from 'rxjs';
|
|
||||||
|
|
||||||
import { getAvailableStorageImplementations } from '../impls';
|
|
||||||
import {
|
|
||||||
BlobStorage,
|
|
||||||
DocStorage,
|
|
||||||
HistoricalDocStorage,
|
|
||||||
SpaceStorage,
|
|
||||||
type Storage,
|
|
||||||
type StorageOptions,
|
|
||||||
SyncStorage,
|
|
||||||
} from '../storage';
|
|
||||||
import type { SpaceStorageOps } from './ops';
|
|
||||||
|
|
||||||
export class SpaceStorageConsumer extends SpaceStorage {
|
|
||||||
constructor(private readonly consumer: OpConsumer<SpaceStorageOps>) {
|
|
||||||
super([]);
|
|
||||||
this.registerConnectionHandlers();
|
|
||||||
this.listen();
|
|
||||||
}
|
|
||||||
|
|
||||||
listen() {
|
|
||||||
this.consumer.listen();
|
|
||||||
}
|
|
||||||
|
|
||||||
add(name: string, options: StorageOptions) {
|
|
||||||
const Storage = getAvailableStorageImplementations(name);
|
|
||||||
const storage = new Storage(options);
|
|
||||||
this.storages.set(storage.storageType, storage);
|
|
||||||
this.registerStorageHandlers(storage);
|
|
||||||
}
|
|
||||||
|
|
||||||
override async destroy() {
|
|
||||||
await super.destroy();
|
|
||||||
this.consumer.destroy();
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerConnectionHandlers() {
|
|
||||||
this.consumer.register('addStorage', ({ name, opts }) => {
|
|
||||||
this.add(name, opts);
|
|
||||||
});
|
|
||||||
this.consumer.register('connect', this.connect.bind(this));
|
|
||||||
this.consumer.register('disconnect', this.disconnect.bind(this));
|
|
||||||
this.consumer.register('destroy', this.destroy.bind(this));
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerStorageHandlers(storage: Storage) {
|
|
||||||
if (storage instanceof DocStorage) {
|
|
||||||
this.registerDocHandlers(storage);
|
|
||||||
} else if (storage instanceof BlobStorage) {
|
|
||||||
this.registerBlobHandlers(storage);
|
|
||||||
} else if (storage instanceof SyncStorage) {
|
|
||||||
this.registerSyncHandlers(storage);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerDocHandlers(storage: DocStorage) {
|
|
||||||
this.consumer.register('getDoc', storage.getDoc.bind(storage));
|
|
||||||
this.consumer.register('getDocDiff', ({ docId, state }) => {
|
|
||||||
return storage.getDocDiff(docId, state);
|
|
||||||
});
|
|
||||||
this.consumer.register('pushDocUpdate', ({ update, origin }) => {
|
|
||||||
return storage.pushDocUpdate(update, origin);
|
|
||||||
});
|
|
||||||
this.consumer.register(
|
|
||||||
'getDocTimestamps',
|
|
||||||
storage.getDocTimestamps.bind(storage)
|
|
||||||
);
|
|
||||||
this.consumer.register('deleteDoc', storage.deleteDoc.bind(storage));
|
|
||||||
this.consumer.register('subscribeDocUpdate', () => {
|
|
||||||
return new Observable(subscriber => {
|
|
||||||
subscriber.add(
|
|
||||||
storage.subscribeDocUpdate((update, origin) => {
|
|
||||||
subscriber.next({ update, origin });
|
|
||||||
})
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
if (storage instanceof HistoricalDocStorage) {
|
|
||||||
this.consumer.register('listHistory', ({ docId, filter }) => {
|
|
||||||
return storage.listHistories(docId, filter);
|
|
||||||
});
|
|
||||||
this.consumer.register('getHistory', ({ docId, timestamp }) => {
|
|
||||||
return storage.getHistory(docId, timestamp);
|
|
||||||
});
|
|
||||||
this.consumer.register('deleteHistory', ({ docId, timestamp }) => {
|
|
||||||
return storage.deleteHistory(docId, timestamp);
|
|
||||||
});
|
|
||||||
this.consumer.register('rollbackDoc', ({ docId, timestamp }) => {
|
|
||||||
return storage.rollbackDoc(docId, timestamp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerBlobHandlers(storage: BlobStorage) {
|
|
||||||
this.consumer.register('getBlob', key => {
|
|
||||||
return storage.get(key);
|
|
||||||
});
|
|
||||||
this.consumer.register('setBlob', blob => {
|
|
||||||
return storage.set(blob);
|
|
||||||
});
|
|
||||||
this.consumer.register('deleteBlob', ({ key, permanently }) => {
|
|
||||||
return storage.delete(key, permanently);
|
|
||||||
});
|
|
||||||
this.consumer.register('listBlobs', storage.list.bind(storage));
|
|
||||||
this.consumer.register('releaseBlobs', storage.release.bind(storage));
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerSyncHandlers(storage: SyncStorage) {
|
|
||||||
this.consumer.register(
|
|
||||||
'getPeerClocks',
|
|
||||||
storage.getPeerRemoteClocks.bind(storage)
|
|
||||||
);
|
|
||||||
this.consumer.register('setPeerClock', ({ peer, ...clock }) => {
|
|
||||||
return storage.setPeerRemoteClock(peer, clock);
|
|
||||||
});
|
|
||||||
this.consumer.register(
|
|
||||||
'getPeerPushedClocks',
|
|
||||||
storage.getPeerPushedClocks.bind(storage)
|
|
||||||
);
|
|
||||||
this.consumer.register('setPeerPushedClock', ({ peer, ...clock }) => {
|
|
||||||
return storage.setPeerPushedClock(peer, clock);
|
|
||||||
});
|
|
||||||
this.consumer.register('clearClocks', storage.clearClocks.bind(storage));
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,53 +0,0 @@
|
|||||||
import { OpClient } from '@toeverything/infra/op';
|
|
||||||
|
|
||||||
import type { Storage } from '../storage';
|
|
||||||
import type { SpaceStorageOps } from './ops';
|
|
||||||
|
|
||||||
export { SpaceStorageConsumer } from './consumer';
|
|
||||||
|
|
||||||
export class SpaceStorageClient extends OpClient<SpaceStorageOps> {
|
|
||||||
/**
|
|
||||||
* Adding a storage implementation to the backend.
|
|
||||||
*
|
|
||||||
* NOTE:
|
|
||||||
* Because the storage beckend might be put behind a worker, we cant pass the instance but only
|
|
||||||
* the constructor name and its options to let the backend construct the instance.
|
|
||||||
*/
|
|
||||||
async addStorage<T extends new (...args: any) => Storage>(
|
|
||||||
Impl: T,
|
|
||||||
...opts: ConstructorParameters<T>
|
|
||||||
) {
|
|
||||||
await this.call('addStorage', { name: Impl.name, opts: opts[0] });
|
|
||||||
}
|
|
||||||
|
|
||||||
async connect() {
|
|
||||||
await this.call('connect');
|
|
||||||
}
|
|
||||||
|
|
||||||
async disconnect() {
|
|
||||||
await this.call('disconnect');
|
|
||||||
}
|
|
||||||
|
|
||||||
override destroy() {
|
|
||||||
this.call('destroy').catch(console.error);
|
|
||||||
super.destroy();
|
|
||||||
}
|
|
||||||
|
|
||||||
connection$() {
|
|
||||||
return this.ob$('connection');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class SpaceStorageWorkerClient extends SpaceStorageClient {
|
|
||||||
private readonly worker: Worker;
|
|
||||||
constructor() {
|
|
||||||
const worker = new Worker(new URL('./worker.ts', import.meta.url));
|
|
||||||
super(worker);
|
|
||||||
this.worker = worker;
|
|
||||||
}
|
|
||||||
|
|
||||||
override destroy() {
|
|
||||||
super.destroy();
|
|
||||||
this.worker.terminate();
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,58 +0,0 @@
|
|||||||
import { type OpSchema } from '@toeverything/infra/op';
|
|
||||||
|
|
||||||
import type { ConnectionStatus } from '../connection';
|
|
||||||
import type {
|
|
||||||
BlobRecord,
|
|
||||||
DocClock,
|
|
||||||
DocClocks,
|
|
||||||
DocDiff,
|
|
||||||
DocRecord,
|
|
||||||
DocUpdate,
|
|
||||||
HistoryFilter,
|
|
||||||
ListedBlobRecord,
|
|
||||||
ListedHistory,
|
|
||||||
StorageOptions,
|
|
||||||
StorageType,
|
|
||||||
} from '../storage';
|
|
||||||
|
|
||||||
export interface SpaceStorageOps extends OpSchema {
|
|
||||||
// init
|
|
||||||
addStorage: [{ name: string; opts: StorageOptions }, void];
|
|
||||||
|
|
||||||
// connection
|
|
||||||
connect: [void, void];
|
|
||||||
disconnect: [void, void];
|
|
||||||
connection: [
|
|
||||||
void,
|
|
||||||
{ storage: StorageType; status: ConnectionStatus; error?: Error },
|
|
||||||
];
|
|
||||||
destroy: [void, void];
|
|
||||||
|
|
||||||
// doc
|
|
||||||
getDoc: [string, DocRecord | null];
|
|
||||||
getDocDiff: [{ docId: string; state?: Uint8Array }, DocDiff | null];
|
|
||||||
pushDocUpdate: [{ update: DocUpdate; origin?: string }, DocClock];
|
|
||||||
getDocTimestamps: [Date, DocClocks];
|
|
||||||
deleteDoc: [string, void];
|
|
||||||
subscribeDocUpdate: [void, { update: DocRecord; origin?: string }];
|
|
||||||
|
|
||||||
// history
|
|
||||||
listHistory: [{ docId: string; filter?: HistoryFilter }, ListedHistory[]];
|
|
||||||
getHistory: [DocClock, DocRecord | null];
|
|
||||||
deleteHistory: [DocClock, void];
|
|
||||||
rollbackDoc: [DocClock & { editor?: string }, void];
|
|
||||||
|
|
||||||
// blob
|
|
||||||
getBlob: [string, BlobRecord | null];
|
|
||||||
setBlob: [BlobRecord, void];
|
|
||||||
deleteBlob: [{ key: string; permanently: boolean }, void];
|
|
||||||
releaseBlobs: [void, void];
|
|
||||||
listBlobs: [void, ListedBlobRecord[]];
|
|
||||||
|
|
||||||
// sync
|
|
||||||
getPeerClocks: [string, DocClocks];
|
|
||||||
setPeerClock: [{ peer: string } & DocClock, void];
|
|
||||||
getPeerPushedClocks: [string, DocClocks];
|
|
||||||
setPeerPushedClock: [{ peer: string } & DocClock, void];
|
|
||||||
clearClocks: [void, void];
|
|
||||||
}
|
|
@ -1,11 +0,0 @@
|
|||||||
import { OpConsumer } from '@toeverything/infra/op';
|
|
||||||
|
|
||||||
import { SpaceStorageConsumer } from './consumer';
|
|
||||||
import type { SpaceStorageOps } from './ops';
|
|
||||||
|
|
||||||
const consumer = new SpaceStorageConsumer(
|
|
||||||
// @ts-expect-error safe
|
|
||||||
new OpConsumer<SpaceStorageOps>(self)
|
|
||||||
);
|
|
||||||
|
|
||||||
consumer.listen();
|
|
@ -1,4 +1,4 @@
|
|||||||
import { Storage, type StorageOptions } from './storage';
|
import { type Storage, StorageBase, type StorageOptions } from './storage';
|
||||||
|
|
||||||
export interface AwarenessStorageOptions extends StorageOptions {}
|
export interface AwarenessStorageOptions extends StorageOptions {}
|
||||||
|
|
||||||
@ -7,21 +7,35 @@ export type AwarenessRecord = {
|
|||||||
bin: Uint8Array;
|
bin: Uint8Array;
|
||||||
};
|
};
|
||||||
|
|
||||||
export abstract class AwarenessStorage<
|
export interface AwarenessStorage extends Storage {
|
||||||
Options extends AwarenessStorageOptions = AwarenessStorageOptions,
|
readonly storageType: 'awareness';
|
||||||
> extends Storage<Options> {
|
|
||||||
override readonly storageType = 'awareness';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update the awareness record.
|
* Update the awareness record.
|
||||||
*
|
*
|
||||||
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
|
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
|
||||||
*/
|
*/
|
||||||
|
update(record: AwarenessRecord, origin?: string): Promise<void>;
|
||||||
|
subscribeUpdate(
|
||||||
|
id: string,
|
||||||
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
|
): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export abstract class AwarenessStorageBase<
|
||||||
|
Options extends AwarenessStorageOptions = AwarenessStorageOptions,
|
||||||
|
>
|
||||||
|
extends StorageBase<Options>
|
||||||
|
implements AwarenessStorage
|
||||||
|
{
|
||||||
|
override readonly storageType = 'awareness';
|
||||||
|
|
||||||
abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
|
abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
|
||||||
|
|
||||||
abstract subscribeUpdate(
|
abstract subscribeUpdate(
|
||||||
id: string,
|
id: string,
|
||||||
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
onCollect: () => AwarenessRecord
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
): () => void;
|
): () => void;
|
||||||
}
|
}
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
import { Storage, type StorageOptions } from './storage';
|
import { type Storage, StorageBase, type StorageOptions } from './storage';
|
||||||
|
|
||||||
export interface BlobStorageOptions extends StorageOptions {}
|
export interface BlobStorageOptions extends StorageOptions {}
|
||||||
|
|
||||||
@ -16,9 +16,25 @@ export interface ListedBlobRecord {
|
|||||||
createdAt?: Date;
|
createdAt?: Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
export abstract class BlobStorage<
|
export interface BlobStorage extends Storage {
|
||||||
|
readonly storageType: 'blob';
|
||||||
|
get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;
|
||||||
|
set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
|
||||||
|
delete(
|
||||||
|
key: string,
|
||||||
|
permanently: boolean,
|
||||||
|
signal?: AbortSignal
|
||||||
|
): Promise<void>;
|
||||||
|
release(signal?: AbortSignal): Promise<void>;
|
||||||
|
list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export abstract class BlobStorageBase<
|
||||||
Options extends BlobStorageOptions = BlobStorageOptions,
|
Options extends BlobStorageOptions = BlobStorageOptions,
|
||||||
> extends Storage<Options> {
|
>
|
||||||
|
extends StorageBase<Options>
|
||||||
|
implements BlobStorage
|
||||||
|
{
|
||||||
override readonly storageType = 'blob';
|
override readonly storageType = 'blob';
|
||||||
|
|
||||||
abstract get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;
|
abstract get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;
|
||||||
|
@ -4,7 +4,7 @@ import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs';
|
|||||||
import { isEmptyUpdate } from '../utils/is-empty-update';
|
import { isEmptyUpdate } from '../utils/is-empty-update';
|
||||||
import type { Locker } from './lock';
|
import type { Locker } from './lock';
|
||||||
import { SingletonLocker } from './lock';
|
import { SingletonLocker } from './lock';
|
||||||
import { Storage, type StorageOptions } from './storage';
|
import { type Storage, StorageBase, type StorageOptions } from './storage';
|
||||||
|
|
||||||
export interface DocClock {
|
export interface DocClock {
|
||||||
docId: string;
|
docId: string;
|
||||||
@ -37,17 +37,67 @@ export interface DocStorageOptions extends StorageOptions {
|
|||||||
mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
|
mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
|
||||||
}
|
}
|
||||||
|
|
||||||
export abstract class DocStorage<
|
export interface DocStorage extends Storage {
|
||||||
|
readonly storageType: 'doc';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a doc record with latest binary.
|
||||||
|
*/
|
||||||
|
getDoc(docId: string): Promise<DocRecord | null>;
|
||||||
|
/**
|
||||||
|
* Get a yjs binary diff with the given state vector.
|
||||||
|
*/
|
||||||
|
getDocDiff(docId: string, state?: Uint8Array): Promise<DocDiff | null>;
|
||||||
|
/**
|
||||||
|
* Push updates into storage
|
||||||
|
*
|
||||||
|
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
|
||||||
|
*/
|
||||||
|
pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the timestamp of the latest update of a doc.
|
||||||
|
*/
|
||||||
|
getDocTimestamp(docId: string): Promise<DocClock | null>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all docs timestamps info. especially for useful in sync process.
|
||||||
|
*/
|
||||||
|
getDocTimestamps(after?: Date): Promise<DocClocks>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete a specific doc data with all snapshots and updates
|
||||||
|
*/
|
||||||
|
deleteDoc(docId: string): Promise<void>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Subscribe on doc updates emitted from storage itself.
|
||||||
|
*
|
||||||
|
* NOTE:
|
||||||
|
*
|
||||||
|
* There is not always update emitted from storage itself.
|
||||||
|
*
|
||||||
|
* For example, in Sqlite storage, the update will only come from user's updating on docs,
|
||||||
|
* in other words, the update will never somehow auto generated in storage internally.
|
||||||
|
*
|
||||||
|
* But for Cloud storage, there will be updates broadcasted from other clients,
|
||||||
|
* so the storage will emit updates to notify the client to integrate them.
|
||||||
|
*/
|
||||||
|
subscribeDocUpdate(
|
||||||
|
callback: (update: DocRecord, origin?: string) => void
|
||||||
|
): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export abstract class DocStorageBase<
|
||||||
Opts extends DocStorageOptions = DocStorageOptions,
|
Opts extends DocStorageOptions = DocStorageOptions,
|
||||||
> extends Storage<Opts> {
|
>
|
||||||
|
extends StorageBase<Opts>
|
||||||
|
implements DocStorage
|
||||||
|
{
|
||||||
private readonly event = new EventEmitter2();
|
private readonly event = new EventEmitter2();
|
||||||
override readonly storageType = 'doc';
|
override readonly storageType = 'doc';
|
||||||
protected readonly locker: Locker = new SingletonLocker();
|
protected readonly locker: Locker = new SingletonLocker();
|
||||||
|
|
||||||
// REGION: open apis by Op system
|
|
||||||
/**
|
|
||||||
* Get a doc record with latest binary.
|
|
||||||
*/
|
|
||||||
async getDoc(docId: string) {
|
async getDoc(docId: string) {
|
||||||
await using _lock = await this.lockDocForUpdate(docId);
|
await using _lock = await this.lockDocForUpdate(docId);
|
||||||
|
|
||||||
@ -78,9 +128,6 @@ export abstract class DocStorage<
|
|||||||
return snapshot;
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a yjs binary diff with the given state vector.
|
|
||||||
*/
|
|
||||||
async getDocDiff(docId: string, state?: Uint8Array) {
|
async getDocDiff(docId: string, state?: Uint8Array) {
|
||||||
const doc = await this.getDoc(docId);
|
const doc = await this.getDoc(docId);
|
||||||
|
|
||||||
@ -96,41 +143,14 @@ export abstract class DocStorage<
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Push updates into storage
|
|
||||||
*
|
|
||||||
* @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
|
|
||||||
*/
|
|
||||||
abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
|
abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the timestamp of the latest update of a doc.
|
|
||||||
*/
|
|
||||||
abstract getDocTimestamp(docId: string): Promise<DocClock | null>;
|
abstract getDocTimestamp(docId: string): Promise<DocClock | null>;
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all docs timestamps info. especially for useful in sync process.
|
|
||||||
*/
|
|
||||||
abstract getDocTimestamps(after?: Date): Promise<DocClocks>;
|
abstract getDocTimestamps(after?: Date): Promise<DocClocks>;
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete a specific doc data with all snapshots and updates
|
|
||||||
*/
|
|
||||||
abstract deleteDoc(docId: string): Promise<void>;
|
abstract deleteDoc(docId: string): Promise<void>;
|
||||||
|
|
||||||
/**
|
|
||||||
* Subscribe on doc updates emitted from storage itself.
|
|
||||||
*
|
|
||||||
* NOTE:
|
|
||||||
*
|
|
||||||
* There is not always update emitted from storage itself.
|
|
||||||
*
|
|
||||||
* For example, in Sqlite storage, the update will only come from user's updating on docs,
|
|
||||||
* in other words, the update will never somehow auto generated in storage internally.
|
|
||||||
*
|
|
||||||
* But for Cloud storage, there will be updates broadcasted from other clients,
|
|
||||||
* so the storage will emit updates to notify the client to integrate them.
|
|
||||||
*/
|
|
||||||
subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
|
subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
|
||||||
this.event.on('update', callback);
|
this.event.on('update', callback);
|
||||||
|
|
||||||
@ -138,7 +158,6 @@ export abstract class DocStorage<
|
|||||||
this.event.off('update', callback);
|
this.event.off('update', callback);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
// ENDREGION
|
|
||||||
|
|
||||||
// REGION: api for internal usage
|
// REGION: api for internal usage
|
||||||
protected on(
|
protected on(
|
||||||
|
@ -7,7 +7,7 @@ import {
|
|||||||
UndoManager,
|
UndoManager,
|
||||||
} from 'yjs';
|
} from 'yjs';
|
||||||
|
|
||||||
import { type DocRecord, DocStorage, type DocStorageOptions } from './doc';
|
import { type DocRecord, DocStorageBase, type DocStorageOptions } from './doc';
|
||||||
|
|
||||||
export interface HistoryFilter {
|
export interface HistoryFilter {
|
||||||
before?: Date;
|
before?: Date;
|
||||||
@ -21,7 +21,7 @@ export interface ListedHistory {
|
|||||||
|
|
||||||
export abstract class HistoricalDocStorage<
|
export abstract class HistoricalDocStorage<
|
||||||
Options extends DocStorageOptions = DocStorageOptions,
|
Options extends DocStorageOptions = DocStorageOptions,
|
||||||
> extends DocStorage<Options> {
|
> extends DocStorageBase<Options> {
|
||||||
constructor(opts: Options) {
|
constructor(opts: Options) {
|
||||||
super(opts);
|
super(opts);
|
||||||
|
|
||||||
|
@ -40,20 +40,20 @@ export class SpaceStorage {
|
|||||||
|
|
||||||
connect() {
|
connect() {
|
||||||
Array.from(this.storages.values()).forEach(storage => {
|
Array.from(this.storages.values()).forEach(storage => {
|
||||||
storage.connect();
|
storage.connection.connect();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
disconnect() {
|
disconnect() {
|
||||||
Array.from(this.storages.values()).forEach(storage => {
|
Array.from(this.storages.values()).forEach(storage => {
|
||||||
storage.disconnect();
|
storage.connection.disconnect();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async waitForConnected() {
|
async waitForConnected(signal?: AbortSignal) {
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
Array.from(this.storages.values()).map(storage =>
|
Array.from(this.storages.values()).map(storage =>
|
||||||
storage.waitForConnected()
|
storage.connection.waitForConnected(signal)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -65,6 +65,7 @@ export class SpaceStorage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export * from './awareness';
|
||||||
export * from './blob';
|
export * from './blob';
|
||||||
export * from './doc';
|
export * from './doc';
|
||||||
export * from './history';
|
export * from './history';
|
||||||
|
@ -80,7 +80,18 @@ export function parseUniversalId(id: string) {
|
|||||||
return result as any;
|
return result as any;
|
||||||
}
|
}
|
||||||
|
|
||||||
export abstract class Storage<Opts extends StorageOptions = StorageOptions> {
|
export interface Storage {
|
||||||
|
readonly storageType: StorageType;
|
||||||
|
readonly connection: Connection;
|
||||||
|
readonly peer: string;
|
||||||
|
readonly spaceType: string;
|
||||||
|
readonly spaceId: string;
|
||||||
|
readonly universalId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export abstract class StorageBase<Opts extends StorageOptions = StorageOptions>
|
||||||
|
implements Storage
|
||||||
|
{
|
||||||
abstract readonly storageType: StorageType;
|
abstract readonly storageType: StorageType;
|
||||||
abstract readonly connection: Connection;
|
abstract readonly connection: Connection;
|
||||||
|
|
||||||
@ -101,16 +112,4 @@ export abstract class Storage<Opts extends StorageOptions = StorageOptions> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
constructor(public readonly options: Opts) {}
|
constructor(public readonly options: Opts) {}
|
||||||
|
|
||||||
connect() {
|
|
||||||
this.connection.connect();
|
|
||||||
}
|
|
||||||
|
|
||||||
disconnect() {
|
|
||||||
this.connection.disconnect();
|
|
||||||
}
|
|
||||||
|
|
||||||
async waitForConnected() {
|
|
||||||
await this.connection.waitForConnected();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -1,11 +1,32 @@
|
|||||||
import type { DocClock, DocClocks } from './doc';
|
import type { DocClock, DocClocks } from './doc';
|
||||||
import { Storage, type StorageOptions } from './storage';
|
import { type Storage, StorageBase, type StorageOptions } from './storage';
|
||||||
|
|
||||||
export interface SyncStorageOptions extends StorageOptions {}
|
export interface SyncStorageOptions extends StorageOptions {}
|
||||||
|
|
||||||
export abstract class SyncStorage<
|
export interface SyncStorage extends Storage {
|
||||||
|
readonly storageType: 'sync';
|
||||||
|
|
||||||
|
getPeerRemoteClock(peer: string, docId: string): Promise<DocClock | null>;
|
||||||
|
getPeerRemoteClocks(peer: string): Promise<DocClocks>;
|
||||||
|
setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
|
||||||
|
getPeerPulledRemoteClock(
|
||||||
|
peer: string,
|
||||||
|
docId: string
|
||||||
|
): Promise<DocClock | null>;
|
||||||
|
getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
|
||||||
|
setPeerPulledRemoteClock(peer: string, clock: DocClock): Promise<void>;
|
||||||
|
getPeerPushedClock(peer: string, docId: string): Promise<DocClock | null>;
|
||||||
|
getPeerPushedClocks(peer: string): Promise<DocClocks>;
|
||||||
|
setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
|
||||||
|
clearClocks(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export abstract class BasicSyncStorage<
|
||||||
Opts extends SyncStorageOptions = SyncStorageOptions,
|
Opts extends SyncStorageOptions = SyncStorageOptions,
|
||||||
> extends Storage<Opts> {
|
>
|
||||||
|
extends StorageBase<Opts>
|
||||||
|
implements SyncStorage
|
||||||
|
{
|
||||||
override readonly storageType = 'sync';
|
override readonly storageType = 'sync';
|
||||||
|
|
||||||
abstract getPeerRemoteClock(
|
abstract getPeerRemoteClock(
|
||||||
|
@ -3,7 +3,16 @@ import type {
|
|||||||
AwarenessStorage,
|
AwarenessStorage,
|
||||||
} from '../../storage/awareness';
|
} from '../../storage/awareness';
|
||||||
|
|
||||||
export class AwarenessSync {
|
export interface AwarenessSync {
|
||||||
|
update(record: AwarenessRecord, origin?: string): Promise<void>;
|
||||||
|
subscribeUpdate(
|
||||||
|
id: string,
|
||||||
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
|
): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AwarenessSyncImpl implements AwarenessSync {
|
||||||
constructor(
|
constructor(
|
||||||
readonly local: AwarenessStorage,
|
readonly local: AwarenessStorage,
|
||||||
readonly remotes: AwarenessStorage[]
|
readonly remotes: AwarenessStorage[]
|
||||||
@ -18,7 +27,7 @@ export class AwarenessSync {
|
|||||||
subscribeUpdate(
|
subscribeUpdate(
|
||||||
id: string,
|
id: string,
|
||||||
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
onCollect: () => AwarenessRecord
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
): () => void {
|
): () => void {
|
||||||
const unsubscribes = [this.local, ...this.remotes].map(peer =>
|
const unsubscribes = [this.local, ...this.remotes].map(peer =>
|
||||||
peer.subscribeUpdate(id, onUpdate, onCollect)
|
peer.subscribeUpdate(id, onUpdate, onCollect)
|
||||||
|
@ -3,7 +3,15 @@ import { difference } from 'lodash-es';
|
|||||||
import type { BlobRecord, BlobStorage } from '../../storage';
|
import type { BlobRecord, BlobStorage } from '../../storage';
|
||||||
import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
|
import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
|
||||||
|
|
||||||
export class BlobSync {
|
export interface BlobSync {
|
||||||
|
downloadBlob(
|
||||||
|
blobId: string,
|
||||||
|
signal?: AbortSignal
|
||||||
|
): Promise<BlobRecord | null>;
|
||||||
|
uploadBlob(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class BlobSyncImpl implements BlobSync {
|
||||||
private abort: AbortController | null = null;
|
private abort: AbortController | null = null;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
|
@ -17,7 +17,13 @@ export interface DocSyncDocState {
|
|||||||
errorMessage: string | null;
|
errorMessage: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class DocSync {
|
export interface DocSync {
|
||||||
|
readonly state$: Observable<DocSyncState>;
|
||||||
|
docState$(docId: string): Observable<DocSyncDocState>;
|
||||||
|
addPriority(id: string, priority: number): () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DocSyncImpl implements DocSync {
|
||||||
private readonly peers: DocSyncPeer[] = this.remotes.map(
|
private readonly peers: DocSyncPeer[] = this.remotes.map(
|
||||||
remote => new DocSyncPeer(this.local, this.sync, remote)
|
remote => new DocSyncPeer(this.local, this.sync, remote)
|
||||||
);
|
);
|
||||||
|
@ -92,7 +92,7 @@ export class DocSyncPeer {
|
|||||||
/**
|
/**
|
||||||
* random unique id for recognize self in "update" event
|
* random unique id for recognize self in "update" event
|
||||||
*/
|
*/
|
||||||
private readonly uniqueId = `sync:${this.local.peer}:${this.remote.peer}:${nanoid()}`;
|
private readonly uniqueId = `sync:${this.local.universalId}:${this.remote.universalId}:${nanoid()}`;
|
||||||
private readonly prioritySettings = new Map<string, number>();
|
private readonly prioritySettings = new Map<string, number>();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@ -435,7 +435,6 @@ export class DocSyncPeer {
|
|||||||
};
|
};
|
||||||
|
|
||||||
async mainLoop(signal?: AbortSignal) {
|
async mainLoop(signal?: AbortSignal) {
|
||||||
// eslint-disable-next-line no-constant-condition
|
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
await this.retryLoop(signal);
|
await this.retryLoop(signal);
|
||||||
@ -594,12 +593,12 @@ export class DocSyncPeer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// begin to process jobs
|
// begin to process jobs
|
||||||
// eslint-disable-next-line no-constant-condition
|
|
||||||
while (true) {
|
while (true) {
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
|
|
||||||
const docId = await this.status.jobDocQueue.asyncPop(signal);
|
const docId = await this.status.jobDocQueue.asyncPop(signal);
|
||||||
// eslint-disable-next-line no-constant-condition
|
|
||||||
while (true) {
|
while (true) {
|
||||||
// batch process jobs for the same doc
|
// batch process jobs for the same doc
|
||||||
const jobs = this.status.jobMap.get(docId);
|
const jobs = this.status.jobMap.get(docId);
|
||||||
|
@ -1,19 +1,23 @@
|
|||||||
import { combineLatest, map, type Observable, of } from 'rxjs';
|
import { combineLatest, map, type Observable, of } from 'rxjs';
|
||||||
|
|
||||||
import type { BlobStorage, DocStorage, SpaceStorage } from '../storage';
|
import type {
|
||||||
import type { AwarenessStorage } from '../storage/awareness';
|
AwarenessStorage,
|
||||||
import { AwarenessSync } from './awareness';
|
BlobStorage,
|
||||||
import { BlobSync } from './blob';
|
DocStorage,
|
||||||
import { DocSync, type DocSyncState } from './doc';
|
SpaceStorage,
|
||||||
|
} from '../storage';
|
||||||
|
import { AwarenessSyncImpl } from './awareness';
|
||||||
|
import { BlobSyncImpl } from './blob';
|
||||||
|
import { DocSyncImpl, type DocSyncState } from './doc';
|
||||||
|
|
||||||
export interface SyncState {
|
export interface SyncState {
|
||||||
doc?: DocSyncState;
|
doc?: DocSyncState;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class Sync {
|
export class Sync {
|
||||||
readonly doc: DocSync | null;
|
readonly doc: DocSyncImpl | null;
|
||||||
readonly blob: BlobSync | null;
|
readonly blob: BlobSyncImpl | null;
|
||||||
readonly awareness: AwarenessSync | null;
|
readonly awareness: AwarenessSyncImpl | null;
|
||||||
|
|
||||||
readonly state$: Observable<SyncState>;
|
readonly state$: Observable<SyncState>;
|
||||||
|
|
||||||
@ -28,7 +32,7 @@ export class Sync {
|
|||||||
|
|
||||||
this.doc =
|
this.doc =
|
||||||
doc && sync
|
doc && sync
|
||||||
? new DocSync(
|
? new DocSyncImpl(
|
||||||
doc,
|
doc,
|
||||||
sync,
|
sync,
|
||||||
peers
|
peers
|
||||||
@ -37,7 +41,7 @@ export class Sync {
|
|||||||
)
|
)
|
||||||
: null;
|
: null;
|
||||||
this.blob = blob
|
this.blob = blob
|
||||||
? new BlobSync(
|
? new BlobSyncImpl(
|
||||||
blob,
|
blob,
|
||||||
peers
|
peers
|
||||||
.map(peer => peer.tryGet('blob'))
|
.map(peer => peer.tryGet('blob'))
|
||||||
@ -45,7 +49,7 @@ export class Sync {
|
|||||||
)
|
)
|
||||||
: null;
|
: null;
|
||||||
this.awareness = awareness
|
this.awareness = awareness
|
||||||
? new AwarenessSync(
|
? new AwarenessSyncImpl(
|
||||||
awareness,
|
awareness,
|
||||||
peers
|
peers
|
||||||
.map(peer => peer.tryGet('awareness'))
|
.map(peer => peer.tryGet('awareness'))
|
||||||
|
294
packages/common/nbstore/src/worker/client.ts
Normal file
294
packages/common/nbstore/src/worker/client.ts
Normal file
@ -0,0 +1,294 @@
|
|||||||
|
import type { OpClient } from '@toeverything/infra/op';
|
||||||
|
|
||||||
|
import { DummyConnection } from '../connection';
|
||||||
|
import { DocFrontend } from '../frontend/doc';
|
||||||
|
import {
|
||||||
|
type AwarenessRecord,
|
||||||
|
type AwarenessStorage,
|
||||||
|
type BlobRecord,
|
||||||
|
type BlobStorage,
|
||||||
|
type DocRecord,
|
||||||
|
type DocStorage,
|
||||||
|
type DocUpdate,
|
||||||
|
type ListedBlobRecord,
|
||||||
|
type StorageOptions,
|
||||||
|
universalId,
|
||||||
|
} from '../storage';
|
||||||
|
import type { AwarenessSync } from '../sync/awareness';
|
||||||
|
import type { BlobSync } from '../sync/blob';
|
||||||
|
import type { DocSync } from '../sync/doc';
|
||||||
|
import type { WorkerOps } from './ops';
|
||||||
|
|
||||||
|
export class WorkerClient {
|
||||||
|
constructor(
|
||||||
|
private readonly client: OpClient<WorkerOps>,
|
||||||
|
private readonly options: StorageOptions
|
||||||
|
) {}
|
||||||
|
|
||||||
|
readonly docStorage = new WorkerDocStorage(this.client, this.options);
|
||||||
|
readonly blobStorage = new WorkerBlobStorage(this.client, this.options);
|
||||||
|
readonly awarenessStorage = new WorkerAwarenessStorage(
|
||||||
|
this.client,
|
||||||
|
this.options
|
||||||
|
);
|
||||||
|
readonly docSync = new WorkerDocSync(this.client);
|
||||||
|
readonly blobSync = new WorkerBlobSync(this.client);
|
||||||
|
readonly awarenessSync = new WorkerAwarenessSync(this.client);
|
||||||
|
|
||||||
|
readonly docFrontend = new DocFrontend(this.docStorage, this.docSync);
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerDocStorage implements DocStorage {
|
||||||
|
constructor(
|
||||||
|
private readonly client: OpClient<WorkerOps>,
|
||||||
|
private readonly options: StorageOptions
|
||||||
|
) {}
|
||||||
|
|
||||||
|
readonly peer = this.options.peer;
|
||||||
|
readonly spaceType = this.options.type;
|
||||||
|
readonly spaceId = this.options.id;
|
||||||
|
readonly universalId = universalId(this.options);
|
||||||
|
readonly storageType = 'doc';
|
||||||
|
|
||||||
|
async getDoc(docId: string) {
|
||||||
|
return this.client.call('docStorage.getDoc', docId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getDocDiff(docId: string, state?: Uint8Array) {
|
||||||
|
return this.client.call('docStorage.getDocDiff', { docId, state });
|
||||||
|
}
|
||||||
|
|
||||||
|
async pushDocUpdate(update: DocUpdate, origin?: string) {
|
||||||
|
return this.client.call('docStorage.pushDocUpdate', { update, origin });
|
||||||
|
}
|
||||||
|
|
||||||
|
async getDocTimestamp(docId: string) {
|
||||||
|
return this.client.call('docStorage.getDocTimestamp', docId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getDocTimestamps(after?: Date) {
|
||||||
|
return this.client.call('docStorage.getDocTimestamps', after ?? null);
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteDoc(docId: string) {
|
||||||
|
return this.client.call('docStorage.deleteDoc', docId);
|
||||||
|
}
|
||||||
|
|
||||||
|
subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
|
||||||
|
const subscription = this.client
|
||||||
|
.ob$('docStorage.subscribeDocUpdate')
|
||||||
|
.subscribe(value => {
|
||||||
|
callback(value.update, value.origin);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
subscription.unsubscribe();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
connection = new WorkerDocConnection(this.client);
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerDocConnection extends DummyConnection {
|
||||||
|
constructor(private readonly client: OpClient<WorkerOps>) {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
override waitForConnected(signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const abortListener = () => {
|
||||||
|
reject(signal?.reason);
|
||||||
|
subscription.unsubscribe();
|
||||||
|
};
|
||||||
|
|
||||||
|
signal?.addEventListener('abort', abortListener);
|
||||||
|
|
||||||
|
const subscription = this.client
|
||||||
|
.ob$('docStorage.waitForConnected')
|
||||||
|
.subscribe({
|
||||||
|
next() {
|
||||||
|
signal?.removeEventListener('abort', abortListener);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
error(err) {
|
||||||
|
signal?.removeEventListener('abort', abortListener);
|
||||||
|
reject(err);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerBlobStorage implements BlobStorage {
|
||||||
|
constructor(
|
||||||
|
private readonly client: OpClient<WorkerOps>,
|
||||||
|
private readonly options: StorageOptions
|
||||||
|
) {}
|
||||||
|
|
||||||
|
readonly storageType = 'blob';
|
||||||
|
readonly peer = this.options.peer;
|
||||||
|
readonly spaceType = this.options.type;
|
||||||
|
readonly spaceId = this.options.id;
|
||||||
|
readonly universalId = universalId(this.options);
|
||||||
|
|
||||||
|
get(key: string, _signal?: AbortSignal): Promise<BlobRecord | null> {
|
||||||
|
return this.client.call('blobStorage.getBlob', key);
|
||||||
|
}
|
||||||
|
set(blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
|
||||||
|
return this.client.call('blobStorage.setBlob', blob);
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(
|
||||||
|
key: string,
|
||||||
|
permanently: boolean,
|
||||||
|
_signal?: AbortSignal
|
||||||
|
): Promise<void> {
|
||||||
|
return this.client.call('blobStorage.deleteBlob', { key, permanently });
|
||||||
|
}
|
||||||
|
|
||||||
|
release(_signal?: AbortSignal): Promise<void> {
|
||||||
|
return this.client.call('blobStorage.releaseBlobs');
|
||||||
|
}
|
||||||
|
|
||||||
|
list(_signal?: AbortSignal): Promise<ListedBlobRecord[]> {
|
||||||
|
return this.client.call('blobStorage.listBlobs');
|
||||||
|
}
|
||||||
|
|
||||||
|
connection = new DummyConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerAwarenessStorage implements AwarenessStorage {
|
||||||
|
constructor(
|
||||||
|
private readonly client: OpClient<WorkerOps>,
|
||||||
|
private readonly options: StorageOptions
|
||||||
|
) {}
|
||||||
|
|
||||||
|
readonly storageType = 'awareness';
|
||||||
|
readonly peer = this.options.peer;
|
||||||
|
readonly spaceType = this.options.type;
|
||||||
|
readonly spaceId = this.options.id;
|
||||||
|
readonly universalId = universalId(this.options);
|
||||||
|
|
||||||
|
update(record: AwarenessRecord, origin?: string): Promise<void> {
|
||||||
|
return this.client.call('awarenessStorage.update', {
|
||||||
|
awareness: record,
|
||||||
|
origin,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
subscribeUpdate(
|
||||||
|
id: string,
|
||||||
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
|
): () => void {
|
||||||
|
const subscription = this.client
|
||||||
|
.ob$('awarenessStorage.subscribeUpdate', id)
|
||||||
|
.subscribe({
|
||||||
|
next: update => {
|
||||||
|
if (update.type === 'awareness-update') {
|
||||||
|
onUpdate(update.awareness, update.origin);
|
||||||
|
}
|
||||||
|
if (update.type === 'awareness-collect') {
|
||||||
|
onCollect()
|
||||||
|
.then(record => {
|
||||||
|
if (record) {
|
||||||
|
this.client
|
||||||
|
.call('awarenessStorage.collect', {
|
||||||
|
awareness: record,
|
||||||
|
collectId: update.collectId,
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
console.error('error feedback collected awareness', err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
console.error('error collecting awareness', err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
subscription.unsubscribe();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
connection = new DummyConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerDocSync implements DocSync {
|
||||||
|
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||||
|
|
||||||
|
readonly state$ = this.client.ob$('docSync.state');
|
||||||
|
|
||||||
|
docState$(docId: string) {
|
||||||
|
return this.client.ob$('docSync.docState', docId);
|
||||||
|
}
|
||||||
|
|
||||||
|
addPriority(docId: string, priority: number) {
|
||||||
|
const subscription = this.client
|
||||||
|
.ob$('docSync.addPriority', { docId, priority })
|
||||||
|
.subscribe();
|
||||||
|
return () => {
|
||||||
|
subscription.unsubscribe();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerBlobSync implements BlobSync {
|
||||||
|
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||||
|
downloadBlob(
|
||||||
|
blobId: string,
|
||||||
|
_signal?: AbortSignal
|
||||||
|
): Promise<BlobRecord | null> {
|
||||||
|
return this.client.call('blobSync.downloadBlob', blobId);
|
||||||
|
}
|
||||||
|
uploadBlob(blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
|
||||||
|
return this.client.call('blobSync.uploadBlob', blob);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class WorkerAwarenessSync implements AwarenessSync {
|
||||||
|
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||||
|
|
||||||
|
update(record: AwarenessRecord, origin?: string): Promise<void> {
|
||||||
|
return this.client.call('awarenessSync.update', {
|
||||||
|
awareness: record,
|
||||||
|
origin,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
subscribeUpdate(
|
||||||
|
id: string,
|
||||||
|
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||||
|
onCollect: () => Promise<AwarenessRecord | null>
|
||||||
|
): () => void {
|
||||||
|
const subscription = this.client
|
||||||
|
.ob$('awarenessSync.subscribeUpdate', id)
|
||||||
|
.subscribe({
|
||||||
|
next: update => {
|
||||||
|
if (update.type === 'awareness-update') {
|
||||||
|
onUpdate(update.awareness, update.origin);
|
||||||
|
}
|
||||||
|
if (update.type === 'awareness-collect') {
|
||||||
|
onCollect()
|
||||||
|
.then(record => {
|
||||||
|
if (record) {
|
||||||
|
this.client
|
||||||
|
.call('awarenessSync.collect', {
|
||||||
|
awareness: record,
|
||||||
|
collectId: update.collectId,
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
console.error('error feedback collected awareness', err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
console.error('error collecting awareness', err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
subscription.unsubscribe();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
256
packages/common/nbstore/src/worker/consumer.ts
Normal file
256
packages/common/nbstore/src/worker/consumer.ts
Normal file
@ -0,0 +1,256 @@
|
|||||||
|
import type { OpConsumer } from '@toeverything/infra/op';
|
||||||
|
import { Observable } from 'rxjs';
|
||||||
|
|
||||||
|
import { getAvailableStorageImplementations } from '../impls';
|
||||||
|
import { SpaceStorage, type StorageOptions } from '../storage';
|
||||||
|
import type { AwarenessRecord } from '../storage/awareness';
|
||||||
|
import { Sync } from '../sync';
|
||||||
|
import type { WorkerOps } from './ops';
|
||||||
|
|
||||||
|
export class WorkerConsumer {
|
||||||
|
private remotes: SpaceStorage[] = [];
|
||||||
|
private local: SpaceStorage | null = null;
|
||||||
|
private sync: Sync | null = null;
|
||||||
|
|
||||||
|
get ensureLocal() {
|
||||||
|
if (!this.local) {
|
||||||
|
throw new Error('Not initialized');
|
||||||
|
}
|
||||||
|
return this.local;
|
||||||
|
}
|
||||||
|
|
||||||
|
get ensureSync() {
|
||||||
|
if (!this.sync) {
|
||||||
|
throw new Error('Not initialized');
|
||||||
|
}
|
||||||
|
return this.sync;
|
||||||
|
}
|
||||||
|
|
||||||
|
get docStorage() {
|
||||||
|
return this.ensureLocal.get('doc');
|
||||||
|
}
|
||||||
|
|
||||||
|
get docSync() {
|
||||||
|
const docSync = this.ensureSync.doc;
|
||||||
|
if (!docSync) {
|
||||||
|
throw new Error('Doc sync not initialized');
|
||||||
|
}
|
||||||
|
return docSync;
|
||||||
|
}
|
||||||
|
|
||||||
|
get blobStorage() {
|
||||||
|
return this.ensureLocal.get('blob');
|
||||||
|
}
|
||||||
|
|
||||||
|
get blobSync() {
|
||||||
|
const blobSync = this.ensureSync.blob;
|
||||||
|
if (!blobSync) {
|
||||||
|
throw new Error('Blob sync not initialized');
|
||||||
|
}
|
||||||
|
return blobSync;
|
||||||
|
}
|
||||||
|
|
||||||
|
get syncStorage() {
|
||||||
|
return this.ensureLocal.get('sync');
|
||||||
|
}
|
||||||
|
|
||||||
|
get awarenessStorage() {
|
||||||
|
return this.ensureLocal.get('awareness');
|
||||||
|
}
|
||||||
|
|
||||||
|
get awarenessSync() {
|
||||||
|
const awarenessSync = this.ensureSync.awareness;
|
||||||
|
if (!awarenessSync) {
|
||||||
|
throw new Error('Awareness sync not initialized');
|
||||||
|
}
|
||||||
|
return awarenessSync;
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(private readonly consumer: OpConsumer<WorkerOps>) {}
|
||||||
|
|
||||||
|
listen() {
|
||||||
|
this.registerHandlers();
|
||||||
|
this.consumer.listen();
|
||||||
|
}
|
||||||
|
|
||||||
|
async init(init: {
|
||||||
|
local: { name: string; opts: StorageOptions }[];
|
||||||
|
remotes: { name: string; opts: StorageOptions }[][];
|
||||||
|
}) {
|
||||||
|
this.local = new SpaceStorage(
|
||||||
|
init.local.map(opt => {
|
||||||
|
const Storage = getAvailableStorageImplementations(opt.name);
|
||||||
|
return new Storage(opt.opts);
|
||||||
|
})
|
||||||
|
);
|
||||||
|
this.remotes = init.remotes.map(opts => {
|
||||||
|
return new SpaceStorage(
|
||||||
|
opts.map(opt => {
|
||||||
|
const Storage = getAvailableStorageImplementations(opt.name);
|
||||||
|
return new Storage(opt.opts);
|
||||||
|
})
|
||||||
|
);
|
||||||
|
});
|
||||||
|
this.sync = new Sync(this.local, this.remotes);
|
||||||
|
this.local.connect();
|
||||||
|
for (const remote of this.remotes) {
|
||||||
|
remote.connect();
|
||||||
|
}
|
||||||
|
this.sync.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
async destroy() {
|
||||||
|
this.sync?.stop();
|
||||||
|
this.local?.disconnect();
|
||||||
|
await this.local?.destroy();
|
||||||
|
for (const remote of this.remotes) {
|
||||||
|
remote.disconnect();
|
||||||
|
await remote.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private registerHandlers() {
|
||||||
|
const collectJobs = new Map<
|
||||||
|
string,
|
||||||
|
(awareness: AwarenessRecord | null) => void
|
||||||
|
>();
|
||||||
|
let collectId = 0;
|
||||||
|
this.consumer.registerAll({
|
||||||
|
'worker.init': this.init.bind(this),
|
||||||
|
'worker.destroy': this.destroy.bind(this),
|
||||||
|
'docStorage.getDoc': (docId: string) => this.docStorage.getDoc(docId),
|
||||||
|
'docStorage.getDocDiff': ({ docId, state }) =>
|
||||||
|
this.docStorage.getDocDiff(docId, state),
|
||||||
|
'docStorage.pushDocUpdate': ({ update, origin }) =>
|
||||||
|
this.docStorage.pushDocUpdate(update, origin),
|
||||||
|
'docStorage.getDocTimestamps': after =>
|
||||||
|
this.docStorage.getDocTimestamps(after ?? undefined),
|
||||||
|
'docStorage.getDocTimestamp': docId =>
|
||||||
|
this.docStorage.getDocTimestamp(docId),
|
||||||
|
'docStorage.deleteDoc': (docId: string) =>
|
||||||
|
this.docStorage.deleteDoc(docId),
|
||||||
|
'docStorage.subscribeDocUpdate': () =>
|
||||||
|
new Observable(subscriber => {
|
||||||
|
return this.docStorage.subscribeDocUpdate((update, origin) => {
|
||||||
|
subscriber.next({ update, origin });
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
'docStorage.waitForConnected': () =>
|
||||||
|
new Observable(subscriber => {
|
||||||
|
const abortController = new AbortController();
|
||||||
|
this.docStorage.connection
|
||||||
|
.waitForConnected(abortController.signal)
|
||||||
|
.then(() => {
|
||||||
|
subscriber.next(true);
|
||||||
|
subscriber.complete();
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
subscriber.error(error);
|
||||||
|
});
|
||||||
|
return () => abortController.abort();
|
||||||
|
}),
|
||||||
|
'blobStorage.getBlob': key => this.blobStorage.get(key),
|
||||||
|
'blobStorage.setBlob': blob => this.blobStorage.set(blob),
|
||||||
|
'blobStorage.deleteBlob': ({ key, permanently }) =>
|
||||||
|
this.blobStorage.delete(key, permanently),
|
||||||
|
'blobStorage.releaseBlobs': () => this.blobStorage.release(),
|
||||||
|
'blobStorage.listBlobs': () => this.blobStorage.list(),
|
||||||
|
'syncStorage.clearClocks': () => this.syncStorage.clearClocks(),
|
||||||
|
'syncStorage.getPeerPulledRemoteClock': ({ peer, docId }) =>
|
||||||
|
this.syncStorage.getPeerPulledRemoteClock(peer, docId),
|
||||||
|
'syncStorage.getPeerPulledRemoteClocks': ({ peer }) =>
|
||||||
|
this.syncStorage.getPeerPulledRemoteClocks(peer),
|
||||||
|
'syncStorage.setPeerPulledRemoteClock': ({ peer, clock }) =>
|
||||||
|
this.syncStorage.setPeerPulledRemoteClock(peer, clock),
|
||||||
|
'syncStorage.getPeerRemoteClock': ({ peer, docId }) =>
|
||||||
|
this.syncStorage.getPeerRemoteClock(peer, docId),
|
||||||
|
'syncStorage.getPeerRemoteClocks': ({ peer }) =>
|
||||||
|
this.syncStorage.getPeerRemoteClocks(peer),
|
||||||
|
'syncStorage.setPeerRemoteClock': ({ peer, clock }) =>
|
||||||
|
this.syncStorage.setPeerRemoteClock(peer, clock),
|
||||||
|
'syncStorage.getPeerPushedClock': ({ peer, docId }) =>
|
||||||
|
this.syncStorage.getPeerPushedClock(peer, docId),
|
||||||
|
'syncStorage.getPeerPushedClocks': ({ peer }) =>
|
||||||
|
this.syncStorage.getPeerPushedClocks(peer),
|
||||||
|
'syncStorage.setPeerPushedClock': ({ peer, clock }) =>
|
||||||
|
this.syncStorage.setPeerPushedClock(peer, clock),
|
||||||
|
'awarenessStorage.update': ({ awareness, origin }) =>
|
||||||
|
this.awarenessStorage.update(awareness, origin),
|
||||||
|
'awarenessStorage.subscribeUpdate': docId =>
|
||||||
|
new Observable(subscriber => {
|
||||||
|
return this.awarenessStorage.subscribeUpdate(
|
||||||
|
docId,
|
||||||
|
(update, origin) => {
|
||||||
|
subscriber.next({
|
||||||
|
type: 'awareness-update',
|
||||||
|
awareness: update,
|
||||||
|
origin,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
() => {
|
||||||
|
const currentCollectId = collectId++;
|
||||||
|
const promise = new Promise<AwarenessRecord | null>(resolve => {
|
||||||
|
collectJobs.set(currentCollectId.toString(), awareness => {
|
||||||
|
resolve(awareness);
|
||||||
|
collectJobs.delete(currentCollectId.toString());
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
'awarenessStorage.collect': ({ collectId, awareness }) =>
|
||||||
|
collectJobs.get(collectId)?.(awareness),
|
||||||
|
'docSync.state': () =>
|
||||||
|
new Observable(subscriber => {
|
||||||
|
const subscription = this.docSync.state$.subscribe(state => {
|
||||||
|
subscriber.next(state);
|
||||||
|
});
|
||||||
|
return () => subscription.unsubscribe();
|
||||||
|
}),
|
||||||
|
'docSync.docState': docId =>
|
||||||
|
new Observable(subscriber => {
|
||||||
|
const subscription = this.docSync
|
||||||
|
.docState$(docId)
|
||||||
|
.subscribe(state => {
|
||||||
|
subscriber.next(state);
|
||||||
|
});
|
||||||
|
return () => subscription.unsubscribe();
|
||||||
|
}),
|
||||||
|
'docSync.addPriority': ({ docId, priority }) =>
|
||||||
|
new Observable(() => {
|
||||||
|
const undo = this.docSync.addPriority(docId, priority);
|
||||||
|
return () => undo();
|
||||||
|
}),
|
||||||
|
'blobSync.downloadBlob': key => this.blobSync.downloadBlob(key),
|
||||||
|
'blobSync.uploadBlob': blob => this.blobSync.uploadBlob(blob),
|
||||||
|
'awarenessSync.update': ({ awareness, origin }) =>
|
||||||
|
this.awarenessSync.update(awareness, origin),
|
||||||
|
'awarenessSync.subscribeUpdate': docId =>
|
||||||
|
new Observable(subscriber => {
|
||||||
|
return this.awarenessStorage.subscribeUpdate(
|
||||||
|
docId,
|
||||||
|
(update, origin) => {
|
||||||
|
subscriber.next({
|
||||||
|
type: 'awareness-update',
|
||||||
|
awareness: update,
|
||||||
|
origin,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
() => {
|
||||||
|
const currentCollectId = collectId++;
|
||||||
|
const promise = new Promise<AwarenessRecord | null>(resolve => {
|
||||||
|
collectJobs.set(currentCollectId.toString(), awareness => {
|
||||||
|
resolve(awareness);
|
||||||
|
collectJobs.delete(currentCollectId.toString());
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
'awarenessSync.collect': ({ collectId, awareness }) =>
|
||||||
|
collectJobs.get(collectId)?.(awareness),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
122
packages/common/nbstore/src/worker/ops.ts
Normal file
122
packages/common/nbstore/src/worker/ops.ts
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
import type {
|
||||||
|
BlobRecord,
|
||||||
|
DocClock,
|
||||||
|
DocClocks,
|
||||||
|
DocDiff,
|
||||||
|
DocRecord,
|
||||||
|
DocUpdate,
|
||||||
|
ListedBlobRecord,
|
||||||
|
StorageOptions,
|
||||||
|
} from '../storage';
|
||||||
|
import type { AwarenessRecord } from '../storage/awareness';
|
||||||
|
import type { DocSyncDocState, DocSyncState } from '../sync/doc';
|
||||||
|
|
||||||
|
interface GroupedWorkerOps {
|
||||||
|
worker: {
|
||||||
|
init: [
|
||||||
|
{
|
||||||
|
local: { name: string; opts: StorageOptions }[];
|
||||||
|
remotes: { name: string; opts: StorageOptions }[][];
|
||||||
|
},
|
||||||
|
void,
|
||||||
|
];
|
||||||
|
destroy: [void, void];
|
||||||
|
};
|
||||||
|
|
||||||
|
docStorage: {
|
||||||
|
getDoc: [string, DocRecord | null];
|
||||||
|
getDocDiff: [{ docId: string; state?: Uint8Array }, DocDiff | null];
|
||||||
|
pushDocUpdate: [{ update: DocUpdate; origin?: string }, DocClock];
|
||||||
|
getDocTimestamps: [Date | null, DocClocks];
|
||||||
|
getDocTimestamp: [string, DocClock | null];
|
||||||
|
deleteDoc: [string, void];
|
||||||
|
subscribeDocUpdate: [void, { update: DocRecord; origin?: string }];
|
||||||
|
waitForConnected: [void, boolean];
|
||||||
|
};
|
||||||
|
|
||||||
|
blobStorage: {
|
||||||
|
getBlob: [string, BlobRecord | null];
|
||||||
|
setBlob: [BlobRecord, void];
|
||||||
|
deleteBlob: [{ key: string; permanently: boolean }, void];
|
||||||
|
releaseBlobs: [void, void];
|
||||||
|
listBlobs: [void, ListedBlobRecord[]];
|
||||||
|
};
|
||||||
|
|
||||||
|
syncStorage: {
|
||||||
|
getPeerPulledRemoteClocks: [{ peer: string }, DocClocks];
|
||||||
|
getPeerPulledRemoteClock: [
|
||||||
|
{ peer: string; docId: string },
|
||||||
|
DocClock | null,
|
||||||
|
];
|
||||||
|
setPeerPulledRemoteClock: [{ peer: string; clock: DocClock }, void];
|
||||||
|
getPeerRemoteClocks: [{ peer: string }, DocClocks];
|
||||||
|
getPeerRemoteClock: [{ peer: string; docId: string }, DocClock | null];
|
||||||
|
setPeerRemoteClock: [{ peer: string; clock: DocClock }, void];
|
||||||
|
getPeerPushedClocks: [{ peer: string }, DocClocks];
|
||||||
|
getPeerPushedClock: [{ peer: string; docId: string }, DocClock | null];
|
||||||
|
setPeerPushedClock: [{ peer: string; clock: DocClock }, void];
|
||||||
|
clearClocks: [void, void];
|
||||||
|
};
|
||||||
|
|
||||||
|
awarenessStorage: {
|
||||||
|
update: [{ awareness: AwarenessRecord; origin?: string }, void];
|
||||||
|
subscribeUpdate: [
|
||||||
|
string,
|
||||||
|
(
|
||||||
|
| {
|
||||||
|
type: 'awareness-update';
|
||||||
|
awareness: AwarenessRecord;
|
||||||
|
origin?: string;
|
||||||
|
}
|
||||||
|
| { type: 'awareness-collect'; collectId: string }
|
||||||
|
),
|
||||||
|
];
|
||||||
|
collect: [{ collectId: string; awareness: AwarenessRecord }, void];
|
||||||
|
};
|
||||||
|
|
||||||
|
docSync: {
|
||||||
|
state: [void, DocSyncState];
|
||||||
|
docState: [string, DocSyncDocState];
|
||||||
|
addPriority: [{ docId: string; priority: number }, boolean];
|
||||||
|
};
|
||||||
|
|
||||||
|
blobSync: {
|
||||||
|
downloadBlob: [string, BlobRecord | null];
|
||||||
|
uploadBlob: [BlobRecord, void];
|
||||||
|
};
|
||||||
|
|
||||||
|
awarenessSync: {
|
||||||
|
update: [{ awareness: AwarenessRecord; origin?: string }, void];
|
||||||
|
subscribeUpdate: [
|
||||||
|
string,
|
||||||
|
(
|
||||||
|
| {
|
||||||
|
type: 'awareness-update';
|
||||||
|
awareness: AwarenessRecord;
|
||||||
|
origin?: string;
|
||||||
|
}
|
||||||
|
| { type: 'awareness-collect'; collectId: string }
|
||||||
|
),
|
||||||
|
];
|
||||||
|
collect: [{ collectId: string; awareness: AwarenessRecord }, void];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
type Values<T> = T extends { [k in keyof T]: any } ? T[keyof T] : never;
|
||||||
|
type UnionToIntersection<U> = (U extends any ? (x: U) => void : never) extends (
|
||||||
|
x: infer I
|
||||||
|
) => void
|
||||||
|
? I
|
||||||
|
: never;
|
||||||
|
|
||||||
|
export type WorkerOps = UnionToIntersection<
|
||||||
|
Values<
|
||||||
|
Values<{
|
||||||
|
[k in keyof GroupedWorkerOps]: {
|
||||||
|
[k2 in keyof GroupedWorkerOps[k]]: k2 extends string
|
||||||
|
? Record<`${k}.${k2}`, GroupedWorkerOps[k][k2]>
|
||||||
|
: never;
|
||||||
|
};
|
||||||
|
}>
|
||||||
|
>
|
||||||
|
>;
|
@ -1,8 +1,8 @@
|
|||||||
import { type BlobRecord, BlobStorage, share } from '@affine/nbstore';
|
import { type BlobRecord, BlobStorageBase, share } from '@affine/nbstore';
|
||||||
|
|
||||||
import { NativeDBConnection } from './db';
|
import { NativeDBConnection } from './db';
|
||||||
|
|
||||||
export class SqliteBlobStorage extends BlobStorage {
|
export class SqliteBlobStorage extends BlobStorageBase {
|
||||||
override connection = share(
|
override connection = share(
|
||||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||||
);
|
);
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
import { DocStorage as NativeDocStorage } from '@affine/native';
|
import { DocStorage as NativeDocStorage } from '@affine/native';
|
||||||
import { Connection, type SpaceType } from '@affine/nbstore';
|
import { AutoReconnectConnection, type SpaceType } from '@affine/nbstore';
|
||||||
import fs from 'fs-extra';
|
import fs from 'fs-extra';
|
||||||
|
|
||||||
import { logger } from '../logger';
|
import { logger } from '../logger';
|
||||||
import { getSpaceDBPath } from '../workspace/meta';
|
import { getSpaceDBPath } from '../workspace/meta';
|
||||||
|
|
||||||
export class NativeDBConnection extends Connection<NativeDocStorage> {
|
export class NativeDBConnection extends AutoReconnectConnection<NativeDocStorage> {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly peer: string,
|
private readonly peer: string,
|
||||||
private readonly type: SpaceType,
|
private readonly type: SpaceType,
|
||||||
|
@ -1,14 +1,14 @@
|
|||||||
import {
|
import {
|
||||||
type DocClocks,
|
type DocClocks,
|
||||||
type DocRecord,
|
type DocRecord,
|
||||||
DocStorage,
|
DocStorageBase,
|
||||||
type DocUpdate,
|
type DocUpdate,
|
||||||
share,
|
share,
|
||||||
} from '@affine/nbstore';
|
} from '@affine/nbstore';
|
||||||
|
|
||||||
import { NativeDBConnection } from './db';
|
import { NativeDBConnection } from './db';
|
||||||
|
|
||||||
export class SqliteDocStorage extends DocStorage {
|
export class SqliteDocStorage extends DocStorageBase {
|
||||||
override connection = share(
|
override connection = share(
|
||||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||||
);
|
);
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
import {
|
import {
|
||||||
|
BasicSyncStorage,
|
||||||
type DocClock,
|
type DocClock,
|
||||||
type DocClocks,
|
type DocClocks,
|
||||||
share,
|
share,
|
||||||
SyncStorage,
|
|
||||||
} from '@affine/nbstore';
|
} from '@affine/nbstore';
|
||||||
|
|
||||||
import { NativeDBConnection } from './db';
|
import { NativeDBConnection } from './db';
|
||||||
|
|
||||||
export class SqliteSyncStorage extends SyncStorage {
|
export class SqliteSyncStorage extends BasicSyncStorage {
|
||||||
override connection = share(
|
override connection = share(
|
||||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||||
);
|
);
|
||||||
|
Loading…
Reference in New Issue
Block a user