feat: basic db support

This commit is contained in:
DarkSky 2022-08-09 18:58:18 +08:00
parent 73d7d4e1a8
commit 0083b7445a
9 changed files with 445 additions and 24 deletions

View File

@ -12,6 +12,7 @@
"cssmodule",
"datasource",
"fflate",
"fstore",
"groq",
"howpublished",
"immer",

View File

@ -148,6 +148,12 @@ module.exports = function (webpackConfig) {
}
}
config.module.rules.unshift({
test: /\.wasm$/,
type: 'asset/resource',
});
config.resolve.fallback = { crypto: false, fs: false, path: false };
addEmotionBabelPlugin(config);
config.plugins = [

View File

@ -5,7 +5,11 @@
"author": "DarkSky <darksky2048@gmail.com>",
"dependencies": {
"lib0": "^0.2.52",
"sql.js": "^1.7.0",
"yjs": "^13.5.41",
"y-protocols": "^1.0.5"
},
"devDependencies": {
"@types/sql.js": "^1.4.3"
}
}

View File

@ -1 +1,3 @@
export { IndexedDBProvider } from './indexeddb';
export { WebsocketProvider } from './provider';
export { SQLiteProvider } from './sqlite';

View File

@ -0,0 +1,185 @@
import * as Y from 'yjs';
import * as idb from 'lib0/indexeddb.js';
import * as mutex from 'lib0/mutex.js';
import { Observable } from 'lib0/observable.js';
const customStoreName = 'custom';
const updatesStoreName = 'updates';
const PREFERRED_TRIM_SIZE = 500;
const fetchUpdates = async (provider: IndexedDBProvider) => {
const [updatesStore] = idb.transact(provider.db as IDBDatabase, [
updatesStoreName,
]); // , 'readonly')
const updates = await idb.getAll(
updatesStore,
idb.createIDBKeyRangeLowerBound(provider._dbref, false)
);
Y.transact(
provider.doc,
() => {
updates.forEach(val => Y.applyUpdate(provider.doc, val));
},
provider,
false
);
const lastKey = await idb.getLastKey(updatesStore);
provider._dbref = lastKey + 1;
const cnt = await idb.count(updatesStore);
provider._dbsize = cnt;
return updatesStore;
};
const storeState = (provider: IndexedDBProvider, forceStore = true) =>
fetchUpdates(provider).then(updatesStore => {
if (forceStore || provider._dbsize >= PREFERRED_TRIM_SIZE) {
idb.addAutoKey(updatesStore, Y.encodeStateAsUpdate(provider.doc))
.then(() =>
idb.del(
updatesStore,
idb.createIDBKeyRangeUpperBound(provider._dbref, true)
)
)
.then(() =>
idb.count(updatesStore).then(cnt => {
provider._dbsize = cnt;
})
);
}
});
export class IndexedDBProvider extends Observable<string> {
doc: Y.Doc;
name: string;
private _mux: mutex.mutex;
_dbref: number;
_dbsize: number;
private _destroyed: boolean;
whenSynced: Promise<IndexedDBProvider>;
db: IDBDatabase | null;
private _db: Promise<IDBDatabase>;
private synced: boolean;
private _storeTimeout: number;
private _storeTimeoutId: NodeJS.Timeout | null;
private _storeUpdate: (update: Uint8Array, origin: any) => void;
constructor(name: string, doc: Y.Doc) {
super();
this.doc = doc;
this.name = name;
this._mux = mutex.createMutex();
this._dbref = 0;
this._dbsize = 0;
this._destroyed = false;
this.db = null;
this.synced = false;
this._db = idb.openDB(name, db =>
idb.createStores(db, [
['updates', { autoIncrement: true }],
['custom'],
])
);
this.whenSynced = this._db.then(async db => {
this.db = db;
const currState = Y.encodeStateAsUpdate(doc);
const updatesStore = await fetchUpdates(this);
await idb.addAutoKey(updatesStore, currState);
if (this._destroyed) return this;
this.emit('synced', [this]);
this.synced = true;
return this;
});
// Timeout in ms untill data is merged and persisted in idb.
this._storeTimeout = 1000;
this._storeTimeoutId = null;
this._storeUpdate = (update: Uint8Array, origin: any) => {
if (this.db && origin !== this) {
const [updatesStore] = idb.transact(
/** @type {IDBDatabase} */ this.db,
[updatesStoreName]
);
idb.addAutoKey(updatesStore, update);
if (++this._dbsize >= PREFERRED_TRIM_SIZE) {
// debounce store call
if (this._storeTimeoutId !== null) {
clearTimeout(this._storeTimeoutId);
}
this._storeTimeoutId = setTimeout(() => {
storeState(this, false);
this._storeTimeoutId = null;
}, this._storeTimeout);
}
}
};
doc.on('update', this._storeUpdate);
this.destroy = this.destroy.bind(this);
doc.on('destroy', this.destroy);
}
override destroy() {
if (this._storeTimeoutId) {
clearTimeout(this._storeTimeoutId);
}
this.doc.off('update', this._storeUpdate);
this.doc.off('destroy', this.destroy);
this._destroyed = true;
return this._db.then(db => {
db.close();
});
}
/**
* Destroys this instance and removes all data from SQLite.
*
* @return {Promise<void>}
*/
async clearData(): Promise<void> {
return this.destroy().then(() => {
idb.deleteDB(this.name);
});
}
/**
* @param {String | number | ArrayBuffer | Date} key
* @return {Promise<String | number | ArrayBuffer | Date | any>}
*/
async get(
key: string | number | ArrayBuffer | Date
): Promise<string | number | ArrayBuffer | Date | any> {
return this._db.then(db => {
const [custom] = idb.transact(db, [customStoreName], 'readonly');
return idb.get(custom, key);
});
}
/**
* @param {String | number | ArrayBuffer | Date} key
* @param {String | number | ArrayBuffer | Date} value
* @return {Promise<String | number | ArrayBuffer | Date>}
*/
async set(
key: string | number | ArrayBuffer | Date,
value: string | number | ArrayBuffer | Date
): Promise<string | number | ArrayBuffer | Date> {
return this._db.then(db => {
const [custom] = idb.transact(db, [customStoreName]);
return idb.put(custom, value, key);
});
}
/**
* @param {String | number | ArrayBuffer | Date} key
* @return {Promise<undefined>}
*/
async del(key: string | number | ArrayBuffer | Date): Promise<undefined> {
return this._db.then(db => {
const [custom] = idb.transact(db, [customStoreName]);
return idb.del(custom, key);
});
}
}

View File

@ -0,0 +1,166 @@
import * as Y from 'yjs';
import sqlite, { Database, SqlJsStatic } from 'sql.js';
import { Observable } from 'lib0/observable.js';
const PREFERRED_TRIM_SIZE = 500;
const STMTS = {
create: 'CREATE TABLE updates (key INTEGER PRIMARY KEY AUTOINCREMENT, value BLOB);',
selectAll: 'SELECT * FROM updates where key >= $idx',
selectCount: 'SELECT count(*) FROM updates',
insert: 'INSERT INTO updates VALUES (null, $data);',
delete: 'DELETE FROM updates WHERE key < $idx',
drop: 'DROP TABLE updates;',
};
const countUpdates = (db: Database) => {
const [cnt] = db.exec(STMTS.selectCount);
return cnt.values[0]?.[0] as number;
};
const clearUpdates = (db: Database, idx: number) => {
db.exec(STMTS.delete, { $idx: idx });
};
const fetchUpdates = async (provider: SQLiteProvider) => {
const db = provider.db!;
const updates = db
.exec(STMTS.selectAll, { $idx: provider._dbref })
.flatMap(val => val.values as [number, Uint8Array][])
.sort(([a], [b]) => a - b);
Y.transact(
provider.doc,
() => {
updates.forEach(([, update]) =>
Y.applyUpdate(provider.doc, update)
);
},
provider,
false
);
const lastKey = Math.max(...updates.map(([idx]) => idx));
provider._dbref = lastKey + 1;
provider._dbsize = countUpdates(db);
return db;
};
const storeState = async (provider: SQLiteProvider, forceStore = true) => {
const db = await fetchUpdates(provider);
if (forceStore || provider._dbsize >= PREFERRED_TRIM_SIZE) {
db.exec(STMTS.insert, { $data: Y.encodeStateAsUpdate(provider.doc) });
clearUpdates(db, provider._dbref);
provider._dbsize = countUpdates(db);
console.log(db.export());
}
};
let _sqliteInstance: SqlJsStatic | undefined;
let _sqliteProcessing = false;
const sleep = () => new Promise(resolve => setTimeout(resolve, 500));
const initSQLiteInstance = async () => {
while (_sqliteProcessing) {
await sleep();
}
if (_sqliteInstance) return _sqliteInstance;
_sqliteProcessing = true;
_sqliteInstance = await sqlite({
locateFile: () =>
new URL('sql.js/dist/sql-wasm.wasm', import.meta.url).href,
});
_sqliteProcessing = false;
return _sqliteInstance;
};
export class SQLiteProvider extends Observable<string> {
doc: Y.Doc;
name: string;
_dbref: number;
_dbsize: number;
private _destroyed: boolean;
whenSynced: Promise<SQLiteProvider>;
db: Database | null;
private _db: Promise<Database>;
synced: boolean;
_storeTimeout: number;
_storeTimeoutId: NodeJS.Timeout | null;
_storeUpdate: (update: Uint8Array, origin: any) => void;
constructor(dbname: string, doc: Y.Doc) {
super();
this.doc = doc;
this.name = dbname;
this._dbref = 0;
this._dbsize = 0;
this._destroyed = false;
this.db = null;
this.synced = false;
this._db = initSQLiteInstance().then(db => {
const sqlite = new db.Database();
return sqlite.run(STMTS.create);
});
this.whenSynced = this._db.then(async db => {
this.db = db;
const currState = Y.encodeStateAsUpdate(doc);
await fetchUpdates(this);
db.exec(STMTS.insert, { $data: currState });
if (this._destroyed) return this;
this.emit('synced', [this]);
this.synced = true;
return this;
});
// Timeout in ms untill data is merged and persisted in idb.
this._storeTimeout = 1000;
this._storeTimeoutId = null;
this._storeUpdate = (update: Uint8Array, origin: any) => {
if (this.db && origin !== this) {
this.db.exec(STMTS.insert, { $data: update });
if (++this._dbsize >= PREFERRED_TRIM_SIZE) {
// debounce store call
if (this._storeTimeoutId !== null) {
clearTimeout(this._storeTimeoutId);
}
this._storeTimeoutId = setTimeout(() => {
storeState(this, false);
this._storeTimeoutId = null;
}, this._storeTimeout);
}
}
};
doc.on('update', this._storeUpdate);
this.destroy = this.destroy.bind(this);
doc.on('destroy', this.destroy);
}
override destroy(): Promise<void> {
if (this._storeTimeoutId) {
clearTimeout(this._storeTimeoutId);
}
this.doc.off('update', this._storeUpdate);
this.doc.off('destroy', this.destroy);
this._destroyed = true;
return this._db.then(db => {
db.close();
});
}
// Destroys this instance and removes all data from SQLite.
async clearData(): Promise<void> {
return this._db.then(db => {
db.exec(STMTS.drop);
return this.destroy();
});
}
}

View File

@ -13,8 +13,7 @@
"flexsearch": "^0.7.21",
"lib0": "^0.2.52",
"lru-cache": "^7.13.2",
"ts-debounce": "^4.0.0",
"y-indexeddb": "^9.0.9"
"ts-debounce": "^4.0.0"
},
"dependencies": {
"@types/flexsearch": "^0.7.3",

View File

@ -7,7 +7,6 @@ import { fromEvent } from 'file-selector';
import LRUCache from 'lru-cache';
import { debounce } from 'ts-debounce';
import { nanoid } from 'nanoid';
import { IndexeddbPersistence } from 'y-indexeddb';
import { Awareness } from 'y-protocols/awareness.js';
import {
Doc,
@ -19,7 +18,11 @@ import {
snapshot,
} from 'yjs';
import { WebsocketProvider } from '@toeverything/datasource/jwt-rpc';
import {
IndexedDBProvider,
SQLiteProvider,
WebsocketProvider,
} from '@toeverything/datasource/jwt-rpc';
import {
AsyncDatabaseAdapter,
@ -46,8 +49,9 @@ const logger = getLogger('BlockDB:yjs');
type YjsProviders = {
awareness: Awareness;
idb: IndexeddbPersistence;
binariesIdb: IndexeddbPersistence;
idb: IndexedDBProvider;
binariesIdb: IndexedDBProvider;
fstore?: SQLiteProvider;
ws?: WebsocketProvider;
backend: string;
gatekeeper: GateKeeper;
@ -117,7 +121,9 @@ async function _initYjsDatabase(
const doc = new Doc({ autoLoad: true, shouldLoad: true });
const idbp = new IndexeddbPersistence(workspace, doc).whenSynced;
const idbp = new IndexedDBProvider(workspace, doc).whenSynced;
const fsp: SQLiteProvider | undefined = undefined; // new SQLiteProvider(workspace, doc).whenSynced;
const wsp = _initWebsocketProvider(
backend,
workspace,
@ -126,10 +132,10 @@ async function _initYjsDatabase(
params
);
const [idb, [awareness, ws]] = await Promise.all([idbp, wsp]);
const [idb, [awareness, ws], fstore] = await Promise.all([idbp, wsp, fsp]);
const binaries = new Doc({ autoLoad: true, shouldLoad: true });
const binariesIdb = await new IndexeddbPersistence(
const binariesIdb = await new IndexedDBProvider(
`${workspace}_binaries`,
binaries
).whenSynced;
@ -147,6 +153,7 @@ async function _initYjsDatabase(
awareness,
idb,
binariesIdb,
fstore,
ws,
backend,
gatekeeper,
@ -374,7 +381,7 @@ export class YjsAdapter implements AsyncDatabaseAdapter<YjsContentOperation> {
};
check();
});
await new IndexeddbPersistence(this._provider.idb.name, doc)
await new IndexedDBProvider(this._provider.idb.name, doc)
.whenSynced;
applyUpdate(doc, new Uint8Array(binary));
await update_check;

View File

@ -194,7 +194,7 @@ importers:
yjs: ^13.5.41
dependencies:
authing-js-sdk: 4.23.35
firebase-admin: 11.0.1_@firebase+app-types@0.7.0
firebase-admin: 11.0.1
lib0: 0.2.52
lru-cache: 7.13.2
nanoid: 4.0.0
@ -571,6 +571,9 @@ importers:
dependencies:
ffc-js-client-side-sdk: 1.1.5
libs/datasource/jwst/pkg:
specifiers: {}
libs/datasource/jwt:
specifiers:
'@types/debug': ^4.1.7
@ -594,7 +597,6 @@ importers:
sift: ^16.0.0
ts-debounce: ^4.0.0
uuid: ^8.3.2
y-indexeddb: ^9.0.9
y-protocols: ^1.0.5
yjs: ^13.5.41
dependencies:
@ -622,17 +624,21 @@ importers:
lib0: 0.2.52
lru-cache: 7.13.2
ts-debounce: 4.0.0
y-indexeddb: 9.0.9_yjs@13.5.41
libs/datasource/jwt-rpc:
specifiers:
'@types/sql.js': ^1.4.3
lib0: ^0.2.52
sql.js: ^1.7.0
y-protocols: ^1.0.5
yjs: ^13.5.41
dependencies:
lib0: 0.2.52
sql.js: 1.7.0
y-protocols: 1.0.5
yjs: 13.5.41
devDependencies:
'@types/sql.js': 1.4.3
libs/datasource/remote-kv:
specifiers:
@ -3288,6 +3294,15 @@ packages:
- utf-8-validate
dev: true
/@firebase/auth-interop-types/0.1.6_@firebase+util@1.6.3:
resolution: {integrity: sha512-etIi92fW3CctsmR9e3sYM3Uqnoq861M0Id9mdOPF6PWIg38BXL5k4upCNBggGUpLIS0H1grMOvy/wn1xymwe2g==}
peerDependencies:
'@firebase/app-types': 0.x
'@firebase/util': 1.x
dependencies:
'@firebase/util': 1.6.3
dev: false
/@firebase/auth-interop-types/0.1.6_pbfwexsq7uf6mrzcwnikj3g37m:
resolution: {integrity: sha512-etIi92fW3CctsmR9e3sYM3Uqnoq861M0Id9mdOPF6PWIg38BXL5k4upCNBggGUpLIS0H1grMOvy/wn1xymwe2g==}
peerDependencies:
@ -3296,6 +3311,7 @@ packages:
dependencies:
'@firebase/app-types': 0.7.0
'@firebase/util': 1.6.3
dev: true
/@firebase/auth-types/0.11.0_pbfwexsq7uf6mrzcwnikj3g37m:
resolution: {integrity: sha512-q7Bt6cx+ySj9elQHTsKulwk3+qDezhzRBFC9zlQ1BjgMueUOnGMcvqmU0zuKlQ4RhLSH7MNAdBV2znVaoN3Vxw==}
@ -3331,6 +3347,19 @@ packages:
'@firebase/util': 1.6.3
tslib: 2.4.0
/@firebase/database-compat/0.2.4:
resolution: {integrity: sha512-VtsGixO5mTjNMJn6PwxAJEAR70fj+3blCXIdQKel3q+eYGZAfdqxox1+tzZDnf9NWBJpaOgAHPk3JVDxEo9NFQ==}
dependencies:
'@firebase/component': 0.5.17
'@firebase/database': 0.13.4
'@firebase/database-types': 0.9.12
'@firebase/logger': 0.3.3
'@firebase/util': 1.6.3
tslib: 2.4.0
transitivePeerDependencies:
- '@firebase/app-types'
dev: false
/@firebase/database-compat/0.2.4_@firebase+app-types@0.7.0:
resolution: {integrity: sha512-VtsGixO5mTjNMJn6PwxAJEAR70fj+3blCXIdQKel3q+eYGZAfdqxox1+tzZDnf9NWBJpaOgAHPk3JVDxEo9NFQ==}
dependencies:
@ -3342,6 +3371,7 @@ packages:
tslib: 2.4.0
transitivePeerDependencies:
- '@firebase/app-types'
dev: true
/@firebase/database-types/0.9.10:
resolution: {integrity: sha512-2ji6nXRRsY+7hgU6zRhUtK0RmSjVWM71taI7Flgaw+BnopCo/lDF5HSwxp8z7LtiHlvQqeRA3Ozqx5VhlAbiKg==}
@ -3356,6 +3386,19 @@ packages:
'@firebase/app-types': 0.7.0
'@firebase/util': 1.6.3
/@firebase/database/0.13.4:
resolution: {integrity: sha512-NW7bOoiaC4sJCj6DY/m9xHoFNa0CK32YPMCh6FiMweLCDQbOZM8Ql/Kn6yyuxCb7K7ypz9eSbRlCWQJsJRQjhg==}
dependencies:
'@firebase/auth-interop-types': 0.1.6_@firebase+util@1.6.3
'@firebase/component': 0.5.17
'@firebase/logger': 0.3.3
'@firebase/util': 1.6.3
faye-websocket: 0.11.4
tslib: 2.4.0
transitivePeerDependencies:
- '@firebase/app-types'
dev: false
/@firebase/database/0.13.4_@firebase+app-types@0.7.0:
resolution: {integrity: sha512-NW7bOoiaC4sJCj6DY/m9xHoFNa0CK32YPMCh6FiMweLCDQbOZM8Ql/Kn6yyuxCb7K7ypz9eSbRlCWQJsJRQjhg==}
dependencies:
@ -3367,6 +3410,7 @@ packages:
tslib: 2.4.0
transitivePeerDependencies:
- '@firebase/app-types'
dev: true
/@firebase/firestore-compat/0.1.23_53yvy43rwpg2c45kgeszsxtrca:
resolution: {integrity: sha512-QfcuyMAavp//fQnjSfCEpnbWi7spIdKaXys1kOLu7395fLr+U6ykmto1HUMCSz8Yus9cEr/03Ujdi2SUl2GUAA==}
@ -6691,6 +6735,10 @@ packages:
resolution: {integrity: sha512-uw8eYMIReOwstQ0QKF0sICefSy8cNO/v7gOTiIy9SbwuHyEecJUm7qlgueOO5S1udZ5I/irVydHVwMchgzbKTg==}
dev: true
/@types/emscripten/1.39.6:
resolution: {integrity: sha512-H90aoynNhhkQP6DRweEjJp5vfUVdIj7tdPLsu7pq89vODD/lcugKfZOsfgwpvM6XUewEp2N5dCg1Uf3Qe55Dcg==}
dev: true
/@types/eslint-scope/3.7.4:
resolution: {integrity: sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==}
dependencies:
@ -6988,6 +7036,13 @@ packages:
'@types/node': 18.0.1
dev: true
/@types/sql.js/1.4.3:
resolution: {integrity: sha512-3bz1LJIiJtKMEL8tYf7c9Nrb1lYcFeWQkE8vhWvobE29ZzizW79DtoTjqx1bR82DS2Ch2K30nOwNhuLclZ1vYg==}
dependencies:
'@types/emscripten': 1.39.6
'@types/node': 18.0.1
dev: true
/@types/stack-utils/2.0.1:
resolution: {integrity: sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==}
dev: true
@ -10849,12 +10904,12 @@ packages:
semver-regex: 2.0.0
dev: true
/firebase-admin/11.0.1_@firebase+app-types@0.7.0:
/firebase-admin/11.0.1:
resolution: {integrity: sha512-rL3wlZbi2Kb/KJgcmj1YHlD4ZhfmhfgRO2YJialxAllm0tj1IQea878hHuBLGmv4DpbW9t9nLvX9kddNR2Y65Q==}
engines: {node: '>=14'}
dependencies:
'@fastify/busboy': 1.1.0
'@firebase/database-compat': 0.2.4_@firebase+app-types@0.7.0
'@firebase/database-compat': 0.2.4
'@firebase/database-types': 0.9.10
'@types/node': 18.0.1
jsonwebtoken: 8.5.1
@ -17879,6 +17934,10 @@ packages:
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
dev: true
/sql.js/1.7.0:
resolution: {integrity: sha512-qAfft3xkSgHqmmfNugWTp/59PsqIw8gbeao5TZmpmzQQsAJ49de3iDDKuxVixidYs6dkHNksY8m27v2dZNn2jw==}
dev: false
/sshpk/1.17.0:
resolution: {integrity: sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==}
engines: {node: '>=0.10.0'}
@ -19572,15 +19631,6 @@ packages:
engines: {node: '>=0.4'}
dev: true
/y-indexeddb/9.0.9_yjs@13.5.41:
resolution: {integrity: sha512-GcJbiJa2eD5hankj46Hea9z4hbDnDjvh1fT62E5SpZRsv8GcEemw34l1hwI2eknGcv5Ih9JfusT37JLx9q3LFg==}
peerDependencies:
yjs: ^13.0.0
dependencies:
lib0: 0.2.52
yjs: 13.5.41
dev: true
/y-protocols/1.0.5:
resolution: {integrity: sha512-Wil92b7cGk712lRHDqS4T90IczF6RkcvCwAD0A2OPg+adKmOe+nOiT/N2hvpQIWS3zfjmtL4CPaH5sIW1Hkm/A==}
dependencies:
@ -19649,6 +19699,7 @@ packages:
resolution: {integrity: sha512-4eSTrrs8OeI0heXKKioRY4ag7V5Bk85Z4MeniUyown3o3y0G7G4JpAZWrZWfTp7pzw2b53GkAQWKqHsHi9j9JA==}
dependencies:
lib0: 0.2.52
dev: false
/yn/3.1.1:
resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}