toeverything/AFFiNE
1
1
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2024-12-22 15:11:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

perf(y-indexeddb): improve boost and loading time (#1879)

Browse Source
This commit is contained in:
Himself65 2023-04-11 17:29:44 -05:00 committed by GitHub
parent a599364218
commit f3af128baf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 220 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
apps/electron/yarn.lock
View File

@ -5662,6 +5662,7 @@ __metadata:
idb: ^7.1.1
vite: ^4.2.1
vite-plugin-dts: ^2.2.0
y-indexeddb: ^9.0.10
peerDependencies:
yjs: ^13.5.51
languageName: unknown
@ -12383,7 +12384,7 @@ __metadata:
languageName: node
linkType: hard
"lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
"lib0@npm:^0.2.35, lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
version: 0.2.73
resolution: "lib0@npm:0.2.73"
dependencies:
@ -17474,6 +17475,17 @@ __metadata:
languageName: node
linkType: hard
"y-indexeddb@npm:^9.0.10":
version: 9.0.10
resolution: "y-indexeddb@npm:9.0.10"
dependencies:
lib0: ^0.2.35
peerDependencies:
yjs: ^13.0.0
checksum: 6a57825b599cdf77da7c9857b1acc0f782492fc41531618bd7392bdfbcf11c783ff1a30b82ae080b050a5ebafd54754a978de7a6ac42144ec59eb1fbdebd090b
languageName: node
linkType: hard
"y-protocols@npm:^1.0.5":
version: 1.0.5
resolution: "y-protocols@npm:1.0.5"

103
packages/y-indexeddb/benchmark/index.ts Normal file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env ts-node-esm
import 'fake-indexeddb/auto';
const map = new Map();
const localStorage = {
getItem: (key: string) => map.get(key),
setItem: (key: string, value: string) => map.set(key, value),
clear: () => map.clear(),
};
// @ts-expect-error
globalThis.localStorage = localStorage;
import { Workspace } from '@blocksuite/store';
import { IndexeddbPersistence } from 'y-indexeddb';
const Y = Workspace.Y;
// eslint-disable-next-line @typescript-eslint/no-restricted-imports
import { createIndexedDBProvider } from '../src/index.js';
async function yjs_create_persistence(n = 1e3) {
for (let i = 0; i < n; i++) {
const yDoc = new Y.Doc();
const persistence = new IndexeddbPersistence('test', yDoc);
await persistence.whenSynced;
persistence.destroy();
}
}
async function yjs_single_persistence(n = 1e5) {
const yDoc = new Y.Doc();
const map = yDoc.getMap();
for (let i = 0; i < n; i++) {
map.set(`${i}`, i);
}
{
const persistence = new IndexeddbPersistence('test', yDoc);
await persistence.whenSynced;
persistence.destroy();
}
{
const persistence = new IndexeddbPersistence('test', yDoc);
await persistence.whenSynced;
persistence.destroy();
}
}
async function toeverything_create_provider(n = 1e3) {
for (let i = 0; i < n; i++) {
const yDoc = new Y.Doc();
const provider = createIndexedDBProvider('test', yDoc);
provider.connect();
await provider.whenSynced;
provider.disconnect();
}
}
async function toeverything_single_persistence(n = 1e5) {
const yDoc = new Y.Doc();
const map = yDoc.getMap();
for (let i = 0; i < n; i++) {
map.set(`${i}`, i);
}
const provider = createIndexedDBProvider('test', yDoc, 'test');
provider.connect();
await provider.whenSynced;
provider.disconnect();
provider.connect();
await provider.whenSynced;
provider.disconnect();
}
async function main() {
console.log('create many persistence');
performance.mark('start');
await yjs_create_persistence();
performance.mark('end');
performance.measure('yjs', 'start', 'end');
indexedDB.deleteDatabase('test');
performance.mark('start');
await toeverything_create_provider();
performance.mark('end');
performance.measure('toeverything', 'start', 'end');
console.log(performance.getEntriesByType('measure'));
indexedDB.deleteDatabase('test');
performance.clearMarks();
performance.clearMeasures();
localStorage.clear();
console.log('single persistence with huge updates');
performance.mark('start');
await yjs_single_persistence();
performance.mark('end');
performance.measure('yjs', 'start', 'end');
indexedDB.deleteDatabase('test');
performance.mark('start');
await toeverything_single_persistence();
performance.mark('end');
performance.measure('toeverything', 'start', 'end');
console.log(performance.getEntriesByType('measure'));
}
main().then();

3
packages/y-indexeddb/package.json
View File

@ -28,7 +28,8 @@
"@blocksuite/blocks": "0.0.0-20230411141436-ec6b051d-nightly",
"@blocksuite/store": "0.0.0-20230411141436-ec6b051d-nightly",
"vite": "^4.2.1",
"vite-plugin-dts": "^2.2.0"
"vite-plugin-dts": "^2.2.0",
"y-indexeddb": "^9.0.10"
},
"peerDependencies": {
"yjs": "^13.5.51"

61
packages/y-indexeddb/src/__tests__/index.spec.ts
View File

@ -6,7 +6,7 @@ import 'fake-indexeddb/auto';
import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
import { assertExists, uuidv4, Workspace } from '@blocksuite/store';
import { openDB } from 'idb';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs';
import type { WorkspacePersist } from '../index';
@ -20,7 +20,7 @@ import {
} from '../index';
async function getUpdates(id: string): Promise<ArrayBuffer[]> {
const db = await openDB('affine-local', dbVersion);
const db = await openDB(rootDBName, dbVersion);
const store = await db
.transaction('workspace', 'readonly')
.objectStore('workspace');
@ -32,6 +32,7 @@ async function getUpdates(id: string): Promise<ArrayBuffer[]> {
let id: string;
let workspace: Workspace;
const rootDBName = 'affine-local';
beforeEach(() => {
id = uuidv4();
@ -42,12 +43,16 @@ beforeEach(() => {
workspace.register(AffineSchemas).register(__unstableSchemas);
});
afterEach(() => {
indexedDB.deleteDatabase('affine-local');
});
describe('indexeddb provider', () => {
test('connect', async () => {
const provider = createIndexedDBProvider(workspace.id, workspace.doc);
provider.connect();
await provider.whenSynced;
const db = await openDB('affine-local', dbVersion);
const db = await openDB(rootDBName, dbVersion);
{
const store = await db
.transaction('workspace', 'readonly')
@ -89,7 +94,8 @@ describe('indexeddb provider', () => {
.register(__unstableSchemas);
const provider2 = createIndexedDBProvider(
secondWorkspace.id,
secondWorkspace.doc
secondWorkspace.doc,
rootDBName
);
provider2.connect();
await provider2.whenSynced;
@ -99,7 +105,11 @@ describe('indexeddb provider', () => {
});
test('disconnect suddenly', async () => {
const provider = createIndexedDBProvider(workspace.id, workspace.doc);
const provider = createIndexedDBProvider(
workspace.id,
workspace.doc,
rootDBName
);
const fn = vi.fn();
provider.connect();
provider.disconnect();
@ -109,10 +119,14 @@ describe('indexeddb provider', () => {
});
test('connect and disconnect', async () => {
const provider = createIndexedDBProvider(workspace.id, workspace.doc);
const provider = createIndexedDBProvider(
workspace.id,
workspace.doc,
rootDBName
);
provider.connect();
const p1 = provider.whenSynced;
await provider.whenSynced;
await p1;
provider.disconnect();
{
const page = workspace.createPage('page0');
@ -126,7 +140,7 @@ describe('indexeddb provider', () => {
}
provider.connect();
const p2 = provider.whenSynced;
await provider.whenSynced;
await p2;
{
const updates = await getUpdates(workspace.id);
expect(updates).not.toEqual([]);
@ -137,7 +151,11 @@ describe('indexeddb provider', () => {
test('merge', async () => {
setMergeCount(5);
const provider = createIndexedDBProvider(workspace.id, workspace.doc);
const provider = createIndexedDBProvider(
workspace.id,
workspace.doc,
rootDBName
);
provider.connect();
{
const page = workspace.createPage('page0');
@ -153,6 +171,31 @@ describe('indexeddb provider', () => {
expect(updates.length).lessThanOrEqual(5);
}
});
test("data won't be lost", async () => {
const id = uuidv4();
const doc = new Workspace.Y.Doc();
const map = doc.getMap('map');
for (let i = 0; i < 100; i++) {
map.set(`${i}`, i);
}
{
const provider = createIndexedDBProvider(id, doc, rootDBName);
provider.connect();
await provider.whenSynced;
provider.disconnect();
}
{
const newDoc = new Workspace.Y.Doc();
const provider = createIndexedDBProvider(id, newDoc, rootDBName);
provider.connect();
await provider.whenSynced;
provider.disconnect();
newDoc.getMap('map').forEach((value, key) => {
expect(value).toBe(parseInt(key));
});
}
});
});
describe('milestone', () => {

12
packages/y-indexeddb/src/index.ts
View File

@ -299,8 +299,16 @@ export const createIndexedDBProvider = (
});
} else {
const updates = data.updates.map(({ update }) => update);
const update = mergeUpdates(updates);
const newUpdate = diffUpdate(encodeStateAsUpdate(doc), update);
const fakeDoc = new Doc();
fakeDoc.transact(() => {
updates.forEach(update => {
applyUpdate(fakeDoc, update);
});
}, indexeddbOrigin);
const newUpdate = diffUpdate(
encodeStateAsUpdate(doc),
encodeStateAsUpdate(fakeDoc)
);
await store.put({
...data,
updates: [

10
tests/libs/utils.ts
View File

@ -1,3 +1,6 @@
import { ok } from 'node:assert';
import { resolve } from 'node:path';
import type { PageMeta } from '@blocksuite/store';
import { faker } from '@faker-js/faker';
import type { Page } from '@playwright/test';
@ -7,6 +10,13 @@ const user1 = require('@affine-test/fixtures/built-in-user1.json');
// eslint-disable-next-line @typescript-eslint/no-var-requires
const user2 = require('@affine-test/fixtures/built-in-user2.json');
export const rootDir = resolve(__dirname, '..', '..');
// assert that the rootDir is the root of the project
// eslint-disable-next-line @typescript-eslint/no-var-requires
ok(require(resolve(rootDir, 'package.json')).name.toLowerCase() === 'affine');
export const testResultDir = resolve(rootDir, 'test-results');
export async function getBuiltInUser() {
return Promise.all([
fetch('http://localhost:3000/api/user/token', {

24
tests/parallels/local-first-setting-page.spec.ts
View File

@ -1,9 +1,12 @@
import { resolve } from 'node:path';
import { expect } from '@playwright/test';
import { openHomePage } from '../libs/load-page';
import { waitMarkdownImported } from '../libs/page-logic';
import { test } from '../libs/playwright';
import { clickSideBarSettingButton } from '../libs/sidebar';
import { testResultDir } from '../libs/utils';
test.describe('Local first setting page', () => {
test('Should highlight the setting page menu when selected', async ({
@ -14,13 +17,20 @@ test.describe('Local first setting page', () => {
const element = await page.getByTestId(
'slider-bar-workspace-setting-button'
);
const prevColor = await element.evaluate(
element => window.getComputedStyle(element).color
);
const prev = await element.screenshot({
path: resolve(
testResultDir,
'slider-bar-workspace-setting-button-prev.png'
),
});
await clickSideBarSettingButton(page);
const currentColor = await element.evaluate(
element => window.getComputedStyle(element).color
);
expect(prevColor).not.toBe(currentColor);
await page.waitForTimeout(50);
const after = await element.screenshot({
path: resolve(
testResultDir,
'slider-bar-workspace-setting-button-after.png'
),
});
expect(prev).not.toEqual(after);
});
});

14
yarn.lock
View File

@ -6219,6 +6219,7 @@ __metadata:
idb: ^7.1.1
vite: ^4.2.1
vite-plugin-dts: ^2.2.0
y-indexeddb: ^9.0.10
peerDependencies:
yjs: ^13.5.51
languageName: unknown
@ -14229,7 +14230,7 @@ __metadata:
languageName: node
linkType: hard
"lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
"lib0@npm:^0.2.35, lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
version: 0.2.73
resolution: "lib0@npm:0.2.73"
dependencies:
@ -20163,6 +20164,17 @@ __metadata:
languageName: node
linkType: hard
"y-indexeddb@npm:^9.0.10":
version: 9.0.10
resolution: "y-indexeddb@npm:9.0.10"
dependencies:
lib0: ^0.2.35
peerDependencies:
yjs: ^13.0.0
checksum: 6a57825b599cdf77da7c9857b1acc0f782492fc41531618bd7392bdfbcf11c783ff1a30b82ae080b050a5ebafd54754a978de7a6ac42144ec59eb1fbdebd090b
languageName: node
linkType: hard
"y-protocols@npm:^1.0.5":
version: 1.0.5
resolution: "y-protocols@npm:1.0.5"