toeverything/AFFiNE
1
1
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2024-09-11 19:47:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor(core): use manual upgrade to replace auto migration when web setup (#5022)

Browse Source 
1. Split logic in `packages/common/infra/src/blocksuite/index.ts` to multiple single files
2. Move migration logic from setup to upgrade module, to prevent auto migration problems and loading problem
This commit is contained in:
Joooye_34 2023-11-23 02:26:06 +00:00
parent 3710bcdc14
commit 4c8d54b3a7
No known key found for this signature in database
GPG Key ID: 3645F0B246B8EFC7
21 changed files with 947 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

749
packages/common/infra/src/blocksuite/index.ts
View File

@ -1,750 +1,17 @@
import type { Page, PageMeta, Workspace } from '@blocksuite/store';
import { createIndexeddbStorage } from '@blocksuite/store';
import type { createStore, WritableAtom } from 'jotai/vanilla';
import type { Doc } from 'yjs';
import { Array as YArray, Doc as YDoc, Map as YMap, transact } from 'yjs';
export async function initEmptyPage(page: Page, title?: string) {
await page.waitForLoaded();
const pageBlockId = page.addBlock('affine:page', {
title: new page.Text(title ?? ''),
});
page.addBlock('affine:surface', {}, pageBlockId);
const noteBlockId = page.addBlock('affine:note', {}, pageBlockId);
page.addBlock('affine:paragraph', {}, noteBlockId);
}
export async function buildEmptyBlockSuite(workspace: Workspace) {
const page = workspace.createPage();
await initEmptyPage(page);
workspace.setPageMeta(page.id, {
jumpOnce: true,
});
}
export async function buildShowcaseWorkspace(
workspace: Workspace,
options: {
schema: Schema;
atoms: {
pageMode: WritableAtom<
undefined,
[pageId: string, mode: 'page' | 'edgeless'],
void
>;
};
store: ReturnType<typeof createStore>;
}
) {
const prototypes = {
tags: {
options: [
{
id: 'icg1n5UdkP',
value: 'Travel',
color: 'var(--affine-tag-gray)',
},
{
id: 'Oe5dSe1DDJ',
value: 'Quick summary',
color: 'var(--affine-tag-green)',
},
{
id: 'g1L5dXKctL',
value: 'OKR',
color: 'var(--affine-tag-purple)',
},
{
id: 'q3mceOl_zi',
value: 'Streamline your workflow',
color: 'var(--affine-tag-teal)',
},
{
id: 'ze07JVwBu4',
value: 'Plan',
color: 'var(--affine-tag-teal)',
},
{
id: '8qcYPCTK0h',
value: 'Review',
color: 'var(--affine-tag-orange)',
},
{
id: 'wg-fBtd2eI',
value: 'Engage',
color: 'var(--affine-tag-pink)',
},
{
id: 'QYFD_HeQc-',
value: 'Create',
color: 'var(--affine-tag-blue)',
},
{
id: 'ZHBa2NtdSo',
value: 'Learn',
color: 'var(--affine-tag-yellow)',
},
],
},
};
workspace.meta.setProperties(prototypes);
const edgelessPage1 = nanoid();
const edgelessPage2 = nanoid();
const edgelessPage3 = nanoid();
const { store, atoms } = options;
[edgelessPage1, edgelessPage2, edgelessPage3].forEach(pageId => {
store.set(atoms.pageMode, pageId, 'edgeless');
});
const pageMetas = {
'9f6f3c04-cf32-470c-9648-479dc838f10e': {
createDate: 1691548231530,
tags: ['ZHBa2NtdSo', 'QYFD_HeQc-', 'wg-fBtd2eI'],
updatedDate: 1691676331623,
favorite: true,
jumpOnce: true,
},
'0773e198-5de0-45d4-a35e-de22ea72b96b': {
createDate: 1691548220794,
tags: [],
updatedDate: 1691676775642,
favorite: false,
},
'59b140eb-4449-488f-9eeb-42412dcc044e': {
createDate: 1691551731225,
tags: [],
updatedDate: 1691654611175,
favorite: false,
},
'7217fbe2-61db-4a91-93c6-ad5c800e5a43': {
createDate: 1691552082822,
tags: [],
updatedDate: 1691654606912,
favorite: false,
},
'6eb43ea8-8c11-456d-bb1d-5193937961ab': {
createDate: 1691552090989,
tags: [],
updatedDate: 1691646748171,
favorite: false,
},
'3ddc8a4f-62c7-4fd4-8064-9ed9f61e437a': {
createDate: 1691564303138,
tags: [],
updatedDate: 1691646845195,
},
'512b1cb3-d22d-4b20-a7aa-58e2afcb1238': {
createDate: 1691574743531,
tags: ['icg1n5UdkP'],
updatedDate: 1691647117761,
},
'22163830-8252-43fe-b62d-fd9bbeaa4caa': {
createDate: 1691574859042,
tags: [],
updatedDate: 1691648159371,
},
'b7a9e1bc-e205-44aa-8dad-7e328269d00b': {
createDate: 1691575011078,
tags: ['8qcYPCTK0h'],
updatedDate: 1691645074511,
favorite: false,
},
'646305d9-93e0-48df-bb92-d82944ceb5a3': {
createDate: 1691634722239,
tags: ['ze07JVwBu4'],
updatedDate: 1691647069662,
favorite: false,
},
'0350509d-8702-4797-b4d7-168f5e9359c7': {
createDate: 1691635388447,
tags: ['Oe5dSe1DDJ'],
updatedDate: 1691645873930,
},
'aa02af3c-5c5c-4856-b7ce-947ad17331f3': {
createDate: 1691636192263,
tags: ['q3mceOl_zi', 'g1L5dXKctL'],
updatedDate: 1691645102104,
},
'9d6e716e-a071-45a2-88ac-2f2f6eec0109': {
createDate: 1691574743531,
tags: ['icg1n5UdkP'],
updatedDate: 1691574743531,
},
} satisfies Record<string, Partial<PageMeta>>;
const data = [
[
'9f6f3c04-cf32-470c-9648-479dc838f10e',
import('@affine/templates/v1/getting-started.json'),
nanoid(),
],
[
'0773e198-5de0-45d4-a35e-de22ea72b96b',
import('@affine/templates/v1/preloading.json'),
edgelessPage1,
],
[
'59b140eb-4449-488f-9eeb-42412dcc044e',
import('@affine/templates/v1/template-galleries.json'),
nanoid(),
],
[
'7217fbe2-61db-4a91-93c6-ad5c800e5a43',
import('@affine/templates/v1/personal-home.json'),
nanoid(),
],
[
'6eb43ea8-8c11-456d-bb1d-5193937961ab',
import('@affine/templates/v1/working-home.json'),
nanoid(),
],
[
'3ddc8a4f-62c7-4fd4-8064-9ed9f61e437a',
import('@affine/templates/v1/personal-project-management.json'),
nanoid(),
],
[
'512b1cb3-d22d-4b20-a7aa-58e2afcb1238',
import('@affine/templates/v1/travel-plan.json'),
edgelessPage2,
],
[
'22163830-8252-43fe-b62d-fd9bbeaa4caa',
import('@affine/templates/v1/personal-knowledge-management.json'),
nanoid(),
],
[
'b7a9e1bc-e205-44aa-8dad-7e328269d00b',
import('@affine/templates/v1/annual-performance-review.json'),
nanoid(),
],
[
'646305d9-93e0-48df-bb92-d82944ceb5a3',
import('@affine/templates/v1/brief-event-planning.json'),
nanoid(),
],
[
'0350509d-8702-4797-b4d7-168f5e9359c7',
import('@affine/templates/v1/meeting-summary.json'),
nanoid(),
],
[
'aa02af3c-5c5c-4856-b7ce-947ad17331f3',
import('@affine/templates/v1/okr-template.json'),
nanoid(),
],
[
'9d6e716e-a071-45a2-88ac-2f2f6eec0109',
import('@affine/templates/v1/travel-note.json'),
edgelessPage3,
],
] as const;
const idMap = await Promise.all(data).then(async data => {
return data.reduce<Record<string, string>>(
(record, currentValue) => {
const [oldId, _, newId] = currentValue;
record[oldId] = newId;
return record;
},
{} as Record<string, string>
);
});
await Promise.all(
data.map(async ([id, promise, newId]) => {
const { default: template } = await promise;
let json = JSON.stringify(template);
Object.entries(idMap).forEach(([oldId, newId]) => {
json = json.replaceAll(oldId, newId);
});
json = JSON.parse(json);
await workspace
.importPageSnapshot(structuredClone(json), newId)
.catch(error => {
console.error('error importing page', id, error);
});
const page = workspace.getPage(newId);
assertExists(page);
await page.waitForLoaded();
workspace.schema.upgradePage(
0,
{
'affine:note': 1,
'affine:bookmark': 1,
'affine:database': 2,
'affine:divider': 1,
'affine:image': 1,
'affine:list': 1,
'affine:code': 1,
'affine:page': 2,
'affine:paragraph': 1,
'affine:surface': 3,
},
page.spaceDoc
);
})
);
Object.entries(pageMetas).forEach(([oldId, meta]) => {
const newId = idMap[oldId];
workspace.setPageMeta(newId, meta);
});
}
import { applyUpdate, encodeStateAsUpdate } from 'yjs';
const migrationOrigin = 'affine-migration';
import { assertExists } from '@blocksuite/global/utils';
import type { Schema } from '@blocksuite/store';
import { nanoid } from 'nanoid';
type XYWH = [number, number, number, number];
function deserializeXYWH(xywh: string): XYWH {
return JSON.parse(xywh) as XYWH;
}
const getLatestVersions = (schema: Schema): Record<string, number> => {
return [...schema.flavourSchemaMap.entries()].reduce(
(record, [flavour, schema]) => {
record[flavour] = schema.version;
return record;
},
{} as Record<string, number>
);
};
function migrateDatabase(data: YMap<unknown>) {
data.delete('prop:mode');
data.set('prop:views', new YArray());
const columns = (data.get('prop:columns') as YArray<unknown>).toJSON() as {
id: string;
name: string;
hide: boolean;
type: string;
width: number;
selection?: unknown[];
}[];
const views = [
{
id: 'default',
name: 'Table',
columns: columns.map(col => ({
id: col.id,
width: col.width,
hide: col.hide,
})),
filter: { type: 'group', op: 'and', conditions: [] },
mode: 'table',
},
];
const cells = (data.get('prop:cells') as YMap<unknown>).toJSON() as Record<
string,
Record<
string,
{
id: string;
value: unknown;
}
>
>;
const convertColumn = (
id: string,
update: (cell: { id: string; value: unknown }) => void
) => {
Object.values(cells).forEach(row => {
if (row[id] != null) {
update(row[id]);
}
});
};
const newColumns = columns.map(v => {
let data: Record<string, unknown> = {};
if (v.type === 'select' || v.type === 'multi-select') {
data = { options: v.selection };
if (v.type === 'select') {
convertColumn(v.id, cell => {
if (Array.isArray(cell.value)) {
cell.value = cell.value[0]?.id;
}
});
} else {
convertColumn(v.id, cell => {
if (Array.isArray(cell.value)) {
cell.value = cell.value.map(v => v.id);
}
});
}
}
if (v.type === 'number') {
convertColumn(v.id, cell => {
if (typeof cell.value === 'string') {
cell.value = Number.parseFloat(cell.value.toString());
}
});
}
return {
id: v.id,
type: v.type,
name: v.name,
data,
};
});
data.set('prop:columns', newColumns);
data.set('prop:views', views);
data.set('prop:cells', cells);
}
function runBlockMigration(
flavour: string,
data: YMap<unknown>,
version: number
) {
if (flavour === 'affine:frame') {
data.set('sys:flavour', 'affine:note');
return;
}
if (flavour === 'affine:surface' && version <= 3) {
if (data.has('elements')) {
const elements = data.get('elements') as YMap<unknown>;
migrateSurface(elements);
data.set('prop:elements', elements.clone());
data.delete('elements');
} else {
data.set('prop:elements', new YMap());
}
}
if (flavour === 'affine:embed') {
data.set('sys:flavour', 'affine:image');
data.delete('prop:type');
}
if (flavour === 'affine:database' && version < 2) {
migrateDatabase(data);
}
}
function migrateSurface(data: YMap<unknown>) {
for (const [, value] of <IterableIterator<[string, YMap<unknown>]>>(
data.entries()
)) {
if (value.get('type') === 'connector') {
migrateSurfaceConnector(value);
}
}
}
function migrateSurfaceConnector(data: YMap<any>) {
let id = data.get('startElement')?.id;
const controllers = data.get('controllers');
const length = controllers.length;
const xywh = deserializeXYWH(data.get('xywh'));
if (id) {
data.set('source', { id });
} else {
data.set('source', {
position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]],
});
}
id = data.get('endElement')?.id;
if (id) {
data.set('target', { id });
} else {
data.set('target', {
position: [
controllers[length - 1].x + xywh[0],
controllers[length - 1].y + xywh[1],
],
});
}
const width = data.get('lineWidth') ?? 4;
data.set('strokeWidth', width);
const color = data.get('color');
data.set('stroke', color);
data.delete('startElement');
data.delete('endElement');
data.delete('controllers');
data.delete('lineWidth');
data.delete('color');
data.delete('xywh');
}
function updateBlockVersions(versions: YMap<number>) {
const frameVersion = versions.get('affine:frame');
if (frameVersion !== undefined) {
versions.set('affine:note', frameVersion);
versions.delete('affine:frame');
}
const embedVersion = versions.get('affine:embed');
if (embedVersion !== undefined) {
versions.set('affine:image', embedVersion);
versions.delete('affine:embed');
}
const databaseVersion = versions.get('affine:database');
if (databaseVersion !== undefined && databaseVersion < 2) {
versions.set('affine:database', 2);
}
}
function migrateMeta(
oldDoc: YDoc,
newDoc: YDoc,
idMap: Record<string, string>
) {
const originalMeta = oldDoc.getMap('space:meta');
const originalVersions = originalMeta.get('versions') as YMap<number>;
const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
const meta = newDoc.getMap('meta');
const pages = new YArray();
const blockVersions = originalVersions.clone();
meta.set('workspaceVersion', 1);
meta.set('blockVersions', blockVersions);
meta.set('pages', pages);
meta.set('name', originalMeta.get('name') as string);
updateBlockVersions(blockVersions);
const mapList = originalPages.map(page => {
const map = new YMap();
Array.from(page.entries())
.filter(([key]) => key !== 'subpageIds')
.forEach(([key, value]) => {
if (key === 'id') {
idMap[value] = nanoid();
map.set(key, idMap[value]);
} else {
map.set(key, value);
}
});
return map;
});
pages.push(mapList);
}
function migrateBlocks(
oldDoc: YDoc,
newDoc: YDoc,
idMap: Record<string, string>
) {
const spaces = newDoc.getMap('spaces');
const originalMeta = oldDoc.getMap('space:meta');
const originalVersions = originalMeta.get('versions') as YMap<number>;
const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
originalPages.forEach(page => {
const id = page.get('id') as string;
const newId = idMap[id];
const spaceId = id.startsWith('space:') ? id : `space:${id}`;
const originalBlocks = oldDoc.getMap(spaceId) as YMap<unknown>;
const subdoc = new YDoc();
spaces.set(newId, subdoc);
subdoc.guid = id;
const blocks = subdoc.getMap('blocks');
Array.from(originalBlocks.entries()).forEach(([key, value]) => {
const blockData = value.clone();
blocks.set(key, blockData);
const flavour = blockData.get('sys:flavour') as string;
const version = originalVersions.get(flavour);
if (version !== undefined) {
runBlockMigration(flavour, blockData, version);
}
});
});
}
export function migrateToSubdoc(oldDoc: YDoc): YDoc {
const needMigration =
Array.from(oldDoc.getMap('space:meta').keys()).length > 0;
if (!needMigration) {
return oldDoc;
}
const newDoc = new YDoc();
const idMap = {} as Record<string, string>;
migrateMeta(oldDoc, newDoc, idMap);
migrateBlocks(oldDoc, newDoc, idMap);
return newDoc;
}
export type UpgradeOptions = {
getCurrentRootDoc: () => Promise<YDoc>;
createWorkspace: () => Promise<Workspace>;
getSchema: () => Schema;
};
const upgradeV1ToV2 = async (options: UpgradeOptions) => {
const oldDoc = await options.getCurrentRootDoc();
const newDoc = migrateToSubdoc(oldDoc);
const newWorkspace = await options.createWorkspace();
applyUpdate(newWorkspace.doc, encodeStateAsUpdate(newDoc), migrationOrigin);
newDoc.getSubdocs().forEach(subdoc => {
newWorkspace.doc.getSubdocs().forEach(newDoc => {
if (subdoc.guid === newDoc.guid) {
applyUpdate(newDoc, encodeStateAsUpdate(subdoc), migrationOrigin);
}
});
});
return newWorkspace;
};
export * from './initialization';
export * from './migration/blob';
export { migratePages as forceUpgradePages } from './migration/blocksuite'; // campatible with electron
export * from './migration/fixing';
export { migrateToSubdoc } from './migration/subdoc';
export * from './migration/workspace';
/**
* Force upgrade block schema to the latest.
* Don't force to upgrade the pages without the check.
*
* Please note that this function will not upgrade the workspace version.
*
* @returns true if any schema is upgraded.
* @returns false if no schema is upgraded.
* @deprecated
* Use workspace meta data to determine the workspace version.
*/
export async function forceUpgradePages(
options: Omit<UpgradeOptions, 'createWorkspace'>
): Promise<boolean> {
const rootDoc = await options.getCurrentRootDoc();
guidCompatibilityFix(rootDoc);
const spaces = rootDoc.getMap('spaces') as YMap<any>;
const meta = rootDoc.getMap('meta') as YMap<unknown>;
const versions = meta.get('blockVersions') as YMap<number>;
const schema = options.getSchema();
const oldVersions = versions?.toJSON() ?? {};
spaces.forEach((space: Doc) => {
try {
schema.upgradePage(0, oldVersions, space);
} catch (e) {
console.error(`page ${space.guid} upgrade failed`, e);
}
});
const newVersions = getLatestVersions(schema);
meta.set('blockVersions', new YMap(Object.entries(newVersions)));
return Object.entries(oldVersions).some(
([flavour, version]) => newVersions[flavour] !== version
);
}
// database from 2 to 3
async function upgradeV2ToV3(options: UpgradeOptions): Promise<boolean> {
const rootDoc = await options.getCurrentRootDoc();
const spaces = rootDoc.getMap('spaces') as YMap<any>;
const meta = rootDoc.getMap('meta') as YMap<unknown>;
const versions = meta.get('blockVersions') as YMap<number>;
const schema = options.getSchema();
guidCompatibilityFix(rootDoc);
spaces.forEach((space: Doc) => {
schema.upgradePage(
0,
{
'affine:note': 1,
'affine:bookmark': 1,
'affine:database': 2,
'affine:divider': 1,
'affine:image': 1,
'affine:list': 1,
'affine:code': 1,
'affine:page': 2,
'affine:paragraph': 1,
'affine:surface': 3,
},
space
);
});
if ('affine:database' in versions) {
meta.set(
'blockVersions',
new YMap(Object.entries(getLatestVersions(schema)))
);
} else {
Object.entries(getLatestVersions(schema)).map(([flavour, version]) =>
versions.set(flavour, version)
);
}
return true;
}
// patch root doc's space guid compatibility issue
//
// in version 0.10, page id in spaces no longer has prefix "space:"
// The data flow for fetching a doc's updates is:
// - page id in `meta.pages` -> find `${page-id}` in `doc.spaces` -> `doc` -> `doc.guid`
// if `doc` is not found in `doc.spaces`, a new doc will be created and its `doc.guid` is the same with its pageId
// - because of guid logic change, the doc that previously prefixed with "space:" will not be found in `doc.spaces`
// - when fetching the rows of this doc using the doc id === page id,
// it will return empty since there is no updates associated with the page id
export function guidCompatibilityFix(rootDoc: YDoc) {
let changed = false;
transact(rootDoc, () => {
const meta = rootDoc.getMap('meta') as YMap<unknown>;
const pages = meta.get('pages') as YArray<YMap<unknown>>;
pages?.forEach(page => {
const pageId = page.get('id') as string | undefined;
if (pageId?.includes(':')) {
// remove the prefix "space:" from page id
page.set('id', pageId.split(':').at(-1));
}
});
const spaces = rootDoc.getMap('spaces') as YMap<YDoc>;
spaces?.forEach((doc: YDoc, pageId: string) => {
if (pageId.includes(':')) {
const newPageId = pageId.split(':').at(-1) ?? pageId;
const newDoc = new YDoc();
// clone the original doc. yjs is not happy to use the same doc instance
applyUpdate(newDoc, encodeStateAsUpdate(doc));
newDoc.guid = doc.guid;
spaces.set(newPageId, newDoc);
// should remove the old doc, otherwise we will do it again in the next run
spaces.delete(pageId);
changed = true;
console.debug(
`fixed space id ${pageId} -> ${newPageId}, doc id: ${doc.guid}`
);
}
});
});
return changed;
}
export enum WorkspaceVersion {
// v1 is treated as undefined
SubDoc = 2,
DatabaseV3 = 3,
Surface = 4,
}
/**
* If returns false, it means no migration is needed.
* If returns true, it means migration is done.
* If returns Workspace, it means new workspace is created,
* and the old workspace should be deleted.
*/
export async function migrateWorkspace(
currentVersion: WorkspaceVersion | undefined,
options: UpgradeOptions
): Promise<Workspace | boolean> {
if (currentVersion === undefined) {
const workspace = await upgradeV1ToV2(options);
await upgradeV2ToV3({
...options,
getCurrentRootDoc: () => Promise.resolve(workspace.doc),
});
return workspace;
}
if (currentVersion === WorkspaceVersion.SubDoc) {
return upgradeV2ToV3(options);
} else if (currentVersion === WorkspaceVersion.DatabaseV3) {
// surface from 3 to 5
return forceUpgradePages(options);
} else {
return false;
}
}
export async function migrateLocalBlobStorage(from: string, to: string) {
const fromStorage = createIndexeddbStorage(from);
const toStorage = createIndexeddbStorage(to);
const keys = await fromStorage.crud.list();
for (const key of keys) {
const value = await fromStorage.crud.get(key);
if (!value) {
console.warn('cannot find blob:', key);
continue;
}
await toStorage.crud.set(key, value);
}
}

291
packages/common/infra/src/blocksuite/initialization/index.ts Normal file
View File

@ -0,0 +1,291 @@
import { assertExists } from '@blocksuite/global/utils';
import type { Page, PageMeta, Workspace } from '@blocksuite/store';
import type { createStore, WritableAtom } from 'jotai/vanilla';
import { nanoid } from 'nanoid';
import { migratePages } from '../migration/blocksuite';
export async function initEmptyPage(page: Page, title?: string) {
await page.load(() => {
const pageBlockId = page.addBlock('affine:page', {
title: new page.Text(title ?? ''),
});
page.addBlock('affine:surface', {}, pageBlockId);
const noteBlockId = page.addBlock('affine:note', {}, pageBlockId);
page.addBlock('affine:paragraph', {}, noteBlockId);
});
}
/**
* FIXME: Use exported json data to instead of building data.
*/
export async function buildShowcaseWorkspace(
workspace: Workspace,
options: {
atoms: {
pageMode: WritableAtom<
undefined,
[pageId: string, mode: 'page' | 'edgeless'],
void
>;
};
store: ReturnType<typeof createStore>;
}
) {
const prototypes = {
tags: {
options: [
{
id: 'icg1n5UdkP',
value: 'Travel',
color: 'var(--affine-tag-gray)',
},
{
id: 'Oe5dSe1DDJ',
value: 'Quick summary',
color: 'var(--affine-tag-green)',
},
{
id: 'g1L5dXKctL',
value: 'OKR',
color: 'var(--affine-tag-purple)',
},
{
id: 'q3mceOl_zi',
value: 'Streamline your workflow',
color: 'var(--affine-tag-teal)',
},
{
id: 'ze07JVwBu4',
value: 'Plan',
color: 'var(--affine-tag-teal)',
},
{
id: '8qcYPCTK0h',
value: 'Review',
color: 'var(--affine-tag-orange)',
},
{
id: 'wg-fBtd2eI',
value: 'Engage',
color: 'var(--affine-tag-pink)',
},
{
id: 'QYFD_HeQc-',
value: 'Create',
color: 'var(--affine-tag-blue)',
},
{
id: 'ZHBa2NtdSo',
value: 'Learn',
color: 'var(--affine-tag-yellow)',
},
],
},
};
workspace.meta.setProperties(prototypes);
const edgelessPage1 = nanoid();
const edgelessPage2 = nanoid();
const edgelessPage3 = nanoid();
const { store, atoms } = options;
[edgelessPage1, edgelessPage2, edgelessPage3].forEach(pageId => {
store.set(atoms.pageMode, pageId, 'edgeless');
});
const pageMetas = {
'9f6f3c04-cf32-470c-9648-479dc838f10e': {
createDate: 1691548231530,
tags: ['ZHBa2NtdSo', 'QYFD_HeQc-', 'wg-fBtd2eI'],
updatedDate: 1691676331623,
favorite: true,
jumpOnce: true,
},
'0773e198-5de0-45d4-a35e-de22ea72b96b': {
createDate: 1691548220794,
tags: [],
updatedDate: 1691676775642,
favorite: false,
},
'59b140eb-4449-488f-9eeb-42412dcc044e': {
createDate: 1691551731225,
tags: [],
updatedDate: 1691654611175,
favorite: false,
},
'7217fbe2-61db-4a91-93c6-ad5c800e5a43': {
createDate: 1691552082822,
tags: [],
updatedDate: 1691654606912,
favorite: false,
},
'6eb43ea8-8c11-456d-bb1d-5193937961ab': {
createDate: 1691552090989,
tags: [],
updatedDate: 1691646748171,
favorite: false,
},
'3ddc8a4f-62c7-4fd4-8064-9ed9f61e437a': {
createDate: 1691564303138,
tags: [],
updatedDate: 1691646845195,
},
'512b1cb3-d22d-4b20-a7aa-58e2afcb1238': {
createDate: 1691574743531,
tags: ['icg1n5UdkP'],
updatedDate: 1691647117761,
},
'22163830-8252-43fe-b62d-fd9bbeaa4caa': {
createDate: 1691574859042,
tags: [],
updatedDate: 1691648159371,
},
'b7a9e1bc-e205-44aa-8dad-7e328269d00b': {
createDate: 1691575011078,
tags: ['8qcYPCTK0h'],
updatedDate: 1691645074511,
favorite: false,
},
'646305d9-93e0-48df-bb92-d82944ceb5a3': {
createDate: 1691634722239,
tags: ['ze07JVwBu4'],
updatedDate: 1691647069662,
favorite: false,
},
'0350509d-8702-4797-b4d7-168f5e9359c7': {
createDate: 1691635388447,
tags: ['Oe5dSe1DDJ'],
updatedDate: 1691645873930,
},
'aa02af3c-5c5c-4856-b7ce-947ad17331f3': {
createDate: 1691636192263,
tags: ['q3mceOl_zi', 'g1L5dXKctL'],
updatedDate: 1691645102104,
},
'9d6e716e-a071-45a2-88ac-2f2f6eec0109': {
createDate: 1691574743531,
tags: ['icg1n5UdkP'],
updatedDate: 1691574743531,
},
} satisfies Record<string, Partial<PageMeta>>;
const data = [
[
'9f6f3c04-cf32-470c-9648-479dc838f10e',
import('@affine/templates/v1/getting-started.json'),
nanoid(),
],
[
'0773e198-5de0-45d4-a35e-de22ea72b96b',
import('@affine/templates/v1/preloading.json'),
edgelessPage1,
],
[
'59b140eb-4449-488f-9eeb-42412dcc044e',
import('@affine/templates/v1/template-galleries.json'),
nanoid(),
],
[
'7217fbe2-61db-4a91-93c6-ad5c800e5a43',
import('@affine/templates/v1/personal-home.json'),
nanoid(),
],
[
'6eb43ea8-8c11-456d-bb1d-5193937961ab',
import('@affine/templates/v1/working-home.json'),
nanoid(),
],
[
'3ddc8a4f-62c7-4fd4-8064-9ed9f61e437a',
import('@affine/templates/v1/personal-project-management.json'),
nanoid(),
],
[
'512b1cb3-d22d-4b20-a7aa-58e2afcb1238',
import('@affine/templates/v1/travel-plan.json'),
edgelessPage2,
],
[
'22163830-8252-43fe-b62d-fd9bbeaa4caa',
import('@affine/templates/v1/personal-knowledge-management.json'),
nanoid(),
],
[
'b7a9e1bc-e205-44aa-8dad-7e328269d00b',
import('@affine/templates/v1/annual-performance-review.json'),
nanoid(),
],
[
'646305d9-93e0-48df-bb92-d82944ceb5a3',
import('@affine/templates/v1/brief-event-planning.json'),
nanoid(),
],
[
'0350509d-8702-4797-b4d7-168f5e9359c7',
import('@affine/templates/v1/meeting-summary.json'),
nanoid(),
],
[
'aa02af3c-5c5c-4856-b7ce-947ad17331f3',
import('@affine/templates/v1/okr-template.json'),
nanoid(),
],
[
'9d6e716e-a071-45a2-88ac-2f2f6eec0109',
import('@affine/templates/v1/travel-note.json'),
edgelessPage3,
],
] as const;
const idMap = await Promise.all(data).then(async data => {
return data.reduce<Record<string, string>>(
(record, currentValue) => {
const [oldId, _, newId] = currentValue;
record[oldId] = newId;
return record;
},
{} as Record<string, string>
);
});
await Promise.all(
data.map(async ([id, promise, newId]) => {
const { default: template } = await promise;
let json = JSON.stringify(template);
Object.entries(idMap).forEach(([oldId, newId]) => {
json = json.replaceAll(oldId, newId);
});
json = JSON.parse(json);
await workspace
.importPageSnapshot(structuredClone(json), newId)
.catch(error => {
console.error('error importing page', id, error);
});
const page = workspace.getPage(newId);
assertExists(page);
await page.load();
workspace.schema.upgradePage(
0,
{
'affine:note': 1,
'affine:bookmark': 1,
'affine:database': 2,
'affine:divider': 1,
'affine:image': 1,
'affine:list': 1,
'affine:code': 1,
'affine:page': 2,
'affine:paragraph': 1,
'affine:surface': 3,
},
page.spaceDoc
);
// The showcase building will create multiple pages once, and may skip the version writing.
// https://github.com/toeverything/blocksuite/blob/master/packages/store/src/workspace/page.ts#L662
if (!workspace.meta.blockVersions) {
await migratePages(workspace.doc, workspace.schema);
}
})
);
Object.entries(pageMetas).forEach(([oldId, meta]) => {
const newId = idMap[oldId];
workspace.setPageMeta(newId, meta);
});
}

15
packages/common/infra/src/blocksuite/migration/blob.ts Normal file
View File

@ -0,0 +1,15 @@
import { createIndexeddbStorage } from '@blocksuite/store';
export async function migrateLocalBlobStorage(from: string, to: string) {
const fromStorage = createIndexeddbStorage(from);
const toStorage = createIndexeddbStorage(to);
const keys = await fromStorage.crud.list();
for (const key of keys) {
const value = await fromStorage.crud.get(key);
if (!value) {
console.warn('cannot find blob:', key);
continue;
}
await toStorage.crud.set(key, value);
}
}

36
packages/common/infra/src/blocksuite/migration/blocksuite.ts Normal file
View File

@ -0,0 +1,36 @@
import type { Schema } from '@blocksuite/store';
import type { Doc as YDoc } from 'yjs';
import { Map as YMap } from 'yjs';
const getLatestVersions = (schema: Schema): Record<string, number> => {
return [...schema.flavourSchemaMap.entries()].reduce(
(record, [flavour, schema]) => {
record[flavour] = schema.version;
return record;
},
{} as Record<string, number>
);
};
export async function migratePages(
rootDoc: YDoc,
schema: Schema
): Promise<boolean> {
const spaces = rootDoc.getMap('spaces') as YMap<any>;
const meta = rootDoc.getMap('meta') as YMap<unknown>;
const versions = meta.get('blockVersions') as YMap<number>;
const oldVersions = versions?.toJSON() ?? {};
spaces.forEach((space: YDoc) => {
try {
schema.upgradePage(0, oldVersions, space);
} catch (e) {
console.error(`page ${space.guid} upgrade failed`, e);
}
});
const newVersions = getLatestVersions(schema);
meta.set('blockVersions', new YMap(Object.entries(newVersions)));
return Object.entries(oldVersions).some(
([flavour, version]) => newVersions[flavour] !== version
);
}

45
packages/common/infra/src/blocksuite/migration/fixing.ts Normal file
View File

@ -0,0 +1,45 @@
import type { Array as YArray, Map as YMap } from 'yjs';
import { Doc as YDoc, transact } from 'yjs';
import { applyUpdate, encodeStateAsUpdate } from 'yjs';
// patch root doc's space guid compatibility issue
//
// in version 0.10, page id in spaces no longer has prefix "space:"
// The data flow for fetching a doc's updates is:
// - page id in `meta.pages` -> find `${page-id}` in `doc.spaces` -> `doc` -> `doc.guid`
// if `doc` is not found in `doc.spaces`, a new doc will be created and its `doc.guid` is the same with its pageId
// - because of guid logic change, the doc that previously prefixed with "space:" will not be found in `doc.spaces`
// - when fetching the rows of this doc using the doc id === page id,
// it will return empty since there is no updates associated with the page id
export function guidCompatibilityFix(rootDoc: YDoc) {
let changed = false;
transact(rootDoc, () => {
const meta = rootDoc.getMap('meta') as YMap<unknown>;
const pages = meta.get('pages') as YArray<YMap<unknown>>;
pages?.forEach(page => {
const pageId = page.get('id') as string | undefined;
if (pageId?.includes(':')) {
// remove the prefix "space:" from page id
page.set('id', pageId.split(':').at(-1));
}
});
const spaces = rootDoc.getMap('spaces') as YMap<YDoc>;
spaces?.forEach((doc: YDoc, pageId: string) => {
if (pageId.includes(':')) {
const newPageId = pageId.split(':').at(-1) ?? pageId;
const newDoc = new YDoc();
// clone the original doc. yjs is not happy to use the same doc instance
applyUpdate(newDoc, encodeStateAsUpdate(doc));
newDoc.guid = doc.guid;
spaces.set(newPageId, newDoc);
// should remove the old doc, otherwise we will do it again in the next run
spaces.delete(pageId);
changed = true;
console.debug(
`fixed space id ${pageId} -> ${newPageId}, doc id: ${doc.guid}`
);
}
});
});
return changed;
}

281
packages/common/infra/src/blocksuite/migration/subdoc.ts Normal file
View File

@ -0,0 +1,281 @@
import type { Workspace } from '@blocksuite/store';
import { nanoid } from 'nanoid';
import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs';
import { applyUpdate, encodeStateAsUpdate } from 'yjs';
const migrationOrigin = 'affine-migration';
type XYWH = [number, number, number, number];
function deserializeXYWH(xywh: string): XYWH {
return JSON.parse(xywh) as XYWH;
}
function migrateDatabase(data: YMap<unknown>) {
data.delete('prop:mode');
data.set('prop:views', new YArray());
const columns = (data.get('prop:columns') as YArray<unknown>).toJSON() as {
id: string;
name: string;
hide: boolean;
type: string;
width: number;
selection?: unknown[];
}[];
const views = [
{
id: 'default',
name: 'Table',
columns: columns.map(col => ({
id: col.id,
width: col.width,
hide: col.hide,
})),
filter: { type: 'group', op: 'and', conditions: [] },
mode: 'table',
},
];
const cells = (data.get('prop:cells') as YMap<unknown>).toJSON() as Record<
string,
Record<
string,
{
id: string;
value: unknown;
}
>
>;
const convertColumn = (
id: string,
update: (cell: { id: string; value: unknown }) => void
) => {
Object.values(cells).forEach(row => {
if (row[id] != null) {
update(row[id]);
}
});
};
const newColumns = columns.map(v => {
let data: Record<string, unknown> = {};
if (v.type === 'select' || v.type === 'multi-select') {
data = { options: v.selection };
if (v.type === 'select') {
convertColumn(v.id, cell => {
if (Array.isArray(cell.value)) {
cell.value = cell.value[0]?.id;
}
});
} else {
convertColumn(v.id, cell => {
if (Array.isArray(cell.value)) {
cell.value = cell.value.map(v => v.id);
}
});
}
}
if (v.type === 'number') {
convertColumn(v.id, cell => {
if (typeof cell.value === 'string') {
cell.value = Number.parseFloat(cell.value.toString());
}
});
}
return {
id: v.id,
type: v.type,
name: v.name,
data,
};
});
data.set('prop:columns', newColumns);
data.set('prop:views', views);
data.set('prop:cells', cells);
}
function runBlockMigration(
flavour: string,
data: YMap<unknown>,
version: number
) {
if (flavour === 'affine:frame') {
data.set('sys:flavour', 'affine:note');
return;
}
if (flavour === 'affine:surface' && version <= 3) {
if (data.has('elements')) {
const elements = data.get('elements') as YMap<unknown>;
migrateSurface(elements);
data.set('prop:elements', elements.clone());
data.delete('elements');
} else {
data.set('prop:elements', new YMap());
}
}
if (flavour === 'affine:embed') {
data.set('sys:flavour', 'affine:image');
data.delete('prop:type');
}
if (flavour === 'affine:database' && version < 2) {
migrateDatabase(data);
}
}
function migrateSurface(data: YMap<unknown>) {
for (const [, value] of <IterableIterator<[string, YMap<unknown>]>>(
data.entries()
)) {
if (value.get('type') === 'connector') {
migrateSurfaceConnector(value);
}
}
}
function migrateSurfaceConnector(data: YMap<any>) {
let id = data.get('startElement')?.id;
const controllers = data.get('controllers');
const length = controllers.length;
const xywh = deserializeXYWH(data.get('xywh'));
if (id) {
data.set('source', { id });
} else {
data.set('source', {
position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]],
});
}
id = data.get('endElement')?.id;
if (id) {
data.set('target', { id });
} else {
data.set('target', {
position: [
controllers[length - 1].x + xywh[0],
controllers[length - 1].y + xywh[1],
],
});
}
const width = data.get('lineWidth') ?? 4;
data.set('strokeWidth', width);
const color = data.get('color');
data.set('stroke', color);
data.delete('startElement');
data.delete('endElement');
data.delete('controllers');
data.delete('lineWidth');
data.delete('color');
data.delete('xywh');
}
function updateBlockVersions(versions: YMap<number>) {
const frameVersion = versions.get('affine:frame');
if (frameVersion !== undefined) {
versions.set('affine:note', frameVersion);
versions.delete('affine:frame');
}
const embedVersion = versions.get('affine:embed');
if (embedVersion !== undefined) {
versions.set('affine:image', embedVersion);
versions.delete('affine:embed');
}
const databaseVersion = versions.get('affine:database');
if (databaseVersion !== undefined && databaseVersion < 2) {
versions.set('affine:database', 2);
}
}
function migrateMeta(
oldDoc: YDoc,
newDoc: YDoc,
idMap: Record<string, string>
) {
const originalMeta = oldDoc.getMap('space:meta');
const originalVersions = originalMeta.get('versions') as YMap<number>;
const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
const meta = newDoc.getMap('meta');
const pages = new YArray();
const blockVersions = originalVersions.clone();
meta.set('workspaceVersion', 1);
meta.set('blockVersions', blockVersions);
meta.set('pages', pages);
meta.set('name', originalMeta.get('name') as string);
updateBlockVersions(blockVersions);
const mapList = originalPages.map(page => {
const map = new YMap();
Array.from(page.entries())
.filter(([key]) => key !== 'subpageIds')
.forEach(([key, value]) => {
if (key === 'id') {
idMap[value] = nanoid();
map.set(key, idMap[value]);
} else {
map.set(key, value);
}
});
return map;
});
pages.push(mapList);
}
function migrateBlocks(
oldDoc: YDoc,
newDoc: YDoc,
idMap: Record<string, string>
) {
const spaces = newDoc.getMap('spaces');
const originalMeta = oldDoc.getMap('space:meta');
const originalVersions = originalMeta.get('versions') as YMap<number>;
const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
originalPages.forEach(page => {
const id = page.get('id') as string;
const newId = idMap[id];
const spaceId = id.startsWith('space:') ? id : `space:${id}`;
const originalBlocks = oldDoc.getMap(spaceId) as YMap<unknown>;
const subdoc = new YDoc();
spaces.set(newId, subdoc);
subdoc.guid = id;
const blocks = subdoc.getMap('blocks');
Array.from(originalBlocks.entries()).forEach(([key, value]) => {
const blockData = value.clone();
blocks.set(key, blockData);
const flavour = blockData.get('sys:flavour') as string;
const version = originalVersions.get(flavour);
if (version !== undefined) {
runBlockMigration(flavour, blockData, version);
}
});
});
}
export function migrateToSubdoc(oldDoc: YDoc): YDoc {
const needMigration =
Array.from(oldDoc.getMap('space:meta').keys()).length > 0;
if (!needMigration) {
return oldDoc;
}
const newDoc = new YDoc();
const idMap = {} as Record<string, string>;
migrateMeta(oldDoc, newDoc, idMap);
migrateBlocks(oldDoc, newDoc, idMap);
return newDoc;
}
export const upgradeV1ToV2 = async (
oldDoc: YDoc,
createWorkspace: () => Promise<Workspace>
) => {
const newDoc = migrateToSubdoc(oldDoc);
const newWorkspace = await createWorkspace();
applyUpdate(newWorkspace.doc, encodeStateAsUpdate(newDoc), migrationOrigin);
newDoc.getSubdocs().forEach(subdoc => {
newWorkspace.doc.getSubdocs().forEach(newDoc => {
if (subdoc.guid === newDoc.guid) {
applyUpdate(newDoc, encodeStateAsUpdate(subdoc), migrationOrigin);
}
});
});
return newWorkspace;
};

77
packages/common/infra/src/blocksuite/migration/workspace.ts Normal file
View File

@ -0,0 +1,77 @@
import type { Workspace } from '@blocksuite/store';
import type { Schema } from '@blocksuite/store';
import type { Doc as YDoc } from 'yjs';
import { migratePages } from './blocksuite';
import { upgradeV1ToV2 } from './subdoc';
interface MigrationOptions {
doc: YDoc;
schema: Schema;
createWorkspace: () => Promise<Workspace>;
}
function createMigrationQueue(options: MigrationOptions) {
return [
async (doc: YDoc) => {
const newWorkspace = await upgradeV1ToV2(doc, options.createWorkspace);
return newWorkspace.doc;
},
async (doc: YDoc) => {
await migratePages(doc, options.schema);
return doc;
},
];
}
/**
* For split migrate function from MigrationQueue.
*/
export enum MigrationPoint {
SubDoc = 1,
BlockVersion = 2,
}
export async function migrateWorkspace(
point: MigrationPoint,
options: MigrationOptions
) {
const migrationQueue = createMigrationQueue(options);
const migrationFns = migrationQueue.slice(point - 1);
let doc = options.doc;
for (const migrate of migrationFns) {
doc = await migrate(doc);
}
return doc;
}
export function checkWorkspaceCompatibility(
workspace: Workspace
): MigrationPoint | null {
const workspaceDocJSON = workspace.doc.toJSON();
const spaceMetaObj = workspaceDocJSON['space:meta'];
const docKeys = Object.keys(workspaceDocJSON);
const haveSpaceMeta = !!spaceMetaObj && Object.keys(spaceMetaObj).length > 0;
const haveLegacySpace = docKeys.some(key => key.startsWith('space:'));
if (haveSpaceMeta || haveLegacySpace) {
return MigrationPoint.SubDoc;
}
// Sometimes, blocksuite will not write blockVersions to meta.
// Just fix it when user open the workspace.
const blockVersions = workspace.meta.blockVersions;
if (!blockVersions) {
return MigrationPoint.BlockVersion;
}
// From v2, we depend on blocksuite to check and migrate data.
for (const [flavour, version] of Object.entries(blockVersions)) {
const schema = workspace.schema.flavourSchemaMap.get(flavour);
if (schema?.version !== version) {
return MigrationPoint.BlockVersion;
}
}
return null;
}

6
packages/frontend/core/src/adapters/local/index.tsx
View File

@ -13,10 +13,7 @@ import {
CRUD,
saveWorkspaceToLocalStorage,
} from '@affine/workspace/local/crud';
import {
getOrCreateWorkspace,
globalBlockSuiteSchema,
} from '@affine/workspace/manager';
import { getOrCreateWorkspace } from '@affine/workspace/manager';
import { getBlockSuiteWorkspaceAtom } from '@toeverything/infra/__internal__/workspace';
import { getCurrentStore } from '@toeverything/infra/atom';
import { initEmptyPage } from '@toeverything/infra/blocksuite';
@ -47,7 +44,6 @@ export const LocalAdapter: WorkspaceAdapter<WorkspaceFlavour.LOCAL> = {
blockSuiteWorkspace.meta.setName(DEFAULT_WORKSPACE_NAME);
if (runtimeConfig.enablePreloading) {
buildShowcaseWorkspace(blockSuiteWorkspace, {
schema: globalBlockSuiteSchema,
store: getCurrentStore(),
atoms: {
pageMode: setPageModeAtom,

117
packages/frontend/core/src/bootstrap/setup.ts
View File

@ -1,129 +1,18 @@
import { setupGlobal } from '@affine/env/global';
import type { WorkspaceAdapter } from '@affine/env/workspace';
import { WorkspaceFlavour } from '@affine/env/workspace';
import type { RootWorkspaceMetadata } from '@affine/workspace/atom';
import type { WorkspaceFlavour } from '@affine/env/workspace';
import {
type RootWorkspaceMetadataV2,
rootWorkspacesMetadataAtom,
workspaceAdaptersAtom,
} from '@affine/workspace/atom';
import {
getOrCreateWorkspace,
globalBlockSuiteSchema,
} from '@affine/workspace/manager';
import { assertExists } from '@blocksuite/global/utils';
import {
migrateLocalBlobStorage,
migrateWorkspace,
WorkspaceVersion,
} from '@toeverything/infra/blocksuite';
import { downloadBinary, overwriteBinary } from '@toeverything/y-indexeddb';
import type { createStore } from 'jotai/vanilla';
import { nanoid } from 'nanoid';
import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
import { WorkspaceAdapters } from '../adapters/workspace';
import { performanceLogger } from '../shared';
const performanceSetupLogger = performanceLogger.namespace('setup');
async function tryMigration() {
const value = localStorage.getItem('jotai-workspaces');
if (value) {
try {
const metadata = JSON.parse(value) as RootWorkspaceMetadata[];
const promises: Promise<void>[] = [];
const newMetadata = [...metadata];
metadata.forEach(oldMeta => {
if (oldMeta.flavour === WorkspaceFlavour.LOCAL) {
let doc: YDoc;
const options = {
getCurrentRootDoc: async () => {
doc = new YDoc({
guid: oldMeta.id,
});
const downloadWorkspace = async (doc: YDoc): Promise<void> => {
const binary = await downloadBinary(doc.guid);
if (binary) {
applyUpdate(doc, binary);
}
await Promise.all(
[...doc.subdocs.values()].map(subdoc =>
downloadWorkspace(subdoc)
)
);
};
await downloadWorkspace(doc);
return doc;
},
createWorkspace: async () =>
getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL),
getSchema: () => globalBlockSuiteSchema,
};
promises.push(
migrateWorkspace(
'version' in oldMeta ? oldMeta.version : undefined,
options
).then(async status => {
if (typeof status !== 'boolean') {
const adapter = WorkspaceAdapters[oldMeta.flavour];
const oldWorkspace = await adapter.CRUD.get(oldMeta.id);
const newId = await adapter.CRUD.create(status);
assertExists(
oldWorkspace,
'workspace should exist after migrate'
);
await adapter.CRUD.delete(oldWorkspace.blockSuiteWorkspace);
const index = newMetadata.findIndex(
meta => meta.id === oldMeta.id
);
newMetadata[index] = {
...oldMeta,
id: newId,
version: WorkspaceVersion.Surface,
};
await migrateLocalBlobStorage(status.id, newId);
console.log('workspace migrated', oldMeta.id, newId);
} else if (status) {
const index = newMetadata.findIndex(
meta => meta.id === oldMeta.id
);
newMetadata[index] = {
...oldMeta,
version: WorkspaceVersion.Surface,
};
const overWrite = async (doc: YDoc): Promise<void> => {
await overwriteBinary(doc.guid, encodeStateAsUpdate(doc));
return Promise.all(
[...doc.subdocs.values()].map(subdoc => overWrite(subdoc))
).then();
};
await overWrite(doc);
console.log('workspace migrated', oldMeta.id);
}
})
);
}
});
await Promise.all(promises)
.then(() => {
console.log('migration done');
})
.catch(e => {
console.error('migration failed', e);
})
.finally(() => {
localStorage.setItem('jotai-workspaces', JSON.stringify(newMetadata));
window.dispatchEvent(new CustomEvent('migration-done'));
window.$migrationDone = true;
});
} catch (e) {
console.error('error when migrating data', e);
}
}
}
export function createFirstAppData(store: ReturnType<typeof createStore>) {
const createFirst = (): RootWorkspaceMetadataV2[] => {
const Plugins = Object.values(WorkspaceAdapters).sort(
@ -136,7 +25,6 @@ export function createFirstAppData(store: ReturnType<typeof createStore>) {
<RootWorkspaceMetadataV2>{
id,
flavour: Plugin.flavour,
version: WorkspaceVersion.DatabaseV3,
}
);
}).filter((ids): ids is RootWorkspaceMetadataV2 => !!ids);
@ -163,9 +51,6 @@ export async function setup(store: ReturnType<typeof createStore>) {
performanceSetupLogger.info('setup global');
setupGlobal();
performanceSetupLogger.info('try migration');
await tryMigration();
performanceSetupLogger.info('get root workspace meta');
// do not read `rootWorkspacesMetadataAtom` before migration
await store.get(rootWorkspacesMetadataAtom);

114
packages/frontend/core/src/components/workspace-upgrade/upgrade-hooks.ts
View File

@ -1,42 +1,122 @@
import { forceUpgradePages } from '@toeverything/infra/blocksuite';
import { useCallback, useState } from 'react';
import { WorkspaceFlavour } from '@affine/env/workspace';
import type { RootWorkspaceMetadata } from '@affine/workspace/atom';
import { getOrCreateWorkspace } from '@affine/workspace/manager';
import type { Workspace } from '@blocksuite/store';
import { useAsyncCallback } from '@toeverything/hooks/affine-async-hooks';
import { getBlockSuiteWorkspaceAtom } from '@toeverything/infra/__internal__/workspace';
import { getCurrentStore } from '@toeverything/infra/atom';
import type { MigrationPoint } from '@toeverything/infra/blocksuite';
import {
migrateLocalBlobStorage,
migrateWorkspace,
} from '@toeverything/infra/blocksuite';
import { nanoid } from 'nanoid';
import { useState } from 'react';
import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
import { WorkspaceAdapters } from '../../adapters/workspace';
import { useCurrentSyncEngine } from '../../hooks/current/use-current-sync-engine';
import { useCurrentWorkspace } from '../../hooks/current/use-current-workspace';
export type UpgradeState = 'pending' | 'upgrading' | 'done' | 'error';
export function useUpgradeWorkspace() {
function applyDoc(target: YDoc, result: YDoc) {
applyUpdate(target, encodeStateAsUpdate(result));
for (const targetSubDoc of target.subdocs.values()) {
const resultSubDocs = Array.from(result.subdocs.values());
const resultSubDoc = resultSubDocs.find(
item => item.guid === targetSubDoc.guid
);
if (resultSubDoc) {
applyDoc(targetSubDoc, resultSubDoc);
}
}
}
export function useUpgradeWorkspace(migration: MigrationPoint) {
const [state, setState] = useState<UpgradeState>('pending');
const [error, setError] = useState<Error | null>(null);
const [newWorkspaceId, setNewWorkspaceId] = useState<string | null>(null);
const [workspace] = useCurrentWorkspace();
const syncEngine = useCurrentSyncEngine();
const rootStore = getCurrentStore();
const upgradeWorkspace = useCallback(() => {
const upgradeWorkspace = useAsyncCallback(async () => {
setState('upgrading');
setError(null);
(async () => {
try {
// Migration need to wait for root doc and all subdocs loaded.
await syncEngine?.waitForSynced();
await forceUpgradePages({
getCurrentRootDoc: async () => workspace.blockSuiteWorkspace.doc,
getSchema: () => workspace.blockSuiteWorkspace.schema,
// Clone a new doc to prevent change events.
const clonedDoc = new YDoc({
guid: workspace.blockSuiteWorkspace.doc.guid,
});
applyDoc(clonedDoc, workspace.blockSuiteWorkspace.doc);
const schema = workspace.blockSuiteWorkspace.schema;
let newWorkspace: Workspace | null = null;
const resultDoc = await migrateWorkspace(migration, {
doc: clonedDoc,
schema,
createWorkspace: () => {
// Migrate to subdoc version need to create a new workspace.
// It will only happened for old local workspace.
newWorkspace = getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL);
return Promise.resolve(newWorkspace);
},
});
if (newWorkspace) {
const localMetaString =
localStorage.getItem('jotai-workspaces') ?? '[]';
const localMetadataList = JSON.parse(
localMetaString
) as RootWorkspaceMetadata[];
const currentLocalMetadata = localMetadataList.find(
item => item.id === workspace.id
);
const flavour = currentLocalMetadata?.flavour ?? WorkspaceFlavour.LOCAL;
// Legacy logic moved from `setup.ts`.
// It works well before, should be refactor or remove in the future.
const adapter = WorkspaceAdapters[flavour];
const newId = await adapter.CRUD.create(newWorkspace);
const [workspaceAtom] = getBlockSuiteWorkspaceAtom(newId);
await rootStore.get(workspaceAtom); // Trigger provider sync to persist data.
await adapter.CRUD.delete(workspace.blockSuiteWorkspace);
await migrateLocalBlobStorage(workspace.id, newId);
setNewWorkspaceId(newId);
const index = localMetadataList.findIndex(
meta => meta.id === workspace.id
);
localMetadataList[index] = {
...currentLocalMetadata,
id: newId,
flavour,
};
localStorage.setItem(
'jotai-workspaces',
JSON.stringify(localMetadataList)
);
localStorage.setItem('last_workspace_id', newId);
localStorage.removeItem('last_page_id');
} else {
applyDoc(workspace.blockSuiteWorkspace.doc, resultDoc);
}
await syncEngine?.waitForSynced();
setState('done');
})().catch((e: any) => {
} catch (e: any) {
console.error(e);
setError(e);
setState('error');
});
}, [
workspace.blockSuiteWorkspace.doc,
workspace.blockSuiteWorkspace.schema,
syncEngine,
]);
}
}, [rootStore, workspace, syncEngine, migration]);
return [state, error, upgradeWorkspace] as const;
return [state, error, upgradeWorkspace, newWorkspaceId] as const;
}

21
packages/frontend/core/src/components/workspace-upgrade/upgrade.tsx
View File

@ -1,8 +1,10 @@
import { AffineShapeIcon } from '@affine/component/page-list'; // TODO: import from page-list temporarily, need to defined common svg icon/images management.
import { useAFFiNEI18N } from '@affine/i18n/hooks';
import { Button } from '@toeverything/components/button';
import type { MigrationPoint } from '@toeverything/infra/blocksuite';
import { useCallback, useMemo } from 'react';
import { pathGenerator } from '../../shared';
import * as styles from './upgrade.css';
import { type UpgradeState, useUpgradeWorkspace } from './upgrade-hooks';
import { ArrowCircleIcon, HeartBreakIcon } from './upgrade-icon';
@ -32,11 +34,18 @@ function UpgradeIcon({ upgradeState }: { upgradeState: UpgradeState }) {
);
}
interface WorkspaceUpgradeProps {
migration: MigrationPoint;
}
/**
* TODO: Help info is not implemented yet.
*/
export const WorkspaceUpgrade = function MigrationFallback() {
const [upgradeState, , upgradeWorkspace] = useUpgradeWorkspace();
export const WorkspaceUpgrade = function WorkspaceUpgrade(
props: WorkspaceUpgradeProps
) {
const [upgradeState, , upgradeWorkspace, newWorkspaceId] =
useUpgradeWorkspace(props.migration);
const t = useAFFiNEI18N();
const refreshPage = useCallback(() => {
@ -45,6 +54,12 @@ export const WorkspaceUpgrade = function MigrationFallback() {
const onButtonClick = useMemo(() => {
if (upgradeState === 'done') {
if (newWorkspaceId) {
return () => {
window.location.replace(pathGenerator.all(newWorkspaceId));
};
}
return refreshPage;
}
@ -53,7 +68,7 @@ export const WorkspaceUpgrade = function MigrationFallback() {
}
return undefined;
}, [upgradeState, upgradeWorkspace, refreshPage]);
}, [upgradeState, upgradeWorkspace, refreshPage, newWorkspaceId]);
return (
<div className={styles.layout}>

6
packages/frontend/core/src/hooks/use-workspaces.ts
View File

@ -2,10 +2,7 @@ import { DebugLogger } from '@affine/debug';
import { WorkspaceFlavour } from '@affine/env/workspace';
import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud';
import {
getOrCreateWorkspace,
globalBlockSuiteSchema,
} from '@affine/workspace/manager';
import { getOrCreateWorkspace } from '@affine/workspace/manager';
import { getWorkspace } from '@toeverything/infra/__internal__/workspace';
import { getCurrentStore } from '@toeverything/infra/atom';
import {
@ -76,7 +73,6 @@ export function useAppHelper() {
WorkspaceFlavour.LOCAL
);
await buildShowcaseWorkspace(blockSuiteWorkspace, {
schema: globalBlockSuiteSchema,
store: getCurrentStore(),
atoms: {
pageMode: setPageModeAtom,

15
packages/frontend/core/src/layouts/workspace-layout.tsx
View File

@ -27,6 +27,7 @@ import {
} from '@dnd-kit/core';
import { useBlockSuitePageMeta } from '@toeverything/hooks/use-block-suite-page-meta';
import { currentWorkspaceIdAtom } from '@toeverything/infra/atom';
import type { MigrationPoint } from '@toeverything/infra/blocksuite';
import { useAtom, useAtomValue, useSetAtom } from 'jotai';
import type { PropsWithChildren, ReactNode } from 'react';
import { lazy, Suspense, useCallback, useEffect, useState } from 'react';
@ -112,12 +113,12 @@ export const CurrentWorkspaceContext = ({
};
type WorkspaceLayoutProps = {
incompatible?: boolean;
migration?: MigrationPoint;
};
export const WorkspaceLayout = function WorkspacesSuspense({
children,
incompatible = false,
migration,
}: PropsWithChildren<WorkspaceLayoutProps>) {
return (
<AdapterProviderWrapper>
@ -128,7 +129,7 @@ export const WorkspaceLayout = function WorkspacesSuspense({
<CurrentWorkspaceModals />
</Suspense>
<Suspense fallback={<WorkspaceFallback />}>
<WorkspaceLayoutInner incompatible={incompatible}>
<WorkspaceLayoutInner migration={migration}>
{children}
</WorkspaceLayoutInner>
</Suspense>
@ -139,7 +140,7 @@ export const WorkspaceLayout = function WorkspacesSuspense({
export const WorkspaceLayoutInner = ({
children,
incompatible = false,
migration,
}: PropsWithChildren<WorkspaceLayoutProps>) => {
const [currentWorkspace] = useCurrentWorkspace();
const { openPage } = useNavigateHelper();
@ -262,7 +263,11 @@ export const WorkspaceLayoutInner = ({
padding={appSettings.clientBorder}
inTrashPage={inTrashPage}
>
{incompatible ? <WorkspaceUpgrade /> : children}
{migration ? (
<WorkspaceUpgrade migration={migration} />
) : (
children
)}
<ToolContainer inTrashPage={inTrashPage}>
<RootBlockHub />
<HelpIsland showList={pageId ? undefined : showList} />

30
packages/frontend/core/src/pages/workspace/index.tsx
View File

@ -1,4 +1,3 @@
import { WorkspaceFlavour } from '@affine/env/workspace';
import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
import { getBlockSuiteWorkspaceAtom } from '@toeverything/infra/__internal__/workspace';
import {
@ -6,7 +5,11 @@ import {
currentWorkspaceIdAtom,
getCurrentStore,
} from '@toeverything/infra/atom';
import { guidCompatibilityFix } from '@toeverything/infra/blocksuite';
import type { MigrationPoint } from '@toeverything/infra/blocksuite';
import {
checkWorkspaceCompatibility,
guidCompatibilityFix,
} from '@toeverything/infra/blocksuite';
import { useSetAtom } from 'jotai';
import { type ReactElement, useEffect } from 'react';
import {
@ -49,22 +52,9 @@ export const loader: LoaderFunction = async args => {
const workspace = await rootStore.get(workspaceAtom);
workspaceLoaderLogger.info('workspace loaded');
if (currentMetadata.flavour === WorkspaceFlavour.AFFINE_CLOUD) {
return (() => {
guidCompatibilityFix(workspace.doc);
const blockVersions = workspace.meta.blockVersions;
if (!blockVersions) {
return true;
}
for (const [flavour, schema] of workspace.schema.flavourSchemaMap) {
if (blockVersions[flavour] !== schema.version) {
return true;
}
}
return false;
})();
}
return null;
guidCompatibilityFix(workspace.doc);
return checkWorkspaceCompatibility(workspace);
};
export const Component = (): ReactElement => {
@ -81,10 +71,10 @@ export const Component = (): ReactElement => {
}
}, [params, setCurrentWorkspaceId]);
const incompatible = useLoaderData();
const migration = useLoaderData() as MigrationPoint | undefined;
return (
<AffineErrorBoundary height="100vh">
<WorkspaceLayout incompatible={!!incompatible}>
<WorkspaceLayout migration={migration}>
<Outlet />
</WorkspaceLayout>
</AffineErrorBoundary>

5
packages/frontend/electron/src/helper/db/migration.ts
View File

@ -77,10 +77,7 @@ export const migrateToLatest = async (
);
};
await downloadBinary(rootDoc, true);
const result = await forceUpgradePages({
getSchema: () => schema,
getCurrentRootDoc: () => Promise.resolve(rootDoc),
});
const result = await forceUpgradePages(rootDoc, schema);
equal(result, true, 'migrateWorkspace should return boolean value');
const uploadBinary = async (doc: YDoc, isRoot: boolean) => {
await connection.replaceUpdates(doc.guid, [

6
packages/frontend/hooks/src/affine-async-hooks.ts
View File

@ -5,8 +5,10 @@ export type AsyncErrorHandler = (error: Error) => void;
/**
* App should provide a global error handler for async callback in the root.
*/
export const AsyncCallbackContext = React.createContext<AsyncErrorHandler>(e =>
console.error(e)
export const AsyncCallbackContext = React.createContext<AsyncErrorHandler>(
e => {
console.error(e);
}
);
/**

34
packages/frontend/workspace/src/atom.ts
View File

@ -74,15 +74,12 @@ const rootWorkspacesMetadataPrimitiveAtom = atom<Promise<
type Getter = <Value>(atom: Atom<Value>) => Value;
type FetchMetadata = (
get: Getter,
options: { signal: AbortSignal }
) => Promise<RootWorkspaceMetadata[]>;
type FetchMetadata = (get: Getter) => Promise<RootWorkspaceMetadata[]>;
/**
* @internal
*/
const fetchMetadata: FetchMetadata = async (get, { signal }) => {
const fetchMetadata: FetchMetadata = async get => {
performanceJotaiLogger.info('fetch metadata start');
const WorkspaceAdapters = get(workspaceAdaptersAtom);
@ -111,23 +108,6 @@ const fetchMetadata: FetchMetadata = async (get, { signal }) => {
}
return [];
};
const maybeMetadata = loadFromLocalStorage();
// migration step, only data in `METADATA_STORAGE_KEY` will be migrated
if (
maybeMetadata.some(meta => !('version' in meta)) &&
!window.$migrationDone
) {
await new Promise<void>((resolve, reject) => {
signal.addEventListener('abort', () => reject(), { once: true });
window.addEventListener('migration-done', () => resolve(), {
once: true,
});
});
performanceJotaiLogger.info('migration done');
}
metadata.push(...loadFromLocalStorage());
}
// step 2: fetch from adapters
@ -211,14 +191,14 @@ const fetchMetadata: FetchMetadata = async (get, { signal }) => {
const rootWorkspacesMetadataPromiseAtom = atom<
Promise<RootWorkspaceMetadata[]>
>(async (get, { signal }) => {
>(async get => {
const primitiveMetadata = get(rootWorkspacesMetadataPrimitiveAtom);
assertEquals(
primitiveMetadata,
null,
'rootWorkspacesMetadataPrimitiveAtom should be null'
);
return fetchMetadata(get, { signal });
return fetchMetadata(get);
});
type SetStateAction<Value> = Value | ((prev: Value) => Value);
@ -276,11 +256,7 @@ export const rootWorkspacesMetadataAtom = atom<
);
export const refreshRootMetadataAtom = atom(null, (get, set) => {
const abortController = new AbortController();
set(
rootWorkspacesMetadataPrimitiveAtom,
fetchMetadata(get, { signal: abortController.signal })
);
set(rootWorkspacesMetadataPrimitiveAtom, fetchMetadata(get));
});
// blocksuite atoms,

3
tests/affine-migration/README.md
View File

@ -7,3 +7,6 @@ BUILD_TYPE=canary yarn run build
cd tests/affine-migration
yarn run e2e
```
> Tips:
> Run `yarn dev` to start dev server in 8080 could make debugging more quickly.

112
tests/affine-migration/e2e/basic.spec.ts
View File

@ -34,26 +34,21 @@ test('v1 to v4', async ({ page }) => {
await page.goto(coreUrl);
await clickSideBarAllPageButton(page);
await page.getByText('hello').click();
//#region fixme(himself65): blocksuite issue, data cannot be loaded to store
const url = page.url();
await page.waitForTimeout(5000);
await page.goto(url);
//#endregion
await expect(page.getByTestId('upgrade-workspace-button')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await expect(page.getByText('Refresh Current Page')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await expect(page.getByTestId('page-list-item')).toHaveCount(2);
await page
.getByTestId('page-list-item-title-text')
.getByText('hello')
.click();
await waitForEditorLoad(page);
expect(await page.locator('v-line').nth(0).textContent()).toBe('hello');
const changedLocalStorageData = await page.evaluate(() =>
window.readAffineLocalStorage()
);
const workspaces = JSON.parse(
changedLocalStorageData['jotai-workspaces']
) as any[];
for (const workspace of workspaces) {
expect(workspace.version).toBe(4);
}
await expect(page.locator('v-line').nth(0)).toHaveText('hello');
});
test('v2 to v4, database migration', async ({ page }) => {
@ -62,99 +57,70 @@ test('v2 to v4, database migration', async ({ page }) => {
'0.8.0-canary.7'
);
//#region fixme(himself65): blocksuite issue, data cannot be loaded to store
const allPagePath = `${coreUrl}/workspace/${localStorageData.last_workspace_id}/all`;
await page.goto(allPagePath);
await page.waitForTimeout(5000);
//#endregion
const detailPagePath = `${coreUrl}/workspace/${localStorageData.last_workspace_id}/${localStorageData.last_page_id}`;
await page.goto(detailPagePath);
await expect(page.getByTestId('upgrade-workspace-button')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await expect(page.getByText('Refresh Current Page')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await waitForEditorLoad(page);
// check page mode is correct
expect(await page.locator('v-line').nth(0).textContent()).toBe('hello');
expect(await page.locator('affine-database').isVisible()).toBe(true);
await expect(page.locator('v-line').nth(0)).toHaveText('hello');
await expect(page.locator('affine-database')).toBeVisible();
// check edgeless mode is correct
await clickEdgelessModeButton(page);
await page.waitForTimeout(200);
expect(await page.locator('affine-database').isVisible()).toBe(true);
const changedLocalStorageData = await page.evaluate(() =>
window.readAffineLocalStorage()
);
const workspaces = JSON.parse(
changedLocalStorageData['jotai-workspaces']
) as any[];
for (const workspace of workspaces) {
expect(workspace.version).toBe(4);
}
await expect(page.locator('affine-database')).toBeVisible();
});
test('v3 to v4, surface migration', async ({ page }) => {
const { localStorageData } = await open404PageToInitData(page, '0.8.4');
//#region fixme(himself65): blocksuite issue, data cannot be loaded to store
const allPagePath = `${coreUrl}/workspace/${localStorageData.last_workspace_id}/all`;
await page.goto(allPagePath);
await page.waitForTimeout(5000);
//#endregion
const detailPagePath = `${coreUrl}/workspace/${localStorageData.last_workspace_id}/${localStorageData.last_page_id}`;
await page.goto(detailPagePath);
await expect(page.getByTestId('upgrade-workspace-button')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await expect(page.getByText('Refresh Current Page')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await waitForEditorLoad(page);
// check edgeless mode is correct
await clickEdgelessModeButton(page);
await expect(page.locator('edgeless-toolbar')).toBeVisible();
await expect(page.locator('affine-edgeless-page')).toBeVisible();
const changedLocalStorageData = await page.evaluate(() =>
window.readAffineLocalStorage()
);
const workspaces = JSON.parse(
changedLocalStorageData['jotai-workspaces']
) as any[];
for (const workspace of workspaces) {
expect(workspace.version).toBe(4);
}
});
test('v0 to v4, subdoc migration', async ({ page }) => {
await open404PageToInitData(page, '0.6.1-beta.1');
await page.goto(coreUrl);
await page.waitForTimeout(5000);
// go to all page
await clickSideBarAllPageButton(page);
// find if page name with "hello" exists and click it
await expect(page.getByTestId('upgrade-workspace-button')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await expect(page.getByText('Refresh Current Page')).toBeVisible();
await page.getByTestId('upgrade-workspace-button').click();
await expect(page.getByTestId('page-list-item')).toHaveCount(2);
await page
.locator('[data-testid="page-list-item-title-text"]:has-text("hello")')
.getByTestId('page-list-item-title-text')
.getByText('hello')
.click();
await waitForEditorLoad(page);
// check if content is correct
expect(await page.locator('v-line').nth(0).textContent()).toBe('hello');
expect(await page.locator('v-line').nth(1).textContent()).toBe(
'TEST CONTENT'
);
// check page mode is correct
await expect(page.locator('v-line').nth(0)).toHaveText('hello');
await expect(page.locator('v-line').nth(1)).toHaveText('TEST CONTENT');
// check edgeless mode is correct
await clickEdgelessModeButton(page);
await expect(page.locator('edgeless-toolbar')).toBeVisible();
await expect(page.locator('affine-edgeless-page')).toBeVisible();
const changedLocalStorageData = await page.evaluate(() =>
window.readAffineLocalStorage()
);
const workspaces = JSON.parse(
changedLocalStorageData['jotai-workspaces']
) as any[];
for (const workspace of workspaces) {
expect(workspace.version).toBe(4);
}
});

5
tests/kit/utils/workspace.ts
View File

@ -30,10 +30,13 @@ export async function createLocalWorkspace(
await page.getByPlaceholder('Set a Workspace name').fill(params.name);
// click create button
await page.getByRole('button', { name: 'Create' }).click({
await page.getByTestId('create-workspace-create-button').click({
delay: 500,
});
await expect(
page.getByTestId('create-workspace-create-button')
).not.toBeAttached();
await waitForEditorLoad(page);
await expect(page.getByTestId('workspace-name')).toHaveText(params.name);

5
tools/@types/env/__all.d.ts vendored
View File

@ -44,11 +44,6 @@ declare global {
): this;
};
};
$migrationDone: boolean | undefined;
}
interface WindowEventMap {
'migration-done': CustomEvent;
}
// eslint-disable-next-line no-var