feat: support get datasource status (#3645)

.github/workflows/build.yml

@@ -59,6 +59,25 @@ jobs:
       - name: Run Type Check
         run: yarn typecheck
 
+  build-prototype:
+    name: Build Prototype
+    runs-on: ubuntu-latest
+    environment: development
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+      - name: Build Prototype
+        run: yarn nx build prototype
+      - name: Upload prototype artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: prototype
+          path: ./apps/prototype/dist
+          if-no-files-found: error
+
   build-server:
     name: Build Server
     runs-on: ubuntu-latest
@@ -280,6 +299,49 @@ jobs:
           path: ./test-results
           if-no-files-found: ignore
 
+  e2e-prototype-test:
+    name: E2E Prototype Test
+    runs-on: ubuntu-latest
+    environment: development
+    needs: build-prototype
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+      - name: Download prototype artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: prototype
+          path: ./apps/prototype/dist
+      - name: Run playwright tests
+        run: yarn e2e --forbid-only
+        working-directory: tests/affine-prototype
+        env:
+          COVERAGE: true
+
+      #      - name: Collect code coverage report
+      #        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
+
+      #      - name: Upload e2e test coverage results
+      #        uses: codecov/codecov-action@v3
+      #        with:
+      #          token: ${{ secrets.CODECOV_TOKEN }}
+      #          files: ./.coverage/lcov.info
+      #          flags: e2etest-prototype
+      #          name: affine
+      #          fail_ci_if_error: false
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-e2e-prototype
+          path: ./test-results
+          if-no-files-found: ignore
+
   e2e-test:
     name: E2E Test
     runs-on: ubuntu-latest

apps/README.md

@@ -8,7 +8,7 @@ AFFiNE Developer Documentation using [waku](https://github.com/dai-shi/waku).
 
 ## electron
 
-> `web` needs to be built before electron.
+> `core` needs to be built before electron.
 
 AFFiNE Desktop (macOS, Linux and Windows Distribution) using [Electron](https://www.electronjs.org/).
 
@@ -20,6 +20,10 @@ Server using [Nest.js](https://nestjs.com/).
 
 Storybook using [Storybook](https://storybook.js.org/).
 
-## Core
+## prototype
 
-AFFiNE Core Application using [React.js](https://reactjs.org/).
+AFFiNE Prototype using [React.js](https://reactjs.org/) + [Vite](https://vitejs.dev/).
+
+## core
+
+AFFiNE Core Application using [React.js](https://reactjs.org/) + [Webpack](https://webpack.js.org/).

apps/core/src/components/pure/workspace-slider-bar/WorkspaceSelector/workspace-selector.tsx

@@ -4,7 +4,6 @@ import { useBlockSuiteWorkspaceName } from '@toeverything/hooks/use-block-suite-
 import type React from 'react';
 import { useCallback } from 'react';
 
-import { useCurrentWorkspace } from '../../../../hooks/current/use-current-workspace';
 import type { AllWorkspace } from '../../../../shared';
 import { workspaceAvatarStyle } from './index.css';
 import {
@@ -28,9 +27,8 @@ export const WorkspaceSelector = ({
   onClick,
 }: WorkspaceSelectorProps) => {
   const [name] = useBlockSuiteWorkspaceName(
-    currentWorkspace?.blockSuiteWorkspace
+    currentWorkspace.blockSuiteWorkspace
   );
-  const [workspace] = useCurrentWorkspace();
 
   // Open dialog when `Enter` or `Space` pressed
   // TODO-Doma Refactor with `@radix-ui/react-dialog` or other libraries that handle these out of the box and be accessible by default
@@ -57,22 +55,20 @@ export const WorkspaceSelector = ({
         data-testid="workspace-avatar"
         className={workspaceAvatarStyle}
         size={40}
-        workspace={currentWorkspace?.blockSuiteWorkspace ?? null}
+        workspace={currentWorkspace.blockSuiteWorkspace}
       />
       <StyledSelectorWrapper>
         <StyledWorkspaceName data-testid="workspace-name">
           {name}
         </StyledWorkspaceName>
-        {workspace && (
-          <StyledWorkspaceStatus>
-            {workspace.flavour === 'local' ? (
-              <LocalWorkspaceIcon />
-            ) : (
-              <CloudWorkspaceIcon />
-            )}
-            {workspace.flavour === 'local' ? 'Local' : 'AFFiNE Cloud'}
-          </StyledWorkspaceStatus>
-        )}
+        <StyledWorkspaceStatus>
+          {currentWorkspace.flavour === 'local' ? (
+            <LocalWorkspaceIcon />
+          ) : (
+            <CloudWorkspaceIcon />
+          )}
+          {currentWorkspace.flavour === 'local' ? 'Local' : 'AFFiNE Cloud'}
+        </StyledWorkspaceStatus>
       </StyledSelectorWrapper>
     </StyledSelectorContainer>
   );

apps/prototype/README.md

@@ -0,0 +1,5 @@
+# AFFiNE Prototype
+
+> This is a prototype of the AFFiNE system to test the feasibility of the approach.
+>
+> It is not intended for production use.

apps/prototype/index.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>AFFiNE Prototype</title>
+  </head>
+  <body>
+    <ul>
+      <li>
+        <a href="suite/provider-status.html">Provider status test</a>
+      </li>
+    </ul>
+  </body>
+</html>

apps/prototype/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@affine/prototype",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite --host --port 3003",
+    "build": "tsc -b && vite build",
+    "preview": "vite preview --host --port 3003"
+  },
+  "dependencies": {
+    "@affine-test/fixtures": "workspace:*",
+    "@affine/component": "workspace:*",
+    "@affine/debug": "workspace:*",
+    "@affine/env": "workspace:*",
+    "@affine/graphql": "workspace:*",
+    "@affine/i18n": "workspace:*",
+    "@affine/jotai": "workspace:*",
+    "@affine/templates": "workspace:*",
+    "@affine/workspace": "workspace:*",
+    "@blocksuite/block-std": "0.0.0-20230809030546-32e6e21d-nightly",
+    "@blocksuite/blocks": "0.0.0-20230809030546-32e6e21d-nightly",
+    "@blocksuite/editor": "0.0.0-20230809030546-32e6e21d-nightly",
+    "@blocksuite/global": "0.0.0-20230809030546-32e6e21d-nightly",
+    "@blocksuite/icons": "^2.1.31",
+    "@blocksuite/lit": "0.0.0-20230809030546-32e6e21d-nightly",
+    "@blocksuite/store": "0.0.0-20230809030546-32e6e21d-nightly",
+    "@toeverything/hooks": "workspace:*",
+    "@toeverything/y-indexeddb": "workspace:*",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.20",
+    "@types/react-dom": "^18.2.7",
+    "@vitejs/plugin-react-swc": "^3.3.2",
+    "typescript": "^5.1.6",
+    "vite": "^4.4.9"
+  }
+}

apps/prototype/project.json

@@ -0,0 +1,16 @@
+{
+  "name": "prototype",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "projectType": "application",
+  "sourceRoot": "apps/prototype/src",
+  "targets": {
+    "build": {
+      "executor": "nx:run-script",
+      "dependsOn": ["^build"],
+      "options": {
+        "script": "build"
+      },
+      "outputs": ["{projectRoot}/dist"]
+    }
+  }
+}

apps/prototype/src/provider-status.tsx

@@ -0,0 +1,57 @@
+import type { LocalIndexedDBBackgroundProvider } from '@affine/env/workspace';
+import { createIndexedDBBackgroundProvider } from '@affine/workspace/providers';
+import { assertExists } from '@blocksuite/global/utils';
+import { useDataSourceStatus } from '@toeverything/hooks/use-data-source-status';
+import React, { useCallback, useRef } from 'react';
+import ReactDOM from 'react-dom/client';
+import { Awareness } from 'y-protocols/awareness';
+import { Doc } from 'yjs';
+
+const doc = new Doc();
+const map = doc.getMap();
+const awareness = new Awareness(doc);
+
+const indexeddbProvider = createIndexedDBBackgroundProvider('test', doc, {
+  awareness,
+}) as LocalIndexedDBBackgroundProvider;
+indexeddbProvider.connect();
+
+const App = () => {
+  const counterRef = useRef(0);
+  const disposeRef = useRef<number>(0);
+  const status = useDataSourceStatus(indexeddbProvider);
+  return (
+    <div>
+      <button
+        data-testid="start-button"
+        onClick={useCallback(() => {
+          disposeRef.current = setInterval(() => {
+            const counter = counterRef.current;
+            map.set('counter', counter + 1);
+            counterRef.current = counter + 1;
+          }, 0) as any;
+        }, [])}
+      >
+        start writing
+      </button>
+      <button
+        data-testid="stop-button"
+        onClick={useCallback(() => {
+          clearInterval(disposeRef.current);
+        }, [])}
+      >
+        stop writing
+      </button>
+      <div data-testid="status">{status.type}</div>
+    </div>
+  );
+};
+
+const root = document.getElementById('root');
+assertExists(root);
+
+ReactDOM.createRoot(root).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);

apps/prototype/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

apps/prototype/suite/provider-status.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Provider status test</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="../src/provider-status.tsx"></script>
+  </body>
+</html>

apps/prototype/tsconfig.json

@@ -0,0 +1,40 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "moduleResolution": "bundler",
+    "outDir": "./lib"
+  },
+  "include": ["./src"],
+  "references": [
+    {
+      "path": "../../packages/component"
+    },
+    {
+      "path": "../../packages/debug"
+    },
+    {
+      "path": "../../packages/env"
+    },
+    {
+      "path": "../../packages/graphql"
+    },
+    {
+      "path": "../../packages/hooks"
+    },
+    {
+      "path": "../../packages/i18n"
+    },
+    {
+      "path": "../../packages/jotai"
+    },
+    {
+      "path": "../../packages/y-indexeddb"
+    },
+    {
+      "path": "../../packages/workspace"
+    },
+    {
+      "path": "./tsconfig.node.json"
+    }
+  ]
+}

apps/prototype/tsconfig.node.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "outDir": "./lib",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}

apps/prototype/vite.config.ts

@@ -0,0 +1,22 @@
+import { resolve } from 'node:path';
+
+import react from '@vitejs/plugin-react-swc';
+import { defineConfig } from 'vite';
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  build: {
+    target: 'ES2022',
+    sourcemap: true,
+    rollupOptions: {
+      input: {
+        'suite/provider-status': resolve(
+          __dirname,
+          'suite',
+          'provider-status.html'
+        ),
+      },
+    },
+  },
+  plugins: [react()],
+});

package.json

@@ -14,7 +14,8 @@
     "tests/kit",
     "tests/affine-legacy/*",
     "tests/affine-local",
-    "tests/affine-plugin"
+    "tests/affine-plugin",
+    "tests/affine-prototype"
   ],
   "engines": {
     "node": ">=18.16.1 <19.0.0"

packages/env/src/blocksuite/index.ts

@@ -2,11 +2,7 @@ import type { Page } from '@blocksuite/store';
 
 export async function initPageWithPreloading(page: Page) {
   const workspace = page.workspace;
-  const {
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-expect-error
-    data,
-  } = await import('@affine/templates/preloading.json');
+  const { data } = await import('@affine/templates/preloading.json');
   await page.waitForLoaded();
   await workspace.importPageSnapshot(data['space:hello-world'], page.id);
 }

packages/env/src/workspace.ts

@@ -1,3 +1,4 @@
+import type { StatusAdapter } from '@affine/y-provider';
 import type { EditorContainer } from '@blocksuite/editor';
 import type { Page } from '@blocksuite/store';
 import type {
@@ -35,7 +36,9 @@ export interface BroadCastChannelProvider extends PassiveDocProvider {
 /**
  * Long polling provider with local indexeddb
  */
-export interface LocalIndexedDBBackgroundProvider extends PassiveDocProvider {
+export interface LocalIndexedDBBackgroundProvider
+  extends StatusAdapter,
+    PassiveDocProvider {
   flavour: 'local-indexeddb-background';
 }
 
@@ -43,7 +46,7 @@ export interface LocalIndexedDBDownloadProvider extends ActiveDocProvider {
   flavour: 'local-indexeddb';
 }
 
-export interface SQLiteProvider extends PassiveDocProvider {
+export interface SQLiteProvider extends PassiveDocProvider, StatusAdapter {
   flavour: 'sqlite';
 }
 

packages/env/tsconfig.json

@@ -4,7 +4,6 @@
   "compilerOptions": {
     "composite": true,
     "noEmit": false,
-    "moduleResolution": "Node16",
     "outDir": "lib"
   },
   "references": [

packages/hooks/package.json

@@ -6,10 +6,11 @@
   },
   "private": true,
   "dependencies": {
-    "@affine/env": "workspace:*",
-    "@toeverything/y-indexeddb": "workspace:*"
+    "foxact": "^0.2.17"
   },
   "devDependencies": {
+    "@affine/env": "workspace:*",
+    "@affine/y-provider": "workspace:*",
     "@blocksuite/block-std": "0.0.0-20230810005427-25adb757-nightly",
     "@blocksuite/blocks": "0.0.0-20230810005427-25adb757-nightly",
     "@blocksuite/editor": "0.0.0-20230810005427-25adb757-nightly",
@@ -18,6 +19,7 @@
     "@blocksuite/store": "0.0.0-20230810005427-25adb757-nightly"
   },
   "peerDependencies": {
+    "@affine/y-provider": "workspace:*",
     "@blocksuite/block-std": "*",
     "@blocksuite/blocks": "*",
     "@blocksuite/editor": "*",
@@ -25,5 +27,31 @@
     "@blocksuite/lit": "*",
     "@blocksuite/store": "*"
   },
+  "peerDependenciesMeta": {
+    "@affine/env": {
+      "optional": true
+    },
+    "@affine/y-provider": {
+      "optional": true
+    },
+    "@blocksuite/block-std": {
+      "optional": true
+    },
+    "@blocksuite/blocks": {
+      "optional": true
+    },
+    "@blocksuite/editor": {
+      "optional": true
+    },
+    "@blocksuite/global": {
+      "optional": true
+    },
+    "@blocksuite/lit": {
+      "optional": true
+    },
+    "@blocksuite/store": {
+      "optional": true
+    }
+  },
   "version": "0.8.0-canary.16"
 }

packages/hooks/src/use-data-source-status.ts

@@ -0,0 +1,15 @@
+import type { Status, StatusAdapter } from '@affine/y-provider';
+import { useCallback, useSyncExternalStore } from 'react';
+
+type UIStatus =
+  | Status
+  | {
+      type: 'unknown';
+    };
+
+export function useDataSourceStatus(datasource: StatusAdapter): UIStatus {
+  return useSyncExternalStore(
+    datasource.subscribeStatusChange,
+    useCallback(() => datasource.status, [datasource])
+  );
+}

packages/workspace/src/providers/index.ts

@@ -26,10 +26,15 @@ const createIndexedDBBackgroundProvider: DocProviderCreator = (
   blockSuiteWorkspace
 ): LocalIndexedDBBackgroundProvider => {
   const indexeddbProvider = create(blockSuiteWorkspace);
+
   let connected = false;
   return {
     flavour: 'local-indexeddb-background',
     passive: true,
+    get status() {
+      return indexeddbProvider.status;
+    },
+    subscribeStatusChange: indexeddbProvider.subscribeStatusChange,
     get connected() {
       return connected;
     },

packages/workspace/src/providers/sqlite-providers.ts

@@ -6,6 +6,7 @@ import {
   createLazyProvider,
   type DatasourceDocAdapter,
 } from '@affine/y-provider';
+import { assertExists } from '@blocksuite/global/utils';
 import type { DocProviderCreator } from '@blocksuite/store';
 import { Workspace as BlockSuiteWorkspace } from '@blocksuite/store';
 import type { Doc } from 'yjs';
@@ -51,6 +52,14 @@ export const createSQLiteProvider: DocProviderCreator = (
   return {
     flavour: 'sqlite',
     passive: true,
+    get status() {
+      assertExists(provider);
+      return provider.status;
+    },
+    subscribeStatusChange(onStatusChange) {
+      assertExists(provider);
+      return provider.subscribeStatusChange(onStatusChange);
+    },
     connect: () => {
       datasource = createDatasource(id);
       provider = createLazyProvider(rootDoc, datasource, { origin: 'sqlite' });

packages/y-indexeddb/src/provider.ts

@@ -3,6 +3,7 @@ import {
   type DatasourceDocAdapter,
   writeOperation,
 } from '@affine/y-provider';
+import { assertExists } from '@blocksuite/global/utils';
 import { openDB } from 'idb';
 import type { Doc } from 'yjs';
 import { diffUpdate, mergeUpdates } from 'yjs';
@@ -77,7 +78,6 @@ const createDatasource = ({
           const merged = mergeUpdates(rows.map(({ update }) => update));
           rows = [{ timestamp: Date.now(), update: merged }];
         }
-
         await writeOperation(
           store.put({
             id: guid,
@@ -112,6 +112,14 @@ export const createIndexedDBProvider = (
   let provider: ReturnType<typeof createLazyProvider> | null = null;
 
   const apis = {
+    get status() {
+      assertExists(provider);
+      return provider.status;
+    },
+    subscribeStatusChange(onStatusChange) {
+      assertExists(provider);
+      return provider.subscribeStatusChange(onStatusChange);
+    },
     connect: () => {
       if (apis.connected) {
         apis.disconnect();
@@ -132,7 +140,7 @@ export const createIndexedDBProvider = (
     get connected() {
       return provider?.connected || false;
     },
-  };
+  } satisfies IndexedDBProvider;
 
   return apis;
 };

packages/y-indexeddb/src/shared.ts

@@ -1,3 +1,4 @@
+import type { StatusAdapter } from '@affine/y-provider';
 import type { DBSchema, IDBPDatabase } from 'idb';
 
 export const dbVersion = 1;
@@ -8,7 +9,7 @@ export function upgradeDB(db: IDBPDatabase<BlockSuiteBinaryDB>) {
   db.createObjectStore('milestone', { keyPath: 'id' });
 }
 
-export interface IndexedDBProvider {
+export interface IndexedDBProvider extends StatusAdapter {
   connect: () => void;
   disconnect: () => void;
   cleanup: () => Promise<void>;

packages/y-provider/package.json

@@ -4,6 +4,10 @@
   "version": "0.8.0-canary.16",
   "description": "Yjs provider utilities for AFFiNE",
   "main": "./src/index.ts",
+  "module": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
   "devDependencies": {
     "@blocksuite/store": "0.0.0-20230810005427-25adb757-nightly"
   },

packages/y-provider/src/lazy-provider.ts

@@ -7,7 +7,8 @@ import {
   encodeStateVectorFromUpdate,
 } from 'yjs';
 
-import type { DatasourceDocAdapter } from './types';
+import type { DatasourceDocAdapter, StatusAdapter } from './types';
+import type { Status } from './types';
 
 function getDoc(doc: Doc, guid: string): Doc | undefined {
   if (doc.guid === guid) {
@@ -33,7 +34,7 @@ export const createLazyProvider = (
   rootDoc: Doc,
   datasource: DatasourceDocAdapter,
   options: LazyProviderOptions = {}
-): Omit<PassiveDocProvider, 'flavour'> => {
+): Omit<PassiveDocProvider, 'flavour'> & StatusAdapter => {
   let connected = false;
   const pendingMap = new Map<string, Uint8Array[]>(); // guid -> pending-updates
   const disposableMap = new Map<string, Set<() => void>>();
@@ -42,11 +43,59 @@ export const createLazyProvider = (
 
   const { origin = 'lazy-provider' } = options;
 
+  // todo: should we use a real state machine here like `xstate`?
+  let currentStatus: Status = {
+    type: 'idle',
+  };
+  let syncingStack = 0;
+  const callbackSet = new Set<() => void>();
+  const changeStatus = (newStatus: Status) => {
+    // simulate a stack, each syncing and synced should be paired
+    if (newStatus.type === 'idle') {
+      if (syncingStack !== 0) {
+        console.error('syncingStatus !== 0, this should not happen');
+      }
+      syncingStack = 0;
+    }
+    if (newStatus.type === 'syncing') {
+      syncingStack++;
+    }
+    if (newStatus.type === 'synced' || newStatus.type === 'error') {
+      syncingStack--;
+    }
+
+    if (syncingStack < 0) {
+      console.error('syncingStatus < 0, this should not happen');
+    }
+
+    if (syncingStack === 0) {
+      currentStatus = newStatus;
+    }
+    if (newStatus.type !== 'synced') {
+      currentStatus = newStatus;
+    }
+    callbackSet.forEach(cb => cb());
+  };
+
   async function syncDoc(doc: Doc) {
     const guid = doc.guid;
 
-    const remoteUpdate = await datasource.queryDocState(guid, {
-      stateVector: encodeStateVector(doc),
+    changeStatus({
+      type: 'syncing',
+    });
+    const remoteUpdate = await datasource
+      .queryDocState(guid, {
+        stateVector: encodeStateVector(doc),
+      })
+      .catch(error => {
+        changeStatus({
+          type: 'error',
+          error,
+        });
+        throw error;
+      });
+    changeStatus({
+      type: 'synced',
     });
 
     pendingMap.set(guid, []);
@@ -59,6 +108,9 @@ export const createLazyProvider = (
       ? encodeStateVectorFromUpdate(remoteUpdate)
       : undefined;
 
+    if (!connected) {
+      return;
+    }
     // perf: optimize me
     // it is possible the doc is only in memory but not yet in the datasource
     // we need to send the whole update to the datasource
@@ -76,7 +128,23 @@ export const createLazyProvider = (
       if (origin === updateOrigin) {
         return;
       }
-      datasource.sendDocUpdate(doc.guid, update).catch(console.error);
+      changeStatus({
+        type: 'syncing',
+      });
+      datasource
+        .sendDocUpdate(doc.guid, update)
+        .then(() => {
+          changeStatus({
+            type: 'synced',
+          });
+        })
+        .catch(error => {
+          changeStatus({
+            type: 'error',
+            error,
+          });
+          console.error(error);
+        });
     };
 
     const subdocsHandler = (event: { loaded: Set<Doc>; removed: Set<Doc> }) => {
@@ -103,6 +171,9 @@ export const createLazyProvider = (
    */
   function setupDatasourceListeners() {
     datasourceUnsub = datasource.onDocUpdate?.((guid, update) => {
+      changeStatus({
+        type: 'syncing',
+      });
       const doc = getDoc(rootDoc, guid);
       if (doc) {
         applyUpdate(doc, update, origin);
@@ -120,6 +191,9 @@ export const createLazyProvider = (
         console.warn('idb: doc not found', guid);
         pendingMap.set(guid, (pendingMap.get(guid) ?? []).concat(update));
       }
+      changeStatus({
+        type: 'synced',
+      });
     });
   }
 
@@ -165,20 +239,44 @@ export const createLazyProvider = (
   function connect() {
     connected = true;
 
+    changeStatus({
+      type: 'syncing',
+    });
     // root doc should be already loaded,
     // but we want to populate the cache for later update events
-    connectDoc(rootDoc).catch(console.error);
+    connectDoc(rootDoc).catch(error => {
+      changeStatus({
+        type: 'error',
+        error,
+      });
+      console.error(error);
+    });
+    changeStatus({
+      type: 'synced',
+    });
     setupDatasourceListeners();
   }
 
   async function disconnect() {
     connected = false;
+    changeStatus({
+      type: 'idle',
+    });
     disposeAll();
     datasourceUnsub?.();
     datasourceUnsub = undefined;
   }
 
   return {
+    get status() {
+      return currentStatus;
+    },
+    subscribeStatusChange(cb: () => void) {
+      callbackSet.add(cb);
+      return () => {
+        callbackSet.delete(cb);
+      };
+    },
     get connected() {
       return connected;
     },

packages/y-provider/src/types.ts

@@ -1,4 +1,24 @@
-export interface DatasourceDocAdapter {
+export type Status =
+  | {
+      type: 'idle';
+    }
+  | {
+      type: 'syncing';
+    }
+  | {
+      type: 'synced';
+    }
+  | {
+      type: 'error';
+      error: Error;
+    };
+
+export interface StatusAdapter {
+  readonly status: Status;
+  subscribeStatusChange(onStatusChange: () => void): () => void;
+}
+
+export interface DatasourceDocAdapter extends Partial<StatusAdapter> {
   // request diff update from other clients
   queryDocState: (
     guid: string,

tests/affine-local/e2e/router.spec.ts

@@ -1,5 +1,5 @@
 import { test } from '@affine-test/kit/playwright';
-import { openHomePage, webUrl } from '@affine-test/kit/utils/load-page';
+import { coreUrl, openHomePage } from '@affine-test/kit/utils/load-page';
 import { waitEditorLoad } from '@affine-test/kit/utils/page-logic';
 import { expect } from '@playwright/test';
 
@@ -17,7 +17,7 @@ test('goto not found workspace', async ({ page }) => {
   await waitEditorLoad(page);
   // if doesn't wait for timeout, data won't be saved into indexedDB
   await page.waitForTimeout(1000);
-  await page.goto(new URL('/workspace/invalid/all', webUrl).toString());
+  await page.goto(new URL('/workspace/invalid/all', coreUrl).toString());
   await page.waitForTimeout(1000);
-  expect(page.url()).toBe(new URL('/404', webUrl).toString());
+  expect(page.url()).toBe(new URL('/404', coreUrl).toString());
 });

tests/affine-prototype/e2e/basic.spec.ts

@@ -0,0 +1,12 @@
+import { test } from '@affine-test/kit/playwright';
+import { openPrototypeProviderStatusPage } from '@affine-test/kit/utils/load-page';
+import { expect } from '@playwright/test';
+
+test('syncing and synced status should works', async ({ page }) => {
+  await openPrototypeProviderStatusPage(page);
+  await expect(page.getByTestId('status')).toHaveText('synced');
+  await page.getByTestId('start-button').click();
+  await expect(page.getByTestId('status')).toHaveText('syncing');
+  await page.getByTestId('stop-button').click();
+  await expect(page.getByTestId('status')).toHaveText('synced');
+});

tests/affine-prototype/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@affine-test/affine-prototype",
+  "private": true,
+  "scripts": {
+    "e2e": "yarn playwright test"
+  },
+  "devDependencies": {
+    "@affine-test/fixtures": "workspace:*",
+    "@affine-test/kit": "workspace:*",
+    "@playwright/test": "^1.36.2"
+  },
+  "version": "0.8.0-canary.14"
+}

tests/affine-prototype/playwright.config.ts

@@ -0,0 +1,63 @@
+import type {
+  PlaywrightTestConfig,
+  PlaywrightWorkerOptions,
+} from '@playwright/test';
+// import { devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: './e2e',
+  fullyParallel: true,
+  timeout: process.env.CI ? 50_000 : 30_000,
+  use: {
+    baseURL: 'http://localhost:8080/',
+    browserName:
+      (process.env.BROWSER as PlaywrightWorkerOptions['browserName']) ??
+      'chromium',
+    permissions: ['clipboard-read', 'clipboard-write'],
+    viewport: { width: 1440, height: 800 },
+    actionTimeout: 5 * 1000,
+    locale: 'en-US',
+    // Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer
+    // You can open traces locally(`npx playwright show-trace trace.zip`)
+    // or in your browser on [Playwright Trace Viewer](https://trace.playwright.dev/).
+    trace: 'on-first-retry',
+    // Record video only when retrying a test for the first time.
+    video: 'on-first-retry',
+  },
+  forbidOnly: !!process.env.CI,
+  workers: 4,
+  retries: 1,
+  // 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot'
+  // default 'list' when running locally
+  // See https://playwright.dev/docs/test-reporters#github-actions-annotations
+  reporter: process.env.CI ? 'github' : 'list',
+
+  webServer: [
+    // Intentionally not building the web, reminds you to run it by yourself.
+    {
+      command: 'yarn workspace @affine/prototype preview',
+      port: 3003,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+      env: {
+        COVERAGE: process.env.COVERAGE || 'false',
+      },
+    },
+  ],
+};
+
+if (process.env.CI) {
+  config.retries = 3;
+  config.workers = '50%';
+}
+
+export default config;

tests/affine-prototype/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "outDir": "lib"
+  },
+  "include": ["e2e"],
+  "references": [
+    {
+      "path": "../kit"
+    },
+    {
+      "path": "../fixtures"
+    }
+  ]
+}

tests/kit/utils/load-page.ts

@@ -1,11 +1,16 @@
 import type { Page } from '@playwright/test';
 
-export const webUrl = 'http://localhost:8080';
+export const coreUrl = 'http://localhost:8080';
+export const prototypeUrl = 'http://localhost:3003';
 
 export async function openHomePage(page: Page) {
-  await page.goto(webUrl);
+  await page.goto(coreUrl);
 }
 
 export async function openPluginPage(page: Page) {
-  await page.goto(`${webUrl}/_plugin/index.html`);
+  await page.goto(`${coreUrl}/_plugin/index.html`);
+}
+
+export async function openPrototypeProviderStatusPage(page: Page) {
+  await page.goto(`${prototypeUrl}/suite/provider-status.html`);
 }

tsconfig.json

@@ -169,8 +169,14 @@
     {
       "path": "./tests/affine-plugin"
     },
+    {
+      "path": "./tests/affine-prototype"
+    },
     {
       "path": "./tests/affine-legacy/0.7.0-canary.18"
+    },
+    {
+      "path": "./tests/affine-legacy/0.8.0-canary.7"
     }
   ],
   "files": [],

yarn.lock

@@ -73,6 +73,16 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@affine-test/affine-prototype@workspace:tests/affine-prototype":
+  version: 0.0.0-use.local
+  resolution: "@affine-test/affine-prototype@workspace:tests/affine-prototype"
+  dependencies:
+    "@affine-test/fixtures": "workspace:*"
+    "@affine-test/kit": "workspace:*"
+    "@playwright/test": ^1.36.2
+  languageName: unknown
+  linkType: soft
+
 "@affine-test/fixtures@workspace:*, @affine-test/fixtures@workspace:tests/fixtures":
   version: 0.0.0-use.local
   resolution: "@affine-test/fixtures@workspace:tests/fixtures"
@@ -558,6 +568,38 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@affine/prototype@workspace:apps/prototype":
+  version: 0.0.0-use.local
+  resolution: "@affine/prototype@workspace:apps/prototype"
+  dependencies:
+    "@affine-test/fixtures": "workspace:*"
+    "@affine/component": "workspace:*"
+    "@affine/debug": "workspace:*"
+    "@affine/env": "workspace:*"
+    "@affine/graphql": "workspace:*"
+    "@affine/i18n": "workspace:*"
+    "@affine/jotai": "workspace:*"
+    "@affine/templates": "workspace:*"
+    "@affine/workspace": "workspace:*"
+    "@blocksuite/block-std": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/blocks": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/editor": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/icons": ^2.1.31
+    "@blocksuite/lit": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/store": 0.0.0-20230809030546-32e6e21d-nightly
+    "@toeverything/hooks": "workspace:*"
+    "@toeverything/y-indexeddb": "workspace:*"
+    "@types/react": ^18.2.20
+    "@types/react-dom": ^18.2.7
+    "@vitejs/plugin-react-swc": ^3.3.2
+    react: ^18.2.0
+    react-dom: ^18.2.0
+    typescript: ^5.1.6
+    vite: ^4.4.9
+  languageName: unknown
+  linkType: soft
+
 "@affine/sdk@workspace:*, @affine/sdk@workspace:packages/sdk":
   version: 0.0.0-use.local
   resolution: "@affine/sdk@workspace:packages/sdk"
@@ -3349,6 +3391,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/block-std@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/block-std@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    w3c-keyname: ^2.2.8
+  peerDependencies:
+    "@blocksuite/store": 0.0.0-20230809030546-32e6e21d-nightly
+  checksum: 728387fe20e4b3534d6723172479b9116621a1398c9f28b4ef4e008d028717ba960e3a6d59508a45f07cfff43c71175466c10f71d48a0062938a16de2d3462c6
+  languageName: node
+  linkType: hard
+
 "@blocksuite/block-std@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/block-std@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3361,6 +3415,34 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/blocks@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/blocks@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/phasor": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/virgo": 0.0.0-20230809030546-32e6e21d-nightly
+    "@floating-ui/dom": ^1.5.1
+    buffer: ^6.0.3
+    date-fns: ^2.30.0
+    file-type: ^16.5.4
+    html2canvas: ^1.4.1
+    jszip: ^3.10.1
+    lit: ^2.7.6
+    marked: ^4.3.0
+    pdf-lib: ^1.17.1
+    shiki: ^0.14.3
+    turndown: ^7.1.2
+    zod: ^3.21.4
+  peerDependencies:
+    "@blocksuite/block-std": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/lit": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/store": 0.0.0-20230809030546-32e6e21d-nightly
+    yjs: ^13
+  checksum: 8f4f4942541b6c0efd5b6527e906dfba36fb7a1b9ef32042bcfa3cbadcf0da3f55bd8f22edade27b630c62290483a1d464b300edf7406ade909aaf41e3c88b5b
+  languageName: node
+  linkType: hard
+
 "@blocksuite/blocks@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/blocks@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3390,6 +3472,23 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/editor@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/editor@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    lit: ^2.7.6
+    marked: ^4.3.0
+    turndown: ^7.1.2
+  peerDependencies:
+    "@blocksuite/blocks": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/lit": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/store": 0.0.0-20230809030546-32e6e21d-nightly
+    "@toeverything/theme": ^0.7.9
+  checksum: e01ae29d424f1273a5c0ab9f969baad4b58ad8510255df4ea03addab06e12203d51edef87a36fcc20a05233d3863418540b6846229b8dd4fed1d295ce5abef78
+  languageName: node
+  linkType: hard
+
 "@blocksuite/editor@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/editor@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3405,6 +3504,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/global@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/global@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    ansi-colors: ^4.1.3
+    zod: ^3.21.4
+  peerDependencies:
+    lit: ^2.7
+  peerDependenciesMeta:
+    lit:
+      optional: true
+  checksum: 03eb2fe544f4122f0b4369c637de0238cf8e6731ac7d47e9890a401da5ee20a21bb93c1fdd306f4379bdd5c7956b861fcad8b279145cdf370f009314a6feed72
+  languageName: node
+  linkType: hard
+
 "@blocksuite/global@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/global@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3440,6 +3554,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/lit@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/lit@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    lit: ^2.7.6
+  peerDependencies:
+    "@blocksuite/block-std": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/store": 0.0.0-20230809030546-32e6e21d-nightly
+  checksum: c8ba7e600839fe463368d804f26fd6c369f22693681cd9c7f94500988c2a83054ef156e56e71621f1b6a67e2667703ff657d4f10e98de52f3b616ec285ee8a0c
+  languageName: node
+  linkType: hard
+
 "@blocksuite/lit@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/lit@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3453,6 +3580,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/phasor@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/phasor@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    fractional-indexing: ^3.2.0
+  peerDependencies:
+    nanoid: ^4
+    yjs: ^13
+  checksum: 46ee3d98ed054df635db30eccb716495c313ab341e4d471ae23912c855fa7c946c4b830294e93d6a7435b2c65fb1e7bdba53cf61c24cc85b13d14f50cabb9f1d
+  languageName: node
+  linkType: hard
+
 "@blocksuite/phasor@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/phasor@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3466,6 +3606,30 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/store@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/store@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    "@blocksuite/virgo": 0.0.0-20230809030546-32e6e21d-nightly
+    "@types/flexsearch": ^0.7.3
+    buffer: ^6.0.3
+    flexsearch: 0.7.21
+    idb-keyval: ^6.2.1
+    ky: ^0.33.3
+    lib0: ^0.2.78
+    merge: ^2.1.1
+    minimatch: ^9.0.3
+    nanoid: ^4.0.2
+    y-protocols: ^1.0.5
+    zod: ^3.21.4
+  peerDependencies:
+    async-call-rpc: ^6
+    yjs: ^13
+  checksum: 262b0858917f05eafba6c440bf7b5310e2509c1df44c67c6cc32b417af906970b1073a2864fa3c494dd993e7b6da7b16a975dbb31a240767b8e0963dd6334b3d
+  languageName: node
+  linkType: hard
+
 "@blocksuite/store@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/store@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -3490,6 +3654,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@blocksuite/virgo@npm:0.0.0-20230809030546-32e6e21d-nightly":
+  version: 0.0.0-20230809030546-32e6e21d-nightly
+  resolution: "@blocksuite/virgo@npm:0.0.0-20230809030546-32e6e21d-nightly"
+  dependencies:
+    "@blocksuite/global": 0.0.0-20230809030546-32e6e21d-nightly
+    zod: ^3.21.4
+  peerDependencies:
+    lit: ^2.7
+    yjs: ^13
+  checksum: b870ef551a856e44eca743962c8b25d97e80aece9a0172c983b34ab7bb8cc4a2d3f8fa93d1550b0d2e500cf409a72aa8e72a415ccbb791598c4c8354cec1389e
+  languageName: node
+  linkType: hard
+
 "@blocksuite/virgo@npm:0.0.0-20230810005427-25adb757-nightly":
   version: 0.0.0-20230810005427-25adb757-nightly
   resolution: "@blocksuite/virgo@npm:0.0.0-20230810005427-25adb757-nightly"
@@ -11307,20 +11484,39 @@ __metadata:
   resolution: "@toeverything/hooks@workspace:packages/hooks"
   dependencies:
     "@affine/env": "workspace:*"
+    "@affine/y-provider": "workspace:*"
     "@blocksuite/block-std": 0.0.0-20230810005427-25adb757-nightly
     "@blocksuite/blocks": 0.0.0-20230810005427-25adb757-nightly
     "@blocksuite/editor": 0.0.0-20230810005427-25adb757-nightly
     "@blocksuite/global": 0.0.0-20230810005427-25adb757-nightly
     "@blocksuite/lit": 0.0.0-20230810005427-25adb757-nightly
     "@blocksuite/store": 0.0.0-20230810005427-25adb757-nightly
-    "@toeverything/y-indexeddb": "workspace:*"
+    foxact: ^0.2.17
   peerDependencies:
+    "@affine/y-provider": "workspace:*"
     "@blocksuite/block-std": "*"
     "@blocksuite/blocks": "*"
     "@blocksuite/editor": "*"
     "@blocksuite/global": "*"
     "@blocksuite/lit": "*"
     "@blocksuite/store": "*"
+  peerDependenciesMeta:
+    "@affine/env":
+      optional: true
+    "@affine/y-provider":
+      optional: true
+    "@blocksuite/block-std":
+      optional: true
+    "@blocksuite/blocks":
+      optional: true
+    "@blocksuite/editor":
+      optional: true
+    "@blocksuite/global":
+      optional: true
+    "@blocksuite/lit":
+      optional: true
+    "@blocksuite/store":
+      optional: true
   languageName: unknown
   linkType: soft
 
@@ -12141,6 +12337,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/react@npm:^18.2.20":
+  version: 18.2.20
+  resolution: "@types/react@npm:18.2.20"
+  dependencies:
+    "@types/prop-types": "*"
+    "@types/scheduler": "*"
+    csstype: ^3.0.2
+  checksum: 30f699c60e5e4bfef273ce64d320651cdd60f5c6a08361c6c7eca8cebcccda1ac953d2ee57c9f321b5ae87f8a62c72b6d35ca42df0e261d337849952daab2141
+  languageName: node
+  linkType: hard
+
 "@types/responselike@npm:^1.0.0":
   version: 1.0.0
   resolution: "@types/responselike@npm:1.0.0"
@@ -27533,7 +27740,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"react-dom@npm:18.2.0":
+"react-dom@npm:18.2.0, react-dom@npm:^18.2.0":
   version: 18.2.0
   resolution: "react-dom@npm:18.2.0"
   dependencies: