fix: sqlite provider import sub doc db file (#2991)

Co-authored-by: Alex Yang <himself65@outlook.com>

.eslintignore

@@ -8,3 +8,4 @@ _next
 lib
 .eslintrc.js
 packages/i18n/src/i18n-generated.ts
+e2e-dist-*

.github/workflows/build.yml

@@ -352,18 +352,18 @@ jobs:
         env:
           NATIVE_TEST: 'true'
 
+      - name: Download static resource artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: next-js-static
+          path: apps/electron/resources/web-static
+
       - name: Build Plugins
         run: yarn run build:plugins
 
       - name: Build Desktop Layers
         run: yarn workspace @affine/electron build
 
-      - name: Download static resource artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: next-js-static
-          path: ./apps/electron/resources/web-static
-
       - name: Run desktop tests
         if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }}
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine/electron test

apps/electron/e2e/fixture.ts

@@ -16,6 +16,8 @@ function generateUUID() {
   return crypto.randomUUID();
 }
 
+type RoutePath = 'setting';
+
 export const test = base.extend<{
   page: Page;
   electronApp: ElectronApplication;
@@ -28,6 +30,9 @@ export const test = base.extend<{
     // get current workspace
     current: () => Promise<any>; // todo: type
   };
+  router: {
+    goto: (path: RoutePath) => Promise<void>;
+  };
 }>({
   page: async ({ electronApp }, use) => {
     const page = await electronApp.firstWindow();
@@ -41,10 +46,6 @@ export const test = base.extend<{
         });
       });
     }
-    const logFilePath = await page.evaluate(async () => {
-      // @ts-expect-error
-      return window.apis?.debug.logFilePath();
-    });
     // wat for blocksuite to be loaded
     await page.waitForSelector('v-line');
     if (enableCoverage) {
@@ -71,10 +72,6 @@ export const test = base.extend<{
       );
     }
     await page.close();
-    if (logFilePath) {
-      const logs = await fs.readFile(logFilePath, 'utf-8');
-      console.log(logs);
-    }
   },
   electronApp: async ({}, use) => {
     // a random id to avoid conflicts between tests

apps/electron/e2e/workspace.spec.ts

@@ -5,7 +5,7 @@ import fs from 'fs-extra';
 
 import { test } from './fixture';
 
-test.skip('check workspace has a DB file', async ({ appInfo, workspace }) => {
+test('check workspace has a DB file', async ({ appInfo, workspace }) => {
   const w = await workspace.current();
   const dbPath = path.join(
     appInfo.sessionData,

apps/web/preset.config.mjs

@@ -30,7 +30,7 @@ const buildPreset = {
     enablePreloading: true,
     enableNewSettingModal: true,
     enableNewSettingUnstableApi: false,
-    enableSQLiteProvider: false,
+    enableSQLiteProvider: true,
     enableNotificationCenter: false,
     enableCloud: false,
   },
@@ -47,7 +47,7 @@ const buildPreset = {
     enablePreloading: true,
     enableNewSettingModal: true,
     enableNewSettingUnstableApi: false,
-    enableSQLiteProvider: false,
+    enableSQLiteProvider: true,
     enableNotificationCenter: true,
     enableCloud: false,
   },

packages/native/src/sqlite/mod.rs

@@ -232,29 +232,40 @@ impl SqliteConnection {
 
   #[napi]
   pub async fn validate(path: String) -> bool {
-    if let Ok(pool) = SqlitePoolOptions::new()
+    let pool = match SqlitePoolOptions::new()
       .max_connections(1)
       .connect(&path)
       .await
     {
-      if let Ok(res) = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
-        .fetch_all(&pool)
-        .await
-      {
-        let names = res.iter().map(|row| row.get(0));
-        names.fold(0, |acc, cur: String| {
-          if cur == "updates" || cur == "blobs" {
-            acc + 1
-          } else {
-            acc
-          }
-        }) == 2
-      } else {
-        false
+      Ok(pool) => pool,
+      Err(_) => return false,
+    };
+
+    let tables_res = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
+      .fetch_all(&pool)
+      .await;
+
+    let tables_exist = match tables_res {
+      Ok(res) => {
+        let names: Vec<String> = res.iter().map(|row| row.get(0)).collect();
+        names.contains(&"updates".to_string()) && names.contains(&"blobs".to_string())
       }
-    } else {
-      false
-    }
+      Err(_) => return false,
+    };
+
+    let columns_res = sqlx::query("PRAGMA table_info(updates)")
+      .fetch_all(&pool)
+      .await;
+
+    let columns_exist = match columns_res {
+      Ok(res) => {
+        let names: Vec<String> = res.iter().map(|row| row.get(1)).collect();
+        names.contains(&"data".to_string()) && names.contains(&"doc_id".to_string())
+      }
+      Err(_) => return false,
+    };
+
+    tables_exist && columns_exist
   }
 
   // todo: have a better way to handle migration

packages/workspace/src/providers/sqlite-providers.ts

@@ -173,17 +173,18 @@ export const createSQLiteDBDownloadProvider: DocProviderCreator = (
       Y.applyUpdate(doc, updates, sqliteOrigin);
     }
 
-    const diff = Y.encodeStateAsUpdate(doc, updates);
+    const mergedUpdates = Y.encodeStateAsUpdate(doc);
 
     // also apply updates to sqlite
-    await apis.db.applyDocUpdate(id, diff, subdocId);
+    await apis.db.applyDocUpdate(id, mergedUpdates, subdocId);
 
     return true;
   }
 
   async function syncAllUpdates(doc: Doc) {
     if (await syncUpdates(doc)) {
-      const subdocs = Array.from(doc.subdocs).filter(d => d.shouldLoad);
+      // load all subdocs
+      const subdocs = Array.from(doc.subdocs);
       await Promise.all(subdocs.map(syncAllUpdates));
     }
   }
@@ -198,7 +199,7 @@ export const createSQLiteDBDownloadProvider: DocProviderCreator = (
       disconnected = true;
     },
     sync: async () => {
-      logger.info('connect indexeddb provider', id);
+      logger.info('connect sqlite download provider', id);
       try {
         await syncAllUpdates(rootDoc);
         _resolve();