From f3db427f1d420a2d3631c89eec4b2433f32dda63 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Fri, 6 Oct 2023 14:20:34 +0700
Subject: [PATCH] Client model persistence (#3796)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
 packages/core/src/client.ts           | 46 +++++++++++++++++++++------
 plugins/client-resources/src/index.ts | 19 ++++++++++-
 server/core/src/indexer/indexer.ts    | 28 +++++++++++++++-
 3 files changed, 82 insertions(+), 11 deletions(-)

diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts
index a8601caa03..566b2b4223 100644
--- a/packages/core/src/client.ts
+++ b/packages/core/src/client.ts
@@ -158,13 +158,22 @@ class ClientImpl implements AccountClient, BackupClient {
   }
 }
 
+/**
+ * @public
+ */
+export interface TxPersistenceStore {
+  load: () => Promise<Tx[]>
+  store: (tx: Tx[]) => Promise<void>
+}
+
 /**
  * @public
  */
 export async function createClient (
   connect: (txHandler: TxHandler) => Promise<ClientConnection>,
   // If set will build model with only allowed plugins.
-  allowedPlugins?: Plugin[]
+  allowedPlugins?: Plugin[],
+  txPersistence?: TxPersistenceStore
 ): Promise<AccountClient> {
   let client: ClientImpl | null = null
 
@@ -193,7 +202,7 @@ export async function createClient (
 
   const conn = await connect(txHandler)
 
-  lastTxTime = await loadModel(conn, lastTxTime, allowedPlugins, configs, hierarchy, model)
+  lastTxTime = await loadModel(conn, lastTxTime, allowedPlugins, configs, hierarchy, model, false, txPersistence)
 
   txBuffer = txBuffer.filter((tx) => tx.space !== core.space.Model || tx.modifiedOn > lastTxTime)
 
@@ -255,11 +264,18 @@ async function loadModel (
   configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
   hierarchy: Hierarchy,
   model: ModelDb,
-  reload = false
+  reload = false,
+  persistence?: TxPersistenceStore
 ): Promise<Timestamp> {
   const t = Date.now()
 
-  let atxes = []
+  let ltxes: Tx[] = []
+  if (lastTxTime === 0 && persistence !== undefined) {
+    ltxes = await persistence.load()
+    lastTxTime = getLastTxTime(ltxes)
+  }
+
+  let atxes: Tx[] = []
   try {
     atxes = await conn.loadModel(lastTxTime)
   } catch (err: any) {
@@ -274,6 +290,12 @@ async function loadModel (
     return -1
   }
 
+  if (atxes.length < modelTransactionThreshold) {
+    atxes = ltxes.concat(atxes)
+  }
+
+  await persistence?.store(atxes)
+
   let systemTx: Tx[] = []
   const userTx: Tx[] = []
   console.log('find' + (lastTxTime >= 0 ? 'full model' : 'model diff'), atxes.length, Date.now() - t)
@@ -302,11 +324,7 @@ async function loadModel (
 
   const txes = systemTx.concat(userTx)
 
-  for (const tx of txes) {
-    if (tx.modifiedOn > lastTxTime) {
-      lastTxTime = tx.modifiedOn
-    }
-  }
+  lastTxTime = getLastTxTime(txes)
 
   for (const tx of txes) {
     try {
@@ -325,6 +343,16 @@ async function loadModel (
   return lastTxTime
 }
 
+function getLastTxTime (txes: Tx[]): number {
+  let lastTxTime = 0
+  for (const tx of txes) {
+    if (tx.modifiedOn > lastTxTime) {
+      lastTxTime = tx.modifiedOn
+    }
+  }
+  return lastTxTime
+}
+
 function fillConfiguration (systemTx: Tx[], configs: Map<Ref<PluginConfiguration>, PluginConfiguration>): void {
   for (const t of systemTx) {
     if (t._class === core.class.TxCreateDoc) {
diff --git a/plugins/client-resources/src/index.ts b/plugins/client-resources/src/index.ts
index 496b712ec6..424c7393fc 100644
--- a/plugins/client-resources/src/index.ts
+++ b/plugins/client-resources/src/index.ts
@@ -60,7 +60,24 @@ export default async () => {
 
             return connect(url.href, upgradeHandler, onUpgrade, onUnauthorized, onConnect)
           },
-          filterModel ? [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])] : undefined
+          filterModel ? [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])] : undefined,
+          {
+            load: async () => {
+              if (typeof localStorage !== 'undefined') {
+                const dta = localStorage.getItem('stored_model_' + token) ?? null
+                if (dta === null) {
+                  return []
+                }
+                return JSON.parse(dta)
+              }
+              return []
+            },
+            store: async (txes) => {
+              if (typeof localStorage !== 'undefined') {
+                localStorage.setItem('stored_model_' + token, JSON.stringify(txes))
+              }
+            }
+          }
         )
         // Check if we had dev hook for client.
         client = hookClient(client)
diff --git a/server/core/src/indexer/indexer.ts b/server/core/src/indexer/indexer.ts
index d70f241558..d74c0cc47e 100644
--- a/server/core/src/indexer/indexer.ts
+++ b/server/core/src/indexer/indexer.ts
@@ -30,6 +30,7 @@ import core, {
   WorkspaceId,
   _getOperator,
   setObjectValue,
+  toFindResult,
   versionToString
 } from '@hcengineering/core'
 import { DbAdapter } from '../adapter'
@@ -371,7 +372,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
             .filter((it) => it[1] > 3)
             .map((it) => it[0])
 
-          const result = await this.metrics.with(
+          let result = await this.metrics.with(
             'get-to-index',
             {},
             async () =>
@@ -390,6 +391,31 @@ export class FullTextIndexPipeline implements FullTextPipeline {
                 }
               )
           )
+          const toRemove: DocIndexState[] = []
+          // Check and remove missing class documents.
+          result = toFindResult(
+            result.filter((doc) => {
+              const _class = this.model.findObject(doc.objectClass)
+              if (_class === undefined) {
+                // no _class present, remove doc
+                toRemove.push(doc)
+                return false
+              }
+              return true
+            }),
+            result.total
+          )
+
+          if (toRemove.length > 0) {
+            try {
+              await this.storage.clean(
+                DOMAIN_DOC_INDEX_STATE,
+                toRemove.map((it) => it._id)
+              )
+            } catch (err: any) {
+              // QuotaExceededError, ignore
+            }
+          }
 
           if (result.length > 0) {
             console.log(