UBER-356 Facility to apply initial workspace (#3344)

* UBER-356 Facility to apply initial workspace

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

* Don't allow join to source workspace

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

---------

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

dev/docker-compose.yaml

@@ -60,6 +60,7 @@ services:
       - MINIO_SECRET_KEY=minioadmin
       - FRONT_URL=http://front:8080
       - SES_URL=http://localhost:8091
+      - INIT_WORKSPACE=demo-tracker
       - MODEL_ENABLED=*
     restart: unless-stopped
   collaborator:

dev/tool/src/index.ts

@@ -91,6 +91,11 @@ export function devTool (
     return elasticUrl
   }
 
+  const initWS = process.env.INIT_WORKSPACE
+  if (initWS !== undefined) {
+    setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
+  }
+
   setMetadata(toolPlugin.metadata.Endpoint, transactorUrl)
   setMetadata(toolPlugin.metadata.Transactor, transactorUrl)
   setMetadata(serverToken.metadata.Secret, serverSecret)

pods/account/src/index.ts

@@ -57,6 +57,11 @@ export function serveAccount (methods: Record<string, AccountMethod>, productId
   setMetadata(account.metadata.FrontURL, frontURL)
 
   setMetadata(serverToken.metadata.Secret, serverSecret)
+
+  const initWS = process.env.INIT_WORKSPACE
+  if (initWS !== undefined) {
+    setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
+  }
   setMetadata(toolPlugin.metadata.Endpoint, endpointUri)
   setMetadata(toolPlugin.metadata.Transactor, transactorUri)
 

server/account/package.json

@@ -36,6 +36,7 @@
     "@hcengineering/client": "^0.6.11",
     "ws": "^8.10.0",
     "@hcengineering/model": "^0.6.4",
+    "@hcengineering/server-backup": "^0.6.0",
     "@hcengineering/server-tool": "^0.6.0",
     "@hcengineering/server-token": "^0.6.4",
     "@hcengineering/model-all": "^0.6.0",

server/account/src/index.ts

@@ -44,6 +44,7 @@ import platform, {
   Status,
   StatusCode
 } from '@hcengineering/platform'
+import { cloneWorkspace } from '@hcengineering/server-backup'
 import { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { connect, initModel, upgradeModel } from '@hcengineering/server-tool'
 import { pbkdf2Sync, randomBytes } from 'crypto'
@@ -568,6 +569,12 @@ export async function createWorkspace (
     })
     .then((e) => e.insertedId.toHexString())
   await initModel(getTransactor(), getWorkspaceId(workspace, productId), txes, migrationOperation)
+  const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
+  if (initWS !== undefined) {
+    if ((await getWorkspace(db, productId, initWS)) !== null) {
+      await cloneWorkspace(getTransactor(), getWorkspaceId(initWS, productId), getWorkspaceId(workspace, productId))
+    }
+  }
   return result
 }
 
@@ -720,6 +727,10 @@ export async function setRole (email: string, workspace: string, productId: stri
  * @public
  */
 export async function assignWorkspace (db: Db, productId: string, email: string, workspace: string): Promise<void> {
+  const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
+  if (initWS !== undefined && initWS === workspace) {
+    throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {}))
+  }
   const { workspaceId, accountId } = await getWorkspaceAndAccount(db, productId, email, workspace)
   const account = await db.collection<Account>(ACCOUNT_COLLECTION).findOne({ _id: accountId })
 

server/backup/src/index.ts

@@ -15,6 +15,7 @@
 //
 
 import core, {
+  AttachedDoc,
   BackupClient,
   BlobData,
   Client as CoreClient,
@@ -23,6 +24,7 @@ import core, {
   DOMAIN_MODEL,
   DOMAIN_TRANSIENT,
   Ref,
+  TxCollectionCUD,
   WorkspaceId
 } from '@hcengineering/core'
 import { connect } from '@hcengineering/server-tool'
@@ -191,6 +193,131 @@ async function writeChanges (storage: BackupStorage, snapshot: string, changes:
   )
 }
 
+/**
+ * @public
+ */
+export async function cloneWorkspace (
+  transactorUrl: string,
+  sourceWorkspaceId: WorkspaceId,
+  targetWorkspaceId: WorkspaceId,
+  clearTime: boolean = true
+): Promise<void> {
+  const sourceConnection = (await connect(transactorUrl, sourceWorkspaceId, undefined, {
+    mode: 'backup'
+  })) as unknown as CoreClient & BackupClient
+  const targetConnection = (await connect(transactorUrl, targetWorkspaceId, undefined, {
+    mode: 'backup'
+  })) as unknown as CoreClient & BackupClient
+  try {
+    const domains = sourceConnection
+      .getHierarchy()
+      .domains()
+      .filter((it) => it !== DOMAIN_TRANSIENT && it !== DOMAIN_MODEL)
+
+    for (const c of domains) {
+      console.log('clone domain...', c)
+
+      const changes: Snapshot = {
+        added: new Map(),
+        updated: new Map(),
+        removed: []
+      }
+
+      let idx: number | undefined
+
+      // update digest tar
+      const needRetrieveChunks: Ref<Doc>[][] = []
+
+      let processed = 0
+      let st = Date.now()
+      // Load all digest from collection.
+      while (true) {
+        try {
+          const it = await sourceConnection.loadChunk(c, idx)
+          idx = it.idx
+
+          const needRetrieve: Ref<Doc>[] = []
+
+          for (const [k, v] of Object.entries(it.docs)) {
+            processed++
+            if (processed % 10000 === 0) {
+              console.log('processed', processed, Date.now() - st)
+              st = Date.now()
+            }
+
+            changes.added.set(k as Ref<Doc>, v)
+            needRetrieve.push(k as Ref<Doc>)
+          }
+          if (needRetrieve.length > 0) {
+            needRetrieveChunks.push(needRetrieve)
+          }
+          if (it.finished) {
+            await sourceConnection.closeChunk(idx)
+            break
+          }
+        } catch (err: any) {
+          console.error(err)
+          if (idx !== undefined) {
+            await sourceConnection.closeChunk(idx)
+          }
+          // Try again
+          idx = undefined
+          processed = 0
+        }
+      }
+      while (needRetrieveChunks.length > 0) {
+        const needRetrieve = needRetrieveChunks.shift() as Ref<Doc>[]
+
+        console.log('Retrieve chunk:', needRetrieve.length)
+        let docs: Doc[] = []
+        try {
+          docs = await sourceConnection.loadDocs(c, needRetrieve)
+          if (clearTime) {
+            docs = docs.map((p) => {
+              if (sourceConnection.getHierarchy().isDerived(p._class, core.class.TxCollectionCUD)) {
+                return {
+                  ...p,
+                  createdBy: core.account.System,
+                  modifiedBy: core.account.System,
+                  modifiedOn: Date.now(),
+                  createdOn: Date.now(),
+                  tx: {
+                    ...(p as TxCollectionCUD<Doc, AttachedDoc>).tx,
+                    createdBy: core.account.System,
+                    modifiedBy: core.account.System,
+                    modifiedOn: Date.now(),
+                    createdOn: Date.now()
+                  }
+                }
+              } else {
+                return {
+                  ...p,
+                  createdBy: core.account.System,
+                  modifiedBy: core.account.System,
+                  modifiedOn: Date.now(),
+                  createdOn: Date.now()
+                }
+              }
+            })
+          }
+          await targetConnection.upload(c, docs)
+        } catch (err: any) {
+          console.log(err)
+          // Put back.
+          needRetrieveChunks.push(needRetrieve)
+          continue
+        }
+      }
+    }
+  } catch (err: any) {
+    console.error(err)
+  } finally {
+    console.log('end clone')
+    await sourceConnection.close()
+    await targetConnection.close()
+  }
+}
+
 /**
  * @public
  */

server/tool/src/plugin.ts

@@ -11,7 +11,8 @@ export const toolId = 'tool' as Plugin
 const toolPlugin = plugin(toolId, {
   metadata: {
     Endpoint: '' as Metadata<string>,
-    Transactor: '' as Metadata<string>
+    Transactor: '' as Metadata<string>,
+    InitWorkspace: '' as Metadata<string>
   }
 })