Move import tool to a separate image (#6719)

Signed-off-by: Anna Khismatullina <anna.khismatullina@gmail.com>

.vscode/launch.json

@@ -5,21 +5,19 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Debug notion import",
+      "name": "Debug Notion import",
       "type": "node",
       "request": "launch",
-      "args": ["src/__start.ts", "import-notion-to-teamspace", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-ws", "notion-test", "-u", "user1", "-pw", "1234", "-ts", "natalya"],
-      // "args": ["src/__start.ts", "import-notion-with-teamspaces", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-ws", "ws1", "-u", "user1", "-pw", "1234"],
+      // "args": ["src/__start.ts", "import-notion-to-teamspace", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws1", "-ts", "notion", ],
+      "args": ["src/__start.ts", "import-notion-with-teamspaces", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws1"],
       "env": {
-        "SERVER_SECRET": "secret",
         "FRONT_URL": "http://localhost:8087",
-        "ACCOUNTS_URL": "http://localhost:3000",
       },
       "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "sourceMaps": true,
       "outputCapture": "std",
-      "cwd": "${workspaceRoot}/dev/tool"
+      "cwd": "${workspaceRoot}/dev/import-tool"
     },
     {
       "address": "127.0.0.1",

common/config/rush/command-line.json

@@ -238,7 +238,7 @@
       "summary": "Build docker with platform",
       "description": "use to build all docker containers required for platform",
       "safeForSimultaneousRushProcesses": true,
-      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot"
+      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool"
     },        
     {
       "commandKind": "global",

dev/import-tool/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

dev/import-tool/Dockerfile

@@ -0,0 +1,7 @@
+FROM node:20
+
+WORKDIR /usr/src/app
+
+COPY bundle/bundle.js ./
+
+CMD [ "bash" ]

dev/import-tool/README.md

@@ -27,7 +27,7 @@ rushx run-local import-notion-to-teamspace {dir} \
 * *dir* - path to the root of the extracted archive
 * *user* - your username or email
 * *password* - password
-* *workspace* - workspace name where the documents should be imported to
+* *workspace* - workspace url where the documents should be imported to
 * *teamspace* - teamspace to be created for newly imported docs
 
 
@@ -58,11 +58,9 @@ rushx run-local import-notion-to-teamspace /home/john/extracted-notion-docs \
 * To import Notion workspace with teamspaces
 ```
 docker run \
-  -e SERVER_SECRET="" \
-  -e ACCOUNTS_URL="https://account.huly.app" \
   -e FRONT_URL="https://huly.app" \
   -v $(pwd):/data \
-  hardcoreeng/tool:latest \
+  hardcoreeng/import-tool:latest \
   -- bundle.js import-notion-with-teamspaces /data \
 --user jane.doe@gmail.com \
 --password 4321qwe \
@@ -71,14 +69,12 @@ docker run \
 * To import Notion workspace without teamspaces or a page with subpages.
 ```
 docker run \
-  -e SERVER_SECRET="" \
-  -e ACCOUNTS_URL="https://account.huly.app" \
   -e FRONT_URL="https://huly.app" \
   -v $(pwd):/data \
-  hardcoreeng/tool:latest \
+  hardcoreeng/import-tool:latest \
   -- bundle.js import-notion-to-teamspace /data \
 --user jane.doe@gmail.com \
 --password 4321qwe \
 --workspace ws1 \
 --teamspace notion
-```
+```

dev/import-tool/build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+#  Copyright © 2020, 2021 Anticrm Platform Contributors.
+#  Copyright © 2021 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+rushx bundle
+rushx docker:build $@
+rushx docker:push

dev/import-tool/esbuild.js

@@ -0,0 +1,9 @@
+const esbuild = require('esbuild')
+
+esbuild.build({
+  entryPoints: ['src/index.ts'],
+  bundle: true,
+  minify: true,
+  platform: 'node',
+  outfile: 'bundle/bundle.js'
+})

dev/import-tool/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@hcengineering/import-tool",
+  "version": "0.1.0",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "author": "Anticrm Platform Contributors",
+  "template": "@hcengineering/node-package",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "compile",
+    "build:watch": "compile",
+    "start": "ts-node src/__start.ts",
+    "_phase:bundle": "rushx bundle",
+    "_phase:docker-build": "rushx docker:build",
+    "_phase:docker-staging": "rushx docker:staging",
+    "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --keep-names --sourcemap=external --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --log-level=error --outfile=bundle/bundle.js",
+    "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/import-tool",
+    "docker:tbuild": "docker build -t hardcoreeng/import-tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/import-tool",
+    "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/import-tool staging",
+    "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/import-tool",
+    "run-local": "rush bundle --to @hcengineering/import-tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --max-old-space-size=18000 ./bundle/bundle.js",
+    "run-local-brk": "rush bundle --to @hcengineering/import-tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js",
+    "run": "rush bundle --to @hcengineering/import-tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js",
+    "upgrade": "rushx run-local upgrade",
+    "format": "format src",
+    "test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "cross-env": "~7.0.3",
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "ts-node": "^10.8.0",
+    "esbuild": "^0.20.0",
+    "@types/mime-types": "~2.1.1",
+    "@types/node": "~20.11.16",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5"
+  },
+  "dependencies": {
+    "@hcengineering/attachment": "^0.6.14",
+    "@hcengineering/collaboration": "^0.6.0",
+    "@hcengineering/document": "^0.6.0",
+    "@hcengineering/text": "^0.6.5",
+    "@hcengineering/model-attachment": "^0.6.0",
+    "@hcengineering/model-core": "^0.6.0",
+    "@hcengineering/core": "^0.6.32",
+    "@hcengineering/platform": "^0.6.11",
+    "@hcengineering/server-tool": "^0.6.0",
+    "@hcengineering/server-client": "^0.6.0",
+    "commander": "^8.1.0",
+    "mime-types": "~2.1.34"
+  }
+}

dev/import-tool/src/__start.ts

@@ -0,0 +1,17 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { importTool } from '.'
+
+importTool()

dev/import-tool/src/index.ts

@@ -0,0 +1,122 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { concatLink, TxOperations } from '@hcengineering/core'
+import serverClientPlugin, {
+  createClient,
+  getUserWorkspaces,
+  login,
+  selectWorkspace
+} from '@hcengineering/server-client'
+import { program } from 'commander'
+import { importNotion } from './notion'
+import { setMetadata } from '@hcengineering/platform'
+
+/**
+ * @public
+ */
+export function importTool (): void {
+  function getFrontUrl (): string {
+    const frontUrl = process.env.FRONT_URL
+    if (frontUrl === undefined) {
+      console.error('please provide front url')
+      process.exit(1)
+    }
+    return frontUrl
+  }
+
+  program.version('0.0.1')
+
+  // import-notion-with-teamspaces /home/anna/work/notion/pages/exported --workspace workspace
+  program
+    .command('import-notion-with-teamspaces <dir>')
+    .description('import extracted archive exported from Notion as "Markdown & CSV"')
+    .requiredOption('-u, --user <user>', 'user')
+    .requiredOption('-pw, --password <password>', 'password')
+    .requiredOption('-ws, --workspace <workspace>', 'workspace url where the documents should be imported to')
+    .action(async (dir: string, cmd) => {
+      await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace)
+    })
+
+  // import-notion-to-teamspace /home/anna/work/notion/pages/exported --workspace workspace --teamspace notion
+  program
+    .command('import-notion-to-teamspace <dir>')
+    .description('import extracted archive exported from Notion as "Markdown & CSV"')
+    .requiredOption('-u, --user <user>', 'user')
+    .requiredOption('-pw, --password <password>', 'password')
+    .requiredOption('-ws, --workspace <workspace>', 'workspace url where the documents should be imported to')
+    .requiredOption('-ts, --teamspace <teamspace>', 'new teamspace name where the documents should be imported to')
+    .action(async (dir: string, cmd) => {
+      await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace, cmd.teamspace)
+    })
+
+  async function importFromNotion (
+    dir: string,
+    user: string,
+    password: string,
+    workspaceUrl: string,
+    teamspace?: string
+  ): Promise<void> {
+    if (workspaceUrl === '' || user === '' || password === '' || teamspace === '') {
+      return
+    }
+
+    const config = await (await fetch(concatLink(getFrontUrl(), '/config.json'))).json()
+    console.log('Setting up Accounts URL: ', config.ACCOUNTS_URL)
+    setMetadata(serverClientPlugin.metadata.Endpoint, config.ACCOUNTS_URL)
+    console.log('Trying to login user: ', user)
+    const userToken = await login(user, password, workspaceUrl)
+    if (userToken === undefined) {
+      console.log('Login failed for user: ', user)
+      return
+    }
+
+    console.log('Looking for workspace: ', workspaceUrl)
+    const allWorkspaces = await getUserWorkspaces(userToken)
+    const workspaces = allWorkspaces.filter((ws) => ws.workspaceUrl === workspaceUrl)
+    if (workspaces.length < 1) {
+      console.log('Workspace not found: ', workspaceUrl)
+      return
+    }
+    console.log('Workspace found')
+    const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace)
+    console.log(selectedWs)
+
+    function uploader (token: string) {
+      return (id: string, data: any) => {
+        return fetch(concatLink(getFrontUrl(), config.UPLOAD_URL), {
+          method: 'POST',
+          headers: {
+            Authorization: 'Bearer ' + token
+          },
+          body: data
+        })
+      }
+    }
+
+    console.log('Connecting to Transactor URL: ', selectedWs.endpoint)
+    const connection = await createClient(selectedWs.endpoint, selectedWs.token)
+    const acc = connection.getModel().getAccountByEmail(user)
+    if (acc === undefined) {
+      console.log('Account not found for email: ', user)
+      return
+    }
+    const client = new TxOperations(connection, acc._id)
+    console.log('OK. Start the import directory: ', dir)
+    await importNotion(client, uploader(selectedWs.token), dir, teamspace)
+    await connection.close()
+  }
+
+  program.parse(process.argv)
+}

dev/import-tool/src/notion.ts

@@ -1,3 +1,17 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 import {
   generateId,
   type AttachedData,

dev/import-tool/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "declarationDir": "./types",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  }
+}

dev/tool/src/index.ts

@@ -46,14 +46,7 @@ import {
   createStorageBackupStorage,
   restore
 } from '@hcengineering/server-backup'
-import serverClientPlugin, {
-  BlobClient,
-  createClient,
-  getTransactorEndpoint,
-  getUserWorkspaces,
-  login,
-  selectWorkspace
-} from '@hcengineering/server-client'
+import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
 import { getServerPipeline } from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
@@ -68,13 +61,11 @@ import { diffWorkspace, recreateElastic, updateField } from './workspace'
 
 import core, {
   AccountRole,
-  concatLink,
   generateId,
   getWorkspaceId,
   MeasureMetricsContext,
   metricsToString,
   systemAccountEmail,
-  TxOperations,
   versionToString,
   type Data,
   type Doc,
@@ -115,7 +106,6 @@ import { changeConfiguration } from './configuration'
 import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db'
 import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
-import { importNotion } from './notion'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 import { moveFiles, showLostFiles, syncFiles } from './storage'
 
@@ -172,15 +162,6 @@ export function devTool (
     return elasticUrl
   }
 
-  function getFrontUrl (): string {
-    const frontUrl = process.env.FRONT_URL
-    if (frontUrl === undefined) {
-      console.error('please provide front url')
-      process.exit(1)
-    }
-    return frontUrl
-  }
-
   const initWS = process.env.INIT_WORKSPACE
   if (initWS !== undefined) {
     setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
@@ -242,73 +223,6 @@ export function devTool (
       })
     })
 
-  // import-notion-with-teamspaces /home/anna/work/notion/pages/exported --workspace workspace
-  program
-    .command('import-notion-with-teamspaces <dir>')
-    .description('import extracted archive exported from Notion as "Markdown & CSV"')
-    .requiredOption('-u, --user <user>', 'user')
-    .requiredOption('-pw, --password <password>', 'password')
-    .requiredOption('-ws, --workspace <workspace>', 'workspace where the documents should be imported to')
-    .action(async (dir: string, cmd) => {
-      await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace)
-    })
-
-  // import-notion-to-teamspace /home/anna/work/notion/pages/exported --workspace workspace --teamspace notion
-  program
-    .command('import-notion-to-teamspace <dir>')
-    .description('import extracted archive exported from Notion as "Markdown & CSV"')
-    .requiredOption('-u, --user <user>', 'user')
-    .requiredOption('-pw, --password <password>', 'password')
-    .requiredOption('-ws, --workspace <workspace>', 'workspace where the documents should be imported to')
-    .requiredOption('-ts, --teamspace <teamspace>', 'new teamspace name where the documents should be imported to')
-    .action(async (dir: string, cmd) => {
-      await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace, cmd.teamspace)
-    })
-
-  async function importFromNotion (
-    dir: string,
-    user: string,
-    password: string,
-    workspace: string,
-    teamspace?: string
-  ): Promise<void> {
-    if (workspace === '' || user === '' || password === '' || teamspace === '') {
-      return
-    }
-
-    const userToken = await login(user, password, workspace)
-    const allWorkspaces = await getUserWorkspaces(userToken)
-    const workspaces = allWorkspaces.filter((ws) => ws.workspace === workspace)
-    if (workspaces.length < 1) {
-      console.log('Workspace not found: ', workspace)
-      return
-    }
-    const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace)
-    console.log(selectedWs)
-
-    function uploader (token: string) {
-      return (id: string, data: any) => {
-        return fetch(concatLink(getFrontUrl(), '/files'), {
-          method: 'POST',
-          headers: {
-            Authorization: 'Bearer ' + token
-          },
-          body: data
-        })
-      }
-    }
-
-    const connection = await createClient(selectedWs.endpoint, selectedWs.token)
-    const acc = connection.getModel().getAccountByEmail(user)
-    if (acc === undefined) {
-      console.log('Account not found for email: ', user)
-      return
-    }
-    const client = new TxOperations(connection, acc._id)
-    await importNotion(client, uploader(selectedWs.token), dir, teamspace)
-    await connection.close()
-  }
-
   program
     .command('reset-account <email>')
     .description('create user and corresponding account in master database')

rush.json

@@ -855,6 +855,11 @@
       "projectFolder": "dev/tool",
       "shouldPublish": false
     },
+    {
+      "packageName": "@hcengineering/import-tool",
+      "projectFolder": "dev/import-tool",
+      "shouldPublish": false
+    },
     {
       "packageName": "@hcengineering/pod-account",
       "projectFolder": "pods/account",

server/client/src/account.ts

@@ -203,8 +203,7 @@ export async function login (user: string, password: string, workspace: string):
   })
 
   const result = await response.json()
-  const { token } = result.result
-  return token
+  return result.result?.token
 }
 
 export async function getUserWorkspaces (token: string): Promise<BaseWorkspaceInfo[]> {