Backup pod (#1975)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
2024-11-22 11:42:30 +03:00 · 2022-06-02 09:26:42 +07:00 · 2022-06-02 09:26:42 +07:00 · b3e516beb4
commit b3e516beb4
parent 56d764d451
16 changed files with 402 additions and 38 deletions
--- a/common/config/rush/pnpm-lock.yaml
+++ b/common/config/rush/pnpm-lock.yaml
@ -104,6 +104,7 @@ specifiers:
  '@rush-temp/platform': file:./projects/platform.tgz
  '@rush-temp/platform-rig': file:./projects/platform-rig.tgz
  '@rush-temp/pod-account': file:./projects/pod-account.tgz
+  '@rush-temp/pod-backup': file:./projects/pod-backup.tgz
  '@rush-temp/preference': file:./projects/preference.tgz
  '@rush-temp/preference-assets': file:./projects/preference-assets.tgz
  '@rush-temp/presentation': file:./projects/presentation.tgz
@ -386,6 +387,7 @@ dependencies:
  '@rush-temp/platform': file:projects/platform.tgz
  '@rush-temp/platform-rig': file:projects/platform-rig.tgz_6fa5d152813644f44d2852d983dc9002
  '@rush-temp/pod-account': file:projects/pod-account.tgz
+  '@rush-temp/pod-backup': file:projects/pod-backup.tgz
  '@rush-temp/preference': file:projects/preference.tgz
  '@rush-temp/preference-assets': file:projects/preference-assets.tgz
  '@rush-temp/presentation': file:projects/presentation.tgz_3b42a51b6c974062237d417c554d9dd7
@ -3162,7 +3164,7 @@ packages:
    dev: false

  /assert-plus/1.0.0:
-    resolution: {integrity: sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=}
+    resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==}
    engines: {node: '>=0.8'}
    dev: false

@ -3242,7 +3244,7 @@ packages:
    dev: false

  /aws-sign2/0.7.0:
-    resolution: {integrity: sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=}
+    resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==}
    dev: false

  /aws4/1.11.0:
@ -3371,7 +3373,7 @@ packages:
    dev: false

  /bcrypt-pbkdf/1.0.2:
-    resolution: {integrity: sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=}
+    resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==}
    dependencies:
      tweetnacl: 0.14.5
    dev: false
@ -3699,7 +3701,7 @@ packages:
    dev: false

  /caseless/0.12.0:
-    resolution: {integrity: sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=}
+    resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}
    dev: false

  /chalk/2.4.2:
@ -4179,7 +4181,7 @@ packages:
    dev: false

  /dashdash/1.14.1:
-    resolution: {integrity: sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=}
+    resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==}
    engines: {node: '>=0.10'}
    dependencies:
      assert-plus: 1.0.0
@ -4496,7 +4498,7 @@ packages:
    dev: false

  /ecc-jsbn/0.1.2:
-    resolution: {integrity: sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=}
+    resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==}
    dependencies:
      jsbn: 0.1.1
      safer-buffer: 2.1.2
@ -5274,7 +5276,7 @@ packages:
    dev: false

  /extsprintf/1.3.0:
-    resolution: {integrity: sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=}
+    resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==}
    engines: {'0': node >=0.6.0}
    dev: false

@ -5492,7 +5494,7 @@ packages:
    dev: false

  /forever-agent/0.6.1:
-    resolution: {integrity: sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=}
+    resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==}
    dev: false

  /form-data/2.3.3:
@ -5633,7 +5635,7 @@ packages:
    dev: false

  /getpass/0.1.7:
-    resolution: {integrity: sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=}
+    resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==}
    dependencies:
      assert-plus: 1.0.0
    dev: false
@ -5733,7 +5735,7 @@ packages:
    dev: false

  /har-schema/2.0.0:
-    resolution: {integrity: sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=}
+    resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==}
    engines: {node: '>=4'}
    dev: false

@ -5983,7 +5985,7 @@ packages:
    dev: false

  /http-signature/1.2.0:
-    resolution: {integrity: sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=}
+    resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==}
    engines: {node: '>=0.8', npm: '>=1.3.7'}
    dependencies:
      assert-plus: 1.0.0
@ -6371,7 +6373,7 @@ packages:
    dev: false

  /isstream/0.1.2:
-    resolution: {integrity: sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=}
+    resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==}
    dev: false

  /istanbul-lib-coverage/3.2.0:
@ -6923,7 +6925,7 @@ packages:
    dev: false

  /jsbn/0.1.1:
-    resolution: {integrity: sha1-peZUwuWi3rXyAdls77yoDA7y9RM=}
+    resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==}
    dev: false

  /jsdom/16.7.0:
@ -7007,7 +7009,7 @@ packages:
    dev: false

  /json-stringify-safe/5.0.1:
-    resolution: {integrity: sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=}
+    resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
    dev: false

  /json5/0.5.1:
@ -13045,6 +13047,37 @@ packages:
      - supports-color
    dev: false

+  file:projects/pod-backup.tgz:
+    resolution: {integrity: sha512-DMl1JkvmMx9OJgpA9yGV0OOqpGBDDRCdiyrnk+cBr519KYLSkhakYULl7qvejWgP7eSAKs+ZVTOqBg8rCMKzyg==, tarball: file:projects/pod-backup.tgz}
+    name: '@rush-temp/pod-backup'
+    version: 0.0.0
+    dependencies:
+      '@rushstack/heft': 0.44.13
+      '@types/heft-jest': 1.0.2
+      '@types/minio': 7.0.13
+      '@types/node': 16.11.33
+      '@typescript-eslint/eslint-plugin': 5.22.0_27efc1da00e78084f5aa1809ff6483a1
+      '@typescript-eslint/parser': 5.22.0_eslint@7.32.0+typescript@4.6.4
+      cross-env: 7.0.3
+      dotenv: 16.0.0
+      esbuild: 0.12.29
+      eslint: 7.32.0
+      eslint-config-standard-with-typescript: 21.0.1_d91f81f4c73639e41ff8a6e8953d9ef2
+      eslint-plugin-import: 2.26.0_eslint@7.32.0
+      eslint-plugin-node: 11.1.0_eslint@7.32.0
+      eslint-plugin-promise: 5.2.0_eslint@7.32.0
+      got: 11.8.5
+      minio: 7.0.28
+      mongodb: 4.5.0
+      prettier: 2.6.2
+      ts-node: 10.5.0_fd4b5ff48b408bf425abfec4a641da13
+      typescript: 4.6.4
+    transitivePeerDependencies:
+      - '@swc/core'
+      - '@swc/wasm'
+      - supports-color
+    dev: false
+
  file:projects/preference-assets.tgz:
    resolution: {integrity: sha512-XMs/Zw64oLNq4NoRjxGI9vBIEdZrCXC8nHKW78fk7GYN+lXSvE4bSY3H8EFiBVAIL5i9F9bXK/VXZVUYAEY8Aw==, tarball: file:projects/preference-assets.tgz}
    name: '@rush-temp/preference-assets'
--- a/common/scripts/build_docker.sh
+++ b/common/scripts/build_docker.sh
@ -1,6 +1,6 @@
 #!/bin/bash

-roots='./server/server ./server/front ./pods/account ./products/tracker'
+roots='./server/server ./server/front ./pods/account ./pods/backup ./products/tracker'

 for r in $roots
 do
--- a/pods/backup/.eslintrc.js
+++ b/pods/backup/.eslintrc.js
@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@anticrm/platform-rig/profiles/default/config/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}
--- a/pods/backup/.npmignore
+++ b/pods/backup/.npmignore
@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/
--- a/pods/backup/Dockerfile
+++ b/pods/backup/Dockerfile
@ -0,0 +1,9 @@
+
+FROM node
+
+WORKDIR /usr/src/app
+
+COPY bundle.js ./
+
+EXPOSE 3000
+CMD [ "node", "bundle.js" ]
--- a/pods/backup/build.sh
+++ b/pods/backup/build.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+#
+#  Copyright © 2022 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+rushx bundle
+rushx docker:build
+rushx docker:push
--- a/pods/backup/config/rig.json
+++ b/pods/backup/config/rig.json
@ -0,0 +1,18 @@
+// The "rig.json" file directs tools to look for their config files in an external package.
+// Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+  /**
+   * (Required) The name of the rig package to inherit from.
+   * It should be an NPM package name with the "-rig" suffix.
+   */
+  "rigPackageName": "@anticrm/platform-rig"
+
+  /**
+   * (Optional) Selects a config profile from the rig package.  The name must consist of
+   * lowercase alphanumeric words separated by hyphens, for example "sample-profile".
+   * If omitted, then the "default" profile will be used."
+   */
+  // "rigProfile": "your-profile-name"
+}
--- a/pods/backup/package.json
+++ b/pods/backup/package.json
@ -0,0 +1,52 @@
+{
+  "name": "@anticrm/pod-backup",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "author": "Anticrm Platform Contributors",
+  "license": "EPL-2.0",
+  "scripts": {
+    "start": "ts-node src/index.ts",
+    "build": "heft build",
+    "build:watch": "tsc",
+    "lint:fix": "eslint --fix src",
+    "bundle": "esbuild src/index.ts --bundle --minify --platform=node > bundle.js",
+    "docker:build": "docker build -t hardcoreeng/backup .",
+    "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/backup staging",
+    "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/backup",
+    "run-local": "cross-env ACCOUNTS_URL=http://localhost:3000/ SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost TRANSACTOR_URL=ws:/localhost:3333 BUCKET_NAME=backups INTERVAL=30 ts-node src/index.ts",
+    "lint": "eslint src",
+    "format": "prettier --write src && eslint --fix src"
+  },
+  "devDependencies": {
+    "cross-env": "~7.0.3",
+    "@anticrm/platform-rig": "~0.6.0",
+    "@types/heft-jest": "^1.0.2",
+    "@types/node": "~16.11.12",
+    "@typescript-eslint/eslint-plugin": "^5.21.0",
+    "eslint-plugin-import": "^2.25.3",
+    "eslint-plugin-promise": "^5.1.1",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint": "^7.32.0",
+    "esbuild": "^0.12.26",
+    "@typescript-eslint/parser": "^5.4.0",
+    "eslint-config-standard-with-typescript": "^21.0.1",
+    "prettier": "^2.4.1",
+    "ts-node": "~10.5.0",
+    "@rushstack/heft": "^0.44.13",
+    "typescript": "^4.3.5",
+    "@types/minio": "~7.0.11"
+  },
+  "dependencies": {
+    "@anticrm/platform": "~0.6.6",
+    "mongodb": "^4.1.1",
+    "@anticrm/server-tool": "~0.6.0",
+    "@anticrm/server-token": "~0.6.0",
+    "@anticrm/client": "~0.6.2",
+    "@anticrm/client-resources": "~0.6.4",
+    "@anticrm/core": "~0.6.16",
+    "dotenv": "~16.0.0",
+    "got": "^11.8.3",
+    "@anticrm/server-backup": "~0.6.0",
+    "minio": "^7.0.26"
+  }
+}
--- a/pods/backup/src/config.ts
+++ b/pods/backup/src/config.ts
@ -0,0 +1,75 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+interface Config {
+  TransactorURL: string
+  AccountsURL: string
+  ServiceID: string
+  Secret: string
+
+  Interval: number
+  BucketName: string
+
+  MinioEndpoint: string
+  MinioAccessKey: string
+  MinioSecretKey: string
+}
+
+const envMap: { [key in keyof Config]: string } = {
+  TransactorURL: 'TRANSACTOR_URL',
+  AccountsURL: 'ACCOUNTS_URL',
+  ServiceID: 'SERVICE_ID',
+  Secret: 'SECRET',
+  BucketName: 'BUCKET_NAME',
+  Interval: 'INTERVAL',
+  MinioEndpoint: 'MINIO_ENDPOINT',
+  MinioAccessKey: 'MINIO_ACCESS_KEY',
+  MinioSecretKey: 'MINIO_SECRET_KEY'
+}
+
+const required: Array<keyof Config> = [
+  'TransactorURL',
+  'AccountsURL',
+  'Secret',
+  'ServiceID',
+  'BucketName',
+  'MinioEndpoint',
+  'MinioAccessKey',
+  'MinioSecretKey'
+]
+
+const config: Config = (() => {
+  const params: Partial<Config> = {
+    TransactorURL: process.env[envMap.TransactorURL],
+    AccountsURL: process.env[envMap.AccountsURL],
+    Secret: process.env[envMap.Secret],
+    BucketName: process.env[envMap.BucketName],
+    ServiceID: process.env[envMap.ServiceID] ?? 'backup-service',
+    Interval: parseInt(process.env[envMap.Interval] ?? '3600'),
+    MinioEndpoint: process.env[envMap.MinioEndpoint],
+    MinioAccessKey: process.env[envMap.MinioAccessKey],
+    MinioSecretKey: process.env[envMap.MinioSecretKey]
+  }
+
+  const missingEnv = required.filter((key) => params[key] === undefined).map((key) => envMap[key])
+
+  if (missingEnv.length > 0) {
+    throw Error(`Missing env variables: ${missingEnv.join(', ')}`)
+  }
+
+  return params as Config
+})()
+
+export default config
--- a/pods/backup/src/index.ts
+++ b/pods/backup/src/index.ts
@ -0,0 +1,36 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { PlatformWorker } from './platform'
+
+const main = async (): Promise<void> => {
+  const platformWorker = await PlatformWorker.create()
+
+  const shutdown = (): void => {
+    void platformWorker.close().then(() => {
+      process.exit()
+    })
+  }
+
+  process.on('SIGINT', shutdown)
+  process.on('SIGTERM', shutdown)
+  process.on('uncaughtException', (e) => {
+    console.error(e)
+  })
+  process.on('unhandledRejection', (e) => {
+    console.error(e)
+  })
+}
+void main()
--- a/pods/backup/src/platform.ts
+++ b/pods/backup/src/platform.ts
@ -0,0 +1,96 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { setMetadata } from '@anticrm/platform'
+import { backup, createMinioBackupStorage } from '@anticrm/server-backup'
+import got from 'got'
+import { Client as MinioClient } from 'minio'
+import config from './config'
+import serverToken from '@anticrm/server-token'
+
+async function getWorkspaces (): Promise<string[]> {
+  const { body }: { body: { error?: string, result?: any[] } } = await got.post(config.AccountsURL, {
+    json: {
+      method: 'listWorkspaces',
+      params: []
+    },
+    responseType: 'json'
+  })
+
+  if (body.error !== undefined) {
+    throw Error(body.error)
+  }
+
+  return (body.result ?? []).map((x) => x.workspace)
+}
+
+export class PlatformWorker {
+  minio?: MinioClient
+
+  async close (): Promise<void> {}
+
+  async init (): Promise<void> {
+    setMetadata(serverToken.metadata.Secret, config.Secret)
+    let minioPort = 9000
+    let minioEndpoint = config.MinioEndpoint
+    const sp = minioEndpoint.split(':')
+    if (sp.length > 1) {
+      minioEndpoint = sp[0]
+      minioPort = parseInt(sp[1])
+    }
+
+    this.minio = new MinioClient({
+      endPoint: minioEndpoint,
+      port: minioPort,
+      useSSL: false,
+      accessKey: config.MinioAccessKey,
+      secretKey: config.MinioSecretKey
+    })
+
+    await this.backup().then(() => {
+      void this.schedule()
+    })
+  }
+
+  async schedule (): Promise<void> {
+    console.log('schedule timeout for', config.Interval, ' seconds')
+    setTimeout(() => {
+      void this.backup().then(() => {
+        void this.schedule()
+      })
+    }, config.Interval * 1000)
+  }
+
+  async backup (): Promise<void> {
+    const workspaces = await getWorkspaces()
+    for (const ws of workspaces) {
+      console.log('\n\nBACKUP WORKSPACE ', ws)
+      try {
+        const storage = await createMinioBackupStorage(this.minio as MinioClient, 'backups', ws)
+        await backup(config.TransactorURL, ws, storage)
+      } catch (err: any) {
+        console.error('\n\nFAILED to BACKUP', ws, err)
+      }
+    }
+  }
+
+  static async create (): Promise<PlatformWorker> {
+    const worker = new PlatformWorker()
+    await worker.init()
+    return worker
+  }
+
+  private constructor () {}
+}
--- a/pods/backup/tsconfig.json
+++ b/pods/backup/tsconfig.json
@ -0,0 +1,9 @@
+{
+  "extends": "./node_modules/@anticrm/platform-rig/profiles/default/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "esModuleInterop": true
+  }
+}
--- a/products/tracker/tsconfig.json
+++ b/products/tracker/tsconfig.json
@ -7,6 +7,7 @@
    "allowJs": true,
    "sourceMap": true,
    "moduleResolution": "node",
+    "skipLibCheck": true,
    "allowSyntheticDefaultImports": true,
    "lib": [
      "es2016",
--- a/rush.json
+++ b/rush.json
@ -1307,6 +1307,11 @@
      "packageName": "@anticrm/server-backup",
      "projectFolder": "server/backup",
      "shouldPublish": true
-    }
+    },
+    {
+      "packageName": "@anticrm/pod-backup",
+      "projectFolder": "pods/backup",
+      "shouldPublish": false
+    },
  ]
 }