From bf1de1f43610326ff481e47cb3d62edc3d6fde2b Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Wed, 11 Sep 2024 13:31:26 +0500 Subject: [PATCH] Cockroach (#6334) Signed-off-by: Denis Bykhov --- .github/workflows/main.yml | 66 + .vscode/launch.json | 31 +- common/config/rush/pnpm-lock.yaml | 224 ++- dev/docker-compose.yaml | 10 + dev/tool/package.json | 2 + dev/tool/src/__start.ts | 1 + dev/tool/src/benchmark.ts | 121 ++ dev/tool/src/db.ts | 68 + dev/tool/src/index.ts | 84 +- dev/tool/src/markup.ts | 8 +- models/contact/src/migration.ts | 62 +- models/core/src/index.ts | 4 +- models/core/src/migration.ts | 12 +- models/core/src/security.ts | 4 +- packages/core/src/classes.ts | 5 + packages/core/src/operator.ts | 14 +- packages/core/src/server.ts | 22 +- packages/model/src/migration.ts | 8 +- .../src/components/CreateEmployee.svelte | 13 +- plugins/tags-resources/src/utils.ts | 19 +- pods/server/package.json | 2 +- pods/server/src/server.ts | 12 +- rush.json | 5 + server/account-service/src/index.ts | 9 +- server/account/src/operations.ts | 3 +- server/core/src/adapter.ts | 16 +- server/core/src/domainHelper.ts | 2 +- server/core/src/mem.ts | 28 +- server/elastic/src/backup.ts | 28 +- server/middleware/src/dbAdapter.ts | 49 +- server/middleware/src/lowLevel.ts | 31 +- server/middleware/src/spaceSecurity.ts | 13 +- server/middleware/src/triggers.ts | 3 + server/mongo/src/storage.ts | 125 +- server/mongo/src/utils.ts | 7 +- server/postgres/.eslintrc.js | 7 + server/postgres/.npmignore | 4 + server/postgres/config/rig.json | 5 + server/postgres/jest.config.js | 7 + server/postgres/package.json | 43 + server/postgres/src/__tests__/minmodel.ts | 233 +++ server/postgres/src/__tests__/storage.test.ts | 328 ++++ server/postgres/src/__tests__/tasks.ts | 112 ++ server/postgres/src/index.ts | 17 + server/postgres/src/storage.ts | 1439 +++++++++++++++++ server/postgres/src/utils.ts | 391 +++++ server/postgres/tsconfig.json | 10 + server/server-pipeline/package.json | 1 + server/server-pipeline/src/pipeline.ts | 277 +++- server/server-storage/src/blobStorage.ts | 25 + server/server/package.json | 1 - server/tool/src/index.ts | 450 +++--- server/tool/src/upgrade.ts | 184 +-- server/workspace-service/src/ws-operations.ts | 90 +- tests/docker-compose.override.yaml | 10 + tests/docker-compose.yaml | 7 + tests/prepare-pg.sh | 30 + tests/prepare-tests.sh | 2 +- tests/prepare.sh | 2 +- tests/restore-pg.sh | 18 + tests/sanity/tests/recruiting/talents.spec.ts | 2 - tests/tool-pg.sh | 13 + 62 files changed, 4187 insertions(+), 632 deletions(-) create mode 100644 dev/tool/src/db.ts create mode 100644 server/postgres/.eslintrc.js create mode 100644 server/postgres/.npmignore create mode 100644 server/postgres/config/rig.json create mode 100644 server/postgres/jest.config.js create mode 100644 server/postgres/package.json create mode 100644 server/postgres/src/__tests__/minmodel.ts create mode 100644 server/postgres/src/__tests__/storage.test.ts create mode 100644 server/postgres/src/__tests__/tasks.ts create mode 100644 server/postgres/src/index.ts create mode 100644 server/postgres/src/storage.ts create mode 100644 server/postgres/src/utils.ts create mode 100644 server/postgres/tsconfig.json create mode 100644 tests/docker-compose.override.yaml create mode 100755 tests/prepare-pg.sh create mode 100755 tests/restore-pg.sh create mode 100755 tests/tool-pg.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd938ddeca..3747dd2f36 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,6 +190,7 @@ jobs: - name: Testing... run: node common/scripts/install-run-rush.js test env: + DB_URL: 'postgresql://postgres:example@localhost:5433' ELASTIC_URL: 'http://localhost:9201' MONGO_URL: 'mongodb://localhost:27018' uitest: @@ -309,6 +310,71 @@ jobs: # with: # name: db-snapshot # path: ./tests/db_dump + uitest-pg: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: tree:0 + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + - name: Cache node modules + uses: actions/cache@v4 + env: + cache-name: cache-node-platform + with: + path: | + common/temp + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }} + + - name: Checking for mis-matching dependencies... + run: node common/scripts/install-run-rush.js check + + - name: Installing... + run: node common/scripts/install-run-rush.js install + + - name: Docker Build + run: node common/scripts/install-run-rush.js docker:build -p 20 + env: + DOCKER_CLI_HINTS: false + - name: Prepare server + run: | + cd ./tests + export DO_CLEAN="true" + ./prepare-pg.sh + - name: Install Playwright + run: | + cd ./tests/sanity + node ../../common/scripts/install-run-rushx.js ci + - name: Run UI tests + run: | + cd ./tests/sanity + node ../../common/scripts/install-run-rushx.js uitest + - name: 'Store docker logs' + if: always() + run: | + cd ./tests/sanity + mkdir logs + docker logs $(docker ps | grep transactor | cut -f 1 -d ' ') > logs/transactor.log + docker logs $(docker ps | grep account | cut -f 1 -d ' ') > logs/account.log + docker logs $(docker ps | grep front | cut -f 1 -d ' ') > logs/front.log + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-results-pg + path: ./tests/sanity/playwright-report/ + - name: Upload Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: docker-logs-pg + path: ./tests/sanity/logs uitest-qms: runs-on: ubuntu-latest timeout-minutes: 60 diff --git a/.vscode/launch.json b/.vscode/launch.json index 7f3d2c0ac7..b9dc845ca2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -59,15 +59,15 @@ "args": ["src/__start.ts"], "env": { "ELASTIC_URL": "http://localhost:9200", - "MONGO_URL": "mongodb://localhost:27017", + "MONGO_URL": "postgresql://postgres:example@localhost:5432;mongodb://localhost:27017", "APM_SERVER_URL2": "http://localhost:8200", "METRICS_CONSOLE": "false", "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds., - "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", + "STORAGE_CONFIG": "minio|localhost:9000?accessKey=minioadmin&secretKey=minioadmin", "SERVER_SECRET": "secret", "ENABLE_CONSOLE": "true", "COLLABORATOR_URL": "ws://localhost:3078", - "REKONI_URL": "http://localhost:4004", + "REKONI_URL": "http://localhost:4000", "FRONT_URL": "http://localhost:8080", "ACCOUNTS_URL": "http://localhost:3000", // "SERVER_PROVIDER":"uweb" @@ -240,7 +240,29 @@ "name": "Debug tool upgrade", "type": "node", "request": "launch", - "args": ["src/__start.ts", "stress", "ws://localhost:3333", "wrong"], + "args": ["src/__start.ts", "create-workspace", "sanity-ws", "-w sanity-ws"], + "env": { + "SERVER_SECRET": "secret", + "MINIO_ACCESS_KEY": "minioadmin", + "MINIO_SECRET_KEY": "minioadmin", + "MINIO_ENDPOINT": "localhost:9000", + "TRANSACTOR_URL": "ws://localhost:3333", + "MONGO_URL": "mongodb://localhost:27017", + "ACCOUNTS_URL": "http://localhost:3000", + "TELEGRAM_DATABASE": "telegram-service", + "ELASTIC_URL": "http://localhost:9200", + "REKONI_URL": "http://localhost:4000" + }, + "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], + "sourceMaps": true, + "outputCapture": "std", + "cwd": "${workspaceRoot}/dev/tool" + }, + { + "name": "Debug tool move", + "type": "node", + "request": "launch", + "args": ["src/__start.ts", "move-to-pg"], "env": { "SERVER_SECRET": "secret", "MINIO_ACCESS_KEY": "minioadmin", @@ -248,6 +270,7 @@ "MINIO_ENDPOINT": "localhost", "TRANSACTOR_URL": "ws://localhost:3333", "MONGO_URL": "mongodb://localhost:27017", + "DB_URL": "postgresql://postgres:example@localhost:5432", "ACCOUNTS_URL": "http://localhost:3000", "TELEGRAM_DATABASE": "telegram-service", "ELASTIC_URL": "http://localhost:9200", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b0f232ad2c..6367e53b2d 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -629,6 +629,9 @@ dependencies: '@rush-temp/pod-workspace': specifier: file:./projects/pod-workspace.tgz version: file:projects/pod-workspace.tgz + '@rush-temp/postgres': + specifier: file:./projects/postgres.tgz + version: file:projects/postgres.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/preference': specifier: file:./projects/preference.tgz version: file:projects/preference.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) @@ -1298,6 +1301,9 @@ dependencies: '@types/pdfjs-dist': specifier: 2.10.378 version: 2.10.378 + '@types/pg': + specifier: ^8.11.6 + version: 8.11.6 '@types/png-chunks-extract': specifier: ^1.0.2 version: 1.0.2 @@ -1703,6 +1709,9 @@ dependencies: pdfjs-dist: specifier: 2.12.313 version: 2.12.313 + pg: + specifier: 8.12.0 + version: 8.12.0 png-chunks-extract: specifier: ^1.0.0 version: 1.0.0 @@ -4690,7 +4699,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.5 + debug: 4.3.4 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -4845,7 +4854,7 @@ packages: engines: {node: '>=10.10.0'} dependencies: '@humanwhocodes/object-schema': 2.0.2 - debug: 4.3.5 + debug: 4.3.4 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -9364,6 +9373,14 @@ packages: - worker-loader dev: false + /@types/pg@8.11.6: + resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} + dependencies: + '@types/node': 20.11.19 + pg-protocol: 1.6.1 + pg-types: 4.0.2 + dev: false + /@types/plist@3.0.5: resolution: {integrity: sha512-E6OCaRmAe4WDmWNsL/9RMqdkkzDCY1etutkflWk4c+AcjDU07Pcz1fQwTX0TQz+Pxqn9i4L1TU3UFpjnrcDgxA==} requiresBuild: true @@ -9728,7 +9745,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.3.3) '@typescript-eslint/utils': 6.21.0(eslint@8.56.0)(typescript@5.3.3) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.56.0 ts-api-utils: 1.2.1(typescript@5.3.3) typescript: 5.3.3 @@ -9778,7 +9795,7 @@ packages: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.5 + debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -10566,7 +10583,7 @@ packages: builder-util: 24.13.1 builder-util-runtime: 9.2.4 chromium-pickle-js: 0.2.0 - debug: 4.3.5 + debug: 4.3.4 dmg-builder: 24.13.3 ejs: 3.1.9 electron-publish: 24.13.1 @@ -11403,7 +11420,7 @@ packages: resolution: {integrity: sha512-upp+biKpN/XZMLim7aguUyW8s0FUpDvOtK6sbanMFDAMBzpHDqdhgVYm6zc9HJ6nWo7u2Lxk60i2M6Jd3aiNrA==} engines: {node: '>=12.0.0'} dependencies: - debug: 4.3.5 + debug: 4.3.4 sax: 1.3.0 transitivePeerDependencies: - supports-color @@ -11419,7 +11436,7 @@ packages: builder-util-runtime: 9.2.4 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.5 + debug: 4.3.4 fs-extra: 10.1.0 http-proxy-agent: 5.0.0 https-proxy-agent: 5.0.1 @@ -13385,7 +13402,7 @@ packages: has-property-descriptors: 1.0.2 has-proto: 1.0.3 has-symbols: 1.0.3 - hasown: 2.0.2 + hasown: 2.0.1 internal-slot: 1.0.7 is-array-buffer: 3.0.4 is-callable: 1.2.7 @@ -13531,13 +13548,13 @@ packages: dependencies: get-intrinsic: 1.2.4 has-tostringtag: 1.0.2 - hasown: 2.0.2 + hasown: 2.0.1 dev: false /es-shim-unscopables@1.0.2: resolution: {integrity: sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==} dependencies: - hasown: 2.0.2 + hasown: 2.0.1 dev: false /es-to-primitive@1.2.1: @@ -14139,7 +14156,7 @@ packages: optionator: 0.9.3 progress: 2.0.3 regexpp: 3.2.0 - semver: 7.6.0 + semver: 7.6.3 strip-ansi: 6.0.1 strip-json-comments: 3.1.1 text-table: 0.2.0 @@ -14850,7 +14867,7 @@ packages: minimatch: 3.1.2 node-abort-controller: 3.1.1 schema-utils: 3.3.0 - semver: 7.6.0 + semver: 7.6.3 tapable: 2.2.1 typescript: 5.3.3 webpack: 5.90.3(@swc/core@1.4.2)(esbuild@0.20.1)(webpack-cli@5.1.4) @@ -15105,7 +15122,7 @@ packages: function-bind: 1.1.2 has-proto: 1.0.3 has-symbols: 1.0.3 - hasown: 2.0.2 + hasown: 2.0.1 dev: false /get-nonce@1.0.1: @@ -15868,7 +15885,7 @@ packages: engines: {node: '>= 6'} dependencies: agent-base: 6.0.2 - debug: 4.3.5 + debug: 4.3.4 transitivePeerDependencies: - supports-color dev: false @@ -16037,7 +16054,7 @@ packages: engines: {node: '>= 0.4'} dependencies: es-errors: 1.3.0 - hasown: 2.0.2 + hasown: 2.0.1 side-channel: 1.0.6 dev: false @@ -16979,7 +16996,7 @@ packages: jest-util: 29.7.0 natural-compare: 1.4.0 pretty-format: 29.7.0 - semver: 7.6.0 + semver: 7.6.3 transitivePeerDependencies: - supports-color dev: false @@ -19490,6 +19507,86 @@ packages: is-reference: 3.0.2 dev: false + /pg-cloudflare@1.1.1: + resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} + requiresBuild: true + dev: false + optional: true + + /pg-connection-string@2.6.4: + resolution: {integrity: sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==} + dev: false + + /pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + dev: false + + /pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + dev: false + + /pg-pool@3.6.2(pg@8.12.0): + resolution: {integrity: sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==} + peerDependencies: + pg: '>=8.0' + dependencies: + pg: 8.12.0 + dev: false + + /pg-protocol@1.6.1: + resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==} + dev: false + + /pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.0 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + dev: false + + /pg-types@4.0.2: + resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} + engines: {node: '>=10'} + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.2 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + dev: false + + /pg@8.12.0: + resolution: {integrity: sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==} + engines: {node: '>= 8.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + dependencies: + pg-connection-string: 2.6.4 + pg-pool: 3.6.2(pg@8.12.0) + pg-protocol: 1.6.1 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.1.1 + dev: false + + /pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + dependencies: + split2: 4.2.0 + dev: false + /phin@2.9.3: resolution: {integrity: sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. @@ -19782,6 +19879,54 @@ packages: source-map-js: 1.0.2 dev: false + /postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + dev: false + + /postgres-array@3.0.2: + resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} + engines: {node: '>=12'} + dev: false + + /postgres-bytea@1.0.0: + resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} + engines: {node: '>=0.10.0'} + dev: false + + /postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + dependencies: + obuf: 1.1.2 + dev: false + + /postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + dev: false + + /postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + dev: false + + /postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + dependencies: + xtend: 4.0.2 + dev: false + + /postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + dev: false + + /postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + dev: false + /posthog-js@1.122.0: resolution: {integrity: sha512-+8R2/nLaWyI5Jp2Ly7L52qcgDFU3xryyoNG52DPJ8dlGnagphxIc0mLNGurgyKeeTGycsOsuOIP4dtofv3ZoBA==} deprecated: This version of posthog-js is deprecated, please update posthog-js, and do not use this version! Check out our JS docs at https://posthog.com/docs/libraries/js @@ -20646,7 +20791,7 @@ packages: resolution: {integrity: sha512-efCx3b+0Z69/LGJmm9Yvi4cqEdxnoGnxYxGxBghkkTTFeXRtTCmmhO0AnAfHz59k957uTSuy8WaHqOs8wbYUWg==} engines: {node: '>=6'} dependencies: - debug: 4.3.5 + debug: 4.3.4 module-details-from-path: 1.0.3 resolve: 1.22.8 transitivePeerDependencies: @@ -21479,6 +21624,11 @@ packages: engines: {node: '>=6'} dev: false + /split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + dev: false + /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: false @@ -30219,6 +30369,39 @@ packages: - supports-color dev: false + file:projects/postgres.tgz(esbuild@0.20.1)(ts-node@10.9.2): + resolution: {integrity: sha512-qZVG4Pk9RAvQfkKRB1iQPPOWsKFvFuFyNBtD/ksT/eW0/ByyGABvYMeQPNenrh3ZH2n/hZ5lH5DON042MXebPg==, tarball: file:projects/postgres.tgz} + id: file:projects/postgres.tgz + name: '@rush-temp/postgres' + version: 0.0.0 + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@types/pg': 8.11.6 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + pg: 8.12.0 + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) + typescript: 5.3.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - pg-native + - supports-color + - ts-node + dev: false + file:projects/preference-assets.tgz(esbuild@0.20.1)(ts-node@10.9.2): resolution: {integrity: sha512-VlBSKBg3XmuMLtxNAS703aS+dhhb5a7H5Ns2nzhhv7w3KlAqtwp6cQ5VLxceNuRaPbTtI+2K+YkjFb2S1ld5VQ==, tarball: file:projects/preference-assets.tgz} id: file:projects/preference-assets.tgz @@ -32536,7 +32719,7 @@ packages: dev: false file:projects/server-pipeline.tgz: - resolution: {integrity: sha512-LU5dxarK3XG4FXu2W/Wmu1IZh93faWFIdjKmurdgNZ9AV2Mv7B6fTALyVZoA0O7gysXFNDlxvH/qq7PyfAHivQ==, tarball: file:projects/server-pipeline.tgz} + resolution: {integrity: sha512-W5khmA6sUi5kU1UiRR7YTOYH40t96jeBa9ww0g1EUtraXSlO12g2NBpMXswt8/HkMVe6j/56Jj2vq8QVnXSY5w==, tarball: file:projects/server-pipeline.tgz} name: '@rush-temp/server-pipeline' version: 0.0.0 dependencies: @@ -33412,7 +33595,7 @@ packages: dev: false file:projects/server.tgz(esbuild@0.20.1): - resolution: {integrity: sha512-fTgDuks26uZ0IhbPq9N/65R1I/hQG/gRRygfBsTmwAQqcJfo25EMz9gfLsuxu6YgZIWgI+UU6wbphtNeWLW/2A==, tarball: file:projects/server.tgz} + resolution: {integrity: sha512-93PHAZJSkt0uDRAhNNcAGlJUHFM7/RLUAmzu2CZxcknO0hRM6QTDIkRg6T+GfE3fqyMuRsIJA0CDJdlziht/sA==, tarball: file:projects/server.tgz} id: file:projects/server.tgz name: '@rush-temp/server' version: 0.0.0 @@ -34612,7 +34795,7 @@ packages: dev: false file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-gjmLNkjPV0dXJNjkowQTvuR1ELM+S3aT3t/y9QJScPMxDIUPZRjkXIoM//4Qk3/RqUOFVhcRQTVoZVcTtK6rCg==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-LwQbmBaSOZ5IKwCHz2mULcIuEr9rZ2b/7tqUGICHCawUzexUlQVxv2Yt0oFf2aZu83Sittt7dZwnN3sXHX9t9g==, tarball: file:projects/tool.tgz} id: file:projects/tool.tgz name: '@rush-temp/tool' version: 0.0.0 @@ -35337,7 +35520,6 @@ packages: koa: 2.15.3 koa-bodyparser: 4.4.1 koa-router: 12.0.1 - mongodb: 6.8.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index ad51c2a1ce..78ea7ab210 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -19,6 +19,16 @@ services: ports: - 27017:27017 restart: unless-stopped + postgres: + image: postgres + container_name: postgres + environment: + - POSTGRES_PASSWORD=example + volumes: + - db:/data/db + ports: + - 5432:5432 + restart: unless-stopped minio: image: 'minio/minio' command: server /data --address ":9000" --console-address ":9001" diff --git a/dev/tool/package.json b/dev/tool/package.json index 700f121318..476c331442 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -79,10 +79,12 @@ "@hcengineering/model-task": "^0.6.0", "@hcengineering/model-activity": "^0.6.0", "@hcengineering/model-lead": "^0.6.0", + "@hcengineering/postgres": "^0.6.0", "@hcengineering/mongo": "^0.6.1", "@hcengineering/platform": "^0.6.11", "@hcengineering/recruit": "^0.6.29", "@hcengineering/rekoni": "^0.6.0", + "@hcengineering/server-pipeline": "^0.6.0", "@hcengineering/server-attachment": "^0.6.1", "@hcengineering/server-attachment-resources": "^0.6.0", "@hcengineering/server-collaboration": "^0.6.0", diff --git a/dev/tool/src/__start.ts b/dev/tool/src/__start.ts index ab0e6a74bc..fd1f7f0676 100644 --- a/dev/tool/src/__start.ts +++ b/dev/tool/src/__start.ts @@ -73,6 +73,7 @@ addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources' function prepareTools (): { mongodbUri: string + dbUrl: string | undefined txes: Tx[] version: Data migrateOperations: [string, MigrateOperation][] diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index dfa440535f..fcae5fd39f 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -14,11 +14,14 @@ // import core, { + AccountRole, MeasureMetricsContext, RateLimiter, TxOperations, concatLink, generateId, + getWorkspaceId, + makeCollaborativeDoc, metricsToString, newMetrics, systemAccountEmail, @@ -40,6 +43,10 @@ import os from 'os' import { Worker, isMainThread, parentPort } from 'worker_threads' import { CSVWriter } from './csv' +import { AvatarType, type PersonAccount } from '@hcengineering/contact' +import contact from '@hcengineering/model-contact' +import recruit from '@hcengineering/model-recruit' +import { type Vacancy } from '@hcengineering/recruit' import { WebSocket } from 'ws' interface StartMessage { @@ -503,3 +510,117 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark } } } + +export async function testFindAll (endpoint: string, workspace: string, email: string): Promise { + const connection = await connect(endpoint, getWorkspaceId(workspace), email) + try { + const client = new TxOperations(connection, core.account.System) + const start = Date.now() + const res = await client.findAll( + recruit.class.Applicant, + {}, + { + lookup: { + attachedTo: recruit.mixin.Candidate, + space: recruit.class.Vacancy + } + } + ) + console.log('Find all', res.length, 'time', Date.now() - start) + } finally { + await connection.close() + } +} + +export async function generateWorkspaceData ( + endpoint: string, + workspace: string, + parallel: boolean, + user: string +): Promise { + const connection = await connect(endpoint, getWorkspaceId(workspace)) + const client = new TxOperations(connection, core.account.System) + try { + const acc = await client.findOne(contact.class.PersonAccount, { email: user }) + if (acc == null) { + throw new Error('User not found') + } + const employees: Ref[] = [acc._id] + const start = Date.now() + for (let i = 0; i < 100; i++) { + const acc = await generateEmployee(client) + employees.push(acc) + } + if (parallel) { + const promises: Promise[] = [] + for (let i = 0; i < 10; i++) { + promises.push(generateVacancy(client, employees)) + } + await Promise.all(promises) + } else { + for (let i = 0; i < 10; i++) { + await generateVacancy(client, employees) + } + } + console.log('Generate', Date.now() - start) + } finally { + await connection.close() + } +} + +export async function generateEmployee (client: TxOperations): Promise> { + const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, { + name: generateId().toString(), + city: '', + avatarType: AvatarType.COLOR + }) + await client.createMixin(personId, contact.class.Person, contact.space.Contacts, contact.mixin.Employee, { + active: true + }) + const acc = await client.createDoc(contact.class.PersonAccount, core.space.Model, { + person: personId, + role: AccountRole.User, + email: personId + }) + return acc +} + +async function generateVacancy (client: TxOperations, members: Ref[]): Promise { + // generate vacancies + const _id = generateId() + await client.createDoc( + recruit.class.Vacancy, + core.space.Space, + { + name: generateId().toString(), + number: 0, + fullDescription: makeCollaborativeDoc(_id, 'fullDescription'), + type: recruit.template.DefaultVacancy, + description: '', + private: false, + members, + archived: false + }, + _id + ) + for (let i = 0; i < 100; i++) { + // generate candidate + const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, { + name: generateId().toString(), + city: '', + avatarType: AvatarType.COLOR + }) + await client.createMixin(personId, contact.class.Person, contact.space.Contacts, recruit.mixin.Candidate, {}) + // generate applicants + await client.addCollection(recruit.class.Applicant, _id, personId, recruit.mixin.Candidate, 'applications', { + status: recruit.taskTypeStatus.Backlog, + number: i + 1, + identifier: `APP-${i + 1}`, + assignee: null, + rank: '', + startDate: null, + dueDate: null, + kind: recruit.taskTypes.Applicant + }) + } +} diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts new file mode 100644 index 0000000000..51301c2203 --- /dev/null +++ b/dev/tool/src/db.ts @@ -0,0 +1,68 @@ +import { type Doc, type WorkspaceId } from '@hcengineering/core' +import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' +import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres' + +export async function moveFromMongoToPG ( + mongoUrl: string, + dbUrl: string | undefined, + workspaces: WorkspaceId[] +): Promise { + if (dbUrl === undefined) { + throw new Error('dbUrl is required') + } + const client = getMongoClient(mongoUrl) + const mongo = await client.getClient() + const pg = getDBClient(dbUrl) + const pgClient = await pg.getClient() + + for (let index = 0; index < workspaces.length; index++) { + const ws = workspaces[index] + try { + const mongoDB = getWorkspaceDB(mongo, ws) + const collections = await mongoDB.collections() + await createTable( + pgClient, + collections.map((c) => c.collectionName) + ) + for (const collection of collections) { + const cursor = collection.find() + const domain = translateDomain(collection.collectionName) + while (true) { + const doc = (await cursor.next()) as Doc | null + if (doc === null) break + try { + const converted = convertDoc(doc, ws.name) + await retryTxn(pgClient, async (client) => { + await client.query( + `INSERT INTO ${domain} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + [ + converted._id, + converted.workspaceId, + converted._class, + converted.createdBy, + converted.modifiedBy, + converted.modifiedOn, + converted.createdOn, + converted.space, + converted.attachedTo, + converted.data + ] + ) + }) + } catch (err) { + console.log('error when move doc', doc._id, doc._class, err) + continue + } + } + } + if (index % 100 === 0) { + console.log('Move workspace', index, workspaces.length) + } + } catch (err) { + console.log('Error when move workspace', ws.name, err) + throw err + } + } + pg.close() + client.close() +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 4fcc7c2690..2fbb945a10 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -19,6 +19,7 @@ import accountPlugin, { assignWorkspace, confirmEmail, createAcc, + createWorkspace as createWorkspaceRecord, dropAccount, dropWorkspace, dropWorkspaceFull, @@ -32,10 +33,8 @@ import accountPlugin, { setAccountAdmin, setRole, updateWorkspace, - createWorkspace as createWorkspaceRecord, type Workspace } from '@hcengineering/account' -import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' import { setMetadata } from '@hcengineering/platform' import { backup, @@ -54,8 +53,10 @@ import serverClientPlugin, { login, selectWorkspace } from '@hcengineering/server-client' +import { getServerPipeline } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool' +import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' import path from 'path' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' @@ -66,6 +67,8 @@ import { diffWorkspace, recreateElastic, updateField } from './workspace' import core, { AccountRole, + concatLink, + generateId, getWorkspaceId, MeasureMetricsContext, metricsToString, @@ -79,7 +82,7 @@ import core, { type Tx, type Version, type WorkspaceId, - concatLink + type WorkspaceIdWithUrl } from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' import contact from '@hcengineering/model-contact' @@ -87,7 +90,14 @@ import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' import type { StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core' import { deepEqual } from 'fast-equals' import { createWriteStream, readFileSync } from 'fs' -import { benchmark, benchmarkWorker, stressBenchmark, type StressBenchmarkMode } from './benchmark' +import { + benchmark, + benchmarkWorker, + generateWorkspaceData, + stressBenchmark, + testFindAll, + type StressBenchmarkMode +} from './benchmark' import { cleanArchivedSpaces, cleanRemovedTransactions, @@ -101,11 +111,12 @@ import { restoreRecruitingTaskTypes } from './clean' import { changeConfiguration } from './configuration' +import { moveFromMongoToPG } from './db' import { fixJsonMarkup, migrateMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' +import { importNotion } from './notion' import { fixAccountEmails, renameAccount } from './renameAccount' import { moveFiles, syncFiles } from './storage' -import { importNotion } from './notion' const colorConstants = { colorRed: '\u001b[31m', @@ -125,6 +136,7 @@ const colorConstants = { export function devTool ( prepareTools: () => { mongodbUri: string + dbUrl: string | undefined txes: Tx[] version: Data migrateOperations: [string, MigrateOperation][] @@ -1470,7 +1482,7 @@ export function devTool ( .option('-w, --workspace ', 'Selected workspace only', '') .option('-c, --concurrency ', 'Number of documents being processed concurrently', '10') .action(async (cmd: { workspace: string, concurrency: string }) => { - const { mongodbUri } = prepareTools() + const { mongodbUri, dbUrl } = prepareTools() await withDatabase(mongodbUri, async (db, client) => { await withStorage(mongodbUri, async (adapter) => { const workspaces = await listWorkspacesPure(db) @@ -1482,8 +1494,15 @@ export function devTool ( const wsId = getWorkspaceId(workspace.workspace) console.log('processing workspace', workspace.workspace, index, workspaces.length) + const wsUrl: WorkspaceIdWithUrl = { + name: workspace.workspace, + workspaceName: workspace.workspaceName ?? '', + workspaceUrl: workspace.workspaceUrl ?? '' + } - await migrateMarkup(toolCtx, adapter, wsId, client, mongodbUri, parseInt(cmd.concurrency)) + const { pipeline } = await getServerPipeline(toolCtx, mongodbUri, dbUrl, wsUrl) + + await migrateMarkup(toolCtx, adapter, wsId, client, pipeline, parseInt(cmd.concurrency)) console.log('...done', workspace.workspace) index++ @@ -1502,6 +1521,57 @@ export function devTool ( }) }) + program.command('move-to-pg').action(async () => { + const { mongodbUri, dbUrl } = prepareTools() + await withDatabase(mongodbUri, async (db) => { + const workspaces = await listWorkspacesRaw(db) + await moveFromMongoToPG( + mongodbUri, + dbUrl, + workspaces.map((it) => getWorkspaceId(it.workspace)) + ) + }) + }) + + program + .command('perfomance') + .option('-p, --parallel', '', false) + .action(async (cmd: { parallel: boolean }) => { + const { mongodbUri, txes, version, migrateOperations } = prepareTools() + await withDatabase(mongodbUri, async (db) => { + const email = generateId() + const ws = generateId() + const wsid = getWorkspaceId(ws) + const start = new Date() + const measureCtx = new MeasureMetricsContext('create-workspace', {}) + const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws) + + // update the record so it's not taken by one of the workers for the next 60 seconds + await updateWorkspace(db, wsInfo, { + mode: 'creating', + progress: 0, + lastProcessingTime: Date.now() + 1000 * 60 + }) + + await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations) + + await updateWorkspace(db, wsInfo, { + mode: 'active', + progress: 100, + disabled: false, + version + }) + await createAcc(toolCtx, db, null, email, '1234', '', '', true) + await assignWorkspace(toolCtx, db, null, email, ws, AccountRole.User) + console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms') + const token = generateToken(systemAccountEmail, wsid) + const endpoint = await getTransactorEndpoint(token, 'external') + await generateWorkspaceData(endpoint, ws, cmd.parallel, email) + await testFindAll(endpoint, ws, email) + await dropWorkspace(toolCtx, db, null, ws) + }) + }) + extendProgram?.(program) program.parse(process.argv) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 67afdd44db..48f79b1131 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -14,8 +14,8 @@ import core, { makeCollaborativeDoc } from '@hcengineering/core' import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' -import { type StorageAdapter } from '@hcengineering/server-core' -import { connect, fetchModelFromMongo } from '@hcengineering/server-tool' +import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' +import { connect, fetchModel } from '@hcengineering/server-tool' import { jsonToText, markupToYDoc } from '@hcengineering/text' import { type Db, type FindCursor, type MongoClient } from 'mongodb' @@ -120,10 +120,10 @@ export async function migrateMarkup ( storageAdapter: StorageAdapter, workspaceId: WorkspaceId, client: MongoClient, - mongodbUri: string, + pipeline: Pipeline, concurrency: number ): Promise { - const { hierarchy } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId) + const { hierarchy } = await fetchModel(ctx, pipeline) const workspaceDb = client.db(workspaceId.name) diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index 668b6f5d8b..421bfb84f7 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -69,37 +69,41 @@ async function migrateAvatars (client: MigrationClient): Promise { _class: { $in: classes }, avatar: { $regex: 'color|gravatar://.*' } }) - while (true) { - const docs = await i.next(50) - if (docs === null || docs?.length === 0) { - break - } - const updates: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] - for (const d of docs) { - if (d.avatar?.startsWith(colorPrefix) ?? false) { - d.avatarProps = { color: d.avatar?.slice(colorPrefix.length) ?? '' } - updates.push({ - filter: { _id: d._id }, - update: { - avatarType: AvatarType.COLOR, - avatar: null, - avatarProps: { color: d.avatar?.slice(colorPrefix.length) ?? '' } - } - }) - } else if (d.avatar?.startsWith(gravatarPrefix) ?? false) { - updates.push({ - filter: { _id: d._id }, - update: { - avatarType: AvatarType.GRAVATAR, - avatar: null, - avatarProps: { url: d.avatar?.slice(gravatarPrefix.length) ?? '' } - } - }) + try { + while (true) { + const docs = await i.next(50) + if (docs === null || docs?.length === 0) { + break + } + const updates: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + for (const d of docs) { + if (d.avatar?.startsWith(colorPrefix) ?? false) { + d.avatarProps = { color: d.avatar?.slice(colorPrefix.length) ?? '' } + updates.push({ + filter: { _id: d._id }, + update: { + avatarType: AvatarType.COLOR, + avatar: null, + avatarProps: { color: d.avatar?.slice(colorPrefix.length) ?? '' } + } + }) + } else if (d.avatar?.startsWith(gravatarPrefix) ?? false) { + updates.push({ + filter: { _id: d._id }, + update: { + avatarType: AvatarType.GRAVATAR, + avatar: null, + avatarProps: { url: d.avatar?.slice(gravatarPrefix.length) ?? '' } + } + }) + } + } + if (updates.length > 0) { + await client.bulk(DOMAIN_CONTACT, updates) } } - if (updates.length > 0) { - await client.bulk(DOMAIN_CONTACT, updates) - } + } finally { + await i.close() } await client.update( diff --git a/models/core/src/index.ts b/models/core/src/index.ts index cad0f0b54b..f273a93208 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -20,6 +20,7 @@ import { DOMAIN_CONFIGURATION, DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, + DOMAIN_SPACE, DOMAIN_STATUS, DOMAIN_TRANSIENT, DOMAIN_TX, @@ -76,7 +77,6 @@ import { } from './core' import { definePermissions } from './permissions' import { - DOMAIN_SPACE, TAccount, TPermission, TRole, @@ -101,7 +101,7 @@ import { TTxWorkspaceEvent } from './tx' -export { coreId } from '@hcengineering/core' +export { coreId, DOMAIN_SPACE } from '@hcengineering/core' export * from './core' export { coreOperation } from './migration' export * from './security' diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 69d30408ca..00ad834ffb 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -17,6 +17,7 @@ import { saveCollaborativeDoc } from '@hcengineering/collaboration' import core, { DOMAIN_BLOB, DOMAIN_DOC_INDEX_STATE, + DOMAIN_SPACE, DOMAIN_STATUS, DOMAIN_TX, MeasureMetricsContext, @@ -49,7 +50,6 @@ import { } from '@hcengineering/model' import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/storage' import { markupToYDoc } from '@hcengineering/text' -import { DOMAIN_SPACE } from './security' async function migrateStatusesToModel (client: MigrationClient): Promise { // Move statuses to model: @@ -326,6 +326,16 @@ export const coreOperation: MigrateOperation = { core.class.TypedSpace ) } + }, + { + state: 'default-space', + func: async (client) => { + await createDefaultSpace(client, core.space.Tx, { name: 'Space for all txes' }) + await createDefaultSpace(client, core.space.DerivedTx, { name: 'Space for derived txes' }) + await createDefaultSpace(client, core.space.Model, { name: 'Space for model' }) + await createDefaultSpace(client, core.space.Configuration, { name: 'Space for config' }) + await createDefaultSpace(client, core.space.Workspace, { name: 'Space for common things' }) + } } ]) } diff --git a/models/core/src/security.ts b/models/core/src/security.ts index 2c038b6bda..ef0585d61d 100644 --- a/models/core/src/security.ts +++ b/models/core/src/security.ts @@ -15,13 +15,13 @@ import { DOMAIN_MODEL, + DOMAIN_SPACE, IndexKind, type Account, type AccountRole, type Arr, type Class, type CollectionSize, - type Domain, type Permission, type Ref, type Role, @@ -48,8 +48,6 @@ import { getEmbeddedLabel, type Asset, type IntlString } from '@hcengineering/pl import core from './component' import { TAttachedDoc, TDoc } from './core' -export const DOMAIN_SPACE = 'space' as Domain - // S P A C E @Model(core.class.Space, core.class.Doc, DOMAIN_SPACE) diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 96f9e88eda..b81656e2b2 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -318,6 +318,11 @@ export interface TypeAny extends Type { */ export const DOMAIN_MODEL = 'model' as Domain +/** + * @public + */ +export const DOMAIN_SPACE = 'space' as Domain + /** * @public */ diff --git a/packages/core/src/operator.ts b/packages/core/src/operator.ts index 711aca85f8..f929351ee6 100644 --- a/packages/core/src/operator.ts +++ b/packages/core/src/operator.ts @@ -165,6 +165,17 @@ function $unset (document: Doc, keyval: Record): void { } } +function $rename (document: Doc, keyval: Record): void { + const doc = document as any + for (const key in keyval) { + if (doc[key] !== undefined) { + doc[keyval[key]] = doc[key] + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete doc[key] + } + } +} + const operators: Record = { $push, $pull, @@ -172,7 +183,8 @@ const operators: Record = { $move, $pushMixin, $inc, - $unset + $unset, + $rename } /** diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 24a200ec23..b6c86d422f 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -15,7 +15,8 @@ import type { Account, Doc, Domain, Ref } from './classes' import { MeasureContext } from './measurements' -import type { Tx } from './tx' +import { DocumentQuery, FindOptions } from './storage' +import type { DocumentUpdate, Tx } from './tx' import type { WorkspaceIdWithUrl } from './utils' /** @@ -48,6 +49,8 @@ export interface SessionData { sessionId: string admin?: boolean + isTriggerCtx?: boolean + account: Account getAccount: (account: Ref) => Account | undefined @@ -76,6 +79,23 @@ export interface LowLevelStorage { // Low level direct group API groupBy: (ctx: MeasureContext, domain: Domain, field: string) => Promise> + + // migrations + rawFindAll: (domain: Domain, query: DocumentQuery, options?: FindOptions) => Promise + + rawUpdate: (domain: Domain, query: DocumentQuery, operations: DocumentUpdate) => Promise + + // Traverse documents + traverse: ( + domain: Domain, + query: DocumentQuery, + options?: Pick, 'sort' | 'limit' | 'projection'> + ) => Promise> +} + +export interface Iterator { + next: (count: number) => Promise + close: () => Promise } export interface Branding { diff --git a/packages/model/src/migration.ts b/packages/model/src/migration.ts index 070535147f..03f79a1443 100644 --- a/packages/model/src/migration.ts +++ b/packages/model/src/migration.ts @@ -80,7 +80,7 @@ export interface MigrationClient { traverse: ( domain: Domain, query: MigrationDocumentQuery, - options?: Omit, 'lookup'> + options?: Pick, 'sort' | 'limit' | 'projection'> ) => Promise> // Allow to raw update documents inside domain. @@ -88,15 +88,15 @@ export interface MigrationClient { domain: Domain, query: MigrationDocumentQuery, operations: MigrateUpdate - ) => Promise + ) => Promise bulk: ( domain: Domain, operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] - ) => Promise + ) => Promise // Move documents per domain - move: (sourceDomain: Domain, query: DocumentQuery, targetDomain: Domain) => Promise + move: (sourceDomain: Domain, query: DocumentQuery, targetDomain: Domain) => Promise create: (domain: Domain, doc: T | T[]) => Promise delete: (domain: Domain, _id: Ref) => Promise diff --git a/plugins/contact-resources/src/components/CreateEmployee.svelte b/plugins/contact-resources/src/components/CreateEmployee.svelte index 79f4dc0119..32472b7fa4 100644 --- a/plugins/contact-resources/src/components/CreateEmployee.svelte +++ b/plugins/contact-resources/src/components/CreateEmployee.svelte @@ -13,7 +13,15 @@ // limitations under the License. -->