Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>
This commit is contained in:
Denis Bykhov 2024-09-11 13:31:26 +05:00 committed by GitHub
parent 74c27d6dd7
commit bf1de1f436
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
62 changed files with 4187 additions and 632 deletions

View File

@ -190,6 +190,7 @@ jobs:
- name: Testing...
run: node common/scripts/install-run-rush.js test
env:
DB_URL: 'postgresql://postgres:example@localhost:5433'
ELASTIC_URL: 'http://localhost:9201'
MONGO_URL: 'mongodb://localhost:27018'
uitest:
@ -309,6 +310,71 @@ jobs:
# with:
# name: db-snapshot
# path: ./tests/db_dump
uitest-pg:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
filter: tree:0
- uses: actions/setup-node@v4
with:
node-version-file: '.nvmrc'
- name: Cache node modules
uses: actions/cache@v4
env:
cache-name: cache-node-platform
with:
path: |
common/temp
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
- name: Checking for mis-matching dependencies...
run: node common/scripts/install-run-rush.js check
- name: Installing...
run: node common/scripts/install-run-rush.js install
- name: Docker Build
run: node common/scripts/install-run-rush.js docker:build -p 20
env:
DOCKER_CLI_HINTS: false
- name: Prepare server
run: |
cd ./tests
export DO_CLEAN="true"
./prepare-pg.sh
- name: Install Playwright
run: |
cd ./tests/sanity
node ../../common/scripts/install-run-rushx.js ci
- name: Run UI tests
run: |
cd ./tests/sanity
node ../../common/scripts/install-run-rushx.js uitest
- name: 'Store docker logs'
if: always()
run: |
cd ./tests/sanity
mkdir logs
docker logs $(docker ps | grep transactor | cut -f 1 -d ' ') > logs/transactor.log
docker logs $(docker ps | grep account | cut -f 1 -d ' ') > logs/account.log
docker logs $(docker ps | grep front | cut -f 1 -d ' ') > logs/front.log
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-results-pg
path: ./tests/sanity/playwright-report/
- name: Upload Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: docker-logs-pg
path: ./tests/sanity/logs
uitest-qms:
runs-on: ubuntu-latest
timeout-minutes: 60

31
.vscode/launch.json vendored
View File

@ -59,15 +59,15 @@
"args": ["src/__start.ts"],
"env": {
"ELASTIC_URL": "http://localhost:9200",
"MONGO_URL": "mongodb://localhost:27017",
"MONGO_URL": "postgresql://postgres:example@localhost:5432;mongodb://localhost:27017",
"APM_SERVER_URL2": "http://localhost:8200",
"METRICS_CONSOLE": "false",
"METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds.,
"STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
"STORAGE_CONFIG": "minio|localhost:9000?accessKey=minioadmin&secretKey=minioadmin",
"SERVER_SECRET": "secret",
"ENABLE_CONSOLE": "true",
"COLLABORATOR_URL": "ws://localhost:3078",
"REKONI_URL": "http://localhost:4004",
"REKONI_URL": "http://localhost:4000",
"FRONT_URL": "http://localhost:8080",
"ACCOUNTS_URL": "http://localhost:3000",
// "SERVER_PROVIDER":"uweb"
@ -240,7 +240,29 @@
"name": "Debug tool upgrade",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "stress", "ws://localhost:3333", "wrong"],
"args": ["src/__start.ts", "create-workspace", "sanity-ws", "-w sanity-ws"],
"env": {
"SERVER_SECRET": "secret",
"MINIO_ACCESS_KEY": "minioadmin",
"MINIO_SECRET_KEY": "minioadmin",
"MINIO_ENDPOINT": "localhost:9000",
"TRANSACTOR_URL": "ws://localhost:3333",
"MONGO_URL": "mongodb://localhost:27017",
"ACCOUNTS_URL": "http://localhost:3000",
"TELEGRAM_DATABASE": "telegram-service",
"ELASTIC_URL": "http://localhost:9200",
"REKONI_URL": "http://localhost:4000"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"sourceMaps": true,
"outputCapture": "std",
"cwd": "${workspaceRoot}/dev/tool"
},
{
"name": "Debug tool move",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "move-to-pg"],
"env": {
"SERVER_SECRET": "secret",
"MINIO_ACCESS_KEY": "minioadmin",
@ -248,6 +270,7 @@
"MINIO_ENDPOINT": "localhost",
"TRANSACTOR_URL": "ws://localhost:3333",
"MONGO_URL": "mongodb://localhost:27017",
"DB_URL": "postgresql://postgres:example@localhost:5432",
"ACCOUNTS_URL": "http://localhost:3000",
"TELEGRAM_DATABASE": "telegram-service",
"ELASTIC_URL": "http://localhost:9200",

View File

@ -629,6 +629,9 @@ dependencies:
'@rush-temp/pod-workspace':
specifier: file:./projects/pod-workspace.tgz
version: file:projects/pod-workspace.tgz
'@rush-temp/postgres':
specifier: file:./projects/postgres.tgz
version: file:projects/postgres.tgz(esbuild@0.20.1)(ts-node@10.9.2)
'@rush-temp/preference':
specifier: file:./projects/preference.tgz
version: file:projects/preference.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
@ -1298,6 +1301,9 @@ dependencies:
'@types/pdfjs-dist':
specifier: 2.10.378
version: 2.10.378
'@types/pg':
specifier: ^8.11.6
version: 8.11.6
'@types/png-chunks-extract':
specifier: ^1.0.2
version: 1.0.2
@ -1703,6 +1709,9 @@ dependencies:
pdfjs-dist:
specifier: 2.12.313
version: 2.12.313
pg:
specifier: 8.12.0
version: 8.12.0
png-chunks-extract:
specifier: ^1.0.0
version: 1.0.0
@ -4690,7 +4699,7 @@ packages:
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
dependencies:
ajv: 6.12.6
debug: 4.3.5
debug: 4.3.4
espree: 9.6.1
globals: 13.24.0
ignore: 5.3.1
@ -4845,7 +4854,7 @@ packages:
engines: {node: '>=10.10.0'}
dependencies:
'@humanwhocodes/object-schema': 2.0.2
debug: 4.3.5
debug: 4.3.4
minimatch: 3.1.2
transitivePeerDependencies:
- supports-color
@ -9364,6 +9373,14 @@ packages:
- worker-loader
dev: false
/@types/pg@8.11.6:
resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==}
dependencies:
'@types/node': 20.11.19
pg-protocol: 1.6.1
pg-types: 4.0.2
dev: false
/@types/plist@3.0.5:
resolution: {integrity: sha512-E6OCaRmAe4WDmWNsL/9RMqdkkzDCY1etutkflWk4c+AcjDU07Pcz1fQwTX0TQz+Pxqn9i4L1TU3UFpjnrcDgxA==}
requiresBuild: true
@ -9728,7 +9745,7 @@ packages:
dependencies:
'@typescript-eslint/typescript-estree': 6.21.0(typescript@5.3.3)
'@typescript-eslint/utils': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
debug: 4.3.5
debug: 4.3.4
eslint: 8.56.0
ts-api-utils: 1.2.1(typescript@5.3.3)
typescript: 5.3.3
@ -9778,7 +9795,7 @@ packages:
dependencies:
'@typescript-eslint/types': 6.21.0
'@typescript-eslint/visitor-keys': 6.21.0
debug: 4.3.5
debug: 4.3.4
globby: 11.1.0
is-glob: 4.0.3
minimatch: 9.0.3
@ -10566,7 +10583,7 @@ packages:
builder-util: 24.13.1
builder-util-runtime: 9.2.4
chromium-pickle-js: 0.2.0
debug: 4.3.5
debug: 4.3.4
dmg-builder: 24.13.3
ejs: 3.1.9
electron-publish: 24.13.1
@ -11403,7 +11420,7 @@ packages:
resolution: {integrity: sha512-upp+biKpN/XZMLim7aguUyW8s0FUpDvOtK6sbanMFDAMBzpHDqdhgVYm6zc9HJ6nWo7u2Lxk60i2M6Jd3aiNrA==}
engines: {node: '>=12.0.0'}
dependencies:
debug: 4.3.5
debug: 4.3.4
sax: 1.3.0
transitivePeerDependencies:
- supports-color
@ -11419,7 +11436,7 @@ packages:
builder-util-runtime: 9.2.4
chalk: 4.1.2
cross-spawn: 7.0.3
debug: 4.3.5
debug: 4.3.4
fs-extra: 10.1.0
http-proxy-agent: 5.0.0
https-proxy-agent: 5.0.1
@ -13385,7 +13402,7 @@ packages:
has-property-descriptors: 1.0.2
has-proto: 1.0.3
has-symbols: 1.0.3
hasown: 2.0.2
hasown: 2.0.1
internal-slot: 1.0.7
is-array-buffer: 3.0.4
is-callable: 1.2.7
@ -13531,13 +13548,13 @@ packages:
dependencies:
get-intrinsic: 1.2.4
has-tostringtag: 1.0.2
hasown: 2.0.2
hasown: 2.0.1
dev: false
/es-shim-unscopables@1.0.2:
resolution: {integrity: sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==}
dependencies:
hasown: 2.0.2
hasown: 2.0.1
dev: false
/es-to-primitive@1.2.1:
@ -14139,7 +14156,7 @@ packages:
optionator: 0.9.3
progress: 2.0.3
regexpp: 3.2.0
semver: 7.6.0
semver: 7.6.3
strip-ansi: 6.0.1
strip-json-comments: 3.1.1
text-table: 0.2.0
@ -14850,7 +14867,7 @@ packages:
minimatch: 3.1.2
node-abort-controller: 3.1.1
schema-utils: 3.3.0
semver: 7.6.0
semver: 7.6.3
tapable: 2.2.1
typescript: 5.3.3
webpack: 5.90.3(@swc/core@1.4.2)(esbuild@0.20.1)(webpack-cli@5.1.4)
@ -15105,7 +15122,7 @@ packages:
function-bind: 1.1.2
has-proto: 1.0.3
has-symbols: 1.0.3
hasown: 2.0.2
hasown: 2.0.1
dev: false
/get-nonce@1.0.1:
@ -15868,7 +15885,7 @@ packages:
engines: {node: '>= 6'}
dependencies:
agent-base: 6.0.2
debug: 4.3.5
debug: 4.3.4
transitivePeerDependencies:
- supports-color
dev: false
@ -16037,7 +16054,7 @@ packages:
engines: {node: '>= 0.4'}
dependencies:
es-errors: 1.3.0
hasown: 2.0.2
hasown: 2.0.1
side-channel: 1.0.6
dev: false
@ -16979,7 +16996,7 @@ packages:
jest-util: 29.7.0
natural-compare: 1.4.0
pretty-format: 29.7.0
semver: 7.6.0
semver: 7.6.3
transitivePeerDependencies:
- supports-color
dev: false
@ -19490,6 +19507,86 @@ packages:
is-reference: 3.0.2
dev: false
/pg-cloudflare@1.1.1:
resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==}
requiresBuild: true
dev: false
optional: true
/pg-connection-string@2.6.4:
resolution: {integrity: sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==}
dev: false
/pg-int8@1.0.1:
resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==}
engines: {node: '>=4.0.0'}
dev: false
/pg-numeric@1.0.2:
resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==}
engines: {node: '>=4'}
dev: false
/pg-pool@3.6.2(pg@8.12.0):
resolution: {integrity: sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==}
peerDependencies:
pg: '>=8.0'
dependencies:
pg: 8.12.0
dev: false
/pg-protocol@1.6.1:
resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==}
dev: false
/pg-types@2.2.0:
resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==}
engines: {node: '>=4'}
dependencies:
pg-int8: 1.0.1
postgres-array: 2.0.0
postgres-bytea: 1.0.0
postgres-date: 1.0.7
postgres-interval: 1.2.0
dev: false
/pg-types@4.0.2:
resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==}
engines: {node: '>=10'}
dependencies:
pg-int8: 1.0.1
pg-numeric: 1.0.2
postgres-array: 3.0.2
postgres-bytea: 3.0.0
postgres-date: 2.1.0
postgres-interval: 3.0.0
postgres-range: 1.1.4
dev: false
/pg@8.12.0:
resolution: {integrity: sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==}
engines: {node: '>= 8.0.0'}
peerDependencies:
pg-native: '>=3.0.1'
peerDependenciesMeta:
pg-native:
optional: true
dependencies:
pg-connection-string: 2.6.4
pg-pool: 3.6.2(pg@8.12.0)
pg-protocol: 1.6.1
pg-types: 2.2.0
pgpass: 1.0.5
optionalDependencies:
pg-cloudflare: 1.1.1
dev: false
/pgpass@1.0.5:
resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==}
dependencies:
split2: 4.2.0
dev: false
/phin@2.9.3:
resolution: {integrity: sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==}
deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
@ -19782,6 +19879,54 @@ packages:
source-map-js: 1.0.2
dev: false
/postgres-array@2.0.0:
resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==}
engines: {node: '>=4'}
dev: false
/postgres-array@3.0.2:
resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==}
engines: {node: '>=12'}
dev: false
/postgres-bytea@1.0.0:
resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==}
engines: {node: '>=0.10.0'}
dev: false
/postgres-bytea@3.0.0:
resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==}
engines: {node: '>= 6'}
dependencies:
obuf: 1.1.2
dev: false
/postgres-date@1.0.7:
resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==}
engines: {node: '>=0.10.0'}
dev: false
/postgres-date@2.1.0:
resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==}
engines: {node: '>=12'}
dev: false
/postgres-interval@1.2.0:
resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==}
engines: {node: '>=0.10.0'}
dependencies:
xtend: 4.0.2
dev: false
/postgres-interval@3.0.0:
resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==}
engines: {node: '>=12'}
dev: false
/postgres-range@1.1.4:
resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==}
dev: false
/posthog-js@1.122.0:
resolution: {integrity: sha512-+8R2/nLaWyI5Jp2Ly7L52qcgDFU3xryyoNG52DPJ8dlGnagphxIc0mLNGurgyKeeTGycsOsuOIP4dtofv3ZoBA==}
deprecated: This version of posthog-js is deprecated, please update posthog-js, and do not use this version! Check out our JS docs at https://posthog.com/docs/libraries/js
@ -20646,7 +20791,7 @@ packages:
resolution: {integrity: sha512-efCx3b+0Z69/LGJmm9Yvi4cqEdxnoGnxYxGxBghkkTTFeXRtTCmmhO0AnAfHz59k957uTSuy8WaHqOs8wbYUWg==}
engines: {node: '>=6'}
dependencies:
debug: 4.3.5
debug: 4.3.4
module-details-from-path: 1.0.3
resolve: 1.22.8
transitivePeerDependencies:
@ -21479,6 +21624,11 @@ packages:
engines: {node: '>=6'}
dev: false
/split2@4.2.0:
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
engines: {node: '>= 10.x'}
dev: false
/sprintf-js@1.0.3:
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
dev: false
@ -30219,6 +30369,39 @@ packages:
- supports-color
dev: false
file:projects/postgres.tgz(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-qZVG4Pk9RAvQfkKRB1iQPPOWsKFvFuFyNBtD/ksT/eW0/ByyGABvYMeQPNenrh3ZH2n/hZ5lH5DON042MXebPg==, tarball: file:projects/postgres.tgz}
id: file:projects/postgres.tgz
name: '@rush-temp/postgres'
version: 0.0.0
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/pg': 8.11.6
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
pg: 8.12.0
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- pg-native
- supports-color
- ts-node
dev: false
file:projects/preference-assets.tgz(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-VlBSKBg3XmuMLtxNAS703aS+dhhb5a7H5Ns2nzhhv7w3KlAqtwp6cQ5VLxceNuRaPbTtI+2K+YkjFb2S1ld5VQ==, tarball: file:projects/preference-assets.tgz}
id: file:projects/preference-assets.tgz
@ -32536,7 +32719,7 @@ packages:
dev: false
file:projects/server-pipeline.tgz:
resolution: {integrity: sha512-LU5dxarK3XG4FXu2W/Wmu1IZh93faWFIdjKmurdgNZ9AV2Mv7B6fTALyVZoA0O7gysXFNDlxvH/qq7PyfAHivQ==, tarball: file:projects/server-pipeline.tgz}
resolution: {integrity: sha512-W5khmA6sUi5kU1UiRR7YTOYH40t96jeBa9ww0g1EUtraXSlO12g2NBpMXswt8/HkMVe6j/56Jj2vq8QVnXSY5w==, tarball: file:projects/server-pipeline.tgz}
name: '@rush-temp/server-pipeline'
version: 0.0.0
dependencies:
@ -33412,7 +33595,7 @@ packages:
dev: false
file:projects/server.tgz(esbuild@0.20.1):
resolution: {integrity: sha512-fTgDuks26uZ0IhbPq9N/65R1I/hQG/gRRygfBsTmwAQqcJfo25EMz9gfLsuxu6YgZIWgI+UU6wbphtNeWLW/2A==, tarball: file:projects/server.tgz}
resolution: {integrity: sha512-93PHAZJSkt0uDRAhNNcAGlJUHFM7/RLUAmzu2CZxcknO0hRM6QTDIkRg6T+GfE3fqyMuRsIJA0CDJdlziht/sA==, tarball: file:projects/server.tgz}
id: file:projects/server.tgz
name: '@rush-temp/server'
version: 0.0.0
@ -34612,7 +34795,7 @@ packages:
dev: false
file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-gjmLNkjPV0dXJNjkowQTvuR1ELM+S3aT3t/y9QJScPMxDIUPZRjkXIoM//4Qk3/RqUOFVhcRQTVoZVcTtK6rCg==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-LwQbmBaSOZ5IKwCHz2mULcIuEr9rZ2b/7tqUGICHCawUzexUlQVxv2Yt0oFf2aZu83Sittt7dZwnN3sXHX9t9g==, tarball: file:projects/tool.tgz}
id: file:projects/tool.tgz
name: '@rush-temp/tool'
version: 0.0.0
@ -35337,7 +35520,6 @@ packages:
koa: 2.15.3
koa-bodyparser: 4.4.1
koa-router: 12.0.1
mongodb: 6.8.0
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)

View File

@ -19,6 +19,16 @@ services:
ports:
- 27017:27017
restart: unless-stopped
postgres:
image: postgres
container_name: postgres
environment:
- POSTGRES_PASSWORD=example
volumes:
- db:/data/db
ports:
- 5432:5432
restart: unless-stopped
minio:
image: 'minio/minio'
command: server /data --address ":9000" --console-address ":9001"

View File

@ -79,10 +79,12 @@
"@hcengineering/model-task": "^0.6.0",
"@hcengineering/model-activity": "^0.6.0",
"@hcengineering/model-lead": "^0.6.0",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/recruit": "^0.6.29",
"@hcengineering/rekoni": "^0.6.0",
"@hcengineering/server-pipeline": "^0.6.0",
"@hcengineering/server-attachment": "^0.6.1",
"@hcengineering/server-attachment-resources": "^0.6.0",
"@hcengineering/server-collaboration": "^0.6.0",

View File

@ -73,6 +73,7 @@ addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources'
function prepareTools (): {
mongodbUri: string
dbUrl: string | undefined
txes: Tx[]
version: Data<Version>
migrateOperations: [string, MigrateOperation][]

View File

@ -14,11 +14,14 @@
//
import core, {
AccountRole,
MeasureMetricsContext,
RateLimiter,
TxOperations,
concatLink,
generateId,
getWorkspaceId,
makeCollaborativeDoc,
metricsToString,
newMetrics,
systemAccountEmail,
@ -40,6 +43,10 @@ import os from 'os'
import { Worker, isMainThread, parentPort } from 'worker_threads'
import { CSVWriter } from './csv'
import { AvatarType, type PersonAccount } from '@hcengineering/contact'
import contact from '@hcengineering/model-contact'
import recruit from '@hcengineering/model-recruit'
import { type Vacancy } from '@hcengineering/recruit'
import { WebSocket } from 'ws'
interface StartMessage {
@ -503,3 +510,117 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark
}
}
}
export async function testFindAll (endpoint: string, workspace: string, email: string): Promise<void> {
const connection = await connect(endpoint, getWorkspaceId(workspace), email)
try {
const client = new TxOperations(connection, core.account.System)
const start = Date.now()
const res = await client.findAll(
recruit.class.Applicant,
{},
{
lookup: {
attachedTo: recruit.mixin.Candidate,
space: recruit.class.Vacancy
}
}
)
console.log('Find all', res.length, 'time', Date.now() - start)
} finally {
await connection.close()
}
}
export async function generateWorkspaceData (
endpoint: string,
workspace: string,
parallel: boolean,
user: string
): Promise<void> {
const connection = await connect(endpoint, getWorkspaceId(workspace))
const client = new TxOperations(connection, core.account.System)
try {
const acc = await client.findOne(contact.class.PersonAccount, { email: user })
if (acc == null) {
throw new Error('User not found')
}
const employees: Ref<PersonAccount>[] = [acc._id]
const start = Date.now()
for (let i = 0; i < 100; i++) {
const acc = await generateEmployee(client)
employees.push(acc)
}
if (parallel) {
const promises: Promise<void>[] = []
for (let i = 0; i < 10; i++) {
promises.push(generateVacancy(client, employees))
}
await Promise.all(promises)
} else {
for (let i = 0; i < 10; i++) {
await generateVacancy(client, employees)
}
}
console.log('Generate', Date.now() - start)
} finally {
await connection.close()
}
}
export async function generateEmployee (client: TxOperations): Promise<Ref<PersonAccount>> {
const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, {
name: generateId().toString(),
city: '',
avatarType: AvatarType.COLOR
})
await client.createMixin(personId, contact.class.Person, contact.space.Contacts, contact.mixin.Employee, {
active: true
})
const acc = await client.createDoc(contact.class.PersonAccount, core.space.Model, {
person: personId,
role: AccountRole.User,
email: personId
})
return acc
}
async function generateVacancy (client: TxOperations, members: Ref<PersonAccount>[]): Promise<void> {
// generate vacancies
const _id = generateId<Vacancy>()
await client.createDoc(
recruit.class.Vacancy,
core.space.Space,
{
name: generateId().toString(),
number: 0,
fullDescription: makeCollaborativeDoc(_id, 'fullDescription'),
type: recruit.template.DefaultVacancy,
description: '',
private: false,
members,
archived: false
},
_id
)
for (let i = 0; i < 100; i++) {
// generate candidate
const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, {
name: generateId().toString(),
city: '',
avatarType: AvatarType.COLOR
})
await client.createMixin(personId, contact.class.Person, contact.space.Contacts, recruit.mixin.Candidate, {})
// generate applicants
await client.addCollection(recruit.class.Applicant, _id, personId, recruit.mixin.Candidate, 'applications', {
status: recruit.taskTypeStatus.Backlog,
number: i + 1,
identifier: `APP-${i + 1}`,
assignee: null,
rank: '',
startDate: null,
dueDate: null,
kind: recruit.taskTypes.Applicant
})
}
}

68
dev/tool/src/db.ts Normal file
View File

@ -0,0 +1,68 @@
import { type Doc, type WorkspaceId } from '@hcengineering/core'
import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres'
export async function moveFromMongoToPG (
mongoUrl: string,
dbUrl: string | undefined,
workspaces: WorkspaceId[]
): Promise<void> {
if (dbUrl === undefined) {
throw new Error('dbUrl is required')
}
const client = getMongoClient(mongoUrl)
const mongo = await client.getClient()
const pg = getDBClient(dbUrl)
const pgClient = await pg.getClient()
for (let index = 0; index < workspaces.length; index++) {
const ws = workspaces[index]
try {
const mongoDB = getWorkspaceDB(mongo, ws)
const collections = await mongoDB.collections()
await createTable(
pgClient,
collections.map((c) => c.collectionName)
)
for (const collection of collections) {
const cursor = collection.find()
const domain = translateDomain(collection.collectionName)
while (true) {
const doc = (await cursor.next()) as Doc | null
if (doc === null) break
try {
const converted = convertDoc(doc, ws.name)
await retryTxn(pgClient, async (client) => {
await client.query(
`INSERT INTO ${domain} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
[
converted._id,
converted.workspaceId,
converted._class,
converted.createdBy,
converted.modifiedBy,
converted.modifiedOn,
converted.createdOn,
converted.space,
converted.attachedTo,
converted.data
]
)
})
} catch (err) {
console.log('error when move doc', doc._id, doc._class, err)
continue
}
}
}
if (index % 100 === 0) {
console.log('Move workspace', index, workspaces.length)
}
} catch (err) {
console.log('Error when move workspace', ws.name, err)
throw err
}
}
pg.close()
client.close()
}

View File

@ -19,6 +19,7 @@ import accountPlugin, {
assignWorkspace,
confirmEmail,
createAcc,
createWorkspace as createWorkspaceRecord,
dropAccount,
dropWorkspace,
dropWorkspaceFull,
@ -32,10 +33,8 @@ import accountPlugin, {
setAccountAdmin,
setRole,
updateWorkspace,
createWorkspace as createWorkspaceRecord,
type Workspace
} from '@hcengineering/account'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
import { setMetadata } from '@hcengineering/platform'
import {
backup,
@ -54,8 +53,10 @@ import serverClientPlugin, {
login,
selectWorkspace
} from '@hcengineering/server-client'
import { getServerPipeline } from '@hcengineering/server-pipeline'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
import path from 'path'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
@ -66,6 +67,8 @@ import { diffWorkspace, recreateElastic, updateField } from './workspace'
import core, {
AccountRole,
concatLink,
generateId,
getWorkspaceId,
MeasureMetricsContext,
metricsToString,
@ -79,7 +82,7 @@ import core, {
type Tx,
type Version,
type WorkspaceId,
concatLink
type WorkspaceIdWithUrl
} from '@hcengineering/core'
import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
import contact from '@hcengineering/model-contact'
@ -87,7 +90,14 @@ import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
import type { StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
import { deepEqual } from 'fast-equals'
import { createWriteStream, readFileSync } from 'fs'
import { benchmark, benchmarkWorker, stressBenchmark, type StressBenchmarkMode } from './benchmark'
import {
benchmark,
benchmarkWorker,
generateWorkspaceData,
stressBenchmark,
testFindAll,
type StressBenchmarkMode
} from './benchmark'
import {
cleanArchivedSpaces,
cleanRemovedTransactions,
@ -101,11 +111,12 @@ import {
restoreRecruitingTaskTypes
} from './clean'
import { changeConfiguration } from './configuration'
import { moveFromMongoToPG } from './db'
import { fixJsonMarkup, migrateMarkup } from './markup'
import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
import { importNotion } from './notion'
import { fixAccountEmails, renameAccount } from './renameAccount'
import { moveFiles, syncFiles } from './storage'
import { importNotion } from './notion'
const colorConstants = {
colorRed: '\u001b[31m',
@ -125,6 +136,7 @@ const colorConstants = {
export function devTool (
prepareTools: () => {
mongodbUri: string
dbUrl: string | undefined
txes: Tx[]
version: Data<Version>
migrateOperations: [string, MigrateOperation][]
@ -1470,7 +1482,7 @@ export function devTool (
.option('-w, --workspace <workspace>', 'Selected workspace only', '')
.option('-c, --concurrency <concurrency>', 'Number of documents being processed concurrently', '10')
.action(async (cmd: { workspace: string, concurrency: string }) => {
const { mongodbUri } = prepareTools()
const { mongodbUri, dbUrl } = prepareTools()
await withDatabase(mongodbUri, async (db, client) => {
await withStorage(mongodbUri, async (adapter) => {
const workspaces = await listWorkspacesPure(db)
@ -1482,8 +1494,15 @@ export function devTool (
const wsId = getWorkspaceId(workspace.workspace)
console.log('processing workspace', workspace.workspace, index, workspaces.length)
const wsUrl: WorkspaceIdWithUrl = {
name: workspace.workspace,
workspaceName: workspace.workspaceName ?? '',
workspaceUrl: workspace.workspaceUrl ?? ''
}
await migrateMarkup(toolCtx, adapter, wsId, client, mongodbUri, parseInt(cmd.concurrency))
const { pipeline } = await getServerPipeline(toolCtx, mongodbUri, dbUrl, wsUrl)
await migrateMarkup(toolCtx, adapter, wsId, client, pipeline, parseInt(cmd.concurrency))
console.log('...done', workspace.workspace)
index++
@ -1502,6 +1521,57 @@ export function devTool (
})
})
program.command('move-to-pg').action(async () => {
const { mongodbUri, dbUrl } = prepareTools()
await withDatabase(mongodbUri, async (db) => {
const workspaces = await listWorkspacesRaw(db)
await moveFromMongoToPG(
mongodbUri,
dbUrl,
workspaces.map((it) => getWorkspaceId(it.workspace))
)
})
})
program
.command('perfomance')
.option('-p, --parallel', '', false)
.action(async (cmd: { parallel: boolean }) => {
const { mongodbUri, txes, version, migrateOperations } = prepareTools()
await withDatabase(mongodbUri, async (db) => {
const email = generateId()
const ws = generateId()
const wsid = getWorkspaceId(ws)
const start = new Date()
const measureCtx = new MeasureMetricsContext('create-workspace', {})
const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws)
// update the record so it's not taken by one of the workers for the next 60 seconds
await updateWorkspace(db, wsInfo, {
mode: 'creating',
progress: 0,
lastProcessingTime: Date.now() + 1000 * 60
})
await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations)
await updateWorkspace(db, wsInfo, {
mode: 'active',
progress: 100,
disabled: false,
version
})
await createAcc(toolCtx, db, null, email, '1234', '', '', true)
await assignWorkspace(toolCtx, db, null, email, ws, AccountRole.User)
console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms')
const token = generateToken(systemAccountEmail, wsid)
const endpoint = await getTransactorEndpoint(token, 'external')
await generateWorkspaceData(endpoint, ws, cmd.parallel, email)
await testFindAll(endpoint, ws, email)
await dropWorkspace(toolCtx, db, null, ws)
})
})
extendProgram?.(program)
program.parse(process.argv)

View File

@ -14,8 +14,8 @@ import core, {
makeCollaborativeDoc
} from '@hcengineering/core'
import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
import { type StorageAdapter } from '@hcengineering/server-core'
import { connect, fetchModelFromMongo } from '@hcengineering/server-tool'
import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
import { connect, fetchModel } from '@hcengineering/server-tool'
import { jsonToText, markupToYDoc } from '@hcengineering/text'
import { type Db, type FindCursor, type MongoClient } from 'mongodb'
@ -120,10 +120,10 @@ export async function migrateMarkup (
storageAdapter: StorageAdapter,
workspaceId: WorkspaceId,
client: MongoClient,
mongodbUri: string,
pipeline: Pipeline,
concurrency: number
): Promise<void> {
const { hierarchy } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
const { hierarchy } = await fetchModel(ctx, pipeline)
const workspaceDb = client.db(workspaceId.name)

View File

@ -69,6 +69,7 @@ async function migrateAvatars (client: MigrationClient): Promise<void> {
_class: { $in: classes },
avatar: { $regex: 'color|gravatar://.*' }
})
try {
while (true) {
const docs = await i.next(50)
if (docs === null || docs?.length === 0) {
@ -101,6 +102,9 @@ async function migrateAvatars (client: MigrationClient): Promise<void> {
await client.bulk(DOMAIN_CONTACT, updates)
}
}
} finally {
await i.close()
}
await client.update(
DOMAIN_CONTACT,

View File

@ -20,6 +20,7 @@ import {
DOMAIN_CONFIGURATION,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION,
DOMAIN_SPACE,
DOMAIN_STATUS,
DOMAIN_TRANSIENT,
DOMAIN_TX,
@ -76,7 +77,6 @@ import {
} from './core'
import { definePermissions } from './permissions'
import {
DOMAIN_SPACE,
TAccount,
TPermission,
TRole,
@ -101,7 +101,7 @@ import {
TTxWorkspaceEvent
} from './tx'
export { coreId } from '@hcengineering/core'
export { coreId, DOMAIN_SPACE } from '@hcengineering/core'
export * from './core'
export { coreOperation } from './migration'
export * from './security'

View File

@ -17,6 +17,7 @@ import { saveCollaborativeDoc } from '@hcengineering/collaboration'
import core, {
DOMAIN_BLOB,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_SPACE,
DOMAIN_STATUS,
DOMAIN_TX,
MeasureMetricsContext,
@ -49,7 +50,6 @@ import {
} from '@hcengineering/model'
import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/storage'
import { markupToYDoc } from '@hcengineering/text'
import { DOMAIN_SPACE } from './security'
async function migrateStatusesToModel (client: MigrationClient): Promise<void> {
// Move statuses to model:
@ -326,6 +326,16 @@ export const coreOperation: MigrateOperation = {
core.class.TypedSpace
)
}
},
{
state: 'default-space',
func: async (client) => {
await createDefaultSpace(client, core.space.Tx, { name: 'Space for all txes' })
await createDefaultSpace(client, core.space.DerivedTx, { name: 'Space for derived txes' })
await createDefaultSpace(client, core.space.Model, { name: 'Space for model' })
await createDefaultSpace(client, core.space.Configuration, { name: 'Space for config' })
await createDefaultSpace(client, core.space.Workspace, { name: 'Space for common things' })
}
}
])
}

View File

@ -15,13 +15,13 @@
import {
DOMAIN_MODEL,
DOMAIN_SPACE,
IndexKind,
type Account,
type AccountRole,
type Arr,
type Class,
type CollectionSize,
type Domain,
type Permission,
type Ref,
type Role,
@ -48,8 +48,6 @@ import { getEmbeddedLabel, type Asset, type IntlString } from '@hcengineering/pl
import core from './component'
import { TAttachedDoc, TDoc } from './core'
export const DOMAIN_SPACE = 'space' as Domain
// S P A C E
@Model(core.class.Space, core.class.Doc, DOMAIN_SPACE)

View File

@ -318,6 +318,11 @@ export interface TypeAny<AnyComponent = any> extends Type<any> {
*/
export const DOMAIN_MODEL = 'model' as Domain
/**
* @public
*/
export const DOMAIN_SPACE = 'space' as Domain
/**
* @public
*/

View File

@ -165,6 +165,17 @@ function $unset (document: Doc, keyval: Record<string, PropertyType>): void {
}
}
function $rename (document: Doc, keyval: Record<string, string>): void {
const doc = document as any
for (const key in keyval) {
if (doc[key] !== undefined) {
doc[keyval[key]] = doc[key]
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete doc[key]
}
}
}
const operators: Record<string, _OperatorFunc> = {
$push,
$pull,
@ -172,7 +183,8 @@ const operators: Record<string, _OperatorFunc> = {
$move,
$pushMixin,
$inc,
$unset
$unset,
$rename
}
/**

View File

@ -15,7 +15,8 @@
import type { Account, Doc, Domain, Ref } from './classes'
import { MeasureContext } from './measurements'
import type { Tx } from './tx'
import { DocumentQuery, FindOptions } from './storage'
import type { DocumentUpdate, Tx } from './tx'
import type { WorkspaceIdWithUrl } from './utils'
/**
@ -48,6 +49,8 @@ export interface SessionData {
sessionId: string
admin?: boolean
isTriggerCtx?: boolean
account: Account
getAccount: (account: Ref<Account>) => Account | undefined
@ -76,6 +79,23 @@ export interface LowLevelStorage {
// Low level direct group API
groupBy: <T>(ctx: MeasureContext, domain: Domain, field: string) => Promise<Set<T>>
// migrations
rawFindAll: <T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>) => Promise<T[]>
rawUpdate: <T extends Doc>(domain: Domain, query: DocumentQuery<T>, operations: DocumentUpdate<T>) => Promise<void>
// Traverse documents
traverse: <T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
) => Promise<Iterator<T>>
}
export interface Iterator<T extends Doc> {
next: (count: number) => Promise<T[] | null>
close: () => Promise<void>
}
export interface Branding {

View File

@ -80,7 +80,7 @@ export interface MigrationClient {
traverse: <T extends Doc>(
domain: Domain,
query: MigrationDocumentQuery<T>,
options?: Omit<FindOptions<T>, 'lookup'>
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
) => Promise<MigrationIterator<T>>
// Allow to raw update documents inside domain.
@ -88,15 +88,15 @@ export interface MigrationClient {
domain: Domain,
query: MigrationDocumentQuery<T>,
operations: MigrateUpdate<T>
) => Promise<MigrationResult>
) => Promise<void>
bulk: <T extends Doc>(
domain: Domain,
operations: { filter: MigrationDocumentQuery<T>, update: MigrateUpdate<T> }[]
) => Promise<MigrationResult>
) => Promise<void>
// Move documents per domain
move: <T extends Doc>(sourceDomain: Domain, query: DocumentQuery<T>, targetDomain: Domain) => Promise<MigrationResult>
move: <T extends Doc>(sourceDomain: Domain, query: DocumentQuery<T>, targetDomain: Domain) => Promise<void>
create: <T extends Doc>(domain: Domain, doc: T | T[]) => Promise<void>
delete: <T extends Doc>(domain: Domain, _id: Ref<T>) => Promise<void>

View File

@ -13,7 +13,15 @@
// limitations under the License.
-->
<script lang="ts">
import { AvatarType, Channel, combineName, ContactEvents, Employee, PersonAccount } from '@hcengineering/contact'
import {
AvatarType,
Channel,
combineName,
ContactEvents,
Employee,
Person,
PersonAccount
} from '@hcengineering/contact'
import core, { AccountRole, AttachedData, Data, generateId, Ref } from '@hcengineering/core'
import login from '@hcengineering/login'
import { getResource } from '@hcengineering/platform'
@ -42,10 +50,9 @@
let saving: boolean = false
const person: Data<Employee> = {
const person: Data<Person> = {
name: '',
city: '',
active: true,
avatarType: AvatarType.COLOR
}

View File

@ -1,5 +1,6 @@
// Copyright © 2022 Hardcore Engineering Inc.
import { Analytics } from '@hcengineering/analytics'
import core, {
type Class,
type Data,
@ -10,19 +11,12 @@ import core, {
} from '@hcengineering/core'
import { type Asset } from '@hcengineering/platform'
import { getClient } from '@hcengineering/presentation'
import {
type InitialKnowledge,
type TagCategory,
type TagElement,
type TagReference,
TagsEvents
} from '@hcengineering/tags'
import { type TagCategory, type TagElement, type TagReference, TagsEvents } from '@hcengineering/tags'
import { type ColorDefinition, getColorNumberByText } from '@hcengineering/ui'
import { type Filter } from '@hcengineering/view'
import { FilterQuery } from '@hcengineering/view-resources'
import { writable } from 'svelte/store'
import tags from './plugin'
import { Analytics } from '@hcengineering/analytics'
export function getTagStyle (color: ColorDefinition, selected = false): string {
return `
@ -37,11 +31,10 @@ export async function getRefs (filter: Filter, onUpdate: () => void): Promise<Ar
const promise = new Promise<Array<Ref<Doc>>>((resolve, reject) => {
const level = filter.props?.level ?? 0
const q: DocumentQuery<TagReference> = {
tag: { $in: filter.value },
weight:
level === 0
? { $in: [null as unknown as InitialKnowledge, 0, 1, 2, 3, 4, 5, 6, 7, 8] }
: { $gte: level as TagReference['weight'] }
tag: { $in: filter.value }
}
if (level > 0) {
q.weight = { $gte: level as TagReference['weight'] }
}
const refresh = lq.query(tags.class.TagReference, q, (refs: FindResult<TagReference>) => {
const result = Array.from(new Set(refs.map((p) => p.attachedTo)))

View File

@ -15,7 +15,7 @@
"_phase:bundle": "rushx bundle",
"_phase:docker-build": "rushx docker:build",
"_phase:docker-staging": "rushx docker:staging",
"bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --bundle --keep-names --platform=node --external:*.node --external:bufferutil --external:snappy --external:utf-8-validate --external:msgpackr-extract --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
"bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --bundle --keep-names --platform=node --external:*.node --external:bufferutil --external:snappy --external:utf-8-validate --external:msgpackr-extract --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/transactor",
"docker:tbuild": "docker build -t hardcoreeng/transactor . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor",
"docker:abuild": "docker build -t hardcoreeng/transactor . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor",

View File

@ -31,7 +31,7 @@ registerStringLoaders()
* @public
*/
export function start (
dbUrl: string,
dbUrls: string,
opt: {
fullTextUrl: string
storageConfig: StorageConfiguration
@ -57,18 +57,20 @@ export function start (
registerServerPlugins()
const externalStorage = buildStorageFromConfig(opt.storageConfig, dbUrl)
const [mainDbUrl, rawDbUrl] = dbUrls.split(';')
const externalStorage = buildStorageFromConfig(opt.storageConfig, rawDbUrl ?? mainDbUrl)
const pipelineFactory = createServerPipeline(
metrics,
dbUrl,
{ ...opt, externalStorage },
dbUrls,
{ ...opt, externalStorage, adapterSecurity: rawDbUrl !== undefined },
{
serviceAdapters: {
[serverAiBotId]: {
factory: createAIBotAdapter,
db: '%ai-bot',
url: dbUrl
url: rawDbUrl ?? mainDbUrl
}
}
}

View File

@ -836,6 +836,11 @@
"projectFolder": "server/mongo",
"shouldPublish": false
},
{
"packageName": "@hcengineering/postgres",
"projectFolder": "server/postgres",
"shouldPublish": false
},
{
"packageName": "@hcengineering/elastic",
"projectFolder": "server/elastic",

View File

@ -35,11 +35,12 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap
console.log('Starting account service with brandings: ', brandings)
const methods = getMethods()
const ACCOUNT_PORT = parseInt(process.env.ACCOUNT_PORT ?? '3000')
const dbUri = process.env.MONGO_URL
if (dbUri === undefined) {
console.log('Please provide mongodb url')
const dbUrls = process.env.MONGO_URL
if (dbUrls === undefined) {
console.log('Please provide db url')
process.exit(1)
}
const [dbUrl, mongoUrl] = dbUrls.split(';')
const transactorUri = process.env.TRANSACTOR_URL
if (transactorUri === undefined) {
@ -89,7 +90,7 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap
}
setMetadata(serverClientPlugin.metadata.UserAgent, 'AccountService')
const client: MongoClientReference = getMongoClient(dbUri)
const client: MongoClientReference = getMongoClient(mongoUrl ?? dbUrl)
let _client: MongoClient | Promise<MongoClient> = client.getClient()
const app = new Koa()

View File

@ -33,6 +33,7 @@ import core, {
generateId,
getWorkspaceId,
groupByArray,
isWorkspaceCreating,
MeasureContext,
RateLimiter,
Ref,
@ -42,13 +43,11 @@ import core, {
TxOperations,
Version,
versionToString,
isWorkspaceCreating,
WorkspaceId,
type Branding,
type WorkspaceMode
} from '@hcengineering/core'
import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform'
import { type StorageAdapter } from '@hcengineering/server-core'
import { decodeToken as decodeTokenRaw, generateToken, type Token } from '@hcengineering/server-token'
import toolPlugin, { connect } from '@hcengineering/server-tool'

View File

@ -14,6 +14,7 @@
//
import {
type LowLevelStorage,
type Class,
type Doc,
type DocumentQuery,
@ -26,7 +27,6 @@ import {
type MeasureContext,
type ModelDb,
type Ref,
type StorageIterator,
type Tx,
type TxResult,
type WorkspaceId
@ -36,7 +36,7 @@ import type { ServerFindOptions } from './types'
export interface DomainHelperOperations {
create: (domain: Domain) => Promise<void>
exists: (domain: Domain) => boolean
exists: (domain: Domain) => Promise<boolean>
listDomains: () => Promise<Set<Domain>>
createIndex: (domain: Domain, value: string | FieldIndexConfig<Doc>, options?: { name: string }) => Promise<void>
@ -99,8 +99,8 @@ export type DbAdapterHandler = (
/**
* @public
*/
export interface DbAdapter {
init?: () => Promise<void>
export interface DbAdapter extends LowLevelStorage {
init?: (domains?: string[], excludeDomains?: string[]) => Promise<void>
helper: () => DomainHelperOperations
@ -114,14 +114,6 @@ export interface DbAdapter {
tx: (ctx: MeasureContext, ...tx: Tx[]) => Promise<TxResult[]>
find: (ctx: MeasureContext, domain: Domain, recheck?: boolean) => StorageIterator
load: (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]) => Promise<Doc[]>
upload: (ctx: MeasureContext, domain: Domain, docs: Doc[]) => Promise<void>
clean: (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
groupBy: <T>(ctx: MeasureContext, domain: Domain, field: string) => Promise<Set<T>>
// Bulk update operations
update: (ctx: MeasureContext, domain: Domain, operations: Map<Ref<Doc>, DocumentUpdate<Doc>>) => Promise<void>

View File

@ -93,7 +93,7 @@ export class DomainIndexHelperImpl implements DomainHelper {
const added = new Set<string>()
try {
if (!operations.exists(domain)) {
if (!(await operations.exists(domain))) {
return
}
const has50Documents = documents > 50

View File

@ -23,6 +23,7 @@ import core, {
type FindResult,
type Hierarchy,
type IndexingConfiguration,
type Iterator,
type MeasureContext,
ModelDb,
type Ref,
@ -32,12 +33,25 @@ import core, {
type TxResult,
type WorkspaceId
} from '@hcengineering/core'
import { type DbAdapter, type DomainHelperOperations } from './adapter'
import { type DbAdapterHandler, type DbAdapter, type DomainHelperOperations } from './adapter'
/**
* @public
*/
export class DummyDbAdapter implements DbAdapter {
on?: ((handler: DbAdapterHandler) => void) | undefined
async traverse<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
): Promise<Iterator<T>> {
return {
next: async () => [],
close: async () => {}
}
}
async findAll<T extends Doc>(
ctx: MeasureContext,
_class: Ref<Class<T>>,
@ -52,7 +66,7 @@ export class DummyDbAdapter implements DbAdapter {
helper (): DomainHelperOperations {
return {
create: async () => {},
exists: () => true,
exists: async () => true,
listDomains: async () => new Set(),
createIndex: async () => {},
dropIndex: async () => {},
@ -90,6 +104,16 @@ export class DummyDbAdapter implements DbAdapter {
async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
return new Set()
}
async rawFindAll<T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<T[]> {
return []
}
async rawUpdate<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
operations: DocumentUpdate<T>
): Promise<void> {}
}
class InMemoryAdapter extends DummyDbAdapter implements DbAdapter {

View File

@ -24,6 +24,7 @@ import {
FindResult,
Hierarchy,
IndexingConfiguration,
Iterator,
MeasureContext,
Ref,
StorageIterator,
@ -33,7 +34,7 @@ import {
WorkspaceId
} from '@hcengineering/core'
import { getMetadata } from '@hcengineering/platform'
import serverCore, { DbAdapter, type DomainHelperOperations } from '@hcengineering/server-core'
import serverCore, { DbAdapter, DbAdapterHandler, type DomainHelperOperations } from '@hcengineering/server-core'
function getIndexName (): string {
return getMetadata(serverCore.metadata.ElasticIndexName) ?? 'storage_index'
@ -61,10 +62,21 @@ class ElasticDataAdapter implements DbAdapter {
this.getDocId = (fulltext) => fulltext.slice(0, -1 * (this.workspaceString.length + 1)) as Ref<Doc>
}
init?: ((domains?: string[], excludeDomains?: string[]) => Promise<void>) | undefined
on?: ((handler: DbAdapterHandler) => void) | undefined
async traverse<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
): Promise<Iterator<T>> {
throw new Error('Method not implemented.')
}
helper (): DomainHelperOperations {
return {
create: async () => {},
exists: () => true,
exists: async () => true,
listDomains: async () => new Set(),
createIndex: async () => {},
dropIndex: async () => {},
@ -118,6 +130,18 @@ class ElasticDataAdapter implements DbAdapter {
throw new Error('Method not implemented.')
}
async rawFindAll<T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<T[]> {
throw new Error('Method not implemented.')
}
async rawUpdate<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
operations: DocumentUpdate<T>
): Promise<void> {
throw new Error('Method not implemented.')
}
async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
const indexExists = await this.client.indices.exists({
index: this.indexName

View File

@ -13,14 +13,15 @@
// limitations under the License.
//
import { type MeasureContext } from '@hcengineering/core'
import { DOMAIN_TX, type MeasureContext } from '@hcengineering/core'
import { PlatformError, unknownStatus } from '@hcengineering/platform'
import type {
DbAdapter,
DbConfiguration,
Middleware,
MiddlewareCreator,
PipelineContext
PipelineContext,
TxAdapter
} from '@hcengineering/server-core'
import { BaseMiddleware, createServiceAdaptersManager, DbAdapterManagerImpl } from '@hcengineering/server-core'
@ -67,11 +68,51 @@ export class DBAdapterMiddleware extends BaseMiddleware implements Middleware {
}
})
const txAdapterName = this.conf.domains[DOMAIN_TX]
const txAdapter = adapters.get(txAdapterName) as TxAdapter
const txAdapterDomains: string[] = []
for (const key in this.conf.domains) {
if (this.conf.domains[key] === txAdapterName) {
txAdapterDomains.push(key)
}
}
await txAdapter.init?.(txAdapterDomains)
const model = await txAdapter.getModel(ctx)
for (const tx of model) {
try {
this.context.hierarchy.tx(tx)
} catch (err: any) {
ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err })
}
}
await ctx.with('init-adapters', {}, async (ctx) => {
for (const adapter of adapters.values()) {
await adapter.init?.()
for (const [key, adapter] of adapters) {
// already initialized
if (key !== this.conf.domains[DOMAIN_TX] && adapter.init !== undefined) {
let excludeDomains: string[] | undefined
let domains: string[] | undefined
if (this.conf.defaultAdapter === key) {
excludeDomains = []
for (const domain in this.conf.domains) {
if (this.conf.domains[domain] !== key) {
excludeDomains.push(domain)
}
}
} else {
domains = []
for (const domain in this.conf.domains) {
if (this.conf.domains[domain] === key) {
domains.push(domain)
}
}
}
await adapter.init(domains, excludeDomains)
}
}
})
const metrics = this.conf.metrics.newChild('📔 server-storage', {})
const defaultAdapter = adapters.get(this.conf.defaultAdapter)

View File

@ -13,7 +13,17 @@
// limitations under the License.
//
import { type Doc, type Domain, type MeasureContext, type Ref, type StorageIterator } from '@hcengineering/core'
import {
DocumentQuery,
DocumentUpdate,
FindOptions,
type Doc,
type Domain,
type MeasureContext,
type Ref,
type StorageIterator,
type Iterator
} from '@hcengineering/core'
import { PlatformError, unknownStatus } from '@hcengineering/platform'
import type { Middleware, PipelineContext } from '@hcengineering/server-core'
import { BaseMiddleware } from '@hcengineering/server-core'
@ -48,8 +58,25 @@ export class LowLevelMiddleware extends BaseMiddleware implements Middleware {
async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
await adapterManager.getAdapter(domain, true).clean(ctx, domain, docs)
},
async groupBy (ctx, domain, field) {
async groupBy<T>(ctx: MeasureContext, domain: Domain, field: string): Promise<Set<T>> {
return await adapterManager.getAdapter(domain, false).groupBy(ctx, domain, field)
},
async rawFindAll<T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<T[]> {
return await adapterManager.getAdapter(domain, false).rawFindAll(domain, query, options)
},
async rawUpdate<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
operations: DocumentUpdate<T>
): Promise<void> {
await adapterManager.getAdapter(domain, true).rawUpdate(domain, query, operations)
},
async traverse<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
): Promise<Iterator<T>> {
return await adapterManager.getAdapter(domain, false).traverse(domain, query, options)
}
}
return undefined

View File

@ -73,12 +73,21 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
core.space.Tx
]
private constructor (
private readonly skipFindCheck: boolean,
context: PipelineContext,
next?: Middleware
) {
super(context, next)
}
static async create (
skipFindCheck: boolean,
ctx: MeasureContext,
context: PipelineContext,
next: Middleware | undefined
): Promise<SpaceSecurityMiddleware> {
return new SpaceSecurityMiddleware(context, next)
return new SpaceSecurityMiddleware(skipFindCheck, context, next)
}
private resyncDomains (): void {
@ -496,7 +505,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
let clientFilterSpaces: Set<Ref<Space>> | undefined
if (!isSystem(account) && account.role !== AccountRole.DocGuest && domain !== DOMAIN_MODEL) {
if (!this.skipFindCheck && !isSystem(account) && account.role !== AccountRole.DocGuest && domain !== DOMAIN_MODEL) {
if (!isOwner(account, ctx) || !isSpace) {
if (query[field] !== undefined) {
const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace)

View File

@ -87,6 +87,9 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware {
const findAll: SessionFindAll = async (ctx, _class, query, options) => {
const _ctx: MeasureContext = (options as ServerFindOptions<Doc>)?.ctx ?? ctx
delete (options as ServerFindOptions<Doc>)?.ctx
if (_ctx.contextData !== undefined) {
_ctx.contextData.isTriggerCtx = true
}
const results = await this.findAll(_ctx, _class, query, options)
return toFindResult(

View File

@ -16,6 +16,7 @@
import core, {
DOMAIN_MODEL,
DOMAIN_TX,
type Iterator,
SortingOrder,
TxProcessor,
addOperation,
@ -162,6 +163,103 @@ abstract class MongoAdapterBase implements DbAdapter {
this._db = new DBCollectionHelper(db)
}
async traverse<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
): Promise<Iterator<T>> {
let cursor = this.db.collection(domain).find<T>(this.translateRawQuery(query))
if (options?.limit !== undefined) {
cursor = cursor.limit(options.limit)
}
if (options !== null && options !== undefined) {
if (options.sort !== undefined) {
const sort: Sort = {}
for (const key in options.sort) {
const order = options.sort[key] === SortingOrder.Ascending ? 1 : -1
sort[key] = order
}
cursor = cursor.sort(sort)
}
}
return {
next: async (size: number) => {
const docs: T[] = []
while (docs.length < size && (await cursor.hasNext())) {
try {
const d = await cursor.next()
if (d !== null) {
docs.push(d)
} else {
break
}
} catch (err) {
console.error(err)
return null
}
}
return docs
},
close: async () => {
await cursor.close()
}
}
}
private translateRawQuery<T extends Doc>(query: DocumentQuery<T>): Filter<Document> {
const translated: any = {}
for (const key in query) {
const value = (query as any)[key]
if (value !== null && typeof value === 'object') {
const keys = Object.keys(value)
if (keys[0] === '$like') {
const pattern = value.$like as string
translated[key] = {
$regex: `^${pattern.split('%').join('.*')}$`,
$options: 'i'
}
continue
}
}
translated[key] = value
}
return translated
}
async rawFindAll<T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<T[]> {
let cursor = this.db.collection(domain).find<T>(this.translateRawQuery(query))
if (options?.limit !== undefined) {
cursor = cursor.limit(options.limit)
}
if (options !== null && options !== undefined) {
if (options.sort !== undefined) {
const sort: Sort = {}
for (const key in options.sort) {
const order = options.sort[key] === SortingOrder.Ascending ? 1 : -1
sort[key] = order
}
cursor = cursor.sort(sort)
}
}
return await cursor.toArray()
}
async rawUpdate<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
operations: DocumentUpdate<T>
): Promise<void> {
if (isOperator(operations)) {
await this.db
.collection(domain)
.updateMany(this.translateRawQuery(query), { ...operations } as unknown as UpdateFilter<Document>)
} else {
await this.db
.collection(domain)
.updateMany(this.translateRawQuery(query), { $set: { ...operations, '%hash%': null } })
}
}
abstract init (): Promise<void>
collection<TSchema extends Document = Document>(domain: Domain): Collection<TSchema> {
@ -255,15 +353,15 @@ abstract class MongoAdapterBase implements DbAdapter {
return { base: translatedBase, lookup: translatedLookup }
}
private async getLookupValue<T extends Doc>(
private getLookupValue<T extends Doc>(
clazz: Ref<Class<T>>,
lookup: Lookup<T>,
result: LookupStep[],
parent?: string
): Promise<void> {
): void {
for (const key in lookup) {
if (key === '_id') {
await this.getReverseLookupValue(lookup, result, parent)
this.getReverseLookupValue(lookup, result, parent)
continue
}
const value = (lookup as any)[key]
@ -280,7 +378,7 @@ abstract class MongoAdapterBase implements DbAdapter {
as: fullKey.split('.').join('') + '_lookup'
})
}
await this.getLookupValue(_class, nested, result, fullKey + '_lookup')
this.getLookupValue(_class, nested, result, fullKey + '_lookup')
} else {
const _class = value as Ref<Class<Doc>>
const tkey = this.checkMixinKey(key, clazz)
@ -298,11 +396,7 @@ abstract class MongoAdapterBase implements DbAdapter {
}
}
private async getReverseLookupValue (
lookup: ReverseLookups,
result: LookupStep[],
parent?: string
): Promise<any | undefined> {
private getReverseLookupValue (lookup: ReverseLookups, result: LookupStep[], parent?: string): void {
const fullKey = parent !== undefined ? parent + '.' + '_id' : '_id'
const lid = lookup?._id ?? {}
for (const key in lid) {
@ -319,7 +413,9 @@ abstract class MongoAdapterBase implements DbAdapter {
_class = value
}
const domain = this.hierarchy.getDomain(_class)
const desc = this.hierarchy.getDescendants(_class)
const desc = this.hierarchy
.getDescendants(this.hierarchy.getBaseClass(_class))
.filter((it) => !this.hierarchy.isMixin(it))
if (domain !== DOMAIN_MODEL) {
const asVal = as.split('.').join('') + '_lookup'
const step: LookupStep = {
@ -340,14 +436,14 @@ abstract class MongoAdapterBase implements DbAdapter {
}
}
private async getLookups<T extends Doc>(
private getLookups<T extends Doc>(
_class: Ref<Class<T>>,
lookup: Lookup<T> | undefined,
parent?: string
): Promise<LookupStep[]> {
): LookupStep[] {
if (lookup === undefined) return []
const result: [] = []
await this.getLookupValue(_class, lookup, result, parent)
this.getLookupValue(_class, lookup, result, parent)
return result
}
@ -482,8 +578,7 @@ abstract class MongoAdapterBase implements DbAdapter {
const tquery = this.translateQuery(clazz, query, options)
const slowPipeline = isLookupQuery(query) || isLookupSort(options?.sort)
const steps = await ctx.with('get-lookups', {}, async () => await this.getLookups(clazz, options?.lookup))
const steps = this.getLookups(clazz, options?.lookup)
if (slowPipeline) {
if (Object.keys(tquery.base).length > 0) {
pipeline.push({ $match: tquery.base })

View File

@ -81,7 +81,8 @@ class MongoClientReferenceImpl {
}
this.onclose()
void (async () => {
await (await this.client).close()
const cl = await this.client
await cl.close()
})()
}
}
@ -214,7 +215,7 @@ export class DBCollectionHelper implements DomainHelperOperations {
}
}
exists (domain: Domain): boolean {
async exists (domain: Domain): Promise<boolean> {
return this.collections.has(domain)
}
@ -242,7 +243,7 @@ export class DBCollectionHelper implements DomainHelperOperations {
}
async estimatedCount (domain: Domain): Promise<number> {
if (this.exists(domain)) {
if (await this.exists(domain)) {
const c = this.collection(domain)
return await c.estimatedDocumentCount()
}

View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

View File

@ -0,0 +1,5 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig",
"rigProfile": "node"
}

View File

@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}

View File

@ -0,0 +1,43 @@
{
"name": "@hcengineering/postgres",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"author": "Copyright © Hardcore Engineering Inc.",
"template": "@hcengineering/node-package",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"@types/node": "~20.11.16",
"@types/pg": "^8.11.6"
},
"dependencies": {
"pg": "8.12.0",
"@hcengineering/core": "^0.6.32",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-core": "^0.6.1"
}
}

View File

@ -0,0 +1,233 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
type Account,
type Arr,
type AttachedDoc,
type Class,
ClassifierKind,
type Data,
type Doc,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL,
DOMAIN_TX,
type Mixin,
type Obj,
type Ref,
type TxCreateDoc,
type TxCUD,
TxFactory,
AccountRole
} from '@hcengineering/core'
import type { IntlString, Plugin } from '@hcengineering/platform'
import { plugin } from '@hcengineering/platform'
export const txFactory = new TxFactory(core.account.System)
export function createClass (_class: Ref<Class<Obj>>, attributes: Data<Class<Obj>>): TxCreateDoc<Doc> {
return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class)
}
/**
* @public
*/
export function createDoc<T extends Doc> (
_class: Ref<Class<T>>,
attributes: Data<T>,
id?: Ref<T>,
modifiedBy?: Ref<Account>
): TxCreateDoc<Doc> {
const result = txFactory.createTxCreateDoc(_class, core.space.Model, attributes, id)
if (modifiedBy !== undefined) {
result.modifiedBy = modifiedBy
}
return result
}
/**
* @public
*/
export interface TestMixin extends Doc {
arr: Arr<string>
}
/**
* @public
*/
export interface AttachedComment extends AttachedDoc {
message: string
}
/**
* @public
*/
export const test = plugin('test' as Plugin, {
mixin: {
TestMixin: '' as Ref<Mixin<TestMixin>>
},
class: {
TestComment: '' as Ref<Class<AttachedComment>>
}
})
/**
* @public
* Generate minimal model for testing purposes.
* @returns R
*/
export function genMinModel (): TxCUD<Doc>[] {
const txes = []
// Fill Tx'es with basic model classes.
txes.push(createClass(core.class.Obj, { label: 'Obj' as IntlString, kind: ClassifierKind.CLASS }))
txes.push(
createClass(core.class.Doc, { label: 'Doc' as IntlString, extends: core.class.Obj, kind: ClassifierKind.CLASS })
)
txes.push(
createClass(core.class.AttachedDoc, {
label: 'AttachedDoc' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.MIXIN
})
)
txes.push(
createClass(core.class.Class, {
label: 'Class' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_MODEL
})
)
txes.push(
createClass(core.class.Space, {
label: 'Space' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_MODEL
})
)
txes.push(
createClass(core.class.DocIndexState, {
label: 'DocIndexState' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_DOC_INDEX_STATE
})
)
txes.push(
createClass(core.class.Account, {
label: 'Account' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_MODEL
})
)
txes.push(
createClass(core.class.Tx, {
label: 'Tx' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_TX
})
)
txes.push(
createClass(core.class.TxCUD, {
label: 'TxCUD' as IntlString,
extends: core.class.Tx,
kind: ClassifierKind.CLASS,
domain: DOMAIN_TX
})
)
txes.push(
createClass(core.class.TxCreateDoc, {
label: 'TxCreateDoc' as IntlString,
extends: core.class.TxCUD,
kind: ClassifierKind.CLASS
})
)
txes.push(
createClass(core.class.TxUpdateDoc, {
label: 'TxUpdateDoc' as IntlString,
extends: core.class.TxCUD,
kind: ClassifierKind.CLASS
})
)
txes.push(
createClass(core.class.TxRemoveDoc, {
label: 'TxRemoveDoc' as IntlString,
extends: core.class.TxCUD,
kind: ClassifierKind.CLASS
})
)
txes.push(
createClass(core.class.TxCollectionCUD, {
label: 'TxCollectionCUD' as IntlString,
extends: core.class.TxCUD,
kind: ClassifierKind.CLASS
})
)
txes.push(
createClass(test.mixin.TestMixin, {
label: 'TestMixin' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.MIXIN
})
)
txes.push(
createClass(test.class.TestComment, {
label: 'TestComment' as IntlString,
extends: core.class.AttachedDoc,
kind: ClassifierKind.CLASS
})
)
const u1 = 'User1' as Ref<Account>
const u2 = 'User2' as Ref<Account>
txes.push(
createDoc(core.class.Account, { email: 'user1@site.com', role: AccountRole.User }, u1),
createDoc(core.class.Account, { email: 'user2@site.com', role: AccountRole.User }, u2),
createDoc(core.class.Space, {
name: 'Sp1',
description: '',
private: false,
archived: false,
members: [u1, u2]
})
)
txes.push(
createDoc(core.class.Space, {
name: 'Sp2',
description: '',
private: false,
archived: false,
members: [u1]
})
)
txes.push(
createClass(core.class.DomainIndexConfiguration, {
label: 'DomainIndexConfiguration' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_MODEL
})
)
return txes
}

View File

@ -0,0 +1,328 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
type Client,
type ClientConnection,
createClient,
type Doc,
type DocChunk,
type Domain,
generateId,
getWorkspaceId,
Hierarchy,
MeasureMetricsContext,
ModelDb,
type Ref,
SortingOrder,
type Space,
TxOperations
} from '@hcengineering/core'
import { type DbAdapter } from '@hcengineering/server-core'
import { createPostgresAdapter, createPostgresTxAdapter } from '..'
import { getDBClient, type PostgresClientReference, shutdown } from '../utils'
import { genMinModel } from './minmodel'
import { createTaskModel, type Task, type TaskComment, taskPlugin } from './tasks'
const txes = genMinModel()
createTaskModel(txes)
describe('postgres operations', () => {
const baseDbUri: string = process.env.DB_URL ?? 'postgresql://postgres:example@localhost:5433'
let dbId: string = 'pg_testdb_' + generateId()
let dbUri: string = baseDbUri + '/' + dbId
const clientRef: PostgresClientReference = getDBClient(baseDbUri)
let hierarchy: Hierarchy
let model: ModelDb
let client: Client
let operations: TxOperations
let serverStorage: DbAdapter
afterAll(async () => {
clientRef.close()
await shutdown()
})
beforeEach(async () => {
try {
dbId = 'pg_testdb_' + generateId()
dbUri = baseDbUri + '/' + dbId
const client = await clientRef.getClient()
await client.query(`CREATE DATABASE ${dbId}`)
} catch (err) {
console.error(err)
}
})
afterEach(async () => {
try {
// await client.close()
// await (await clientRef.getClient()).query(`DROP DATABASE ${dbId}`)
} catch (err) {
console.log(err)
}
await serverStorage?.close()
})
async function initDb (): Promise<void> {
// Remove all stuff from database.
hierarchy = new Hierarchy()
model = new ModelDb(hierarchy)
for (const t of txes) {
hierarchy.tx(t)
}
for (const t of txes) {
await model.tx(t)
}
const mctx = new MeasureMetricsContext('', {})
const txStorage = await createPostgresTxAdapter(mctx, hierarchy, dbUri, getWorkspaceId(dbId), model)
// Put all transactions to Tx
for (const t of txes) {
await txStorage.tx(mctx, t)
}
await txStorage.close()
const ctx = new MeasureMetricsContext('client', {})
const serverStorage = await createPostgresAdapter(ctx, hierarchy, dbUri, getWorkspaceId(dbId), model)
await serverStorage.init?.()
client = await createClient(async (handler) => {
const st: ClientConnection = {
isConnected: () => true,
findAll: async (_class, query, options) => await serverStorage.findAll(ctx, _class, query, options),
tx: async (tx) => await serverStorage.tx(ctx, tx),
searchFulltext: async () => ({ docs: [] }),
close: async () => {},
loadChunk: async (domain): Promise<DocChunk> => await Promise.reject(new Error('unsupported')),
closeChunk: async (idx) => {},
loadDocs: async (domain: Domain, docs: Ref<Doc>[]) => [],
upload: async (domain: Domain, docs: Doc[]) => {},
clean: async (domain: Domain, docs: Ref<Doc>[]) => {},
loadModel: async () => txes,
getAccount: async () => ({}) as any,
sendForceClose: async () => {}
}
return st
})
operations = new TxOperations(client, core.account.System)
}
beforeEach(async () => {
jest.setTimeout(30000)
await initDb()
})
it('check add', async () => {
const times: number[] = []
for (let i = 0; i < 50; i++) {
const t = Date.now()
await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
name: `my-task-${i}`,
description: `${i * i}`,
rate: 20 + i
})
times.push(Date.now() - t)
}
console.log('createDoc times', times)
const r = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(r.length).toEqual(50)
})
it('check find by criteria', async () => {
jest.setTimeout(20000)
for (let i = 0; i < 50; i++) {
await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
name: `my-task-${i}`,
description: `${i * i}`,
rate: 20 + i
})
}
const r = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(r.length).toEqual(50)
const first = await client.findAll<Task>(taskPlugin.class.Task, { name: 'my-task-0' })
expect(first.length).toEqual(1)
const second = await client.findAll<Task>(taskPlugin.class.Task, { name: { $like: '%0' } })
expect(second.length).toEqual(5)
const third = await client.findAll<Task>(taskPlugin.class.Task, { rate: { $in: [25, 26, 27, 28] } })
expect(third.length).toEqual(4)
})
it('check update', async () => {
await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
name: 'my-task',
description: 'some data ',
rate: 20,
arr: []
})
const doc = (await client.findAll<Task>(taskPlugin.class.Task, {}))[0]
await operations.updateDoc(doc._class, doc.space, doc._id, { rate: 30 })
let tasks = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(tasks.length).toEqual(1)
expect(tasks[0].rate).toEqual(30)
await operations.updateDoc(doc._class, doc.space, doc._id, { $inc: { rate: 1 } })
tasks = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(tasks.length).toEqual(1)
expect(tasks[0].rate).toEqual(31)
await operations.updateDoc(doc._class, doc.space, doc._id, { $inc: { rate: -1 } })
tasks = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(tasks.length).toEqual(1)
expect(tasks[0].rate).toEqual(30)
await operations.updateDoc(doc._class, doc.space, doc._id, { $push: { arr: 1 } })
tasks = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(tasks.length).toEqual(1)
expect(tasks[0].arr?.length).toEqual(1)
expect(tasks[0].arr?.[0]).toEqual(1)
await operations.updateDoc(doc._class, doc.space, doc._id, { $push: { arr: 3 } })
tasks = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(tasks.length).toEqual(1)
expect(tasks[0].arr?.length).toEqual(2)
expect(tasks[0].arr?.[0]).toEqual(1)
expect(tasks[0].arr?.[1]).toEqual(3)
})
it('check remove', async () => {
for (let i = 0; i < 10; i++) {
await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
name: `my-task-${i}`,
description: `${i * i}`,
rate: 20 + i
})
}
let r = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(r.length).toEqual(10)
await operations.removeDoc<Task>(taskPlugin.class.Task, '' as Ref<Space>, r[0]._id)
r = await client.findAll<Task>(taskPlugin.class.Task, {})
expect(r.length).toEqual(9)
})
it('limit and sorting', async () => {
for (let i = 0; i < 5; i++) {
await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
name: `my-task-${i}`,
description: `${i * i}`,
rate: 20 + i
})
}
const without = await client.findAll(taskPlugin.class.Task, {})
expect(without).toHaveLength(5)
const limit = await client.findAll(taskPlugin.class.Task, {}, { limit: 1 })
expect(limit).toHaveLength(1)
const sortAsc = await client.findAll(taskPlugin.class.Task, {}, { sort: { name: SortingOrder.Ascending } })
expect(sortAsc[0].name).toMatch('my-task-0')
const sortDesc = await client.findAll(taskPlugin.class.Task, {}, { sort: { name: SortingOrder.Descending } })
expect(sortDesc[0].name).toMatch('my-task-4')
})
it('check attached', async () => {
const docId = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
name: 'my-task',
description: 'Descr',
rate: 20
})
const commentId = await operations.addCollection(
taskPlugin.class.TaskComment,
'' as Ref<Space>,
docId,
taskPlugin.class.Task,
'tasks',
{
message: 'my-msg',
date: new Date()
}
)
await operations.addCollection(
taskPlugin.class.TaskComment,
'' as Ref<Space>,
docId,
taskPlugin.class.Task,
'tasks',
{
message: 'my-msg2',
date: new Date()
}
)
const r2 = await client.findAll<TaskComment>(
taskPlugin.class.TaskComment,
{},
{
lookup: {
attachedTo: taskPlugin.class.Task
}
}
)
expect(r2.length).toEqual(2)
expect((r2[0].$lookup?.attachedTo as Task)?._id).toEqual(docId)
const r3 = await client.findAll<Task>(
taskPlugin.class.Task,
{},
{
lookup: {
_id: { comment: taskPlugin.class.TaskComment }
}
}
)
expect(r3).toHaveLength(1)
expect((r3[0].$lookup as any).comment).toHaveLength(2)
const comment2Id = await operations.addCollection(
taskPlugin.class.TaskComment,
'' as Ref<Space>,
commentId,
taskPlugin.class.TaskComment,
'comments',
{
message: 'my-msg3',
date: new Date()
}
)
const r4 = await client.findAll<TaskComment>(
taskPlugin.class.TaskComment,
{
_id: comment2Id
},
{
lookup: { attachedTo: [taskPlugin.class.TaskComment, { attachedTo: taskPlugin.class.Task } as any] }
}
)
expect((r4[0].$lookup?.attachedTo as TaskComment)?._id).toEqual(commentId)
expect(((r4[0].$lookup?.attachedTo as any)?.$lookup.attachedTo as Task)?._id).toEqual(docId)
})
})

View File

@ -0,0 +1,112 @@
import {
type Account,
type AttachedDoc,
type Class,
ClassifierKind,
type Data,
type Doc,
type Domain,
type Ref,
type Space,
type Tx
} from '@hcengineering/core'
import { type IntlString, plugin, type Plugin } from '@hcengineering/platform'
import { createClass } from './minmodel'
export interface TaskComment extends AttachedDoc {
message: string
date: Date
}
export enum TaskStatus {
Open,
Close,
Resolved = 100,
InProgress
}
export enum TaskReproduce {
Always = 'always',
Rare = 'rare',
Sometimes = 'sometimes'
}
export interface Task extends Doc {
name: string
description: string
rate?: number
status?: TaskStatus
reproduce?: TaskReproduce
eta?: TaskEstimate | null
arr?: number[]
}
/**
* Define ROM and Estimated Time to arrival
*/
export interface TaskEstimate extends AttachedDoc {
rom: number // in hours
eta: number // in hours
}
export interface TaskMixin extends Task {
textValue?: string
}
export interface TaskWithSecond extends Task {
secondTask: string | null
}
const taskIds = 'taskIds' as Plugin
export const taskPlugin = plugin(taskIds, {
class: {
Task: '' as Ref<Class<Task>>,
TaskEstimate: '' as Ref<Class<TaskEstimate>>,
TaskComment: '' as Ref<Class<TaskComment>>
}
})
/**
* Create a random task with name specified
* @param name
*/
export function createTask (name: string, rate: number, description: string): Data<Task> {
return {
name,
description,
rate
}
}
export const doc1: Task = {
_id: 'd1' as Ref<Task>,
_class: taskPlugin.class.Task,
name: 'my-space',
description: 'some-value',
rate: 20,
modifiedBy: 'user' as Ref<Account>,
modifiedOn: 10,
// createdOn: 10,
space: '' as Ref<Space>
}
export function createTaskModel (txes: Tx[]): void {
txes.push(
createClass(taskPlugin.class.Task, {
kind: ClassifierKind.CLASS,
label: 'Task' as IntlString,
domain: 'test-task' as Domain
}),
createClass(taskPlugin.class.TaskEstimate, {
kind: ClassifierKind.CLASS,
label: 'Estimate' as IntlString,
domain: 'test-task' as Domain
}),
createClass(taskPlugin.class.TaskComment, {
kind: ClassifierKind.CLASS,
label: 'Comment' as IntlString,
domain: 'test-task' as Domain
})
)
}

View File

@ -0,0 +1,17 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export * from './storage'
export { getDBClient, convertDoc, createTable, retryTxn, translateDomain } from './utils'

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,391 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
type Account,
AccountRole,
type Class,
type Doc,
type Domain,
type FieldIndexConfig,
generateId,
type Projection,
type Ref,
type WorkspaceId
} from '@hcengineering/core'
import { PlatformError, unknownStatus } from '@hcengineering/platform'
import { type DomainHelperOperations } from '@hcengineering/server-core'
import { Pool, type PoolClient } from 'pg'
const connections = new Map<string, PostgresClientReferenceImpl>()
// Register close on process exit.
process.on('exit', () => {
shutdown().catch((err) => {
console.error(err)
})
})
const clientRefs = new Map<string, ClientRef>()
export async function retryTxn (pool: Pool, operation: (client: PoolClient) => Promise<any>): Promise<any> {
const backoffInterval = 100 // millis
const maxTries = 5
let tries = 0
const client = await pool.connect()
try {
while (true) {
await client.query('BEGIN;')
tries++
try {
const result = await operation(client)
await client.query('COMMIT;')
return result
} catch (err: any) {
await client.query('ROLLBACK;')
if (err.code !== '40001' || tries === maxTries) {
throw err
} else {
console.log('Transaction failed. Retrying.')
console.log(err.message)
await new Promise((resolve) => setTimeout(resolve, tries * backoffInterval))
}
}
}
} finally {
client.release()
}
}
export async function createTable (client: Pool, domains: string[]): Promise<void> {
if (domains.length === 0) {
return
}
const mapped = domains.map((p) => translateDomain(p))
const inArr = mapped.map((it) => `'${it}'`).join(', ')
const exists = await client.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_name IN (${inArr})
`)
const toCreate = mapped.filter((it) => !exists.rows.map((it) => it.table_name).includes(it))
await retryTxn(client, async (client) => {
for (const domain of toCreate) {
await client.query(
`CREATE TABLE ${domain} (
"workspaceId" VARCHAR(255) NOT NULL,
_id VARCHAR(255) NOT NULL,
_class VARCHAR(255) NOT NULL,
"createdBy" VARCHAR(255),
"modifiedBy" VARCHAR(255) NOT NULL,
"modifiedOn" bigint NOT NULL,
"createdOn" bigint,
space VARCHAR(255) NOT NULL,
"attachedTo" VARCHAR(255),
data JSONB NOT NULL,
PRIMARY KEY("workspaceId", _id)
)`
)
await client.query(`
CREATE INDEX ${domain}_attachedTo ON ${domain} ("attachedTo")
`)
await client.query(`
CREATE INDEX ${domain}_class ON ${domain} (_class)
`)
await client.query(`
CREATE INDEX ${domain}_space ON ${domain} (space)
`)
await client.query(`
CREATE INDEX ${domain}_idxgin ON ${domain} USING GIN (data)
`)
}
})
}
/**
* @public
*/
export async function shutdown (): Promise<void> {
for (const c of connections.values()) {
c.close(true)
}
connections.clear()
}
export interface PostgresClientReference {
getClient: () => Promise<Pool>
close: () => void
}
class PostgresClientReferenceImpl {
count: number
client: Pool | Promise<Pool>
constructor (
client: Pool | Promise<Pool>,
readonly onclose: () => void
) {
this.count = 0
this.client = client
}
async getClient (): Promise<Pool> {
if (this.client instanceof Promise) {
this.client = await this.client
}
return this.client
}
close (force: boolean = false): void {
this.count--
if (this.count === 0 || force) {
if (force) {
this.count = 0
}
void (async () => {
this.onclose()
const cl = await this.client
await cl.end()
console.log('Closed postgres connection')
})()
}
}
addRef (): void {
this.count++
}
}
export class ClientRef implements PostgresClientReference {
id = generateId()
constructor (readonly client: PostgresClientReferenceImpl) {
clientRefs.set(this.id, this)
}
closed = false
async getClient (): Promise<Pool> {
if (!this.closed) {
return await this.client.getClient()
} else {
throw new PlatformError(unknownStatus('DB client is already closed'))
}
}
close (): void {
// Do not allow double close of mongo connection client
if (!this.closed) {
clientRefs.delete(this.id)
this.closed = true
this.client.close()
}
}
}
/**
* Initialize a workspace connection to DB
* @public
*/
export function getDBClient (connectionString: string, database?: string): PostgresClientReference {
const key = `${connectionString}${process.env.postgree_OPTIONS ?? '{}'}`
let existing = connections.get(key)
if (existing === undefined) {
const pool = new Pool({
connectionString,
application_name: 'transactor',
database
})
existing = new PostgresClientReferenceImpl(pool, () => {
connections.delete(key)
})
connections.set(key, existing)
}
// Add reference and return once closable
existing.addRef()
return new ClientRef(existing)
}
export function convertDoc<T extends Doc> (doc: T, workspaceId: string): DBDoc {
const { _id, _class, createdBy, modifiedBy, modifiedOn, createdOn, space, attachedTo, ...data } = doc as any
return {
_id,
_class,
createdBy,
modifiedBy,
modifiedOn,
createdOn,
space,
attachedTo,
workspaceId,
data
}
}
export function escapeBackticks (str: string): string {
return str.replaceAll("'", "''")
}
export function isOwner (account: Account): boolean {
return account.role === AccountRole.Owner || account._id === core.account.System
}
export class DBCollectionHelper implements DomainHelperOperations {
constructor (
protected readonly client: Pool,
protected readonly workspaceId: WorkspaceId
) {}
domains = new Set<Domain>()
async create (domain: Domain): Promise<void> {}
async exists (domain: Domain): Promise<boolean> {
const exists = await this.client.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_name = '${translateDomain(domain)}'
`)
return exists.rows.length > 0
}
async listDomains (): Promise<Set<Domain>> {
return this.domains
}
async createIndex (domain: Domain, value: string | FieldIndexConfig<Doc>, options?: { name: string }): Promise<void> {}
async dropIndex (domain: Domain, name: string): Promise<void> {}
async listIndexes (domain: Domain): Promise<{ name: string }[]> {
return []
}
async estimatedCount (domain: Domain): Promise<number> {
const res = await this.client.query(`SELECT COUNT(_id) FROM ${translateDomain(domain)} WHERE "workspaceId" = $1`, [
this.workspaceId.name
])
return res.rows[0].count
}
}
export function translateDomain (domain: string): string {
return domain.replaceAll('-', '_')
}
export function parseDocWithProjection<T extends Doc> (doc: DBDoc, projection: Projection<T> | undefined): T {
const { workspaceId, data, ...rest } = doc
for (const key in rest) {
if ((rest as any)[key] === 'NULL') {
if (key === 'attachedTo') {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete rest[key]
} else {
;(rest as any)[key] = null
}
}
if (key === 'modifiedOn' || key === 'createdOn') {
;(rest as any)[key] = Number.parseInt((rest as any)[key])
}
}
if (projection !== undefined) {
for (const key in data) {
if (!Object.prototype.hasOwnProperty.call(projection, key) || (projection as any)[key] === 0) {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete data[key]
}
}
}
const res = {
...data,
...rest
} as any as T
return res
}
export function parseDoc<T extends Doc> (doc: DBDoc): T {
const { workspaceId, data, ...rest } = doc
for (const key in rest) {
if ((rest as any)[key] === 'NULL') {
if (key === 'attachedTo') {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete rest[key]
} else {
;(rest as any)[key] = null
}
}
if (key === 'modifiedOn' || key === 'createdOn') {
;(rest as any)[key] = Number.parseInt((rest as any)[key])
}
}
const res = {
...data,
...rest
} as any as T
return res
}
export interface DBDoc extends Doc {
workspaceId: string
attachedTo?: Ref<Doc>
data: Record<string, any>
}
export function isDataField (field: string): boolean {
return !docFields.includes(field)
}
export const docFields: string[] = [
'_id',
'_class',
'createdBy',
'modifiedBy',
'modifiedOn',
'createdOn',
'space',
'attachedTo'
] as const
export function getUpdateValue (value: any): string {
if (typeof value === 'string') {
return '"' + escapeDoubleQuotes(value) + '"'
}
if (typeof value === 'object') {
return JSON.stringify(value)
}
return value
}
function escapeDoubleQuotes (jsonString: string): string {
const unescapedQuotes = /(?<!\\)"/g
return jsonString.replace(unescapedQuotes, '\\"')
}
export interface JoinProps {
table: string // table to join
path: string // _id.roles, attachedTo.attachedTo, space...
fromAlias: string
fromField: string
toAlias: string // alias for the table
toField: string // field to join on
isReverse: boolean
toClass: Ref<Class<Doc>>
classes?: Ref<Class<Doc>>[] // filter by classes
}

View File

@ -0,0 +1,10 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}

View File

@ -48,6 +48,7 @@
"@hcengineering/server-collaboration-resources": "^0.6.0",
"@hcengineering/server": "^0.6.4",
"@hcengineering/server-storage": "^0.6.0",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1",
"@hcengineering/elastic": "^0.6.0",
"elastic-apm-node": "~3.26.0",

View File

@ -1,11 +1,13 @@
/* eslint-disable @typescript-eslint/unbound-method */
import {
type Branding,
DOMAIN_BENCHMARK,
DOMAIN_BLOB,
DOMAIN_FULLTEXT_BLOB,
DOMAIN_MODEL,
DOMAIN_TRANSIENT,
DOMAIN_TX,
type WorkspaceIdWithUrl,
Hierarchy,
ModelDb,
type MeasureContext
@ -33,23 +35,32 @@ import {
TriggersMiddleware,
TxMiddleware
} from '@hcengineering/middleware'
import { createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import {
buildStorageFromConfig,
createNullAdapter,
createRekoniAdapter,
createStorageDataAdapter,
createYDocAdapter
createYDocAdapter,
storageConfigFromEnv
} from '@hcengineering/server'
import {
createBenchmarkAdapter,
createInMemoryAdapter,
createPipeline,
type Middleware,
type DbAdapterFactory,
FullTextMiddleware,
type DbConfiguration,
type MiddlewareCreator,
type PipelineContext,
type PipelineFactory,
type StorageAdapter
type StorageAdapter,
type Pipeline,
type StorageConfiguration,
DummyFullTextAdapter,
type AggregatorStorageAdapter
} from '@hcengineering/server-core'
import { createIndexStages } from './indexing'
@ -57,9 +68,11 @@ import { createIndexStages } from './indexing'
* @public
*/
export function createServerPipeline (
export function getTxAdapterFactory (
metrics: MeasureContext,
dbUrl: string,
dbUrls: string,
workspace: WorkspaceIdWithUrl,
branding: Branding | null,
opt: {
fullTextUrl: string
rekoniUrl: string
@ -71,13 +84,152 @@ export function createServerPipeline (
externalStorage: StorageAdapter
},
extensions?: Partial<DbConfiguration>
): DbAdapterFactory {
const conf = getConfig(metrics, dbUrls, workspace, branding, metrics, opt, extensions)
const adapterName = conf.domains[DOMAIN_TX] ?? conf.defaultAdapter
const adapter = conf.adapters[adapterName]
return adapter.factory
}
/**
* @public
*/
export function createServerPipeline (
metrics: MeasureContext,
dbUrls: string,
opt: {
fullTextUrl: string
rekoniUrl: string
indexProcessing: number // 1000
indexParallel: number // 2
disableTriggers?: boolean
usePassedCtx?: boolean
adapterSecurity?: boolean
externalStorage: StorageAdapter
},
extensions?: Partial<DbConfiguration>
): PipelineFactory {
return (ctx, workspace, upgrade, broadcast, branding) => {
const metricsCtx = opt.usePassedCtx === true ? ctx : metrics
const wsMetrics = metricsCtx.newChild('🧲 session', {})
const conf = getConfig(metrics, dbUrls, workspace, branding, wsMetrics, opt, extensions)
const middlewares: MiddlewareCreator[] = [
LookupMiddleware.create,
ModifiedMiddleware.create,
PrivateMiddleware.create,
(ctx: MeasureContext, context: PipelineContext, next?: Middleware) =>
SpaceSecurityMiddleware.create(opt.adapterSecurity ?? false, ctx, context, next),
SpacePermissionsMiddleware.create,
ConfigurationMiddleware.create,
LowLevelMiddleware.create,
ContextNameMiddleware.create,
MarkDerivedEntryMiddleware.create,
ApplyTxMiddleware.create, // Extract apply
TxMiddleware.create, // Store tx into transaction domain
...(opt.disableTriggers === true ? [] : [TriggersMiddleware.create]),
FullTextMiddleware.create(conf, upgrade),
QueryJoinMiddleware.create,
LiveQueryMiddleware.create,
DomainFindMiddleware.create,
DomainTxMiddleware.create,
DBAdapterHelperMiddleware.create,
ModelMiddleware.create,
DBAdapterMiddleware.create(conf), // Configure DB adapters
BroadcastMiddleware.create(broadcast)
]
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
const context: PipelineContext = {
workspace,
branding,
modelDb,
hierarchy,
storageAdapter: opt.externalStorage
}
return createPipeline(ctx, middlewares, context)
}
}
export async function getServerPipeline (
ctx: MeasureContext,
mongodbUri: string,
dbUrl: string | undefined,
wsUrl: WorkspaceIdWithUrl
): Promise<{
pipeline: Pipeline
storageAdapter: AggregatorStorageAdapter
}> {
const dbUrls = dbUrl !== undefined ? `${dbUrl};${mongodbUri}` : mongodbUri
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
const pipelineFactory = createServerPipeline(
ctx,
dbUrls,
{
externalStorage: storageAdapter,
fullTextUrl: 'http://localhost:9200',
indexParallel: 0,
indexProcessing: 0,
rekoniUrl: '',
usePassedCtx: true,
disableTriggers: true
},
{
fulltextAdapter: {
factory: async () => new DummyFullTextAdapter(),
url: '',
stages: (adapter, storage, storageAdapter, contentAdapter) =>
createIndexStages(
ctx.newChild('stages', {}),
wsUrl,
null,
adapter,
storage,
storageAdapter,
contentAdapter,
0,
0
)
}
}
)
return {
pipeline: await pipelineFactory(ctx, wsUrl, true, () => {}, null),
storageAdapter
}
}
function getConfig (
metrics: MeasureContext,
dbUrls: string,
workspace: WorkspaceIdWithUrl,
branding: Branding | null,
ctx: MeasureContext,
opt: {
fullTextUrl: string
rekoniUrl: string
indexProcessing: number // 1000
indexParallel: number // 2
disableTriggers?: boolean
usePassedCtx?: boolean
externalStorage: StorageAdapter
},
extensions?: Partial<DbConfiguration>
): DbConfiguration {
const metricsCtx = opt.usePassedCtx === true ? ctx : metrics
const wsMetrics = metricsCtx.newChild('🧲 session', {})
const [dbUrl, mongoUrl] = dbUrls.split(';')
const conf: DbConfiguration = {
domains: {
[DOMAIN_TX]: 'MongoTx',
[DOMAIN_TX]: 'Tx',
[DOMAIN_TRANSIENT]: 'InMemory',
[DOMAIN_BLOB]: 'StorageData',
[DOMAIN_FULLTEXT_BLOB]: 'FullTextBlob',
@ -86,14 +238,14 @@ export function createServerPipeline (
...extensions?.domains
},
metrics: wsMetrics,
defaultAdapter: extensions?.defaultAdapter ?? 'Mongo',
defaultAdapter: extensions?.defaultAdapter ?? 'Main',
adapters: {
MongoTx: {
factory: createMongoTxAdapter,
Tx: {
factory: mongoUrl !== undefined ? createPostgresTxAdapter : createMongoTxAdapter,
url: dbUrl
},
Mongo: {
factory: createMongoAdapter,
Main: {
factory: mongoUrl !== undefined ? createPostgresAdapter : createMongoAdapter,
url: dbUrl
},
Null: {
@ -106,7 +258,7 @@ export function createServerPipeline (
},
StorageData: {
factory: createStorageDataAdapter,
url: dbUrl
url: mongoUrl ?? dbUrl
},
FullTextBlob: {
factory: createElasticBackupDataAdapter,
@ -150,40 +302,5 @@ export function createServerPipeline (
},
defaultContentAdapter: extensions?.defaultContentAdapter ?? 'Rekoni'
}
const middlewares: MiddlewareCreator[] = [
LookupMiddleware.create,
ModifiedMiddleware.create,
PrivateMiddleware.create,
SpaceSecurityMiddleware.create,
SpacePermissionsMiddleware.create,
ConfigurationMiddleware.create,
LowLevelMiddleware.create,
ContextNameMiddleware.create,
MarkDerivedEntryMiddleware.create,
ApplyTxMiddleware.create, // Extract apply
TxMiddleware.create, // Store tx into transaction domain
...(opt.disableTriggers === true ? [] : [TriggersMiddleware.create]),
FullTextMiddleware.create(conf, upgrade),
QueryJoinMiddleware.create,
LiveQueryMiddleware.create,
DomainFindMiddleware.create,
DomainTxMiddleware.create,
DBAdapterHelperMiddleware.create,
ModelMiddleware.create,
DBAdapterMiddleware.create(conf), // Configure DB adapters
BroadcastMiddleware.create(broadcast)
]
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
const context: PipelineContext = {
workspace,
branding,
modelDb,
hierarchy,
storageAdapter: opt.externalStorage
}
return createPipeline(ctx, middlewares, context)
}
return conf
}

View File

@ -23,6 +23,7 @@ import core, {
FindResult,
Hierarchy,
IndexingConfiguration,
Iterator,
MeasureContext,
ModelDb,
Ref,
@ -36,6 +37,7 @@ import { createMongoAdapter } from '@hcengineering/mongo'
import { PlatformError, unknownError } from '@hcengineering/platform'
import {
DbAdapter,
DbAdapterHandler,
StorageAdapter,
type DomainHelperOperations,
type StorageAdapterEx
@ -49,6 +51,29 @@ class StorageBlobAdapter implements DbAdapter {
readonly blobAdapter: DbAdapter // A real blob adapter for Blob documents.
) {}
async traverse<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: Pick<FindOptions<T>, 'sort' | 'limit' | 'projection'>
): Promise<Iterator<T>> {
return await this.blobAdapter.traverse(domain, query, options)
}
init?: ((domains?: string[], excludeDomains?: string[]) => Promise<void>) | undefined
on?: ((handler: DbAdapterHandler) => void) | undefined
async rawFindAll<T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<T[]> {
return await this.blobAdapter.rawFindAll(domain, query, options)
}
async rawUpdate<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
operations: DocumentUpdate<T>
): Promise<void> {
await this.blobAdapter.rawUpdate(domain, query, operations)
}
async findAll<T extends Doc>(
ctx: MeasureContext,
_class: Ref<Class<T>>,

View File

@ -45,7 +45,6 @@
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/server-ws": "^0.6.11",
"@hcengineering/mongo": "^0.6.1",
"@hcengineering/minio": "^0.6.0",
"@hcengineering/elastic": "^0.6.0",
"elastic-apm-node": "~3.26.0",

View File

@ -39,21 +39,19 @@ import core, {
type TxCUD
} from '@hcengineering/core'
import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate } from '@hcengineering/model'
import { createMongoTxAdapter, DBCollectionHelper, getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
import {
AggregatorStorageAdapter,
DbAdapter,
DomainIndexHelperImpl,
StorageAdapter,
StorageConfiguration
Pipeline,
StorageAdapter
} from '@hcengineering/server-core'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { Db, Document } from 'mongodb'
import { connect } from './connect'
import { InitScript, WorkspaceInitializer } from './initializer'
import toolPlugin from './plugin'
import { MigrateClientImpl } from './upgrade'
import { getMetadata } from '@hcengineering/platform'
import { getMetadata, PlatformError, unknownError } from '@hcengineering/platform'
import { generateToken } from '@hcengineering/server-token'
import fs from 'fs'
import * as yaml from 'js-yaml'
@ -89,6 +87,7 @@ export class FileModelLogger implements ModelLogger {
*/
export function prepareTools (rawTxes: Tx[]): {
mongodbUri: string
dbUrl: string | undefined
txes: Tx[]
} {
const mongodbUri = process.env.MONGO_URL
@ -97,8 +96,11 @@ export function prepareTools (rawTxes: Tx[]): {
process.exit(1)
}
const dbUrl = process.env.DB_URL
return {
mongodbUri,
dbUrl,
txes: JSON.parse(JSON.stringify(rawTxes)) as Tx[]
}
}
@ -110,40 +112,38 @@ export async function initModel (
ctx: MeasureContext,
workspaceId: WorkspaceId,
rawTxes: Tx[],
adapter: DbAdapter,
storageAdapter: AggregatorStorageAdapter,
logger: ModelLogger = consoleModelLogger,
progress: (value: number) => Promise<void>,
deleteFirst: boolean = false
): Promise<void> {
const { mongodbUri, txes } = prepareTools(rawTxes)
const { txes } = prepareTools(rawTxes)
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
throw Error('Model txes must target only core.space.Model')
}
const _client = getMongoClient(mongodbUri)
const client = await _client.getClient()
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
try {
const db = getWorkspaceDB(client, workspaceId)
if (deleteFirst) {
logger.log('deleting model...', workspaceId)
const result = await ctx.with(
'mongo-delete',
{},
async () =>
await db.collection(DOMAIN_TX).deleteMany({
await ctx.with('mongo-delete', {}, async () => {
const toRemove = await adapter.rawFindAll(DOMAIN_TX, {
objectSpace: core.space.Model,
modifiedBy: core.account.System,
objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] }
})
await adapter.clean(
ctx,
DOMAIN_TX,
toRemove.map((p) => p._id)
)
logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: result.deletedCount })
})
logger.log('transactions deleted.', { workspaceId: workspaceId.name })
}
logger.log('creating model...', workspaceId)
const result = await db.collection(DOMAIN_TX).insertMany(txes as Document[])
logger.log('model transactions inserted.', { count: result.insertedCount })
await adapter.upload(ctx, DOMAIN_TX, txes)
logger.log('model transactions inserted.', { count: txes.length })
await progress(30)
@ -159,8 +159,7 @@ export async function initModel (
ctx.error('Failed to create workspace', { error: err })
throw err
} finally {
await storageAdapter.close()
_client.close()
await adapter.close()
}
}
@ -169,6 +168,7 @@ export async function updateModel (
workspaceId: WorkspaceId,
migrateOperations: [string, MigrateOperation][],
connection: TxOperations,
pipeline: Pipeline,
logger: ModelLogger = consoleModelLogger,
progress: (value: number) => Promise<void>
): Promise<void> {
@ -178,14 +178,7 @@ export async function updateModel (
const sts = Array.from(groupByArray(states, (it) => it.plugin).entries())
const migrateState = new Map(sts.map((it) => [it[0], new Set(it[1].map((q) => q.state))]))
const { mongodbUri } = prepareTools([])
const _client = getMongoClient(mongodbUri)
const client = await _client.getClient()
try {
const db = getWorkspaceDB(client, workspaceId)
let i = 0
for (const op of migrateOperations) {
logger.log('Migrate', { name: op[0] })
@ -199,8 +192,7 @@ export async function updateModel (
ctx,
connection.getHierarchy(),
connection.getModel(),
db,
logger,
pipeline,
async (value) => {
await progress(30 + (Math.min(value, 100) / 100) * 70)
},
@ -210,8 +202,6 @@ export async function updateModel (
} catch (e: any) {
logger.error('error', { error: e })
throw e
} finally {
_client.close()
}
}
@ -260,31 +250,23 @@ export async function initializeWorkspace (
export async function upgradeModel (
ctx: MeasureContext,
transactorUrl: string,
workspaceId: WorkspaceId,
rawTxes: Tx[],
workspaceId: WorkspaceIdWithUrl,
txes: Tx[],
pipeline: Pipeline,
storageAdapter: AggregatorStorageAdapter,
migrateOperations: [string, MigrateOperation][],
logger: ModelLogger = consoleModelLogger,
skipTxUpdate: boolean = false,
progress: (value: number) => Promise<void>,
forceIndexes: boolean = false
): Promise<Tx[]> {
const { mongodbUri, txes } = prepareTools(rawTxes)
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
throw Error('Model txes must target only core.space.Model')
}
const _client = getMongoClient(mongodbUri)
const client = await _client.getClient()
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
try {
const db = getWorkspaceDB(client, workspaceId)
const prevModel = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
const prevModel = await fetchModel(ctx, pipeline)
const { migrateClient: preMigrateClient } = await prepareMigrationClient(
db,
pipeline,
prevModel.hierarchy,
prevModel.modelDb,
logger,
@ -295,6 +277,7 @@ export async function upgradeModel (
await progress(0)
await ctx.with('pre-migrate', {}, async (ctx) => {
let i = 0
for (const op of migrateOperations) {
if (op[1].preMigrate === undefined) {
continue
@ -317,28 +300,26 @@ export async function upgradeModel (
})
if (!skipTxUpdate) {
if (pipeline.context.lowLevelStorage === undefined) {
throw new PlatformError(unknownError('Low level storage is not available'))
}
logger.log('removing model...', { workspaceId: workspaceId.name })
await progress(10)
// we're preserving accounts (created by core.account.System).
const result = await ctx.with(
'mongo-delete',
{},
async () =>
await db.collection(DOMAIN_TX).deleteMany({
const toRemove = await pipeline.findAll(ctx, core.class.Tx, {
objectSpace: core.space.Model,
modifiedBy: core.account.System,
objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] }
})
await pipeline.context.lowLevelStorage.clean(
ctx,
DOMAIN_TX,
toRemove.map((p) => p._id)
)
logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: result.deletedCount })
logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: toRemove.length })
logger.log('creating model...', { workspaceId: workspaceId.name })
const insert = await ctx.with(
'mongo-insert',
{},
async () => await db.collection(DOMAIN_TX).insertMany(txes as Document[])
)
await pipeline.context.lowLevelStorage.upload(ctx, DOMAIN_TX, txes)
logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: insert.insertedCount })
logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: txes.length })
await progress(20)
}
const newModel = [
@ -353,9 +334,9 @@ export async function upgradeModel (
)
]
const { hierarchy, modelDb, model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId, newModel)
const { hierarchy, modelDb, model } = await fetchModel(ctx, pipeline, newModel)
const { migrateClient, migrateState } = await prepareMigrationClient(
db,
pipeline,
hierarchy,
modelDb,
logger,
@ -370,8 +351,7 @@ export async function upgradeModel (
ctx,
hierarchy,
modelDb,
db,
logger,
pipeline,
async (value) => {
await progress(90 + (Math.min(value, 100) / 100) * 10)
},
@ -449,8 +429,7 @@ export async function upgradeModel (
const token = generateToken(systemAccountEmail, workspaceId, { admin: 'true' })
try {
await fetch(
serverEndpoint +
`/api/v1/manage?token=${token}&operation=force-close&wsId=${toWorkspaceString(workspaceId)}`,
serverEndpoint + `/api/v1/manage?token=${token}&operation=force-close&wsId=${toWorkspaceString(workspaceId)}`,
{
method: 'PUT'
}
@ -464,14 +443,10 @@ export async function upgradeModel (
await connection?.close()
}
return model
} finally {
await storageAdapter.close()
_client.close()
}
}
async function prepareMigrationClient (
db: Db,
pipeline: Pipeline,
hierarchy: Hierarchy,
model: ModelDb,
logger: ModelLogger,
@ -481,7 +456,7 @@ async function prepareMigrationClient (
migrateClient: MigrateClientImpl
migrateState: Map<string, Set<string>>
}> {
const migrateClient = new MigrateClientImpl(db, hierarchy, model, logger, storageAdapter, workspaceId)
const migrateClient = new MigrateClientImpl(pipeline, hierarchy, model, logger, storageAdapter, workspaceId)
const states = await migrateClient.find<MigrationState>(DOMAIN_MIGRATION, { _class: core.class.MigrationState })
const sts = Array.from(groupByArray(states, (it) => it.plugin).entries())
const migrateState = new Map(sts.map((it) => [it[0], new Set(it[1].map((q) => q.state))]))
@ -490,19 +465,18 @@ async function prepareMigrationClient (
return { migrateClient, migrateState }
}
export async function fetchModelFromMongo (
export async function fetchModel (
ctx: MeasureContext,
mongodbUri: string,
workspaceId: WorkspaceId,
pipeline: Pipeline,
model?: Tx[]
): Promise<{ hierarchy: Hierarchy, modelDb: ModelDb, model: Tx[] }> {
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
const txAdapter = await createMongoTxAdapter(ctx, hierarchy, mongodbUri, workspaceId, modelDb)
try {
model = model ?? (await ctx.with('get-model', {}, async (ctx) => await txAdapter.getModel(ctx)))
if (model === undefined) {
const res = await ctx.with('get-model', {}, async (ctx) => await pipeline.loadModel(ctx, 0))
model = Array.isArray(res) ? res : res.transactions
}
await ctx.with('build local model', {}, async () => {
for (const tx of model ?? []) {
@ -512,30 +486,30 @@ export async function fetchModelFromMongo (
}
modelDb.addTxes(ctx, model as Tx[], false)
})
} finally {
await txAdapter.close()
}
return { hierarchy, modelDb, model }
return { hierarchy, modelDb, model: model ?? [] }
}
async function createUpdateIndexes (
ctx: MeasureContext,
hierarchy: Hierarchy,
model: ModelDb,
db: Db,
logger: ModelLogger,
pipeline: Pipeline,
progress: (value: number) => Promise<void>,
workspaceId: WorkspaceId
): Promise<void> {
const domainHelper = new DomainIndexHelperImpl(ctx, hierarchy, model, workspaceId)
const dbHelper = new DBCollectionHelper(db)
await dbHelper.init()
let completed = 0
const allDomains = hierarchy.domains()
for (const domain of allDomains) {
if (domain === DOMAIN_MODEL || domain === DOMAIN_TRANSIENT || domain === DOMAIN_BENCHMARK) {
continue
}
const adapter = pipeline.context.adapterManager?.getAdapter(domain, false)
if (adapter === undefined) {
throw new PlatformError(unknownError(`Adapter for domain ${domain} not found`))
}
const dbHelper = adapter.helper()
await domainHelper.checkDomain(ctx, domain, await dbHelper.estimatedCount(domain), dbHelper)
completed++
await progress((100 / allDomains.length) * completed)

View File

@ -4,71 +4,42 @@ import {
Domain,
FindOptions,
Hierarchy,
isOperator,
LowLevelStorage,
MeasureMetricsContext,
ModelDb,
Ref,
SortingOrder,
WorkspaceId
} from '@hcengineering/core'
import { MigrateUpdate, MigrationClient, MigrationIterator, MigrationResult, ModelLogger } from '@hcengineering/model'
import { StorageAdapter } from '@hcengineering/server-core'
import { Db, Document, Filter, Sort, UpdateFilter } from 'mongodb'
import { MigrateUpdate, MigrationClient, MigrationIterator, ModelLogger } from '@hcengineering/model'
import { Pipeline, StorageAdapter } from '@hcengineering/server-core'
/**
* Upgrade client implementation.
*/
export class MigrateClientImpl implements MigrationClient {
private readonly lowLevel: LowLevelStorage
constructor (
readonly db: Db,
readonly pipeline: Pipeline,
readonly hierarchy: Hierarchy,
readonly model: ModelDb,
readonly logger: ModelLogger,
readonly storageAdapter: StorageAdapter,
readonly workspaceId: WorkspaceId
) {}
) {
if (this.pipeline.context.lowLevelStorage === undefined) {
throw new Error('lowLevelStorage is not defined')
}
this.lowLevel = this.pipeline.context.lowLevelStorage
}
migrateState = new Map<string, Set<string>>()
private translateQuery<T extends Doc>(query: DocumentQuery<T>): Filter<Document> {
const translated: any = {}
for (const key in query) {
const value = (query as any)[key]
if (value !== null && typeof value === 'object') {
const keys = Object.keys(value)
if (keys[0] === '$like') {
const pattern = value.$like as string
translated[key] = {
$regex: `^${pattern.split('%').join('.*')}$`,
$options: 'i'
}
continue
}
}
translated[key] = value
}
return translated
}
async find<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
options?: FindOptions<T> | undefined
): Promise<T[]> {
let cursor = this.db.collection(domain).find<T>(this.translateQuery(query))
if (options?.limit !== undefined) {
cursor = cursor.limit(options.limit)
}
if (options !== null && options !== undefined) {
if (options.sort !== undefined) {
const sort: Sort = {}
for (const key in options.sort) {
const order = options.sort[key] === SortingOrder.Ascending ? 1 : -1
sort[key] = order
}
cursor = cursor.sort(sort)
}
}
return await cursor.toArray()
return await this.lowLevel.rawFindAll(domain, query, options)
}
async traverse<T extends Doc>(
@ -76,68 +47,13 @@ export class MigrateClientImpl implements MigrationClient {
query: DocumentQuery<T>,
options?: FindOptions<T> | undefined
): Promise<MigrationIterator<T>> {
let cursor = this.db.collection(domain).find<T>(this.translateQuery(query))
if (options?.limit !== undefined) {
cursor = cursor.limit(options.limit)
}
if (options !== null && options !== undefined) {
if (options.sort !== undefined) {
const sort: Sort = {}
for (const key in options.sort) {
const order = options.sort[key] === SortingOrder.Ascending ? 1 : -1
sort[key] = order
}
cursor = cursor.sort(sort)
}
}
return {
next: async (size: number) => {
const docs: T[] = []
while (docs.length < size && (await cursor.hasNext())) {
try {
const d = await cursor.next()
if (d !== null) {
docs.push(d)
} else {
break
}
} catch (err) {
console.error(err)
return null
}
}
return docs
},
close: async () => {
await cursor.close()
}
}
return await this.lowLevel.traverse(domain, query, options)
}
async update<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
operations: MigrateUpdate<T>
): Promise<MigrationResult> {
async update<T extends Doc>(domain: Domain, query: DocumentQuery<T>, operations: MigrateUpdate<T>): Promise<void> {
const t = Date.now()
try {
if (isOperator(operations)) {
if (operations?.$set !== undefined) {
operations.$set['%hash%'] = null
} else {
operations = { ...operations, $set: { '%hash%': null } }
}
const result = await this.db
.collection(domain)
.updateMany(this.translateQuery(query), { ...operations } as unknown as UpdateFilter<Document>)
return { matched: result.matchedCount, updated: result.modifiedCount }
} else {
const result = await this.db
.collection(domain)
.updateMany(this.translateQuery(query), { $set: { ...operations, '%hash%': null } })
return { matched: result.matchedCount, updated: result.modifiedCount }
}
await this.lowLevel.rawUpdate(domain, query, operations)
} finally {
if (Date.now() - t > 1000) {
this.logger.log(`update${Date.now() - t > 5000 ? 'slow' : ''}`, { domain, query, time: Date.now() - t })
@ -148,60 +64,44 @@ export class MigrateClientImpl implements MigrationClient {
async bulk<T extends Doc>(
domain: Domain,
operations: { filter: DocumentQuery<T>, update: MigrateUpdate<T> }[]
): Promise<MigrationResult> {
const result = await this.db.collection(domain).bulkWrite(
operations.map((it) => ({
updateOne: {
filter: this.translateQuery(it.filter),
update: { $set: { ...it.update, '%hash%': null } }
): Promise<void> {
for (const ops of operations) {
await this.lowLevel.rawUpdate(domain, ops.filter, ops.update)
}
}))
)
return { matched: result.matchedCount, updated: result.modifiedCount }
}
async move<T extends Doc>(
sourceDomain: Domain,
query: DocumentQuery<T>,
targetDomain: Domain
): Promise<MigrationResult> {
async move<T extends Doc>(sourceDomain: Domain, query: DocumentQuery<T>, targetDomain: Domain): Promise<void> {
const ctx = new MeasureMetricsContext('move', {})
this.logger.log('move', { sourceDomain, query })
const q = this.translateQuery(query)
const cursor = this.db.collection(sourceDomain).find<T>(q)
const target = this.db.collection(targetDomain)
const result: MigrationResult = {
matched: 0,
updated: 0
while (true) {
const source = await this.lowLevel.rawFindAll(sourceDomain, query, { limit: 500 })
if (source.length === 0) break
await this.lowLevel.upload(ctx, targetDomain, source)
await this.lowLevel.clean(
ctx,
sourceDomain,
source.map((p) => p._id)
)
}
let doc: Document | null
while ((doc = await cursor.next()) != null) {
if ('%hash%' in doc) {
delete doc['%hash%']
}
await target.insertOne(doc)
result.matched++
result.updated++
}
await this.db.collection(sourceDomain).deleteMany(q)
return result
}
async create<T extends Doc>(domain: Domain, doc: T | T[]): Promise<void> {
if (Array.isArray(doc)) {
if (doc.length > 0) {
await this.db.collection(domain).insertMany(doc as Document[])
}
} else {
await this.db.collection(domain).insertOne(doc as Document)
}
const ctx = new MeasureMetricsContext('create', {})
await this.lowLevel.upload(ctx, domain, Array.isArray(doc) ? doc : [doc])
}
async delete<T extends Doc>(domain: Domain, _id: Ref<T>): Promise<void> {
await this.db.collection(domain).deleteOne({ _id })
const ctx = new MeasureMetricsContext('delete', {})
await this.lowLevel.clean(ctx, domain, [_id])
}
async deleteMany<T extends Doc>(domain: Domain, query: DocumentQuery<T>): Promise<void> {
await this.db.collection<Doc>(domain).deleteMany(query as any)
const ctx = new MeasureMetricsContext('deleteMany', {})
const docs = await this.lowLevel.rawFindAll(domain, query)
await this.lowLevel.clean(
ctx,
domain,
docs.map((d) => d._id)
)
}
}

View File

@ -1,5 +1,7 @@
import core, {
getWorkspaceId,
Hierarchy,
ModelDb,
systemAccountEmail,
TxOperations,
versionToString,
@ -24,6 +26,8 @@ import {
import {
createIndexStages,
createServerPipeline,
getServerPipeline,
getTxAdapterFactory,
registerServerPlugins,
registerStringLoaders
} from '@hcengineering/server-pipeline'
@ -118,6 +122,25 @@ export async function createWorkspace (
await handleWsEvent?.('create-started', version, 10)
const { mongodbUri, dbUrl } = prepareTools([])
const dbUrls = dbUrl !== undefined ? `${dbUrl};${mongodbUri}` : mongodbUri
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
try {
const txFactory = getTxAdapterFactory(ctx, dbUrls, wsUrl, null, {
externalStorage: storageAdapter,
fullTextUrl: 'http://localhost:9200',
indexParallel: 0,
indexProcessing: 0,
rekoniUrl: '',
usePassedCtx: true
})
const txAdapter = await txFactory(ctx, hierarchy, dbUrl ?? mongodbUri, wsId, modelDb, storageAdapter)
await childLogger.withLog('init-workspace', {}, async (ctx) => {
const deleteModelFirst = workspaceInfo.mode === 'creating'
@ -125,6 +148,8 @@ export async function createWorkspace (
ctx,
wsId,
txes,
txAdapter,
storageAdapter,
ctxModellogger,
async (value) => {
await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10))
@ -132,17 +157,11 @@ export async function createWorkspace (
deleteModelFirst
)
})
const { mongodbUri } = prepareTools([])
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
try {
registerServerPlugins()
registerStringLoaders()
const factory: PipelineFactory = createServerPipeline(
ctx,
mongodbUri,
dbUrls,
{
externalStorage: storageAdapter,
fullTextUrl: 'http://localhost:9200',
@ -174,7 +193,7 @@ export async function createWorkspace (
const pipeline = await factory(ctx, wsUrl, true, () => {}, null)
const client = new TxOperations(wrapPipeline(ctx, pipeline, wsUrl), core.account.System)
await updateModel(ctx, wsId, migrationOperation, client, ctxModellogger, async (value) => {
await updateModel(ctx, wsId, migrationOperation, client, pipeline, ctxModellogger, async (value) => {
await handleWsEvent?.('progress', version, 20 + Math.round((Math.min(value, 100) / 100) * 10))
})
@ -221,13 +240,19 @@ export async function upgradeWorkspace (
if (ws?.version !== undefined && !forceUpdate && versionStr === versionToString(ws.version)) {
return
}
ctx.info('upgrading', {
force: forceUpdate,
currentVersion: ws?.version !== undefined ? versionToString(ws.version) : '',
toVersion: versionStr,
workspace: ws.workspace
})
const wsId = getWorkspaceId(ws.workspace)
const wsId: WorkspaceIdWithUrl = {
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
}
const token = generateToken(systemAccountEmail, wsId, { service: 'workspace' })
let progress = 0
@ -235,14 +260,38 @@ export async function upgradeWorkspace (
void handleWsEvent?.('progress', version, progress)
}, 5000)
const { mongodbUri, dbUrl } = prepareTools([])
const wsUrl: WorkspaceIdWithUrl = {
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
}
const { pipeline, storageAdapter } = await getServerPipeline(ctx, mongodbUri, dbUrl, wsUrl)
const contextData = new SessionDataImpl(
systemAccountEmail,
'backup',
true,
{ targets: {}, txes: [] },
wsUrl,
null,
false,
new Map(),
new Map(),
pipeline.context.modelDb
)
ctx.contextData = contextData
try {
await handleWsEvent?.('upgrade-started', version, 0)
await upgradeModel(
ctx,
await getTransactorEndpoint(token, external ? 'external' : 'internal'),
getWorkspaceId(ws.workspace),
wsId,
txes,
pipeline,
storageAdapter,
migrationOperation,
logger,
false,
@ -254,8 +303,11 @@ export async function upgradeWorkspace (
await handleWsEvent?.('upgrade-done', version, 100, '')
} catch (err: any) {
ctx.error('upgrade-failed', { message: err.message })
await handleWsEvent?.('ping', version, 0, `Upgrade failed: ${err.message}`)
} finally {
await pipeline.close()
await storageAdapter.close()
clearInterval(updateProgressHandle)
}
}

View File

@ -0,0 +1,10 @@
services:
account:
environment:
- DB_URL=postgresql://postgres:example@postgres:5432
transactor:
environment:
- MONGO_URL=postgresql://postgres:example@postgres:5432;mongodb://mongodb:27018
workspace:
environment:
- DB_URL=postgresql://postgres:example@postgres:5432

View File

@ -8,6 +8,13 @@ services:
ports:
- 27018:27018
restart: unless-stopped
postgres:
image: postgres
environment:
- POSTGRES_PASSWORD=example
ports:
- 5433:5432
restart: unless-stopped
minio:
image: 'minio/minio'
command: server /data --address ":9000" --console-address ":9001"

30
tests/prepare-pg.sh Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env bash
docker compose -p sanity kill
docker compose -p sanity down --volumes
docker compose -p sanity up -d --force-recreate --renew-anon-volumes
docker_exit=$?
if [ ${docker_exit} -eq 0 ]; then
echo "Container started successfully"
else
echo "Container started with errors"
exit ${docker_exit}
fi
if [ "x$DO_CLEAN" == 'xtrue' ]; then
echo 'Do docker Clean'
docker system prune -a -f
fi
./wait-elastic.sh 9201
# Create workspace record in accounts
./tool-pg.sh create-workspace sanity-ws -w SanityTest
# Create user record in accounts
./tool-pg.sh create-account user1 -f John -l Appleseed -p 1234
./tool-pg.sh create-account user2 -f Kainin -l Dirak -p 1234
# Make user the workspace maintainer
./tool-pg.sh confirm-email user1
./tool-pg.sh confirm-email user2
./restore-pg.sh

View File

@ -2,7 +2,7 @@
docker compose -p sanity kill
docker compose -p sanity down --volumes
docker compose -p sanity up elastic mongodb -d --force-recreate --renew-anon-volumes
docker compose -f docker-compose.yaml -p sanity up elastic mongodb postgres -d --force-recreate --renew-anon-volumes
docker_exit=$?
if [ ${docker_exit} -eq 0 ]; then
echo "Container started successfully"

View File

@ -2,7 +2,7 @@
docker compose -p sanity kill
docker compose -p sanity down --volumes
docker compose -p sanity up -d --force-recreate --renew-anon-volumes
docker compose -p sanity -f docker-compose.yaml up -d --force-recreate --renew-anon-volumes
docker_exit=$?
if [ ${docker_exit} -eq 0 ]; then
echo "Container started successfully"

18
tests/restore-pg.sh Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Restore workspace contents in mongo/elastic
./tool-pg.sh backup-restore ./sanity-ws sanity-ws
./tool-pg.sh upgrade-workspace sanity-ws
# Re-assign user to workspace.
./tool-pg.sh assign-workspace user1 sanity-ws
./tool-pg.sh assign-workspace user2 sanity-ws
./tool-pg.sh set-user-role user1 sanity-ws OWNER
./tool-pg.sh set-user-role user2 sanity-ws OWNER
./tool-pg.sh configure sanity-ws --enable=*
./tool-pg.sh configure sanity-ws --list
# setup issue createdOn for yesterday
./tool-pg.sh change-field sanity-ws --objectId 65e47f1f1b875b51e3b4b983 --objectClass tracker:class:Issue --attribute createdOn --value $(($(date +%s)*1000 - 86400000)) --type number --domain task

View File

@ -84,7 +84,6 @@ test.describe('candidate/talents tests', () => {
const sourceTalent1 = 'SourceTalent1'
await talentDetailsPage.addSource(sourceTalent1)
await talentDetailsPage.addSocialLinks('Phone', '123123213213')
await talentDetailsPage.checkSocialLinks('Phone', '123123213213')
// talent 2
await navigationMenuPage.clickButtonTalents()
@ -96,7 +95,6 @@ test.describe('candidate/talents tests', () => {
const sourceTalent2 = 'SourceTalent2'
await talentDetailsPage.addSource(sourceTalent2)
await talentDetailsPage.addSocialLinks('Email', 'test-merge-2@gmail.com')
await talentDetailsPage.checkSocialLinks('Email', 'test-merge-2@gmail.com')
// merge
await navigationMenuPage.clickButtonTalents()

13
tests/tool-pg.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
export MINIO_ACCESS_KEY=minioadmin
export MINIO_SECRET_KEY=minioadmin
export MINIO_ENDPOINT=localhost:9002
export ACCOUNTS_URL=http://localhost:3003
export TRANSACTOR_URL=ws://localhost:3334
export MONGO_URL=mongodb://localhost:27018
export ELASTIC_URL=http://localhost:9201
export SERVER_SECRET=secret
export DB_URL=postgresql://postgres:example@localhost:5433
node ../dev/tool/bundle/bundle.js $@