hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 19:11:33 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-6756: Tracker performance fixes (#5488)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-05-03 12:57:05 +07:00 committed by GitHub
parent d5af222ec2
commit 1c3edbc395
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
36 changed files with 1032 additions and 499 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
common/config/rush/pnpm-lock.yaml
View File

@ -13,10 +13,10 @@ dependencies:
version: 7.17.13
'@hocuspocus/provider':
specifier: ^2.9.0
version: 2.11.2(bufferutil@4.0.8)(yjs@13.6.12)
version: 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12)
'@hocuspocus/server':
specifier: ^2.9.0
version: 2.11.2(bufferutil@4.0.8)(yjs@13.6.12)
version: 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12)
'@hocuspocus/transformer':
specifier: ^2.9.0
version: 2.11.2(@tiptap/pm@2.2.4)(y-prosemirror@1.2.2)(yjs@13.6.12)
@ -28,7 +28,7 @@ dependencies:
version: 1.41.2
'@rush-temp/account':
specifier: file:./projects/account.tgz
version: file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)
version: file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3)
'@rush-temp/account-service':
specifier: file:./projects/account-service.tgz
version: file:projects/account-service.tgz
@ -109,7 +109,7 @@ dependencies:
version: file:projects/collaboration.tgz(esbuild@0.20.1)(ts-node@10.9.2)
'@rush-temp/collaborator':
specifier: file:./projects/collaborator.tgz
version: file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)
version: file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)(utf-8-validate@6.0.3)
'@rush-temp/collaborator-client':
specifier: file:./projects/collaborator-client.tgz
version: file:projects/collaborator-client.tgz(ts-node@10.9.2)
@ -451,7 +451,7 @@ dependencies:
version: file:projects/presentation.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2)
'@rush-temp/prod':
specifier: file:./projects/prod.tgz
version: file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2)
version: file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2)(utf-8-validate@6.0.3)
'@rush-temp/query':
specifier: file:./projects/query.tgz
version: file:projects/query.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
@ -622,7 +622,7 @@ dependencies:
version: file:projects/server-token.tgz(esbuild@0.20.1)(ts-node@10.9.2)
'@rush-temp/server-tool':
specifier: file:./projects/server-tool.tgz
version: file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)
version: file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3)
'@rush-temp/server-tracker':
specifier: file:./projects/server-tracker.tgz
version: file:projects/server-tracker.tgz(esbuild@0.20.1)(ts-node@10.9.2)
@ -649,7 +649,7 @@ dependencies:
version: file:projects/setting-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2)
'@rush-temp/storybook':
specifier: file:./projects/storybook.tgz
version: file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(webpack-cli@5.1.4)(webpack@5.90.3)
version: file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3)
'@rush-temp/support':
specifier: file:./projects/support.tgz
version: file:projects/support.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
@ -700,10 +700,10 @@ dependencies:
version: file:projects/tests-sanity.tgz
'@rush-temp/text':
specifier: file:./projects/text.tgz
version: file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)
version: file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3)
'@rush-temp/text-editor':
specifier: file:./projects/text-editor.tgz
version: file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)
version: file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)(utf-8-validate@6.0.3)
'@rush-temp/theme':
specifier: file:./projects/theme.tgz
version: file:projects/theme.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2)
@ -718,7 +718,7 @@ dependencies:
version: file:projects/time-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2)
'@rush-temp/tool':
specifier: file:./projects/tool.tgz
version: file:projects/tool.tgz(bufferutil@4.0.8)
version: file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.3)
'@rush-temp/tracker':
specifier: file:./projects/tracker.tgz
version: file:projects/tracker.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
@ -1111,7 +1111,7 @@ dependencies:
version: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
jest-environment-jsdom:
specifier: 29.7.0
version: 29.7.0(bufferutil@4.0.8)
version: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
jwt-simple:
specifier: ^0.5.6
version: 0.5.6
@ -1231,7 +1231,7 @@ dependencies:
version: 2.0.5
storybook:
specifier: ^7.0.6
version: 7.6.17(bufferutil@4.0.8)
version: 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3)
storybook-addon-themes:
specifier: ^6.1.0
version: 6.1.0(react-dom@18.2.0)(react@18.2.0)(svelte@4.2.12)
@ -1280,6 +1280,9 @@ dependencies:
url-loader:
specifier: ~4.1.1
version: 4.1.1(file-loader@6.2.0)(webpack@5.90.3)
utf-8-validate:
specifier: ^6.0.3
version: 6.0.3
uuid:
specifier: ^8.3.2
version: 8.3.2
@ -1291,16 +1294,16 @@ dependencies:
version: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4)
webpack-bundle-analyzer:
specifier: ^4.7.0
version: 4.10.1(bufferutil@4.0.8)
version: 4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)
webpack-cli:
specifier: ^5.0.1
version: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3)
webpack-dev-server:
specifier: ^4.11.1
version: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3)
version: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3)
ws:
specifier: ^8.10.0
version: 8.16.0(bufferutil@4.0.8)
specifier: ^8.16.0
version: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
y-prosemirror:
specifier: ^1.2.1
version: 1.2.2(prosemirror-model@1.19.4)(yjs@13.6.12)
@ -3396,7 +3399,7 @@ packages:
lib0: 0.2.89
dev: false
/@hocuspocus/provider@2.11.2(bufferutil@4.0.8)(yjs@13.6.12):
/@hocuspocus/provider@2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12):
resolution: {integrity: sha512-h6/2KZC8Z3Wz+rP17WQ5eqxdgtsq5JZ4iC72WSXdIOJn3RO4sv63ZvT+EWc4aA1YM7UVDBYVjdOjMtEYZF0Flg==}
peerDependencies:
y-protocols: ^1.0.6
@ -3405,14 +3408,14 @@ packages:
'@hocuspocus/common': 2.11.2
'@lifeomic/attempt': 3.0.3
lib0: 0.2.89
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
yjs: 13.6.12
transitivePeerDependencies:
- bufferutil
- utf-8-validate
dev: false
/@hocuspocus/server@2.11.2(bufferutil@4.0.8)(yjs@13.6.12):
/@hocuspocus/server@2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12):
resolution: {integrity: sha512-/djaNUSS9vYmz5H/bformB9BXKVhSXS10WIURT1OqFLnN0dZnOB0gxL/IyGCZgXCNyiD8U03n7FIgad9MckV8g==}
peerDependencies:
y-protocols: ^1.0.6
@ -3423,7 +3426,7 @@ packages:
kleur: 4.1.5
lib0: 0.2.89
uuid: 9.0.1
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
yjs: 13.6.12
transitivePeerDependencies:
- bufferutil
@ -4889,7 +4892,7 @@ packages:
tiny-invariant: 1.3.1
dev: false
/@storybook/cli@7.6.17(bufferutil@4.0.8):
/@storybook/cli@7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-1sCo+nCqyR+nKfTcEidVu8XzNoECC7Y1l+uW38/r7s2f/TdDorXaIGAVrpjbSaXSoQpx5DxYJVaKCcQuOgqwcA==}
hasBin: true
dependencies:
@ -4900,7 +4903,7 @@ packages:
'@storybook/codemod': 7.6.17
'@storybook/core-common': 7.6.17
'@storybook/core-events': 7.6.17
'@storybook/core-server': 7.6.17(bufferutil@4.0.8)
'@storybook/core-server': 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3)
'@storybook/csf-tools': 7.6.17
'@storybook/node-logger': 7.6.17
'@storybook/telemetry': 7.6.17
@ -5065,7 +5068,7 @@ packages:
ts-dedent: 2.2.0
dev: false
/@storybook/core-server@7.6.17(bufferutil@4.0.8):
/@storybook/core-server@7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-KWGhTTaL1Q14FolcoKKZgytlPJUbH6sbJ1Ptj/84EYWFewcnEgVs0Zlnh1VStRZg+Rd1WC1V4yVd/bbDzxrvQA==}
dependencies:
'@aw-web-design/x-default-browser': 1.4.126
@ -5108,7 +5111,7 @@ packages:
util: 0.12.5
util-deprecate: 1.0.2
watchpack: 2.4.0
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- bufferutil
- encoding
@ -6900,7 +6903,7 @@ packages:
dependencies:
webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4)
webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3)
webpack-dev-server: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3)
webpack-dev-server: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3)
dev: false
/@xtuc/ieee754@1.2.0:
@ -9998,6 +10001,10 @@ packages:
resolution: {integrity: sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==}
dev: false
/fflate@0.8.2:
resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
dev: false
/file-entry-cache@6.0.1:
resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==}
engines: {node: ^10.12.0 || >=12.0.0}
@ -11584,7 +11591,7 @@ packages:
pretty-format: 29.7.0
dev: false
/jest-environment-jsdom@29.7.0(bufferutil@4.0.8):
/jest-environment-jsdom@29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
peerDependencies:
@ -11600,7 +11607,7 @@ packages:
'@types/node': 20.11.19
jest-mock: 29.7.0
jest-util: 29.7.0
jsdom: 20.0.3(bufferutil@4.0.8)
jsdom: 20.0.3(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- bufferutil
- supports-color
@ -11958,7 +11965,7 @@ packages:
- supports-color
dev: false
/jsdom@20.0.3(bufferutil@4.0.8):
/jsdom@20.0.3(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-SYhBvTh89tTfCD/CRdSOm13mOBa42iTaTyfyEWBdKcGdPxPtLFBXuHR8XHb33YNYaP+lLbmSvBTsnoesCNJEsQ==}
engines: {node: '>=14'}
peerDependencies:
@ -11991,7 +11998,7 @@ packages:
whatwg-encoding: 2.0.0
whatwg-mimetype: 3.0.0
whatwg-url: 11.0.0
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
xml-name-validator: 4.0.0
transitivePeerDependencies:
- bufferutil
@ -15192,11 +15199,11 @@ packages:
- react-dom
dev: false
/storybook@7.6.17(bufferutil@4.0.8):
/storybook@7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-8+EIo91bwmeFWPg1eysrxXlhIYv3OsXrznTr4+4Eq0NikqAoq6oBhtlN5K2RGS2lBVF537eN+9jTCNbR+WrzDA==}
hasBin: true
dependencies:
'@storybook/cli': 7.6.17(bufferutil@4.0.8)
'@storybook/cli': 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- bufferutil
- encoding
@ -16333,6 +16340,14 @@ packages:
tslib: 2.6.2
dev: false
/utf-8-validate@6.0.3:
resolution: {integrity: sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==}
engines: {node: '>=6.14.2'}
requiresBuild: true
dependencies:
node-gyp-build: 4.8.0
dev: false
/util-deprecate@1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
dev: false
@ -16463,7 +16478,7 @@ packages:
engines: {node: '>=12'}
dev: false
/webpack-bundle-analyzer@4.10.1(bufferutil@4.0.8):
/webpack-bundle-analyzer@4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-s3P7pgexgT/HTUSYgxJyn28A+99mmLq4HsJepMPzu0R8ImJc52QNqaFYW1Z2z2uIb1/J3eYgaAWVpaC+v/1aAQ==}
engines: {node: '>= 10.13.0'}
hasBin: true
@ -16480,7 +16495,7 @@ packages:
opener: 1.5.2
picocolors: 1.0.0
sirv: 2.0.4
ws: 7.5.9(bufferutil@4.0.8)
ws: 7.5.9(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- bufferutil
- utf-8-validate
@ -16516,8 +16531,8 @@ packages:
interpret: 3.1.1
rechoir: 0.8.0
webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4)
webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8)
webpack-dev-server: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3)
webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)
webpack-dev-server: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3)
webpack-merge: 5.10.0
dev: false
@ -16552,7 +16567,7 @@ packages:
webpack: 5.90.3(@swc/core@1.4.2)(esbuild@0.20.1)(webpack-cli@5.1.4)
dev: false
/webpack-dev-server@4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3):
/webpack-dev-server@4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3):
resolution: {integrity: sha512-5hbAst3h3C3L8w6W4P96L5vaV0PxSmJhxZvWKYIdgxOQm8pNZ5dEOmmSLBVpP85ReeyRt6AS1QJNyo/oFFPeVA==}
engines: {node: '>= 12.13.0'}
hasBin: true
@ -16596,7 +16611,7 @@ packages:
webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4)
webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3)
webpack-dev-middleware: 5.3.3(webpack@5.90.3)
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- bufferutil
- debug
@ -16862,7 +16877,7 @@ packages:
async-limiter: 1.0.1
dev: false
/ws@7.5.9(bufferutil@4.0.8):
/ws@7.5.9(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==}
engines: {node: '>=8.3.0'}
peerDependencies:
@ -16875,9 +16890,10 @@ packages:
optional: true
dependencies:
bufferutil: 4.0.8
utf-8-validate: 6.0.3
dev: false
/ws@8.16.0(bufferutil@4.0.8):
/ws@8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==}
engines: {node: '>=10.0.0'}
peerDependencies:
@ -16890,6 +16906,7 @@ packages:
optional: true
dependencies:
bufferutil: 4.0.8
utf-8-validate: 6.0.3
dev: false
/xml-name-validator@4.0.0:
@ -17112,8 +17129,8 @@ packages:
- supports-color
dev: false
file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-9RuPhqNNHTYjQwezirzcJ6WZpMJTzbjc72jZSJC6dQFG7WqW0a2QZ8VTaa32IM902stPP950kaRUDGGEKlxEwg==, tarball: file:projects/account.tgz}
file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-5tvVeDuogAFbWW1kdbXHh0UYCEuA4w0d5mv1euYiTTq6Wrw+MUMCHVGylKlbc5DCApvmkjGYyGTBEQ6gtVDGSg==, tarball: file:projects/account.tgz}
id: file:projects/account.tgz
name: '@rush-temp/account'
version: 0.0.0
@ -17132,10 +17149,9 @@ packages:
mongodb: 6.3.0
node-fetch: 2.7.0
prettier: 3.2.5
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- '@aws-sdk/credential-providers'
- '@babel/core'
@ -18092,13 +18108,13 @@ packages:
- ts-node
dev: false
file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4):
resolution: {integrity: sha512-dz9Gj6jr2YoZbl+ZbR3XtYsHqcVRA0qjQ318BwKD+SjgiETupwiFVay/ltHyc+l2V0Ai7m6qSY5ymedU0EevPg==, tarball: file:projects/collaborator.tgz}
file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-r+sgjj6JmjY8Lkk/dYVeEm7AJtq3uCZDosWdJZPNIhtJo4fDeVYnMZQEawUKkjW0YcGE/KRnLvS6TCjFMH3ugw==, tarball: file:projects/collaborator.tgz}
id: file:projects/collaborator.tgz
name: '@rush-temp/collaborator'
version: 0.0.0
dependencies:
'@hocuspocus/server': 2.11.2(bufferutil@4.0.8)(yjs@13.6.12)
'@hocuspocus/server': 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12)
'@hocuspocus/transformer': 2.11.2(@tiptap/pm@2.2.4)(y-prosemirror@1.2.2)(yjs@13.6.12)
'@tiptap/core': 2.2.4(@tiptap/pm@2.2.4)
'@tiptap/html': 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
@ -18128,7 +18144,7 @@ packages:
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
y-prosemirror: 1.2.2(prosemirror-model@1.19.4)(yjs@13.6.12)
yjs: 13.6.12
transitivePeerDependencies:
@ -18614,7 +18630,7 @@ packages:
dev: false
file:projects/elastic.tgz(@types/node@20.11.19)(esbuild@0.20.1):
resolution: {integrity: sha512-X3dHb4JxiTeFuXt6bUwzCBklB8BipruxEhhtGWhFbRxvTx689V0bfzoO62QUwc+Kqmz2Q9Df8IPYy51tCKyMvQ==, tarball: file:projects/elastic.tgz}
resolution: {integrity: sha512-iUKt8+1LhjnXTCYZv0DssFNwm1pHB+ioIePMutpYZrtO7HvwaZN4kHVneHjk5hA0EeE96JOkCwgYcPka35UvuA==, tarball: file:projects/elastic.tgz}
id: file:projects/elastic.tgz
name: '@rush-temp/elastic'
version: 0.0.0
@ -21254,7 +21270,7 @@ packages:
- ts-node
dev: false
file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2):
file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-/CKLzPjaTfkDNjT0evklXES1ISDy11ikLCJKFuyI4OQ6nfbt84wTqkO80Egr9IZScuNA0M0MEeJP+01pO2LNdA==, tarball: file:projects/prod.tgz}
id: file:projects/prod.tgz
name: '@rush-temp/prod'
@ -21286,9 +21302,9 @@ packages:
typescript: 5.3.3
update-browserslist-db: 1.0.13(browserslist@4.21.5)
webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4)
webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8)
webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)
webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3)
webpack-dev-server: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3)
webpack-dev-server: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3)
transitivePeerDependencies:
- '@babel/core'
- '@rspack/core'
@ -23062,8 +23078,8 @@ packages:
- ts-node
dev: false
file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-La/JqbJn8mFb4MYQ5DKwjn5sxUJ0VYBI8UEcP2iC3KIcvjRrQ0Rcy6q5QBeVpYRpfnAcvUvFCuwR8i1TklseZw==, tarball: file:projects/server-tool.tgz}
file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-dPn+fdjcD0BlDVCQ4DZD7zT16BfgObaomZEf+3r/ZpLxZTI+OiEpZUIQlEjgPkW6mgFui74nIcW5MFNO5uJnNA==, tarball: file:projects/server-tool.tgz}
id: file:projects/server-tool.tgz
name: '@rush-temp/server-tool'
version: 0.0.0
@ -23081,10 +23097,9 @@ packages:
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.3.0
prettier: 3.2.5
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- '@aws-sdk/credential-providers'
- '@babel/core'
@ -23231,7 +23246,7 @@ packages:
dev: false
file:projects/server-ws.tgz(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-U8RVqOkhC0aFRzjZzXrGwKzS+keJrlDuQOtap4w7XOXaHaZO+/dWWw2u3cTTRWdt3xKDFwDAAwiHaBkviaGGtg==, tarball: file:projects/server-ws.tgz}
resolution: {integrity: sha512-H+LX8du8Fflohly6Uv20gI7fhm+T6RRerOESTho9L1Dxgn0E8zXqD3XGrjEblNm94HR9/09uHrs2a7CWPaRAig==, tarball: file:projects/server-ws.tgz}
id: file:projects/server-ws.tgz
name: '@rush-temp/server-ws'
version: 0.0.0
@ -23253,12 +23268,14 @@ packages:
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
express: 4.18.3
fflate: 0.8.2
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
prettier: 3.2.5
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
ws: 8.16.0(bufferutil@4.0.8)
utf-8-validate: 6.0.3
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@ -23268,7 +23285,6 @@ packages:
- node-notifier
- supports-color
- ts-node
- utf-8-validate
dev: false
file:projects/server.tgz(esbuild@0.20.1):
@ -23415,7 +23431,7 @@ packages:
- ts-node
dev: false
file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(webpack-cli@5.1.4)(webpack@5.90.3):
file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3):
resolution: {integrity: sha512-7yMMWo7RZ6G1yC1wGli2iB6twTB9Tz0cGGgH8xxK/nAbiXdU6CVsv/qdxV/aCnLq9hvLMJY9CaOrFMrqskV3mA==, tarball: file:projects/storybook.tgz}
id: file:projects/storybook.tgz
name: '@rush-temp/storybook'
@ -23434,7 +23450,7 @@ packages:
resolve-url-loader: 5.0.0
sass: 1.71.1
sass-loader: 13.3.3(sass@1.71.1)(webpack@5.90.3)
storybook: 7.6.17(bufferutil@4.0.8)
storybook: 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3)
storybook-addon-themes: 6.1.0(react-dom@18.2.0)(react@18.2.0)(svelte@4.2.12)
svelte-preprocess: 5.1.3(postcss-load-config@4.0.2)(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.12)(typescript@5.3.3)
transitivePeerDependencies:
@ -24028,13 +24044,13 @@ packages:
- supports-color
dev: false
file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2):
file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-HrOMU/Hpc05gePqahXl1WJ/jZC+FUOTC4qbSeK9whq4PYxp6A7Uy5apkN8ww5gDlHs7TW5c69rVGBnbHnqJhsw==, tarball: file:projects/text-editor.tgz}
id: file:projects/text-editor.tgz
name: '@rush-temp/text-editor'
version: 0.0.0
dependencies:
'@hocuspocus/provider': 2.11.2(bufferutil@4.0.8)(yjs@13.6.12)
'@hocuspocus/provider': 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12)
'@tiptap/core': 2.2.4(@tiptap/pm@2.2.4)
'@tiptap/extension-bubble-menu': 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
'@tiptap/extension-code': 2.2.4(@tiptap/core@2.2.4)
@ -24062,7 +24078,6 @@ packages:
'@tiptap/suggestion': 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
'@types/diff': 5.0.9
'@types/jest': 29.5.12
'@types/png-chunks-extract': 1.0.2
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
diff: 5.2.0
@ -24076,7 +24091,6 @@ packages:
filesize: 8.0.7
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
lib0: 0.2.89
png-chunks-extract: 1.0.0
prettier: 3.2.5
prettier-plugin-svelte: 3.2.2(prettier@3.2.5)(svelte@4.2.12)
prosemirror-codemark: 0.4.2(prosemirror-model@1.19.4)
@ -24119,7 +24133,7 @@ packages:
- y-protocols
dev: false
file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2):
file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-+71BisKHNmsLjMHsBChfyRUJXOP7P2gOAIHC1JP8z/AHvuGfWZirZRyYtRBY1E5PFu7qaI6G4SM2ARIvKBAaxw==, tarball: file:projects/text.tgz}
id: file:projects/text.tgz
name: '@rush-temp/text'
@ -24155,7 +24169,7 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
fast-equals: 5.0.1
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
jest-environment-jsdom: 29.7.0(bufferutil@4.0.8)
jest-environment-jsdom: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
prettier: 3.2.5
prosemirror-model: 1.19.4
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
@ -24333,8 +24347,8 @@ packages:
- ts-node
dev: false
file:projects/tool.tgz(bufferutil@4.0.8):
resolution: {integrity: sha512-46NgBZuhcTT7XEc+NxbKu9abwTd63lNVTDR8WrDt8aLuSxorolUYMJSiZcaagz85/eGcn21Q25yvzvhEt2N0Sw==, tarball: file:projects/tool.tgz}
file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.3):
resolution: {integrity: sha512-FapBriQDbhOPHpCCh3hiJAY68tHjzbF6r94xISTJxwsOsJNijAV9v7RFhTUnLiLcCuBSj9eovI7Mljd3CDZrAw==, tarball: file:projects/tool.tgz}
id: file:projects/tool.tgz
name: '@rush-temp/tool'
version: 0.0.0
@ -24365,11 +24379,10 @@ packages:
mime-types: 2.1.35
mongodb: 6.3.0
prettier: 3.2.5
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
ws: 8.16.0(bufferutil@4.0.8)
ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
transitivePeerDependencies:
- '@aws-sdk/credential-providers'
- '@babel/core'

2
dev/tool/package.json
View File

@ -136,6 +136,6 @@
"libphonenumber-js": "^1.9.46",
"mime-types": "~2.1.34",
"mongodb": "^6.3.0",
"ws": "^8.10.0"
"ws": "^8.16.0"
}
}

295
dev/tool/src/benchmark.ts
View File

@ -13,24 +13,20 @@
// limitations under the License.
//
import contact from '@hcengineering/contact'
import core, {
type Account,
type BackupClient,
ClassifierKind,
type Client,
DOMAIN_TX,
type Doc,
type DocumentUpdate,
MeasureMetricsContext,
type Metrics,
type Ref,
RateLimiter,
TxOperations,
type WorkspaceId,
generateId,
metricsToString,
newMetrics,
systemAccountEmail
systemAccountEmail,
type Account,
type BackupClient,
type BenchmarkDoc,
type Client,
type Metrics,
type Ref,
type WorkspaceId
} from '@hcengineering/core'
import { generateToken } from '@hcengineering/server-token'
import { connect } from '@hcengineering/server-tool'
@ -48,8 +44,9 @@ interface StartMessage {
idd: number
workId: string
options: {
readRequests: number
modelRequests: number
smallRequests: number
rate: number
bigRequests: number
limit: {
min: number
rand: number
@ -62,7 +59,7 @@ interface StartMessage {
compression: boolean
}
interface Msg {
type: 'complete' | 'operate'
type: 'complete' | 'operate' | 'pending'
}
interface CompleteMsg extends Msg {
@ -70,19 +67,18 @@ interface CompleteMsg extends Msg {
workId: string
}
// interface OperatingMsg extends Msg {
// type: 'operate'
// workId: string
// }
const benchAccount = (core.account.System + '_benchmark') as Ref<Account>
interface PendingMsg extends Msg {
type: 'pending'
workId: string
pending: number
}
export async function benchmark (
workspaceId: WorkspaceId[],
transactorUrl: string,
cmd: { from: number, steps: number, sleep: number, binary: boolean, write: boolean, compression: boolean }
): Promise<void> {
let operating = 0
const operating = new Set<string>()
const workers: Worker[] = []
const works = new Map<string, () => void>()
@ -92,7 +88,6 @@ export async function benchmark (
os.cpus().forEach(() => {
/* Spawn a new thread running this source file */
console.error('__filename', __filename)
const worker = new Worker(__filename, {
argv: ['benchmarkWorker']
})
@ -102,13 +97,19 @@ export async function benchmark (
return
}
if (data.type === 'operate') {
operating += 1
operating.add((data as any).workId)
}
if (data.type === 'pending') {
const msg = data as PendingMsg
console.log('info', `worker:${msg.workId}`, msg.pending)
}
if (data.type === 'complete') {
const resolve = works.get((data as CompleteMsg).workId)
if (resolve != null) {
resolve()
operating -= 1
operating.delete((data as any).workId)
} else {
console.log('Worker failed to done', (data as CompleteMsg).workId)
}
}
})
@ -150,7 +151,9 @@ export async function benchmark (
let elapsed = 0
let requestTime: number = 0
let operations = 0
let oldOperations = 0
let transfer: number = 0
let oldTransfer: number = 0
const token = generateToken(systemAccountEmail, workspaceId[0])
@ -162,16 +165,6 @@ export async function benchmark (
)) as BackupClient & Client)
: undefined
if (monitorConnection !== undefined) {
// We need to cleanup all transactions from our benchmark account.
const txes = await monitorConnection.findAll(
core.class.Tx,
{ modifiedBy: benchAccount },
{ projection: { _id: 1 } }
)
await monitorConnection.clean(DOMAIN_TX, Array.from(txes.map((it) => it._id)))
}
let running = false
function extract (metrics: Metrics, ...path: string[]): Metrics | null {
@ -193,7 +186,7 @@ export async function benchmark (
}
let timer: any
if (isMainThread) {
if (isMainThread && monitorConnection !== undefined) {
timer = setInterval(() => {
const st = Date.now()
@ -207,9 +200,9 @@ export async function benchmark (
memUsed = json.statistics.memoryUsed
memTotal = json.statistics.memoryTotal
cpu = json.statistics.cpuUsage
operations = 0
// operations = 0
requestTime = 0
transfer = 0
// transfer = 0
const r = extract(
json.metrics as Metrics,
'🧲 session',
@ -218,11 +211,14 @@ export async function benchmark (
'process',
'find-all'
)
operations += r?.operations ?? 0
requestTime += (r?.value ?? 0) / (((r?.operations as number) ?? 0) + 1)
operations = (r?.operations ?? 0) - oldOperations
oldOperations = r?.operations ?? 0
requestTime = (r?.value ?? 0) / (((r?.operations as number) ?? 0) + 1)
const tr = extract(json.metrics as Metrics, '🧲 session', 'client', 'handleRequest', '#send-data')
transfer += tr?.value ?? 0
transfer = (tr?.value ?? 0) - oldTransfer
oldTransfer = tr?.value ?? 0
})
.catch((err) => {
console.log(err)
@ -237,83 +233,89 @@ export async function benchmark (
if (!running) {
running = true
void ctx.with(
'avg',
{},
async () =>
await monitorConnection?.findAll(contact.mixin.Employee, {}).then((res) => {
void ctx.with('avg', {}, async () => {
await monitorConnection
?.findAll(core.class.BenchmarkDoc, {
source: 'monitor',
request: {
documents: 1,
size: 1
}
})
.then((res) => {
const cur = Date.now() - st
opTime += cur
moment = cur
ops++
running = false
})
)
})
}
elapsed++
// console.log('Sheduled', scheduled)
csvWriter.add(
{
time: elapsed,
clients: operating,
clients: operating.size,
moment,
average: Math.round(opTime / (ops + 1)),
mem: memUsed,
memTotal,
cpu,
requestTime,
operations: Math.round(operations / (elapsed + 1)),
transfer: Math.round(transfer / (elapsed + 1)) / 1024
operations,
transfer: transfer / 1024
},
true
)
void csvWriter.write(`report${cmd.binary ? '-bin' : ''}${cmd.write ? '-wr' : ''}.csv`)
}, 1000)
}
for (let i = cmd.from; i < cmd.from + cmd.steps; i++) {
await ctx.with('iteration', { i }, async () => {
await Promise.all(
Array.from(Array(i))
.map((it, idx) => idx)
.map((it) => {
const wsid = workspaceId[randNum(workspaceId.length)]
const workId = generateId()
const msg: StartMessage = {
workspaceId: wsid,
transactorUrl,
id: i,
idd: it,
workId,
options: {
readRequests: 100,
modelRequests: 0,
limit: {
min: 10,
rand: 1000
},
sleep: cmd.sleep,
write: cmd.write
},
binary: cmd.binary,
compression: cmd.compression
}
workers[i % workers.length].postMessage(msg)
return new Promise((resolve) => {
works.set(workId, () => {
resolve(null)
for (let i = cmd.from; i < cmd.from + cmd.steps; i++) {
await ctx.with('iteration', { i }, async () => {
await Promise.all(
Array.from(Array(i))
.map((it, idx) => idx)
.map((it) => {
const wsid = workspaceId[randNum(workspaceId.length)]
const workId = 'w-' + i + '-' + it
const msg: StartMessage = {
workspaceId: wsid,
transactorUrl,
id: i,
idd: it,
workId,
options: {
smallRequests: 100,
rate: 1,
bigRequests: 0,
limit: {
min: 64,
rand: 512
},
sleep: cmd.sleep,
write: cmd.write
},
binary: cmd.binary,
compression: cmd.compression
}
workers[i % workers.length].postMessage(msg)
return new Promise((resolve) => {
works.set(workId, () => {
resolve(null)
})
})
})
})
)
})
console.log(metricsToString(m, `iteration-${i}`, 120))
}
)
})
console.log(metricsToString(m, `iteration-${i}`, 120))
}
if (isMainThread) {
for (const w of workers) {
await w.terminate()
}
clearInterval(timer)
await csvWriter.write(`report${cmd.binary ? '-bin' : ''}${cmd.write ? '-wr' : ''}.csv`)
await monitorConnection?.close()
}
}
@ -344,65 +346,86 @@ export function benchmarkWorker (): void {
workId: msg.workId
})
const h = connection.getHierarchy()
const allClasses = await connection.getModel().findAll(core.class.Class, {})
const classes = allClasses.filter((it) => it.kind === ClassifierKind.CLASS && h.findDomain(it._id) !== undefined)
while (msg.options.readRequests + msg.options.modelRequests > 0) {
if (msg.options.modelRequests > 0) {
await connection?.findAll(core.class.Tx, {}, { sort: { _id: -1 } })
msg.options.modelRequests--
}
let doc: Doc | undefined
if (msg.options.readRequests > 0) {
const cl = classes[randNum(classes.length - 1)]
if (cl !== undefined) {
const docs = await connection?.findAll(
cl._id,
{},
{
sort: { _id: -1 },
limit: msg.options.limit.min + randNum(msg.options.limit.rand)
}
)
if (docs.length > 0) {
doc = docs[randNum(docs.length - 1)]
}
msg.options.readRequests--
}
if (msg.options.write && doc !== undefined) {
const attrs = connection.getHierarchy().getAllAttributes(doc._class)
const upd: DocumentUpdate<Doc> = {}
for (const [key, value] of attrs.entries()) {
if (value.type._class === core.class.TypeString || value.type._class === core.class.TypeBoolean) {
if (
key !== '_id' &&
key !== '_class' &&
key !== 'space' &&
key !== 'attachedTo' &&
key !== 'attachedToClass'
) {
const v = (doc as any)[key]
if (v != null) {
;(upd as any)[key] = v
}
const rateLimiter = new RateLimiter(msg.options.rate)
let bigRunning = 0
while (msg.options.smallRequests + msg.options.bigRequests > 0) {
const variant = Math.random()
// console.log(`Thread ${msg.workId} ${msg.options.smallRequests} ${msg.options.bigRequests}`)
if (msg.options.bigRequests > 0 && variant < 0.5 && bigRunning === 0) {
await rateLimiter.add(async () => {
bigRunning = 1
await connection?.findAll(core.class.BenchmarkDoc, {
source: msg.workId,
request: {
documents: {
from: 1024,
to: 1000
},
size: {
// 1kb to 5kb
from: 1 * 1024,
to: 4 * 1024
}
}
}
if (Object.keys(upd).length > 0) {
await opt.update(doc, upd)
}
}
})
})
bigRunning = 0
msg.options.bigRequests--
}
if (msg.options.smallRequests > 0 && variant > 0.5) {
await rateLimiter.add(async () => {
await connection?.findAll(core.class.BenchmarkDoc, {
source: msg.workId,
request: {
documents: {
from: msg.options.limit.min,
to: msg.options.limit.min + msg.options.limit.rand
},
size: {
from: 4,
to: 16
}
}
})
})
msg.options.smallRequests--
}
if (msg.options.write) {
await opt.updateDoc(core.class.BenchmarkDoc, core.space.DerivedTx, '' as Ref<BenchmarkDoc>, {
response: 'qwe'
})
}
if (msg.options.sleep > 0) {
await new Promise((resolve) => setTimeout(resolve, randNum(msg.options.sleep)))
}
}
// clearInterval(infoInterval)
await rateLimiter.waitProcessing()
const to1 = setTimeout(() => {
parentPort?.postMessage({
type: 'log',
workId: msg.workId,
msg: `timeout waiting processing${msg.workId}`
})
}, 5000)
clearTimeout(to1)
//
// console.log(`${msg.idd} perform complete`)
} catch (err: any) {
console.error(msg.workspaceId, err)
} finally {
const to = setTimeout(() => {
parentPort?.postMessage({
type: 'log',
workId: msg.workId,
msg: `timeout closing connection${msg.workId}`
})
}, 5000)
await connection?.close()
clearTimeout(to)
}
parentPort?.postMessage({
type: 'complete',

27
models/core/src/benchmark.ts Normal file
View File

@ -0,0 +1,27 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import type { BenchmarkDoc } from '@hcengineering/core'
import { DOMAIN_BENCHMARK } from '@hcengineering/core'
import { Model, UX } from '@hcengineering/model'
import { getEmbeddedLabel } from '@hcengineering/platform'
import core from './component'
import { TDoc } from './core'
// B E N C H M A R K
@Model(core.class.BenchmarkDoc, core.class.Doc, DOMAIN_BENCHMARK)
@UX(getEmbeddedLabel('Benchmark'), undefined, undefined, undefined, 'name')
export class TBenchmarkDoc extends TDoc implements BenchmarkDoc {}

11
models/core/src/index.ts
View File

@ -15,6 +15,7 @@
import {
AccountRole,
DOMAIN_BENCHMARK,
DOMAIN_BLOB,
DOMAIN_BLOB_DATA,
DOMAIN_CONFIGURATION,
@ -32,6 +33,7 @@ import {
type TxCollectionCUD
} from '@hcengineering/core'
import { type Builder } from '@hcengineering/model'
import { TBenchmarkDoc } from './benchmark'
import core from './component'
import {
TArrOf,
@ -172,7 +174,8 @@ export function createModel (builder: Builder): void {
TStatusCategory,
TMigrationState,
TBlob,
TDomainIndexConfiguration
TDomainIndexConfiguration,
TBenchmarkDoc
)
builder.createDoc(
@ -226,6 +229,12 @@ export function createModel (builder: Builder): void {
]
})
builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
domain: DOMAIN_BENCHMARK,
disableCollection: true,
disabled: []
})
builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
domain: DOMAIN_CONFIGURATION,
disabled: [

29
packages/core/src/benchmark.ts Normal file
View File

@ -0,0 +1,29 @@
import { Doc, type Domain } from './classes'
/**
* @public
*/
export const DOMAIN_BENCHMARK = 'benchmark' as Domain
export type BenchmarkDocRange =
| number
| {
// Or random in range
from: number
to: number
}
export interface BenchmarkDoc extends Doc {
source?: string
// Query fields to perform different set of workload
request?: {
// On response will return a set of BenchmarkDoc with requested fields.
documents: BenchmarkDocRange
// A random sized document with size from to sizeTo
size: BenchmarkDocRange
// Produce a set of derived documents payload
derived?: BenchmarkDocRange
}
response?: string // A dummy random data to match document's size
}

5
packages/core/src/component.ts
View File

@ -69,6 +69,7 @@ import type {
TxUpdateDoc,
TxWorkspaceEvent
} from './tx'
import type { BenchmarkDoc } from './benchmark'
/**
* @public
@ -142,7 +143,9 @@ export default plugin(coreId, {
Status: '' as Ref<Class<Status>>,
StatusCategory: '' as Ref<Class<StatusCategory>>,
MigrationState: '' as Ref<Class<MigrationState>>
MigrationState: '' as Ref<Class<MigrationState>>,
BenchmarkDoc: '' as Ref<Class<BenchmarkDoc>>
},
mixin: {
FullTextSearchContext: '' as Ref<Mixin<FullTextSearchContext>>,

1
packages/core/src/index.ts
View File

@ -33,3 +33,4 @@ export * from './status'
export * from './clone'
export * from './common'
export * from './time'
export * from './benchmark'

1
packages/core/src/storage.ts
View File

@ -196,6 +196,7 @@ export type WithLookup<T extends Doc> = T & {
*/
export type FindResult<T extends Doc> = WithLookup<T>[] & {
total: number
lookupMap?: Record<string, Doc>
}
/**

53
packages/core/src/utils.ts
View File

@ -106,9 +106,9 @@ export function escapeLikeForRegexp (value: string): string {
/**
* @public
*/
export function toFindResult<T extends Doc> (docs: T[], total?: number): FindResult<T> {
export function toFindResult<T extends Doc> (docs: T[], total?: number, lookupMap?: Record<string, Doc>): FindResult<T> {
const length = total ?? docs.length
return Object.assign(docs, { total: length })
return Object.assign(docs, { total: length, lookupMap })
}
/**
@ -370,7 +370,7 @@ export class RateLimiter {
}
async waitProcessing (): Promise<void> {
await Promise.race(this.processingQueue.values())
await Promise.all(this.processingQueue.values())
}
}
@ -447,6 +447,14 @@ function mergeField (field1: any, field2: any): any | undefined {
for (const x in predicate) {
const result = mergePredicateWithValue(x, predicate[x], value)
if (
Array.isArray(result?.$in) &&
result.$in.length > 0 &&
Array.isArray(result?.$nin) &&
result.$nin.length === 0
) {
delete result.$nin
}
if (result !== undefined) {
return result
}
@ -714,3 +722,42 @@ export function isClassIndexable (hierarchy: Hierarchy, c: Ref<Class<Doc>>): boo
hierarchy.setClassifierProp(c, 'class_indexed', result)
return result
}
type ReduceParameters<T extends (...args: any) => any> = T extends (...args: infer P) => any ? P : never
interface NextCall {
op: () => Promise<void>
}
/**
* Utility method to skip middle update calls, optimistically if update function is called multiple times with few different parameters, only the last variant will be executed.
* The last invocation is executed after a few cycles, allowing to skip middle ones.
*
* This method can be used inside Svelte components to collapse complex update logic and handle interactions.
*/
export function reduceCalls<T extends (...args: ReduceParameters<T>) => Promise<void>> (
operation: T
): (...args: ReduceParameters<T>) => Promise<void> {
let nextCall: NextCall | undefined
let currentCall: NextCall | undefined
const next = (): void => {
currentCall = nextCall
nextCall = undefined
if (currentCall !== undefined) {
void currentCall.op()
}
}
return async function (...args: ReduceParameters<T>): Promise<void> {
const myOp = async (): Promise<void> => {
await operation(...args)
next()
}
nextCall = { op: myOp }
await Promise.resolve()
if (currentCall === undefined) {
next()
}
}
}

55
packages/presentation/src/utils.ts
View File

@ -18,6 +18,7 @@ import { Analytics } from '@hcengineering/analytics'
import core, {
TxOperations,
getCurrentAccount,
reduceCalls,
type AnyAttribute,
type ArrOf,
type AttachedDoc,
@ -52,6 +53,7 @@ import { onDestroy } from 'svelte'
import { type KeyedAttribute } from '..'
import { OptimizeQueryMiddleware, PresentationPipelineImpl, type PresentationPipeline } from './pipeline'
import plugin from './plugin'
export { reduceCalls } from '@hcengineering/core'
let liveQuery: LQ
let client: TxOperations & MeasureClient
@ -248,10 +250,22 @@ export class LiveQuery {
// We need to prevent callback with old values to be happening
// One time refresh in case of client recreation
this.clientRecreated = false
this.doQuery<T>(++this.reqId, _class, query, callback, options)
void this.reducedDoQuery(++this.reqId, _class, query, callback as any, options)
return true
}
reducedDoQuery = reduceCalls(
async (
id: number,
_class: Ref<Class<Doc>>,
query: DocumentQuery<Doc>,
callback: (result: FindResult<Doc>) => void | Promise<void>,
options: FindOptions<Doc> | undefined
) => {
this.doQuery(id, _class, query, callback, options)
}
)
private doQuery<T extends Doc>(
id: number,
_class: Ref<Class<T>>,
@ -552,42 +566,3 @@ export function decodeTokenPayload (token: string): any {
export function isAdminUser (): boolean {
return decodeTokenPayload(getMetadata(plugin.metadata.Token) ?? '').admin === 'true'
}
type ReduceParameters<T extends (...args: any) => any> = T extends (...args: infer P) => any ? P : never
interface NextCall {
op: () => Promise<void>
}
/**
* Utility method to skip middle update calls, optimistically if update function is called multiple times with few different parameters, only the last variant will be executed.
* The last invocation is executed after a few cycles, allowing to skip middle ones.
*
* This method can be used inside Svelte components to collapse complex update logic and handle interactions.
*/
export function reduceCalls<T extends (...args: ReduceParameters<T>) => Promise<void>> (
operation: T
): (...args: ReduceParameters<T>) => Promise<void> {
let nextCall: NextCall | undefined
let currentCall: NextCall | undefined
const next = (): void => {
currentCall = nextCall
nextCall = undefined
if (currentCall !== undefined) {
void currentCall.op()
}
}
return async function (...args: ReduceParameters<T>): Promise<void> {
const myOp = async (): Promise<void> => {
await operation(...args)
next()
}
nextCall = { op: myOp }
await Promise.resolve()
if (currentCall === undefined) {
next()
}
}
}

2
plugins/chunter-resources/src/components/chat/Chat.svelte
View File

@ -70,7 +70,7 @@
$: void loadObject(selectedData?._id, selectedData?._class)
async function loadObject (_id?: Ref<Doc>, _class?: Ref<Class<Doc>>): Promise<void> {
if (_id === undefined || _class === undefined) {
if (_id == null || _class == null || _class === '') {
object = undefined
objectQuery.unsubscribe()
return

144
plugins/client-resources/src/connection.ts
View File

@ -41,7 +41,8 @@ import core, {
TxResult,
TxWorkspaceEvent,
WorkspaceEvent,
generateId
generateId,
toFindResult
} from '@hcengineering/core'
import { PlatformError, UNAUTHORIZED, broadcastEvent, getMetadata, unknownError } from '@hcengineering/platform'
@ -70,14 +71,23 @@ class RequestPromise {
})
}
chunks?: { index: number, data: any[] }[]
chunks?: { index: number, data: FindResult<any> }[]
}
class Connection implements ClientConnection {
private websocket: ClientSocket | null = null
binaryMode = false
private readonly requests = new Map<ReqId, RequestPromise>()
private lastId = 0
private interval: number | undefined
private dialTimer: any | undefined
private sockets = 0
private incomingTimer: any
private openAction: any
private sessionId: string | undefined
private closed = false
@ -136,15 +146,11 @@ class Connection implements ClientConnection {
async close (): Promise<void> {
this.closed = true
clearTimeout(this.openAction)
clearTimeout(this.dialTimer)
clearInterval(this.interval)
if (this.websocket !== null) {
if (this.websocket instanceof Promise) {
await this.websocket.then((ws) => {
ws.close(1000)
})
} else {
this.websocket.close(1000)
}
this.websocket.close(1000)
this.websocket = null
}
}
@ -166,12 +172,6 @@ class Connection implements ClientConnection {
})
}
sockets = 0
incomingTimer: any
openAction: any
scheduleOpen (force: boolean): void {
if (force) {
if (this.websocket !== null) {
@ -181,6 +181,7 @@ class Connection implements ClientConnection {
clearTimeout(this.openAction)
this.openAction = undefined
}
clearInterval(this.interval)
if (!this.closed && this.openAction === undefined) {
if (this.websocket === null) {
const socketId = ++this.sockets
@ -198,6 +199,7 @@ class Connection implements ClientConnection {
}
private openConnection (socketId: number): void {
this.binaryMode = false
// Use defined factory or browser default one.
const clientSocketFactory =
getMetadata(client.metadata.ClientSocketFactory) ??
@ -229,19 +231,21 @@ class Connection implements ClientConnection {
}
this.websocket = wsocket
const opened = false
let binaryResponse = false
const dialTimer = setTimeout(() => {
this.dialTimer = setTimeout(() => {
if (!opened && !this.closed) {
this.scheduleOpen(true)
}
}, dialTimeout)
wsocket.onmessage = (event: MessageEvent) => {
if (this.closed) {
return
}
if (this.websocket !== wsocket) {
return
}
const resp = readResponse<any>(event.data, binaryResponse)
const resp = readResponse<any>(event.data, this.binaryMode)
if (resp.error !== undefined) {
if (resp.error?.code === UNAUTHORIZED.code) {
@ -280,7 +284,7 @@ class Connection implements ClientConnection {
return
}
if ((resp as HelloResponse).binary) {
binaryResponse = true
this.binaryMode = true
}
// Notify all waiting connection listeners
const handlers = this.onConnectHandlers.splice(0, this.onConnectHandlers.length)
@ -309,7 +313,11 @@ class Connection implements ClientConnection {
if (resp.id !== undefined) {
const promise = this.requests.get(resp.id)
if (promise === undefined) {
throw new Error(`unknown response id: ${resp.id as string} ${this.workspace} ${this.email}`)
console.error(
new Error(`unknown response id: ${resp.id as string} ${this.workspace} ${this.email}`),
JSON.stringify(this.requests)
)
return
}
if (resp.chunk !== undefined) {
@ -317,17 +325,26 @@ class Connection implements ClientConnection {
...(promise.chunks ?? []),
{
index: resp.chunk.index,
data: resp.result as []
data: resp.result as FindResult<any>
}
]
// console.log(socketId, 'chunk', promise.chunks.length, resp.chunk.total)
// console.log(socketId, 'chunk', promise.method, promise.params, promise.chunks.length, (resp.result as []).length)
if (resp.chunk.final) {
promise.chunks.sort((a, b) => a.index - b.index)
let result: any[] = []
let total = -1
let lookupMap: Record<string, Doc> | undefined
for (const c of promise.chunks) {
if (c.data.total !== 0) {
total = c.data.total
}
if (c.data.lookupMap !== undefined) {
lookupMap = c.data.lookupMap
}
result = result.concat(c.data)
}
resp.result = result
resp.result = toFindResult(result, total, lookupMap)
resp.chunk = undefined
} else {
// Not all chunks are available yet.
@ -387,10 +404,10 @@ class Connection implements ClientConnection {
}
}
wsocket.onclose = (ev) => {
clearTimeout(dialTimer)
clearTimeout(this.dialTimer)
if (this.websocket !== wsocket) {
wsocket.close()
clearTimeout(dialTimer)
clearTimeout(this.dialTimer)
return
}
// console.log('client websocket closed', socketId, ev?.reason)
@ -403,7 +420,7 @@ class Connection implements ClientConnection {
}
const useBinary = getMetadata(client.metadata.UseBinaryProtocol) ?? true
const useCompression = getMetadata(client.metadata.UseProtocolCompression) ?? false
clearTimeout(dialTimer)
clearTimeout(this.dialTimer)
const helloRequest: HelloRequest = {
method: 'hello',
params: [],
@ -416,7 +433,7 @@ class Connection implements ClientConnection {
}
wsocket.onerror = (event: any) => {
clearTimeout(dialTimer)
clearTimeout(this.dialTimer)
if (this.websocket !== wsocket) {
return
}
@ -438,6 +455,7 @@ class Connection implements ClientConnection {
handleResult?: (result: any) => Promise<void>
once?: boolean // Require handleResult to retrieve result
measure?: (time: number, result: any, serverTime: number, queue: number) => void
allowReconnect?: boolean
}): Promise<any> {
if (this.closed) {
throw new PlatformError(unknownError('connection closed'))
@ -462,7 +480,7 @@ class Connection implements ClientConnection {
await w
}
this.requests.set(id, promise)
const sendData = (): void => {
const sendData = async (): Promise<void> => {
if (this.websocket?.readyState === ClientSocketReadyState.OPEN) {
promise.startTime = Date.now()
this.websocket?.send(
@ -470,22 +488,25 @@ class Connection implements ClientConnection {
{
method: data.method,
params: data.params,
id
id,
time: Date.now()
},
false
this.binaryMode
)
)
}
}
promise.reconnect = () => {
setTimeout(async () => {
// In case we don't have response yet.
if (this.requests.has(id) && ((await data.retry?.()) ?? true)) {
sendData()
}
}, 500)
if (data.allowReconnect ?? true) {
promise.reconnect = () => {
setTimeout(async () => {
// In case we don't have response yet.
if (this.requests.has(id) && ((await data.retry?.()) ?? true)) {
void sendData()
}
}, 50)
}
}
sendData()
void sendData()
void broadcastEvent(client.event.NetworkRequests, this.requests.size)
return await promise.promise
}
@ -527,11 +548,52 @@ class Connection implements ClientConnection {
method: 'findAll',
params: [_class, query, options],
measure: (time, result, serverTime, queue) => {
if (typeof window !== 'undefined' && time > 1000) {
console.error('measure slow findAll', time, serverTime, queue, _class, query, options, result)
if (typeof window !== 'undefined' && (time > 1000 || serverTime > 500)) {
console.error(
'measure slow findAll',
time,
serverTime,
queue,
_class,
query,
options,
result,
JSON.stringify(result).length
)
}
}
})
if (result.lookupMap !== undefined) {
// We need to extract lookup map to document lookups
for (const d of result) {
if (d.$lookup !== undefined) {
for (const [k, v] of Object.entries(d.$lookup)) {
if (!Array.isArray(v)) {
d.$lookup[k] = result.lookupMap[v as any]
} else {
d.$lookup[k] = v.map((it) => result.lookupMap?.[it])
}
}
}
}
delete result.lookupMap
}
// We need to revert deleted query simple values.
// We need to get rid of simple query parameters matched in documents
for (const doc of result) {
if (doc._class == null) {
doc._class = _class
}
for (const [k, v] of Object.entries(query)) {
if (typeof v === 'string' || typeof v === 'number') {
if (doc[k] == null) {
doc[k] = v
}
}
}
}
return result
}
@ -586,7 +648,7 @@ class Connection implements ClientConnection {
}
sendForceClose (): Promise<void> {
return this.sendRequest({ method: 'forceClose', params: [] })
return this.sendRequest({ method: 'forceClose', params: [], allowReconnect: false })
}
}

3
plugins/devmodel-resources/src/index.ts
View File

@ -135,7 +135,8 @@ class ModelClient implements AccountClient {
' =>model',
this.client.getModel(),
getMetadata(devmodel.metadata.DevModel),
Date.now() - startTime
Date.now() - startTime,
JSON.stringify(result).length
)
}
return result

4
plugins/task-resources/src/index.ts
View File

@ -238,8 +238,10 @@ async function statusSort (
const aIndex = getStatusIndex(type, taskTypes, a)
const bIndex = getStatusIndex(type, taskTypes, b)
return aIndex - bIndex
} else {
} else if (aVal != null && bVal != null) {
return aVal.name.localeCompare(bVal.name)
} else {
return 0
}
})
} else {

2
plugins/tracker-resources/src/components/myissues/MyIssues.svelte
View File

@ -71,7 +71,7 @@
subscribed = { _id: { $in: newSub } }
}
},
{ sort: { _id: 1 } }
{ sort: { _id: 1 }, projection: { _id: 1 } }
)
const archivedProjectQuery = createQuery()

2
plugins/tracker-resources/src/utils.ts
View File

@ -352,7 +352,7 @@ export async function issueStatusSort (
const aIndex = getStatusIndex(type, taskTypes, a)
const bIndex = getStatusIndex(type, taskTypes, b)
return aIndex - bIndex
} else {
} else if (aVal != null && bVal != null) {
return aVal.name.localeCompare(bVal.name)
}
}

19
plugins/view-resources/src/components/list/List.svelte
View File

@ -101,14 +101,17 @@
},
{ ...categoryQueryOptions, limit: 1000 }
)
$: docsQuerySlow.query(
_class,
queryNoLookup,
(res) => {
slowDocs = res
},
categoryQueryOptions
)
$: if (fastDocs.length === 1000) {
docsQuerySlow.query(
_class,
queryNoLookup,
(res) => {
slowDocs = res
},
categoryQueryOptions
)
}
$: docs = [...fastDocs, ...slowDocs.filter((it) => !fastQueryIds.has(it._id))]

2
pods/server/package.json
View File

@ -13,7 +13,7 @@
"_phase:bundle": "rushx bundle",
"_phase:docker-build": "rushx docker:build",
"_phase:docker-staging": "rushx docker:staging",
"bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node --external:bufferutil --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js",
"bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node --external:bufferutil --external:utf-8-validate --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js",
"bundle:u": "mkdir -p bundle && esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node > bundle/bundle.js && mkdir -p ./dist && cp -r ./node_modules/uWebSockets.js/*.node ./dist",
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/transactor",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/transactor staging",

8
pods/server/src/server.ts
View File

@ -28,6 +28,7 @@ import {
import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic'
import {
ConfigurationMiddleware,
LookupMiddleware,
ModifiedMiddleware,
PrivateMiddleware,
QueryJoinMiddleware,
@ -60,14 +61,14 @@ import {
FullTextPushStage,
globalIndexer,
IndexedFieldStage,
type StorageConfiguration,
type ContentTextAdapter,
type DbConfiguration,
type FullTextAdapter,
type FullTextPipelineStage,
type MiddlewareCreator,
type Pipeline,
type StorageAdapter
type StorageAdapter,
type StorageConfiguration
} from '@hcengineering/server-core'
import { serverDocumentId } from '@hcengineering/server-document'
import { serverGmailId } from '@hcengineering/server-gmail'
@ -228,12 +229,13 @@ export function start (
addLocation(serverTimeId, () => import('@hcengineering/server-time-resources'))
const middlewares: MiddlewareCreator[] = [
LookupMiddleware.create,
ModifiedMiddleware.create,
PrivateMiddleware.create,
SpaceSecurityMiddleware.create,
SpacePermissionsMiddleware.create,
ConfigurationMiddleware.create,
QueryJoinMiddleware.create // Should be last one
QueryJoinMiddleware.create
]
const metrics = getMetricsContext()

2
server/account/package.json
View File

@ -40,7 +40,7 @@
"@hcengineering/contact": "^0.6.20",
"@hcengineering/client-resources": "^0.6.23",
"@hcengineering/client": "^0.6.14",
"ws": "^8.10.0",
"ws": "^8.16.0",
"@hcengineering/model": "^0.6.7",
"@hcengineering/server-backup": "^0.6.0",
"@hcengineering/server-tool": "^0.6.0",

2
server/collaborator/package.json
View File

@ -73,6 +73,6 @@
"body-parser": "^1.20.2",
"cors": "^2.8.5",
"compression": "~1.7.4",
"ws": "^8.10.0"
"ws": "^8.16.0"
}
}

14
server/collaborator/src/server.ts
View File

@ -68,7 +68,8 @@ export async function start (
// fallback to standard filter function
return compression.filter(req, res)
},
level: 6
level: 1,
memLevel: 9
})
)
@ -206,13 +207,14 @@ export async function start (
noServer: true,
perMessageDeflate: {
zlibDeflateOptions: {
// See zlib defaults.
chunkSize: 1024,
memLevel: 7,
level: 3
chunkSize: 32 * 1024,
memLevel: 9,
level: 1
},
zlibInflateOptions: {
chunkSize: 10 * 1024
chunkSize: 32 * 1024,
memLevel: 9,
level: 1
},
// Below options specified as default values.
concurrencyLimit: 10, // Limits zlib concurrency for perf.

100
server/core/src/benchmark/index.ts Normal file
View File

@ -0,0 +1,100 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
generateId,
toFindResult,
type BenchmarkDoc,
type Class,
type Doc,
type DocumentQuery,
type FindOptions,
type FindResult,
type Hierarchy,
type MeasureContext,
type ModelDb,
type Ref,
type Space,
type WorkspaceId
} from '@hcengineering/core'
import type { DbAdapter } from '../adapter'
import { DummyDbAdapter } from '../mem'
function genData (dataSize: number): string {
let result = ''
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
const charactersLength = characters.length
for (let i = 0; i < dataSize; i++) {
result += characters.charAt(Math.floor(Math.random() * charactersLength))
}
return result
}
let benchData = ''
class BenchmarkDbAdapter extends DummyDbAdapter {
async findAll<T extends Doc<Space>>(
ctx: MeasureContext,
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T> | undefined
): Promise<FindResult<T>> {
if (_class !== core.class.BenchmarkDoc) {
return toFindResult([])
}
if (benchData === '') {
benchData = genData(1024 * 1024)
}
const result: BenchmarkDoc[] = []
const request: BenchmarkDoc['request'] = ((query as DocumentQuery<BenchmarkDoc>)
.request as BenchmarkDoc['request']) ?? {
documents: 1,
size: 1
}
const docsToAdd =
typeof request.documents === 'number'
? request.documents
: request.documents.from + Math.random() * request.documents.to
for (let i = 0; i < docsToAdd; i++) {
const dataSize =
typeof request.size === 'number' ? request.size : request.size.from + Math.random() * request.size.to
result.push({
_class: core.class.BenchmarkDoc,
_id: generateId(),
modifiedBy: core.account.System,
modifiedOn: Date.now(),
space: core.space.DerivedTx, // To be available for all
response: benchData.slice(0, dataSize)
})
}
return toFindResult<T>(result as T[])
}
}
/**
* @public
*/
export async function createBenchmarkAdapter (
ctx: MeasureContext,
hierarchy: Hierarchy,
url: string,
workspaceId: WorkspaceId,
modelDb: ModelDb
): Promise<DbAdapter> {
return new BenchmarkDbAdapter()
}

1
server/core/src/index.ts
View File

@ -26,3 +26,4 @@ export * from './server'
export * from './storage'
export * from './types'
export * from './utils'
export * from './benchmark'

1
server/middleware/src/index.ts
View File

@ -20,3 +20,4 @@ export * from './private'
export * from './queryJoin'
export * from './spaceSecurity'
export * from './spacePermissions'
export * from './lookup'

114
server/middleware/src/lookup.ts Normal file
View File

@ -0,0 +1,114 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
Class,
Doc,
DocumentQuery,
FindOptions,
FindResult,
MeasureContext,
Ref,
ServerStorage,
Tx,
clone,
toFindResult
} from '@hcengineering/core'
import { BroadcastFunc, Middleware, SessionContext, TxMiddlewareResult } from '@hcengineering/server-core'
import { BaseMiddleware } from './base'
/**
* @public
*/
export class LookupMiddleware extends BaseMiddleware implements Middleware {
private constructor (storage: ServerStorage, next?: Middleware) {
super(storage, next)
}
static async create (
ctx: MeasureContext,
broadcast: BroadcastFunc,
storage: ServerStorage,
next?: Middleware
): Promise<LookupMiddleware> {
return new LookupMiddleware(storage, next)
}
async tx (ctx: SessionContext, tx: Tx): Promise<TxMiddlewareResult> {
return await this.provideTx(ctx, tx)
}
handleBroadcast (tx: Tx[], targets?: string[]): Tx[] {
return this.provideHandleBroadcast(tx, targets)
}
override async findAll<T extends Doc>(
ctx: SessionContext,
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<FindResult<T>> {
const result = await this.provideFindAll(ctx, _class, query, options)
// Fill lookup map to make more compact representation
if (options?.lookup !== undefined) {
const newResult: T[] = []
let counter = 0
const idClassMap: Record<string, { id: number, doc: Doc, count: number }> = {}
function mapDoc (doc: Doc): number {
const key = doc._class + '@' + doc._id
let docRef = idClassMap[key]
if (docRef === undefined) {
docRef = { id: ++counter, doc, count: -1 }
idClassMap[key] = docRef
}
docRef.count++
return docRef.id
}
for (const d of result) {
if (d.$lookup !== undefined) {
const newDoc = clone(d)
newResult.push(newDoc)
for (const [k, v] of Object.entries(d.$lookup)) {
if (!Array.isArray(v)) {
newDoc.$lookup[k] = mapDoc(v)
} else {
newDoc.$lookup[k] = v.map((it) => mapDoc(it))
}
}
}
}
const lookupMap = Object.fromEntries(Array.from(Object.values(idClassMap)).map((it) => [it.id, it.doc]))
if (Object.keys(lookupMap).length > 0) {
return toFindResult(newResult, result.total, lookupMap)
}
}
// We need to get rid of simple query parameters matched in documents
for (const doc of result) {
for (const [k, v] of Object.entries(query)) {
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') {
if ((doc as any)[k] === v) {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete (doc as any)[k]
}
}
}
}
return result
}
}

9
server/mongo/src/storage.ts
View File

@ -229,7 +229,8 @@ abstract class MongoAdapterBase implements DbAdapter {
from: domain,
localField: fullKey,
foreignField: '_id',
as: fullKey.split('.').join('') + '_lookup'
as: fullKey.split('.').join('') + '_lookup',
pipeline: [{ $project: { '%hash%': 0 } }]
})
}
await this.getLookupValue(_class, nested, result, fullKey + '_lookup')
@ -243,7 +244,8 @@ abstract class MongoAdapterBase implements DbAdapter {
from: domain,
localField: fullKey,
foreignField: '_id',
as: fullKey.split('.').join('') + '_lookup'
as: fullKey.split('.').join('') + '_lookup',
pipeline: [{ $project: { '%hash%': 0 } }]
})
}
}
@ -283,7 +285,8 @@ abstract class MongoAdapterBase implements DbAdapter {
$match: {
_class: { $in: desc }
}
}
},
{ $project: { '%hash%': 0 } }
],
as: asVal
}

14
server/rpc/src/rpc.ts
View File

@ -30,6 +30,8 @@ export interface Request<P extends any[]> {
id?: ReqId
method: string
params: P
time?: number // Server time to perform operation
}
/**
@ -86,7 +88,12 @@ export function protoSerialize (object: object, binary: boolean): any {
*/
export function protoDeserialize (data: any, binary: boolean): any {
if (!binary) {
return JSON.parse(data, receiver)
let _data = data
if (_data instanceof ArrayBuffer) {
const decoder = new TextDecoder()
_data = decoder.decode(_data)
}
return JSON.parse(_data.toString(), receiver)
}
return packr.unpack(new Uint8Array(replacer('', data)))
}
@ -117,10 +124,11 @@ export function readResponse<D> (response: any, binary: boolean): Response<D> {
}
function replacer (key: string, value: any): any {
if (Array.isArray(value) && (value as any).total !== undefined) {
if (Array.isArray(value) && ((value as any).total !== undefined || (value as any).lookupMap !== undefined)) {
return {
dataType: 'TotalArray',
total: (value as any).total,
lookupMap: (value as any).lookupMap,
value: [...value]
}
} else {
@ -131,7 +139,7 @@ function replacer (key: string, value: any): any {
function receiver (key: string, value: any): any {
if (typeof value === 'object' && value !== null) {
if (value.dataType === 'TotalArray') {
return Object.assign(value.value, { total: value.total })
return Object.assign(value.value, { total: value.total, lookupMap: value.lookupMap })
}
}
return value

2
server/tool/package.json
View File

@ -39,7 +39,7 @@
"@hcengineering/contact": "^0.6.20",
"@hcengineering/client-resources": "^0.6.23",
"@hcengineering/client": "^0.6.14",
"ws": "^8.10.0",
"ws": "^8.16.0",
"@hcengineering/model": "^0.6.7",
"@hcengineering/server-token": "^0.6.7",
"@hcengineering/server-core": "^0.6.1",

39
server/ws/package.json
View File

@ -19,35 +19,36 @@
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@types/ws": "^8.5.3",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"@types/express": "^4.17.13",
"@types/cors": "^2.8.12",
"@types/compression": "~1.7.2",
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"@types/jest": "^29.5.5",
"@types/node": "~20.11.16",
"@types/ws": "^8.5.3",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint": "^8.54.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.4.0",
"eslint-plugin-promise": "^6.1.1",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
"typescript": "^5.3.3"
},
"dependencies": {
"ws": "^8.10.0",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/analytics": "^0.6.0",
"@hcengineering/core": "^0.6.28",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/rpc": "^0.6.1",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/server-token": "^0.6.7",
"@hcengineering/rpc": "^0.6.1",
"bufferutil": "^4.0.7",
"express": "^4.18.3",
"compression": "~1.7.4",
"cors": "^2.8.5",
"@hcengineering/analytics": "^0.6.0"
"express": "^4.18.3",
"utf-8-validate": "^6.0.3",
"ws": "^8.16.0"
}
}

14
server/ws/src/__tests__/server.test.ts
View File

@ -21,6 +21,11 @@ import WebSocket from 'ws'
import { start } from '../server'
import {
Hierarchy,
MeasureMetricsContext,
ModelDb,
getWorkspaceId,
toFindResult,
type Account,
type Class,
type Doc,
@ -28,15 +33,10 @@ import {
type Domain,
type FindOptions,
type FindResult,
getWorkspaceId,
Hierarchy,
type MeasureContext,
MeasureMetricsContext,
ModelDb,
type Ref,
type ServerStorage,
type Space,
toFindResult,
type Tx,
type TxResult
} from '@hcengineering/core'
@ -239,13 +239,15 @@ describe('server', () => {
try {
//
const token: string = generateToken('my@email.com', getWorkspaceId('latest', ''))
let clearTo: any
const timeoutPromise = new Promise<void>((resolve) => {
setTimeout(resolve, 4000)
clearTo = setTimeout(resolve, 4000)
})
const t1 = await findClose(token, timeoutPromise, 1005)
const t2 = await findClose(token, timeoutPromise, 1000)
expect(t1).toBe(t2)
clearTimeout(clearTo)
} catch (err: any) {
console.error(err)
} finally {

38
server/ws/src/client.ts
View File

@ -14,8 +14,12 @@
//
import core, {
type Account,
AccountRole,
TxFactory,
TxProcessor,
WorkspaceEvent,
generateId,
type Account,
type BulkUpdateEvent,
type Class,
type Doc,
@ -33,12 +37,8 @@ import core, {
type TxApplyIf,
type TxApplyResult,
type TxCUD,
TxFactory,
TxProcessor,
type TxResult,
type TxWorkspaceEvent,
WorkspaceEvent,
generateId
type TxWorkspaceEvent
} from '@hcengineering/core'
import { type Pipeline, type SessionContext } from '@hcengineering/server-core'
import { type Token } from '@hcengineering/server-token'
@ -50,7 +50,7 @@ import { type BroadcastCall, type Session, type SessionRequest, type StatisticsE
export class ClientSession implements Session {
createTime = Date.now()
requests = new Map<string, SessionRequest>()
binaryResponseMode: boolean = false
binaryMode: boolean = false
useCompression: boolean = true
useBroadcast: boolean = false
sessionId = ''
@ -190,17 +190,23 @@ export class ClientSession implements Session {
if (tx._class === core.class.TxApplyIf) {
;(result as TxApplyResult).derived.push(...derived)
}
while (derived.length > 0) {
const part = derived.splice(0, 250)
console.log('Broadcasting part', part.length, derived.length)
this.broadcast(null, this.token.workspace, { result: part }, target)
}
// Let's send after our response will go out
setImmediate(() => {
while (derived.length > 0) {
const part = derived.splice(0, 250)
console.log('Broadcasting part', part.length, derived.length)
this.broadcast(null, this.token.workspace, { result: part }, target)
}
})
}
} else {
while (derived.length > 0) {
const part = derived.splice(0, 250)
this.broadcast(null, this.token.workspace, { result: part }, target)
}
// Let's send after our response will go out
setImmediate(() => {
while (derived.length > 0) {
const part = derived.splice(0, 250)
this.broadcast(null, this.token.workspace, { result: part }, target)
}
})
}
}
if (tx._class === core.class.TxApplyIf) {

190
server/ws/src/server.ts
View File

@ -19,10 +19,12 @@ import core, {
WorkspaceEvent,
generateId,
systemAccountEmail,
toFindResult,
toWorkspaceString,
versionToString,
withContext,
type BaseWorkspaceInfo,
type FindResult,
type MeasureContext,
type Tx,
type TxWorkspaceEvent,
@ -54,9 +56,25 @@ interface WorkspaceLoginInfo extends Omit<BaseWorkspaceInfo, 'workspace'> {
workspaceId: string
}
function timeoutPromise (time: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, time)
function timeoutPromise (time: number): { promise: Promise<void>, cancelHandle: () => void } {
let timer: any
return {
promise: new Promise((resolve) => {
timer = setTimeout(resolve, time)
}),
cancelHandle: () => {
clearTimeout(timer)
}
}
}
function onNextTick (op: () => void): void {
setImmediate(op)
}
function waitNextTick (): Promise<void> {
return new Promise<void>((resolve) => {
setImmediate(resolve)
})
}
@ -75,7 +93,7 @@ class TSessionManager implements SessionManager {
checkInterval: any
sessions = new Map<string, { session: Session, socket: ConnectionSocket }>()
reconnectIds = new Set<string>()
reconnectIds = new Map<string, any>()
maintenanceTimer: any
timeMinutes = 0
@ -164,14 +182,14 @@ class TSessionManager implements SessionManager {
this.ctx.warn('session hang, closing...', { wsId, user: s[1].session.getUser() })
// Force close workspace if only one client and it hang.
void this.close(s[1].socket, workspace.workspaceId)
void this.close(s[1].socket, wsId)
continue
}
if (diff > 20000 && diff < 60000 && this.ticks % 10 === 0) {
void s[1].socket.send(
workspace.context,
{ result: 'ping' },
s[1].session.binaryResponseMode,
s[1].session.binaryMode,
s[1].session.useCompression
)
}
@ -366,7 +384,7 @@ class TSessionManager implements SessionManager {
void ctx.with('set-status', {}, (ctx) => this.trySetStatus(ctx, session, true))
if (this.timeMinutes > 0) {
void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryResponseMode, session.useCompression)
void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression)
}
return { session, context: workspace.context, workspaceId: wsString }
}
@ -438,12 +456,7 @@ class TSessionManager implements SessionManager {
if (targets !== undefined && !targets.includes(session.session.getUser())) continue
for (const _tx of tx) {
try {
void session.socket.send(
ctx,
{ result: _tx },
session.session.binaryResponseMode,
session.session.useCompression
)
void session.socket.send(ctx, { result: _tx }, session.session.binaryMode, session.session.useCompression)
} catch (err: any) {
Analytics.handleError(err)
ctx.error('error during send', { error: err })
@ -451,12 +464,17 @@ class TSessionManager implements SessionManager {
}
}
if (sessions.length > 0) {
setImmediate(send)
onNextTick(send)
} else {
ctx.end()
}
}
send()
if (sessions.length > 0) {
// We need to send broadcast after our client response so put it after all IO
onNextTick(send)
} else {
ctx.end()
}
}
private createWorkspace (
@ -532,8 +550,7 @@ class TSessionManager implements SessionManager {
} catch {}
}
async close (ws: ConnectionSocket, workspaceId: WorkspaceId): Promise<void> {
const wsid = toWorkspaceString(workspaceId)
async close (ws: ConnectionSocket, wsid: string): Promise<void> {
const workspace = this.workspaces.get(wsid)
const sessionRef = this.sessions.get(ws.id)
@ -542,23 +559,25 @@ class TSessionManager implements SessionManager {
if (workspace !== undefined) {
workspace.sessions.delete(sessionRef.session.sessionId)
}
this.reconnectIds.add(sessionRef.session.sessionId)
this.reconnectIds.set(
sessionRef.session.sessionId,
setTimeout(() => {
this.reconnectIds.delete(sessionRef.session.sessionId)
setTimeout(() => {
this.reconnectIds.delete(sessionRef.session.sessionId)
}, this.timeouts.reconnectTimeout)
const user = sessionRef.session.getUser()
if (workspace !== undefined) {
const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user)
if (another === -1 && !workspace.upgrade) {
void this.trySetStatus(workspace.context, sessionRef.session, false)
}
}
}, this.timeouts.reconnectTimeout)
)
try {
sessionRef.socket.close()
} catch (err) {
// Ignore if closed
}
const user = sessionRef.session.getUser()
if (workspace !== undefined) {
const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user)
if (another === -1 && !workspace.upgrade) {
await this.trySetStatus(workspace.context, sessionRef.session, false)
}
}
}
}
@ -596,7 +615,7 @@ class TSessionManager implements SessionManager {
s.workspaceClosed = true
if (reason === 'upgrade' || reason === 'force-close') {
// Override message handler, to wait for upgrading response from clients.
await this.sendUpgrade(workspace.context, webSocket, s.binaryResponseMode)
this.sendUpgrade(workspace.context, webSocket, s.binaryMode)
}
webSocket.close()
}
@ -623,15 +642,17 @@ class TSessionManager implements SessionManager {
}
}
await this.ctx.with('closing', {}, async () => {
await Promise.race([closePipeline(), timeoutPromise(120000)])
const to = timeoutPromise(120000)
await Promise.race([closePipeline(), to.promise])
to.cancelHandle()
})
if (LOGGING_ENABLED) {
this.ctx.warn('Workspace closed...', { workspace: workspace.id, wsId, wsName: workspace.workspaceName })
}
}
private async sendUpgrade (ctx: MeasureContext, webSocket: ConnectionSocket, binary: boolean): Promise<void> {
await webSocket.send(
private sendUpgrade (ctx: MeasureContext, webSocket: ConnectionSocket, binary: boolean): void {
void webSocket.send(
ctx,
{
result: {
@ -644,9 +665,7 @@ class TSessionManager implements SessionManager {
}
async closeWorkspaces (ctx: MeasureContext): Promise<void> {
if (this.checkInterval !== undefined) {
clearInterval(this.checkInterval)
}
clearInterval(this.checkInterval)
for (const w of this.workspaces) {
await this.closeAll(w[0], w[1], 1, 'shutdown')
}
@ -666,8 +685,12 @@ class TSessionManager implements SessionManager {
try {
if (workspace.sessions.size === 0) {
const pl = await workspace.pipeline
await Promise.race([pl, timeoutPromise(60000)])
await Promise.race([pl.close(), timeoutPromise(60000)])
let to = timeoutPromise(60000)
await Promise.race([pl, to.promise])
to.cancelHandle()
to = timeoutPromise(60000)
await Promise.race([pl.close(), to])
to.cancelHandle()
if (this.workspaces.get(wsid)?.id === wsUID) {
this.workspaces.delete(wsid)
@ -719,30 +742,23 @@ class TSessionManager implements SessionManager {
function send (): void {
for (const sessionRef of sessions.splice(0, 1)) {
if (sessionRef.session.sessionId !== from?.sessionId) {
if (target === undefined) {
void sessionRef.socket.send(
ctx,
resp,
sessionRef.session.binaryResponseMode,
sessionRef.session.useCompression
)
} else if (target.includes(sessionRef.session.getUser())) {
void sessionRef.socket.send(
ctx,
resp,
sessionRef.session.binaryResponseMode,
sessionRef.session.useCompression
)
if (target === undefined || target.includes(sessionRef.session.getUser())) {
void sessionRef.socket.send(ctx, resp, sessionRef.session.binaryMode, sessionRef.session.useCompression)
}
}
}
if (sessions.length > 0) {
setImmediate(send)
onNextTick(send)
} else {
ctx.end()
}
}
send()
if (sessions.length > 0) {
// We need to send broadcast after our client response so put it after all IO
onNextTick(send)
} else {
ctx.end()
}
}
async handleRequest<S extends Session>(
@ -764,7 +780,12 @@ class TSessionManager implements SessionManager {
try {
const backupMode = 'loadChunk' in service
await userCtx.with(`🧭 ${backupMode ? 'handleBackup' : 'handleRequest'}`, {}, async (ctx) => {
const request = await ctx.with('📥 read', {}, async () => readRequest(msg, false))
const request = readRequest(msg, service.binaryMode)
if (request.time != null) {
const delta = Date.now() - request.time
userCtx.measure('receive msg', delta)
}
if (request.method === 'forceClose') {
const wsRef = this.workspaces.get(workspace)
let done = false
@ -780,12 +801,12 @@ class TSessionManager implements SessionManager {
id: request.id,
result: done
}
await ws.send(ctx, forceCloseResponse, service.binaryResponseMode, service.useCompression)
await ws.send(ctx, forceCloseResponse, service.binaryMode, service.useCompression)
return
}
if (request.id === -1 && request.method === 'hello') {
const hello = request as HelloRequest
service.binaryResponseMode = hello.binary ?? false
service.binaryMode = hello.binary ?? false
service.useCompression = hello.compression ?? false
service.useBroadcast = hello.broadcast ?? false
@ -793,18 +814,24 @@ class TSessionManager implements SessionManager {
ctx.info('hello happen', {
workspace,
user: service.getUser(),
binary: service.binaryResponseMode,
binary: service.binaryMode,
compression: service.useCompression,
timeToHello: Date.now() - service.createTime,
workspaceUsers: this.workspaces.get(workspace)?.sessions?.size,
totalUsers: this.sessions.size
})
}
const reconnect = this.reconnectIds.has(service.sessionId)
if (reconnect) {
const reconnectTimeout = this.reconnectIds.get(service.sessionId)
clearTimeout(reconnectTimeout)
this.reconnectIds.delete(service.sessionId)
}
const helloResponse: HelloResponse = {
id: -1,
result: 'hello',
binary: service.binaryResponseMode,
reconnect: this.reconnectIds.has(service.sessionId)
binary: service.binaryMode,
reconnect
}
await ws.send(ctx, helloResponse, false, false)
return
@ -839,14 +866,7 @@ class TSessionManager implements SessionManager {
queue: service.requests.size
}
await handleSend(
ctx,
ws,
resp,
this.sessions.size < 100 ? 10000 : 1001,
service.binaryResponseMode,
service.useCompression
)
await handleSend(ctx, ws, resp, 32 * 1024, service.binaryMode, service.useCompression)
} catch (err: any) {
Analytics.handleError(err)
if (LOGGING_ENABLED) {
@ -857,7 +877,7 @@ class TSessionManager implements SessionManager {
error: unknownError(err),
result: JSON.parse(JSON.stringify(err?.stack))
}
await ws.send(ctx, resp, service.binaryResponseMode, service.useCompression)
await ws.send(ctx, resp, service.binaryMode, service.useCompression)
}
})
} finally {
@ -884,14 +904,7 @@ class TSessionManager implements SessionManager {
try {
const resp: Response<any> = { id: request.id, result: request.method === 'measure' ? 'started' : serverTime }
await handleSend(
ctx,
ws,
resp,
this.sessions.size < 100 ? 10000 : 1001,
service.binaryResponseMode,
service.useCompression
)
await handleSend(ctx, ws, resp, 32 * 1024, service.binaryMode, service.useCompression)
} catch (err: any) {
Analytics.handleError(err)
if (LOGGING_ENABLED) {
@ -902,7 +915,7 @@ class TSessionManager implements SessionManager {
error: unknownError(err),
result: JSON.parse(JSON.stringify(err?.stack))
}
await ws.send(ctx, resp, service.binaryResponseMode, service.useCompression)
await ws.send(ctx, resp, service.binaryMode, service.useCompression)
}
}
}
@ -916,13 +929,26 @@ async function handleSend (
useCompression: boolean
): Promise<void> {
// ws.send(msg)
if (Array.isArray(msg.result) && chunkLimit > 0 && msg.result.length > chunkLimit) {
if (Array.isArray(msg.result) && msg.result.length > 1 && chunkLimit > 0) {
// Split and send by chunks
const data = [...msg.result]
let cid = 1
while (data.length > 0) {
const chunk = data.splice(0, chunkLimit)
const dataSize = JSON.stringify(data).length
const avg = Math.round(dataSize / data.length)
const itemChunk = Math.round(chunkLimit / avg) + 1
while (data.length > 0 && !ws.isClosed) {
let itemChunkCurrent = itemChunk
if (data.length - itemChunk < itemChunk / 2) {
itemChunkCurrent = data.length
}
const chunk: FindResult<any> = toFindResult(data.splice(0, itemChunkCurrent))
if (data.length === 0) {
const orig = msg.result as FindResult<any>
chunk.total = orig.total ?? 0
chunk.lookupMap = orig.lookupMap
}
if (chunk !== undefined) {
await ws.send(
ctx,
@ -932,6 +958,10 @@ async function handleSend (
)
}
cid++
if (data.length > 0 && !ws.isClosed) {
await waitNextTick()
}
}
} else {
await ws.send(ctx, msg, useBinary, useCompression)

159
server/ws/src/server_http.ts
View File

@ -14,7 +14,7 @@
//
import { Analytics } from '@hcengineering/analytics'
import { generateId, type MeasureContext } from '@hcengineering/core'
import { generateId, toWorkspaceString, type MeasureContext } from '@hcengineering/core'
import { UNAUTHORIZED } from '@hcengineering/platform'
import { serialize, type Response } from '@hcengineering/rpc'
import { decodeToken, type Token } from '@hcengineering/server-token'
@ -22,16 +22,21 @@ import compression from 'compression'
import cors from 'cors'
import express from 'express'
import http, { type IncomingMessage } from 'http'
import os from 'os'
import { WebSocketServer, type RawData, type WebSocket } from 'ws'
import { getStatistics, wipeStatistics } from './stats'
import {
LOGGING_ENABLED,
type AddSessionResponse,
type ConnectionSocket,
type HandleRequestFunction,
type PipelineFactory,
type SessionManager
} from './types'
import 'bufferutil'
import 'utf-8-validate'
/**
* @public
* @param sessionFactory -
@ -49,7 +54,13 @@ export function startHttpServer (
accountsUrl: string
): () => Promise<void> {
if (LOGGING_ENABLED) {
ctx.info('starting server on', { port, productId, enableCompression, accountsUrl })
ctx.info('starting server on', {
port,
productId,
enableCompression,
accountsUrl,
parallel: os.availableParallelism()
})
}
const app = express()
@ -65,7 +76,8 @@ export function startHttpServer (
// fallback to standard filter function
return compression.filter(req, res)
},
level: 6
level: 1,
memLevel: 9
})
)
@ -157,16 +169,19 @@ export function startHttpServer (
? {
zlibDeflateOptions: {
// See zlib defaults.
chunkSize: 10 * 1024,
memLevel: 7,
level: 3
chunkSize: 32 * 1024,
memLevel: 9,
level: 1
},
zlibInflateOptions: {
chunkSize: 10 * 1024,
level: 3
chunkSize: 32 * 1024,
level: 1,
memLevel: 9
},
serverNoContextTakeover: true,
clientNoContextTakeover: true,
// Below options specified as default values.
concurrencyLimit: 20, // Limits zlib concurrency for perf.
concurrencyLimit: Math.max(10, os.availableParallelism()), // Limits zlib concurrency for perf.
threshold: 1024 // Size (in bytes) below which messages
// should not be compressed if context takeover is disabled.
}
@ -181,8 +196,6 @@ export function startHttpServer (
rawToken: string,
sessionId?: string
): Promise<void> => {
let buffer: Buffer[] | undefined = []
const data = {
remoteAddress: request.socket.remoteAddress ?? '',
userAgent: request.headers['user-agent'] ?? '',
@ -193,36 +206,38 @@ export function startHttpServer (
}
const cs: ConnectionSocket = {
id: generateId(),
isClosed: false,
close: () => {
cs.isClosed = true
ws.close()
},
data: () => data,
send: async (ctx: MeasureContext, msg, binary, compression) => {
if (ws.readyState !== ws.OPEN) {
return
if (ws.readyState !== ws.OPEN && !cs.isClosed) {
return 0
}
const smsg = await ctx.with('📦 serialize', {}, async () => serialize(msg, binary))
const smsg = serialize(msg, binary)
ctx.measure('send-data', smsg.length)
await ctx.with('📤 socket-send', {}, async (ctx) => {
await new Promise<void>((resolve, reject) => {
ws.send(smsg, { binary, compress: compression }, (err) => {
if (err != null) {
reject(err)
} else {
resolve()
}
})
while (ws.bufferedAmount > 128 && ws.readyState === ws.OPEN) {
await new Promise<void>((resolve) => {
setImmediate(resolve)
})
}
await new Promise<void>((resolve, reject) => {
ws.send(smsg, { binary: true, compress: compression }, (err) => {
if (err != null) {
reject(err)
}
resolve()
})
})
return smsg.length
}
}
ws.on('message', (msg: Buffer) => {
buffer?.push(msg)
})
const session = await sessions.addSession(
let session: AddSessionResponse | Promise<AddSessionResponse> = sessions.addSession(
ctx,
cs,
token,
@ -232,25 +247,41 @@ export function startHttpServer (
sessionId,
accountsUrl
)
if ('upgrade' in session || 'error' in session) {
if ('error' in session) {
ctx.error('error', { error: session.error?.message, stack: session.error?.stack })
void session.then((s) => {
if ('upgrade' in s || 'error' in s) {
if ('error' in s) {
ctx.error('error', { error: s.error?.message, stack: s.error?.stack })
}
void cs
.send(ctx, { id: -1, result: { state: 'upgrading', stats: (s as any).upgradeInfo } }, false, false)
.then(() => {
cs.close()
})
}
await cs.send(ctx, { id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } }, false, false)
cs.close()
return
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
ws.on('message', (msg: RawData) => {
try {
let buff: any | undefined
if (msg instanceof Buffer) {
buff = msg?.toString()
buff = msg
} else if (Array.isArray(msg)) {
buff = Buffer.concat(msg).toString()
buff = Buffer.concat(msg)
}
if (buff !== undefined) {
void handleRequest(session.context, session.session, cs, buff, session.workspaceId)
if (session instanceof Promise) {
void session.then((_session) => {
session = _session
if ('session' in _session) {
void handleRequest(_session.context, _session.session, cs, buff, _session.workspaceId)
}
})
} else {
if ('session' in session) {
void handleRequest(session.context, session.session, cs, buff, session.workspaceId)
}
}
}
} catch (err: any) {
Analytics.handleError(err)
@ -260,18 +291,30 @@ export function startHttpServer (
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
ws.on('close', (code: number, reason: Buffer) => {
if (session.session.workspaceClosed ?? false) {
return
ws.on('close', async (code: number, reason: Buffer) => {
if (session instanceof Promise) {
session = await session
}
if ('session' in session) {
if (!(session.session.workspaceClosed ?? false)) {
// remove session after 1seconds, give a time to reconnect.
void sessions.close(cs, toWorkspaceString(token.workspace))
}
}
})
ws.on('error', (err) => {
if (session instanceof Promise) {
void session.then((s) => {
if ('session' in session) {
console.error(session.session.getUser(), 'error', err)
}
})
}
if ('session' in session) {
console.error(session.session.getUser(), 'error', err)
}
// remove session after 1seconds, give a time to reconnect.
void sessions.close(cs, token.workspace)
})
const b = buffer
buffer = undefined
for (const msg of b) {
await handleRequest(session.context, session.session, cs, msg, session.workspaceId)
}
}
wss.on('connection', handleConnection as any)
@ -290,7 +333,10 @@ export function startHttpServer (
throw new Error('Invalid workspace product')
}
wss.handleUpgrade(request, socket, head, (ws) => wss.emit('connection', ws, request, payload, token, sessionId))
wss.handleUpgrade(request, socket, head, (ws) => {
void handleConnection(ws, request, payload, token, sessionId ?? undefined)
// wss.emit('connection', ws, request, payload, token, sessionId)
})
} catch (err: any) {
Analytics.handleError(err)
if (LOGGING_ENABLED) {
@ -320,7 +366,22 @@ export function startHttpServer (
httpServer.listen(port)
return async () => {
httpServer.close()
await sessions.closeWorkspaces(ctx)
await new Promise<void>((resolve, reject) => {
wss.close((err) => {
if (err != null) {
reject(err)
}
resolve()
})
})
await new Promise<void>((resolve, reject) =>
httpServer.close((err) => {
if (err != null) {
reject(err)
}
resolve()
})
)
}
}

14
server/ws/src/types.ts
View File

@ -54,7 +54,7 @@ export interface Session {
requests: Map<string, SessionRequest>
binaryResponseMode: boolean
binaryMode: boolean
useCompression: boolean
useBroadcast: boolean
@ -96,8 +96,9 @@ export type PipelineFactory = (
*/
export interface ConnectionSocket {
id: string
isClosed: boolean
close: () => void
send: (ctx: MeasureContext, msg: Response<any>, binary: boolean, compression: boolean) => Promise<void>
send: (ctx: MeasureContext, msg: Response<any>, binary: boolean, compression: boolean) => Promise<number>
data: () => Record<string, any>
}
@ -131,6 +132,11 @@ export interface Workspace {
workspaceName: string
}
export type AddSessionResponse =
| { session: Session, context: MeasureContext, workspaceId: string }
| { upgrade: true }
| { error: any }
/**
* @public
*/
@ -149,11 +155,11 @@ export interface SessionManager {
productId: string,
sessionId: string | undefined,
accountsUrl: string
) => Promise<{ session: Session, context: MeasureContext, workspaceId: string } | { upgrade: true } | { error: any }>
) => Promise<AddSessionResponse>
broadcastAll: (workspace: Workspace, tx: Tx[], targets?: string[]) => void
close: (ws: ConnectionSocket, workspaceId: WorkspaceId) => Promise<void>
close: (ws: ConnectionSocket, workspaceId: string) => Promise<void>
closeAll: (
wsId: string,