From 1c3edbc395c95c7ece4992cc4cd08890a1ed7ae6 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Fri, 3 May 2024 12:57:05 +0700 Subject: [PATCH] UBERF-6756: Tracker performance fixes (#5488) Signed-off-by: Andrey Sobolev --- common/config/rush/pnpm-lock.yaml | 153 ++++----- dev/tool/package.json | 2 +- dev/tool/src/benchmark.ts | 295 ++++++++++-------- models/core/src/benchmark.ts | 27 ++ models/core/src/index.ts | 11 +- packages/core/src/benchmark.ts | 29 ++ packages/core/src/component.ts | 5 +- packages/core/src/index.ts | 1 + packages/core/src/storage.ts | 1 + packages/core/src/utils.ts | 53 +++- packages/presentation/src/utils.ts | 55 +--- .../src/components/chat/Chat.svelte | 2 +- plugins/client-resources/src/connection.ts | 144 ++++++--- plugins/devmodel-resources/src/index.ts | 3 +- plugins/task-resources/src/index.ts | 4 +- .../src/components/myissues/MyIssues.svelte | 2 +- plugins/tracker-resources/src/utils.ts | 2 +- .../src/components/list/List.svelte | 19 +- pods/server/package.json | 2 +- pods/server/src/server.ts | 8 +- server/account/package.json | 2 +- server/collaborator/package.json | 2 +- server/collaborator/src/server.ts | 14 +- server/core/src/benchmark/index.ts | 100 ++++++ server/core/src/index.ts | 1 + server/middleware/src/index.ts | 1 + server/middleware/src/lookup.ts | 114 +++++++ server/mongo/src/storage.ts | 9 +- server/rpc/src/rpc.ts | 14 +- server/tool/package.json | 2 +- server/ws/package.json | 39 +-- server/ws/src/__tests__/server.test.ts | 14 +- server/ws/src/client.ts | 38 ++- server/ws/src/server.ts | 190 ++++++----- server/ws/src/server_http.ts | 159 +++++++--- server/ws/src/types.ts | 14 +- 36 files changed, 1032 insertions(+), 499 deletions(-) create mode 100644 models/core/src/benchmark.ts create mode 100644 packages/core/src/benchmark.ts create mode 100644 server/core/src/benchmark/index.ts create mode 100644 server/middleware/src/lookup.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index cdc154d235..5f44bbd3c2 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -13,10 +13,10 @@ dependencies: version: 7.17.13 '@hocuspocus/provider': specifier: ^2.9.0 - version: 2.11.2(bufferutil@4.0.8)(yjs@13.6.12) + version: 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12) '@hocuspocus/server': specifier: ^2.9.0 - version: 2.11.2(bufferutil@4.0.8)(yjs@13.6.12) + version: 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12) '@hocuspocus/transformer': specifier: ^2.9.0 version: 2.11.2(@tiptap/pm@2.2.4)(y-prosemirror@1.2.2)(yjs@13.6.12) @@ -28,7 +28,7 @@ dependencies: version: 1.41.2 '@rush-temp/account': specifier: file:./projects/account.tgz - version: file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2) + version: file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3) '@rush-temp/account-service': specifier: file:./projects/account-service.tgz version: file:projects/account-service.tgz @@ -109,7 +109,7 @@ dependencies: version: file:projects/collaboration.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/collaborator': specifier: file:./projects/collaborator.tgz - version: file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4) + version: file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)(utf-8-validate@6.0.3) '@rush-temp/collaborator-client': specifier: file:./projects/collaborator-client.tgz version: file:projects/collaborator-client.tgz(ts-node@10.9.2) @@ -451,7 +451,7 @@ dependencies: version: file:projects/presentation.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) '@rush-temp/prod': specifier: file:./projects/prod.tgz - version: file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2) + version: file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2)(utf-8-validate@6.0.3) '@rush-temp/query': specifier: file:./projects/query.tgz version: file:projects/query.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) @@ -622,7 +622,7 @@ dependencies: version: file:projects/server-token.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/server-tool': specifier: file:./projects/server-tool.tgz - version: file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2) + version: file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3) '@rush-temp/server-tracker': specifier: file:./projects/server-tracker.tgz version: file:projects/server-tracker.tgz(esbuild@0.20.1)(ts-node@10.9.2) @@ -649,7 +649,7 @@ dependencies: version: file:projects/setting-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) '@rush-temp/storybook': specifier: file:./projects/storybook.tgz - version: file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(webpack-cli@5.1.4)(webpack@5.90.3) + version: file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3) '@rush-temp/support': specifier: file:./projects/support.tgz version: file:projects/support.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) @@ -700,10 +700,10 @@ dependencies: version: file:projects/tests-sanity.tgz '@rush-temp/text': specifier: file:./projects/text.tgz - version: file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2) + version: file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3) '@rush-temp/text-editor': specifier: file:./projects/text-editor.tgz - version: file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2) + version: file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)(utf-8-validate@6.0.3) '@rush-temp/theme': specifier: file:./projects/theme.tgz version: file:projects/theme.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) @@ -718,7 +718,7 @@ dependencies: version: file:projects/time-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) '@rush-temp/tool': specifier: file:./projects/tool.tgz - version: file:projects/tool.tgz(bufferutil@4.0.8) + version: file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.3) '@rush-temp/tracker': specifier: file:./projects/tracker.tgz version: file:projects/tracker.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) @@ -1111,7 +1111,7 @@ dependencies: version: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) jest-environment-jsdom: specifier: 29.7.0 - version: 29.7.0(bufferutil@4.0.8) + version: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) jwt-simple: specifier: ^0.5.6 version: 0.5.6 @@ -1231,7 +1231,7 @@ dependencies: version: 2.0.5 storybook: specifier: ^7.0.6 - version: 7.6.17(bufferutil@4.0.8) + version: 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3) storybook-addon-themes: specifier: ^6.1.0 version: 6.1.0(react-dom@18.2.0)(react@18.2.0)(svelte@4.2.12) @@ -1280,6 +1280,9 @@ dependencies: url-loader: specifier: ~4.1.1 version: 4.1.1(file-loader@6.2.0)(webpack@5.90.3) + utf-8-validate: + specifier: ^6.0.3 + version: 6.0.3 uuid: specifier: ^8.3.2 version: 8.3.2 @@ -1291,16 +1294,16 @@ dependencies: version: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4) webpack-bundle-analyzer: specifier: ^4.7.0 - version: 4.10.1(bufferutil@4.0.8) + version: 4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3) webpack-cli: specifier: ^5.0.1 version: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3) webpack-dev-server: specifier: ^4.11.1 - version: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3) + version: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3) ws: - specifier: ^8.10.0 - version: 8.16.0(bufferutil@4.0.8) + specifier: ^8.16.0 + version: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) y-prosemirror: specifier: ^1.2.1 version: 1.2.2(prosemirror-model@1.19.4)(yjs@13.6.12) @@ -3396,7 +3399,7 @@ packages: lib0: 0.2.89 dev: false - /@hocuspocus/provider@2.11.2(bufferutil@4.0.8)(yjs@13.6.12): + /@hocuspocus/provider@2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12): resolution: {integrity: sha512-h6/2KZC8Z3Wz+rP17WQ5eqxdgtsq5JZ4iC72WSXdIOJn3RO4sv63ZvT+EWc4aA1YM7UVDBYVjdOjMtEYZF0Flg==} peerDependencies: y-protocols: ^1.0.6 @@ -3405,14 +3408,14 @@ packages: '@hocuspocus/common': 2.11.2 '@lifeomic/attempt': 3.0.3 lib0: 0.2.89 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) yjs: 13.6.12 transitivePeerDependencies: - bufferutil - utf-8-validate dev: false - /@hocuspocus/server@2.11.2(bufferutil@4.0.8)(yjs@13.6.12): + /@hocuspocus/server@2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12): resolution: {integrity: sha512-/djaNUSS9vYmz5H/bformB9BXKVhSXS10WIURT1OqFLnN0dZnOB0gxL/IyGCZgXCNyiD8U03n7FIgad9MckV8g==} peerDependencies: y-protocols: ^1.0.6 @@ -3423,7 +3426,7 @@ packages: kleur: 4.1.5 lib0: 0.2.89 uuid: 9.0.1 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) yjs: 13.6.12 transitivePeerDependencies: - bufferutil @@ -4889,7 +4892,7 @@ packages: tiny-invariant: 1.3.1 dev: false - /@storybook/cli@7.6.17(bufferutil@4.0.8): + /@storybook/cli@7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-1sCo+nCqyR+nKfTcEidVu8XzNoECC7Y1l+uW38/r7s2f/TdDorXaIGAVrpjbSaXSoQpx5DxYJVaKCcQuOgqwcA==} hasBin: true dependencies: @@ -4900,7 +4903,7 @@ packages: '@storybook/codemod': 7.6.17 '@storybook/core-common': 7.6.17 '@storybook/core-events': 7.6.17 - '@storybook/core-server': 7.6.17(bufferutil@4.0.8) + '@storybook/core-server': 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3) '@storybook/csf-tools': 7.6.17 '@storybook/node-logger': 7.6.17 '@storybook/telemetry': 7.6.17 @@ -5065,7 +5068,7 @@ packages: ts-dedent: 2.2.0 dev: false - /@storybook/core-server@7.6.17(bufferutil@4.0.8): + /@storybook/core-server@7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-KWGhTTaL1Q14FolcoKKZgytlPJUbH6sbJ1Ptj/84EYWFewcnEgVs0Zlnh1VStRZg+Rd1WC1V4yVd/bbDzxrvQA==} dependencies: '@aw-web-design/x-default-browser': 1.4.126 @@ -5108,7 +5111,7 @@ packages: util: 0.12.5 util-deprecate: 1.0.2 watchpack: 2.4.0 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - encoding @@ -6900,7 +6903,7 @@ packages: dependencies: webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4) webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3) - webpack-dev-server: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3) + webpack-dev-server: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3) dev: false /@xtuc/ieee754@1.2.0: @@ -9998,6 +10001,10 @@ packages: resolution: {integrity: sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==} dev: false + /fflate@0.8.2: + resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + dev: false + /file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -11584,7 +11591,7 @@ packages: pretty-format: 29.7.0 dev: false - /jest-environment-jsdom@29.7.0(bufferutil@4.0.8): + /jest-environment-jsdom@29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} peerDependencies: @@ -11600,7 +11607,7 @@ packages: '@types/node': 20.11.19 jest-mock: 29.7.0 jest-util: 29.7.0 - jsdom: 20.0.3(bufferutil@4.0.8) + jsdom: 20.0.3(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - supports-color @@ -11958,7 +11965,7 @@ packages: - supports-color dev: false - /jsdom@20.0.3(bufferutil@4.0.8): + /jsdom@20.0.3(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-SYhBvTh89tTfCD/CRdSOm13mOBa42iTaTyfyEWBdKcGdPxPtLFBXuHR8XHb33YNYaP+lLbmSvBTsnoesCNJEsQ==} engines: {node: '>=14'} peerDependencies: @@ -11991,7 +11998,7 @@ packages: whatwg-encoding: 2.0.0 whatwg-mimetype: 3.0.0 whatwg-url: 11.0.0 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) xml-name-validator: 4.0.0 transitivePeerDependencies: - bufferutil @@ -15192,11 +15199,11 @@ packages: - react-dom dev: false - /storybook@7.6.17(bufferutil@4.0.8): + /storybook@7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-8+EIo91bwmeFWPg1eysrxXlhIYv3OsXrznTr4+4Eq0NikqAoq6oBhtlN5K2RGS2lBVF537eN+9jTCNbR+WrzDA==} hasBin: true dependencies: - '@storybook/cli': 7.6.17(bufferutil@4.0.8) + '@storybook/cli': 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - encoding @@ -16333,6 +16340,14 @@ packages: tslib: 2.6.2 dev: false + /utf-8-validate@6.0.3: + resolution: {integrity: sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==} + engines: {node: '>=6.14.2'} + requiresBuild: true + dependencies: + node-gyp-build: 4.8.0 + dev: false + /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} dev: false @@ -16463,7 +16478,7 @@ packages: engines: {node: '>=12'} dev: false - /webpack-bundle-analyzer@4.10.1(bufferutil@4.0.8): + /webpack-bundle-analyzer@4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-s3P7pgexgT/HTUSYgxJyn28A+99mmLq4HsJepMPzu0R8ImJc52QNqaFYW1Z2z2uIb1/J3eYgaAWVpaC+v/1aAQ==} engines: {node: '>= 10.13.0'} hasBin: true @@ -16480,7 +16495,7 @@ packages: opener: 1.5.2 picocolors: 1.0.0 sirv: 2.0.4 - ws: 7.5.9(bufferutil@4.0.8) + ws: 7.5.9(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - utf-8-validate @@ -16516,8 +16531,8 @@ packages: interpret: 3.1.1 rechoir: 0.8.0 webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4) - webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8) - webpack-dev-server: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3) + webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3) + webpack-dev-server: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3) webpack-merge: 5.10.0 dev: false @@ -16552,7 +16567,7 @@ packages: webpack: 5.90.3(@swc/core@1.4.2)(esbuild@0.20.1)(webpack-cli@5.1.4) dev: false - /webpack-dev-server@4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3): + /webpack-dev-server@4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3): resolution: {integrity: sha512-5hbAst3h3C3L8w6W4P96L5vaV0PxSmJhxZvWKYIdgxOQm8pNZ5dEOmmSLBVpP85ReeyRt6AS1QJNyo/oFFPeVA==} engines: {node: '>= 12.13.0'} hasBin: true @@ -16596,7 +16611,7 @@ packages: webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4) webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3) webpack-dev-middleware: 5.3.3(webpack@5.90.3) - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - debug @@ -16862,7 +16877,7 @@ packages: async-limiter: 1.0.1 dev: false - /ws@7.5.9(bufferutil@4.0.8): + /ws@7.5.9(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==} engines: {node: '>=8.3.0'} peerDependencies: @@ -16875,9 +16890,10 @@ packages: optional: true dependencies: bufferutil: 4.0.8 + utf-8-validate: 6.0.3 dev: false - /ws@8.16.0(bufferutil@4.0.8): + /ws@8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3): resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==} engines: {node: '>=10.0.0'} peerDependencies: @@ -16890,6 +16906,7 @@ packages: optional: true dependencies: bufferutil: 4.0.8 + utf-8-validate: 6.0.3 dev: false /xml-name-validator@4.0.0: @@ -17112,8 +17129,8 @@ packages: - supports-color dev: false - file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-9RuPhqNNHTYjQwezirzcJ6WZpMJTzbjc72jZSJC6dQFG7WqW0a2QZ8VTaa32IM902stPP950kaRUDGGEKlxEwg==, tarball: file:projects/account.tgz} + file:projects/account.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): + resolution: {integrity: sha512-5tvVeDuogAFbWW1kdbXHh0UYCEuA4w0d5mv1euYiTTq6Wrw+MUMCHVGylKlbc5DCApvmkjGYyGTBEQ6gtVDGSg==, tarball: file:projects/account.tgz} id: file:projects/account.tgz name: '@rush-temp/account' version: 0.0.0 @@ -17132,10 +17149,9 @@ packages: mongodb: 6.3.0 node-fetch: 2.7.0 prettier: 3.2.5 - prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - '@aws-sdk/credential-providers' - '@babel/core' @@ -18092,13 +18108,13 @@ packages: - ts-node dev: false - file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4): - resolution: {integrity: sha512-dz9Gj6jr2YoZbl+ZbR3XtYsHqcVRA0qjQ318BwKD+SjgiETupwiFVay/ltHyc+l2V0Ai7m6qSY5ymedU0EevPg==, tarball: file:projects/collaborator.tgz} + file:projects/collaborator.tgz(@tiptap/pm@2.2.4)(bufferutil@4.0.8)(prosemirror-model@1.19.4)(utf-8-validate@6.0.3): + resolution: {integrity: sha512-r+sgjj6JmjY8Lkk/dYVeEm7AJtq3uCZDosWdJZPNIhtJo4fDeVYnMZQEawUKkjW0YcGE/KRnLvS6TCjFMH3ugw==, tarball: file:projects/collaborator.tgz} id: file:projects/collaborator.tgz name: '@rush-temp/collaborator' version: 0.0.0 dependencies: - '@hocuspocus/server': 2.11.2(bufferutil@4.0.8)(yjs@13.6.12) + '@hocuspocus/server': 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12) '@hocuspocus/transformer': 2.11.2(@tiptap/pm@2.2.4)(y-prosemirror@1.2.2)(yjs@13.6.12) '@tiptap/core': 2.2.4(@tiptap/pm@2.2.4) '@tiptap/html': 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4) @@ -18128,7 +18144,7 @@ packages: ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) typescript: 5.3.3 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) y-prosemirror: 1.2.2(prosemirror-model@1.19.4)(yjs@13.6.12) yjs: 13.6.12 transitivePeerDependencies: @@ -18614,7 +18630,7 @@ packages: dev: false file:projects/elastic.tgz(@types/node@20.11.19)(esbuild@0.20.1): - resolution: {integrity: sha512-X3dHb4JxiTeFuXt6bUwzCBklB8BipruxEhhtGWhFbRxvTx689V0bfzoO62QUwc+Kqmz2Q9Df8IPYy51tCKyMvQ==, tarball: file:projects/elastic.tgz} + resolution: {integrity: sha512-iUKt8+1LhjnXTCYZv0DssFNwm1pHB+ioIePMutpYZrtO7HvwaZN4kHVneHjk5hA0EeE96JOkCwgYcPka35UvuA==, tarball: file:projects/elastic.tgz} id: file:projects/elastic.tgz name: '@rush-temp/elastic' version: 0.0.0 @@ -21254,7 +21270,7 @@ packages: - ts-node dev: false - file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2): + file:projects/prod.tgz(bufferutil@4.0.8)(sass@1.71.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): resolution: {integrity: sha512-/CKLzPjaTfkDNjT0evklXES1ISDy11ikLCJKFuyI4OQ6nfbt84wTqkO80Egr9IZScuNA0M0MEeJP+01pO2LNdA==, tarball: file:projects/prod.tgz} id: file:projects/prod.tgz name: '@rush-temp/prod' @@ -21286,9 +21302,9 @@ packages: typescript: 5.3.3 update-browserslist-db: 1.0.13(browserslist@4.21.5) webpack: 5.90.3(esbuild@0.20.1)(webpack-cli@5.1.4) - webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8) + webpack-bundle-analyzer: 4.10.1(bufferutil@4.0.8)(utf-8-validate@6.0.3) webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.15.1)(webpack@5.90.3) - webpack-dev-server: 4.15.1(bufferutil@4.0.8)(webpack-cli@5.1.4)(webpack@5.90.3) + webpack-dev-server: 4.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3) transitivePeerDependencies: - '@babel/core' - '@rspack/core' @@ -23062,8 +23078,8 @@ packages: - ts-node dev: false - file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-La/JqbJn8mFb4MYQ5DKwjn5sxUJ0VYBI8UEcP2iC3KIcvjRrQ0Rcy6q5QBeVpYRpfnAcvUvFCuwR8i1TklseZw==, tarball: file:projects/server-tool.tgz} + file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): + resolution: {integrity: sha512-dPn+fdjcD0BlDVCQ4DZD7zT16BfgObaomZEf+3r/ZpLxZTI+OiEpZUIQlEjgPkW6mgFui74nIcW5MFNO5uJnNA==, tarball: file:projects/server-tool.tgz} id: file:projects/server-tool.tgz name: '@rush-temp/server-tool' version: 0.0.0 @@ -23081,10 +23097,9 @@ packages: jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) mongodb: 6.3.0 prettier: 3.2.5 - prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - '@aws-sdk/credential-providers' - '@babel/core' @@ -23231,7 +23246,7 @@ packages: dev: false file:projects/server-ws.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-U8RVqOkhC0aFRzjZzXrGwKzS+keJrlDuQOtap4w7XOXaHaZO+/dWWw2u3cTTRWdt3xKDFwDAAwiHaBkviaGGtg==, tarball: file:projects/server-ws.tgz} + resolution: {integrity: sha512-H+LX8du8Fflohly6Uv20gI7fhm+T6RRerOESTho9L1Dxgn0E8zXqD3XGrjEblNm94HR9/09uHrs2a7CWPaRAig==, tarball: file:projects/server-ws.tgz} id: file:projects/server-ws.tgz name: '@rush-temp/server-ws' version: 0.0.0 @@ -23253,12 +23268,14 @@ packages: eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.18.3 + fflate: 0.8.2 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) prettier: 3.2.5 prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 - ws: 8.16.0(bufferutil@4.0.8) + utf-8-validate: 6.0.3 + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -23268,7 +23285,6 @@ packages: - node-notifier - supports-color - ts-node - - utf-8-validate dev: false file:projects/server.tgz(esbuild@0.20.1): @@ -23415,7 +23431,7 @@ packages: - ts-node dev: false - file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(webpack-cli@5.1.4)(webpack@5.90.3): + file:projects/storybook.tgz(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(svelte-loader@3.2.0)(svelte@4.2.12)(typescript@5.3.3)(utf-8-validate@6.0.3)(webpack-cli@5.1.4)(webpack@5.90.3): resolution: {integrity: sha512-7yMMWo7RZ6G1yC1wGli2iB6twTB9Tz0cGGgH8xxK/nAbiXdU6CVsv/qdxV/aCnLq9hvLMJY9CaOrFMrqskV3mA==, tarball: file:projects/storybook.tgz} id: file:projects/storybook.tgz name: '@rush-temp/storybook' @@ -23434,7 +23450,7 @@ packages: resolve-url-loader: 5.0.0 sass: 1.71.1 sass-loader: 13.3.3(sass@1.71.1)(webpack@5.90.3) - storybook: 7.6.17(bufferutil@4.0.8) + storybook: 7.6.17(bufferutil@4.0.8)(utf-8-validate@6.0.3) storybook-addon-themes: 6.1.0(react-dom@18.2.0)(react@18.2.0)(svelte@4.2.12) svelte-preprocess: 5.1.3(postcss-load-config@4.0.2)(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.12)(typescript@5.3.3) transitivePeerDependencies: @@ -24028,13 +24044,13 @@ packages: - supports-color dev: false - file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2): + file:projects/text-editor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(prosemirror-model@1.19.4)(ts-node@10.9.2)(utf-8-validate@6.0.3): resolution: {integrity: sha512-HrOMU/Hpc05gePqahXl1WJ/jZC+FUOTC4qbSeK9whq4PYxp6A7Uy5apkN8ww5gDlHs7TW5c69rVGBnbHnqJhsw==, tarball: file:projects/text-editor.tgz} id: file:projects/text-editor.tgz name: '@rush-temp/text-editor' version: 0.0.0 dependencies: - '@hocuspocus/provider': 2.11.2(bufferutil@4.0.8)(yjs@13.6.12) + '@hocuspocus/provider': 2.11.2(bufferutil@4.0.8)(utf-8-validate@6.0.3)(yjs@13.6.12) '@tiptap/core': 2.2.4(@tiptap/pm@2.2.4) '@tiptap/extension-bubble-menu': 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4) '@tiptap/extension-code': 2.2.4(@tiptap/core@2.2.4) @@ -24062,7 +24078,6 @@ packages: '@tiptap/suggestion': 2.2.4(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4) '@types/diff': 5.0.9 '@types/jest': 29.5.12 - '@types/png-chunks-extract': 1.0.2 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) diff: 5.2.0 @@ -24076,7 +24091,6 @@ packages: filesize: 8.0.7 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) lib0: 0.2.89 - png-chunks-extract: 1.0.0 prettier: 3.2.5 prettier-plugin-svelte: 3.2.2(prettier@3.2.5)(svelte@4.2.12) prosemirror-codemark: 0.4.2(prosemirror-model@1.19.4) @@ -24119,7 +24133,7 @@ packages: - y-protocols dev: false - file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2): + file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): resolution: {integrity: sha512-+71BisKHNmsLjMHsBChfyRUJXOP7P2gOAIHC1JP8z/AHvuGfWZirZRyYtRBY1E5PFu7qaI6G4SM2ARIvKBAaxw==, tarball: file:projects/text.tgz} id: file:projects/text.tgz name: '@rush-temp/text' @@ -24155,7 +24169,7 @@ packages: eslint-plugin-promise: 6.1.1(eslint@8.56.0) fast-equals: 5.0.1 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - jest-environment-jsdom: 29.7.0(bufferutil@4.0.8) + jest-environment-jsdom: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) prettier: 3.2.5 prosemirror-model: 1.19.4 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) @@ -24333,8 +24347,8 @@ packages: - ts-node dev: false - file:projects/tool.tgz(bufferutil@4.0.8): - resolution: {integrity: sha512-46NgBZuhcTT7XEc+NxbKu9abwTd63lNVTDR8WrDt8aLuSxorolUYMJSiZcaagz85/eGcn21Q25yvzvhEt2N0Sw==, tarball: file:projects/tool.tgz} + file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.3): + resolution: {integrity: sha512-FapBriQDbhOPHpCCh3hiJAY68tHjzbF6r94xISTJxwsOsJNijAV9v7RFhTUnLiLcCuBSj9eovI7Mljd3CDZrAw==, tarball: file:projects/tool.tgz} id: file:projects/tool.tgz name: '@rush-temp/tool' version: 0.0.0 @@ -24365,11 +24379,10 @@ packages: mime-types: 2.1.35 mongodb: 6.3.0 prettier: 3.2.5 - prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) typescript: 5.3.3 - ws: 8.16.0(bufferutil@4.0.8) + ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - '@aws-sdk/credential-providers' - '@babel/core' diff --git a/dev/tool/package.json b/dev/tool/package.json index 448afb2ff5..3985229ebe 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -136,6 +136,6 @@ "libphonenumber-js": "^1.9.46", "mime-types": "~2.1.34", "mongodb": "^6.3.0", - "ws": "^8.10.0" + "ws": "^8.16.0" } } diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index 32594fa8c8..0a125630af 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -13,24 +13,20 @@ // limitations under the License. // -import contact from '@hcengineering/contact' import core, { - type Account, - type BackupClient, - ClassifierKind, - type Client, - DOMAIN_TX, - type Doc, - type DocumentUpdate, MeasureMetricsContext, - type Metrics, - type Ref, + RateLimiter, TxOperations, - type WorkspaceId, - generateId, metricsToString, newMetrics, - systemAccountEmail + systemAccountEmail, + type Account, + type BackupClient, + type BenchmarkDoc, + type Client, + type Metrics, + type Ref, + type WorkspaceId } from '@hcengineering/core' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' @@ -48,8 +44,9 @@ interface StartMessage { idd: number workId: string options: { - readRequests: number - modelRequests: number + smallRequests: number + rate: number + bigRequests: number limit: { min: number rand: number @@ -62,7 +59,7 @@ interface StartMessage { compression: boolean } interface Msg { - type: 'complete' | 'operate' + type: 'complete' | 'operate' | 'pending' } interface CompleteMsg extends Msg { @@ -70,19 +67,18 @@ interface CompleteMsg extends Msg { workId: string } -// interface OperatingMsg extends Msg { -// type: 'operate' -// workId: string -// } - -const benchAccount = (core.account.System + '_benchmark') as Ref +interface PendingMsg extends Msg { + type: 'pending' + workId: string + pending: number +} export async function benchmark ( workspaceId: WorkspaceId[], transactorUrl: string, cmd: { from: number, steps: number, sleep: number, binary: boolean, write: boolean, compression: boolean } ): Promise { - let operating = 0 + const operating = new Set() const workers: Worker[] = [] const works = new Map void>() @@ -92,7 +88,6 @@ export async function benchmark ( os.cpus().forEach(() => { /* Spawn a new thread running this source file */ - console.error('__filename', __filename) const worker = new Worker(__filename, { argv: ['benchmarkWorker'] }) @@ -102,13 +97,19 @@ export async function benchmark ( return } if (data.type === 'operate') { - operating += 1 + operating.add((data as any).workId) + } + if (data.type === 'pending') { + const msg = data as PendingMsg + console.log('info', `worker:${msg.workId}`, msg.pending) } if (data.type === 'complete') { const resolve = works.get((data as CompleteMsg).workId) if (resolve != null) { resolve() - operating -= 1 + operating.delete((data as any).workId) + } else { + console.log('Worker failed to done', (data as CompleteMsg).workId) } } }) @@ -150,7 +151,9 @@ export async function benchmark ( let elapsed = 0 let requestTime: number = 0 let operations = 0 + let oldOperations = 0 let transfer: number = 0 + let oldTransfer: number = 0 const token = generateToken(systemAccountEmail, workspaceId[0]) @@ -162,16 +165,6 @@ export async function benchmark ( )) as BackupClient & Client) : undefined - if (monitorConnection !== undefined) { - // We need to cleanup all transactions from our benchmark account. - const txes = await monitorConnection.findAll( - core.class.Tx, - { modifiedBy: benchAccount }, - { projection: { _id: 1 } } - ) - await monitorConnection.clean(DOMAIN_TX, Array.from(txes.map((it) => it._id))) - } - let running = false function extract (metrics: Metrics, ...path: string[]): Metrics | null { @@ -193,7 +186,7 @@ export async function benchmark ( } let timer: any - if (isMainThread) { + if (isMainThread && monitorConnection !== undefined) { timer = setInterval(() => { const st = Date.now() @@ -207,9 +200,9 @@ export async function benchmark ( memUsed = json.statistics.memoryUsed memTotal = json.statistics.memoryTotal cpu = json.statistics.cpuUsage - operations = 0 + // operations = 0 requestTime = 0 - transfer = 0 + // transfer = 0 const r = extract( json.metrics as Metrics, '🧲 session', @@ -218,11 +211,14 @@ export async function benchmark ( 'process', 'find-all' ) - operations += r?.operations ?? 0 - requestTime += (r?.value ?? 0) / (((r?.operations as number) ?? 0) + 1) + operations = (r?.operations ?? 0) - oldOperations + oldOperations = r?.operations ?? 0 + + requestTime = (r?.value ?? 0) / (((r?.operations as number) ?? 0) + 1) const tr = extract(json.metrics as Metrics, '🧲 session', 'client', 'handleRequest', '#send-data') - transfer += tr?.value ?? 0 + transfer = (tr?.value ?? 0) - oldTransfer + oldTransfer = tr?.value ?? 0 }) .catch((err) => { console.log(err) @@ -237,83 +233,89 @@ export async function benchmark ( if (!running) { running = true - void ctx.with( - 'avg', - {}, - async () => - await monitorConnection?.findAll(contact.mixin.Employee, {}).then((res) => { + void ctx.with('avg', {}, async () => { + await monitorConnection + ?.findAll(core.class.BenchmarkDoc, { + source: 'monitor', + request: { + documents: 1, + size: 1 + } + }) + .then((res) => { const cur = Date.now() - st opTime += cur moment = cur ops++ running = false }) - ) + }) } elapsed++ + // console.log('Sheduled', scheduled) csvWriter.add( { time: elapsed, - clients: operating, + clients: operating.size, moment, average: Math.round(opTime / (ops + 1)), mem: memUsed, memTotal, cpu, requestTime, - operations: Math.round(operations / (elapsed + 1)), - transfer: Math.round(transfer / (elapsed + 1)) / 1024 + operations, + transfer: transfer / 1024 }, true ) - void csvWriter.write(`report${cmd.binary ? '-bin' : ''}${cmd.write ? '-wr' : ''}.csv`) }, 1000) - } - for (let i = cmd.from; i < cmd.from + cmd.steps; i++) { - await ctx.with('iteration', { i }, async () => { - await Promise.all( - Array.from(Array(i)) - .map((it, idx) => idx) - .map((it) => { - const wsid = workspaceId[randNum(workspaceId.length)] - const workId = generateId() - const msg: StartMessage = { - workspaceId: wsid, - transactorUrl, - id: i, - idd: it, - workId, - options: { - readRequests: 100, - modelRequests: 0, - limit: { - min: 10, - rand: 1000 - }, - sleep: cmd.sleep, - write: cmd.write - }, - binary: cmd.binary, - compression: cmd.compression - } - workers[i % workers.length].postMessage(msg) - return new Promise((resolve) => { - works.set(workId, () => { - resolve(null) + for (let i = cmd.from; i < cmd.from + cmd.steps; i++) { + await ctx.with('iteration', { i }, async () => { + await Promise.all( + Array.from(Array(i)) + .map((it, idx) => idx) + .map((it) => { + const wsid = workspaceId[randNum(workspaceId.length)] + const workId = 'w-' + i + '-' + it + const msg: StartMessage = { + workspaceId: wsid, + transactorUrl, + id: i, + idd: it, + workId, + options: { + smallRequests: 100, + rate: 1, + bigRequests: 0, + limit: { + min: 64, + rand: 512 + }, + sleep: cmd.sleep, + write: cmd.write + }, + binary: cmd.binary, + compression: cmd.compression + } + workers[i % workers.length].postMessage(msg) + + return new Promise((resolve) => { + works.set(workId, () => { + resolve(null) + }) }) }) - }) - ) - }) - console.log(metricsToString(m, `iteration-${i}`, 120)) - } + ) + }) + console.log(metricsToString(m, `iteration-${i}`, 120)) + } - if (isMainThread) { for (const w of workers) { await w.terminate() } clearInterval(timer) + await csvWriter.write(`report${cmd.binary ? '-bin' : ''}${cmd.write ? '-wr' : ''}.csv`) await monitorConnection?.close() } } @@ -344,65 +346,86 @@ export function benchmarkWorker (): void { workId: msg.workId }) - const h = connection.getHierarchy() - const allClasses = await connection.getModel().findAll(core.class.Class, {}) - const classes = allClasses.filter((it) => it.kind === ClassifierKind.CLASS && h.findDomain(it._id) !== undefined) - while (msg.options.readRequests + msg.options.modelRequests > 0) { - if (msg.options.modelRequests > 0) { - await connection?.findAll(core.class.Tx, {}, { sort: { _id: -1 } }) - msg.options.modelRequests-- - } - let doc: Doc | undefined - if (msg.options.readRequests > 0) { - const cl = classes[randNum(classes.length - 1)] - if (cl !== undefined) { - const docs = await connection?.findAll( - cl._id, - {}, - { - sort: { _id: -1 }, - limit: msg.options.limit.min + randNum(msg.options.limit.rand) - } - ) - if (docs.length > 0) { - doc = docs[randNum(docs.length - 1)] - } - msg.options.readRequests-- - } - if (msg.options.write && doc !== undefined) { - const attrs = connection.getHierarchy().getAllAttributes(doc._class) - const upd: DocumentUpdate = {} - for (const [key, value] of attrs.entries()) { - if (value.type._class === core.class.TypeString || value.type._class === core.class.TypeBoolean) { - if ( - key !== '_id' && - key !== '_class' && - key !== 'space' && - key !== 'attachedTo' && - key !== 'attachedToClass' - ) { - const v = (doc as any)[key] - if (v != null) { - ;(upd as any)[key] = v - } + const rateLimiter = new RateLimiter(msg.options.rate) + + let bigRunning = 0 + + while (msg.options.smallRequests + msg.options.bigRequests > 0) { + const variant = Math.random() + // console.log(`Thread ${msg.workId} ${msg.options.smallRequests} ${msg.options.bigRequests}`) + if (msg.options.bigRequests > 0 && variant < 0.5 && bigRunning === 0) { + await rateLimiter.add(async () => { + bigRunning = 1 + await connection?.findAll(core.class.BenchmarkDoc, { + source: msg.workId, + request: { + documents: { + from: 1024, + to: 1000 + }, + size: { + // 1kb to 5kb + from: 1 * 1024, + to: 4 * 1024 } } - } - if (Object.keys(upd).length > 0) { - await opt.update(doc, upd) - } - } + }) + }) + bigRunning = 0 + msg.options.bigRequests-- + } + if (msg.options.smallRequests > 0 && variant > 0.5) { + await rateLimiter.add(async () => { + await connection?.findAll(core.class.BenchmarkDoc, { + source: msg.workId, + request: { + documents: { + from: msg.options.limit.min, + to: msg.options.limit.min + msg.options.limit.rand + }, + size: { + from: 4, + to: 16 + } + } + }) + }) + msg.options.smallRequests-- + } + if (msg.options.write) { + await opt.updateDoc(core.class.BenchmarkDoc, core.space.DerivedTx, '' as Ref, { + response: 'qwe' + }) } if (msg.options.sleep > 0) { await new Promise((resolve) => setTimeout(resolve, randNum(msg.options.sleep))) } } + + // clearInterval(infoInterval) + await rateLimiter.waitProcessing() + const to1 = setTimeout(() => { + parentPort?.postMessage({ + type: 'log', + workId: msg.workId, + msg: `timeout waiting processing${msg.workId}` + }) + }, 5000) + clearTimeout(to1) // // console.log(`${msg.idd} perform complete`) } catch (err: any) { console.error(msg.workspaceId, err) } finally { + const to = setTimeout(() => { + parentPort?.postMessage({ + type: 'log', + workId: msg.workId, + msg: `timeout closing connection${msg.workId}` + }) + }, 5000) await connection?.close() + clearTimeout(to) } parentPort?.postMessage({ type: 'complete', diff --git a/models/core/src/benchmark.ts b/models/core/src/benchmark.ts new file mode 100644 index 0000000000..e92c4733b8 --- /dev/null +++ b/models/core/src/benchmark.ts @@ -0,0 +1,27 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import type { BenchmarkDoc } from '@hcengineering/core' +import { DOMAIN_BENCHMARK } from '@hcengineering/core' +import { Model, UX } from '@hcengineering/model' +import { getEmbeddedLabel } from '@hcengineering/platform' +import core from './component' +import { TDoc } from './core' + +// B E N C H M A R K + +@Model(core.class.BenchmarkDoc, core.class.Doc, DOMAIN_BENCHMARK) +@UX(getEmbeddedLabel('Benchmark'), undefined, undefined, undefined, 'name') +export class TBenchmarkDoc extends TDoc implements BenchmarkDoc {} diff --git a/models/core/src/index.ts b/models/core/src/index.ts index a6f875e031..7f7e6fd2b6 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -15,6 +15,7 @@ import { AccountRole, + DOMAIN_BENCHMARK, DOMAIN_BLOB, DOMAIN_BLOB_DATA, DOMAIN_CONFIGURATION, @@ -32,6 +33,7 @@ import { type TxCollectionCUD } from '@hcengineering/core' import { type Builder } from '@hcengineering/model' +import { TBenchmarkDoc } from './benchmark' import core from './component' import { TArrOf, @@ -172,7 +174,8 @@ export function createModel (builder: Builder): void { TStatusCategory, TMigrationState, TBlob, - TDomainIndexConfiguration + TDomainIndexConfiguration, + TBenchmarkDoc ) builder.createDoc( @@ -226,6 +229,12 @@ export function createModel (builder: Builder): void { ] }) + builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { + domain: DOMAIN_BENCHMARK, + disableCollection: true, + disabled: [] + }) + builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { domain: DOMAIN_CONFIGURATION, disabled: [ diff --git a/packages/core/src/benchmark.ts b/packages/core/src/benchmark.ts new file mode 100644 index 0000000000..76d3277971 --- /dev/null +++ b/packages/core/src/benchmark.ts @@ -0,0 +1,29 @@ +import { Doc, type Domain } from './classes' + +/** + * @public + */ +export const DOMAIN_BENCHMARK = 'benchmark' as Domain + +export type BenchmarkDocRange = + | number + | { + // Or random in range + from: number + to: number + } +export interface BenchmarkDoc extends Doc { + source?: string + // Query fields to perform different set of workload + request?: { + // On response will return a set of BenchmarkDoc with requested fields. + documents: BenchmarkDocRange + + // A random sized document with size from to sizeTo + size: BenchmarkDocRange + + // Produce a set of derived documents payload + derived?: BenchmarkDocRange + } + response?: string // A dummy random data to match document's size +} diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index 057221e374..a9d913bb92 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -69,6 +69,7 @@ import type { TxUpdateDoc, TxWorkspaceEvent } from './tx' +import type { BenchmarkDoc } from './benchmark' /** * @public @@ -142,7 +143,9 @@ export default plugin(coreId, { Status: '' as Ref>, StatusCategory: '' as Ref>, - MigrationState: '' as Ref> + MigrationState: '' as Ref>, + + BenchmarkDoc: '' as Ref> }, mixin: { FullTextSearchContext: '' as Ref>, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9652e82543..2677cb35d4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -33,3 +33,4 @@ export * from './status' export * from './clone' export * from './common' export * from './time' +export * from './benchmark' diff --git a/packages/core/src/storage.ts b/packages/core/src/storage.ts index df77cc2cbb..c115faed4c 100644 --- a/packages/core/src/storage.ts +++ b/packages/core/src/storage.ts @@ -196,6 +196,7 @@ export type WithLookup = T & { */ export type FindResult = WithLookup[] & { total: number + lookupMap?: Record } /** diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 26ff3948cc..f8cb3c54c2 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -106,9 +106,9 @@ export function escapeLikeForRegexp (value: string): string { /** * @public */ -export function toFindResult (docs: T[], total?: number): FindResult { +export function toFindResult (docs: T[], total?: number, lookupMap?: Record): FindResult { const length = total ?? docs.length - return Object.assign(docs, { total: length }) + return Object.assign(docs, { total: length, lookupMap }) } /** @@ -370,7 +370,7 @@ export class RateLimiter { } async waitProcessing (): Promise { - await Promise.race(this.processingQueue.values()) + await Promise.all(this.processingQueue.values()) } } @@ -447,6 +447,14 @@ function mergeField (field1: any, field2: any): any | undefined { for (const x in predicate) { const result = mergePredicateWithValue(x, predicate[x], value) + if ( + Array.isArray(result?.$in) && + result.$in.length > 0 && + Array.isArray(result?.$nin) && + result.$nin.length === 0 + ) { + delete result.$nin + } if (result !== undefined) { return result } @@ -714,3 +722,42 @@ export function isClassIndexable (hierarchy: Hierarchy, c: Ref>): boo hierarchy.setClassifierProp(c, 'class_indexed', result) return result } + +type ReduceParameters any> = T extends (...args: infer P) => any ? P : never + +interface NextCall { + op: () => Promise +} + +/** + * Utility method to skip middle update calls, optimistically if update function is called multiple times with few different parameters, only the last variant will be executed. + * The last invocation is executed after a few cycles, allowing to skip middle ones. + * + * This method can be used inside Svelte components to collapse complex update logic and handle interactions. + */ +export function reduceCalls) => Promise> ( + operation: T +): (...args: ReduceParameters) => Promise { + let nextCall: NextCall | undefined + let currentCall: NextCall | undefined + + const next = (): void => { + currentCall = nextCall + nextCall = undefined + if (currentCall !== undefined) { + void currentCall.op() + } + } + return async function (...args: ReduceParameters): Promise { + const myOp = async (): Promise => { + await operation(...args) + next() + } + + nextCall = { op: myOp } + await Promise.resolve() + if (currentCall === undefined) { + next() + } + } +} diff --git a/packages/presentation/src/utils.ts b/packages/presentation/src/utils.ts index 362e484805..c7a8b33a41 100644 --- a/packages/presentation/src/utils.ts +++ b/packages/presentation/src/utils.ts @@ -18,6 +18,7 @@ import { Analytics } from '@hcengineering/analytics' import core, { TxOperations, getCurrentAccount, + reduceCalls, type AnyAttribute, type ArrOf, type AttachedDoc, @@ -52,6 +53,7 @@ import { onDestroy } from 'svelte' import { type KeyedAttribute } from '..' import { OptimizeQueryMiddleware, PresentationPipelineImpl, type PresentationPipeline } from './pipeline' import plugin from './plugin' +export { reduceCalls } from '@hcengineering/core' let liveQuery: LQ let client: TxOperations & MeasureClient @@ -248,10 +250,22 @@ export class LiveQuery { // We need to prevent callback with old values to be happening // One time refresh in case of client recreation this.clientRecreated = false - this.doQuery(++this.reqId, _class, query, callback, options) + void this.reducedDoQuery(++this.reqId, _class, query, callback as any, options) return true } + reducedDoQuery = reduceCalls( + async ( + id: number, + _class: Ref>, + query: DocumentQuery, + callback: (result: FindResult) => void | Promise, + options: FindOptions | undefined + ) => { + this.doQuery(id, _class, query, callback, options) + } + ) + private doQuery( id: number, _class: Ref>, @@ -552,42 +566,3 @@ export function decodeTokenPayload (token: string): any { export function isAdminUser (): boolean { return decodeTokenPayload(getMetadata(plugin.metadata.Token) ?? '').admin === 'true' } - -type ReduceParameters any> = T extends (...args: infer P) => any ? P : never - -interface NextCall { - op: () => Promise -} - -/** - * Utility method to skip middle update calls, optimistically if update function is called multiple times with few different parameters, only the last variant will be executed. - * The last invocation is executed after a few cycles, allowing to skip middle ones. - * - * This method can be used inside Svelte components to collapse complex update logic and handle interactions. - */ -export function reduceCalls) => Promise> ( - operation: T -): (...args: ReduceParameters) => Promise { - let nextCall: NextCall | undefined - let currentCall: NextCall | undefined - - const next = (): void => { - currentCall = nextCall - nextCall = undefined - if (currentCall !== undefined) { - void currentCall.op() - } - } - return async function (...args: ReduceParameters): Promise { - const myOp = async (): Promise => { - await operation(...args) - next() - } - - nextCall = { op: myOp } - await Promise.resolve() - if (currentCall === undefined) { - next() - } - } -} diff --git a/plugins/chunter-resources/src/components/chat/Chat.svelte b/plugins/chunter-resources/src/components/chat/Chat.svelte index 51adcdb179..86e5bd26c5 100644 --- a/plugins/chunter-resources/src/components/chat/Chat.svelte +++ b/plugins/chunter-resources/src/components/chat/Chat.svelte @@ -70,7 +70,7 @@ $: void loadObject(selectedData?._id, selectedData?._class) async function loadObject (_id?: Ref, _class?: Ref>): Promise { - if (_id === undefined || _class === undefined) { + if (_id == null || _class == null || _class === '') { object = undefined objectQuery.unsubscribe() return diff --git a/plugins/client-resources/src/connection.ts b/plugins/client-resources/src/connection.ts index beaa68a92f..8e03c95ba0 100644 --- a/plugins/client-resources/src/connection.ts +++ b/plugins/client-resources/src/connection.ts @@ -41,7 +41,8 @@ import core, { TxResult, TxWorkspaceEvent, WorkspaceEvent, - generateId + generateId, + toFindResult } from '@hcengineering/core' import { PlatformError, UNAUTHORIZED, broadcastEvent, getMetadata, unknownError } from '@hcengineering/platform' @@ -70,14 +71,23 @@ class RequestPromise { }) } - chunks?: { index: number, data: any[] }[] + chunks?: { index: number, data: FindResult }[] } class Connection implements ClientConnection { private websocket: ClientSocket | null = null + binaryMode = false private readonly requests = new Map() private lastId = 0 private interval: number | undefined + private dialTimer: any | undefined + + private sockets = 0 + + private incomingTimer: any + + private openAction: any + private sessionId: string | undefined private closed = false @@ -136,15 +146,11 @@ class Connection implements ClientConnection { async close (): Promise { this.closed = true + clearTimeout(this.openAction) + clearTimeout(this.dialTimer) clearInterval(this.interval) if (this.websocket !== null) { - if (this.websocket instanceof Promise) { - await this.websocket.then((ws) => { - ws.close(1000) - }) - } else { - this.websocket.close(1000) - } + this.websocket.close(1000) this.websocket = null } } @@ -166,12 +172,6 @@ class Connection implements ClientConnection { }) } - sockets = 0 - - incomingTimer: any - - openAction: any - scheduleOpen (force: boolean): void { if (force) { if (this.websocket !== null) { @@ -181,6 +181,7 @@ class Connection implements ClientConnection { clearTimeout(this.openAction) this.openAction = undefined } + clearInterval(this.interval) if (!this.closed && this.openAction === undefined) { if (this.websocket === null) { const socketId = ++this.sockets @@ -198,6 +199,7 @@ class Connection implements ClientConnection { } private openConnection (socketId: number): void { + this.binaryMode = false // Use defined factory or browser default one. const clientSocketFactory = getMetadata(client.metadata.ClientSocketFactory) ?? @@ -229,19 +231,21 @@ class Connection implements ClientConnection { } this.websocket = wsocket const opened = false - let binaryResponse = false - const dialTimer = setTimeout(() => { + this.dialTimer = setTimeout(() => { if (!opened && !this.closed) { this.scheduleOpen(true) } }, dialTimeout) wsocket.onmessage = (event: MessageEvent) => { + if (this.closed) { + return + } if (this.websocket !== wsocket) { return } - const resp = readResponse(event.data, binaryResponse) + const resp = readResponse(event.data, this.binaryMode) if (resp.error !== undefined) { if (resp.error?.code === UNAUTHORIZED.code) { @@ -280,7 +284,7 @@ class Connection implements ClientConnection { return } if ((resp as HelloResponse).binary) { - binaryResponse = true + this.binaryMode = true } // Notify all waiting connection listeners const handlers = this.onConnectHandlers.splice(0, this.onConnectHandlers.length) @@ -309,7 +313,11 @@ class Connection implements ClientConnection { if (resp.id !== undefined) { const promise = this.requests.get(resp.id) if (promise === undefined) { - throw new Error(`unknown response id: ${resp.id as string} ${this.workspace} ${this.email}`) + console.error( + new Error(`unknown response id: ${resp.id as string} ${this.workspace} ${this.email}`), + JSON.stringify(this.requests) + ) + return } if (resp.chunk !== undefined) { @@ -317,17 +325,26 @@ class Connection implements ClientConnection { ...(promise.chunks ?? []), { index: resp.chunk.index, - data: resp.result as [] + data: resp.result as FindResult } ] - // console.log(socketId, 'chunk', promise.chunks.length, resp.chunk.total) + // console.log(socketId, 'chunk', promise.method, promise.params, promise.chunks.length, (resp.result as []).length) if (resp.chunk.final) { promise.chunks.sort((a, b) => a.index - b.index) let result: any[] = [] + let total = -1 + let lookupMap: Record | undefined + for (const c of promise.chunks) { + if (c.data.total !== 0) { + total = c.data.total + } + if (c.data.lookupMap !== undefined) { + lookupMap = c.data.lookupMap + } result = result.concat(c.data) } - resp.result = result + resp.result = toFindResult(result, total, lookupMap) resp.chunk = undefined } else { // Not all chunks are available yet. @@ -387,10 +404,10 @@ class Connection implements ClientConnection { } } wsocket.onclose = (ev) => { - clearTimeout(dialTimer) + clearTimeout(this.dialTimer) if (this.websocket !== wsocket) { wsocket.close() - clearTimeout(dialTimer) + clearTimeout(this.dialTimer) return } // console.log('client websocket closed', socketId, ev?.reason) @@ -403,7 +420,7 @@ class Connection implements ClientConnection { } const useBinary = getMetadata(client.metadata.UseBinaryProtocol) ?? true const useCompression = getMetadata(client.metadata.UseProtocolCompression) ?? false - clearTimeout(dialTimer) + clearTimeout(this.dialTimer) const helloRequest: HelloRequest = { method: 'hello', params: [], @@ -416,7 +433,7 @@ class Connection implements ClientConnection { } wsocket.onerror = (event: any) => { - clearTimeout(dialTimer) + clearTimeout(this.dialTimer) if (this.websocket !== wsocket) { return } @@ -438,6 +455,7 @@ class Connection implements ClientConnection { handleResult?: (result: any) => Promise once?: boolean // Require handleResult to retrieve result measure?: (time: number, result: any, serverTime: number, queue: number) => void + allowReconnect?: boolean }): Promise { if (this.closed) { throw new PlatformError(unknownError('connection closed')) @@ -462,7 +480,7 @@ class Connection implements ClientConnection { await w } this.requests.set(id, promise) - const sendData = (): void => { + const sendData = async (): Promise => { if (this.websocket?.readyState === ClientSocketReadyState.OPEN) { promise.startTime = Date.now() this.websocket?.send( @@ -470,22 +488,25 @@ class Connection implements ClientConnection { { method: data.method, params: data.params, - id + id, + time: Date.now() }, - false + this.binaryMode ) ) } } - promise.reconnect = () => { - setTimeout(async () => { - // In case we don't have response yet. - if (this.requests.has(id) && ((await data.retry?.()) ?? true)) { - sendData() - } - }, 500) + if (data.allowReconnect ?? true) { + promise.reconnect = () => { + setTimeout(async () => { + // In case we don't have response yet. + if (this.requests.has(id) && ((await data.retry?.()) ?? true)) { + void sendData() + } + }, 50) + } } - sendData() + void sendData() void broadcastEvent(client.event.NetworkRequests, this.requests.size) return await promise.promise } @@ -527,11 +548,52 @@ class Connection implements ClientConnection { method: 'findAll', params: [_class, query, options], measure: (time, result, serverTime, queue) => { - if (typeof window !== 'undefined' && time > 1000) { - console.error('measure slow findAll', time, serverTime, queue, _class, query, options, result) + if (typeof window !== 'undefined' && (time > 1000 || serverTime > 500)) { + console.error( + 'measure slow findAll', + time, + serverTime, + queue, + _class, + query, + options, + result, + JSON.stringify(result).length + ) } } }) + if (result.lookupMap !== undefined) { + // We need to extract lookup map to document lookups + for (const d of result) { + if (d.$lookup !== undefined) { + for (const [k, v] of Object.entries(d.$lookup)) { + if (!Array.isArray(v)) { + d.$lookup[k] = result.lookupMap[v as any] + } else { + d.$lookup[k] = v.map((it) => result.lookupMap?.[it]) + } + } + } + } + delete result.lookupMap + } + + // We need to revert deleted query simple values. + // We need to get rid of simple query parameters matched in documents + for (const doc of result) { + if (doc._class == null) { + doc._class = _class + } + for (const [k, v] of Object.entries(query)) { + if (typeof v === 'string' || typeof v === 'number') { + if (doc[k] == null) { + doc[k] = v + } + } + } + } + return result } @@ -586,7 +648,7 @@ class Connection implements ClientConnection { } sendForceClose (): Promise { - return this.sendRequest({ method: 'forceClose', params: [] }) + return this.sendRequest({ method: 'forceClose', params: [], allowReconnect: false }) } } diff --git a/plugins/devmodel-resources/src/index.ts b/plugins/devmodel-resources/src/index.ts index 251f5f1721..04d56d664e 100644 --- a/plugins/devmodel-resources/src/index.ts +++ b/plugins/devmodel-resources/src/index.ts @@ -135,7 +135,8 @@ class ModelClient implements AccountClient { ' =>model', this.client.getModel(), getMetadata(devmodel.metadata.DevModel), - Date.now() - startTime + Date.now() - startTime, + JSON.stringify(result).length ) } return result diff --git a/plugins/task-resources/src/index.ts b/plugins/task-resources/src/index.ts index 00f666a961..183bf196f4 100644 --- a/plugins/task-resources/src/index.ts +++ b/plugins/task-resources/src/index.ts @@ -238,8 +238,10 @@ async function statusSort ( const aIndex = getStatusIndex(type, taskTypes, a) const bIndex = getStatusIndex(type, taskTypes, b) return aIndex - bIndex - } else { + } else if (aVal != null && bVal != null) { return aVal.name.localeCompare(bVal.name) + } else { + return 0 } }) } else { diff --git a/plugins/tracker-resources/src/components/myissues/MyIssues.svelte b/plugins/tracker-resources/src/components/myissues/MyIssues.svelte index 7ee3153cf3..d796c5cb53 100644 --- a/plugins/tracker-resources/src/components/myissues/MyIssues.svelte +++ b/plugins/tracker-resources/src/components/myissues/MyIssues.svelte @@ -71,7 +71,7 @@ subscribed = { _id: { $in: newSub } } } }, - { sort: { _id: 1 } } + { sort: { _id: 1 }, projection: { _id: 1 } } ) const archivedProjectQuery = createQuery() diff --git a/plugins/tracker-resources/src/utils.ts b/plugins/tracker-resources/src/utils.ts index 4220d987ea..b3054bb76d 100644 --- a/plugins/tracker-resources/src/utils.ts +++ b/plugins/tracker-resources/src/utils.ts @@ -352,7 +352,7 @@ export async function issueStatusSort ( const aIndex = getStatusIndex(type, taskTypes, a) const bIndex = getStatusIndex(type, taskTypes, b) return aIndex - bIndex - } else { + } else if (aVal != null && bVal != null) { return aVal.name.localeCompare(bVal.name) } } diff --git a/plugins/view-resources/src/components/list/List.svelte b/plugins/view-resources/src/components/list/List.svelte index ee3b30ef33..8c78361b88 100644 --- a/plugins/view-resources/src/components/list/List.svelte +++ b/plugins/view-resources/src/components/list/List.svelte @@ -101,14 +101,17 @@ }, { ...categoryQueryOptions, limit: 1000 } ) - $: docsQuerySlow.query( - _class, - queryNoLookup, - (res) => { - slowDocs = res - }, - categoryQueryOptions - ) + + $: if (fastDocs.length === 1000) { + docsQuerySlow.query( + _class, + queryNoLookup, + (res) => { + slowDocs = res + }, + categoryQueryOptions + ) + } $: docs = [...fastDocs, ...slowDocs.filter((it) => !fastQueryIds.has(it._id))] diff --git a/pods/server/package.json b/pods/server/package.json index d0c0724bd7..fb17a5bd0d 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -13,7 +13,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node --external:bufferutil --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node --external:bufferutil --external:utf-8-validate --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js", "bundle:u": "mkdir -p bundle && esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node > bundle/bundle.js && mkdir -p ./dist && cp -r ./node_modules/uWebSockets.js/*.node ./dist", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/transactor", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/transactor staging", diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index 9ddccac0d8..8170d2171e 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -28,6 +28,7 @@ import { import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic' import { ConfigurationMiddleware, + LookupMiddleware, ModifiedMiddleware, PrivateMiddleware, QueryJoinMiddleware, @@ -60,14 +61,14 @@ import { FullTextPushStage, globalIndexer, IndexedFieldStage, - type StorageConfiguration, type ContentTextAdapter, type DbConfiguration, type FullTextAdapter, type FullTextPipelineStage, type MiddlewareCreator, type Pipeline, - type StorageAdapter + type StorageAdapter, + type StorageConfiguration } from '@hcengineering/server-core' import { serverDocumentId } from '@hcengineering/server-document' import { serverGmailId } from '@hcengineering/server-gmail' @@ -228,12 +229,13 @@ export function start ( addLocation(serverTimeId, () => import('@hcengineering/server-time-resources')) const middlewares: MiddlewareCreator[] = [ + LookupMiddleware.create, ModifiedMiddleware.create, PrivateMiddleware.create, SpaceSecurityMiddleware.create, SpacePermissionsMiddleware.create, ConfigurationMiddleware.create, - QueryJoinMiddleware.create // Should be last one + QueryJoinMiddleware.create ] const metrics = getMetricsContext() diff --git a/server/account/package.json b/server/account/package.json index c132be166b..7b702f5dc2 100644 --- a/server/account/package.json +++ b/server/account/package.json @@ -40,7 +40,7 @@ "@hcengineering/contact": "^0.6.20", "@hcengineering/client-resources": "^0.6.23", "@hcengineering/client": "^0.6.14", - "ws": "^8.10.0", + "ws": "^8.16.0", "@hcengineering/model": "^0.6.7", "@hcengineering/server-backup": "^0.6.0", "@hcengineering/server-tool": "^0.6.0", diff --git a/server/collaborator/package.json b/server/collaborator/package.json index b2aa9f2910..6a88798091 100644 --- a/server/collaborator/package.json +++ b/server/collaborator/package.json @@ -73,6 +73,6 @@ "body-parser": "^1.20.2", "cors": "^2.8.5", "compression": "~1.7.4", - "ws": "^8.10.0" + "ws": "^8.16.0" } } diff --git a/server/collaborator/src/server.ts b/server/collaborator/src/server.ts index af785bf7d5..2b88d19e0c 100644 --- a/server/collaborator/src/server.ts +++ b/server/collaborator/src/server.ts @@ -68,7 +68,8 @@ export async function start ( // fallback to standard filter function return compression.filter(req, res) }, - level: 6 + level: 1, + memLevel: 9 }) ) @@ -206,13 +207,14 @@ export async function start ( noServer: true, perMessageDeflate: { zlibDeflateOptions: { - // See zlib defaults. - chunkSize: 1024, - memLevel: 7, - level: 3 + chunkSize: 32 * 1024, + memLevel: 9, + level: 1 }, zlibInflateOptions: { - chunkSize: 10 * 1024 + chunkSize: 32 * 1024, + memLevel: 9, + level: 1 }, // Below options specified as default values. concurrencyLimit: 10, // Limits zlib concurrency for perf. diff --git a/server/core/src/benchmark/index.ts b/server/core/src/benchmark/index.ts new file mode 100644 index 0000000000..d38e938dea --- /dev/null +++ b/server/core/src/benchmark/index.ts @@ -0,0 +1,100 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import core, { + generateId, + toFindResult, + type BenchmarkDoc, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + type Hierarchy, + type MeasureContext, + type ModelDb, + type Ref, + type Space, + type WorkspaceId +} from '@hcengineering/core' +import type { DbAdapter } from '../adapter' +import { DummyDbAdapter } from '../mem' + +function genData (dataSize: number): string { + let result = '' + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + const charactersLength = characters.length + for (let i = 0; i < dataSize; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)) + } + return result +} + +let benchData = '' + +class BenchmarkDbAdapter extends DummyDbAdapter { + async findAll>( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions | undefined + ): Promise> { + if (_class !== core.class.BenchmarkDoc) { + return toFindResult([]) + } + + if (benchData === '') { + benchData = genData(1024 * 1024) + } + + const result: BenchmarkDoc[] = [] + + const request: BenchmarkDoc['request'] = ((query as DocumentQuery) + .request as BenchmarkDoc['request']) ?? { + documents: 1, + size: 1 + } + const docsToAdd = + typeof request.documents === 'number' + ? request.documents + : request.documents.from + Math.random() * request.documents.to + for (let i = 0; i < docsToAdd; i++) { + const dataSize = + typeof request.size === 'number' ? request.size : request.size.from + Math.random() * request.size.to + result.push({ + _class: core.class.BenchmarkDoc, + _id: generateId(), + modifiedBy: core.account.System, + modifiedOn: Date.now(), + space: core.space.DerivedTx, // To be available for all + response: benchData.slice(0, dataSize) + }) + } + + return toFindResult(result as T[]) + } +} +/** + * @public + */ +export async function createBenchmarkAdapter ( + ctx: MeasureContext, + hierarchy: Hierarchy, + url: string, + workspaceId: WorkspaceId, + modelDb: ModelDb +): Promise { + return new BenchmarkDbAdapter() +} diff --git a/server/core/src/index.ts b/server/core/src/index.ts index 599ceb2db3..b087351b6a 100644 --- a/server/core/src/index.ts +++ b/server/core/src/index.ts @@ -26,3 +26,4 @@ export * from './server' export * from './storage' export * from './types' export * from './utils' +export * from './benchmark' diff --git a/server/middleware/src/index.ts b/server/middleware/src/index.ts index 22dcd21f3c..747fb311d0 100644 --- a/server/middleware/src/index.ts +++ b/server/middleware/src/index.ts @@ -20,3 +20,4 @@ export * from './private' export * from './queryJoin' export * from './spaceSecurity' export * from './spacePermissions' +export * from './lookup' diff --git a/server/middleware/src/lookup.ts b/server/middleware/src/lookup.ts new file mode 100644 index 0000000000..33356f40d8 --- /dev/null +++ b/server/middleware/src/lookup.ts @@ -0,0 +1,114 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + Class, + Doc, + DocumentQuery, + FindOptions, + FindResult, + MeasureContext, + Ref, + ServerStorage, + Tx, + clone, + toFindResult +} from '@hcengineering/core' +import { BroadcastFunc, Middleware, SessionContext, TxMiddlewareResult } from '@hcengineering/server-core' +import { BaseMiddleware } from './base' + +/** + * @public + */ +export class LookupMiddleware extends BaseMiddleware implements Middleware { + private constructor (storage: ServerStorage, next?: Middleware) { + super(storage, next) + } + + static async create ( + ctx: MeasureContext, + broadcast: BroadcastFunc, + storage: ServerStorage, + next?: Middleware + ): Promise { + return new LookupMiddleware(storage, next) + } + + async tx (ctx: SessionContext, tx: Tx): Promise { + return await this.provideTx(ctx, tx) + } + + handleBroadcast (tx: Tx[], targets?: string[]): Tx[] { + return this.provideHandleBroadcast(tx, targets) + } + + override async findAll( + ctx: SessionContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const result = await this.provideFindAll(ctx, _class, query, options) + // Fill lookup map to make more compact representation + + if (options?.lookup !== undefined) { + const newResult: T[] = [] + let counter = 0 + const idClassMap: Record = {} + + function mapDoc (doc: Doc): number { + const key = doc._class + '@' + doc._id + let docRef = idClassMap[key] + if (docRef === undefined) { + docRef = { id: ++counter, doc, count: -1 } + idClassMap[key] = docRef + } + docRef.count++ + return docRef.id + } + + for (const d of result) { + if (d.$lookup !== undefined) { + const newDoc = clone(d) + newResult.push(newDoc) + for (const [k, v] of Object.entries(d.$lookup)) { + if (!Array.isArray(v)) { + newDoc.$lookup[k] = mapDoc(v) + } else { + newDoc.$lookup[k] = v.map((it) => mapDoc(it)) + } + } + } + } + const lookupMap = Object.fromEntries(Array.from(Object.values(idClassMap)).map((it) => [it.id, it.doc])) + if (Object.keys(lookupMap).length > 0) { + return toFindResult(newResult, result.total, lookupMap) + } + } + + // We need to get rid of simple query parameters matched in documents + for (const doc of result) { + for (const [k, v] of Object.entries(query)) { + if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { + if ((doc as any)[k] === v) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (doc as any)[k] + } + } + } + } + return result + } +} diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index a56d04199b..8ef338f3e1 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -229,7 +229,8 @@ abstract class MongoAdapterBase implements DbAdapter { from: domain, localField: fullKey, foreignField: '_id', - as: fullKey.split('.').join('') + '_lookup' + as: fullKey.split('.').join('') + '_lookup', + pipeline: [{ $project: { '%hash%': 0 } }] }) } await this.getLookupValue(_class, nested, result, fullKey + '_lookup') @@ -243,7 +244,8 @@ abstract class MongoAdapterBase implements DbAdapter { from: domain, localField: fullKey, foreignField: '_id', - as: fullKey.split('.').join('') + '_lookup' + as: fullKey.split('.').join('') + '_lookup', + pipeline: [{ $project: { '%hash%': 0 } }] }) } } @@ -283,7 +285,8 @@ abstract class MongoAdapterBase implements DbAdapter { $match: { _class: { $in: desc } } - } + }, + { $project: { '%hash%': 0 } } ], as: asVal } diff --git a/server/rpc/src/rpc.ts b/server/rpc/src/rpc.ts index 8a79fed0ff..77807cd841 100644 --- a/server/rpc/src/rpc.ts +++ b/server/rpc/src/rpc.ts @@ -30,6 +30,8 @@ export interface Request

{ id?: ReqId method: string params: P + + time?: number // Server time to perform operation } /** @@ -86,7 +88,12 @@ export function protoSerialize (object: object, binary: boolean): any { */ export function protoDeserialize (data: any, binary: boolean): any { if (!binary) { - return JSON.parse(data, receiver) + let _data = data + if (_data instanceof ArrayBuffer) { + const decoder = new TextDecoder() + _data = decoder.decode(_data) + } + return JSON.parse(_data.toString(), receiver) } return packr.unpack(new Uint8Array(replacer('', data))) } @@ -117,10 +124,11 @@ export function readResponse (response: any, binary: boolean): Response { } function replacer (key: string, value: any): any { - if (Array.isArray(value) && (value as any).total !== undefined) { + if (Array.isArray(value) && ((value as any).total !== undefined || (value as any).lookupMap !== undefined)) { return { dataType: 'TotalArray', total: (value as any).total, + lookupMap: (value as any).lookupMap, value: [...value] } } else { @@ -131,7 +139,7 @@ function replacer (key: string, value: any): any { function receiver (key: string, value: any): any { if (typeof value === 'object' && value !== null) { if (value.dataType === 'TotalArray') { - return Object.assign(value.value, { total: value.total }) + return Object.assign(value.value, { total: value.total, lookupMap: value.lookupMap }) } } return value diff --git a/server/tool/package.json b/server/tool/package.json index 44e5714f38..69aecb8942 100644 --- a/server/tool/package.json +++ b/server/tool/package.json @@ -39,7 +39,7 @@ "@hcengineering/contact": "^0.6.20", "@hcengineering/client-resources": "^0.6.23", "@hcengineering/client": "^0.6.14", - "ws": "^8.10.0", + "ws": "^8.16.0", "@hcengineering/model": "^0.6.7", "@hcengineering/server-token": "^0.6.7", "@hcengineering/server-core": "^0.6.1", diff --git a/server/ws/package.json b/server/ws/package.json index 796832ac94..c7125fc870 100644 --- a/server/ws/package.json +++ b/server/ws/package.json @@ -19,35 +19,36 @@ }, "devDependencies": { "@hcengineering/platform-rig": "^0.6.0", - "@types/node": "~20.11.16", - "@typescript-eslint/eslint-plugin": "^6.11.0", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-n": "^15.4.0", - "eslint": "^8.54.0", - "@types/ws": "^8.5.3", - "@typescript-eslint/parser": "^6.11.0", - "eslint-config-standard-with-typescript": "^40.0.0", - "prettier": "^3.1.0", - "typescript": "^5.3.3", - "@types/express": "^4.17.13", - "@types/cors": "^2.8.12", "@types/compression": "~1.7.2", + "@types/cors": "^2.8.12", + "@types/express": "^4.17.13", + "@types/jest": "^29.5.5", + "@types/node": "~20.11.16", + "@types/ws": "^8.5.3", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint": "^8.54.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.4.0", + "eslint-plugin-promise": "^6.1.1", "jest": "^29.7.0", + "prettier": "^3.1.0", "ts-jest": "^29.1.1", - "@types/jest": "^29.5.5" + "typescript": "^5.3.3" }, "dependencies": { - "ws": "^8.10.0", - "@hcengineering/platform": "^0.6.9", + "@hcengineering/analytics": "^0.6.0", "@hcengineering/core": "^0.6.28", + "@hcengineering/platform": "^0.6.9", + "@hcengineering/rpc": "^0.6.1", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-token": "^0.6.7", - "@hcengineering/rpc": "^0.6.1", "bufferutil": "^4.0.7", - "express": "^4.18.3", "compression": "~1.7.4", "cors": "^2.8.5", - "@hcengineering/analytics": "^0.6.0" + "express": "^4.18.3", + "utf-8-validate": "^6.0.3", + "ws": "^8.16.0" } } diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index 602e0bf4e6..e8de1f47af 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -21,6 +21,11 @@ import WebSocket from 'ws' import { start } from '../server' import { + Hierarchy, + MeasureMetricsContext, + ModelDb, + getWorkspaceId, + toFindResult, type Account, type Class, type Doc, @@ -28,15 +33,10 @@ import { type Domain, type FindOptions, type FindResult, - getWorkspaceId, - Hierarchy, type MeasureContext, - MeasureMetricsContext, - ModelDb, type Ref, type ServerStorage, type Space, - toFindResult, type Tx, type TxResult } from '@hcengineering/core' @@ -239,13 +239,15 @@ describe('server', () => { try { // const token: string = generateToken('my@email.com', getWorkspaceId('latest', '')) + let clearTo: any const timeoutPromise = new Promise((resolve) => { - setTimeout(resolve, 4000) + clearTo = setTimeout(resolve, 4000) }) const t1 = await findClose(token, timeoutPromise, 1005) const t2 = await findClose(token, timeoutPromise, 1000) expect(t1).toBe(t2) + clearTimeout(clearTo) } catch (err: any) { console.error(err) } finally { diff --git a/server/ws/src/client.ts b/server/ws/src/client.ts index b78963a211..691387ed94 100644 --- a/server/ws/src/client.ts +++ b/server/ws/src/client.ts @@ -14,8 +14,12 @@ // import core, { - type Account, AccountRole, + TxFactory, + TxProcessor, + WorkspaceEvent, + generateId, + type Account, type BulkUpdateEvent, type Class, type Doc, @@ -33,12 +37,8 @@ import core, { type TxApplyIf, type TxApplyResult, type TxCUD, - TxFactory, - TxProcessor, type TxResult, - type TxWorkspaceEvent, - WorkspaceEvent, - generateId + type TxWorkspaceEvent } from '@hcengineering/core' import { type Pipeline, type SessionContext } from '@hcengineering/server-core' import { type Token } from '@hcengineering/server-token' @@ -50,7 +50,7 @@ import { type BroadcastCall, type Session, type SessionRequest, type StatisticsE export class ClientSession implements Session { createTime = Date.now() requests = new Map() - binaryResponseMode: boolean = false + binaryMode: boolean = false useCompression: boolean = true useBroadcast: boolean = false sessionId = '' @@ -190,17 +190,23 @@ export class ClientSession implements Session { if (tx._class === core.class.TxApplyIf) { ;(result as TxApplyResult).derived.push(...derived) } - while (derived.length > 0) { - const part = derived.splice(0, 250) - console.log('Broadcasting part', part.length, derived.length) - this.broadcast(null, this.token.workspace, { result: part }, target) - } + // Let's send after our response will go out + setImmediate(() => { + while (derived.length > 0) { + const part = derived.splice(0, 250) + console.log('Broadcasting part', part.length, derived.length) + this.broadcast(null, this.token.workspace, { result: part }, target) + } + }) } } else { - while (derived.length > 0) { - const part = derived.splice(0, 250) - this.broadcast(null, this.token.workspace, { result: part }, target) - } + // Let's send after our response will go out + setImmediate(() => { + while (derived.length > 0) { + const part = derived.splice(0, 250) + this.broadcast(null, this.token.workspace, { result: part }, target) + } + }) } } if (tx._class === core.class.TxApplyIf) { diff --git a/server/ws/src/server.ts b/server/ws/src/server.ts index 7e139833ab..1b0151fd90 100644 --- a/server/ws/src/server.ts +++ b/server/ws/src/server.ts @@ -19,10 +19,12 @@ import core, { WorkspaceEvent, generateId, systemAccountEmail, + toFindResult, toWorkspaceString, versionToString, withContext, type BaseWorkspaceInfo, + type FindResult, type MeasureContext, type Tx, type TxWorkspaceEvent, @@ -54,9 +56,25 @@ interface WorkspaceLoginInfo extends Omit { workspaceId: string } -function timeoutPromise (time: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, time) +function timeoutPromise (time: number): { promise: Promise, cancelHandle: () => void } { + let timer: any + return { + promise: new Promise((resolve) => { + timer = setTimeout(resolve, time) + }), + cancelHandle: () => { + clearTimeout(timer) + } + } +} + +function onNextTick (op: () => void): void { + setImmediate(op) +} + +function waitNextTick (): Promise { + return new Promise((resolve) => { + setImmediate(resolve) }) } @@ -75,7 +93,7 @@ class TSessionManager implements SessionManager { checkInterval: any sessions = new Map() - reconnectIds = new Set() + reconnectIds = new Map() maintenanceTimer: any timeMinutes = 0 @@ -164,14 +182,14 @@ class TSessionManager implements SessionManager { this.ctx.warn('session hang, closing...', { wsId, user: s[1].session.getUser() }) // Force close workspace if only one client and it hang. - void this.close(s[1].socket, workspace.workspaceId) + void this.close(s[1].socket, wsId) continue } if (diff > 20000 && diff < 60000 && this.ticks % 10 === 0) { void s[1].socket.send( workspace.context, { result: 'ping' }, - s[1].session.binaryResponseMode, + s[1].session.binaryMode, s[1].session.useCompression ) } @@ -366,7 +384,7 @@ class TSessionManager implements SessionManager { void ctx.with('set-status', {}, (ctx) => this.trySetStatus(ctx, session, true)) if (this.timeMinutes > 0) { - void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryResponseMode, session.useCompression) + void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) } return { session, context: workspace.context, workspaceId: wsString } } @@ -438,12 +456,7 @@ class TSessionManager implements SessionManager { if (targets !== undefined && !targets.includes(session.session.getUser())) continue for (const _tx of tx) { try { - void session.socket.send( - ctx, - { result: _tx }, - session.session.binaryResponseMode, - session.session.useCompression - ) + void session.socket.send(ctx, { result: _tx }, session.session.binaryMode, session.session.useCompression) } catch (err: any) { Analytics.handleError(err) ctx.error('error during send', { error: err }) @@ -451,12 +464,17 @@ class TSessionManager implements SessionManager { } } if (sessions.length > 0) { - setImmediate(send) + onNextTick(send) } else { ctx.end() } } - send() + if (sessions.length > 0) { + // We need to send broadcast after our client response so put it after all IO + onNextTick(send) + } else { + ctx.end() + } } private createWorkspace ( @@ -532,8 +550,7 @@ class TSessionManager implements SessionManager { } catch {} } - async close (ws: ConnectionSocket, workspaceId: WorkspaceId): Promise { - const wsid = toWorkspaceString(workspaceId) + async close (ws: ConnectionSocket, wsid: string): Promise { const workspace = this.workspaces.get(wsid) const sessionRef = this.sessions.get(ws.id) @@ -542,23 +559,25 @@ class TSessionManager implements SessionManager { if (workspace !== undefined) { workspace.sessions.delete(sessionRef.session.sessionId) } - this.reconnectIds.add(sessionRef.session.sessionId) + this.reconnectIds.set( + sessionRef.session.sessionId, + setTimeout(() => { + this.reconnectIds.delete(sessionRef.session.sessionId) - setTimeout(() => { - this.reconnectIds.delete(sessionRef.session.sessionId) - }, this.timeouts.reconnectTimeout) + const user = sessionRef.session.getUser() + if (workspace !== undefined) { + const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user) + if (another === -1 && !workspace.upgrade) { + void this.trySetStatus(workspace.context, sessionRef.session, false) + } + } + }, this.timeouts.reconnectTimeout) + ) try { sessionRef.socket.close() } catch (err) { // Ignore if closed } - const user = sessionRef.session.getUser() - if (workspace !== undefined) { - const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user) - if (another === -1 && !workspace.upgrade) { - await this.trySetStatus(workspace.context, sessionRef.session, false) - } - } } } @@ -596,7 +615,7 @@ class TSessionManager implements SessionManager { s.workspaceClosed = true if (reason === 'upgrade' || reason === 'force-close') { // Override message handler, to wait for upgrading response from clients. - await this.sendUpgrade(workspace.context, webSocket, s.binaryResponseMode) + this.sendUpgrade(workspace.context, webSocket, s.binaryMode) } webSocket.close() } @@ -623,15 +642,17 @@ class TSessionManager implements SessionManager { } } await this.ctx.with('closing', {}, async () => { - await Promise.race([closePipeline(), timeoutPromise(120000)]) + const to = timeoutPromise(120000) + await Promise.race([closePipeline(), to.promise]) + to.cancelHandle() }) if (LOGGING_ENABLED) { this.ctx.warn('Workspace closed...', { workspace: workspace.id, wsId, wsName: workspace.workspaceName }) } } - private async sendUpgrade (ctx: MeasureContext, webSocket: ConnectionSocket, binary: boolean): Promise { - await webSocket.send( + private sendUpgrade (ctx: MeasureContext, webSocket: ConnectionSocket, binary: boolean): void { + void webSocket.send( ctx, { result: { @@ -644,9 +665,7 @@ class TSessionManager implements SessionManager { } async closeWorkspaces (ctx: MeasureContext): Promise { - if (this.checkInterval !== undefined) { - clearInterval(this.checkInterval) - } + clearInterval(this.checkInterval) for (const w of this.workspaces) { await this.closeAll(w[0], w[1], 1, 'shutdown') } @@ -666,8 +685,12 @@ class TSessionManager implements SessionManager { try { if (workspace.sessions.size === 0) { const pl = await workspace.pipeline - await Promise.race([pl, timeoutPromise(60000)]) - await Promise.race([pl.close(), timeoutPromise(60000)]) + let to = timeoutPromise(60000) + await Promise.race([pl, to.promise]) + to.cancelHandle() + to = timeoutPromise(60000) + await Promise.race([pl.close(), to]) + to.cancelHandle() if (this.workspaces.get(wsid)?.id === wsUID) { this.workspaces.delete(wsid) @@ -719,30 +742,23 @@ class TSessionManager implements SessionManager { function send (): void { for (const sessionRef of sessions.splice(0, 1)) { if (sessionRef.session.sessionId !== from?.sessionId) { - if (target === undefined) { - void sessionRef.socket.send( - ctx, - resp, - sessionRef.session.binaryResponseMode, - sessionRef.session.useCompression - ) - } else if (target.includes(sessionRef.session.getUser())) { - void sessionRef.socket.send( - ctx, - resp, - sessionRef.session.binaryResponseMode, - sessionRef.session.useCompression - ) + if (target === undefined || target.includes(sessionRef.session.getUser())) { + void sessionRef.socket.send(ctx, resp, sessionRef.session.binaryMode, sessionRef.session.useCompression) } } } if (sessions.length > 0) { - setImmediate(send) + onNextTick(send) } else { ctx.end() } } - send() + if (sessions.length > 0) { + // We need to send broadcast after our client response so put it after all IO + onNextTick(send) + } else { + ctx.end() + } } async handleRequest( @@ -764,7 +780,12 @@ class TSessionManager implements SessionManager { try { const backupMode = 'loadChunk' in service await userCtx.with(`🧭 ${backupMode ? 'handleBackup' : 'handleRequest'}`, {}, async (ctx) => { - const request = await ctx.with('📥 read', {}, async () => readRequest(msg, false)) + const request = readRequest(msg, service.binaryMode) + + if (request.time != null) { + const delta = Date.now() - request.time + userCtx.measure('receive msg', delta) + } if (request.method === 'forceClose') { const wsRef = this.workspaces.get(workspace) let done = false @@ -780,12 +801,12 @@ class TSessionManager implements SessionManager { id: request.id, result: done } - await ws.send(ctx, forceCloseResponse, service.binaryResponseMode, service.useCompression) + await ws.send(ctx, forceCloseResponse, service.binaryMode, service.useCompression) return } if (request.id === -1 && request.method === 'hello') { const hello = request as HelloRequest - service.binaryResponseMode = hello.binary ?? false + service.binaryMode = hello.binary ?? false service.useCompression = hello.compression ?? false service.useBroadcast = hello.broadcast ?? false @@ -793,18 +814,24 @@ class TSessionManager implements SessionManager { ctx.info('hello happen', { workspace, user: service.getUser(), - binary: service.binaryResponseMode, + binary: service.binaryMode, compression: service.useCompression, timeToHello: Date.now() - service.createTime, workspaceUsers: this.workspaces.get(workspace)?.sessions?.size, totalUsers: this.sessions.size }) } + const reconnect = this.reconnectIds.has(service.sessionId) + if (reconnect) { + const reconnectTimeout = this.reconnectIds.get(service.sessionId) + clearTimeout(reconnectTimeout) + this.reconnectIds.delete(service.sessionId) + } const helloResponse: HelloResponse = { id: -1, result: 'hello', - binary: service.binaryResponseMode, - reconnect: this.reconnectIds.has(service.sessionId) + binary: service.binaryMode, + reconnect } await ws.send(ctx, helloResponse, false, false) return @@ -839,14 +866,7 @@ class TSessionManager implements SessionManager { queue: service.requests.size } - await handleSend( - ctx, - ws, - resp, - this.sessions.size < 100 ? 10000 : 1001, - service.binaryResponseMode, - service.useCompression - ) + await handleSend(ctx, ws, resp, 32 * 1024, service.binaryMode, service.useCompression) } catch (err: any) { Analytics.handleError(err) if (LOGGING_ENABLED) { @@ -857,7 +877,7 @@ class TSessionManager implements SessionManager { error: unknownError(err), result: JSON.parse(JSON.stringify(err?.stack)) } - await ws.send(ctx, resp, service.binaryResponseMode, service.useCompression) + await ws.send(ctx, resp, service.binaryMode, service.useCompression) } }) } finally { @@ -884,14 +904,7 @@ class TSessionManager implements SessionManager { try { const resp: Response = { id: request.id, result: request.method === 'measure' ? 'started' : serverTime } - await handleSend( - ctx, - ws, - resp, - this.sessions.size < 100 ? 10000 : 1001, - service.binaryResponseMode, - service.useCompression - ) + await handleSend(ctx, ws, resp, 32 * 1024, service.binaryMode, service.useCompression) } catch (err: any) { Analytics.handleError(err) if (LOGGING_ENABLED) { @@ -902,7 +915,7 @@ class TSessionManager implements SessionManager { error: unknownError(err), result: JSON.parse(JSON.stringify(err?.stack)) } - await ws.send(ctx, resp, service.binaryResponseMode, service.useCompression) + await ws.send(ctx, resp, service.binaryMode, service.useCompression) } } } @@ -916,13 +929,26 @@ async function handleSend ( useCompression: boolean ): Promise { // ws.send(msg) - if (Array.isArray(msg.result) && chunkLimit > 0 && msg.result.length > chunkLimit) { + if (Array.isArray(msg.result) && msg.result.length > 1 && chunkLimit > 0) { // Split and send by chunks const data = [...msg.result] let cid = 1 - while (data.length > 0) { - const chunk = data.splice(0, chunkLimit) + const dataSize = JSON.stringify(data).length + const avg = Math.round(dataSize / data.length) + const itemChunk = Math.round(chunkLimit / avg) + 1 + + while (data.length > 0 && !ws.isClosed) { + let itemChunkCurrent = itemChunk + if (data.length - itemChunk < itemChunk / 2) { + itemChunkCurrent = data.length + } + const chunk: FindResult = toFindResult(data.splice(0, itemChunkCurrent)) + if (data.length === 0) { + const orig = msg.result as FindResult + chunk.total = orig.total ?? 0 + chunk.lookupMap = orig.lookupMap + } if (chunk !== undefined) { await ws.send( ctx, @@ -932,6 +958,10 @@ async function handleSend ( ) } cid++ + + if (data.length > 0 && !ws.isClosed) { + await waitNextTick() + } } } else { await ws.send(ctx, msg, useBinary, useCompression) diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index b523576a1c..a13100bd4a 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -14,7 +14,7 @@ // import { Analytics } from '@hcengineering/analytics' -import { generateId, type MeasureContext } from '@hcengineering/core' +import { generateId, toWorkspaceString, type MeasureContext } from '@hcengineering/core' import { UNAUTHORIZED } from '@hcengineering/platform' import { serialize, type Response } from '@hcengineering/rpc' import { decodeToken, type Token } from '@hcengineering/server-token' @@ -22,16 +22,21 @@ import compression from 'compression' import cors from 'cors' import express from 'express' import http, { type IncomingMessage } from 'http' +import os from 'os' import { WebSocketServer, type RawData, type WebSocket } from 'ws' import { getStatistics, wipeStatistics } from './stats' import { LOGGING_ENABLED, + type AddSessionResponse, type ConnectionSocket, type HandleRequestFunction, type PipelineFactory, type SessionManager } from './types' +import 'bufferutil' +import 'utf-8-validate' + /** * @public * @param sessionFactory - @@ -49,7 +54,13 @@ export function startHttpServer ( accountsUrl: string ): () => Promise { if (LOGGING_ENABLED) { - ctx.info('starting server on', { port, productId, enableCompression, accountsUrl }) + ctx.info('starting server on', { + port, + productId, + enableCompression, + accountsUrl, + parallel: os.availableParallelism() + }) } const app = express() @@ -65,7 +76,8 @@ export function startHttpServer ( // fallback to standard filter function return compression.filter(req, res) }, - level: 6 + level: 1, + memLevel: 9 }) ) @@ -157,16 +169,19 @@ export function startHttpServer ( ? { zlibDeflateOptions: { // See zlib defaults. - chunkSize: 10 * 1024, - memLevel: 7, - level: 3 + chunkSize: 32 * 1024, + memLevel: 9, + level: 1 }, zlibInflateOptions: { - chunkSize: 10 * 1024, - level: 3 + chunkSize: 32 * 1024, + level: 1, + memLevel: 9 }, + serverNoContextTakeover: true, + clientNoContextTakeover: true, // Below options specified as default values. - concurrencyLimit: 20, // Limits zlib concurrency for perf. + concurrencyLimit: Math.max(10, os.availableParallelism()), // Limits zlib concurrency for perf. threshold: 1024 // Size (in bytes) below which messages // should not be compressed if context takeover is disabled. } @@ -181,8 +196,6 @@ export function startHttpServer ( rawToken: string, sessionId?: string ): Promise => { - let buffer: Buffer[] | undefined = [] - const data = { remoteAddress: request.socket.remoteAddress ?? '', userAgent: request.headers['user-agent'] ?? '', @@ -193,36 +206,38 @@ export function startHttpServer ( } const cs: ConnectionSocket = { id: generateId(), + isClosed: false, close: () => { + cs.isClosed = true ws.close() }, data: () => data, send: async (ctx: MeasureContext, msg, binary, compression) => { - if (ws.readyState !== ws.OPEN) { - return + if (ws.readyState !== ws.OPEN && !cs.isClosed) { + return 0 } - const smsg = await ctx.with('📦 serialize', {}, async () => serialize(msg, binary)) + const smsg = serialize(msg, binary) ctx.measure('send-data', smsg.length) - await ctx.with('📤 socket-send', {}, async (ctx) => { - await new Promise((resolve, reject) => { - ws.send(smsg, { binary, compress: compression }, (err) => { - if (err != null) { - reject(err) - } else { - resolve() - } - }) + while (ws.bufferedAmount > 128 && ws.readyState === ws.OPEN) { + await new Promise((resolve) => { + setImmediate(resolve) + }) + } + await new Promise((resolve, reject) => { + ws.send(smsg, { binary: true, compress: compression }, (err) => { + if (err != null) { + reject(err) + } + resolve() }) }) + return smsg.length } } - ws.on('message', (msg: Buffer) => { - buffer?.push(msg) - }) - const session = await sessions.addSession( + let session: AddSessionResponse | Promise = sessions.addSession( ctx, cs, token, @@ -232,25 +247,41 @@ export function startHttpServer ( sessionId, accountsUrl ) - if ('upgrade' in session || 'error' in session) { - if ('error' in session) { - ctx.error('error', { error: session.error?.message, stack: session.error?.stack }) + + void session.then((s) => { + if ('upgrade' in s || 'error' in s) { + if ('error' in s) { + ctx.error('error', { error: s.error?.message, stack: s.error?.stack }) + } + void cs + .send(ctx, { id: -1, result: { state: 'upgrading', stats: (s as any).upgradeInfo } }, false, false) + .then(() => { + cs.close() + }) } - await cs.send(ctx, { id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } }, false, false) - cs.close() - return - } + }) // eslint-disable-next-line @typescript-eslint/no-misused-promises ws.on('message', (msg: RawData) => { try { let buff: any | undefined if (msg instanceof Buffer) { - buff = msg?.toString() + buff = msg } else if (Array.isArray(msg)) { - buff = Buffer.concat(msg).toString() + buff = Buffer.concat(msg) } if (buff !== undefined) { - void handleRequest(session.context, session.session, cs, buff, session.workspaceId) + if (session instanceof Promise) { + void session.then((_session) => { + session = _session + if ('session' in _session) { + void handleRequest(_session.context, _session.session, cs, buff, _session.workspaceId) + } + }) + } else { + if ('session' in session) { + void handleRequest(session.context, session.session, cs, buff, session.workspaceId) + } + } } } catch (err: any) { Analytics.handleError(err) @@ -260,18 +291,30 @@ export function startHttpServer ( } }) // eslint-disable-next-line @typescript-eslint/no-misused-promises - ws.on('close', (code: number, reason: Buffer) => { - if (session.session.workspaceClosed ?? false) { - return + ws.on('close', async (code: number, reason: Buffer) => { + if (session instanceof Promise) { + session = await session + } + if ('session' in session) { + if (!(session.session.workspaceClosed ?? false)) { + // remove session after 1seconds, give a time to reconnect. + void sessions.close(cs, toWorkspaceString(token.workspace)) + } + } + }) + + ws.on('error', (err) => { + if (session instanceof Promise) { + void session.then((s) => { + if ('session' in session) { + console.error(session.session.getUser(), 'error', err) + } + }) + } + if ('session' in session) { + console.error(session.session.getUser(), 'error', err) } - // remove session after 1seconds, give a time to reconnect. - void sessions.close(cs, token.workspace) }) - const b = buffer - buffer = undefined - for (const msg of b) { - await handleRequest(session.context, session.session, cs, msg, session.workspaceId) - } } wss.on('connection', handleConnection as any) @@ -290,7 +333,10 @@ export function startHttpServer ( throw new Error('Invalid workspace product') } - wss.handleUpgrade(request, socket, head, (ws) => wss.emit('connection', ws, request, payload, token, sessionId)) + wss.handleUpgrade(request, socket, head, (ws) => { + void handleConnection(ws, request, payload, token, sessionId ?? undefined) + // wss.emit('connection', ws, request, payload, token, sessionId) + }) } catch (err: any) { Analytics.handleError(err) if (LOGGING_ENABLED) { @@ -320,7 +366,22 @@ export function startHttpServer ( httpServer.listen(port) return async () => { - httpServer.close() await sessions.closeWorkspaces(ctx) + await new Promise((resolve, reject) => { + wss.close((err) => { + if (err != null) { + reject(err) + } + resolve() + }) + }) + await new Promise((resolve, reject) => + httpServer.close((err) => { + if (err != null) { + reject(err) + } + resolve() + }) + ) } } diff --git a/server/ws/src/types.ts b/server/ws/src/types.ts index 31c5c071f8..75380fbe04 100644 --- a/server/ws/src/types.ts +++ b/server/ws/src/types.ts @@ -54,7 +54,7 @@ export interface Session { requests: Map - binaryResponseMode: boolean + binaryMode: boolean useCompression: boolean useBroadcast: boolean @@ -96,8 +96,9 @@ export type PipelineFactory = ( */ export interface ConnectionSocket { id: string + isClosed: boolean close: () => void - send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => Promise + send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => Promise data: () => Record } @@ -131,6 +132,11 @@ export interface Workspace { workspaceName: string } +export type AddSessionResponse = + | { session: Session, context: MeasureContext, workspaceId: string } + | { upgrade: true } + | { error: any } + /** * @public */ @@ -149,11 +155,11 @@ export interface SessionManager { productId: string, sessionId: string | undefined, accountsUrl: string - ) => Promise<{ session: Session, context: MeasureContext, workspaceId: string } | { upgrade: true } | { error: any }> + ) => Promise broadcastAll: (workspace: Workspace, tx: Tx[], targets?: string[]) => void - close: (ws: ConnectionSocket, workspaceId: WorkspaceId) => Promise + close: (ws: ConnectionSocket, workspaceId: string) => Promise closeAll: ( wsId: string,