From 806400f5f20112832bd33c7b2f6fb60d3c73588f Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 22 Dec 2021 16:02:51 +0700 Subject: [PATCH] Performance metrics (#619) Signed-off-by: Andrey Sobolev --- .vscode/launch.json | 24 +- common/config/rush/command-line.json | 8 + common/config/rush/pnpm-lock.yaml | 388 ++++++++++++++++++++- dev/client-resources/src/connection.ts | 11 +- dev/docker-compose.yaml | 46 +++ dev/generator/src/index.ts | 4 +- dev/generator/src/kanban.ts | 18 +- dev/generator/src/recruit.ts | 326 ++++++++++++----- dev/generator/src/utils.ts | 6 +- dev/server/src/server.ts | 4 +- dev/tool/src/elastic.ts | 27 +- dev/tool/src/workspace.ts | 4 +- packages/core/src/index.ts | 1 + packages/core/src/measurements/context.ts | 58 +++ packages/core/src/measurements/index.ts | 3 + packages/core/src/measurements/metrics.ts | 156 +++++++++ packages/core/src/measurements/types.ts | 45 +++ packages/core/src/server.ts | 4 +- plugins/devmodel-resources/src/index.ts | 14 +- server/core/src/fulltext.ts | 97 ++++-- server/core/src/storage.ts | 125 +++++-- server/core/src/types.ts | 4 +- server/elastic/src/adapter.ts | 21 +- server/mongo/src/__tests__/storage.test.ts | 16 +- server/mongo/src/storage.ts | 30 +- server/server/package.json | 3 +- server/server/src/__start.ts | 2 +- server/server/src/apm.ts | 85 +++++ server/server/src/metrics.ts | 46 +++ server/server/src/server.ts | 3 +- server/ws/src/__tests__/server.test.ts | 8 +- server/ws/src/server.ts | 42 ++- 32 files changed, 1372 insertions(+), 257 deletions(-) create mode 100644 packages/core/src/measurements/context.ts create mode 100644 packages/core/src/measurements/index.ts create mode 100644 packages/core/src/measurements/metrics.ts create mode 100644 packages/core/src/measurements/types.ts create mode 100644 server/server/src/apm.ts create mode 100644 server/server/src/metrics.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 099c1f0584..63d193057f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,9 @@ "args": ["src/__start.ts"], "env": { "ELASTIC_URL": "http://localhost:9200", - "MONGO_URL": "mongodb://localhost:27017" + "MONGO_URL": "mongodb://localhost:27017", + "APM_SERVER_URL2": "http://localhost:8200", + "METRICS_CONSOLE": "true" // Show metrics in console evert 30 seconds. }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "sourceMaps": true, @@ -45,5 +47,25 @@ "cwd": "${workspaceRoot}/dev/generator", "protocol": "inspector" }, + { + "name": "Debug tool", + "type": "node", + "request": "launch", + // "args": ["src/index.ts", "import-xml", "ws1", "/Users/haiodo/Develop/private/hardware/suho/Кандидаты/Кандидаты.xml"], + "args": ["src/index.ts", "restore-workspace", "ws1", "../../temp/ws1/"], + "env": { + "MINIO_ACCESS_KEY":"minioadmin", + "MINIO_SECRET_KEY":"minioadmin", + "MINIO_ENDPOINT":"localhost", + "MONGO_URL":"mongodb://localhost:27017", + "TRANSACTOR_URL":"ws:/localhost:3333", + "TELEGRAM_DATABASE":"telegram-service", + "ELASTIC_URL":"http://localhost:9200", + }, + "runtimeArgs": ["--nolazy", "-r", "ts-node/register" ], + "sourceMaps": true, + "cwd": "${workspaceRoot}/dev/tool", + "protocol": "inspector" + }, ] } diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index 991f08420d..da0d174a07 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -236,6 +236,14 @@ "safeForSimultaneousRushProcesses": true, "shellCommand": "node templates/apply.js" }, + { + "commandKind": "global", + "name": "ts-clean", + "summary": "Clean tsconfig.tsbuildinfo", + "description": "Clean typescript incremental cache", + "safeForSimultaneousRushProcesses": true, + "shellCommand": "find .|grep tsconfig.tsbuildinfo | xargs rm | pwd" + }, ], /** diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 4d6b5bf717..b5f13d963c 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -125,6 +125,7 @@ specifiers: css-loader: ^5.2.1 deep-equal: ^2.0.5 dotenv-webpack: ^7.0.2 + elastic-apm-node: ~3.26.0 eslint: ^7.32.0 eslint-plugin-import: ^2.25.3 eslint-plugin-node: ^11.1.0 @@ -293,6 +294,7 @@ dependencies: css-loader: 5.2.7_webpack@5.65.0 deep-equal: 2.0.5 dotenv-webpack: 7.0.3_webpack@5.65.0 + elastic-apm-node: 3.26.0 eslint: 7.32.0 eslint-plugin-import: 2.25.3_eslint@7.32.0 eslint-plugin-node: 11.1.0_eslint@7.32.0 @@ -673,6 +675,20 @@ packages: engines: {node: '>=10.0.0'} dev: false + /@elastic/ecs-helpers/1.1.0: + resolution: {integrity: sha512-MDLb2aFeGjg46O5mLpdCzT5yOUDnXToJSrco2ShqGIXxNJaM8uJjX+4nd+hRYV4Vex8YJyDtOFEVBldQct6ndg==} + engines: {node: '>=10'} + dependencies: + fast-json-stringify: 2.7.12 + dev: false + + /@elastic/ecs-pino-format/1.3.0: + resolution: {integrity: sha512-U8D57gPECYoRCcwREsrXKBtqeyFFF/KAwHi4rG1u/oQhAg91Kzw8ZtUQJXD/DMDieLOqtbItFr2FRBWI3t3wog==} + engines: {node: '>=10'} + dependencies: + '@elastic/ecs-helpers': 1.1.0 + dev: false + /@elastic/elasticsearch/7.16.0: resolution: {integrity: sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==} engines: {node: '>=12'} @@ -2323,6 +2339,10 @@ packages: hasBin: true dev: false + /after-all-results/2.0.0: + resolution: {integrity: sha1-asL8ICtQD4jaj09VMM+hAPTGotA=} + dev: false + /ajv-errors/1.0.1_ajv@6.12.6: resolution: {integrity: sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==} peerDependencies: @@ -2584,6 +2604,12 @@ packages: engines: {node: '>=8'} dev: false + /async-cache/1.1.0: + resolution: {integrity: sha1-SppaidBl7F2OUlS9nulrp2xTK1o=} + dependencies: + lru-cache: 4.1.5 + dev: false + /async-each/1.0.3: resolution: {integrity: sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==} dev: false @@ -2592,6 +2618,16 @@ packages: resolution: {integrity: sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==} dev: false + /async-value-promise/1.1.1: + resolution: {integrity: sha512-c2RFDKjJle1rHa0YxN9Ysu97/QBu3Wa+NOejJxsX+1qVDJrkD3JL/GN1B3gaILAEXJXbu/4Z1lcoCHFESe/APA==} + dependencies: + async-value: 1.2.2 + dev: false + + /async-value/1.2.2: + resolution: {integrity: sha1-hFF6Hny2saW14YH6Mb4QQ3t/sSU=} + dev: false + /async/2.6.3: resolution: {integrity: sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==} dependencies: @@ -2612,6 +2648,11 @@ packages: hasBin: true dev: false + /atomic-sleep/1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + dev: false + /autoprefixer/10.4.0_postcss@8.4.5: resolution: {integrity: sha512-7FdJ1ONtwzV1G43GDD0kpVMn/qbiNqyOPMFTX5nRffI+7vgWoFEc6DcXOxHJxrWNDXrZh18eDsZjvZGUljSRGA==} engines: {node: ^10 || ^12 || >=14} @@ -2745,6 +2786,13 @@ packages: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} dev: false + /basic-auth/2.0.1: + resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} + engines: {node: '>= 0.8'} + dependencies: + safe-buffer: 5.1.2 + dev: false + /batch/0.6.1: resolution: {integrity: sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=} dev: false @@ -2769,6 +2817,10 @@ packages: engines: {node: '>=8'} dev: false + /binary-search/1.3.6: + resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==} + dev: false + /bindings/1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} requiresBuild: true @@ -2852,6 +2904,12 @@ packages: fill-range: 7.0.1 dev: false + /breadth-filter/2.0.0: + resolution: {integrity: sha512-thQShDXnFWSk2oVBixRCyrWsFoV5tfOpWKHmxwafHQDNxCfDBk539utpvytNjmlFrTMqz41poLwJvA1MW3z0MQ==} + dependencies: + object.entries: 1.1.5 + dev: false + /brfs/2.0.2: resolution: {integrity: sha512-IrFjVtwu4eTJZyu8w/V2gxU7iLTtcHih67sgEdzrhjLBMHp2uYefUBfdM4k2UvcuWMgV7PQDZHSLeNWnLFKWVQ==} hasBin: true @@ -3329,6 +3387,14 @@ packages: engines: {node: '>=0.8'} dev: false + /console-log-level/1.4.1: + resolution: {integrity: sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==} + dev: false + + /container-info/1.1.0: + resolution: {integrity: sha512-eD2zLAmxGS2kmL4f1jY8BdOqnmpL6X70kvzTBW/9FIQnxoxiBJ4htMsTmtPLPWRs7NHYFvqKQ1VtppV08mdsQA==} + dev: false + /content-disposition/0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -3908,6 +3974,61 @@ packages: resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=} dev: false + /elastic-apm-http-client/10.3.0: + resolution: {integrity: sha512-BAqB7k5JA/x09L8BVj04WRoknRptmW2rLAoHQVrPvPhUm/IgNz63wPfiBuhWVE//Hl7xEpURO5pMV6az0UArkA==} + engines: {node: ^8.6.0 || 10 || >=12} + dependencies: + breadth-filter: 2.0.0 + container-info: 1.1.0 + end-of-stream: 1.4.4 + fast-safe-stringify: 2.1.1 + fast-stream-to-buffer: 1.0.0 + object-filter-sequence: 1.0.0 + readable-stream: 3.6.0 + stream-chopper: 3.0.1 + dev: false + + /elastic-apm-node/3.26.0: + resolution: {integrity: sha512-MwYFlBTlcHI8GGQXLnnEm70JJ4RRFkHCY1D3Wt2027l8T/Ye5tgssMSiKyRbjb9bVdibbte73Xn8HF8i35UaxA==} + engines: {node: ^8.6.0 || 10 || 12 || 14 || 15 || 16 || 17} + dependencies: + '@elastic/ecs-pino-format': 1.3.0 + after-all-results: 2.0.0 + async-cache: 1.1.0 + async-value-promise: 1.1.1 + basic-auth: 2.0.1 + cookie: 0.4.1 + core-util-is: 1.0.3 + elastic-apm-http-client: 10.3.0 + end-of-stream: 1.4.4 + error-callsites: 2.0.4 + error-stack-parser: 2.0.6 + escape-string-regexp: 4.0.0 + fast-safe-stringify: 2.1.1 + http-headers: 3.0.2 + is-native: 1.0.1 + load-source-map: 2.0.0 + lru-cache: 6.0.0 + measured-reporting: 1.51.1 + monitor-event-loop-delay: 1.0.0 + object-filter-sequence: 1.0.0 + object-identity-map: 1.0.2 + original-url: 1.2.3 + pino: 6.13.3 + read-pkg-up: 7.0.1 + relative-microtime: 2.0.0 + require-in-the-middle: 5.1.0 + semver: 6.3.0 + set-cookie-serde: 1.0.0 + shallow-clone-shim: 2.0.0 + sql-summary: 1.0.1 + traceparent: 1.0.0 + traverse: 0.6.6 + unicode-byte-truncate: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: false + /electron-to-chromium/1.4.24: resolution: {integrity: sha512-erwx5r69B/WFfFuF2jcNN0817BfDBdC4765kQ6WltOMuwsimlQo3JTEq0Cle+wpHralwdeX3OfAtw/mHxPK0Wg==} dev: false @@ -3980,12 +4101,23 @@ packages: prr: 1.0.1 dev: false + /error-callsites/2.0.4: + resolution: {integrity: sha512-V877Ch4FC4FN178fDK1fsrHN4I1YQIBdtjKrHh3BUHMnh3SMvwUVrqkaOgDpUuevgSNna0RBq6Ox9SGlxYrigA==} + engines: {node: '>=6.x'} + dev: false + /error-ex/1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: is-arrayish: 0.2.1 dev: false + /error-stack-parser/2.0.6: + resolution: {integrity: sha512-d51brTeqC+BHlwF0BhPtcYgF5nlzf9ZZ0ZIUQNZpc9ZB9qw5IJ2diTrBY9jlCJkTLITYPjmiX6OWCwH+fuyNgQ==} + dependencies: + stackframe: 1.2.0 + dev: false + /es-abstract/1.19.1: resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==} engines: {node: '>= 0.4'} @@ -4684,10 +4816,35 @@ packages: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} dev: false + /fast-json-stringify/2.7.12: + resolution: {integrity: sha512-4hjwZDPmgj/ZUKXhEWovGPciE/5mWtAIQQxN+2VBDFun7DRTk2oOItbu9ZZp6kqj+eZ/u7z+dgBgM74cfGRnBQ==} + engines: {node: '>= 10.0.0'} + dependencies: + ajv: 6.12.6 + deepmerge: 4.2.2 + rfdc: 1.3.0 + string-similarity: 4.0.4 + dev: false + /fast-levenshtein/2.0.6: resolution: {integrity: sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=} dev: false + /fast-redact/3.0.2: + resolution: {integrity: sha512-YN+CYfCVRVMUZOUPeinHNKgytM1wPI/C/UCLEi56EsY2dwwvI00kIJHJoI7pMVqGoMew8SMZ2SSfHKHULHXDsg==} + engines: {node: '>=6'} + dev: false + + /fast-safe-stringify/2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + dev: false + + /fast-stream-to-buffer/1.0.0: + resolution: {integrity: sha512-bI/544WUQlD2iXBibQbOMSmG07Hay7YrpXlKaeGTPT7H7pC0eitt3usak5vUwEvCGK/O7rUAM3iyQValGU22TQ==} + dependencies: + end-of-stream: 1.4.4 + dev: false + /fast-xml-parser/3.21.1: resolution: {integrity: sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg==} hasBin: true @@ -4699,6 +4856,10 @@ packages: resolution: {integrity: sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow==} dev: false + /fastify-warning/0.2.0: + resolution: {integrity: sha512-s1EQguBw/9qtc1p/WTY4eq9WMRIACkj+HTcOIK1in4MV5aFaQC9ZCIt0dJ7pr5bIf4lPpHvAtP2ywpTNgs7hqw==} + dev: false + /fastq/1.13.0: resolution: {integrity: sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==} dependencies: @@ -4807,6 +4968,10 @@ packages: rimraf: 3.0.2 dev: false + /flatstr/1.0.12: + resolution: {integrity: sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==} + dev: false + /flatted/3.2.4: resolution: {integrity: sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==} dev: false @@ -4861,6 +5026,10 @@ packages: mime-types: 2.1.34 dev: false + /forwarded-parse/2.1.2: + resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} + dev: false + /forwarded/0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -5268,6 +5437,12 @@ packages: toidentifier: 1.0.1 dev: false + /http-headers/3.0.2: + resolution: {integrity: sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==} + dependencies: + next-line: 1.1.0 + dev: false + /http-parser-js/0.5.5: resolution: {integrity: sha512-x+JVEkO2PoM8qqpbPbOL3cqHPwerep7OwzK7Ay+sMQjKzaKCqWvjoXm5tqMP9tXWWTnTzAjIhXg+J99XYuPhPA==} dev: false @@ -5612,6 +5787,11 @@ packages: engines: {node: '>=0.10.0'} dev: false + /is-finite/1.1.0: + resolution: {integrity: sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==} + engines: {node: '>=0.10.0'} + dev: false + /is-fullwidth-code-point/2.0.0: resolution: {integrity: sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=} engines: {node: '>=4'} @@ -5648,15 +5828,32 @@ packages: is-extglob: 2.1.1 dev: false + /is-integer/1.0.7: + resolution: {integrity: sha1-a96Bqs3feLZZtmKdYpytxRqIbVw=} + dependencies: + is-finite: 1.1.0 + dev: false + /is-map/2.0.2: resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==} dev: false + /is-native/1.0.1: + resolution: {integrity: sha1-zRjMFi6EUNaDtbq+eayZwUVElnU=} + dependencies: + is-nil: 1.0.1 + to-source-code: 1.0.2 + dev: false + /is-negative-zero/2.0.2: resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} engines: {node: '>= 0.4'} dev: false + /is-nil/1.0.1: + resolution: {integrity: sha1-LauingtYUGOHXntTnQcfWxWTeWk=} + dev: false + /is-number-object/1.0.6: resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==} engines: {node: '>= 0.4'} @@ -6619,6 +6816,13 @@ packages: resolution: {integrity: sha512-JWw1HHMx54g8mEsG7JwI8I/xh7qeJbF6L9u1dQOYW91RdRqDYpnTn1UaNXYkmLD967Vk0xGuyHzuRnkSApby3w==} dev: false + /load-source-map/2.0.0: + resolution: {integrity: sha512-QNZzJ2wMrTmCdeobMuMNEXHN1QGk8HG6louEkzD/zwQ7EU2RarrzlhQ4GnUYEFzLhK+Jq7IGyF/qy+XYBSO7AQ==} + engines: {node: '>= 8'} + dependencies: + source-map: 0.7.3 + dev: false + /loader-runner/4.2.0: resolution: {integrity: sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==} engines: {node: '>=6.11.5'} @@ -6700,6 +6904,13 @@ packages: '@sinonjs/commons': 1.8.3 dev: false + /lru-cache/4.1.5: + resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} + dependencies: + pseudomap: 1.0.2 + yallist: 2.1.2 + dev: false + /lru-cache/6.0.0: resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} engines: {node: '>=10'} @@ -6748,6 +6959,10 @@ packages: object-visit: 1.0.1 dev: false + /mapcap/1.0.0: + resolution: {integrity: sha512-KcNlZSlFPx+r1jYZmxEbTVymG+dIctf10WmWkuhrhrblM+KMoF77HelwihL5cxYlORye79KoR4IlOOk99lUJ0g==} + dev: false + /md5.js/1.3.5: resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==} dependencies: @@ -6760,6 +6975,24 @@ packages: resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} dev: false + /measured-core/1.51.1: + resolution: {integrity: sha512-DZQP9SEwdqqYRvT2slMK81D/7xwdxXosZZBtLVfPSo6y5P672FBTbzHVdN4IQyUkUpcVOR9pIvtUy5Ryl7NKyg==} + engines: {node: '>= 5.12'} + dependencies: + binary-search: 1.3.6 + optional-js: 2.3.0 + dev: false + + /measured-reporting/1.51.1: + resolution: {integrity: sha512-JCt+2u6XT1I5lG3SuYqywE0e62DJuAzBcfMzWGUhIYtPQV2Vm4HiYt/durqmzsAbZV181CEs+o/jMKWJKkYIWw==} + engines: {node: '>= 5.12'} + dependencies: + console-log-level: 1.4.1 + mapcap: 1.0.0 + measured-core: 1.51.1 + optional-js: 2.3.0 + dev: false + /media-typer/0.3.0: resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=} engines: {node: '>= 0.6'} @@ -6935,6 +7168,10 @@ packages: minimist: 1.2.5 dev: false + /module-details-from-path/1.0.3: + resolution: {integrity: sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=} + dev: false + /mongodb-connection-string-url/2.3.2: resolution: {integrity: sha512-2LkmS0ny7LamAyhEs2Q+zuFFxeGNSc2DaGHBevjqkoPt7bgh+67mg1sFU6awnMsdLKpdEt7zUy466K9x7RsYcQ==} dependencies: @@ -6953,6 +7190,10 @@ packages: saslprep: 1.0.3 dev: false + /monitor-event-loop-delay/1.0.0: + resolution: {integrity: sha512-YRIr1exCIfBDLZle8WHOfSo7Xg3M+phcZfq9Fx1L6Abo+atGp7cge5pM7PjyBn4s1oZI/BRD4EMrzQBbPpVb5Q==} + dev: false + /mri/1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -7029,6 +7270,10 @@ packages: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} dev: false + /next-line/1.1.0: + resolution: {integrity: sha1-/K5XhTBStqm66CCOQN19PC0wRgM=} + dev: false + /next-tick/1.0.0: resolution: {integrity: sha1-yobR/ogoFpsBICCOPchCS524NCw=} dev: false @@ -7137,6 +7382,16 @@ packages: kind-of: 3.2.2 dev: false + /object-filter-sequence/1.0.0: + resolution: {integrity: sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==} + dev: false + + /object-identity-map/1.0.2: + resolution: {integrity: sha512-a2XZDGyYTngvGS67kWnqVdpoaJWsY7C1GhPJvejWAFCsUioTAaiTu8oBad7c6cI4McZxr4CmvnZeycK05iav5A==} + dependencies: + object.entries: 1.1.5 + dev: false + /object-inspect/1.12.0: resolution: {integrity: sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==} dev: false @@ -7171,6 +7426,15 @@ packages: object-keys: 1.1.1 dev: false + /object.entries/1.1.5: + resolution: {integrity: sha512-TyxmjUoZggd4OrrU1W66FMDG6CuqJxsFvymeyXI51+vQLN67zYfZseptRge703kKQdo4uccgAKebXFcRCzk4+g==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + es-abstract: 1.19.1 + dev: false + /object.pick/1.3.0: resolution: {integrity: sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=} engines: {node: '>=0.10.0'} @@ -7232,6 +7496,10 @@ packages: is-wsl: 1.1.0 dev: false + /optional-js/2.3.0: + resolution: {integrity: sha512-B0LLi+Vg+eko++0z/b8zIv57kp7HKEzaPJo7LowJXMUKYdf+3XJGu/cw03h/JhIOsLnP+cG5QnTHAuicjA5fMw==} + dev: false + /optionator/0.8.3: resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==} engines: {node: '>= 0.8.0'} @@ -7260,6 +7528,12 @@ packages: resolution: {integrity: sha512-3Ux8um0zXbVacKUkcytc0u3HgC0b0bBLT+I60r2J/En72cI0nZffqrA7Xtf2Hqs27j1g82llR5Mhbd0Z1XW4AQ==} dev: false + /original-url/1.2.3: + resolution: {integrity: sha512-BYm+pKYLtS4mVe/mgT3YKGtWV5HzN/XKiaIu1aK4rsxyjuHeTW9N+xVBEpJcY1onB3nccfH0RbzUEoimMqFUHQ==} + dependencies: + forwarded-parse: 2.1.2 + dev: false + /original/1.0.2: resolution: {integrity: sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg==} dependencies: @@ -7488,6 +7762,23 @@ packages: engines: {node: '>=0.10.0'} dev: false + /pino-std-serializers/3.2.0: + resolution: {integrity: sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==} + dev: false + + /pino/6.13.3: + resolution: {integrity: sha512-tJy6qVgkh9MwNgqX1/oYi3ehfl2Y9H0uHyEEMsBe74KinESIjdMrMQDWpcZPpPicg3VV35d/GLQZmo4QgU2Xkg==} + hasBin: true + dependencies: + fast-redact: 3.0.2 + fast-safe-stringify: 2.1.1 + fastify-warning: 0.2.0 + flatstr: 1.0.12 + pino-std-serializers: 3.2.0 + quick-format-unescaped: 4.0.4 + sonic-boom: 1.4.1 + dev: false + /pirates/4.0.4: resolution: {integrity: sha512-ZIrVPH+A52Dw84R0L3/VS9Op04PuQ2SEoJL6bkshmiTic/HldyW9Tf7oH5mhJZBK7NmDx27vSMrYEXPXclpDKw==} engines: {node: '>= 6'} @@ -7782,6 +8073,10 @@ packages: resolution: {integrity: sha1-0/wRS6BplaRexok/SEzrHXj19HY=} dev: false + /pseudomap/1.0.2: + resolution: {integrity: sha1-8FKijacOYYkX7wqKw0wa5aaChrM=} + dev: false + /psl/1.8.0: resolution: {integrity: sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==} dev: false @@ -7844,6 +8139,10 @@ packages: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} dev: false + /quick-format-unescaped/4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + dev: false + /quote-stream/1.0.2: resolution: {integrity: sha1-hJY/jJwmuULhU/7rU6rnRlK34LI=} hasBin: true @@ -7853,6 +8152,10 @@ packages: through2: 2.0.5 dev: false + /random-poly-fill/1.0.1: + resolution: {integrity: sha512-bMOL0hLfrNs52+EHtIPIXxn2PxYwXb0qjnKruTjXiM/sKfYqj506aB2plFwWW1HN+ri724bAVVGparh4AtlJKw==} + dev: false + /randombytes/2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: @@ -7989,6 +8292,10 @@ packages: engines: {node: '>=8'} dev: false + /relative-microtime/2.0.0: + resolution: {integrity: sha512-l18ha6HEZc+No/uK4GyAnNxgKW7nvEe35IaeN54sShMojtqik2a6GbTyuiezkjpPaqP874Z3lW5ysBo5irz4NA==} + dev: false + /remove-trailing-separator/1.1.0: resolution: {integrity: sha1-wkvOKig62tW8P1jg1IJJuSN52O8=} dev: false @@ -8063,6 +8370,16 @@ packages: engines: {node: '>=0.10.0'} dev: false + /require-in-the-middle/5.1.0: + resolution: {integrity: sha512-M2rLKVupQfJ5lf9OvqFGIT+9iVLnTmjgbOmpil12hiSQNn5zJTKGPoIisETNjfK+09vP3rpm1zJajmErpr2sEQ==} + dependencies: + debug: 4.3.3 + module-details-from-path: 1.0.3 + resolve: 1.20.0 + transitivePeerDependencies: + - supports-color + dev: false + /require-main-filename/2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: false @@ -8150,6 +8467,10 @@ packages: engines: {iojs: '>=1.0.0', node: '>=0.10.0'} dev: false + /rfdc/1.3.0: + resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} + dev: false + /rimraf/2.7.1: resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} hasBin: true @@ -8413,6 +8734,10 @@ packages: resolution: {integrity: sha1-BF+XgtARrppoA93TgrJDkrPYkPc=} dev: false + /set-cookie-serde/1.0.0: + resolution: {integrity: sha512-Vq8e5GsupfJ7okHIvEPcfs5neCo7MZ1ZuWrO3sllYi3DOWt6bSSCpADzqXjz3k0fXehnoFIrmmhty9IN6U6BXQ==} + dev: false + /set-value/2.0.1: resolution: {integrity: sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==} engines: {node: '>=0.10.0'} @@ -8439,6 +8764,10 @@ packages: safe-buffer: 5.2.1 dev: false + /shallow-clone-shim/2.0.0: + resolution: {integrity: sha512-YRNymdiL3KGOoS67d73TEmk4tdPTO9GSMCoiphQsTcC9EtC+AOmMPjkyBkRoCJfW9ASsaZw1craaiw1dPN2D3Q==} + dev: false + /shallow-clone/3.0.1: resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==} engines: {node: '>=8'} @@ -8571,6 +8900,13 @@ packages: websocket-driver: 0.7.4 dev: false + /sonic-boom/1.4.1: + resolution: {integrity: sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==} + dependencies: + atomic-sleep: 1.0.0 + flatstr: 1.0.12 + dev: false + /sorcery/0.10.0: resolution: {integrity: sha1-iukK19fLBfxZ8asMY3hF1cFaUrc=} hasBin: true @@ -8701,6 +9037,10 @@ packages: resolution: {integrity: sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=} dev: false + /sql-summary/1.0.1: + resolution: {integrity: sha512-IpCr2tpnNkP3Jera4ncexsZUp0enJBLr+pHCyTweMUBrbJsTgQeLWx1FXLhoBj/MvcnUQpkgOn2EY8FKOkUzww==} + dev: false + /sshpk/1.16.1: resolution: {integrity: sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==} engines: {node: '>=0.10.0'} @@ -8728,6 +9068,10 @@ packages: escape-string-regexp: 2.0.0 dev: false + /stackframe/1.2.0: + resolution: {integrity: sha512-GrdeshiRmS1YLMYgzF16olf2jJ/IzxXY9lhKOskuVziubpTYcYqyOwYeJKzQkwy7uN0fYSsbsC4RQaXf9LCrYA==} + dev: false + /static-eval/2.1.0: resolution: {integrity: sha512-agtxZ/kWSsCkI5E4QifRwsaPs0P0JmZV6dkLz6ILYfFYQGn+5plctanRN+IC8dJRiFkyXHrwEE3W9Wmx67uDbw==} dependencies: @@ -8771,6 +9115,12 @@ packages: engines: {node: '>=0.10.0'} dev: false + /stream-chopper/3.0.1: + resolution: {integrity: sha512-f7h+ly8baAE26iIjcp3VbnBkbIRGtrvV0X0xxFM/d7fwLTYnLzDPTXRKNxa2HZzohOrc96NTrR+FaV3mzOelNA==} + dependencies: + readable-stream: 3.6.0 + dev: false + /streamsearch/0.1.2: resolution: {integrity: sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=} engines: {node: '>=0.8.0'} @@ -8789,6 +9139,10 @@ packages: strip-ansi: 5.2.0 dev: false + /string-similarity/4.0.4: + resolution: {integrity: sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==} + dev: false + /string-width/3.1.0: resolution: {integrity: sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==} engines: {node: '>=6'} @@ -9311,6 +9665,12 @@ packages: safe-regex: 1.1.0 dev: false + /to-source-code/1.0.2: + resolution: {integrity: sha1-3RNr2x4dvYC76s8IiZJnjpBwv+o=} + dependencies: + is-nil: 1.0.1 + dev: false + /toidentifier/1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -9355,6 +9715,16 @@ packages: punycode: 2.1.1 dev: false + /traceparent/1.0.0: + resolution: {integrity: sha512-b/hAbgx57pANQ6cg2eBguY3oxD6FGVLI1CC2qoi01RmHR7AYpQHPXTig9FkzbWohEsVuHENZHP09aXuw3/LM+w==} + dependencies: + random-poly-fill: 1.0.1 + dev: false + + /traverse/0.6.6: + resolution: {integrity: sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=} + dev: false + /true-case-path/2.2.1: resolution: {integrity: sha512-0z3j8R7MCjy10kc/g+qg7Ln3alJTodw9aDuVWZa3uiWqfuBMKeAeP2ocWcxoyM3D73yz3Jt/Pu4qPr4wHSdB/Q==} dev: false @@ -9555,6 +9925,13 @@ packages: which-boxed-primitive: 1.0.2 dev: false + /unicode-byte-truncate/1.0.0: + resolution: {integrity: sha1-qm8PNHUZP+IMMgrJIT425i6HZKc=} + dependencies: + is-integer: 1.0.7 + unicode-substring: 0.1.0 + dev: false + /unicode-properties/1.3.1: resolution: {integrity: sha512-nIV3Tf3LcUEZttY/2g4ZJtGXhWwSkuLL+rCu0DIAMbjyVPj+8j5gNVz4T/sVbnQybIsd5SFGkPKg/756OY6jlA==} dependencies: @@ -9562,6 +9939,10 @@ packages: unicode-trie: 2.0.0 dev: false + /unicode-substring/0.1.0: + resolution: {integrity: sha1-YSDOPDkDhdvND2DDK5BlxBgdSzY=} + dev: false + /unicode-trie/0.3.1: resolution: {integrity: sha1-1nHd3YkQGgi6w3tqUWEBBgIFIIU=} dependencies: @@ -10152,6 +10533,10 @@ packages: resolution: {integrity: sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==} dev: false + /yallist/2.1.2: + resolution: {integrity: sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=} + dev: false + /yallist/4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} dev: false @@ -11823,7 +12208,7 @@ packages: dev: false file:projects/server.tgz: - resolution: {integrity: sha512-UBFOAisptjpTNIZoU6hWJVBKTk7mftZU0Dz+pJiVhMBJSD+iW393yv4v9GedEpor5rIYDGGcps7IVCXZlr76tw==, tarball: file:projects/server.tgz} + resolution: {integrity: sha512-KX5c6GAIbdrpb9vSiwP/vGovTeHgbcjvlpKZUCmN7/iqGz3S5vSxySnO8wsqJm8fEurl8CAzmcKp73LvHPmSzQ==, tarball: file:projects/server.tgz} name: '@rush-temp/server' version: 0.0.0 dependencies: @@ -11833,6 +12218,7 @@ packages: '@types/ws': 7.4.7 '@typescript-eslint/eslint-plugin': 5.7.0_c25e8c1f4f4f7aaed27aa6f9ce042237 '@typescript-eslint/parser': 5.7.0_eslint@7.32.0+typescript@4.5.4 + elastic-apm-node: 3.26.0 esbuild: 0.12.29 eslint: 7.32.0 eslint-config-standard-with-typescript: 21.0.1_ce2fa0c4dfa1c256100cababd749a13a diff --git a/dev/client-resources/src/connection.ts b/dev/client-resources/src/connection.ts index 2a9c57f2c4..648853521f 100644 --- a/dev/client-resources/src/connection.ts +++ b/dev/client-resources/src/connection.ts @@ -13,24 +13,25 @@ // limitations under the License. // -import type { Class, ClientConnection, Doc, DocumentQuery, FindOptions, FindResult, Ref, ServerStorage, Tx, TxHander, TxResult } from '@anticrm/core' -import { DOMAIN_TX } from '@anticrm/core' +import { Class, ClientConnection, Doc, DocumentQuery, FindOptions, FindResult, Ref, ServerStorage, Tx, TxHander, TxResult, DOMAIN_TX, MeasureMetricsContext } from '@anticrm/core' import { createInMemoryAdapter, createInMemoryTxAdapter } from '@anticrm/dev-storage' import { protoDeserialize, protoSerialize } from '@anticrm/platform' import type { DbConfiguration } from '@anticrm/server-core' import { createServerStorage, FullTextAdapter, IndexedDoc } from '@anticrm/server-core' class ServerStorageWrapper implements ClientConnection { - constructor (private readonly storage: ServerStorage, private readonly handler: TxHander) {} + measureCtx = new MeasureMetricsContext('client', {}) + constructor (private readonly storage: ServerStorage, private readonly handler: TxHander) { + } findAll (_class: Ref>, query: DocumentQuery, options?: FindOptions): Promise> { const [c, q, o] = protoDeserialize(protoSerialize([_class, query, options])) - return this.storage.findAll(c, q, o) + return this.storage.findAll(this.measureCtx, c, q, o) } async tx (tx: Tx): Promise { const _tx = protoDeserialize(protoSerialize(tx)) - const [result, derived] = await this.storage.tx(_tx) + const [result, derived] = await this.storage.tx(this.measureCtx, _tx) for (const tx of derived) { this.handler(tx) } return result } diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 8eb41ea377..56167c1853 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -30,6 +30,10 @@ services: - ELASTICSEARCH_PORT_NUMBER=9200 - BITNAMI_DEBUG=true - discovery.type=single-node + healthcheck: + interval: 20s + retries: 10 + test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"' account: image: anticrm/account links: @@ -66,6 +70,48 @@ services: environment: - ELASTIC_URL=http://elastic:9200 - MONGO_URL=mongodb://mongodb:27017 + - METRICS_CONSOLE=true + # - APM_SERVER_URL=http://apm-server:8200 + # apm-server: + # image: docker.elastic.co/apm/apm-server:7.14.2 + # depends_on: + # elastic: + # condition: service_healthy + # kibana: + # condition: service_healthy + # elastic: + # condition: service_healthy + # cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"] + # cap_drop: ["ALL"] + # ports: + # - 8200:8200 + # command: > + # apm-server -e + # -E apm-server.rum.enabled=true + # -E setup.kibana.host=kibana:5601 + # -E setup.template.settings.index.number_of_replicas=0 + # -E apm-server.kibana.enabled=true + # -E apm-server.kibana.host=kibana:5601 + # -E output.elasticsearch.hosts=["elastic:9200"] + # healthcheck: + # interval: 10s + # retries: 12 + # test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/ + # kibana: + # image: docker.elastic.co/kibana/kibana:7.14.2 + # depends_on: + # elastic: + # condition: service_healthy + # environment: + # ELASTICSEARCH_URL: http://elastic:9200 + # ELASTICSEARCH_HOSTS: http://elastic:9200 + # ports: + # - 5601:5601 + # healthcheck: + # interval: 10s + # retries: 20 + # test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status + volumes: db: files: diff --git a/dev/generator/src/index.ts b/dev/generator/src/index.ts index 4b7c8da207..2a1ac5f32d 100644 --- a/dev/generator/src/index.ts +++ b/dev/generator/src/index.ts @@ -56,6 +56,7 @@ program .command('gen-recruit ') .description('generate a bunch of random candidates with attachemnts and comments.') .option('-r, --random', 'generate random ids. So every call will add count more candidates.', false) + .option('-l, --lite', 'use same pdf and same account for applicant and candidates', false) .action(async (workspace: string, count: number, cmd) => { return await generateContacts(transactorUrl, workspace, { contacts: count, @@ -65,7 +66,8 @@ program min: 1, max: 3, deleteFactor: 20 }, vacancy: 3, - applicantUpdateFactor: 70 + applicants: { min: 50, max: 200, applicantUpdateFactor: 70 }, + lite: (cmd.lite as boolean) }, minio) }) diff --git a/dev/generator/src/kanban.ts b/dev/generator/src/kanban.ts index 03557cbb71..3bcf3f6aa9 100644 --- a/dev/generator/src/kanban.ts +++ b/dev/generator/src/kanban.ts @@ -1,9 +1,9 @@ -import { Ref, TxOperations } from '@anticrm/core' +import { MeasureContext, Ref, TxOperations } from '@anticrm/core' import task, { DoneState, genRanks, Kanban, SpaceWithStates, State } from '@anticrm/task' import { findOrUpdate } from './utils' -export async function createUpdateSpaceKanban (spaceId: Ref, client: TxOperations): Promise[]> { +export async function createUpdateSpaceKanban (ctx: MeasureContext, spaceId: Ref, client: TxOperations): Promise[]> { const rawStates = [ { color: '#7C6FCD', name: 'Initial' }, { color: '#6F7BC5', name: 'Intermidiate' }, @@ -22,14 +22,15 @@ export async function createUpdateSpaceKanban (spaceId: Ref, cl } const sid = ('generated-' + spaceId + '.state.' + st.name.toLowerCase().replace(' ', '_')) as Ref - await findOrUpdate(client, spaceId, task.class.State, + + await ctx.with('find-or-update', {}, (ctx) => findOrUpdate(ctx, client, spaceId, task.class.State, sid, { title: st.name, color: st.color, rank } - ) + )) states.push(sid) } @@ -47,21 +48,20 @@ export async function createUpdateSpaceKanban (spaceId: Ref, cl } const sid = ('generated-' + spaceId + '.done-state.' + st.title.toLowerCase().replace(' ', '_')) as Ref - await findOrUpdate(client, spaceId, st.class, + await ctx.with('gen-done-state', {}, (ctx) => findOrUpdate(ctx, client, spaceId, st.class, sid, { title: st.title, rank } - ) + )) } - await findOrUpdate(client, spaceId, - task.class.Kanban, + await ctx.with('create-kanban', {}, (ctx) => findOrUpdate(ctx, client, spaceId, task.class.Kanban, ('generated-' + spaceId + '.kanban') as Ref, { attachedTo: spaceId } - ) + )) return states } diff --git a/dev/generator/src/recruit.ts b/dev/generator/src/recruit.ts index e4a49a1322..b2a262a07f 100644 --- a/dev/generator/src/recruit.ts +++ b/dev/generator/src/recruit.ts @@ -1,8 +1,15 @@ -import contact from '@anticrm/contact' -import core, { AttachedData, Data, generateId, Ref, TxOperations } from '@anticrm/core' +import contact, { Employee, EmployeeAccount } from '@anticrm/contact' +import core, { + AttachedData, + Data, + generateId, + MeasureContext, + MeasureMetricsContext, metricsToString, Ref, + TxOperations +} from '@anticrm/core' import recruit from '@anticrm/model-recruit' import { Applicant, Candidate, Vacancy } from '@anticrm/recruit' -import { genRanks } from '@anticrm/task' +import { genRanks, State } from '@anticrm/task' import faker from 'faker' import jpeg, { BufferRet } from 'jpeg-js' import { Client } from 'minio' @@ -11,7 +18,6 @@ import { addComments, CommentOptions } from './comments' import { connect } from './connect' import { createUpdateSpaceKanban } from './kanban' import { findOrUpdate, findOrUpdateAttached } from './utils' - export interface RecruitOptions { random: boolean // random id prefix. contacts: number // how many contacts to add @@ -21,15 +27,25 @@ export interface RecruitOptions { // Attachment generation control attachments: AttachmentOptions - applicantUpdateFactor: number + applicants: { + min: number + max: number + applicantUpdateFactor: number + } + lite: boolean } -export async function generateContacts (transactorUrl: string, dbName: string, options: RecruitOptions, minio: Client): Promise { +export async function generateContacts ( + transactorUrl: string, + dbName: string, + options: RecruitOptions, + minio: Client +): Promise { const connection = await connect(transactorUrl, dbName) const accounts = await connection.findAll(contact.class.EmployeeAccount, {}) - const accountIds = accounts.map(a => a._id) - const emoloyeeIds = accounts.map(a => a.employee) + const accountIds = accounts.map((a) => a._id) + const emoloyeeIds = accounts.map((a) => a.employee) const account = faker.random.arrayElement(accounts) @@ -37,100 +53,222 @@ export async function generateContacts (transactorUrl: string, dbName: string, o const candidates: Ref[] = [] + const ctx = new MeasureMetricsContext('recruit', { contacts: options.contacts }) + for (let i = 0; i < options.contacts; i++) { - const fName = faker.name.firstName() - const lName = faker.name.lastName() - - const { imgId, jpegImageData } = generateAvatar(i) - await minio.putObject(dbName, imgId, jpegImageData.data, jpegImageData.data.length, { 'Content-Type': 'image/jpeg' }) - const candidate: Data = { - name: fName + ',' + lName, - city: faker.address.city(), - title: faker.name.title(), - channels: [ - { provider: contact.channelProvider.Email, value: faker.internet.email(fName, lName) } - ], - onsite: faker.datatype.boolean(), - remote: faker.datatype.boolean(), - avatar: imgId, - source: faker.lorem.lines(1) - } - const candidateId = (options.random ? `candidate-${generateId()}-${i}` : `candidate-genid-${i}`) as Ref - candidates.push(candidateId) - - // Update or create candidate - await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candidateId, candidate) - - await addComments(options.comments, client, recruit.space.CandidatesPublic, candidateId, recruit.class.Candidate, 'comments') - - await addAttachments(options.attachments, client, minio, dbName, recruit.space.CandidatesPublic, candidateId, recruit.class.Candidate, 'attachments') - - console.log('Candidate', fName, lName, ' generated') + await ctx.with('candidate', {}, (ctx) => genCandidate(ctx, i, minio, dbName, options, candidates, client)) } // Work on Vacancy/Applications. for (let i = 0; i < options.vacancy; i++) { - const vacancy: Data = { - name: faker.company.companyName(), - description: faker.lorem.sentences(2), - fullDescription: faker.lorem.sentences(10), - location: faker.address.city(), - company: faker.company.companyName(), - members: accountIds, - archived: false, - private: false - } - const vacancyId = (options.random ? `vacancy-${generateId()}-${i}` : `vacancy-genid-${i}`) as Ref - - console.log('Creating vacancy', vacancy.name) - // Update or create candidate - await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy) - - console.log('Vacandy generated', vacancy.name) - - await addAttachments(options.attachments, client, minio, dbName, vacancyId, vacancyId, recruit.class.Vacancy, 'attachments') - - console.log('Vacandy attachments generated', vacancy.name) - - const states = await createUpdateSpaceKanban(vacancyId, client) - - console.log('States generated', vacancy.name) - - const rankGen = genRanks(candidates.length) - for (const candidateId of candidates) { - const rank = rankGen.next().value - - if (rank === undefined) { - throw Error('Failed to generate rank') - } - - const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref - - const applicant: AttachedData = { - number: faker.datatype.number(), - assignee: faker.random.arrayElement(emoloyeeIds), - state: faker.random.arrayElement(states), - doneState: null, - rank - } - - // Update or create candidate - await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' }) - - await addComments(options.comments, client, vacancyId, applicantId, recruit.class.Vacancy, 'comments') - - await addAttachments(options.attachments, client, minio, dbName, vacancyId, applicantId, recruit.class.Applicant, 'attachments') - - if (faker.datatype.number(100) > options.applicantUpdateFactor) { - await client.updateCollection(recruit.class.Applicant, vacancyId, applicantId, candidateId, recruit.class.Applicant, 'applications', { - state: faker.random.arrayElement(states) - }) - } - } + await ctx.with('vacancy', {}, (ctx) => + genVacansyApplicants(ctx, accountIds, options, i, client, minio, dbName, candidates, emoloyeeIds) + ) } await connection.close() + ctx.end() + + console.info(metricsToString(ctx.metrics, 'Client')) } -function generateAvatar (pos: number): {imgId: string, jpegImageData: BufferRet } { +async function genVacansyApplicants ( + ctx: MeasureContext, + accountIds: Ref[], + options: RecruitOptions, + i: number, + client: TxOperations, + minio: Client, + dbName: string, + candidates: Ref[], + emoloyeeIds: Ref[] +): Promise { + const vacancy: Data = { + name: faker.company.companyName(), + description: faker.lorem.sentences(2), + fullDescription: faker.lorem.sentences(10), + location: faker.address.city(), + company: faker.company.companyName(), + members: accountIds, + private: false, + archived: false + } + const vacancyId = (options.random ? `vacancy-${generateId()}-${i}` : `vacancy-genid-${i}`) as Ref + + console.log('Creating vacandy', vacancy.name) + + // Update or create candidate + await ctx.with('update', {}, (ctx) => + findOrUpdate(ctx, client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy) + ) + + console.log('Vacandy generated', vacancy.name) + + if (!options.lite) { + await ctx.with('add-attachments', {}, () => + addAttachments( + options.attachments, + client, + minio, + dbName, + vacancyId, + vacancyId, + recruit.class.Vacancy, + 'attachments' + ) + ) + } + + console.log('Vacandy attachments generated', vacancy.name) + + const states = await ctx.with('create-kanbad', {}, (ctx) => createUpdateSpaceKanban(ctx, vacancyId, client)) + + console.log('States generated', vacancy.name) + + const applicantsForCount = options.applicants.min + faker.datatype.number(options.applicants.max) + + const applicantsFor = faker.random.arrayElements(candidates, applicantsForCount) + const rankGen = genRanks(candidates.length) + for (const candidateId of applicantsFor) { + await ctx.with('applicant', {}, (ctx) => + genApplicant(ctx, vacancyId, candidateId, emoloyeeIds, states, client, options, minio, dbName, rankGen) + ) + } +} + +async function genApplicant ( + ctx: MeasureContext, + vacancyId: Ref, + candidateId: Ref, + emoloyeeIds: Ref[], + states: Ref[], + client: TxOperations, + options: RecruitOptions, + minio: Client, + dbName: string, + rankGen: Generator +): Promise { + const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref + const rank = rankGen.next().value + + const applicant: AttachedData = { + number: faker.datatype.number(), + assignee: faker.random.arrayElement(emoloyeeIds), + state: faker.random.arrayElement(states), + doneState: null, + rank: rank as string + } + + // Update or create candidate + await findOrUpdateAttached(ctx, client, vacancyId, recruit.class.Applicant, applicantId, applicant, { + attachedTo: candidateId, + attachedClass: recruit.class.Candidate, + collection: 'applications' + }) + + await ctx.with('add-comment', {}, () => + addComments(options.comments, client, vacancyId, applicantId, recruit.class.Vacancy, 'comments') + ) + + if (!options.lite) { + await ctx.with('add-attachment', {}, () => + addAttachments( + options.attachments, + client, + minio, + dbName, + vacancyId, + applicantId, + recruit.class.Applicant, + 'attachments' + ) + ) + } + + if (faker.datatype.number(100) > options.applicants.applicantUpdateFactor) { + await ctx.with('update-collection', {}, () => + client.updateCollection( + recruit.class.Applicant, + vacancyId, + applicantId, + candidateId, + recruit.class.Applicant, + 'applications', + { + state: faker.random.arrayElement(states) + } + ) + ) + } +} + +const liteAvatar = generateAvatar(0) + +// @measure('Candidate') +async function genCandidate ( + ctx: MeasureContext, + i: number, + minio: Client, + dbName: string, + options: RecruitOptions, + candidates: Ref[], + client: TxOperations +): Promise { + const fName = faker.name.firstName() + const lName = faker.name.lastName() + + const { imgId, jpegImageData } = options.lite ? liteAvatar : generateAvatar(i) + + if (!options.lite) { + await ctx.with('avatar', {}, () => + minio.putObject(dbName, imgId, jpegImageData.data, jpegImageData.data.length, { 'Content-Type': 'image/jpeg' }) + ) + } + const candidate: Data = { + name: fName + ',' + lName, + city: faker.address.city(), + title: faker.name.title(), + channels: [{ provider: contact.channelProvider.Email, value: faker.internet.email(fName, lName) }], + onsite: faker.datatype.boolean(), + remote: faker.datatype.boolean(), + avatar: imgId, + source: faker.lorem.lines(1) + } + const candidateId = (options.random ? `candidate-${generateId()}-${i}` : `candidate-genid-${i}`) as Ref + candidates.push(candidateId) + + // Update or create candidate + await ctx.with('find-update', {}, () => + findOrUpdate(ctx, client, recruit.space.CandidatesPublic, recruit.class.Candidate, candidateId, candidate) + ) + + await ctx.with('add-comment', {}, () => + addComments( + options.comments, + client, + recruit.space.CandidatesPublic, + candidateId, + recruit.class.Candidate, + 'comments' + ) + ) + + if (!options.lite) { + await ctx.with('add-attachment', {}, () => + addAttachments( + options.attachments, + client, + minio, + dbName, + recruit.space.CandidatesPublic, + candidateId, + recruit.class.Candidate, + 'attachments' + ) + ) + } + console.log('Candidate', candidates.length, fName, lName, ' generated') +} + +function generateAvatar (pos: number): { imgId: string, jpegImageData: BufferRet } { const imgId = generateId() const width = 128 const height = 128 diff --git a/dev/generator/src/utils.ts b/dev/generator/src/utils.ts index a44ca19881..37dde35d0c 100644 --- a/dev/generator/src/utils.ts +++ b/dev/generator/src/utils.ts @@ -1,6 +1,6 @@ -import { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, Space, TxOperations } from '@anticrm/core' +import { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, MeasureContext, Ref, Space, TxOperations } from '@anticrm/core' -export async function findOrUpdate (client: TxOperations, space: Ref, _class: Ref>, objectId: Ref, data: Data): Promise { +export async function findOrUpdate (ctx: MeasureContext, client: TxOperations, space: Ref, _class: Ref>, objectId: Ref, data: Data): Promise { const existingObj = await client.findOne(_class, { _id: objectId, space }) if (existingObj !== undefined) { await client.updateDoc(_class, space, objectId, data) @@ -8,7 +8,7 @@ export async function findOrUpdate (client: TxOperations, space: await client.createDoc(_class, space, data, objectId) } } -export async function findOrUpdateAttached (client: TxOperations, space: Ref, _class: Ref>, objectId: Ref, data: AttachedData, attached: {attachedTo: Ref, attachedClass: Ref>, collection: string}): Promise { +export async function findOrUpdateAttached (ctx: MeasureContext, client: TxOperations, space: Ref, _class: Ref>, objectId: Ref, data: AttachedData, attached: {attachedTo: Ref, attachedClass: Ref>, collection: string}): Promise { const existingObj = await client.findOne(_class, { _id: objectId, space }) if (existingObj !== undefined) { await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate) diff --git a/dev/server/src/server.ts b/dev/server/src/server.ts index c837d148db..80248b8c14 100644 --- a/dev/server/src/server.ts +++ b/dev/server/src/server.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import { DOMAIN_TX } from '@anticrm/core' +import { DOMAIN_TX, MeasureMetricsContext } from '@anticrm/core' import type { Ref, Doc, TxResult } from '@anticrm/core' import { start as startJsonRpc } from '@anticrm/server-ws' import { createInMemoryAdapter, createInMemoryTxAdapter } from '@anticrm/dev-storage' @@ -48,7 +48,7 @@ async function createNullFullTextAdapter (): Promise { export async function start (port: number, host?: string): Promise { addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources')) - startJsonRpc(() => { + startJsonRpc(new MeasureMetricsContext('server', {}), () => { const conf: DbConfiguration = { domains: { [DOMAIN_TX]: 'InMemoryTx' diff --git a/dev/tool/src/elastic.ts b/dev/tool/src/elastic.ts index 1a34b1f522..d529695c9d 100644 --- a/dev/tool/src/elastic.ts +++ b/dev/tool/src/elastic.ts @@ -17,14 +17,10 @@ import core, { Account, Class, - Doc, - FindOptions, - DocumentQuery, - DOMAIN_TX, - FindResult, + Doc, DocumentQuery, + DOMAIN_TX, FindOptions, FindResult, generateId, - Hierarchy, - ModelDb, + Hierarchy, MeasureMetricsContext, ModelDb, Ref, ServerStorage, Tx, @@ -36,10 +32,11 @@ import core, { TxResult, TxUpdateDoc } from '@anticrm/core' -import { Client as ElasticClient } from '@elastic/elasticsearch' -import { Db, MongoClient } from 'mongodb' -import { Client } from 'minio' import { createElasticAdapter } from '@anticrm/elastic' +import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment' +import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo' +import { addLocation } from '@anticrm/platform' +import { serverChunterId } from '@anticrm/server-chunter' import { createServerStorage, DbAdapter, @@ -48,11 +45,10 @@ import { IndexedDoc, TxAdapter } from '@anticrm/server-core' -import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment' -import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo' -import { serverChunterId } from '@anticrm/server-chunter' import { serverRecruitId } from '@anticrm/server-recruit' -import { addLocation } from '@anticrm/platform' +import { Client as ElasticClient } from '@elastic/elasticsearch' +import { Client } from 'minio' +import { Db, MongoClient } from 'mongodb' import { listMinioObjects } from './minio' export async function rebuildElastic ( @@ -106,8 +102,9 @@ async function restoreElastic (mongoUrl: string, dbName: string, minio: Client, const storage = await createStorage(mongoUrl, elasticUrl, dbName) const txes = (await db.collection(DOMAIN_TX).find().sort({ _id: 1 }).toArray()) const data = txes.filter((tx) => tx.objectSpace !== core.space.Model) + const metricsCtx = new MeasureMetricsContext('elastic', {}) for (const tx of data) { - await storage.tx(tx) + await storage.tx(metricsCtx, tx) } if (await minio.bucketExists(dbName)) { const minioObjects = await listMinioObjects(minio, dbName) diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index 89a2519457..e02b6a4974 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -223,7 +223,9 @@ export async function restoreWorkspace ( const collection = db.collection(c.name) await collection.deleteMany({}) const data = JSON.parse((await readFile(fileName + c.name + '.json')).toString()) as Document[] - await collection.insertMany(data) + if (data.length > 0) { + await collection.insertMany(data) + } } console.log('Restore minio objects') diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5f12bec514..ff3976f698 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -22,5 +22,6 @@ export * from './client' export * from './operator' export * from './query' export * from './server' +export * from './measurements' export { default, coreId } from './component' diff --git a/packages/core/src/measurements/context.ts b/packages/core/src/measurements/context.ts new file mode 100644 index 0000000000..7d7395eb67 --- /dev/null +++ b/packages/core/src/measurements/context.ts @@ -0,0 +1,58 @@ +// Basic performance metrics suite. + +import { childMetrics, measure, newMetrics } from './metrics' +import { MeasureContext, MeasureLogger, Metrics, ParamType } from './types' + +/** + * @public + */ +export class MeasureMetricsContext implements MeasureContext { + private readonly name: string + private readonly params: Record + logger: MeasureLogger + metrics: Metrics + private readonly done: () => void + + constructor (name: string, params: Record, metrics: Metrics = newMetrics()) { + this.name = name + this.params = params + this.metrics = metrics + this.done = measure(metrics, params) + + this.logger = { + info: (msg, args) => { + console.info(msg, ...args) + }, + error: (msg, args) => { + console.error(msg, ...args) + } + } + } + + newChild (name: string, params: Record): MeasureContext { + return new MeasureMetricsContext(name, params, childMetrics(this.metrics, [name])) + } + + async with(name: string, params: Record, op: (ctx: MeasureContext) => T | Promise): Promise { + const c = this.newChild(name, params) + try { + let value = op(c) + if (value instanceof Promise) { + value = await value + } + c.end() + return value + } catch (err: any) { + await c.error(err) + throw err + } + } + + async error (err: Error | string): Promise { + console.error(err) + } + + end (): void { + this.done() + } +} diff --git a/packages/core/src/measurements/index.ts b/packages/core/src/measurements/index.ts new file mode 100644 index 0000000000..f15531da84 --- /dev/null +++ b/packages/core/src/measurements/index.ts @@ -0,0 +1,3 @@ +export * from './context' +export * from './metrics' +export * from './types' diff --git a/packages/core/src/measurements/metrics.ts b/packages/core/src/measurements/metrics.ts new file mode 100644 index 0000000000..49628a7b4d --- /dev/null +++ b/packages/core/src/measurements/metrics.ts @@ -0,0 +1,156 @@ +// Basic performance metrics suite. + +import { MetricsData } from '.' +import { Metrics, ParamType } from './types' + +/** + * @public + */ +export const globals: Metrics = newMetrics() + +/** + * @public + * @returns + */ +export function newMetrics (): Metrics { + return { + operations: 0, + time: 0, + measurements: {}, + params: {} + } +} + +/** + * Measure with tree expansion. Operation counter will be added only to leaf's. + * @public + */ +export function measure (metrics: Metrics, params: Record): () => void { + const st = Date.now() + return () => { + const ed = Date.now() + // Update params if required + for (const [k, v] of Object.entries(params)) { + let params = metrics.params[k] + if (params === undefined) { + params = {} + metrics.params[k] = params + } + const vKey = `${v?.toString() ?? ''}` + let param = params[vKey] + if (param === undefined) { + param = { + operations: 0, + time: 0 + } + params[vKey] = param + } + param.time += ed - st + param.operations++ + } + // Update leaf data + metrics.time += ed - st + metrics.operations++ + } +} + +/** + * @public + */ +export function childMetrics (root: Metrics, path: string[]): Metrics { + const segments = path + let oop = root + for (const p of segments) { + const v = oop.measurements[p] ?? { operations: 0, time: 0, measurements: {}, params: {} } + oop.measurements[p] = v + oop = v + } + return oop +} + +function aggregate (m: Metrics): Metrics { + const ms = aggregateMetrics(m.measurements) + + // Use child overage, if there is no top level value specified. + const keysLen = Object.keys(ms).length + const childAverage = m.time === 0 && keysLen > 0 + const sumVal: Metrics | undefined = childAverage + ? Object.values(ms).reduce((p, v) => { + p.operations += v.operations + p.time += v.time + return p + }, { + operations: 0, + time: 0, + measurements: ms, + params: {} + }) + : undefined + if (sumVal !== undefined) { + return { + ...sumVal, + measurements: ms, + params: m.params + } + } + return { + ...m, + measurements: ms + } +} + +function aggregateMetrics (m: Record): Record { + const result: Record = {} + for (const [k, v] of Object.entries(m).sort((a, b) => b[1].time - a[1].time)) { + result[k] = aggregate(v) + } + return result +} + +function toLen (val: string, sep: string, len = 50): string { + while (val.length < len) { + val += sep + } + return val +} + +function printMetricsChildren (params: Record, offset: number): string { + let r = '' + if (Object.keys(params).length > 0) { + r += '\n' + toLen('', ' ', offset) + r += Object.entries(params) + .map(([k, vv]) => toString(k, vv, offset)) + .join('\n' + toLen('', ' ', offset)) + } + return r +} + +function printMetricsParams (params: Record>, offset: number): string { + let r = '' + const joinP = (key: string, data: Record): string[] => { + return Object.entries(data).map(([k, vv]) => + `${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', 70 - offset)}: avg ${vv.time / (vv.operations > 0 ? vv.operations : 1)} total: ${vv.time} ops: ${vv.operations}`.trim() + ) + } + const joinParams = Object.entries(params).reduce((p, c) => [...p, ...joinP(c[0], c[1])], []) + if (Object.keys(joinParams).length > 0) { + r += '\n' + toLen('', ' ', offset) + r += joinParams + .join('\n' + toLen('', ' ', offset)) + } + return r +} + +function toString (name: string, m: Metrics, offset: number): string { + let r = `${toLen('', ' ', offset)}${toLen(name, '-', 70 - offset)}: avg ${m.time / (m.operations > 0 ? m.operations : 1)} total: ${m.time} ops: ${m.operations}`.trim() + r += printMetricsParams(m.params, offset + 4) + r += printMetricsChildren(m.measurements, offset + 4) + return r +} + +/** + * @public + */ +export function metricsToString (metrics: Metrics, name = 'System'): string { + return toString(name, aggregate(metrics), 0) +} diff --git a/packages/core/src/measurements/types.ts b/packages/core/src/measurements/types.ts new file mode 100644 index 0000000000..d91ad5f88d --- /dev/null +++ b/packages/core/src/measurements/types.ts @@ -0,0 +1,45 @@ +/** + * @public + */ +export type ParamType = string | number | boolean | undefined + +/** + * @public + */ +export interface MetricsData { + operations: number + time: number +} + +/** + * @public + */ +export interface Metrics extends MetricsData { + params: Record> + measurements: Record +} + +/** + * @public + */ +export interface MeasureLogger { + info: (message: string, ...args: any[]) => void + error: (message: string, ...args: any[]) => void +} +/** + * @public + */ +export interface MeasureContext { + // Create a child metrics context + newChild: (name: string, params: Record) => MeasureContext + + with: (name: string, params: Record, op: (ctx: MeasureContext) => T | Promise) => Promise + + logger: MeasureLogger + + // Capture error + error: (err: Error | string | any) => Promise + + // Mark current context as complete + end: () => void +} diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 5c55e14808..fbcc9b5300 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -14,6 +14,7 @@ // limitations under the License. // +import { MeasureContext } from '.' import type { Doc, Class, Ref } from './classes' import type { DocumentQuery, FindOptions, FindResult, TxResult } from './storage' import type { Tx } from './tx' @@ -23,9 +24,10 @@ import type { Tx } from './tx' */ export interface ServerStorage { findAll: ( + ctx: MeasureContext, _class: Ref>, query: DocumentQuery, options?: FindOptions ) => Promise> - tx: (tx: Tx) => Promise<[TxResult, Tx[]]> + tx: (ctx: MeasureContext, tx: Tx) => Promise<[TxResult, Tx[]]> } diff --git a/plugins/devmodel-resources/src/index.ts b/plugins/devmodel-resources/src/index.ts index 1aef36032f..b3c970128d 100644 --- a/plugins/devmodel-resources/src/index.ts +++ b/plugins/devmodel-resources/src/index.ts @@ -16,7 +16,7 @@ import { IntlString, Resources } from '@anticrm/platform' import ModelView from './components/ModelView.svelte' import QueryView from './components/QueryView.svelte' -import core, { Class, Client, Doc, DocumentQuery, FindOptions, Ref, FindResult, Hierarchy, ModelDb, Tx, TxResult, WithLookup } from '@anticrm/core' +import core, { Class, Client, Doc, DocumentQuery, FindOptions, Ref, FindResult, Hierarchy, ModelDb, Tx, TxResult, WithLookup, Metrics } from '@anticrm/core' import { Builder } from '@anticrm/model' import workbench from '@anticrm/workbench' import view from '@anticrm/view' @@ -45,6 +45,9 @@ class ModelClient implements Client { this.notify?.(tx) console.info('devmodel# notify=>', tx, this.client.getModel()) notifications.push(tx) + if (notifications.length > 500) { + notifications.shift() + } } } @@ -62,6 +65,9 @@ class ModelClient implements Client { const result = await this.client.findOne(_class, query, options) console.info('devmodel# findOne=>', _class, query, options, 'result => ', result, ' =>model', this.client.getModel()) queries.push({ _class, query, options, result: result !== undefined ? [result] : [], findOne: true }) + if (queries.length > 100) { + queries.shift() + } return result } @@ -69,6 +75,9 @@ class ModelClient implements Client { const result = await this.client.findAll(_class, query, options) console.info('devmodel# findAll=>', _class, query, options, 'result => ', result, ' =>model', this.client.getModel()) queries.push({ _class, query, options, result, findOne: false }) + if (queries.length > 100) { + queries.shift() + } return result } @@ -76,6 +85,9 @@ class ModelClient implements Client { const result = await this.client.tx(tx) console.info('devmodel# tx=>', tx, result) transactions.push({ tx, result }) + if (transactions.length > 100) { + transactions.shift() + } return result } diff --git a/server/core/src/fulltext.ts b/server/core/src/fulltext.ts index 5d28315cb4..80516f01f4 100644 --- a/server/core/src/fulltext.ts +++ b/server/core/src/fulltext.ts @@ -14,22 +14,25 @@ // limitations under the License. // +import type { AttachedDoc, Class, Doc, Obj, Ref, TxCreateDoc, TxResult, TxUpdateDoc } from '@anticrm/core' import core, { - Hierarchy, AnyAttribute, - Storage, + Collection, DocumentQuery, FindOptions, FindResult, - TxProcessor, + Hierarchy, + MeasureContext, + PropertyType, + Tx, + TxBulkWrite, + TxCollectionCUD, TxMixin, + TxProcessor, TxPutBag, - TxRemoveDoc, - Collection + TxRemoveDoc } from '@anticrm/core' -import type { AttachedDoc, TxUpdateDoc, TxCreateDoc, Doc, Ref, Class, Obj, TxResult } from '@anticrm/core' - -import type { IndexedDoc, FullTextAdapter, WithFind } from './types' +import type { FullTextAdapter, IndexedDoc, WithFind } from './types' // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const NO_INDEX = [] as AnyAttribute[] @@ -37,32 +40,76 @@ const NO_INDEX = [] as AnyAttribute[] /** * @public */ -export class FullTextIndex extends TxProcessor implements Storage { +export class FullTextIndex implements WithFind { private readonly indexes = new Map>, AnyAttribute[]>() constructor ( private readonly hierarchy: Hierarchy, private readonly adapter: FullTextAdapter, private readonly dbStorage: WithFind - ) { - super() - } + ) {} - protected override async txPutBag (tx: TxPutBag): Promise { - console.log('FullTextIndex.txPutBag: Method not implemented.') + protected async txPutBag (ctx: MeasureContext, tx: TxPutBag): Promise { + // console.log('FullTextIndex.txPutBag: Method not implemented.') return {} } - protected override async txRemoveDoc (tx: TxRemoveDoc): Promise { - console.log('FullTextIndex.txRemoveDoc: Method not implemented.') + protected async txRemoveDoc (ctx: MeasureContext, tx: TxRemoveDoc): Promise { + // console.log('FullTextIndex.txRemoveDoc: Method not implemented.') return {} } - protected txMixin (tx: TxMixin): Promise { + protected txMixin (ctx: MeasureContext, tx: TxMixin): Promise { throw new Error('Method not implemented.') } + async tx (ctx: MeasureContext, tx: Tx): Promise { + switch (tx._class) { + case core.class.TxCreateDoc: + return await this.txCreateDoc(ctx, tx as TxCreateDoc) + case core.class.TxCollectionCUD: + return await this.txCollectionCUD(ctx, tx as TxCollectionCUD) + case core.class.TxUpdateDoc: + return await this.txUpdateDoc(ctx, tx as TxUpdateDoc) + case core.class.TxRemoveDoc: + return await this.txRemoveDoc(ctx, tx as TxRemoveDoc) + case core.class.TxMixin: + return await this.txMixin(ctx, tx as TxMixin) + case core.class.TxPutBag: + return await this.txPutBag(ctx, tx as TxPutBag) + case core.class.TxBulkWrite: + return await this.txBulkWrite(ctx, tx as TxBulkWrite) + } + throw new Error('TxProcessor: unhandled transaction class: ' + tx._class) + } + + protected txCollectionCUD (ctx: MeasureContext, tx: TxCollectionCUD): Promise { + // We need update only create transactions to contain attached, attachedToClass. + if (tx.tx._class === core.class.TxCreateDoc) { + const createTx = tx.tx as TxCreateDoc + const d: TxCreateDoc = { + ...createTx, + attributes: { + ...createTx.attributes, + attachedTo: tx.objectId, + attachedToClass: tx.objectClass, + collection: tx.collection + } + } + return this.txCreateDoc(ctx, d) + } + return this.tx(ctx, tx.tx) + } + + protected async txBulkWrite (ctx: MeasureContext, bulkTx: TxBulkWrite): Promise { + for (const tx of bulkTx.txes) { + await this.tx(ctx, tx) + } + return {} + } + async findAll( + ctx: MeasureContext, _class: Ref>, query: DocumentQuery, options?: FindOptions @@ -78,7 +125,7 @@ export class FullTextIndex extends TxProcessor implements Storage { ids.push(doc.attachedTo) } } - return await this.dbStorage.findAll(_class, { _id: { $in: ids as any }, ...mainQuery }, options) // TODO: remove `as any` + return await this.dbStorage.findAll(ctx, _class, { _id: { $in: ids as any }, ...mainQuery }, options) // TODO: remove `as any` } private getFullTextAttributes (clazz: Ref>): AnyAttribute[] | undefined { @@ -102,14 +149,14 @@ export class FullTextIndex extends TxProcessor implements Storage { } } - protected override async txCreateDoc (tx: TxCreateDoc): Promise { + protected async txCreateDoc (ctx: MeasureContext, tx: TxCreateDoc): Promise { const attributes = this.getFullTextAttributes(tx.objectClass) const doc = TxProcessor.createDoc2Doc(tx) let parentContent: any[] = [] if (this.hierarchy.isDerived(doc._class, core.class.AttachedDoc)) { const attachedDoc = doc as AttachedDoc const parentDoc = ( - await this.dbStorage.findAll(attachedDoc.attachedToClass, { _id: attachedDoc.attachedTo }, { limit: 1 }) + await this.dbStorage.findAll(ctx, attachedDoc.attachedToClass, { _id: attachedDoc.attachedTo }, { limit: 1 }) )[0] if (parentDoc !== undefined) { const parentAttributes = this.getFullTextAttributes(parentDoc._class) @@ -140,7 +187,7 @@ export class FullTextIndex extends TxProcessor implements Storage { return await this.adapter.index(indexedDoc) } - protected override async txUpdateDoc (tx: TxUpdateDoc): Promise { + protected async txUpdateDoc (ctx: MeasureContext, tx: TxUpdateDoc): Promise { const attributes = this.getFullTextAttributes(tx.objectClass) let result = {} if (attributes === undefined) return result @@ -157,7 +204,7 @@ export class FullTextIndex extends TxProcessor implements Storage { } if (shouldUpdate) { result = await this.adapter.update(tx.objectId, update) - await this.updateAttachedDocs(tx, update) + await this.updateAttachedDocs(ctx, tx, update) } return result } @@ -167,14 +214,14 @@ export class FullTextIndex extends TxProcessor implements Storage { return attributes.map((attr) => (doc as any)[attr.name]?.toString() ?? '') } - private async updateAttachedDocs (tx: TxUpdateDoc, update: any): Promise { - const doc = (await this.dbStorage.findAll(tx.objectClass, { _id: tx.objectId }, { limit: 1 }))[0] + private async updateAttachedDocs (ctx: MeasureContext, tx: TxUpdateDoc, update: any): Promise { + const doc = (await this.dbStorage.findAll(ctx, tx.objectClass, { _id: tx.objectId }, { limit: 1 }))[0] if (doc === undefined) return const attributes = this.hierarchy.getAllAttributes(doc._class) for (const attribute of attributes.values()) { if (this.hierarchy.isDerived(attribute.type._class, core.class.Collection)) { const collection = attribute.type as Collection - const allAttached = await this.dbStorage.findAll(collection.of, { attachedTo: tx.objectId }) + const allAttached = await this.dbStorage.findAll(ctx, collection.of, { attachedTo: tx.objectId }) if (allAttached.length === 0) continue const attributes = this.getFullTextAttributes(tx.objectClass) const shift = attributes?.length ?? 0 diff --git a/server/core/src/storage.ts b/server/core/src/storage.ts index 3507cb5701..1838e4bc61 100644 --- a/server/core/src/storage.ts +++ b/server/core/src/storage.ts @@ -14,10 +14,32 @@ // limitations under the License. // -import core, { ServerStorage, Domain, Tx, TxCUD, Doc, Ref, Class, DocumentQuery, FindResult, FindOptions, Storage, TxBulkWrite, TxResult, TxCollectionCUD, AttachedDoc, DOMAIN_MODEL, Hierarchy, DOMAIN_TX, ModelDb, TxFactory } from '@anticrm/core' -import type { FullTextAdapterFactory, FullTextAdapter } from './types' +import core, { + AttachedDoc, + Class, + Doc, + DocumentQuery, + Domain, + DOMAIN_MODEL, + DOMAIN_TX, + FindOptions, + FindResult, + Hierarchy, + MeasureContext, + ModelDb, + Ref, + ServerStorage, + Storage, + Tx, + TxBulkWrite, + TxCollectionCUD, + TxCUD, + TxFactory, + TxResult +} from '@anticrm/core' import { FullTextIndex } from './fulltext' import { Triggers } from './triggers' +import type { FullTextAdapter, FullTextAdapterFactory } from './types' /** * @public @@ -87,7 +109,7 @@ class TServerStorage implements ServerStorage { return adapter } - private async routeTx (tx: Tx): Promise { + private async routeTx (ctx: MeasureContext, tx: Tx): Promise { if (this.hierarchy.isDerived(tx._class, core.class.TxCUD)) { const txCUD = tx as TxCUD const domain = this.hierarchy.getDomain(txCUD.objectClass) @@ -96,7 +118,7 @@ class TServerStorage implements ServerStorage { if (this.hierarchy.isDerived(tx._class, core.class.TxBulkWrite)) { const bulkWrite = tx as TxBulkWrite for (const tx of bulkWrite.txes) { - await this.tx(tx) + await this.tx(ctx, tx) } } else { throw new Error('not implemented (routeTx)') @@ -105,7 +127,7 @@ class TServerStorage implements ServerStorage { } } - async processCollection (tx: Tx): Promise { + async processCollection (ctx: MeasureContext, tx: Tx): Promise { if (tx._class === core.class.TxCollectionCUD) { const colTx = tx as TxCollectionCUD const _id = colTx.objectId @@ -120,57 +142,88 @@ class TServerStorage implements ServerStorage { const isCreateTx = colTx.tx._class === core.class.TxCreateDoc if (isCreateTx || colTx.tx._class === core.class.TxRemoveDoc) { - attachedTo = (await this.findAll(_class, { _id }, { limit: 1 }))[0] + attachedTo = (await this.findAll(ctx, _class, { _id }, { limit: 1 }))[0] if (attachedTo !== undefined) { const txFactory = new TxFactory(tx.modifiedBy) - return [txFactory.createTxUpdateDoc(_class, attachedTo.space, _id, { $inc: { [colTx.collection]: isCreateTx ? 1 : -1 } })] + return [ + txFactory.createTxUpdateDoc(_class, attachedTo.space, _id, { + $inc: { [colTx.collection]: isCreateTx ? 1 : -1 } + }) + ] } } } return [] } - async findAll ( + async findAll( + ctx: MeasureContext, clazz: Ref>, query: DocumentQuery, options?: FindOptions ): Promise> { - const domain = this.hierarchy.getDomain(clazz) - console.log('server findall', query) - if (Object.keys(query)[0] === '$search') { - return await this.fulltext.findAll(clazz, query, options) - } - return await this.getAdapter(domain).findAll(clazz, query, options) + return await ctx.with('find-all', {}, (ctx) => { + const domain = this.hierarchy.getDomain(clazz) + if (Object.keys(query)[0] === '$search') { + return ctx.with('full-text-find-all', {}, (ctx) => this.fulltext.findAll(ctx, clazz, query, options)) + } + return ctx.with('db-find-all', { _class: clazz, domain }, () => + this.getAdapter(domain).findAll(clazz, query, options) + ) + }) } - async tx (tx: Tx): Promise<[TxResult, Tx[]]> { + async tx (ctx: MeasureContext, tx: Tx): Promise<[TxResult, Tx[]]> { // store tx - await this.getAdapter(DOMAIN_TX).tx(tx) + const _class = txClass(tx) + const objClass = txObjectClass(tx) + return await ctx.with('tx', { _class, objClass }, async (ctx) => { + await ctx.with('domain-tx', { _class, objClass }, async () => await this.getAdapter(DOMAIN_TX).tx(tx)) - if (tx.objectSpace === core.space.Model) { - // maintain hiearachy and triggers - this.hierarchy.tx(tx) - await this.triggers.tx(tx) - await this.modelDb.tx(tx) - } - // store object - const result = await this.routeTx(tx) - // invoke triggers and store derived objects - const derived = [...await this.processCollection(tx), ...await this.triggers.apply(tx.modifiedBy, tx, this.findAll.bind(this), this.hierarchy)] - for (const tx of derived) { - await this.routeTx(tx) - } - // index object - await this.fulltext.tx(tx) - // index derived objects - for (const tx of derived) { - await this.fulltext.tx(tx) - } + if (tx.objectSpace === core.space.Model) { + // maintain hiearachy and triggers + this.hierarchy.tx(tx) + await this.triggers.tx(tx) + await this.modelDb.tx(tx) + } - return [result, derived] + let derived: Tx[] = [] + let result: TxResult = {} + // store object + result = await ctx.with('route-tx', { _class, objClass }, (ctx) => this.routeTx(ctx, tx)) + // invoke triggers and store derived objects + derived = [ + ...(await ctx.with('process-collection', { _class }, () => this.processCollection(ctx, tx))), + ...(await ctx.with('process-triggers', {}, (ctx) => + this.triggers.apply(tx.modifiedBy, tx, this.findAll.bind(this, ctx), this.hierarchy) + )) + ] + + for (const tx of derived) { + await ctx.with('derived-route-tx', { _class: txClass(tx) }, (ctx) => this.routeTx(ctx, tx)) + } + + // index object + await ctx.with('fulltext', { _class, objClass }, (ctx) => this.fulltext.tx(ctx, tx)) + // index derived objects + for (const tx of derived) { + await ctx.with('derived-fulltext', { _class: txClass(tx) }, (ctx) => this.fulltext.tx(ctx, tx)) + } + return [result, derived] + }) } } +function txObjectClass (tx: Tx): string { + return tx._class === core.class.TxCollectionCUD + ? (tx as TxCollectionCUD).tx.objectClass + : (tx as TxCUD).objectClass +} + +function txClass (tx: Tx): string { + return tx._class === core.class.TxCollectionCUD ? (tx as TxCollectionCUD).tx._class : tx._class +} + /** * @public */ diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 4d53e7e75e..a65a64f60b 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import type { Tx, Ref, Doc, Class, Space, Timestamp, Account, FindResult, DocumentQuery, FindOptions, TxResult } from '@anticrm/core' +import type { Tx, Ref, Doc, Class, Space, Timestamp, Account, FindResult, DocumentQuery, FindOptions, TxResult, MeasureContext } from '@anticrm/core' import { TxFactory, Hierarchy } from '@anticrm/core' import type { Resource } from '@anticrm/platform' @@ -88,5 +88,5 @@ export interface Token { * @public */ export interface WithFind { - findAll: (clazz: Ref>, query: DocumentQuery, options?: FindOptions) => Promise> + findAll: (ctx: MeasureContext, clazz: Ref>, query: DocumentQuery, options?: FindOptions) => Promise> } diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts index 897f010960..e701e1092d 100644 --- a/server/elastic/src/adapter.ts +++ b/server/elastic/src/adapter.ts @@ -53,9 +53,7 @@ class ElasticAdapter implements FullTextAdapter { } } }) - console.log(result) const hits = result.body.hits.hits as any[] - console.log('hits', hits) return hits.map(hit => hit._source) } catch (err) { console.error(JSON.stringify(err, null, 2)) @@ -64,34 +62,28 @@ class ElasticAdapter implements FullTextAdapter { } async index (doc: IndexedDoc): Promise { - console.log('eastic: index', doc) if (doc.data === undefined) { try { - const resp = await this.client.index({ + await this.client.index({ index: this.db, id: doc.id, type: '_doc', body: doc }) - console.log('resp', resp) - console.log('error', (resp.meta as any)?.body?.error) } catch (err: any) { - console.log('elastic-exception', err) + console.error('elastic-exception', err) } } else { - console.log('attachment pipeline') try { - const resp = await this.client.index({ + await this.client.index({ index: this.db, id: doc.id, type: '_doc', pipeline: 'attachment', body: doc }) - console.log('resp', resp) - console.log('error', (resp.meta as any)?.body?.error) } catch (err: any) { - console.log('elastic-exception', err) + console.error('elastic-exception', err) } } return {} @@ -99,16 +91,15 @@ class ElasticAdapter implements FullTextAdapter { async update (id: Ref, update: Record): Promise { try { - const resp = await this.client.update({ + await this.client.update({ index: this.db, id, body: { doc: update } }) - console.log('update', resp) } catch (err: any) { - console.log('elastic-exception', err) + console.error('elastic-exception', err) } return {} diff --git a/server/mongo/src/__tests__/storage.test.ts b/server/mongo/src/__tests__/storage.test.ts index 59b118852d..9ddc10f347 100644 --- a/server/mongo/src/__tests__/storage.test.ts +++ b/server/mongo/src/__tests__/storage.test.ts @@ -16,6 +16,7 @@ import core, { Class, Client, + ClientConnection, createClient, Doc, DocumentQuery, @@ -24,13 +25,13 @@ import core, { FindOptions, FindResult, generateId, - Hierarchy, ModelDb, - Ref, + Hierarchy, ModelDb, Ref, SortingOrder, Space, Tx, TxOperations, - TxResult + TxResult, + MeasureMetricsContext } from '@anticrm/core' import { createServerStorage, DbAdapter, DbConfiguration, FullTextAdapter, IndexedDoc } from '@anticrm/server-core' import { MongoClient } from 'mongodb' @@ -155,13 +156,14 @@ describe('mongo operations', () => { workspace: dbId } const serverStorage = await createServerStorage(conf) - + const ctx = new MeasureMetricsContext('client', {}) client = await createClient(async (handler) => { - return { - findAll: async (_class, query, options) => await serverStorage.findAll(_class, query, options), - tx: async (tx) => await serverStorage.tx(tx), + const st: ClientConnection = { + findAll: async (_class, query, options) => await serverStorage.findAll(ctx, _class, query, options), + tx: async (tx) => (await serverStorage.tx(ctx, tx))[0], close: async () => {} } + return st }) operations = new TxOperations(client, core.account.System) diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index 15d4c19048..2fa05e789a 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -13,24 +13,19 @@ // limitations under the License. // -import type { +import core, { Class, Doc, - DocumentQuery, - FindOptions, - FindResult, - Ref, - Tx, + DocumentQuery, DOMAIN_MODEL, DOMAIN_TX, FindOptions, + FindResult, Hierarchy, isOperator, ModelDb, Ref, SortingOrder, Tx, TxCreateDoc, - TxMixin, - TxPutBag, + TxMixin, TxProcessor, TxPutBag, TxRemoveDoc, TxResult, TxUpdateDoc } from '@anticrm/core' -import core, { DOMAIN_MODEL, DOMAIN_TX, Hierarchy, isOperator, ModelDb, SortingOrder, TxProcessor } from '@anticrm/core' import type { DbAdapter, TxAdapter } from '@anticrm/server-core' -import { Db, Document, Filter, Sort } from 'mongodb' +import { Collection, Db, Document, Filter, Sort } from 'mongodb' import { getMongoClient } from './utils' function translateDoc (doc: Doc): Document { @@ -270,12 +265,13 @@ class MongoAdapter extends MongoAdapterBase { } } - override tx (tx: Tx): Promise { - return super.tx(tx) + override async tx (tx: Tx): Promise { + return await super.tx(tx) } } class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { + txColl: Collection | undefined protected txCreateDoc (tx: TxCreateDoc): Promise { throw new Error('Method not implemented.') } @@ -297,10 +293,18 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { } override async tx (tx: Tx): Promise { - await this.db.collection(DOMAIN_TX).insertOne(translateDoc(tx)) + await this.txCollection().insertOne(translateDoc(tx)) return {} } + private txCollection (): Collection { + if (this.txColl !== undefined) { + return this.txColl + } + this.txColl = this.db.collection(DOMAIN_TX) + return this.txColl + } + async getModel (): Promise { return await this.db.collection(DOMAIN_TX).find({ objectSpace: core.space.Model }).sort({ _id: 1 }).toArray() } diff --git a/server/server/package.json b/server/server/package.json index ea3a636123..8f30716701 100644 --- a/server/server/package.json +++ b/server/server/package.json @@ -43,6 +43,7 @@ "@anticrm/server-recruit": "~0.6.0", "@anticrm/server-recruit-resources": "~0.6.0", "@anticrm/mongo": "~0.6.1", - "@anticrm/elastic": "~0.6.0" + "@anticrm/elastic": "~0.6.0", + "elastic-apm-node": "~3.26.0" } } diff --git a/server/server/src/__start.ts b/server/server/src/__start.ts index 58763c7a1a..a8caa1c26b 100644 --- a/server/server/src/__start.ts +++ b/server/server/src/__start.ts @@ -14,6 +14,7 @@ // limitations under the License. // +// Add this to the VERY top of the first file loaded in your app import { start } from '.' const url = process.env.MONGO_URL @@ -39,4 +40,3 @@ const close = (): void => { } process.on('SIGINT', close) process.on('SIGTERM', close) -process.on('exit', close) diff --git a/server/server/src/apm.ts b/server/server/src/apm.ts new file mode 100644 index 0000000000..cb4caa34fb --- /dev/null +++ b/server/server/src/apm.ts @@ -0,0 +1,85 @@ +import { MeasureContext, MeasureLogger, ParamType } from '@anticrm/core' +import apm, { Agent, Span, Transaction } from 'elastic-apm-node' + +export let metricsContext: MeasureContext + +/** + * @public + */ +export function createAPMAgent (apmUrl: string): Agent { + const agent: Agent = apm.start({ + + // Override the service name from package.json + // Allowed characters: a-z, A-Z, 0-9, -, _, and space + serviceName: 'transactor', + + // Use if APM Server requires a secret token + secretToken: '', + + // Set the custom APM Server URL (default: http://localhost:8200) + serverUrl: apmUrl, + logLevel: 'trace' + }) + return agent +} + +/** + * @public + */ +export class APMMeasureContext implements MeasureContext { + logger: MeasureLogger + private readonly transaction?: Transaction | Span + private readonly parent?: Transaction | Span + constructor (private readonly agent: Agent, name: string, params: Record, parent?: Transaction | Span, noTransaction?: boolean) { + this.parent = parent + this.logger = { + info: (msg, args) => { + agent.logger.info(msg, args) + }, + error: (msg, args) => { + agent.logger.error(msg, args) + } + } + if (!(noTransaction ?? false)) { + if (this.parent === undefined) { + this.transaction = agent.startTransaction(name) ?? undefined + } else { + this.transaction = agent.startSpan(name, { childOf: this.parent }) ?? undefined + } + for (const [k, v] of Object.entries(params)) { + this.transaction?.setLabel(k, v) + } + } + } + + newChild (name: string, params: Record): MeasureContext { + return new APMMeasureContext(this.agent, name, params, this.transaction) + } + + async with(name: string, params: Record, op: (ctx: MeasureContext) => T | Promise): Promise { + const c = this.newChild(name, params) + try { + let value = op(c) + if (value instanceof Promise) { + value = await value + } + c.end() + return value + } catch (err: any) { + await c.error(err) + throw err + } + } + + async error (err: any): Promise { + return await new Promise((resolve) => { + this.agent.captureError(err, () => { + resolve() + }) + }) + } + + end (): void { + this.transaction?.end() + } +} diff --git a/server/server/src/metrics.ts b/server/server/src/metrics.ts new file mode 100644 index 0000000000..e41164de1a --- /dev/null +++ b/server/server/src/metrics.ts @@ -0,0 +1,46 @@ +import { MeasureContext, MeasureMetricsContext, metricsToString, newMetrics } from '@anticrm/core' +import { APMMeasureContext, createAPMAgent } from './apm' +import { writeFile } from 'fs/promises' + +const apmUrl = process.env.APM_SERVER_URL +const metricsFile = process.env.METRICS_FILE +const metricsConsole = (process.env.METRICS_CONSOLE ?? 'false') === 'true' + +const METRICS_UPDATE_INTERVAL = 30000 + +export let metricsContext: MeasureContext + +if (apmUrl === undefined) { + console.info('please provide apm server url for monitoring') + + const metrics = newMetrics() + metricsContext = new MeasureMetricsContext('System', {}, metrics) + + if (metricsFile !== undefined || metricsConsole) { + console.info('storing measurements into local file', metricsFile) + let oldMetricsValue = '' + + const intTimer = setInterval(() => { + const val = metricsToString(metrics) + if (val !== oldMetricsValue) { + oldMetricsValue = val + if (metricsFile !== undefined) { + writeFile(metricsFile, val).catch((err) => console.error(err)) + } + if (metricsConsole) { + console.info('METRICS:', val) + } + } + }, METRICS_UPDATE_INTERVAL) + + const closeTimer = (): void => { + clearInterval(intTimer) + } + process.on('SIGINT', closeTimer) + process.on('SIGTERM', closeTimer + ) + } +} else { + console.log('using APM', apmUrl) + metricsContext = new APMMeasureContext(createAPMAgent(apmUrl), 'root', {}, undefined, true) +} diff --git a/server/server/src/server.ts b/server/server/src/server.ts index f2f8e41e65..1680eb4b38 100644 --- a/server/server/src/server.ts +++ b/server/server/src/server.ts @@ -24,6 +24,7 @@ import type { DbConfiguration, DbAdapter } from '@anticrm/server-core' import { addLocation } from '@anticrm/platform' import { serverChunterId } from '@anticrm/server-chunter' import { serverRecruitId } from '@anticrm/server-recruit' +import { metricsContext } from './metrics' class NullDbAdapter implements DbAdapter { async init (model: Tx[]): Promise {} @@ -42,7 +43,7 @@ export function start (dbUrl: string, fullTextUrl: string, port: number, host?: addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources')) addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources')) - return startJsonRpc((workspace: string) => { + return startJsonRpc(metricsContext, (workspace: string) => { const conf: DbConfiguration = { domains: { [DOMAIN_TX]: 'MongoTx', diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index 82d95a9b2b..6c71bb2b2d 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -20,18 +20,20 @@ import type { Token } from '@anticrm/server-core' import { encode } from 'jwt-simple' import WebSocket from 'ws' -import type { Doc, Ref, Class, DocumentQuery, FindOptions, FindResult, Tx, TxResult } from '@anticrm/core' +import type { Doc, Ref, Class, DocumentQuery, FindOptions, FindResult, Tx, TxResult, MeasureContext } from '@anticrm/core' +import { MeasureMetricsContext } from '@anticrm/core' describe('server', () => { disableLogging() - start(async () => ({ + start(new MeasureMetricsContext('test', {}), async () => ({ findAll: async ( + ctx: MeasureContext, _class: Ref>, query: DocumentQuery, options?: FindOptions ): Promise> => ([]), - tx: async (tx: Tx): Promise<[TxResult, Tx[]]> => ([{}, []]) + tx: async (ctx: MeasureContext, tx: Tx): Promise<[TxResult, Tx[]]> => ([{}, []]) }), 3333) function connect (): WebSocket { diff --git a/server/ws/src/server.ts b/server/ws/src/server.ts index 8c74d4c39c..1daf73c3b9 100644 --- a/server/ws/src/server.ts +++ b/server/ws/src/server.ts @@ -14,19 +14,18 @@ // limitations under the License. // -import { readRequest, serialize, Response } from '@anticrm/platform' +import { Class, Doc, DocumentQuery, FindOptions, FindResult, MeasureContext, Ref, ServerStorage, Tx, TxResult } from '@anticrm/core' +import { readRequest, Response, serialize, unknownError } from '@anticrm/platform' import type { Token } from '@anticrm/server-core' import { createServer, IncomingMessage } from 'http' -import WebSocket, { Server } from 'ws' import { decode } from 'jwt-simple' +import WebSocket, { Server } from 'ws' -import type { Doc, Ref, Class, FindOptions, FindResult, Tx, DocumentQuery, Storage, ServerStorage, TxResult } from '@anticrm/core' - -let LOGGING_ENABLED = true +let LOGGING_ENABLED = false export function disableLogging (): void { LOGGING_ENABLED = false } -class Session implements Storage { +class Session { constructor ( private readonly manager: SessionManager, private readonly token: Token, @@ -35,15 +34,16 @@ class Session implements Storage { async ping (): Promise { console.log('ping'); return 'pong!' } - async findAll (_class: Ref>, query: DocumentQuery, options?: FindOptions): Promise> { - return await this.storage.findAll(_class, query, options) + async findAll (ctx: MeasureContext, _class: Ref>, query: DocumentQuery, options?: FindOptions): Promise> { + return await this.storage.findAll(ctx, _class, query, options) } - async tx (tx: Tx): Promise { - const [result, derived] = await this.storage.tx(tx) + async tx (ctx: MeasureContext, tx: Tx): Promise { + const [result, derived] = await this.storage.tx(ctx, tx) + this.manager.broadcast(this, this.token, { result: tx }) - for (const tx of derived) { - this.manager.broadcast(null, this.token, { result: tx }) + for (const dtx of derived) { + this.manager.broadcast(null, this.token, { result: dtx }) } return result } @@ -101,15 +101,19 @@ class SessionManager { } } -async function handleRequest (service: S, ws: WebSocket, msg: string): Promise { +async function handleRequest (ctx: MeasureContext, service: S, ws: WebSocket, msg: string): Promise { const request = readRequest(msg) const f = (service as any)[request.method] try { - const result = await f.apply(service, request.params) - const resp = { id: request.id, result } + const params = [ctx, ...request.params] + const result = await f.apply(service, params) + const resp: Response = { id: request.id, result } ws.send(serialize(resp)) } catch (err: any) { - const resp = { id: request.id, error: err } + const resp: Response = { + id: request.id, + error: unknownError(err) + } ws.send(serialize(resp)) } } @@ -120,7 +124,7 @@ async function handleRequest (service: S, ws: WebSocket, msg: string): Promis * @param port - * @param host - */ -export function start (storageFactory: (workspace: string) => Promise, port: number, host?: string): () => void { +export function start (ctx: MeasureContext, storageFactory: (workspace: string) => Promise, port: number, host?: string): () => void { console.log(`starting server on port ${port} ...`) const sessions = new SessionManager() @@ -133,11 +137,11 @@ export function start (storageFactory: (workspace: string) => Promise { buffer.push(msg) }) const session = await sessions.addSession(ws, token, storageFactory) // eslint-disable-next-line @typescript-eslint/no-misused-promises - ws.on('message', async (msg: string) => await handleRequest(session, ws, msg)) + ws.on('message', async (msg: string) => await handleRequest(ctx, session, ws, msg)) ws.on('close', (code: number, reason: string) => sessions.close(ws, token, code, reason)) for (const msg of buffer) { - await handleRequest(session, ws, msg) + await handleRequest(ctx, session, ws, msg) } })