Performance metrics (#619)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -11,7 +11,9 @@
       "args": ["src/__start.ts"],
       "env": {
         "ELASTIC_URL": "http://localhost:9200",
-        "MONGO_URL": "mongodb://localhost:27017"
+        "MONGO_URL": "mongodb://localhost:27017",
+        "APM_SERVER_URL2": "http://localhost:8200",
+        "METRICS_CONSOLE": "true" // Show metrics in console evert 30 seconds.
       },
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "sourceMaps": true,
@@ -45,5 +47,25 @@
       "cwd": "${workspaceRoot}/dev/generator",
       "protocol": "inspector"
     },    
+    {
+      "name": "Debug tool",
+      "type": "node",
+      "request": "launch",
+      // "args": ["src/index.ts", "import-xml", "ws1", "/Users/haiodo/Develop/private/hardware/suho/Кандидаты/Кандидаты.xml"],
+      "args": ["src/index.ts", "restore-workspace", "ws1", "../../temp/ws1/"],
+      "env": {
+        "MINIO_ACCESS_KEY":"minioadmin",
+        "MINIO_SECRET_KEY":"minioadmin",
+        "MINIO_ENDPOINT":"localhost",
+        "MONGO_URL":"mongodb://localhost:27017",
+        "TRANSACTOR_URL":"ws:/localhost:3333",
+        "TELEGRAM_DATABASE":"telegram-service",
+        "ELASTIC_URL":"http://localhost:9200",
+      },
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register" ],
+      "sourceMaps": true,
+      "cwd": "${workspaceRoot}/dev/tool",
+      "protocol": "inspector"
+    },    
   ]
 }

common/config/rush/command-line.json

@@ -236,6 +236,14 @@
       "safeForSimultaneousRushProcesses": true,
       "shellCommand": "node templates/apply.js"
     },
+    {
+      "commandKind": "global",
+      "name": "ts-clean",
+      "summary": "Clean tsconfig.tsbuildinfo",
+      "description": "Clean typescript incremental cache",
+      "safeForSimultaneousRushProcesses": true,
+      "shellCommand": "find .|grep tsconfig.tsbuildinfo | xargs rm | pwd"
+    },    
   ],
 
   /**

common/config/rush/pnpm-lock.yaml

@@ -125,6 +125,7 @@ specifiers:
   css-loader: ^5.2.1
   deep-equal: ^2.0.5
   dotenv-webpack: ^7.0.2
+  elastic-apm-node: ~3.26.0
   eslint: ^7.32.0
   eslint-plugin-import: ^2.25.3
   eslint-plugin-node: ^11.1.0
@@ -293,6 +294,7 @@ dependencies:
   css-loader: 5.2.7_webpack@5.65.0
   deep-equal: 2.0.5
   dotenv-webpack: 7.0.3_webpack@5.65.0
+  elastic-apm-node: 3.26.0
   eslint: 7.32.0
   eslint-plugin-import: 2.25.3_eslint@7.32.0
   eslint-plugin-node: 11.1.0_eslint@7.32.0
@@ -673,6 +675,20 @@ packages:
     engines: {node: '>=10.0.0'}
     dev: false
 
+  /@elastic/ecs-helpers/1.1.0:
+    resolution: {integrity: sha512-MDLb2aFeGjg46O5mLpdCzT5yOUDnXToJSrco2ShqGIXxNJaM8uJjX+4nd+hRYV4Vex8YJyDtOFEVBldQct6ndg==}
+    engines: {node: '>=10'}
+    dependencies:
+      fast-json-stringify: 2.7.12
+    dev: false
+
+  /@elastic/ecs-pino-format/1.3.0:
+    resolution: {integrity: sha512-U8D57gPECYoRCcwREsrXKBtqeyFFF/KAwHi4rG1u/oQhAg91Kzw8ZtUQJXD/DMDieLOqtbItFr2FRBWI3t3wog==}
+    engines: {node: '>=10'}
+    dependencies:
+      '@elastic/ecs-helpers': 1.1.0
+    dev: false
+
   /@elastic/elasticsearch/7.16.0:
     resolution: {integrity: sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==}
     engines: {node: '>=12'}
@@ -2323,6 +2339,10 @@ packages:
     hasBin: true
     dev: false
 
+  /after-all-results/2.0.0:
+    resolution: {integrity: sha1-asL8ICtQD4jaj09VMM+hAPTGotA=}
+    dev: false
+
   /ajv-errors/1.0.1_ajv@6.12.6:
     resolution: {integrity: sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==}
     peerDependencies:
@@ -2584,6 +2604,12 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /async-cache/1.1.0:
+    resolution: {integrity: sha1-SppaidBl7F2OUlS9nulrp2xTK1o=}
+    dependencies:
+      lru-cache: 4.1.5
+    dev: false
+
   /async-each/1.0.3:
     resolution: {integrity: sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==}
     dev: false
@@ -2592,6 +2618,16 @@ packages:
     resolution: {integrity: sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==}
     dev: false
 
+  /async-value-promise/1.1.1:
+    resolution: {integrity: sha512-c2RFDKjJle1rHa0YxN9Ysu97/QBu3Wa+NOejJxsX+1qVDJrkD3JL/GN1B3gaILAEXJXbu/4Z1lcoCHFESe/APA==}
+    dependencies:
+      async-value: 1.2.2
+    dev: false
+
+  /async-value/1.2.2:
+    resolution: {integrity: sha1-hFF6Hny2saW14YH6Mb4QQ3t/sSU=}
+    dev: false
+
   /async/2.6.3:
     resolution: {integrity: sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==}
     dependencies:
@@ -2612,6 +2648,11 @@ packages:
     hasBin: true
     dev: false
 
+  /atomic-sleep/1.0.0:
+    resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
+    engines: {node: '>=8.0.0'}
+    dev: false
+
   /autoprefixer/10.4.0_postcss@8.4.5:
     resolution: {integrity: sha512-7FdJ1ONtwzV1G43GDD0kpVMn/qbiNqyOPMFTX5nRffI+7vgWoFEc6DcXOxHJxrWNDXrZh18eDsZjvZGUljSRGA==}
     engines: {node: ^10 || ^12 || >=14}
@@ -2745,6 +2786,13 @@ packages:
     resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
     dev: false
 
+  /basic-auth/2.0.1:
+    resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      safe-buffer: 5.1.2
+    dev: false
+
   /batch/0.6.1:
     resolution: {integrity: sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=}
     dev: false
@@ -2769,6 +2817,10 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /binary-search/1.3.6:
+    resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==}
+    dev: false
+
   /bindings/1.5.0:
     resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
     requiresBuild: true
@@ -2852,6 +2904,12 @@ packages:
       fill-range: 7.0.1
     dev: false
 
+  /breadth-filter/2.0.0:
+    resolution: {integrity: sha512-thQShDXnFWSk2oVBixRCyrWsFoV5tfOpWKHmxwafHQDNxCfDBk539utpvytNjmlFrTMqz41poLwJvA1MW3z0MQ==}
+    dependencies:
+      object.entries: 1.1.5
+    dev: false
+
   /brfs/2.0.2:
     resolution: {integrity: sha512-IrFjVtwu4eTJZyu8w/V2gxU7iLTtcHih67sgEdzrhjLBMHp2uYefUBfdM4k2UvcuWMgV7PQDZHSLeNWnLFKWVQ==}
     hasBin: true
@@ -3329,6 +3387,14 @@ packages:
     engines: {node: '>=0.8'}
     dev: false
 
+  /console-log-level/1.4.1:
+    resolution: {integrity: sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==}
+    dev: false
+
+  /container-info/1.1.0:
+    resolution: {integrity: sha512-eD2zLAmxGS2kmL4f1jY8BdOqnmpL6X70kvzTBW/9FIQnxoxiBJ4htMsTmtPLPWRs7NHYFvqKQ1VtppV08mdsQA==}
+    dev: false
+
   /content-disposition/0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
     engines: {node: '>= 0.6'}
@@ -3908,6 +3974,61 @@ packages:
     resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=}
     dev: false
 
+  /elastic-apm-http-client/10.3.0:
+    resolution: {integrity: sha512-BAqB7k5JA/x09L8BVj04WRoknRptmW2rLAoHQVrPvPhUm/IgNz63wPfiBuhWVE//Hl7xEpURO5pMV6az0UArkA==}
+    engines: {node: ^8.6.0 || 10 || >=12}
+    dependencies:
+      breadth-filter: 2.0.0
+      container-info: 1.1.0
+      end-of-stream: 1.4.4
+      fast-safe-stringify: 2.1.1
+      fast-stream-to-buffer: 1.0.0
+      object-filter-sequence: 1.0.0
+      readable-stream: 3.6.0
+      stream-chopper: 3.0.1
+    dev: false
+
+  /elastic-apm-node/3.26.0:
+    resolution: {integrity: sha512-MwYFlBTlcHI8GGQXLnnEm70JJ4RRFkHCY1D3Wt2027l8T/Ye5tgssMSiKyRbjb9bVdibbte73Xn8HF8i35UaxA==}
+    engines: {node: ^8.6.0 || 10 || 12 || 14 || 15 || 16 || 17}
+    dependencies:
+      '@elastic/ecs-pino-format': 1.3.0
+      after-all-results: 2.0.0
+      async-cache: 1.1.0
+      async-value-promise: 1.1.1
+      basic-auth: 2.0.1
+      cookie: 0.4.1
+      core-util-is: 1.0.3
+      elastic-apm-http-client: 10.3.0
+      end-of-stream: 1.4.4
+      error-callsites: 2.0.4
+      error-stack-parser: 2.0.6
+      escape-string-regexp: 4.0.0
+      fast-safe-stringify: 2.1.1
+      http-headers: 3.0.2
+      is-native: 1.0.1
+      load-source-map: 2.0.0
+      lru-cache: 6.0.0
+      measured-reporting: 1.51.1
+      monitor-event-loop-delay: 1.0.0
+      object-filter-sequence: 1.0.0
+      object-identity-map: 1.0.2
+      original-url: 1.2.3
+      pino: 6.13.3
+      read-pkg-up: 7.0.1
+      relative-microtime: 2.0.0
+      require-in-the-middle: 5.1.0
+      semver: 6.3.0
+      set-cookie-serde: 1.0.0
+      shallow-clone-shim: 2.0.0
+      sql-summary: 1.0.1
+      traceparent: 1.0.0
+      traverse: 0.6.6
+      unicode-byte-truncate: 1.0.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /electron-to-chromium/1.4.24:
     resolution: {integrity: sha512-erwx5r69B/WFfFuF2jcNN0817BfDBdC4765kQ6WltOMuwsimlQo3JTEq0Cle+wpHralwdeX3OfAtw/mHxPK0Wg==}
     dev: false
@@ -3980,12 +4101,23 @@ packages:
       prr: 1.0.1
     dev: false
 
+  /error-callsites/2.0.4:
+    resolution: {integrity: sha512-V877Ch4FC4FN178fDK1fsrHN4I1YQIBdtjKrHh3BUHMnh3SMvwUVrqkaOgDpUuevgSNna0RBq6Ox9SGlxYrigA==}
+    engines: {node: '>=6.x'}
+    dev: false
+
   /error-ex/1.3.2:
     resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==}
     dependencies:
       is-arrayish: 0.2.1
     dev: false
 
+  /error-stack-parser/2.0.6:
+    resolution: {integrity: sha512-d51brTeqC+BHlwF0BhPtcYgF5nlzf9ZZ0ZIUQNZpc9ZB9qw5IJ2diTrBY9jlCJkTLITYPjmiX6OWCwH+fuyNgQ==}
+    dependencies:
+      stackframe: 1.2.0
+    dev: false
+
   /es-abstract/1.19.1:
     resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==}
     engines: {node: '>= 0.4'}
@@ -4684,10 +4816,35 @@ packages:
     resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
     dev: false
 
+  /fast-json-stringify/2.7.12:
+    resolution: {integrity: sha512-4hjwZDPmgj/ZUKXhEWovGPciE/5mWtAIQQxN+2VBDFun7DRTk2oOItbu9ZZp6kqj+eZ/u7z+dgBgM74cfGRnBQ==}
+    engines: {node: '>= 10.0.0'}
+    dependencies:
+      ajv: 6.12.6
+      deepmerge: 4.2.2
+      rfdc: 1.3.0
+      string-similarity: 4.0.4
+    dev: false
+
   /fast-levenshtein/2.0.6:
     resolution: {integrity: sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=}
     dev: false
 
+  /fast-redact/3.0.2:
+    resolution: {integrity: sha512-YN+CYfCVRVMUZOUPeinHNKgytM1wPI/C/UCLEi56EsY2dwwvI00kIJHJoI7pMVqGoMew8SMZ2SSfHKHULHXDsg==}
+    engines: {node: '>=6'}
+    dev: false
+
+  /fast-safe-stringify/2.1.1:
+    resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
+    dev: false
+
+  /fast-stream-to-buffer/1.0.0:
+    resolution: {integrity: sha512-bI/544WUQlD2iXBibQbOMSmG07Hay7YrpXlKaeGTPT7H7pC0eitt3usak5vUwEvCGK/O7rUAM3iyQValGU22TQ==}
+    dependencies:
+      end-of-stream: 1.4.4
+    dev: false
+
   /fast-xml-parser/3.21.1:
     resolution: {integrity: sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg==}
     hasBin: true
@@ -4699,6 +4856,10 @@ packages:
     resolution: {integrity: sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow==}
     dev: false
 
+  /fastify-warning/0.2.0:
+    resolution: {integrity: sha512-s1EQguBw/9qtc1p/WTY4eq9WMRIACkj+HTcOIK1in4MV5aFaQC9ZCIt0dJ7pr5bIf4lPpHvAtP2ywpTNgs7hqw==}
+    dev: false
+
   /fastq/1.13.0:
     resolution: {integrity: sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==}
     dependencies:
@@ -4807,6 +4968,10 @@ packages:
       rimraf: 3.0.2
     dev: false
 
+  /flatstr/1.0.12:
+    resolution: {integrity: sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==}
+    dev: false
+
   /flatted/3.2.4:
     resolution: {integrity: sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==}
     dev: false
@@ -4861,6 +5026,10 @@ packages:
       mime-types: 2.1.34
     dev: false
 
+  /forwarded-parse/2.1.2:
+    resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==}
+    dev: false
+
   /forwarded/0.2.0:
     resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
     engines: {node: '>= 0.6'}
@@ -5268,6 +5437,12 @@ packages:
       toidentifier: 1.0.1
     dev: false
 
+  /http-headers/3.0.2:
+    resolution: {integrity: sha512-87E1I+2Wg4dxxz4rcxElo3dxO/w1ZtgL1yA0Sb6vH3qU16vRKq1NjWQv9SCY3ly2OQROcoxHZOUpmelS+k6wOw==}
+    dependencies:
+      next-line: 1.1.0
+    dev: false
+
   /http-parser-js/0.5.5:
     resolution: {integrity: sha512-x+JVEkO2PoM8qqpbPbOL3cqHPwerep7OwzK7Ay+sMQjKzaKCqWvjoXm5tqMP9tXWWTnTzAjIhXg+J99XYuPhPA==}
     dev: false
@@ -5612,6 +5787,11 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /is-finite/1.1.0:
+    resolution: {integrity: sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   /is-fullwidth-code-point/2.0.0:
     resolution: {integrity: sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=}
     engines: {node: '>=4'}
@@ -5648,15 +5828,32 @@ packages:
       is-extglob: 2.1.1
     dev: false
 
+  /is-integer/1.0.7:
+    resolution: {integrity: sha1-a96Bqs3feLZZtmKdYpytxRqIbVw=}
+    dependencies:
+      is-finite: 1.1.0
+    dev: false
+
   /is-map/2.0.2:
     resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==}
     dev: false
 
+  /is-native/1.0.1:
+    resolution: {integrity: sha1-zRjMFi6EUNaDtbq+eayZwUVElnU=}
+    dependencies:
+      is-nil: 1.0.1
+      to-source-code: 1.0.2
+    dev: false
+
   /is-negative-zero/2.0.2:
     resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==}
     engines: {node: '>= 0.4'}
     dev: false
 
+  /is-nil/1.0.1:
+    resolution: {integrity: sha1-LauingtYUGOHXntTnQcfWxWTeWk=}
+    dev: false
+
   /is-number-object/1.0.6:
     resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==}
     engines: {node: '>= 0.4'}
@@ -6619,6 +6816,13 @@ packages:
     resolution: {integrity: sha512-JWw1HHMx54g8mEsG7JwI8I/xh7qeJbF6L9u1dQOYW91RdRqDYpnTn1UaNXYkmLD967Vk0xGuyHzuRnkSApby3w==}
     dev: false
 
+  /load-source-map/2.0.0:
+    resolution: {integrity: sha512-QNZzJ2wMrTmCdeobMuMNEXHN1QGk8HG6louEkzD/zwQ7EU2RarrzlhQ4GnUYEFzLhK+Jq7IGyF/qy+XYBSO7AQ==}
+    engines: {node: '>= 8'}
+    dependencies:
+      source-map: 0.7.3
+    dev: false
+
   /loader-runner/4.2.0:
     resolution: {integrity: sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==}
     engines: {node: '>=6.11.5'}
@@ -6700,6 +6904,13 @@ packages:
       '@sinonjs/commons': 1.8.3
     dev: false
 
+  /lru-cache/4.1.5:
+    resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==}
+    dependencies:
+      pseudomap: 1.0.2
+      yallist: 2.1.2
+    dev: false
+
   /lru-cache/6.0.0:
     resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
     engines: {node: '>=10'}
@@ -6748,6 +6959,10 @@ packages:
       object-visit: 1.0.1
     dev: false
 
+  /mapcap/1.0.0:
+    resolution: {integrity: sha512-KcNlZSlFPx+r1jYZmxEbTVymG+dIctf10WmWkuhrhrblM+KMoF77HelwihL5cxYlORye79KoR4IlOOk99lUJ0g==}
+    dev: false
+
   /md5.js/1.3.5:
     resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==}
     dependencies:
@@ -6760,6 +6975,24 @@ packages:
     resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==}
     dev: false
 
+  /measured-core/1.51.1:
+    resolution: {integrity: sha512-DZQP9SEwdqqYRvT2slMK81D/7xwdxXosZZBtLVfPSo6y5P672FBTbzHVdN4IQyUkUpcVOR9pIvtUy5Ryl7NKyg==}
+    engines: {node: '>= 5.12'}
+    dependencies:
+      binary-search: 1.3.6
+      optional-js: 2.3.0
+    dev: false
+
+  /measured-reporting/1.51.1:
+    resolution: {integrity: sha512-JCt+2u6XT1I5lG3SuYqywE0e62DJuAzBcfMzWGUhIYtPQV2Vm4HiYt/durqmzsAbZV181CEs+o/jMKWJKkYIWw==}
+    engines: {node: '>= 5.12'}
+    dependencies:
+      console-log-level: 1.4.1
+      mapcap: 1.0.0
+      measured-core: 1.51.1
+      optional-js: 2.3.0
+    dev: false
+
   /media-typer/0.3.0:
     resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=}
     engines: {node: '>= 0.6'}
@@ -6935,6 +7168,10 @@ packages:
       minimist: 1.2.5
     dev: false
 
+  /module-details-from-path/1.0.3:
+    resolution: {integrity: sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=}
+    dev: false
+
   /mongodb-connection-string-url/2.3.2:
     resolution: {integrity: sha512-2LkmS0ny7LamAyhEs2Q+zuFFxeGNSc2DaGHBevjqkoPt7bgh+67mg1sFU6awnMsdLKpdEt7zUy466K9x7RsYcQ==}
     dependencies:
@@ -6953,6 +7190,10 @@ packages:
       saslprep: 1.0.3
     dev: false
 
+  /monitor-event-loop-delay/1.0.0:
+    resolution: {integrity: sha512-YRIr1exCIfBDLZle8WHOfSo7Xg3M+phcZfq9Fx1L6Abo+atGp7cge5pM7PjyBn4s1oZI/BRD4EMrzQBbPpVb5Q==}
+    dev: false
+
   /mri/1.2.0:
     resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==}
     engines: {node: '>=4'}
@@ -7029,6 +7270,10 @@ packages:
     resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==}
     dev: false
 
+  /next-line/1.1.0:
+    resolution: {integrity: sha1-/K5XhTBStqm66CCOQN19PC0wRgM=}
+    dev: false
+
   /next-tick/1.0.0:
     resolution: {integrity: sha1-yobR/ogoFpsBICCOPchCS524NCw=}
     dev: false
@@ -7137,6 +7382,16 @@ packages:
       kind-of: 3.2.2
     dev: false
 
+  /object-filter-sequence/1.0.0:
+    resolution: {integrity: sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==}
+    dev: false
+
+  /object-identity-map/1.0.2:
+    resolution: {integrity: sha512-a2XZDGyYTngvGS67kWnqVdpoaJWsY7C1GhPJvejWAFCsUioTAaiTu8oBad7c6cI4McZxr4CmvnZeycK05iav5A==}
+    dependencies:
+      object.entries: 1.1.5
+    dev: false
+
   /object-inspect/1.12.0:
     resolution: {integrity: sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==}
     dev: false
@@ -7171,6 +7426,15 @@ packages:
       object-keys: 1.1.1
     dev: false
 
+  /object.entries/1.1.5:
+    resolution: {integrity: sha512-TyxmjUoZggd4OrrU1W66FMDG6CuqJxsFvymeyXI51+vQLN67zYfZseptRge703kKQdo4uccgAKebXFcRCzk4+g==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+      es-abstract: 1.19.1
+    dev: false
+
   /object.pick/1.3.0:
     resolution: {integrity: sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=}
     engines: {node: '>=0.10.0'}
@@ -7232,6 +7496,10 @@ packages:
       is-wsl: 1.1.0
     dev: false
 
+  /optional-js/2.3.0:
+    resolution: {integrity: sha512-B0LLi+Vg+eko++0z/b8zIv57kp7HKEzaPJo7LowJXMUKYdf+3XJGu/cw03h/JhIOsLnP+cG5QnTHAuicjA5fMw==}
+    dev: false
+
   /optionator/0.8.3:
     resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==}
     engines: {node: '>= 0.8.0'}
@@ -7260,6 +7528,12 @@ packages:
     resolution: {integrity: sha512-3Ux8um0zXbVacKUkcytc0u3HgC0b0bBLT+I60r2J/En72cI0nZffqrA7Xtf2Hqs27j1g82llR5Mhbd0Z1XW4AQ==}
     dev: false
 
+  /original-url/1.2.3:
+    resolution: {integrity: sha512-BYm+pKYLtS4mVe/mgT3YKGtWV5HzN/XKiaIu1aK4rsxyjuHeTW9N+xVBEpJcY1onB3nccfH0RbzUEoimMqFUHQ==}
+    dependencies:
+      forwarded-parse: 2.1.2
+    dev: false
+
   /original/1.0.2:
     resolution: {integrity: sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg==}
     dependencies:
@@ -7488,6 +7762,23 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /pino-std-serializers/3.2.0:
+    resolution: {integrity: sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==}
+    dev: false
+
+  /pino/6.13.3:
+    resolution: {integrity: sha512-tJy6qVgkh9MwNgqX1/oYi3ehfl2Y9H0uHyEEMsBe74KinESIjdMrMQDWpcZPpPicg3VV35d/GLQZmo4QgU2Xkg==}
+    hasBin: true
+    dependencies:
+      fast-redact: 3.0.2
+      fast-safe-stringify: 2.1.1
+      fastify-warning: 0.2.0
+      flatstr: 1.0.12
+      pino-std-serializers: 3.2.0
+      quick-format-unescaped: 4.0.4
+      sonic-boom: 1.4.1
+    dev: false
+
   /pirates/4.0.4:
     resolution: {integrity: sha512-ZIrVPH+A52Dw84R0L3/VS9Op04PuQ2SEoJL6bkshmiTic/HldyW9Tf7oH5mhJZBK7NmDx27vSMrYEXPXclpDKw==}
     engines: {node: '>= 6'}
@@ -7782,6 +8073,10 @@ packages:
     resolution: {integrity: sha1-0/wRS6BplaRexok/SEzrHXj19HY=}
     dev: false
 
+  /pseudomap/1.0.2:
+    resolution: {integrity: sha1-8FKijacOYYkX7wqKw0wa5aaChrM=}
+    dev: false
+
   /psl/1.8.0:
     resolution: {integrity: sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==}
     dev: false
@@ -7844,6 +8139,10 @@ packages:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
     dev: false
 
+  /quick-format-unescaped/4.0.4:
+    resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}
+    dev: false
+
   /quote-stream/1.0.2:
     resolution: {integrity: sha1-hJY/jJwmuULhU/7rU6rnRlK34LI=}
     hasBin: true
@@ -7853,6 +8152,10 @@ packages:
       through2: 2.0.5
     dev: false
 
+  /random-poly-fill/1.0.1:
+    resolution: {integrity: sha512-bMOL0hLfrNs52+EHtIPIXxn2PxYwXb0qjnKruTjXiM/sKfYqj506aB2plFwWW1HN+ri724bAVVGparh4AtlJKw==}
+    dev: false
+
   /randombytes/2.1.0:
     resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}
     dependencies:
@@ -7989,6 +8292,10 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /relative-microtime/2.0.0:
+    resolution: {integrity: sha512-l18ha6HEZc+No/uK4GyAnNxgKW7nvEe35IaeN54sShMojtqik2a6GbTyuiezkjpPaqP874Z3lW5ysBo5irz4NA==}
+    dev: false
+
   /remove-trailing-separator/1.1.0:
     resolution: {integrity: sha1-wkvOKig62tW8P1jg1IJJuSN52O8=}
     dev: false
@@ -8063,6 +8370,16 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /require-in-the-middle/5.1.0:
+    resolution: {integrity: sha512-M2rLKVupQfJ5lf9OvqFGIT+9iVLnTmjgbOmpil12hiSQNn5zJTKGPoIisETNjfK+09vP3rpm1zJajmErpr2sEQ==}
+    dependencies:
+      debug: 4.3.3
+      module-details-from-path: 1.0.3
+      resolve: 1.20.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /require-main-filename/2.0.0:
     resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==}
     dev: false
@@ -8150,6 +8467,10 @@ packages:
     engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
     dev: false
 
+  /rfdc/1.3.0:
+    resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==}
+    dev: false
+
   /rimraf/2.7.1:
     resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==}
     hasBin: true
@@ -8413,6 +8734,10 @@ packages:
     resolution: {integrity: sha1-BF+XgtARrppoA93TgrJDkrPYkPc=}
     dev: false
 
+  /set-cookie-serde/1.0.0:
+    resolution: {integrity: sha512-Vq8e5GsupfJ7okHIvEPcfs5neCo7MZ1ZuWrO3sllYi3DOWt6bSSCpADzqXjz3k0fXehnoFIrmmhty9IN6U6BXQ==}
+    dev: false
+
   /set-value/2.0.1:
     resolution: {integrity: sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==}
     engines: {node: '>=0.10.0'}
@@ -8439,6 +8764,10 @@ packages:
       safe-buffer: 5.2.1
     dev: false
 
+  /shallow-clone-shim/2.0.0:
+    resolution: {integrity: sha512-YRNymdiL3KGOoS67d73TEmk4tdPTO9GSMCoiphQsTcC9EtC+AOmMPjkyBkRoCJfW9ASsaZw1craaiw1dPN2D3Q==}
+    dev: false
+
   /shallow-clone/3.0.1:
     resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==}
     engines: {node: '>=8'}
@@ -8571,6 +8900,13 @@ packages:
       websocket-driver: 0.7.4
     dev: false
 
+  /sonic-boom/1.4.1:
+    resolution: {integrity: sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==}
+    dependencies:
+      atomic-sleep: 1.0.0
+      flatstr: 1.0.12
+    dev: false
+
   /sorcery/0.10.0:
     resolution: {integrity: sha1-iukK19fLBfxZ8asMY3hF1cFaUrc=}
     hasBin: true
@@ -8701,6 +9037,10 @@ packages:
     resolution: {integrity: sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=}
     dev: false
 
+  /sql-summary/1.0.1:
+    resolution: {integrity: sha512-IpCr2tpnNkP3Jera4ncexsZUp0enJBLr+pHCyTweMUBrbJsTgQeLWx1FXLhoBj/MvcnUQpkgOn2EY8FKOkUzww==}
+    dev: false
+
   /sshpk/1.16.1:
     resolution: {integrity: sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==}
     engines: {node: '>=0.10.0'}
@@ -8728,6 +9068,10 @@ packages:
       escape-string-regexp: 2.0.0
     dev: false
 
+  /stackframe/1.2.0:
+    resolution: {integrity: sha512-GrdeshiRmS1YLMYgzF16olf2jJ/IzxXY9lhKOskuVziubpTYcYqyOwYeJKzQkwy7uN0fYSsbsC4RQaXf9LCrYA==}
+    dev: false
+
   /static-eval/2.1.0:
     resolution: {integrity: sha512-agtxZ/kWSsCkI5E4QifRwsaPs0P0JmZV6dkLz6ILYfFYQGn+5plctanRN+IC8dJRiFkyXHrwEE3W9Wmx67uDbw==}
     dependencies:
@@ -8771,6 +9115,12 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /stream-chopper/3.0.1:
+    resolution: {integrity: sha512-f7h+ly8baAE26iIjcp3VbnBkbIRGtrvV0X0xxFM/d7fwLTYnLzDPTXRKNxa2HZzohOrc96NTrR+FaV3mzOelNA==}
+    dependencies:
+      readable-stream: 3.6.0
+    dev: false
+
   /streamsearch/0.1.2:
     resolution: {integrity: sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=}
     engines: {node: '>=0.8.0'}
@@ -8789,6 +9139,10 @@ packages:
       strip-ansi: 5.2.0
     dev: false
 
+  /string-similarity/4.0.4:
+    resolution: {integrity: sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==}
+    dev: false
+
   /string-width/3.1.0:
     resolution: {integrity: sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==}
     engines: {node: '>=6'}
@@ -9311,6 +9665,12 @@ packages:
       safe-regex: 1.1.0
     dev: false
 
+  /to-source-code/1.0.2:
+    resolution: {integrity: sha1-3RNr2x4dvYC76s8IiZJnjpBwv+o=}
+    dependencies:
+      is-nil: 1.0.1
+    dev: false
+
   /toidentifier/1.0.1:
     resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
     engines: {node: '>=0.6'}
@@ -9355,6 +9715,16 @@ packages:
       punycode: 2.1.1
     dev: false
 
+  /traceparent/1.0.0:
+    resolution: {integrity: sha512-b/hAbgx57pANQ6cg2eBguY3oxD6FGVLI1CC2qoi01RmHR7AYpQHPXTig9FkzbWohEsVuHENZHP09aXuw3/LM+w==}
+    dependencies:
+      random-poly-fill: 1.0.1
+    dev: false
+
+  /traverse/0.6.6:
+    resolution: {integrity: sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=}
+    dev: false
+
   /true-case-path/2.2.1:
     resolution: {integrity: sha512-0z3j8R7MCjy10kc/g+qg7Ln3alJTodw9aDuVWZa3uiWqfuBMKeAeP2ocWcxoyM3D73yz3Jt/Pu4qPr4wHSdB/Q==}
     dev: false
@@ -9555,6 +9925,13 @@ packages:
       which-boxed-primitive: 1.0.2
     dev: false
 
+  /unicode-byte-truncate/1.0.0:
+    resolution: {integrity: sha1-qm8PNHUZP+IMMgrJIT425i6HZKc=}
+    dependencies:
+      is-integer: 1.0.7
+      unicode-substring: 0.1.0
+    dev: false
+
   /unicode-properties/1.3.1:
     resolution: {integrity: sha512-nIV3Tf3LcUEZttY/2g4ZJtGXhWwSkuLL+rCu0DIAMbjyVPj+8j5gNVz4T/sVbnQybIsd5SFGkPKg/756OY6jlA==}
     dependencies:
@@ -9562,6 +9939,10 @@ packages:
       unicode-trie: 2.0.0
     dev: false
 
+  /unicode-substring/0.1.0:
+    resolution: {integrity: sha1-YSDOPDkDhdvND2DDK5BlxBgdSzY=}
+    dev: false
+
   /unicode-trie/0.3.1:
     resolution: {integrity: sha1-1nHd3YkQGgi6w3tqUWEBBgIFIIU=}
     dependencies:
@@ -10152,6 +10533,10 @@ packages:
     resolution: {integrity: sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==}
     dev: false
 
+  /yallist/2.1.2:
+    resolution: {integrity: sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=}
+    dev: false
+
   /yallist/4.0.0:
     resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
     dev: false
@@ -11823,7 +12208,7 @@ packages:
     dev: false
 
   file:projects/server.tgz:
-    resolution: {integrity: sha512-UBFOAisptjpTNIZoU6hWJVBKTk7mftZU0Dz+pJiVhMBJSD+iW393yv4v9GedEpor5rIYDGGcps7IVCXZlr76tw==, tarball: file:projects/server.tgz}
+    resolution: {integrity: sha512-KX5c6GAIbdrpb9vSiwP/vGovTeHgbcjvlpKZUCmN7/iqGz3S5vSxySnO8wsqJm8fEurl8CAzmcKp73LvHPmSzQ==, tarball: file:projects/server.tgz}
     name: '@rush-temp/server'
     version: 0.0.0
     dependencies:
@@ -11833,6 +12218,7 @@ packages:
       '@types/ws': 7.4.7
       '@typescript-eslint/eslint-plugin': 5.7.0_c25e8c1f4f4f7aaed27aa6f9ce042237
       '@typescript-eslint/parser': 5.7.0_eslint@7.32.0+typescript@4.5.4
+      elastic-apm-node: 3.26.0
       esbuild: 0.12.29
       eslint: 7.32.0
       eslint-config-standard-with-typescript: 21.0.1_ce2fa0c4dfa1c256100cababd749a13a

dev/client-resources/src/connection.ts

@@ -13,24 +13,25 @@
 // limitations under the License.
 //
 
-import type { Class, ClientConnection, Doc, DocumentQuery, FindOptions, FindResult, Ref, ServerStorage, Tx, TxHander, TxResult } from '@anticrm/core'
+import { Class, ClientConnection, Doc, DocumentQuery, FindOptions, FindResult, Ref, ServerStorage, Tx, TxHander, TxResult, DOMAIN_TX, MeasureMetricsContext } from '@anticrm/core'
-import { DOMAIN_TX } from '@anticrm/core'
 import { createInMemoryAdapter, createInMemoryTxAdapter } from '@anticrm/dev-storage'
 import { protoDeserialize, protoSerialize } from '@anticrm/platform'
 import type { DbConfiguration } from '@anticrm/server-core'
 import { createServerStorage, FullTextAdapter, IndexedDoc } from '@anticrm/server-core'
 
 class ServerStorageWrapper implements ClientConnection {
-  constructor (private readonly storage: ServerStorage, private readonly handler: TxHander) {}
+  measureCtx = new MeasureMetricsContext('client', {})
+  constructor (private readonly storage: ServerStorage, private readonly handler: TxHander) {
+  }
 
   findAll <T extends Doc>(_class: Ref<Class<T>>, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<FindResult<T>> {
     const [c, q, o] = protoDeserialize(protoSerialize([_class, query, options]))
-    return this.storage.findAll(c, q, o)
+    return this.storage.findAll(this.measureCtx, c, q, o)
   }
 
   async tx (tx: Tx): Promise<TxResult> {
     const _tx = protoDeserialize(protoSerialize(tx))
-    const [result, derived] = await this.storage.tx(_tx)
+    const [result, derived] = await this.storage.tx(this.measureCtx, _tx)
     for (const tx of derived) { this.handler(tx) }
     return result
   }

dev/docker-compose.yaml

@@ -30,6 +30,10 @@ services:
       - ELASTICSEARCH_PORT_NUMBER=9200
       - BITNAMI_DEBUG=true
       - discovery.type=single-node
+    healthcheck:
+      interval: 20s
+      retries: 10
+      test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
   account:
     image: anticrm/account
     links:
@@ -66,6 +70,48 @@ services:
     environment:
       - ELASTIC_URL=http://elastic:9200
       - MONGO_URL=mongodb://mongodb:27017
+      - METRICS_CONSOLE=true
+      # - APM_SERVER_URL=http://apm-server:8200
+  # apm-server:
+  #   image: docker.elastic.co/apm/apm-server:7.14.2
+  #   depends_on:
+  #     elastic:
+  #       condition: service_healthy
+  #     kibana:
+  #       condition: service_healthy      
+  #     elastic:
+  #       condition: service_healthy
+  #   cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"]
+  #   cap_drop: ["ALL"]
+  #   ports:
+  #   - 8200:8200
+  #   command: >
+  #      apm-server -e
+  #        -E apm-server.rum.enabled=true
+  #        -E setup.kibana.host=kibana:5601
+  #        -E setup.template.settings.index.number_of_replicas=0
+  #        -E apm-server.kibana.enabled=true
+  #        -E apm-server.kibana.host=kibana:5601
+  #        -E output.elasticsearch.hosts=["elastic:9200"]
+  #   healthcheck:
+  #     interval: 10s
+  #     retries: 12
+  #     test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/  
+  # kibana:
+  #   image: docker.elastic.co/kibana/kibana:7.14.2
+  #   depends_on:
+  #     elastic:
+  #       condition: service_healthy
+  #   environment:
+  #     ELASTICSEARCH_URL: http://elastic:9200
+  #     ELASTICSEARCH_HOSTS: http://elastic:9200
+  #   ports:
+  #   - 5601:5601
+  #   healthcheck:
+  #     interval: 10s
+  #     retries: 20
+  #     test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
+
 volumes:
   db:
   files:

dev/generator/src/index.ts

@@ -56,6 +56,7 @@ program
   .command('gen-recruit <workspace> <count>')
   .description('generate a bunch of random candidates with attachemnts and comments.')
   .option('-r, --random', 'generate random ids. So every call will add count <count> more candidates.', false)
+  .option('-l, --lite', 'use same pdf and same account for applicant and candidates', false)
   .action(async (workspace: string, count: number, cmd) => {
     return await generateContacts(transactorUrl, workspace, {
       contacts: count,
@@ -65,7 +66,8 @@ program
         min: 1, max: 3, deleteFactor: 20
       },
       vacancy: 3,
-      applicantUpdateFactor: 70
+      applicants: { min: 50, max: 200, applicantUpdateFactor: 70 },
+      lite: (cmd.lite as boolean)
     }, minio)
   })
 

dev/generator/src/kanban.ts

@@ -1,9 +1,9 @@
 
-import { Ref, TxOperations } from '@anticrm/core'
+import { MeasureContext, Ref, TxOperations } from '@anticrm/core'
 import task, { DoneState, genRanks, Kanban, SpaceWithStates, State } from '@anticrm/task'
 import { findOrUpdate } from './utils'
 
-export async function createUpdateSpaceKanban (spaceId: Ref<SpaceWithStates>, client: TxOperations): Promise<Ref<State>[]> {
+export async function createUpdateSpaceKanban (ctx: MeasureContext, spaceId: Ref<SpaceWithStates>, client: TxOperations): Promise<Ref<State>[]> {
   const rawStates = [
     { color: '#7C6FCD', name: 'Initial' },
     { color: '#6F7BC5', name: 'Intermidiate' },
@@ -22,14 +22,15 @@ export async function createUpdateSpaceKanban (spaceId: Ref<SpaceWithStates>, cl
     }
 
     const sid = ('generated-' + spaceId + '.state.' + st.name.toLowerCase().replace(' ', '_')) as Ref<State>
-    await findOrUpdate(client, spaceId, task.class.State,
+
+    await ctx.with('find-or-update', {}, (ctx) => findOrUpdate(ctx, client, spaceId, task.class.State,
       sid,
       {
         title: st.name,
         color: st.color,
         rank
       }
-    )
+    ))
     states.push(sid)
   }
 
@@ -47,21 +48,20 @@ export async function createUpdateSpaceKanban (spaceId: Ref<SpaceWithStates>, cl
     }
 
     const sid = ('generated-' + spaceId + '.done-state.' + st.title.toLowerCase().replace(' ', '_')) as Ref<DoneState>
-    await findOrUpdate(client, spaceId, st.class,
+    await ctx.with('gen-done-state', {}, (ctx) => findOrUpdate(ctx, client, spaceId, st.class,
       sid,
       {
         title: st.title,
         rank
       }
-    )
+    ))
   }
 
-  await findOrUpdate(client, spaceId,
+  await ctx.with('create-kanban', {}, (ctx) => findOrUpdate(ctx, client, spaceId, task.class.Kanban,
-    task.class.Kanban,
     ('generated-' + spaceId + '.kanban') as Ref<Kanban>,
     {
       attachedTo: spaceId
     }
-  )
+  ))
   return states
 }

dev/generator/src/recruit.ts

@@ -1,8 +1,15 @@
-import contact from '@anticrm/contact'
+import contact, { Employee, EmployeeAccount } from '@anticrm/contact'
-import core, { AttachedData, Data, generateId, Ref, TxOperations } from '@anticrm/core'
+import core, {
+  AttachedData,
+  Data,
+  generateId,
+  MeasureContext,
+  MeasureMetricsContext, metricsToString, Ref,
+  TxOperations
+} from '@anticrm/core'
 import recruit from '@anticrm/model-recruit'
 import { Applicant, Candidate, Vacancy } from '@anticrm/recruit'
-import { genRanks } from '@anticrm/task'
+import { genRanks, State } from '@anticrm/task'
 import faker from 'faker'
 import jpeg, { BufferRet } from 'jpeg-js'
 import { Client } from 'minio'
@@ -11,7 +18,6 @@ import { addComments, CommentOptions } from './comments'
 import { connect } from './connect'
 import { createUpdateSpaceKanban } from './kanban'
 import { findOrUpdate, findOrUpdateAttached } from './utils'
-
 export interface RecruitOptions {
   random: boolean // random id prefix.
   contacts: number // how many contacts to add
@@ -21,15 +27,25 @@ export interface RecruitOptions {
   // Attachment generation control
   attachments: AttachmentOptions
 
+  applicants: {
+    min: number
+    max: number
     applicantUpdateFactor: number
   }
+  lite: boolean
+}
 
-export async function generateContacts (transactorUrl: string, dbName: string, options: RecruitOptions, minio: Client): Promise<void> {
+export async function generateContacts (
+  transactorUrl: string,
+  dbName: string,
+  options: RecruitOptions,
+  minio: Client
+): Promise<void> {
   const connection = await connect(transactorUrl, dbName)
 
   const accounts = await connection.findAll(contact.class.EmployeeAccount, {})
-  const accountIds = accounts.map(a => a._id)
+  const accountIds = accounts.map((a) => a._id)
-  const emoloyeeIds = accounts.map(a => a.employee)
+  const emoloyeeIds = accounts.map((a) => a.employee)
 
   const account = faker.random.arrayElement(accounts)
 
@@ -37,19 +53,180 @@ export async function generateContacts (transactorUrl: string, dbName: string, o
 
   const candidates: Ref<Candidate>[] = []
 
+  const ctx = new MeasureMetricsContext('recruit', { contacts: options.contacts })
+
   for (let i = 0; i < options.contacts; i++) {
+    await ctx.with('candidate', {}, (ctx) => genCandidate(ctx, i, minio, dbName, options, candidates, client))
+  }
+  // Work on Vacancy/Applications.
+  for (let i = 0; i < options.vacancy; i++) {
+    await ctx.with('vacancy', {}, (ctx) =>
+      genVacansyApplicants(ctx, accountIds, options, i, client, minio, dbName, candidates, emoloyeeIds)
+    )
+  }
+
+  await connection.close()
+  ctx.end()
+
+  console.info(metricsToString(ctx.metrics, 'Client'))
+}
+async function genVacansyApplicants (
+  ctx: MeasureContext,
+  accountIds: Ref<EmployeeAccount>[],
+  options: RecruitOptions,
+  i: number,
+  client: TxOperations,
+  minio: Client,
+  dbName: string,
+  candidates: Ref<Candidate>[],
+  emoloyeeIds: Ref<Employee>[]
+): Promise<void> {
+  const vacancy: Data<Vacancy> = {
+    name: faker.company.companyName(),
+    description: faker.lorem.sentences(2),
+    fullDescription: faker.lorem.sentences(10),
+    location: faker.address.city(),
+    company: faker.company.companyName(),
+    members: accountIds,
+    private: false,
+    archived: false
+  }
+  const vacancyId = (options.random ? `vacancy-${generateId()}-${i}` : `vacancy-genid-${i}`) as Ref<Vacancy>
+
+  console.log('Creating vacandy', vacancy.name)
+
+  // Update or create candidate
+  await ctx.with('update', {}, (ctx) =>
+    findOrUpdate(ctx, client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy)
+  )
+
+  console.log('Vacandy generated', vacancy.name)
+
+  if (!options.lite) {
+    await ctx.with('add-attachments', {}, () =>
+      addAttachments(
+        options.attachments,
+        client,
+        minio,
+        dbName,
+        vacancyId,
+        vacancyId,
+        recruit.class.Vacancy,
+        'attachments'
+      )
+    )
+  }
+
+  console.log('Vacandy attachments generated', vacancy.name)
+
+  const states = await ctx.with('create-kanbad', {}, (ctx) => createUpdateSpaceKanban(ctx, vacancyId, client))
+
+  console.log('States generated', vacancy.name)
+
+  const applicantsForCount = options.applicants.min + faker.datatype.number(options.applicants.max)
+
+  const applicantsFor = faker.random.arrayElements(candidates, applicantsForCount)
+  const rankGen = genRanks(candidates.length)
+  for (const candidateId of applicantsFor) {
+    await ctx.with('applicant', {}, (ctx) =>
+      genApplicant(ctx, vacancyId, candidateId, emoloyeeIds, states, client, options, minio, dbName, rankGen)
+    )
+  }
+}
+
+async function genApplicant (
+  ctx: MeasureContext,
+  vacancyId: Ref<Vacancy>,
+  candidateId: Ref<Candidate>,
+  emoloyeeIds: Ref<Employee>[],
+  states: Ref<State>[],
+  client: TxOperations,
+  options: RecruitOptions,
+  minio: Client,
+  dbName: string,
+  rankGen: Generator<string, void, unknown>
+): Promise<void> {
+  const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref<Applicant>
+  const rank = rankGen.next().value
+
+  const applicant: AttachedData<Applicant> = {
+    number: faker.datatype.number(),
+    assignee: faker.random.arrayElement(emoloyeeIds),
+    state: faker.random.arrayElement(states),
+    doneState: null,
+    rank: rank as string
+  }
+
+  // Update or create candidate
+  await findOrUpdateAttached(ctx, client, vacancyId, recruit.class.Applicant, applicantId, applicant, {
+    attachedTo: candidateId,
+    attachedClass: recruit.class.Candidate,
+    collection: 'applications'
+  })
+
+  await ctx.with('add-comment', {}, () =>
+    addComments(options.comments, client, vacancyId, applicantId, recruit.class.Vacancy, 'comments')
+  )
+
+  if (!options.lite) {
+    await ctx.with('add-attachment', {}, () =>
+      addAttachments(
+        options.attachments,
+        client,
+        minio,
+        dbName,
+        vacancyId,
+        applicantId,
+        recruit.class.Applicant,
+        'attachments'
+      )
+    )
+  }
+
+  if (faker.datatype.number(100) > options.applicants.applicantUpdateFactor) {
+    await ctx.with('update-collection', {}, () =>
+      client.updateCollection(
+        recruit.class.Applicant,
+        vacancyId,
+        applicantId,
+        candidateId,
+        recruit.class.Applicant,
+        'applications',
+        {
+          state: faker.random.arrayElement(states)
+        }
+      )
+    )
+  }
+}
+
+const liteAvatar = generateAvatar(0)
+
+// @measure('Candidate')
+async function genCandidate (
+  ctx: MeasureContext,
+  i: number,
+  minio: Client,
+  dbName: string,
+  options: RecruitOptions,
+  candidates: Ref<Candidate>[],
+  client: TxOperations
+): Promise<void> {
   const fName = faker.name.firstName()
   const lName = faker.name.lastName()
 
-    const { imgId, jpegImageData } = generateAvatar(i)
+  const { imgId, jpegImageData } = options.lite ? liteAvatar : generateAvatar(i)
-    await minio.putObject(dbName, imgId, jpegImageData.data, jpegImageData.data.length, { 'Content-Type': 'image/jpeg' })
+
+  if (!options.lite) {
+    await ctx.with('avatar', {}, () =>
+      minio.putObject(dbName, imgId, jpegImageData.data, jpegImageData.data.length, { 'Content-Type': 'image/jpeg' })
+    )
+  }
   const candidate: Data<Candidate> = {
     name: fName + ',' + lName,
     city: faker.address.city(),
     title: faker.name.title(),
-      channels: [
+    channels: [{ provider: contact.channelProvider.Email, value: faker.internet.email(fName, lName) }],
-        { provider: contact.channelProvider.Email, value: faker.internet.email(fName, lName) }
-      ],
     onsite: faker.datatype.boolean(),
     remote: faker.datatype.boolean(),
     avatar: imgId,
@@ -59,77 +236,38 @@ export async function generateContacts (transactorUrl: string, dbName: string, o
   candidates.push(candidateId)
 
   // Update or create candidate
-    await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candidateId, candidate)
+  await ctx.with('find-update', {}, () =>
+    findOrUpdate(ctx, client, recruit.space.CandidatesPublic, recruit.class.Candidate, candidateId, candidate)
+  )
 
-    await addComments(options.comments, client, recruit.space.CandidatesPublic, candidateId, recruit.class.Candidate, 'comments')
+  await ctx.with('add-comment', {}, () =>
+    addComments(
+      options.comments,
+      client,
+      recruit.space.CandidatesPublic,
+      candidateId,
+      recruit.class.Candidate,
+      'comments'
+    )
+  )
 
-    await addAttachments(options.attachments, client, minio, dbName, recruit.space.CandidatesPublic, candidateId, recruit.class.Candidate, 'attachments')
+  if (!options.lite) {
-
+    await ctx.with('add-attachment', {}, () =>
-    console.log('Candidate', fName, lName, ' generated')
+      addAttachments(
+        options.attachments,
+        client,
+        minio,
+        dbName,
+        recruit.space.CandidatesPublic,
+        candidateId,
+        recruit.class.Candidate,
+        'attachments'
+      )
+    )
   }
-  // Work on Vacancy/Applications.
+  console.log('Candidate', candidates.length, fName, lName, ' generated')
-  for (let i = 0; i < options.vacancy; i++) {
-    const vacancy: Data<Vacancy> = {
-      name: faker.company.companyName(),
-      description: faker.lorem.sentences(2),
-      fullDescription: faker.lorem.sentences(10),
-      location: faker.address.city(),
-      company: faker.company.companyName(),
-      members: accountIds,
-      archived: false,
-      private: false
-    }
-    const vacancyId = (options.random ? `vacancy-${generateId()}-${i}` : `vacancy-genid-${i}`) as Ref<Vacancy>
-
-    console.log('Creating vacancy', vacancy.name)
-    // Update or create candidate
-    await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy)
-
-    console.log('Vacandy generated', vacancy.name)
-
-    await addAttachments(options.attachments, client, minio, dbName, vacancyId, vacancyId, recruit.class.Vacancy, 'attachments')
-
-    console.log('Vacandy attachments generated', vacancy.name)
-
-    const states = await createUpdateSpaceKanban(vacancyId, client)
-
-    console.log('States generated', vacancy.name)
-
-    const rankGen = genRanks(candidates.length)
-    for (const candidateId of candidates) {
-      const rank = rankGen.next().value
-
-      if (rank === undefined) {
-        throw Error('Failed to generate rank')
 }
 
-      const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref<Applicant>
-
-      const applicant: AttachedData<Applicant> = {
-        number: faker.datatype.number(),
-        assignee: faker.random.arrayElement(emoloyeeIds),
-        state: faker.random.arrayElement(states),
-        doneState: null,
-        rank
-      }
-
-      // Update or create candidate
-      await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' })
-
-      await addComments(options.comments, client, vacancyId, applicantId, recruit.class.Vacancy, 'comments')
-
-      await addAttachments(options.attachments, client, minio, dbName, vacancyId, applicantId, recruit.class.Applicant, 'attachments')
-
-      if (faker.datatype.number(100) > options.applicantUpdateFactor) {
-        await client.updateCollection(recruit.class.Applicant, vacancyId, applicantId, candidateId, recruit.class.Applicant, 'applications', {
-          state: faker.random.arrayElement(states)
-        })
-      }
-    }
-  }
-
-  await connection.close()
-}
 function generateAvatar (pos: number): { imgId: string, jpegImageData: BufferRet } {
   const imgId = generateId()
   const width = 128

dev/generator/src/utils.ts

@@ -1,6 +1,6 @@
-import { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, Space, TxOperations } from '@anticrm/core'
+import { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, MeasureContext, Ref, Space, TxOperations } from '@anticrm/core'
 
-export async function findOrUpdate<T extends Doc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: Data<T>): Promise<void> {
+export async function findOrUpdate<T extends Doc> (ctx: MeasureContext, client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: Data<T>): Promise<void> {
   const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
   if (existingObj !== undefined) {
     await client.updateDoc(_class, space, objectId, data)
@@ -8,7 +8,7 @@ export async function findOrUpdate<T extends Doc> (client: TxOperations, space:
     await client.createDoc(_class, space, data, objectId)
   }
 }
-export async function findOrUpdateAttached<T extends AttachedDoc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: AttachedData<T>, attached: {attachedTo: Ref<Doc>, attachedClass: Ref<Class<Doc>>, collection: string}): Promise<void> {
+export async function findOrUpdateAttached<T extends AttachedDoc> (ctx: MeasureContext, client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: AttachedData<T>, attached: {attachedTo: Ref<Doc>, attachedClass: Ref<Class<Doc>>, collection: string}): Promise<void> {
   const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
   if (existingObj !== undefined) {
     await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate<T>)

dev/server/src/server.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 //
 
-import { DOMAIN_TX } from '@anticrm/core'
+import { DOMAIN_TX, MeasureMetricsContext } from '@anticrm/core'
 import type { Ref, Doc, TxResult } from '@anticrm/core'
 import { start as startJsonRpc } from '@anticrm/server-ws'
 import { createInMemoryAdapter, createInMemoryTxAdapter } from '@anticrm/dev-storage'
@@ -48,7 +48,7 @@ async function createNullFullTextAdapter (): Promise<FullTextAdapter> {
 export async function start (port: number, host?: string): Promise<void> {
   addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
 
-  startJsonRpc(() => {
+  startJsonRpc(new MeasureMetricsContext('server', {}), () => {
     const conf: DbConfiguration = {
       domains: {
         [DOMAIN_TX]: 'InMemoryTx'

dev/tool/src/elastic.ts

@@ -17,14 +17,10 @@
 import core, {
   Account,
   Class,
-  Doc,
+  Doc, DocumentQuery,
-  FindOptions,
+  DOMAIN_TX, FindOptions, FindResult,
-  DocumentQuery,
-  DOMAIN_TX,
-  FindResult,
   generateId,
-  Hierarchy,
+  Hierarchy, MeasureMetricsContext, ModelDb,
-  ModelDb,
   Ref,
   ServerStorage,
   Tx,
@@ -36,10 +32,11 @@ import core, {
   TxResult,
   TxUpdateDoc
 } from '@anticrm/core'
-import { Client as ElasticClient } from '@elastic/elasticsearch'
-import { Db, MongoClient } from 'mongodb'
-import { Client } from 'minio'
 import { createElasticAdapter } from '@anticrm/elastic'
+import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment'
+import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo'
+import { addLocation } from '@anticrm/platform'
+import { serverChunterId } from '@anticrm/server-chunter'
 import {
   createServerStorage,
   DbAdapter,
@@ -48,11 +45,10 @@ import {
   IndexedDoc,
   TxAdapter
 } from '@anticrm/server-core'
-import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment'
-import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo'
-import { serverChunterId } from '@anticrm/server-chunter'
 import { serverRecruitId } from '@anticrm/server-recruit'
-import { addLocation } from '@anticrm/platform'
+import { Client as ElasticClient } from '@elastic/elasticsearch'
+import { Client } from 'minio'
+import { Db, MongoClient } from 'mongodb'
 import { listMinioObjects } from './minio'
 
 export async function rebuildElastic (
@@ -106,8 +102,9 @@ async function restoreElastic (mongoUrl: string, dbName: string, minio: Client,
     const storage = await createStorage(mongoUrl, elasticUrl, dbName)
     const txes = (await db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
     const data = txes.filter((tx) => tx.objectSpace !== core.space.Model)
+    const metricsCtx = new MeasureMetricsContext('elastic', {})
     for (const tx of data) {
-      await storage.tx(tx)
+      await storage.tx(metricsCtx, tx)
     }
     if (await minio.bucketExists(dbName)) {
       const minioObjects = await listMinioObjects(minio, dbName)

dev/tool/src/workspace.ts

@@ -223,8 +223,10 @@ export async function restoreWorkspace (
       const collection = db.collection(c.name)
       await collection.deleteMany({})
       const data = JSON.parse((await readFile(fileName + c.name + '.json')).toString()) as Document[]
+      if (data.length > 0) {
         await collection.insertMany(data)
       }
+    }
 
     console.log('Restore minio objects')
     if (await minio.bucketExists(dbName)) {

packages/core/src/index.ts

@@ -22,5 +22,6 @@ export * from './client'
 export * from './operator'
 export * from './query'
 export * from './server'
+export * from './measurements'
 
 export { default, coreId } from './component'

packages/core/src/measurements/context.ts

@@ -0,0 +1,58 @@
+// Basic performance metrics suite.
+
+import { childMetrics, measure, newMetrics } from './metrics'
+import { MeasureContext, MeasureLogger, Metrics, ParamType } from './types'
+
+/**
+ * @public
+ */
+export class MeasureMetricsContext implements MeasureContext {
+  private readonly name: string
+  private readonly params: Record<string, ParamType>
+  logger: MeasureLogger
+  metrics: Metrics
+  private readonly done: () => void
+
+  constructor (name: string, params: Record<string, ParamType>, metrics: Metrics = newMetrics()) {
+    this.name = name
+    this.params = params
+    this.metrics = metrics
+    this.done = measure(metrics, params)
+
+    this.logger = {
+      info: (msg, args) => {
+        console.info(msg, ...args)
+      },
+      error: (msg, args) => {
+        console.error(msg, ...args)
+      }
+    }
+  }
+
+  newChild (name: string, params: Record<string, ParamType>): MeasureContext {
+    return new MeasureMetricsContext(name, params, childMetrics(this.metrics, [name]))
+  }
+
+  async with<T>(name: string, params: Record<string, ParamType>, op: (ctx: MeasureContext) => T | Promise<T>): Promise<T> {
+    const c = this.newChild(name, params)
+    try {
+      let value = op(c)
+      if (value instanceof Promise) {
+        value = await value
+      }
+      c.end()
+      return value
+    } catch (err: any) {
+      await c.error(err)
+      throw err
+    }
+  }
+
+  async error (err: Error | string): Promise<void> {
+    console.error(err)
+  }
+
+  end (): void {
+    this.done()
+  }
+}

packages/core/src/measurements/index.ts

@@ -0,0 +1,3 @@
+export * from './context'
+export * from './metrics'
+export * from './types'

packages/core/src/measurements/metrics.ts

@@ -0,0 +1,156 @@
+// Basic performance metrics suite.
+
+import { MetricsData } from '.'
+import { Metrics, ParamType } from './types'
+
+/**
+ * @public
+ */
+export const globals: Metrics = newMetrics()
+
+/**
+ * @public
+ * @returns
+ */
+export function newMetrics (): Metrics {
+  return {
+    operations: 0,
+    time: 0,
+    measurements: {},
+    params: {}
+  }
+}
+
+/**
+ * Measure with tree expansion. Operation counter will be added only to leaf's.
+ * @public
+ */
+export function measure (metrics: Metrics, params: Record<string, ParamType>): () => void {
+  const st = Date.now()
+  return () => {
+    const ed = Date.now()
+    // Update params if required
+    for (const [k, v] of Object.entries(params)) {
+      let params = metrics.params[k]
+      if (params === undefined) {
+        params = {}
+        metrics.params[k] = params
+      }
+      const vKey = `${v?.toString() ?? ''}`
+      let param = params[vKey]
+      if (param === undefined) {
+        param = {
+          operations: 0,
+          time: 0
+        }
+        params[vKey] = param
+      }
+      param.time += ed - st
+      param.operations++
+    }
+    // Update leaf data
+    metrics.time += ed - st
+    metrics.operations++
+  }
+}
+
+/**
+ * @public
+ */
+export function childMetrics (root: Metrics, path: string[]): Metrics {
+  const segments = path
+  let oop = root
+  for (const p of segments) {
+    const v = oop.measurements[p] ?? { operations: 0, time: 0, measurements: {}, params: {} }
+    oop.measurements[p] = v
+    oop = v
+  }
+  return oop
+}
+
+function aggregate (m: Metrics): Metrics {
+  const ms = aggregateMetrics(m.measurements)
+
+  // Use child overage, if there is no top level value specified.
+  const keysLen = Object.keys(ms).length
+  const childAverage = m.time === 0 && keysLen > 0
+  const sumVal: Metrics | undefined = childAverage
+    ? Object.values(ms).reduce((p, v) => {
+      p.operations += v.operations
+      p.time += v.time
+      return p
+    }, {
+      operations: 0,
+      time: 0,
+      measurements: ms,
+      params: {}
+    })
+    : undefined
+  if (sumVal !== undefined) {
+    return {
+      ...sumVal,
+      measurements: ms,
+      params: m.params
+    }
+  }
+  return {
+    ...m,
+    measurements: ms
+  }
+}
+
+function aggregateMetrics (m: Record<string, Metrics>): Record<string, Metrics> {
+  const result: Record<string, Metrics> = {}
+  for (const [k, v] of Object.entries(m).sort((a, b) => b[1].time - a[1].time)) {
+    result[k] = aggregate(v)
+  }
+  return result
+}
+
+function toLen (val: string, sep: string, len = 50): string {
+  while (val.length < len) {
+    val += sep
+  }
+  return val
+}
+
+function printMetricsChildren (params: Record<string, Metrics>, offset: number): string {
+  let r = ''
+  if (Object.keys(params).length > 0) {
+    r += '\n' + toLen('', ' ', offset)
+    r += Object.entries(params)
+      .map(([k, vv]) => toString(k, vv, offset))
+      .join('\n' + toLen('', ' ', offset))
+  }
+  return r
+}
+
+function printMetricsParams (params: Record<string, Record<string, MetricsData>>, offset: number): string {
+  let r = ''
+  const joinP = (key: string, data: Record<string, MetricsData>): string[] => {
+    return Object.entries(data).map(([k, vv]) =>
+      `${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', 70 - offset)}: avg ${vv.time / (vv.operations > 0 ? vv.operations : 1)} total: ${vv.time} ops: ${vv.operations}`.trim()
+    )
+  }
+  const joinParams = Object.entries(params).reduce<string[]>((p, c) => [...p, ...joinP(c[0], c[1])], [])
+  if (Object.keys(joinParams).length > 0) {
+    r += '\n' + toLen('', ' ', offset)
+    r += joinParams
+      .join('\n' + toLen('', ' ', offset))
+  }
+  return r
+}
+
+function toString (name: string, m: Metrics, offset: number): string {
+  let r = `${toLen('', ' ', offset)}${toLen(name, '-', 70 - offset)}: avg ${m.time / (m.operations > 0 ? m.operations : 1)} total: ${m.time} ops: ${m.operations}`.trim()
+  r += printMetricsParams(m.params, offset + 4)
+  r += printMetricsChildren(m.measurements, offset + 4)
+  return r
+}
+
+/**
+ * @public
+ */
+export function metricsToString (metrics: Metrics, name = 'System'): string {
+  return toString(name, aggregate(metrics), 0)
+}

packages/core/src/measurements/types.ts

@@ -0,0 +1,45 @@
+/**
+ * @public
+ */
+export type ParamType = string | number | boolean | undefined
+
+/**
+  * @public
+  */
+export interface MetricsData {
+  operations: number
+  time: number
+}
+
+/**
+  * @public
+  */
+export interface Metrics extends MetricsData {
+  params: Record<string, Record<string, MetricsData>>
+  measurements: Record<string, Metrics>
+}
+
+/**
+ * @public
+ */
+export interface MeasureLogger {
+  info: (message: string, ...args: any[]) => void
+  error: (message: string, ...args: any[]) => void
+}
+/**
+ * @public
+ */
+export interface MeasureContext {
+  // Create a child metrics context
+  newChild: (name: string, params: Record<string, ParamType>) => MeasureContext
+
+  with: <T>(name: string, params: Record<string, ParamType>, op: (ctx: MeasureContext) => T | Promise<T>) => Promise<T>
+
+  logger: MeasureLogger
+
+  // Capture error
+  error: (err: Error | string | any) => Promise<void>
+
+  // Mark current context as complete
+  end: () => void
+}

packages/core/src/server.ts

@@ -14,6 +14,7 @@
 // limitations under the License.
 //
 
+import { MeasureContext } from '.'
 import type { Doc, Class, Ref } from './classes'
 import type { DocumentQuery, FindOptions, FindResult, TxResult } from './storage'
 import type { Tx } from './tx'
@@ -23,9 +24,10 @@ import type { Tx } from './tx'
  */
 export interface ServerStorage {
   findAll: <T extends Doc>(
+    ctx: MeasureContext,
     _class: Ref<Class<T>>,
     query: DocumentQuery<T>,
     options?: FindOptions<T>
   ) => Promise<FindResult<T>>
-  tx: (tx: Tx) => Promise<[TxResult, Tx[]]>
+  tx: (ctx: MeasureContext, tx: Tx) => Promise<[TxResult, Tx[]]>
 }

plugins/devmodel-resources/src/index.ts

@@ -16,7 +16,7 @@
 import { IntlString, Resources } from '@anticrm/platform'
 import ModelView from './components/ModelView.svelte'
 import QueryView from './components/QueryView.svelte'
-import core, { Class, Client, Doc, DocumentQuery, FindOptions, Ref, FindResult, Hierarchy, ModelDb, Tx, TxResult, WithLookup } from '@anticrm/core'
+import core, { Class, Client, Doc, DocumentQuery, FindOptions, Ref, FindResult, Hierarchy, ModelDb, Tx, TxResult, WithLookup, Metrics } from '@anticrm/core'
 import { Builder } from '@anticrm/model'
 import workbench from '@anticrm/workbench'
 import view from '@anticrm/view'
@@ -45,6 +45,9 @@ class ModelClient implements Client {
       this.notify?.(tx)
       console.info('devmodel# notify=>', tx, this.client.getModel())
       notifications.push(tx)
+      if (notifications.length > 500) {
+        notifications.shift()
+      }
     }
   }
 
@@ -62,6 +65,9 @@ class ModelClient implements Client {
     const result = await this.client.findOne(_class, query, options)
     console.info('devmodel# findOne=>', _class, query, options, 'result => ', result, ' =>model', this.client.getModel())
     queries.push({ _class, query, options, result: result !== undefined ? [result] : [], findOne: true })
+    if (queries.length > 100) {
+      queries.shift()
+    }
     return result
   }
 
@@ -69,6 +75,9 @@ class ModelClient implements Client {
     const result = await this.client.findAll(_class, query, options)
     console.info('devmodel# findAll=>', _class, query, options, 'result => ', result, ' =>model', this.client.getModel())
     queries.push({ _class, query, options, result, findOne: false })
+    if (queries.length > 100) {
+      queries.shift()
+    }
     return result
   }
 
@@ -76,6 +85,9 @@ class ModelClient implements Client {
     const result = await this.client.tx(tx)
     console.info('devmodel# tx=>', tx, result)
     transactions.push({ tx, result })
+    if (transactions.length > 100) {
+      transactions.shift()
+    }
     return result
   }
 

server/core/src/fulltext.ts

@@ -14,22 +14,25 @@
 // limitations under the License.
 //
 
+import type { AttachedDoc, Class, Doc, Obj, Ref, TxCreateDoc, TxResult, TxUpdateDoc } from '@anticrm/core'
 import core, {
-  Hierarchy,
   AnyAttribute,
-  Storage,
+  Collection,
   DocumentQuery,
   FindOptions,
   FindResult,
-  TxProcessor,
+  Hierarchy,
+  MeasureContext,
+  PropertyType,
+  Tx,
+  TxBulkWrite,
+  TxCollectionCUD,
   TxMixin,
+  TxProcessor,
   TxPutBag,
-  TxRemoveDoc,
+  TxRemoveDoc
-  Collection
 } from '@anticrm/core'
-import type { AttachedDoc, TxUpdateDoc, TxCreateDoc, Doc, Ref, Class, Obj, TxResult } from '@anticrm/core'
+import type { FullTextAdapter, IndexedDoc, WithFind } from './types'
-
-import type { IndexedDoc, FullTextAdapter, WithFind } from './types'
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
 const NO_INDEX = [] as AnyAttribute[]
@@ -37,32 +40,76 @@ const NO_INDEX = [] as AnyAttribute[]
 /**
  * @public
  */
-export class FullTextIndex extends TxProcessor implements Storage {
+export class FullTextIndex implements WithFind {
   private readonly indexes = new Map<Ref<Class<Obj>>, AnyAttribute[]>()
 
   constructor (
     private readonly hierarchy: Hierarchy,
     private readonly adapter: FullTextAdapter,
     private readonly dbStorage: WithFind
-  ) {
+  ) {}
-    super()
-  }
 
-  protected override async txPutBag (tx: TxPutBag<any>): Promise<TxResult> {
+  protected async txPutBag (ctx: MeasureContext, tx: TxPutBag<any>): Promise<TxResult> {
-    console.log('FullTextIndex.txPutBag: Method not implemented.')
+    // console.log('FullTextIndex.txPutBag: Method not implemented.')
     return {}
   }
 
-  protected override async txRemoveDoc (tx: TxRemoveDoc<Doc>): Promise<TxResult> {
+  protected async txRemoveDoc (ctx: MeasureContext, tx: TxRemoveDoc<Doc>): Promise<TxResult> {
-    console.log('FullTextIndex.txRemoveDoc: Method not implemented.')
+    // console.log('FullTextIndex.txRemoveDoc: Method not implemented.')
     return {}
   }
 
-  protected txMixin (tx: TxMixin<Doc, Doc>): Promise<TxResult> {
+  protected txMixin (ctx: MeasureContext, tx: TxMixin<Doc, Doc>): Promise<TxResult> {
     throw new Error('Method not implemented.')
   }
 
+  async tx (ctx: MeasureContext, tx: Tx): Promise<TxResult> {
+    switch (tx._class) {
+      case core.class.TxCreateDoc:
+        return await this.txCreateDoc(ctx, tx as TxCreateDoc<Doc>)
+      case core.class.TxCollectionCUD:
+        return await this.txCollectionCUD(ctx, tx as TxCollectionCUD<Doc, AttachedDoc>)
+      case core.class.TxUpdateDoc:
+        return await this.txUpdateDoc(ctx, tx as TxUpdateDoc<Doc>)
+      case core.class.TxRemoveDoc:
+        return await this.txRemoveDoc(ctx, tx as TxRemoveDoc<Doc>)
+      case core.class.TxMixin:
+        return await this.txMixin(ctx, tx as TxMixin<Doc, Doc>)
+      case core.class.TxPutBag:
+        return await this.txPutBag(ctx, tx as TxPutBag<PropertyType>)
+      case core.class.TxBulkWrite:
+        return await this.txBulkWrite(ctx, tx as TxBulkWrite)
+    }
+    throw new Error('TxProcessor: unhandled transaction class: ' + tx._class)
+  }
+
+  protected txCollectionCUD (ctx: MeasureContext, tx: TxCollectionCUD<Doc, AttachedDoc>): Promise<TxResult> {
+    // We need update only create transactions to contain attached, attachedToClass.
+    if (tx.tx._class === core.class.TxCreateDoc) {
+      const createTx = tx.tx as TxCreateDoc<AttachedDoc>
+      const d: TxCreateDoc<AttachedDoc> = {
+        ...createTx,
+        attributes: {
+          ...createTx.attributes,
+          attachedTo: tx.objectId,
+          attachedToClass: tx.objectClass,
+          collection: tx.collection
+        }
+      }
+      return this.txCreateDoc(ctx, d)
+    }
+    return this.tx(ctx, tx.tx)
+  }
+
+  protected async txBulkWrite (ctx: MeasureContext, bulkTx: TxBulkWrite): Promise<TxResult> {
+    for (const tx of bulkTx.txes) {
+      await this.tx(ctx, tx)
+    }
+    return {}
+  }
+
   async findAll<T extends Doc>(
+    ctx: MeasureContext,
     _class: Ref<Class<T>>,
     query: DocumentQuery<T>,
     options?: FindOptions<T>
@@ -78,7 +125,7 @@ export class FullTextIndex extends TxProcessor implements Storage {
         ids.push(doc.attachedTo)
       }
     }
-    return await this.dbStorage.findAll(_class, { _id: { $in: ids as any }, ...mainQuery }, options) // TODO: remove `as any`
+    return await this.dbStorage.findAll(ctx, _class, { _id: { $in: ids as any }, ...mainQuery }, options) // TODO: remove `as any`
   }
 
   private getFullTextAttributes (clazz: Ref<Class<Obj>>): AnyAttribute[] | undefined {
@@ -102,14 +149,14 @@ export class FullTextIndex extends TxProcessor implements Storage {
     }
   }
 
-  protected override async txCreateDoc (tx: TxCreateDoc<Doc>): Promise<TxResult> {
+  protected async txCreateDoc (ctx: MeasureContext, tx: TxCreateDoc<Doc>): Promise<TxResult> {
     const attributes = this.getFullTextAttributes(tx.objectClass)
     const doc = TxProcessor.createDoc2Doc(tx)
     let parentContent: any[] = []
     if (this.hierarchy.isDerived(doc._class, core.class.AttachedDoc)) {
       const attachedDoc = doc as AttachedDoc
       const parentDoc = (
-        await this.dbStorage.findAll(attachedDoc.attachedToClass, { _id: attachedDoc.attachedTo }, { limit: 1 })
+        await this.dbStorage.findAll(ctx, attachedDoc.attachedToClass, { _id: attachedDoc.attachedTo }, { limit: 1 })
       )[0]
       if (parentDoc !== undefined) {
         const parentAttributes = this.getFullTextAttributes(parentDoc._class)
@@ -140,7 +187,7 @@ export class FullTextIndex extends TxProcessor implements Storage {
     return await this.adapter.index(indexedDoc)
   }
 
-  protected override async txUpdateDoc (tx: TxUpdateDoc<Doc>): Promise<TxResult> {
+  protected async txUpdateDoc (ctx: MeasureContext, tx: TxUpdateDoc<Doc>): Promise<TxResult> {
     const attributes = this.getFullTextAttributes(tx.objectClass)
     let result = {}
     if (attributes === undefined) return result
@@ -157,7 +204,7 @@ export class FullTextIndex extends TxProcessor implements Storage {
     }
     if (shouldUpdate) {
       result = await this.adapter.update(tx.objectId, update)
-      await this.updateAttachedDocs(tx, update)
+      await this.updateAttachedDocs(ctx, tx, update)
     }
     return result
   }
@@ -167,14 +214,14 @@ export class FullTextIndex extends TxProcessor implements Storage {
     return attributes.map((attr) => (doc as any)[attr.name]?.toString() ?? '')
   }
 
-  private async updateAttachedDocs (tx: TxUpdateDoc<Doc>, update: any): Promise<void> {
+  private async updateAttachedDocs (ctx: MeasureContext, tx: TxUpdateDoc<Doc>, update: any): Promise<void> {
-    const doc = (await this.dbStorage.findAll(tx.objectClass, { _id: tx.objectId }, { limit: 1 }))[0]
+    const doc = (await this.dbStorage.findAll(ctx, tx.objectClass, { _id: tx.objectId }, { limit: 1 }))[0]
     if (doc === undefined) return
     const attributes = this.hierarchy.getAllAttributes(doc._class)
     for (const attribute of attributes.values()) {
       if (this.hierarchy.isDerived(attribute.type._class, core.class.Collection)) {
         const collection = attribute.type as Collection<AttachedDoc>
-        const allAttached = await this.dbStorage.findAll(collection.of, { attachedTo: tx.objectId })
+        const allAttached = await this.dbStorage.findAll(ctx, collection.of, { attachedTo: tx.objectId })
         if (allAttached.length === 0) continue
         const attributes = this.getFullTextAttributes(tx.objectClass)
         const shift = attributes?.length ?? 0

server/core/src/storage.ts

@@ -14,10 +14,32 @@
 // limitations under the License.
 //
 
-import core, { ServerStorage, Domain, Tx, TxCUD, Doc, Ref, Class, DocumentQuery, FindResult, FindOptions, Storage, TxBulkWrite, TxResult, TxCollectionCUD, AttachedDoc, DOMAIN_MODEL, Hierarchy, DOMAIN_TX, ModelDb, TxFactory } from '@anticrm/core'
+import core, {
-import type { FullTextAdapterFactory, FullTextAdapter } from './types'
+  AttachedDoc,
+  Class,
+  Doc,
+  DocumentQuery,
+  Domain,
+  DOMAIN_MODEL,
+  DOMAIN_TX,
+  FindOptions,
+  FindResult,
+  Hierarchy,
+  MeasureContext,
+  ModelDb,
+  Ref,
+  ServerStorage,
+  Storage,
+  Tx,
+  TxBulkWrite,
+  TxCollectionCUD,
+  TxCUD,
+  TxFactory,
+  TxResult
+} from '@anticrm/core'
 import { FullTextIndex } from './fulltext'
 import { Triggers } from './triggers'
+import type { FullTextAdapter, FullTextAdapterFactory } from './types'
 
 /**
  * @public
@@ -87,7 +109,7 @@ class TServerStorage implements ServerStorage {
     return adapter
   }
 
-  private async routeTx (tx: Tx): Promise<TxResult> {
+  private async routeTx (ctx: MeasureContext, tx: Tx): Promise<TxResult> {
     if (this.hierarchy.isDerived(tx._class, core.class.TxCUD)) {
       const txCUD = tx as TxCUD<Doc>
       const domain = this.hierarchy.getDomain(txCUD.objectClass)
@@ -96,7 +118,7 @@ class TServerStorage implements ServerStorage {
       if (this.hierarchy.isDerived(tx._class, core.class.TxBulkWrite)) {
         const bulkWrite = tx as TxBulkWrite
         for (const tx of bulkWrite.txes) {
-          await this.tx(tx)
+          await this.tx(ctx, tx)
         }
       } else {
         throw new Error('not implemented (routeTx)')
@@ -105,7 +127,7 @@ class TServerStorage implements ServerStorage {
     }
   }
 
-  async processCollection (tx: Tx): Promise<Tx[]> {
+  async processCollection (ctx: MeasureContext, tx: Tx): Promise<Tx[]> {
     if (tx._class === core.class.TxCollectionCUD) {
       const colTx = tx as TxCollectionCUD<Doc, AttachedDoc>
       const _id = colTx.objectId
@@ -120,10 +142,14 @@ class TServerStorage implements ServerStorage {
 
       const isCreateTx = colTx.tx._class === core.class.TxCreateDoc
       if (isCreateTx || colTx.tx._class === core.class.TxRemoveDoc) {
-        attachedTo = (await this.findAll(_class, { _id }, { limit: 1 }))[0]
+        attachedTo = (await this.findAll(ctx, _class, { _id }, { limit: 1 }))[0]
         if (attachedTo !== undefined) {
           const txFactory = new TxFactory(tx.modifiedBy)
-          return [txFactory.createTxUpdateDoc(_class, attachedTo.space, _id, { $inc: { [colTx.collection]: isCreateTx ? 1 : -1 } })]
+          return [
+            txFactory.createTxUpdateDoc(_class, attachedTo.space, _id, {
+              $inc: { [colTx.collection]: isCreateTx ? 1 : -1 }
+            })
+          ]
         }
       }
     }
@@ -131,21 +157,28 @@ class TServerStorage implements ServerStorage {
   }
 
   async findAll<T extends Doc>(
+    ctx: MeasureContext,
     clazz: Ref<Class<T>>,
     query: DocumentQuery<T>,
     options?: FindOptions<T>
   ): Promise<FindResult<T>> {
+    return await ctx.with('find-all', {}, (ctx) => {
       const domain = this.hierarchy.getDomain(clazz)
-    console.log('server findall', query)
       if (Object.keys(query)[0] === '$search') {
-      return await this.fulltext.findAll(clazz, query, options)
+        return ctx.with('full-text-find-all', {}, (ctx) => this.fulltext.findAll(ctx, clazz, query, options))
       }
-    return await this.getAdapter(domain).findAll(clazz, query, options)
+      return ctx.with('db-find-all', { _class: clazz, domain }, () =>
+        this.getAdapter(domain).findAll(clazz, query, options)
+      )
+    })
   }
 
-  async tx (tx: Tx): Promise<[TxResult, Tx[]]> {
+  async tx (ctx: MeasureContext, tx: Tx): Promise<[TxResult, Tx[]]> {
     // store tx
-    await this.getAdapter(DOMAIN_TX).tx(tx)
+    const _class = txClass(tx)
+    const objClass = txObjectClass(tx)
+    return await ctx.with('tx', { _class, objClass }, async (ctx) => {
+      await ctx.with('domain-tx', { _class, objClass }, async () => await this.getAdapter(DOMAIN_TX).tx(tx))
 
       if (tx.objectSpace === core.space.Model) {
         // maintain hiearachy and triggers
@@ -153,22 +186,42 @@ class TServerStorage implements ServerStorage {
         await this.triggers.tx(tx)
         await this.modelDb.tx(tx)
       }
+
+      let derived: Tx[] = []
+      let result: TxResult = {}
       // store object
-    const result = await this.routeTx(tx)
+      result = await ctx.with('route-tx', { _class, objClass }, (ctx) => this.routeTx(ctx, tx))
       // invoke triggers and store derived objects
-    const derived = [...await this.processCollection(tx), ...await this.triggers.apply(tx.modifiedBy, tx, this.findAll.bind(this), this.hierarchy)]
+      derived = [
+        ...(await ctx.with('process-collection', { _class }, () => this.processCollection(ctx, tx))),
+        ...(await ctx.with('process-triggers', {}, (ctx) =>
+          this.triggers.apply(tx.modifiedBy, tx, this.findAll.bind(this, ctx), this.hierarchy)
+        ))
+      ]
+
       for (const tx of derived) {
-      await this.routeTx(tx)
+        await ctx.with('derived-route-tx', { _class: txClass(tx) }, (ctx) => this.routeTx(ctx, tx))
-    }
-    // index object
-    await this.fulltext.tx(tx)
-    // index derived objects
-    for (const tx of derived) {
-      await this.fulltext.tx(tx)
       }
 
-    return [result, derived]
+      // index object
+      await ctx.with('fulltext', { _class, objClass }, (ctx) => this.fulltext.tx(ctx, tx))
+      // index derived objects
+      for (const tx of derived) {
+        await ctx.with('derived-fulltext', { _class: txClass(tx) }, (ctx) => this.fulltext.tx(ctx, tx))
       }
+      return [result, derived]
+    })
+  }
+}
+
+function txObjectClass (tx: Tx): string {
+  return tx._class === core.class.TxCollectionCUD
+    ? (tx as TxCollectionCUD<Doc, AttachedDoc>).tx.objectClass
+    : (tx as TxCUD<Doc>).objectClass
+}
+
+function txClass (tx: Tx): string {
+  return tx._class === core.class.TxCollectionCUD ? (tx as TxCollectionCUD<Doc, AttachedDoc>).tx._class : tx._class
 }
 
 /**

server/core/src/types.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 //
 
-import type { Tx, Ref, Doc, Class, Space, Timestamp, Account, FindResult, DocumentQuery, FindOptions, TxResult } from '@anticrm/core'
+import type { Tx, Ref, Doc, Class, Space, Timestamp, Account, FindResult, DocumentQuery, FindOptions, TxResult, MeasureContext } from '@anticrm/core'
 import { TxFactory, Hierarchy } from '@anticrm/core'
 import type { Resource } from '@anticrm/platform'
 
@@ -88,5 +88,5 @@ export interface Token {
  * @public
  */
 export interface WithFind {
-  findAll: <T extends Doc> (clazz: Ref<Class<T>>, query: DocumentQuery<T>, options?: FindOptions<T>) => Promise<FindResult<T>>
+  findAll: <T extends Doc> (ctx: MeasureContext, clazz: Ref<Class<T>>, query: DocumentQuery<T>, options?: FindOptions<T>) => Promise<FindResult<T>>
 }

server/elastic/src/adapter.ts

@@ -53,9 +53,7 @@ class ElasticAdapter implements FullTextAdapter {
           }
         }
       })
-      console.log(result)
       const hits = result.body.hits.hits as any[]
-      console.log('hits', hits)
       return hits.map(hit => hit._source)
     } catch (err) {
       console.error(JSON.stringify(err, null, 2))
@@ -64,34 +62,28 @@ class ElasticAdapter implements FullTextAdapter {
   }
 
   async index (doc: IndexedDoc): Promise<TxResult> {
-    console.log('eastic: index', doc)
     if (doc.data === undefined) {
       try {
-        const resp = await this.client.index({
+        await this.client.index({
           index: this.db,
           id: doc.id,
           type: '_doc',
           body: doc
         })
-        console.log('resp', resp)
-        console.log('error', (resp.meta as any)?.body?.error)
       } catch (err: any) {
-        console.log('elastic-exception', err)
+        console.error('elastic-exception', err)
       }
     } else {
-      console.log('attachment pipeline')
       try {
-        const resp = await this.client.index({
+        await this.client.index({
           index: this.db,
           id: doc.id,
           type: '_doc',
           pipeline: 'attachment',
           body: doc
         })
-        console.log('resp', resp)
-        console.log('error', (resp.meta as any)?.body?.error)
       } catch (err: any) {
-        console.log('elastic-exception', err)
+        console.error('elastic-exception', err)
       }
     }
     return {}
@@ -99,16 +91,15 @@ class ElasticAdapter implements FullTextAdapter {
 
   async update (id: Ref<Doc>, update: Record<string, any>): Promise<TxResult> {
     try {
-      const resp = await this.client.update({
+      await this.client.update({
         index: this.db,
         id,
         body: {
           doc: update
         }
       })
-      console.log('update', resp)
     } catch (err: any) {
-      console.log('elastic-exception', err)
+      console.error('elastic-exception', err)
     }
 
     return {}

server/mongo/src/__tests__/storage.test.ts

@@ -16,6 +16,7 @@
 import core, {
   Class,
   Client,
+  ClientConnection,
   createClient,
   Doc,
   DocumentQuery,
@@ -24,13 +25,13 @@ import core, {
   FindOptions,
   FindResult,
   generateId,
-  Hierarchy, ModelDb,
+  Hierarchy, ModelDb, Ref,
-  Ref,
   SortingOrder,
   Space,
   Tx,
   TxOperations,
-  TxResult
+  TxResult,
+  MeasureMetricsContext
 } from '@anticrm/core'
 import { createServerStorage, DbAdapter, DbConfiguration, FullTextAdapter, IndexedDoc } from '@anticrm/server-core'
 import { MongoClient } from 'mongodb'
@@ -155,13 +156,14 @@ describe('mongo operations', () => {
       workspace: dbId
     }
     const serverStorage = await createServerStorage(conf)
-
+    const ctx = new MeasureMetricsContext('client', {})
     client = await createClient(async (handler) => {
-      return {
+      const st: ClientConnection = {
-        findAll: async (_class, query, options) => await serverStorage.findAll(_class, query, options),
+        findAll: async (_class, query, options) => await serverStorage.findAll(ctx, _class, query, options),
-        tx: async (tx) => await serverStorage.tx(tx),
+        tx: async (tx) => (await serverStorage.tx(ctx, tx))[0],
         close: async () => {}
       }
+      return st
     })
 
     operations = new TxOperations(client, core.account.System)

server/mongo/src/storage.ts

@@ -13,24 +13,19 @@
 // limitations under the License.
 //
 
-import type {
+import core, {
   Class,
   Doc,
-  DocumentQuery,
+  DocumentQuery, DOMAIN_MODEL, DOMAIN_TX, FindOptions,
-  FindOptions,
+  FindResult, Hierarchy, isOperator, ModelDb, Ref, SortingOrder, Tx,
-  FindResult,
-  Ref,
-  Tx,
   TxCreateDoc,
-  TxMixin,
+  TxMixin, TxProcessor, TxPutBag,
-  TxPutBag,
   TxRemoveDoc,
   TxResult,
   TxUpdateDoc
 } from '@anticrm/core'
-import core, { DOMAIN_MODEL, DOMAIN_TX, Hierarchy, isOperator, ModelDb, SortingOrder, TxProcessor } from '@anticrm/core'
 import type { DbAdapter, TxAdapter } from '@anticrm/server-core'
-import { Db, Document, Filter, Sort } from 'mongodb'
+import { Collection, Db, Document, Filter, Sort } from 'mongodb'
 import { getMongoClient } from './utils'
 
 function translateDoc (doc: Doc): Document {
@@ -270,12 +265,13 @@ class MongoAdapter extends MongoAdapterBase {
     }
   }
 
-  override tx (tx: Tx): Promise<TxResult> {
+  override async tx (tx: Tx): Promise<TxResult> {
-    return super.tx(tx)
+    return await super.tx(tx)
   }
 }
 
 class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
+  txColl: Collection | undefined
   protected txCreateDoc (tx: TxCreateDoc<Doc>): Promise<TxResult> {
     throw new Error('Method not implemented.')
   }
@@ -297,10 +293,18 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
   }
 
   override async tx (tx: Tx): Promise<TxResult> {
-    await this.db.collection(DOMAIN_TX).insertOne(translateDoc(tx))
+    await this.txCollection().insertOne(translateDoc(tx))
     return {}
   }
 
+  private txCollection (): Collection {
+    if (this.txColl !== undefined) {
+      return this.txColl
+    }
+    this.txColl = this.db.collection(DOMAIN_TX)
+    return this.txColl
+  }
+
   async getModel (): Promise<Tx[]> {
     return await this.db.collection(DOMAIN_TX).find<Tx>({ objectSpace: core.space.Model }).sort({ _id: 1 }).toArray()
   }

server/server/package.json

@@ -43,6 +43,7 @@
     "@anticrm/server-recruit": "~0.6.0",
     "@anticrm/server-recruit-resources": "~0.6.0",
     "@anticrm/mongo": "~0.6.1",
-    "@anticrm/elastic": "~0.6.0"
+    "@anticrm/elastic": "~0.6.0",
+    "elastic-apm-node": "~3.26.0"
   }
 }

server/server/src/__start.ts

@@ -14,6 +14,7 @@
 // limitations under the License.
 //
 
+// Add this to the VERY top of the first file loaded in your app
 import { start } from '.'
 
 const url = process.env.MONGO_URL
@@ -39,4 +40,3 @@ const close = (): void => {
 }
 process.on('SIGINT', close)
 process.on('SIGTERM', close)
-process.on('exit', close)

server/server/src/apm.ts

@@ -0,0 +1,85 @@
+import { MeasureContext, MeasureLogger, ParamType } from '@anticrm/core'
+import apm, { Agent, Span, Transaction } from 'elastic-apm-node'
+
+export let metricsContext: MeasureContext
+
+/**
+ * @public
+ */
+export function createAPMAgent (apmUrl: string): Agent {
+  const agent: Agent = apm.start({
+
+    // Override the service name from package.json
+    // Allowed characters: a-z, A-Z, 0-9, -, _, and space
+    serviceName: 'transactor',
+
+    // Use if APM Server requires a secret token
+    secretToken: '',
+
+    // Set the custom APM Server URL (default: http://localhost:8200)
+    serverUrl: apmUrl,
+    logLevel: 'trace'
+  })
+  return agent
+}
+
+/**
+ * @public
+ */
+export class APMMeasureContext implements MeasureContext {
+  logger: MeasureLogger
+  private readonly transaction?: Transaction | Span
+  private readonly parent?: Transaction | Span
+  constructor (private readonly agent: Agent, name: string, params: Record<string, ParamType>, parent?: Transaction | Span, noTransaction?: boolean) {
+    this.parent = parent
+    this.logger = {
+      info: (msg, args) => {
+        agent.logger.info(msg, args)
+      },
+      error: (msg, args) => {
+        agent.logger.error(msg, args)
+      }
+    }
+    if (!(noTransaction ?? false)) {
+      if (this.parent === undefined) {
+        this.transaction = agent.startTransaction(name) ?? undefined
+      } else {
+        this.transaction = agent.startSpan(name, { childOf: this.parent }) ?? undefined
+      }
+      for (const [k, v] of Object.entries(params)) {
+        this.transaction?.setLabel(k, v)
+      }
+    }
+  }
+
+  newChild (name: string, params: Record<string, ParamType>): MeasureContext {
+    return new APMMeasureContext(this.agent, name, params, this.transaction)
+  }
+
+  async with<T>(name: string, params: Record<string, ParamType>, op: (ctx: MeasureContext) => T | Promise<T>): Promise<T> {
+    const c = this.newChild(name, params)
+    try {
+      let value = op(c)
+      if (value instanceof Promise) {
+        value = await value
+      }
+      c.end()
+      return value
+    } catch (err: any) {
+      await c.error(err)
+      throw err
+    }
+  }
+
+  async error (err: any): Promise<void> {
+    return await new Promise((resolve) => {
+      this.agent.captureError(err, () => {
+        resolve()
+      })
+    })
+  }
+
+  end (): void {
+    this.transaction?.end()
+  }
+}

server/server/src/metrics.ts

@@ -0,0 +1,46 @@
+import { MeasureContext, MeasureMetricsContext, metricsToString, newMetrics } from '@anticrm/core'
+import { APMMeasureContext, createAPMAgent } from './apm'
+import { writeFile } from 'fs/promises'
+
+const apmUrl = process.env.APM_SERVER_URL
+const metricsFile = process.env.METRICS_FILE
+const metricsConsole = (process.env.METRICS_CONSOLE ?? 'false') === 'true'
+
+const METRICS_UPDATE_INTERVAL = 30000
+
+export let metricsContext: MeasureContext
+
+if (apmUrl === undefined) {
+  console.info('please provide apm server url for monitoring')
+
+  const metrics = newMetrics()
+  metricsContext = new MeasureMetricsContext('System', {}, metrics)
+
+  if (metricsFile !== undefined || metricsConsole) {
+    console.info('storing measurements into local file', metricsFile)
+    let oldMetricsValue = ''
+
+    const intTimer = setInterval(() => {
+      const val = metricsToString(metrics)
+      if (val !== oldMetricsValue) {
+        oldMetricsValue = val
+        if (metricsFile !== undefined) {
+          writeFile(metricsFile, val).catch((err) => console.error(err))
+        }
+        if (metricsConsole) {
+          console.info('METRICS:', val)
+        }
+      }
+    }, METRICS_UPDATE_INTERVAL)
+
+    const closeTimer = (): void => {
+      clearInterval(intTimer)
+    }
+    process.on('SIGINT', closeTimer)
+    process.on('SIGTERM', closeTimer
+    )
+  }
+} else {
+  console.log('using APM', apmUrl)
+  metricsContext = new APMMeasureContext(createAPMAgent(apmUrl), 'root', {}, undefined, true)
+}

server/server/src/server.ts

@@ -24,6 +24,7 @@ import type { DbConfiguration, DbAdapter } from '@anticrm/server-core'
 import { addLocation } from '@anticrm/platform'
 import { serverChunterId } from '@anticrm/server-chunter'
 import { serverRecruitId } from '@anticrm/server-recruit'
+import { metricsContext } from './metrics'
 
 class NullDbAdapter implements DbAdapter {
   async init (model: Tx[]): Promise<void> {}
@@ -42,7 +43,7 @@ export function start (dbUrl: string, fullTextUrl: string, port: number, host?:
   addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
   addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
 
-  return startJsonRpc((workspace: string) => {
+  return startJsonRpc(metricsContext, (workspace: string) => {
     const conf: DbConfiguration = {
       domains: {
         [DOMAIN_TX]: 'MongoTx',

server/ws/src/__tests__/server.test.ts

@@ -20,18 +20,20 @@ import type { Token } from '@anticrm/server-core'
 import { encode } from 'jwt-simple'
 import WebSocket from 'ws'
 
-import type { Doc, Ref, Class, DocumentQuery, FindOptions, FindResult, Tx, TxResult } from '@anticrm/core'
+import type { Doc, Ref, Class, DocumentQuery, FindOptions, FindResult, Tx, TxResult, MeasureContext } from '@anticrm/core'
+import { MeasureMetricsContext } from '@anticrm/core'
 
 describe('server', () => {
   disableLogging()
 
-  start(async () => ({
+  start(new MeasureMetricsContext('test', {}), async () => ({
     findAll: async <T extends Doc>(
+      ctx: MeasureContext,
       _class: Ref<Class<T>>,
       query: DocumentQuery<T>,
       options?: FindOptions<T>
     ): Promise<FindResult<T>> => ([]),
-    tx: async (tx: Tx): Promise<[TxResult, Tx[]]> => ([{}, []])
+    tx: async (ctx: MeasureContext, tx: Tx): Promise<[TxResult, Tx[]]> => ([{}, []])
   }), 3333)
 
   function connect (): WebSocket {

server/ws/src/server.ts

@@ -14,19 +14,18 @@
 // limitations under the License.
 //
 
-import { readRequest, serialize, Response } from '@anticrm/platform'
+import { Class, Doc, DocumentQuery, FindOptions, FindResult, MeasureContext, Ref, ServerStorage, Tx, TxResult } from '@anticrm/core'
+import { readRequest, Response, serialize, unknownError } from '@anticrm/platform'
 import type { Token } from '@anticrm/server-core'
 import { createServer, IncomingMessage } from 'http'
-import WebSocket, { Server } from 'ws'
 import { decode } from 'jwt-simple'
+import WebSocket, { Server } from 'ws'
 
-import type { Doc, Ref, Class, FindOptions, FindResult, Tx, DocumentQuery, Storage, ServerStorage, TxResult } from '@anticrm/core'
+let LOGGING_ENABLED = false
-
-let LOGGING_ENABLED = true
 
 export function disableLogging (): void { LOGGING_ENABLED = false }
 
-class Session implements Storage {
+class Session {
   constructor (
     private readonly manager: SessionManager,
     private readonly token: Token,
@@ -35,15 +34,16 @@ class Session implements Storage {
 
   async ping (): Promise<string> { console.log('ping'); return 'pong!' }
 
-  async findAll <T extends Doc>(_class: Ref<Class<T>>, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<FindResult<T>> {
+  async findAll <T extends Doc>(ctx: MeasureContext, _class: Ref<Class<T>>, query: DocumentQuery<T>, options?: FindOptions<T>): Promise<FindResult<T>> {
-    return await this.storage.findAll(_class, query, options)
+    return await this.storage.findAll(ctx, _class, query, options)
   }
 
-  async tx (tx: Tx): Promise<TxResult> {
+  async tx (ctx: MeasureContext, tx: Tx): Promise<TxResult> {
-    const [result, derived] = await this.storage.tx(tx)
+    const [result, derived] = await this.storage.tx(ctx, tx)
+
     this.manager.broadcast(this, this.token, { result: tx })
-    for (const tx of derived) {
+    for (const dtx of derived) {
-      this.manager.broadcast(null, this.token, { result: tx })
+      this.manager.broadcast(null, this.token, { result: dtx })
     }
     return result
   }
@@ -101,15 +101,19 @@ class SessionManager {
   }
 }
 
-async function handleRequest<S> (service: S, ws: WebSocket, msg: string): Promise<void> {
+async function handleRequest<S extends Session> (ctx: MeasureContext, service: S, ws: WebSocket, msg: string): Promise<void> {
   const request = readRequest(msg)
   const f = (service as any)[request.method]
   try {
-    const result = await f.apply(service, request.params)
+    const params = [ctx, ...request.params]
-    const resp = { id: request.id, result }
+    const result = await f.apply(service, params)
+    const resp: Response<any> = { id: request.id, result }
     ws.send(serialize(resp))
   } catch (err: any) {
-    const resp = { id: request.id, error: err }
+    const resp: Response<any> = {
+      id: request.id,
+      error: unknownError(err)
+    }
     ws.send(serialize(resp))
   }
 }
@@ -120,7 +124,7 @@ async function handleRequest<S> (service: S, ws: WebSocket, msg: string): Promis
  * @param port -
  * @param host -
  */
-export function start (storageFactory: (workspace: string) => Promise<ServerStorage>, port: number, host?: string): () => void {
+export function start (ctx: MeasureContext, storageFactory: (workspace: string) => Promise<ServerStorage>, port: number, host?: string): () => void {
   console.log(`starting server on port ${port} ...`)
 
   const sessions = new SessionManager()
@@ -133,11 +137,11 @@ export function start (storageFactory: (workspace: string) => Promise<ServerStor
     ws.on('message', (msg: string) => { buffer.push(msg) })
     const session = await sessions.addSession(ws, token, storageFactory)
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
-    ws.on('message', async (msg: string) => await handleRequest(session, ws, msg))
+    ws.on('message', async (msg: string) => await handleRequest(ctx, session, ws, msg))
     ws.on('close', (code: number, reason: string) => sessions.close(ws, token, code, reason))
 
     for (const msg of buffer) {
-      await handleRequest(session, ws, msg)
+      await handleRequest(ctx, session, ws, msg)
     }
   })