UBERF-5511: Fix query and include ibm plex mono (#4764)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.gitignore

@@ -9,6 +9,7 @@ temp/
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+tests/sanity/screenshots
 
 # Runtime data
 *.pid

packages/core/src/utils.ts

@@ -512,6 +512,9 @@ function getInNiN (query1: any, query2: any): any {
 }
 
 export function cutObjectArray (obj: any): any {
+  if (obj == null) {
+    return obj
+  }
   const r = {}
   for (const key of Object.keys(obj)) {
     if (Array.isArray(obj[key])) {

packages/platform-rig/bin/compile.js

@@ -132,7 +132,7 @@ async function performESBuild(filesToTranspile) {
     minify: false,
     outdir: 'lib',
     keepNames: true,
-    sourcemap: 'external',
+    sourcemap: 'inline',
     allowOverwrite: true,
     format: 'cjs',
     plugins: [
@@ -163,7 +163,7 @@ async function validateTSC(st) {
     [
       '-pretty',
       "--emitDeclarationOnly",
-      "--incremental",      
+      "--incremental",
       "--tsBuildInfoFile", ".validate/tsBuildInfoFile.info",
       "--declarationDir", "types",
       ...args.splice(1)

packages/query/src/index.ts

@@ -69,6 +69,8 @@ interface Query {
   options?: FindOptions<Doc>
   total: number
   callbacks: Map<string, Callback>
+
+  requested?: Promise<void>
 }
 
 /**
@@ -105,7 +107,7 @@ export class LiveQuery implements WithTx, Client {
     for (const q of [...this.queue]) {
       if (!this.removeFromQueue(q)) {
         try {
-          await this.refresh(q)
+          void this.refresh(q)
         } catch (err: any) {
           if (err instanceof PlatformError) {
             if (err.message === 'connection closed') {
@@ -119,7 +121,7 @@ export class LiveQuery implements WithTx, Client {
     for (const v of this.queries.values()) {
       for (const q of v) {
         try {
-          await this.refresh(q)
+          void this.refresh(q)
         } catch (err: any) {
           if (err instanceof PlatformError) {
             if (err.message === 'connection closed') {
@@ -728,7 +730,21 @@ export class LiveQuery implements WithTx, Client {
     return this.getHierarchy().clone(results) as T[]
   }
 
-  private async refresh (q: Query): Promise<void> {
+  private async refresh (q: Query, reRequest: boolean = false): Promise<void> {
+    if (q.requested !== undefined && !reRequest) {
+      // we already asked for refresh, just wait.
+      await q.requested
+      return
+    }
+    if (reRequest && q.requested !== undefined) {
+      await q.requested
+    }
+    q.requested = this.doRefresh(q)
+    await q.requested
+    q.requested = undefined
+  }
+
+  private async doRefresh (q: Query): Promise<void> {
     const res = await this.client.findAll(q._class, q.query, q.options)
     if (!deepEqual(res, q.result) || (res.total !== q.total && q.options?.total === true)) {
       q.result = res
@@ -737,18 +753,6 @@ export class LiveQuery implements WithTx, Client {
     }
   }
 
-  private triggerRefresh (q: Query): void {
-    const r: Promise<FindResult<Doc>> | FindResult<Doc> = this.client.findAll(q._class, q.query, q.options)
-    q.result = r
-    void r.then(async (qr) => {
-      const oldResult = q.result
-      if (!deepEqual(qr, oldResult) || (qr.total !== q.total && q.options?.total === true)) {
-        q.total = qr.total
-        await this.callback(q)
-      }
-    })
-  }
-
   // Check if query is partially matched.
   private async matchQuery (q: Query, tx: TxUpdateDoc<Doc>, docCache: Map<string, Doc>): Promise<boolean> {
     const clazz = this.getHierarchy().isMixin(q._class) ? this.getHierarchy().getBaseClass(q._class) : q._class
@@ -1137,16 +1141,31 @@ export class LiveQuery implements WithTx, Client {
     const docCache = new Map<string, Doc>()
     for (const tx of txes) {
       if (tx._class === core.class.TxWorkspaceEvent) {
-        await this.checkUpdateEvents(tx)
-        await this.changePrivateHandler(tx)
+        await this.checkUpdateEvents(tx as TxWorkspaceEvent)
+        await this.changePrivateHandler(tx as TxWorkspaceEvent)
       }
       result.push(await this._tx(tx, docCache))
     }
     return result
   }
 
-  private async checkUpdateEvents (tx: Tx): Promise<void> {
-    const evt = tx as TxWorkspaceEvent
+  triggerCounter = 0
+  searchTriggerTimer: any
+
+  private async checkUpdateEvents (evt: TxWorkspaceEvent, trigger = true): Promise<void> {
+    clearTimeout(this.searchTriggerTimer)
+
+    // We need to add trigger once more, since elastic could have a huge lag with document availability.
+    if (trigger || this.triggerCounter > 0) {
+      if (trigger) {
+        this.triggerCounter = 5 // Schedule 5 refreshes on every 5 seconds.
+      }
+      setTimeout(() => {
+        this.triggerCounter--
+        void this.checkUpdateEvents(evt, false)
+      }, 5000)
+    }
+
     const h = this.client.getHierarchy()
     function hasClass (q: Query, classes: Ref<Class<Doc>>[]): boolean {
       return classes.includes(q._class) || classes.some((it) => h.isDerived(q._class, it) || h.isDerived(it, q._class))
@@ -1157,7 +1176,7 @@ export class LiveQuery implements WithTx, Client {
         if (hasClass(q, indexingParam._class) && q.query.$search !== undefined) {
           if (!this.removeFromQueue(q)) {
             try {
-              this.triggerRefresh(q)
+              await this.refresh(q, true)
             } catch (err) {
               console.error(err)
             }
@@ -1167,11 +1186,14 @@ export class LiveQuery implements WithTx, Client {
       for (const v of this.queries.values()) {
         for (const q of v) {
           if (hasClass(q, indexingParam._class) && q.query.$search !== undefined) {
+            console.log('Query update call', q)
             try {
-              this.triggerRefresh(q)
+              await this.refresh(q, true)
             } catch (err) {
               console.error(err)
             }
+          } else if (q.query.$search !== undefined) {
+            console.log('Query update mismatch class', q._class, indexingParam._class)
           }
         }
       }
@@ -1182,7 +1204,7 @@ export class LiveQuery implements WithTx, Client {
         if (hasClass(q, params._class)) {
           if (!this.removeFromQueue(q)) {
             try {
-              this.triggerRefresh(q)
+              await this.refresh(q, true)
             } catch (err) {
               console.error(err)
             }
@@ -1193,7 +1215,7 @@ export class LiveQuery implements WithTx, Client {
         for (const q of v) {
           if (hasClass(q, params._class)) {
             try {
-              this.triggerRefresh(q)
+              await this.refresh(q, true)
             } catch (err) {
               console.error(err)
             }
@@ -1203,8 +1225,7 @@ export class LiveQuery implements WithTx, Client {
     }
   }
 
-  private async changePrivateHandler (tx: Tx): Promise<void> {
-    const evt = tx as TxWorkspaceEvent
+  private async changePrivateHandler (evt: TxWorkspaceEvent): Promise<void> {
     if (evt.event === WorkspaceEvent.SecurityChange) {
       for (const q of [...this.queue]) {
         if (typeof q.query.space !== 'string') {

packages/theme/styles/global.scss

@@ -29,7 +29,7 @@
 @import "./tables.scss";
 @import "./_text-editor.scss";
 
-@import url('https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:ital,wght@0,400;0,500;1,400;1,500&display=swap');
+@import "./mono.scss";
 
 @font-face {
   font-family: 'IBM Plex Sans';

packages/theme/styles/mono.scss

@@ -0,0 +1,36 @@
+@font-face {
+  font-family: 'IBM Plex Mono';
+  font-style: normal;
+  font-weight: 400;
+  src: local('IBM Plex Mono'),
+    local('IBMPlexMono'),
+    url('../fonts/complete/woff2/mono/IBMPlexMono-Regular.woff2') format('woff2'),
+    url('../fonts/complete/woff/mono/IBMPlexMono-Regular.woff') format('woff');
+}
+@font-face {
+  font-family: 'IBM Plex Mono';
+  font-style: normal;
+  font-weight: 500;
+  src: local('IBM Plex Mono Medium'),
+    local('IBMPlexMono-Medium'),
+    url('../fonts/complete/woff2/mono/IBMPlexMono-Medium.woff2') format('woff2'),
+    url('../fonts/complete/woff/mono/IBMPlexMono-Medium.woff') format('woff');
+}
+@font-face {
+  font-family: 'IBM Plex Mono';
+  font-style: normal;
+  font-weight: 600;
+  src: local('IBM Plex Mono SemiBold'),
+    local('IBMPlexMono-SemiBold'),
+    url('../fonts/complete/woff2/mono/IBMPlexMono-SemiBold.woff2') format('woff2'),
+    url('../fonts/complete/woff/mono/IBMPlexMono-SemiBold.woff') format('woff');
+}
+@font-face {
+  font-family: 'IBM Plex Mono';
+  font-style: normal;
+  font-weight: 700;
+  src: local('IBM Plex Mono Bold'),
+    local('IBMPlexMono-Bold'),
+    url('../fonts/complete/woff2/mono/IBMPlexMono-Bold.woff2') format('woff2'),
+    url('../fonts/complete/woff/mono/IBMPlexMono-Bold.woff') format('woff');
+}

packages/ui/src/components/EditWithIcon.svelte

@@ -25,7 +25,7 @@
 
   export let icon: Asset | AnySvelteComponent | ComponentType
   export let width: string | undefined = undefined
-  export let value: string | undefined = undefined
+  export let value: string | undefined = ''
   export let placeholder: IntlString = plugin.string.EditBoxPlaceholder
   export let placeholderParam: any | undefined = undefined
   export let autoFocus: boolean = false
@@ -34,15 +34,15 @@
 
   const dispatch = createEventDispatcher()
   let textHTML: HTMLInputElement
-  let phTraslate: string = ''
+  let phTranslate: string = ''
 
-  export function focus () {
+  export function focus (): void {
     textHTML.focus()
     autoFocus = false
   }
 
-  $: translate(placeholder, placeholderParam ?? {}).then((res) => {
-    phTraslate = res
+  $: void translate(placeholder, placeholderParam ?? {}).then((res) => {
+    phTranslate = res
   })
   $: if (textHTML !== undefined) {
     if (autoFocus) focus()
@@ -53,13 +53,13 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <div
   class="flex-between editbox {size}"
-  style={width ? 'width: ' + width : ''}
+  style={width != null ? 'width: ' + width : ''}
   on:click={() => {
     textHTML.focus()
   }}
 >
   <div class="mr-2 content-dark-color"><Icon {icon} size={'small'} /></div>
-  <input bind:this={textHTML} type="text" bind:value placeholder={phTraslate} on:change on:input on:keydown />
+  <input bind:this={textHTML} type="text" bind:value placeholder={phTranslate} on:change on:input on:keydown />
   <slot name="extra" />
   <div class="flex-row-center flex-no-shrink">
     {#if value}

packages/ui/src/components/SearchEdit.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { createEventDispatcher } from 'svelte'
+  import { createEventDispatcher, onDestroy } from 'svelte'
   import EditWithIcon from './EditWithIcon.svelte'
   import IconSearch from './icons/Search.svelte'
   import plugin from '../plugin'
@@ -11,13 +11,16 @@
   const dispatch = createEventDispatcher()
   let timer: any
 
-  function restartTimer () {
+  function restartTimer (): void {
     clearTimeout(timer)
     timer = setTimeout(() => {
       value = _search
       dispatch('change', _search)
     }, 500)
   }
+  onDestroy(() => {
+    clearTimeout(timer)
+  })
 </script>
 
 <EditWithIcon
@@ -26,20 +29,14 @@
   placeholder={plugin.string.Search}
   bind:value={_search}
   on:change={() => {
-    if (_search === '') {
-      value = ''
-      dispatch('change', '')
-    }
+    restartTimer()
   }}
   on:input={() => {
     restartTimer()
-    if (_search === '') {
-      value = ''
-      dispatch('change', '')
-    }
   }}
   on:keydown={(evt) => {
     if (evt.key === 'Enter') {
+      clearTimeout(timer)
       value = _search
       dispatch('change', _search)
     }

plugins/devmodel-resources/src/index.ts

@@ -14,6 +14,8 @@
 //
 
 import core, {
+  DOMAIN_MODEL,
+  cutObjectArray,
   type Account,
   type AccountClient,
   type Class,
@@ -23,20 +25,20 @@ import core, {
   type FindOptions,
   type FindResult,
   type Hierarchy,
+  type MeasureDoneOperation,
   type ModelDb,
   type Ref,
+  type SearchOptions,
+  type SearchQuery,
+  type SearchResult,
   type Tx,
   type TxResult,
-  type WithLookup,
-  type SearchQuery,
-  type SearchOptions,
-  type SearchResult,
-  type MeasureDoneOperation,
-  DOMAIN_MODEL
+  type WithLookup
 } from '@hcengineering/core'
 import { devModelId } from '@hcengineering/devmodel'
 import { Builder } from '@hcengineering/model'
-import { type IntlString, type Resources, getMetadata } from '@hcengineering/platform'
+import { getMetadata, type IntlString, type Resources } from '@hcengineering/platform'
+import { testing } from '@hcengineering/ui'
 import view from '@hcengineering/view'
 import workbench from '@hcengineering/workbench'
 import ModelView from './components/ModelView.svelte'
@@ -63,7 +65,11 @@ class ModelClient implements AccountClient {
     client.notify = (...tx) => {
       this.notify?.(...tx)
       if (this.notifyEnabled) {
-        console.debug('devmodel# notify=>', tx, this.client.getModel(), getMetadata(devmodel.metadata.DevModel))
+        console.debug(
+          'devmodel# notify=>',
+          testing ? JSON.stringify(cutObjectArray(tx)).slice(0, 160) : tx,
+          getMetadata(devmodel.metadata.DevModel)
+        )
       }
     }
   }
@@ -97,13 +103,11 @@ class ModelClient implements AccountClient {
     if (this.notifyEnabled && !isModel) {
       console.debug(
         'devmodel# findOne=>',
-        isModel,
-        this.getHierarchy().findDomain(_class),
         _class,
-        query,
+        testing ? JSON.stringify(cutObjectArray(query)) : query,
         options,
         'result => ',
-        result,
+        testing ? JSON.stringify(cutObjectArray(result)) : result,
         ' =>model',
         this.client.getModel(),
         getMetadata(devmodel.metadata.DevModel),
@@ -125,10 +129,10 @@ class ModelClient implements AccountClient {
       console.debug(
         'devmodel# findAll=>',
         _class,
-        query,
+        testing ? JSON.stringify(cutObjectArray(query)).slice(0, 160) : query,
         options,
         'result => ',
-        result,
+        testing ? JSON.stringify(cutObjectArray(result)).slice(0, 160) : result,
         ' =>model',
         this.client.getModel(),
         getMetadata(devmodel.metadata.DevModel),
@@ -141,7 +145,13 @@ class ModelClient implements AccountClient {
   async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
     const result = await this.client.searchFulltext(query, options)
     if (this.notifyEnabled) {
-      console.debug('devmodel# searchFulltext=>', query, options, 'result => ', result)
+      console.debug(
+        'devmodel# searchFulltext=>',
+        testing ? JSON.stringify(cutObjectArray(query)).slice(0, 160) : query,
+        options,
+        'result => ',
+        result
+      )
     }
     return result
   }
@@ -150,7 +160,13 @@ class ModelClient implements AccountClient {
     const startTime = Date.now()
     const result = await this.client.tx(tx)
     if (this.notifyEnabled) {
-      console.debug('devmodel# tx=>', tx, result, getMetadata(devmodel.metadata.DevModel), Date.now() - startTime)
+      console.debug(
+        'devmodel# tx=>',
+        testing ? JSON.stringify(cutObjectArray(tx)).slice(0, 160) : tx,
+        result,
+        getMetadata(devmodel.metadata.DevModel),
+        Date.now() - startTime
+      )
     }
     return result
   }

plugins/view-resources/src/components/list/List.svelte

@@ -54,6 +54,9 @@
 
   const limiter = new RateLimiter(10)
 
+  const client = getClient()
+  const hierarchy = client.getHierarchy()
+
   let docs: Doc[] = []
   let fastDocs: Doc[] = []
   let slowDocs: Doc[] = []
@@ -146,12 +149,9 @@
     return resultOptions
   }
 
-  $: dispatch('content', docs)
-
   const dispatch = createEventDispatcher()
 
-  const client = getClient()
-  const hierarchy = client.getHierarchy()
+  $: dispatch('content', docs)
 
   async function getResultQuery (
     query: DocumentQuery<Doc>,
@@ -174,7 +174,7 @@
     return result
   }
 
-  function uncheckAll () {
+  function uncheckAll (): void {
     dispatch('check', { docs, value: false })
     selectedObjectIds = []
   }
@@ -197,7 +197,7 @@
 <div class="list-container" bind:this={listDiv}>
   <ListCategories
     bind:this={listCategories}
-    newObjectProps={() => (space ? { space } : {})}
+    newObjectProps={() => (space != null ? { space } : {})}
     {docs}
     {_class}
     {space}

server/core/src/indexer/indexer.ts

@@ -267,6 +267,8 @@ export class FullTextIndexPipeline implements FullTextPipeline {
     await this.flush(flush ?? false)
   }
 
+  triggerCounts = 0
+
   triggerIndexing = (): void => {}
   skippedReiterationTimeout: any
   currentStages: Record<string, number> = {}
@@ -341,6 +343,9 @@ export class FullTextIndexPipeline implements FullTextPipeline {
     })
 
     while (!this.cancelling) {
+      // Clear triggers
+      this.triggerCounts = 0
+      this.stageChanged = 0
       await this.metrics.with('initialize-stages', { workspace: this.workspace.name }, async () => {
         await this.initializeStages()
       })
@@ -360,7 +365,11 @@ export class FullTextIndexPipeline implements FullTextPipeline {
 
       _classes.forEach((it) => this.broadcastClasses.add(it))
 
-      if (this.toIndex.size === 0 || this.stageChanged === 0) {
+      if (this.triggerCounts > 0) {
+        console.log('No wait, trigger counts', this.triggerCounts)
+      }
+
+      if (this.toIndex.size === 0 && this.stageChanged === 0 && this.triggerCounts === 0) {
         if (this.toIndex.size === 0) {
           console.log(this.workspace.name, 'Indexing complete', this.indexId)
         }
@@ -374,6 +383,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
 
           await new Promise((resolve) => {
             this.triggerIndexing = () => {
+              this.triggerCounts++
               resolve(null)
               clearTimeout(this.skippedReiterationTimeout)
             }

tests/sanity/tests/model/tracker/issues-page.ts

@@ -1,8 +1,8 @@
 import { expect, type Locator, type Page } from '@playwright/test'
-import { NewIssue } from './types'
 import path from 'path'
-import { CommonTrackerPage } from './common-tracker-page'
 import { attachScreenshot, iterateLocator } from '../../utils'
+import { CommonTrackerPage } from './common-tracker-page'
+import { NewIssue } from './types'
 
 export class IssuesPage extends CommonTrackerPage {
   readonly page: Page
@@ -142,8 +142,14 @@ export class IssuesPage extends CommonTrackerPage {
   }
 
   async searchIssueByName (issueName: string): Promise<void> {
-    await this.inputSearch.fill(issueName)
-    await this.page.waitForTimeout(3000)
+    for (let i = 0; i < 5; i++) {
+      await this.inputSearch.fill(issueName)
+      const v = await this.inputSearch.inputValue()
+      if (v === issueName) {
+        await this.inputSearch.press('Enter')
+        break
+      }
+    }
   }
 
   async openIssueByName (issueName: string): Promise<void> {