UBER-834: Improve list speed (#3692)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2023-09-14 14:17:44 +03:00 committed by GitHub
parent af94b640c5
commit 61cc74d7a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 299 additions and 134 deletions

View File

@ -676,7 +676,7 @@ export function createModel (builder: Builder): void {
const applicantViewOptions = (colors: boolean): ViewOptionsModel => {
const model: ViewOptionsModel = {
groupBy: ['status', 'assignee', 'space', 'createdBy', 'modifiedBy'],
groupBy: ['status', 'doneState', 'assignee', 'space', 'createdBy', 'modifiedBy'],
orderBy: [
['status', SortingOrder.Ascending],
['modifiedOn', SortingOrder.Descending],
@ -784,6 +784,105 @@ export function createModel (builder: Builder): void {
recruit.viewlet.ListApplicant
)
builder.createDoc(
view.class.Viewlet,
core.space.Model,
{
attachTo: recruit.mixin.Candidate,
descriptor: view.viewlet.List,
config: [
{ key: '', displayProps: { fixed: 'left', key: 'app' } },
{
key: 'title',
props: { kind: 'list', size: 'small', shouldShowName: false }
},
{ key: 'comments', displayProps: { key: 'comments', suffix: true } },
{ key: '', displayProps: { grow: true } },
{
key: '$lookup.channels',
label: contact.string.ContactInfo,
sortingKey: ['$lookup.channels.lastMessage', '$lookup.attachedTo.channels'],
props: {
length: 'full',
size: 'small',
kind: 'list'
},
displayProps: { compression: true }
},
{ key: 'modifiedOn', displayProps: { key: 'modified', fixed: 'right', dividerBefore: true } }
],
configOptions: {
strict: true,
hiddenKeys: ['name']
},
viewOptions: {
groupBy: ['createdBy', 'modifiedBy'],
orderBy: [
['modifiedOn', SortingOrder.Descending],
['createdOn', SortingOrder.Descending],
['rank', SortingOrder.Ascending]
],
other: [showColorsViewOption]
}
},
recruit.viewlet.ListTalent
)
builder.createDoc(
view.class.Viewlet,
core.space.Model,
{
attachTo: recruit.mixin.VacancyList,
descriptor: view.viewlet.List,
config: [
{ key: '', displayProps: { fixed: 'left', key: 'app' } },
{
key: '@vacancies',
label: recruit.string.Vacancies,
props: { kind: 'list', size: 'small', shouldShowName: false }
},
{
key: '@applications',
label: recruit.string.Applications,
props: { kind: 'list', size: 'small', shouldShowName: false }
},
{ key: 'comments', displayProps: { key: 'comments', suffix: true } },
{
key: '$lookup.channels',
label: contact.string.ContactInfo,
sortingKey: ['$lookup.channels.lastMessage', '$lookup.attachedTo.channels'],
props: {
length: 'full',
size: 'small',
kind: 'list'
},
displayProps: { compression: true }
},
{ key: '', displayProps: { grow: true } },
{
key: '@applications.modifiedOn',
label: core.string.ModifiedDate,
displayProps: { key: 'modified', fixed: 'right', dividerBefore: true }
}
],
configOptions: {
strict: true,
sortable: true,
hiddenKeys: ['name', 'space', 'modifiedOn']
},
viewOptions: {
groupBy: ['createdBy', 'modifiedBy'],
orderBy: [
['modifiedOn', SortingOrder.Descending],
['createdOn', SortingOrder.Descending],
['rank', SortingOrder.Ascending]
],
other: [showColorsViewOption]
}
},
recruit.viewlet.ListCompanies
)
builder.createDoc(
view.class.Viewlet,
core.space.Model,

View File

@ -120,6 +120,8 @@ export default mergeIds(recruitId, recruit, {
ApplicantTable: '' as Ref<Viewlet>,
ApplicantKanban: '' as Ref<Viewlet>,
ListApplicant: '' as Ref<Viewlet>,
ListTalent: '' as Ref<Viewlet>,
ListCompanies: '' as Ref<Viewlet>,
TableApplicant: '' as Ref<Viewlet>,
TableApplicantMatch: '' as Ref<Viewlet>,
CalendarReview: '' as Ref<Viewlet>,

View File

@ -280,3 +280,47 @@ export class DocManager {
return this.docs.filter(predicate)
}
}
/**
* @public
*/
export class RateLimitter {
idCounter: number = 0
processingQueue = new Map<string, Promise<void>>()
last: number = 0
queue: (() => Promise<void>)[] = []
constructor (readonly config: () => { rate: number, perSecond?: number }) {}
async exec<T, B extends Record<string, any> = {}>(op: (args?: B) => Promise<T>, args?: B): Promise<T> {
const processingId = `${this.idCounter++}`
const cfg = this.config()
while (this.processingQueue.size > cfg.rate) {
await Promise.race(this.processingQueue.values())
}
try {
const p = op(args)
this.processingQueue.set(processingId, p as Promise<void>)
return await p
} finally {
this.processingQueue.delete(processingId)
}
}
async add<T, B extends Record<string, any> = {}>(op: (args?: B) => Promise<T>, args?: B): Promise<void> {
const cfg = this.config()
if (this.processingQueue.size < cfg.rate) {
void this.exec(op, args)
} else {
await this.exec(op, args)
}
}
async waitProcessing (): Promise<void> {
await await Promise.race(this.processingQueue.values())
}
}

View File

@ -1,5 +1,5 @@
<script lang="ts">
export let size: 'small' | 'medium' | 'large'
export let size: 'tiny' | 'small' | 'medium' | 'large'
export let fill: string = 'currentColor'
</script>

View File

@ -14,18 +14,12 @@
-->
<script lang="ts">
import { Organization } from '@hcengineering/contact'
import core, { Doc, DocumentQuery, Ref } from '@hcengineering/core'
import core, { Doc, DocumentQuery, Ref, WithLookup } from '@hcengineering/core'
import { createQuery } from '@hcengineering/presentation'
import { Applicant, Vacancy } from '@hcengineering/recruit'
import { Button, IconAdd, Label, Loading, SearchEdit, showPopup } from '@hcengineering/ui'
import view, { BuildModelKey, Viewlet, ViewletPreference, ViewOptions } from '@hcengineering/view'
import {
FilterBar,
FilterButton,
TableBrowser,
ViewletSelector,
ViewletSettingButton
} from '@hcengineering/view-resources'
import { Button, Component, IconAdd, Label, Loading, SearchEdit, showPopup } from '@hcengineering/ui'
import view, { BuildModelKey, ViewOptions, Viewlet, ViewletPreference } from '@hcengineering/view'
import { FilterBar, FilterButton, ViewletSelector, ViewletSettingButton } from '@hcengineering/view-resources'
import recruit from '../plugin'
import CreateOrganization from './CreateOrganization.svelte'
import VacancyListApplicationsPopup from './organizations/VacancyListApplicationsPopup.svelte'
@ -175,14 +169,17 @@
]
])
let viewlet: Viewlet | undefined
let viewlet: WithLookup<Viewlet> | undefined
let loading = true
let preference: ViewletPreference | undefined
let viewOptions: ViewOptions | undefined
function createConfig (descr: Viewlet, preference: ViewletPreference | undefined): (string | BuildModelKey)[] {
const base = preference?.config ?? descr.config
function createConfig (
descr: Viewlet | undefined,
preference: ViewletPreference | undefined
): (string | BuildModelKey)[] {
const base = preference?.config ?? descr?.config ?? []
const result: (string | BuildModelKey)[] = []
for (const key of base) {
if (typeof key === 'string') {
@ -193,13 +190,24 @@
}
return result
}
$: finalConfig = createConfig(viewlet, preference)
</script>
<div class="ac-header full divide">
<div class="ac-header__wrap-title mr-3">
<span class="ac-header__title"><Label label={recruit.string.Organizations} /></span>
</div>
<div class="clear-mins mb-1">
<div class="ac-header-full medium-gap mb-1">
<ViewletSelector
bind:loading
bind:viewlet
bind:preference
viewletQuery={{
attachTo: recruit.mixin.VacancyList,
descriptor: { $in: [view.viewlet.Table, view.viewlet.List] }
}}
/>
<Button icon={IconAdd} label={recruit.string.CompanyCreateLabel} kind={'accented'} on:click={showCreateDialog} />
</div>
</div>
@ -211,16 +219,6 @@
<FilterButton _class={recruit.mixin.VacancyList} />
</div>
<div class="ac-header-full medium-gap">
<ViewletSelector
hidden
viewletQuery={{
attachTo: recruit.mixin.VacancyList,
descriptor: view.viewlet.Table
}}
bind:preference
bind:loading
bind:viewlet
/>
<ViewletSettingButton bind:viewOptions bind:viewlet />
<!-- <ActionIcon icon={IconMoreH} size={'small'} /> -->
</div>
@ -234,19 +232,22 @@
on:change={(e) => (resultQuery = e.detail)}
/>
{#if viewlet}
{#if loading}
<Loading />
{:else}
<TableBrowser
_class={recruit.mixin.VacancyList}
config={createConfig(viewlet, preference)}
options={viewlet.options}
query={{
{#if loading}
<Loading />
{:else if viewlet && viewlet?.$lookup?.descriptor?.component}
<Component
is={viewlet.$lookup.descriptor.component}
props={{
_class: recruit.mixin.VacancyList,
options: viewlet.options,
config: finalConfig,
viewlet,
viewOptions,
viewOptionsConfig: viewlet.viewOptions?.other,
query: {
...resultQuery
}}
totalQuery={{}}
showNotification
/>
{/if}
},
totalQuery: {}
}}
/>
{/if}

View File

@ -13,7 +13,7 @@
// limitations under the License.
-->
<script lang="ts">
import { Class, Doc, DocumentQuery, FindOptions, Ref, Space } from '@hcengineering/core'
import { Class, Doc, DocumentQuery, FindOptions, Ref, Space, RateLimitter } from '@hcengineering/core'
import { IntlString, getResource } from '@hcengineering/platform'
import { createQuery, getClient } from '@hcengineering/presentation'
import { AnyComponent, AnySvelteComponent } from '@hcengineering/ui'
@ -43,11 +43,16 @@
export let documents: Doc[] | undefined = undefined
const limiter = new RateLimitter(() => ({ rate: 10 }))
let docs: Doc[] = []
let fastDocs: Doc[] = []
let slowDocs: Doc[] = []
$: orderBy = viewOptions.orderBy
const docsQuery = createQuery()
const docsQuerySlow = createQuery()
$: lookup = buildConfigLookup(client.getHierarchy(), _class, config, options?.lookup)
$: resultOptions = { ...options, lookup, ...(orderBy !== undefined ? { sort: { [orderBy[0]]: orderBy[1] } } : {}) }
@ -59,12 +64,35 @@
$: queryNoLookup = noLookup(resultQuery)
let fastQueryIds: Ref<Doc>[] = []
$: if (documents === undefined) {
docsQuery.query(
_class,
queryNoLookup,
(res) => {
docs = res
fastDocs = res
fastQueryIds = res.map((it) => it._id)
},
{
...resultOptions,
projection: {
...resultOptions.projection,
_id: 1,
_class: 1,
...getProjection(viewOptions.groupBy, queryNoLookup)
},
limit: 1000
}
)
} else {
docsQuery.unsubscribe()
}
$: if (fastQueryIds.length > 0) {
docsQuerySlow.query(
_class,
{ ...queryNoLookup, _id: { $nin: fastQueryIds } },
(res) => {
slowDocs = res
},
{
...resultOptions,
@ -77,7 +105,12 @@
}
)
} else {
docsQuery.unsubscribe()
docsQuerySlow.unsubscribe()
}
$: if (documents === undefined) {
docs = [...fastDocs, ...slowDocs]
} else {
docs = documents
}
@ -170,6 +203,7 @@
{viewOptions}
{viewOptionsConfig}
{selectedObjectIds}
{limiter}
level={0}
groupPersistKey={''}
{createItemDialog}

View File

@ -21,6 +21,7 @@
FindOptions,
generateId,
Lookup,
RateLimitter,
Ref,
Space
} from '@hcengineering/core'
@ -50,6 +51,7 @@
import ListCategory from './ListCategory.svelte'
export let docs: Doc[]
export let docKeys: Partial<DocumentQuery<Doc>> = {}
export let _class: Ref<Class<Doc>>
export let space: Ref<Space> | undefined
export let query: DocumentQuery<Doc> | undefined
@ -80,6 +82,7 @@
export let resultQuery: DocumentQuery<Doc>
export let resultOptions: FindOptions<Doc>
export let limiter: RateLimitter
$: groupByKey = viewOptions.groupBy[level] ?? noCategory
let categories: CategoryType[] = []
@ -138,7 +141,7 @@
$: getHeader(_class, groupByKey)
let updateCounter = 0
let configurationsVersion = 0
async function buildModels (
_class: Ref<Class<Doc>>,
config: (string | BuildModelKey)[],
@ -161,6 +164,7 @@
if (id === updateCounter) {
itemModels = newItemModels
configurationsVersion = updateCounter
for (const [, v] of Object.entries(newItemModels)) {
// itemModels = itemModels
;(v as AttributeModel[]).forEach((m: AttributeModel) => {
@ -341,6 +345,7 @@
{#each categories as category, i (typeof category === 'object' ? category.name : category)}
{@const items = groupByKey === noCategory ? docs : getGroupByValues(groupByDocs, category)}
{@const categoryDocKeys = { ...docKeys, [groupByKey]: category }}
<ListCategory
bind:this={listListCategory[i]}
{extraHeaders}
@ -355,14 +360,17 @@
index={i}
{config}
{configurations}
{configurationsVersion}
{itemModels}
{_class}
parentCategories={categories.length}
groupPersistKey={`${groupPersistKey}_${level}_${typeof category === 'object' ? category.name : category}`}
singleCat={level === 0 && categories.length === 1}
oneCat={viewOptions.groupBy.length === 1}
lastCat={i === categories.length - 1}
{category}
itemProj={items}
docKeys={categoryDocKeys}
{newObjectProps}
{createItemDialog}
{createItemDialogProps}
@ -371,6 +379,7 @@
{compactMode}
{resultQuery}
{resultOptions}
{limiter}
on:check
on:uncheckAll
on:row-focus
@ -423,12 +432,14 @@
{flatHeaders}
{props}
{level}
docKeys={categoryDocKeys}
groupPersistKey={`${groupPersistKey}_${level}_${typeof category === 'object' ? category.name : category}`}
{initIndex}
{viewOptionsConfig}
{listDiv}
{resultQuery}
{resultOptions}
{limiter}
bind:dragItem
on:dragItem
on:check

View File

@ -20,8 +20,10 @@
DocumentQuery,
DocumentUpdate,
FindOptions,
Hierarchy,
Lookup,
PrimitiveType,
RateLimitter,
Ref,
Space
} from '@hcengineering/core'
@ -37,7 +39,7 @@
mouseAttractor,
showPopup
} from '@hcengineering/ui'
import { AttributeModel, BuildModelKey, Viewlet, ViewOptionModel, ViewOptions } from '@hcengineering/view'
import { AttributeModel, BuildModelKey, ViewOptionModel, ViewOptions, Viewlet } from '@hcengineering/view'
import { createEventDispatcher } from 'svelte'
import { fade } from 'svelte/transition'
import { FocusSelection, focusStore } from '../../selection'
@ -54,6 +56,7 @@
export let space: Ref<Space> | undefined
export let baseMenuClass: Ref<Class<Doc>> | undefined
export let itemProj: Doc[]
export let docKeys: Partial<DocumentQuery<Doc>> = {}
export let createItemDialog: AnyComponent | AnySvelteComponent | undefined
export let createItemDialogProps: Record<string, any> | undefined
export let createItemLabel: IntlString | undefined
@ -68,6 +71,7 @@
export let _class: Ref<Class<Doc>>
export let config: (string | BuildModelKey)[]
export let configurations: Record<Ref<Class<Doc>>, Viewlet['config']> | undefined
export let configurationsVersion: number
export let viewOptions: ViewOptions
export let newObjectProps: (doc: Doc | undefined) => Record<string, any> | undefined
export let viewOptionsConfig: ViewOptionModel[] | undefined
@ -81,6 +85,8 @@
export let compactMode: boolean = false
export let resultQuery: DocumentQuery<Doc>
export let resultOptions: FindOptions<Doc>
export let parentCategories: number = 0
export let limiter: RateLimitter
$: lastLevel = level + 1 >= viewOptions.groupBy.length
@ -95,14 +101,16 @@
$: limit = initialLimit
$: if (lastLevel) {
docsQuery.query(
_class,
{ ...resultQuery, _id: { $in: itemProj.map((it) => it._id) } },
(res) => {
items = res
},
{ ...resultOptions, limit: limit ?? 200 }
)
limiter.add(async () => {
docsQuery.query(
_class,
{ ...resultQuery, ...docKeys },
(res) => {
items = res
},
{ ...resultOptions, limit: limit ?? 200 }
)
})
} else {
docsQuery.unsubscribe()
}
@ -120,13 +128,18 @@
return res
}
function initCollapsed (singleCat: boolean, lastLevel: boolean): void {
function initCollapsed (singleCat: boolean, lastLevel: boolean, level: number): void {
if (localStorage.getItem(categoryCollapseKey) === null) {
collapsed = !disableHeader && !singleCat && itemProj.length > (lastLevel ? autoFoldLimit : singleCategoryLimit)
collapsed =
(!disableHeader &&
!singleCat &&
itemProj.length > (lastLevel ? autoFoldLimit : singleCategoryLimit) / (level + 1)) ||
parentCategories > 10 ||
(level > 1 && parentCategories > 5)
}
}
$: initCollapsed(singleCat, lastLevel)
$: initCollapsed(singleCat, lastLevel, level)
const handleRowFocused = (object: Doc) => {
dispatch('row-focus', object)
@ -463,6 +476,7 @@
{createItemDialog}
{createItemLabel}
{viewOptions}
{docKeys}
newObjectProps={_newObjectProps}
{flatHeaders}
{props}
@ -475,38 +489,40 @@
/>
{:else if itemModels && itemModels.size > 0 && (!collapsed || wasLoaded || dragItemIndex !== undefined)}
{#if limited && !loading}
{#each limited as docObject, i (docObject._id)}
<ListItem
bind:this={listItems[i]}
{docObject}
model={getDocItemModel(docObject._class)}
{groupByKey}
selected={isSelected(docObject, $focusStore)}
checked={selectedObjectIdsSet.has(docObject._id)}
last={i === limited.length - 1}
lastCat={i === limited.length - 1 && (oneCat || lastCat)}
on:dragstart={(e) => dragStart(e, docObject, i)}
on:dragenter={(e) => {
if (dragItemIndex !== undefined) {
e.stopPropagation()
e.preventDefault()
}
}}
on:dragleave={(e) => dragItemLeave(e, i)}
on:dragover={(e) => dragover(e, i)}
on:drop={dropItemHandle}
on:check={(ev) => dispatch('check', { docs: ev.detail.docs, value: ev.detail.value })}
on:contextmenu={(event) => handleMenuOpened(event, docObject)}
on:focus={() => {}}
on:mouseover={mouseAttractor(() => handleRowFocused(docObject))}
on:mouseenter={mouseAttractor(() => handleRowFocused(docObject))}
{props}
{compactMode}
on:on-mount={() => {
wasLoaded = true
}}
/>
{/each}
{#key configurationsVersion}
{#each limited as docObject, i (docObject._id)}
<ListItem
bind:this={listItems[i]}
{docObject}
model={getDocItemModel(Hierarchy.mixinOrClass(docObject))}
{groupByKey}
selected={isSelected(docObject, $focusStore)}
checked={selectedObjectIdsSet.has(docObject._id)}
last={i === limited.length - 1}
lastCat={i === limited.length - 1 && (oneCat || lastCat)}
on:dragstart={(e) => dragStart(e, docObject, i)}
on:dragenter={(e) => {
if (dragItemIndex !== undefined) {
e.stopPropagation()
e.preventDefault()
}
}}
on:dragleave={(e) => dragItemLeave(e, i)}
on:dragover={(e) => dragover(e, i)}
on:drop={dropItemHandle}
on:check={(ev) => dispatch('check', { docs: ev.detail.docs, value: ev.detail.value })}
on:contextmenu={(event) => handleMenuOpened(event, docObject)}
on:focus={() => {}}
on:mouseover={mouseAttractor(() => handleRowFocused(docObject))}
on:mouseenter={mouseAttractor(() => handleRowFocused(docObject))}
{props}
{compactMode}
on:on-mount={() => {
wasLoaded = true
}}
/>
{/each}
{/key}
{/if}
{:else if loading}
<Spinner size="small" />

View File

@ -111,9 +111,9 @@
on:click={() => dispatch('collapse')}
>
<div class="flex-row-center flex-grow" style:color={headerComponent ? headerTextColor : 'inherit'}>
{#if level === 0}
<div class="chevron"><IconCollapseArrow size={'small'} /></div>
{/if}
<!-- {#if level === 0} -->
<div class="chevron"><IconCollapseArrow size={level === 0 ? 'small' : 'tiny'} /></div>
<!-- {/if} -->
{#if groupByKey === noCategory}
<span class="text-base fs-bold overflow-label pointer-events-none">
<Label label={view.string.NoGrouping} />

View File

@ -1,43 +1 @@
/**
* @public
*/
export class RateLimitter {
idCounter: number = 0
processingQueue = new Map<string, Promise<void>>()
last: number = 0
queue: (() => Promise<void>)[] = []
constructor (readonly config: () => { rate: number, perSecond?: number }) {}
async exec<T, B extends Record<string, any> = {}>(op: (args?: B) => Promise<T>, args?: B): Promise<T> {
const processingId = `${this.idCounter++}`
const cfg = this.config()
while (this.processingQueue.size > cfg.rate) {
await Promise.race(this.processingQueue.values())
}
try {
const p = op(args)
this.processingQueue.set(processingId, p as Promise<void>)
return await p
} finally {
this.processingQueue.delete(processingId)
}
}
async add<T, B extends Record<string, any> = {}>(op: (args?: B) => Promise<T>, args?: B): Promise<void> {
const cfg = this.config()
if (this.processingQueue.size < cfg.rate) {
void this.exec(op, args)
} else {
await this.exec(op, args)
}
}
async waitProcessing (): Promise<void> {
await await Promise.race(this.processingQueue.values())
}
}
export { RateLimitter } from '@hcengineering/core'