hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 02:51:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-8518: Optimize client model (#7000)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-10-21 13:12:45 +07:00 committed by GitHub
parent 001b2dd0d6
commit 0b1af0da90
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 241 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
desktop/src/ui/platform.ts
View File

@ -305,7 +305,7 @@ export async function configurePlatform (): Promise<void> {
addLocation(printId, () => import(/* webpackChunkName: "print" */ '@hcengineering/print-resources'))
addLocation(textEditorId, () => import(/* webpackChunkName: "text-editor" */ '@hcengineering/text-editor-resources'))
setMetadata(client.metadata.FilterModel, true)
setMetadata(client.metadata.FilterModel, 'ui')
setMetadata(client.metadata.ExtraPlugins, ['preference' as Plugin])
// Use binary response transfer for faster performance and small transfer sizes.

2
dev/prod/src/platform.ts
View File

@ -397,7 +397,7 @@ export async function configurePlatform() {
addLocation(textEditorId, () => import(/* webpackChunkName: "text-editor" */ '@hcengineering/text-editor-resources'))
addLocation(uploaderId, () => import(/* webpackChunkName: "uploader" */ '@hcengineering/uploader-resources'))
setMetadata(client.metadata.FilterModel, true)
setMetadata(client.metadata.FilterModel, 'ui')
setMetadata(client.metadata.ExtraPlugins, ['preference' as Plugin])
// Use binary response transfer for faster performance and small transfer sizes.

36
packages/core/src/__tests__/client.test.ts
View File

@ -13,19 +13,30 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Plugin, IntlString } from '@hcengineering/platform'
import { IntlString, Plugin } from '@hcengineering/platform'
import type { Account, Class, Data, Doc, Domain, PluginConfiguration, Ref, Timestamp } from '../classes'
import { Space, ClassifierKind, DOMAIN_MODEL } from '../classes'
import { createClient, ClientConnection } from '../client'
import { ClassifierKind, DOMAIN_MODEL, Space } from '../classes'
import { ClientConnection, createClient } from '../client'
import { clone } from '../clone'
import core from '../component'
import { Hierarchy } from '../hierarchy'
import { ModelDb, TxDb } from '../memdb'
import { TxOperations } from '../operations'
import type { DocumentQuery, FindResult, TxResult, SearchQuery, SearchOptions, SearchResult } from '../storage'
import type { DocumentQuery, FindResult, SearchOptions, SearchQuery, SearchResult, TxResult } from '../storage'
import { Tx, TxFactory, TxProcessor } from '../tx'
import { fillConfiguration, pluginFilterTx } from '../utils'
import { connect } from './connection'
import { genMinModel } from './minmodel'
import { clone } from '../clone'
function filterPlugin (plugin: Plugin): (txes: Tx[]) => Promise<Tx[]> {
return async (txes) => {
const configs = new Map<Ref<PluginConfiguration>, PluginConfiguration>()
fillConfiguration(txes, configs)
const excludedPlugins = Array.from(configs.values()).filter((it) => !it.enabled || it.pluginId !== plugin)
return pluginFilterTx(excludedPlugins, configs, txes)
}
}
describe('client', () => {
it('should create client and spaces', async () => {
@ -136,7 +147,10 @@ describe('client', () => {
}
const txCreateDoc1 = txFactory.createTxCreateDoc(core.class.PluginConfiguration, core.space.Model, pluginData1)
txes.push(txCreateDoc1)
const client1 = new TxOperations(await createClient(connectPlugin, ['testPlugin1' as Plugin]), core.account.System)
const client1 = new TxOperations(
await createClient(connectPlugin, filterPlugin('testPlugin1' as Plugin)),
core.account.System
)
const result1 = await client1.findAll(core.class.PluginConfiguration, {})
expect(result1).toHaveLength(1)
@ -153,7 +167,10 @@ describe('client', () => {
}
const txCreateDoc2 = txFactory.createTxCreateDoc(core.class.PluginConfiguration, core.space.Model, pluginData2)
txes.push(txCreateDoc2)
const client2 = new TxOperations(await createClient(connectPlugin, ['testPlugin1' as Plugin]), core.account.System)
const client2 = new TxOperations(
await createClient(connectPlugin, filterPlugin('testPlugin1' as Plugin)),
core.account.System
)
const result2 = await client2.findAll(core.class.PluginConfiguration, {})
expect(result2).toHaveLength(2)
@ -176,7 +193,10 @@ describe('client', () => {
pluginData3
)
txes.push(txUpdateDoc)
const client3 = new TxOperations(await createClient(connectPlugin, ['testPlugin2' as Plugin]), core.account.System)
const client3 = new TxOperations(
await createClient(connectPlugin, filterPlugin('testPlugin2' as Plugin)),
core.account.System
)
const result3 = await client3.findAll(core.class.PluginConfiguration, {})
expect(result3).toHaveLength(1)

104
packages/core/src/client.ts
View File

@ -13,17 +13,16 @@
// limitations under the License.
//
import { Plugin } from '@hcengineering/platform'
import { BackupClient, DocChunk } from './backup'
import { Account, AttachedDoc, Class, DOMAIN_MODEL, Doc, Domain, PluginConfiguration, Ref, Timestamp } from './classes'
import { Account, AttachedDoc, Class, DOMAIN_MODEL, Doc, Domain, Ref, Timestamp } from './classes'
import core from './component'
import { Hierarchy } from './hierarchy'
import { MeasureContext, MeasureMetricsContext } from './measurements'
import { ModelDb } from './memdb'
import type { DocumentQuery, FindOptions, FindResult, FulltextStorage, Storage, TxResult, WithLookup } from './storage'
import { SearchOptions, SearchQuery, SearchResult, SortingOrder } from './storage'
import { Tx, TxCUD, TxCollectionCUD, TxCreateDoc, TxProcessor, TxUpdateDoc } from './tx'
import { toFindResult, toIdMap } from './utils'
import { Tx, TxCUD, TxCollectionCUD } from './tx'
import { toFindResult } from './utils'
const transactionThreshold = 500
@ -215,13 +214,15 @@ export interface TxPersistenceStore {
store: (model: LoadModelResponse) => Promise<void>
}
export type ModelFilter = (tx: Tx[]) => Promise<Tx[]>
/**
* @public
*/
export async function createClient (
connect: (txHandler: TxHandler) => Promise<ClientConnection>,
// If set will build model with only allowed plugins.
allowedPlugins?: Plugin[],
modelFilter?: ModelFilter,
txPersistence?: TxPersistenceStore,
_ctx?: MeasureContext
): Promise<AccountClient> {
@ -248,14 +249,12 @@ export async function createClient (
}
lastTx = tx.reduce((cur, it) => (it.modifiedOn > cur ? it.modifiedOn : cur), 0)
}
const configs = new Map<Ref<PluginConfiguration>, PluginConfiguration>()
const conn = await ctx.with('connect', {}, async () => await connect(txHandler))
await ctx.with(
'load-model',
{ reload: false },
async (ctx) => await loadModel(ctx, conn, allowedPlugins, configs, hierarchy, model, false, txPersistence)
async (ctx) => await loadModel(ctx, conn, modelFilter, hierarchy, model, false, txPersistence)
)
txBuffer = txBuffer.filter((tx) => tx.space !== core.space.Model)
@ -277,7 +276,7 @@ export async function createClient (
const loadModelResponse = await ctx.with(
'connect',
{ reload: true },
async (ctx) => await loadModel(ctx, conn, allowedPlugins, configs, hierarchy, model, true, txPersistence)
async (ctx) => await loadModel(ctx, conn, modelFilter, hierarchy, model, true, txPersistence)
)
if (event === ClientConnectEvent.Reconnected && loadModelResponse.full) {
@ -286,7 +285,7 @@ export async function createClient (
model = new ModelDb(hierarchy)
await ctx.with('build-model', {}, async (ctx) => {
await buildModel(ctx, loadModelResponse, allowedPlugins, configs, hierarchy, model)
await buildModel(ctx, loadModelResponse, modelFilter, hierarchy, model)
})
await oldOnConnect?.(ClientConnectEvent.Upgraded)
@ -393,8 +392,7 @@ function isPersonAccount (tx: Tx): boolean {
async function loadModel (
ctx: MeasureContext,
conn: ClientConnection,
allowedPlugins: Plugin[] | undefined,
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
modelFilter: ModelFilter | undefined,
hierarchy: Hierarchy,
model: ModelDb,
reload = false,
@ -418,19 +416,18 @@ async function loadModel (
)
}
await ctx.with('build-model', {}, (ctx) => buildModel(ctx, modelResponse, allowedPlugins, configs, hierarchy, model))
await ctx.with('build-model', {}, (ctx) => buildModel(ctx, modelResponse, modelFilter, hierarchy, model))
return modelResponse
}
async function buildModel (
ctx: MeasureContext,
modelResponse: LoadModelResponse,
allowedPlugins: Plugin[] | undefined,
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
modelFilter: ModelFilter | undefined,
hierarchy: Hierarchy,
model: ModelDb
): Promise<void> {
let systemTx: Tx[] = []
const systemTx: Tx[] = []
const userTx: Tx[] = []
const atxes = modelResponse.transactions
@ -444,23 +441,11 @@ async function buildModel (
)
})
if (allowedPlugins != null) {
await ctx.with('fill config system', {}, async () => {
fillConfiguration(systemTx, configs)
})
await ctx.with('fill config user', {}, async () => {
fillConfiguration(userTx, configs)
})
const excludedPlugins = Array.from(configs.values()).filter(
(it) => !it.enabled || !allowedPlugins.includes(it.pluginId)
)
await ctx.with('filter txes', {}, async () => {
systemTx = pluginFilterTx(excludedPlugins, configs, systemTx)
})
let txes = systemTx.concat(userTx)
if (modelFilter !== undefined) {
txes = await modelFilter(txes)
}
const txes = systemTx.concat(userTx)
await ctx.with('build hierarchy', {}, async () => {
for (const tx of txes) {
try {
@ -488,60 +473,3 @@ function getLastTxTime (txes: Tx[]): number {
}
return lastTxTime
}
function fillConfiguration (systemTx: Tx[], configs: Map<Ref<PluginConfiguration>, PluginConfiguration>): void {
for (const t of systemTx) {
if (t._class === core.class.TxCreateDoc) {
const ct = t as TxCreateDoc<Doc>
if (ct.objectClass === core.class.PluginConfiguration) {
configs.set(ct.objectId as Ref<PluginConfiguration>, TxProcessor.createDoc2Doc(ct) as PluginConfiguration)
}
} else if (t._class === core.class.TxUpdateDoc) {
const ut = t as TxUpdateDoc<Doc>
if (ut.objectClass === core.class.PluginConfiguration) {
const c = configs.get(ut.objectId as Ref<PluginConfiguration>)
if (c !== undefined) {
TxProcessor.updateDoc2Doc(c, ut)
}
}
}
}
}
function pluginFilterTx (
excludedPlugins: PluginConfiguration[],
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
systemTx: Tx[]
): Tx[] {
const stx = toIdMap(systemTx)
const totalExcluded = new Set<Ref<Tx>>()
let msg = ''
for (const a of excludedPlugins) {
for (const c of configs.values()) {
if (a.pluginId === c.pluginId) {
for (const id of c.transactions) {
if (c.classFilter !== undefined) {
const filter = new Set(c.classFilter)
const tx = stx.get(id as Ref<Tx>)
if (
tx?._class === core.class.TxCreateDoc ||
tx?._class === core.class.TxUpdateDoc ||
tx?._class === core.class.TxRemoveDoc
) {
const cud = tx as TxCUD<Doc>
if (filter.has(cud.objectClass)) {
totalExcluded.add(id as Ref<Tx>)
}
}
} else {
totalExcluded.add(id as Ref<Tx>)
}
}
msg += ` ${c.pluginId}:${c.transactions.length}`
}
}
}
console.log('exclude plugin', msg)
systemTx = systemTx.filter((t) => !totalExcluded.has(t._id))
return systemTx
}

20
packages/core/src/hierarchy.ts
View File

@ -30,7 +30,7 @@ export class Hierarchy {
private readonly attributes = new Map<Ref<Classifier>, Map<string, AnyAttribute>>()
private readonly attributesById = new Map<Ref<AnyAttribute>, AnyAttribute>()
private readonly descendants = new Map<Ref<Classifier>, Ref<Classifier>[]>()
private readonly ancestors = new Map<Ref<Classifier>, { ordered: Ref<Classifier>[], set: Set<Ref<Classifier>> }>()
private readonly ancestors = new Map<Ref<Classifier>, Set<Ref<Classifier>>>()
private readonly proxies = new Map<Ref<Mixin<Doc>>, ProxyHandler<Doc>>()
private readonly classifierProperties = new Map<Ref<Classifier>, Record<string, any>>()
@ -166,7 +166,7 @@ export class Hierarchy {
if (result === undefined) {
throw new Error('ancestors not found: ' + _class)
}
return result.ordered
return Array.from(result)
}
getClass<T extends Obj = Obj>(_class: Ref<Class<T>>): Class<T> {
@ -301,7 +301,7 @@ export class Hierarchy {
* It will iterate over parents.
*/
isDerived<T extends Obj>(_class: Ref<Class<T>>, from: Ref<Class<T>>): boolean {
return this.ancestors.get(_class)?.set?.has(from) ?? false
return this.ancestors.get(_class)?.has(from) ?? false
}
/**
@ -388,19 +388,17 @@ export class Hierarchy {
const list = this.ancestors.get(_class)
if (list === undefined) {
if (add) {
this.ancestors.set(_class, { ordered: [classifier], set: new Set([classifier]) })
this.ancestors.set(_class, new Set([classifier]))
}
} else {
if (add) {
if (!list.set.has(classifier)) {
list.ordered.push(classifier)
list.set.add(classifier)
if (!list.has(classifier)) {
list.add(classifier)
}
} else {
const pos = list.ordered.indexOf(classifier)
if (pos !== -1) {
list.ordered.splice(pos, 1)
list.set.delete(classifier)
const pos = list.has(classifier)
if (pos) {
list.delete(classifier)
}
}
}

62
packages/core/src/utils.ts
View File

@ -40,7 +40,8 @@ import {
roleOrder,
Space,
TypedSpace,
WorkspaceMode
WorkspaceMode,
type PluginConfiguration
} from './classes'
import core from './component'
import { Hierarchy } from './hierarchy'
@ -48,7 +49,7 @@ import { TxOperations } from './operations'
import { isPredicate } from './predicate'
import { Branding, BrandingMap } from './server'
import { DocumentQuery, FindResult } from './storage'
import { DOMAIN_TX } from './tx'
import { DOMAIN_TX, TxProcessor, type Tx, type TxCreateDoc, type TxCUD, type TxUpdateDoc } from './tx'
function toHex (value: number, chars: number): string {
const result = value.toString(16)
@ -835,3 +836,60 @@ export function getBranding (brandings: BrandingMap, key: string | undefined): B
return Object.values(brandings).find((branding) => branding.key === key) ?? null
}
export function fillConfiguration (systemTx: Tx[], configs: Map<Ref<PluginConfiguration>, PluginConfiguration>): void {
for (const t of systemTx) {
if (t._class === core.class.TxCreateDoc) {
const ct = t as TxCreateDoc<Doc>
if (ct.objectClass === core.class.PluginConfiguration) {
configs.set(ct.objectId as Ref<PluginConfiguration>, TxProcessor.createDoc2Doc(ct) as PluginConfiguration)
}
} else if (t._class === core.class.TxUpdateDoc) {
const ut = t as TxUpdateDoc<Doc>
if (ut.objectClass === core.class.PluginConfiguration) {
const c = configs.get(ut.objectId as Ref<PluginConfiguration>)
if (c !== undefined) {
TxProcessor.updateDoc2Doc(c, ut)
}
}
}
}
}
export function pluginFilterTx (
excludedPlugins: PluginConfiguration[],
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
systemTx: Tx[]
): Tx[] {
const stx = toIdMap(systemTx)
const totalExcluded = new Set<Ref<Tx>>()
let msg = ''
for (const a of excludedPlugins) {
for (const c of configs.values()) {
if (a.pluginId === c.pluginId) {
for (const id of c.transactions) {
if (c.classFilter !== undefined) {
const filter = new Set(c.classFilter)
const tx = stx.get(id as Ref<Tx>)
if (
tx?._class === core.class.TxCreateDoc ||
tx?._class === core.class.TxUpdateDoc ||
tx?._class === core.class.TxRemoveDoc
) {
const cud = tx as TxCUD<Doc>
if (filter.has(cud.objectClass)) {
totalExcluded.add(id as Ref<Tx>)
}
}
} else {
totalExcluded.add(id as Ref<Tx>)
}
}
msg += ` ${c.pluginId}:${c.transactions.length}`
}
}
}
console.log('exclude plugin', msg)
systemTx = systemTx.filter((t) => !totalExcluded.has(t._id))
return systemTx
}

82
plugins/client-resources/src/index.ts
View File

@ -24,7 +24,16 @@ import core, {
TxWorkspaceEvent,
WorkspaceEvent,
concatLink,
createClient
createClient,
fillConfiguration,
pluginFilterTx,
type Class,
type ClientConnection,
type Doc,
type ModelFilter,
type PluginConfiguration,
type Ref,
type TxCUD
} from '@hcengineering/core'
import platform, { Severity, Status, getMetadata, getPlugins, setPlatformStatus } from '@hcengineering/platform'
import { connect } from './connection'
@ -70,10 +79,9 @@ export default async () => {
return {
function: {
GetClient: async (token: string, endpoint: string, opt?: ClientFactoryOptions): Promise<AccountClient> => {
const filterModel = getMetadata(clientPlugin.metadata.FilterModel) ?? false
const filterModel = getMetadata(clientPlugin.metadata.FilterModel) ?? 'none'
const client = createClient(
async (handler: TxHandler) => {
const handler = async (handler: TxHandler): Promise<ClientConnection> => {
const url = concatLink(endpoint, `/${token}`)
const upgradeHandler: TxHandler = (...txes: Tx[]) => {
@ -122,16 +130,72 @@ export default async () => {
await connectPromise
}
return await Promise.resolve(clientConnection)
},
filterModel ? [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])] : undefined,
createModelPersistence(getWSFromToken(token)),
opt?.ctx
)
}
const modelFilter: ModelFilter = async (txes) => {
if (filterModel === 'client') {
return returnClientTxes(txes)
}
if (filterModel === 'ui') {
return returnUITxes(txes)
}
return txes
}
const client = createClient(handler, modelFilter, createModelPersistence(getWSFromToken(token)), opt?.ctx)
return await client
}
}
}
}
function returnUITxes (txes: Tx[]): Tx[] {
const configs = new Map<Ref<PluginConfiguration>, PluginConfiguration>()
fillConfiguration(txes, configs)
const allowedPlugins = [...getPlugins(), ...(getMetadata(clientPlugin.metadata.ExtraPlugins) ?? [])]
const excludedPlugins = Array.from(configs.values()).filter(
(it) => !it.enabled || !allowedPlugins.includes(it.pluginId)
)
return pluginFilterTx(excludedPlugins, configs, txes)
}
function returnClientTxes (txes: Tx[]): Tx[] {
const configs = new Map<Ref<PluginConfiguration>, PluginConfiguration>()
fillConfiguration(txes, configs)
const excludedPlugins = Array.from(configs.values()).filter((it) => !it.enabled || it.pluginId.startsWith('server-'))
const toExclude = new Set([
'workbench:class:Application' as Ref<Class<Doc>>,
'presentation:class:ComponentPointExtension' as Ref<Class<Doc>>,
'presentation:class:ObjectSearchCategory' as Ref<Class<Doc>>,
'notification:class:NotificationGroup' as Ref<Class<Doc>>,
'notification:class:NotificationType' as Ref<Class<Doc>>,
'view:class:Action' as Ref<Class<Doc>>,
'view:class:Viewlet' as Ref<Class<Doc>>,
'text-editor:class:TextEditorAction' as Ref<Class<Doc>>,
'templates:class:TemplateField' as Ref<Class<Doc>>,
'activity:class:DocUpdateMessageViewlet' as Ref<Class<Doc>>,
'core:class:PluginConfiguration' as Ref<Class<Doc>>,
'core:class:DomainIndexConfiguration' as Ref<Class<Doc>>
])
const result = pluginFilterTx(excludedPlugins, configs, txes).filter((tx) => {
// Exclude all matched UI plugins
if (
tx?._class === core.class.TxCreateDoc ||
tx?._class === core.class.TxUpdateDoc ||
tx?._class === core.class.TxRemoveDoc
) {
const cud = tx as TxCUD<Doc>
if (toExclude.has(cud.objectClass)) {
return false
}
}
return true
})
return result
}
function createModelPersistence (workspace: string): TxPersistenceStore | undefined {
const overrideStore = getMetadata(clientPlugin.metadata.OverridePersistenceStore)
if (overrideStore !== undefined) {

7
plugins/client/src/index.ts
View File

@ -69,10 +69,15 @@ export interface ClientFactoryOptions {
*/
export type ClientFactory = (token: string, endpoint: string, opt?: ClientFactoryOptions) => Promise<AccountClient>
// client - will filter out all server model elements
// It will also filter out all UI Elements, like Actions, View declarations etc.
// ui - will filter out all server element's and all UI disabled elements.
export type FilterMode = 'none' | 'client' | 'ui'
export default plugin(clientId, {
metadata: {
ClientSocketFactory: '' as Metadata<ClientSocketFactory>,
FilterModel: '' as Metadata<boolean>,
FilterModel: '' as Metadata<FilterMode>,
ExtraPlugins: '' as Metadata<Plugin[]>,
UseBinaryProtocol: '' as Metadata<boolean>,
UseProtocolCompression: '' as Metadata<boolean>,

3
products/tracker/src/platform.ts
View File

@ -104,6 +104,5 @@ export async function configurePlatform() {
setMetadata(uiPlugin.metadata.PlatformTitle, 'Tracker')
setMetadata(workbench.metadata.PlatformTitle, 'Tracker')
setMetadata(client.metadata.FilterModel, true)
setMetadata(client.metadata.ExtraPlugins, ['preference' as Plugin])
setMetadata(client.metadata.FilterModel, 'ui')
}

3
services/calendar/pod-calendar/src/client.ts
View File

@ -13,10 +13,13 @@
// limitations under the License.
//
import client from '@hcengineering/client'
import { type Client } from '@hcengineering/core'
import { setMetadata } from '@hcengineering/platform'
import { createClient, getTransactorEndpoint } from '@hcengineering/server-client'
export async function getClient (token: string): Promise<Client> {
const endpoint = await getTransactorEndpoint(token)
setMetadata(client.metadata.FilterModel, 'client')
return await createClient(endpoint, token)
}

1
services/github/pod-github/src/client.ts
View File

@ -37,6 +37,7 @@ export async function createPlatformClient (
{ mode: 'github' }
)
setMetadata(client.metadata.ConnectionTimeout, timeout)
setMetadata(client.metadata.FilterModel, 'client')
const endpoint = await getTransactorEndpoint(token)
const connection = await (
await clientResources()