hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 11:01:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-6653: Fix minor issue and add force-close (#5418)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-04-23 15:05:01 +07:00 committed by GitHub
parent ecf0f9d75a
commit df5aa8f7b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 425 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dev/client-resources/src/connection.ts
View File

@ -25,17 +25,17 @@ import core, {
FindOptions,
FindResult,
getWorkspaceId,
MeasureDoneOperation,
MeasureMetricsContext,
Ref,
SearchOptions,
SearchQuery,
SearchResult,
ServerStorage,
Timestamp,
Tx,
TxHandler,
TxResult,
SearchQuery,
SearchOptions,
SearchResult,
MeasureDoneOperation
TxResult
} from '@hcengineering/core'
import { createInMemoryTxAdapter } from '@hcengineering/dev-storage'
import devmodel from '@hcengineering/devmodel'
@ -109,6 +109,8 @@ class ServerStorageWrapper implements ClientConnection {
async measure (operationName: string): Promise<MeasureDoneOperation> {
return async () => ({ time: 0, serverTime: 0 })
}
async sendForceClose (): Promise<void> {}
}
async function createNullFullTextAdapter (): Promise<FullTextAdapter> {

3
dev/prod/package.json
View File

@ -185,6 +185,7 @@
"@hcengineering/document-resources": "^0.6.0",
"@hcengineering/guest": "^0.6.0",
"@hcengineering/guest-assets": "^0.6.0",
"@hcengineering/guest-resources": "^0.6.0"
"@hcengineering/guest-resources": "^0.6.0",
"@hcengineering/analytics": "^0.6.0"
}
}

17
dev/prod/src/platform.ts
View File

@ -13,7 +13,7 @@
// limitations under the License.
//
import { Plugin, addLocation, addStringsLoader, platformId } from '@hcengineering/platform'
import platform, { Plugin, addLocation, addStringsLoader, platformId } from '@hcengineering/platform'
import { activityId } from '@hcengineering/activity'
import { attachmentId } from '@hcengineering/attachment'
@ -86,6 +86,8 @@ import { preferenceId } from '@hcengineering/preference'
import { setDefaultLanguage } from '@hcengineering/theme'
import { uiId } from '@hcengineering/ui/src/plugin'
import { Analytics } from '@hcengineering/analytics'
interface Config {
ACCOUNTS_URL: string
UPLOAD_URL: string
@ -143,6 +145,19 @@ function configureI18n(): void {
}
export async function configurePlatform() {
setMetadata(platform.metadata.LoadHelper, async (loader) => {
for (let i = 0; i < 3; i++) {
try {
return loader()
} catch (err: any) {
if (err.message.includes('Loading chunk') && i != 2) {
continue
}
Analytics.handleError(err)
}
}
})
configureI18n()
const config: Config = await (await fetch(devConfig? '/config-dev.json' : '/config.json')).json()

7
dev/tool/src/index.ts
View File

@ -30,8 +30,8 @@ import {
replacePassword,
setAccountAdmin,
setRole,
upgradeWorkspace,
UpgradeWorker
UpgradeWorker,
upgradeWorkspace
} from '@hcengineering/account'
import { setMetadata } from '@hcengineering/platform'
import {
@ -264,7 +264,7 @@ export function devTool (
.action(async (workspace, cmd) => {
const { mongodbUri, txes, version, migrateOperations } = prepareTools()
await withDatabase(mongodbUri, async (db) => {
const { client } = await createWorkspace(
await createWorkspace(
toolCtx,
version,
txes,
@ -275,7 +275,6 @@ export function devTool (
cmd.workspaceName,
workspace
)
await client?.close()
})
})

3
packages/core/src/__tests__/client.test.ts
View File

@ -123,7 +123,8 @@ describe('client', () => {
getAccount: async () => null as unknown as Account,
measure: async () => {
return async () => ({ time: 0, serverTime: 0 })
}
},
sendForceClose: async () => {}
}
}
const spyCreate = jest.spyOn(TxProcessor, 'createDoc2Doc')

3
packages/core/src/__tests__/connection.ts
View File

@ -72,6 +72,7 @@ export async function connect (handler: (tx: Tx) => void): Promise<ClientConnect
clean: async (domain: Domain, docs: Ref<Doc>[]) => {},
loadModel: async (last: Timestamp) => txes,
getAccount: async () => null as unknown as Account,
measure: async () => async () => ({ time: 0, serverTime: 0 })
measure: async () => async () => ({ time: 0, serverTime: 0 }),
sendForceClose: async () => {}
}
}

2
packages/core/src/backup.ts
View File

@ -23,4 +23,6 @@ export interface BackupClient {
loadDocs: (domain: Domain, docs: Ref<Doc>[]) => Promise<Doc[]>
upload: (domain: Domain, docs: Doc[]) => Promise<void>
clean: (domain: Domain, docs: Ref<Doc>[]) => Promise<void>
sendForceClose: () => Promise<void>
}

4
packages/core/src/client.ts
View File

@ -207,6 +207,10 @@ class ClientImpl implements AccountClient, BackupClient, MeasureClient {
async getAccount (): Promise<Account> {
return await this.conn.getAccount()
}
async sendForceClose (): Promise<void> {
await this.conn.sendForceClose()
}
}
/**

5
packages/platform/src/platform.ts
View File

@ -14,7 +14,7 @@
// limitations under the License.
*/
import { Metadata } from '.'
import { Metadata, PluginLoader, PluginModule, Resources } from '.'
/**
* Id in format 'plugin.resource-kind.id'
@ -156,6 +156,7 @@ export default plugin(platformId, {
ProductIdMismatch: '' as StatusCode<{ productId: string }>
},
metadata: {
locale: '' as Metadata<string>
locale: '' as Metadata<string>,
LoadHelper: '' as Metadata<<T extends Resources>(loader: PluginLoader<T>) => Promise<PluginModule<T>>>
}
})

11
packages/platform/src/resource.ts
View File

@ -19,6 +19,7 @@ import { _parseId } from './ident'
import type { Plugin, Resource } from './platform'
import { PlatformError, Severity, Status } from './status'
import { getMetadata } from './metadata'
import platform from './platform'
/**
@ -77,7 +78,12 @@ async function loadPlugin (id: Plugin): Promise<Resources> {
const status = new Status(Severity.INFO, platform.status.LoadingPlugin, {
plugin: id
})
pluginLoader = monitor(status, getLocation(id)()).then(async (plugin) => {
const loadHelper = getMetadata(platform.metadata.LoadHelper)
const locationLoader = getLocation(id)
pluginLoader = monitor(status, loadHelper !== undefined ? loadHelper(locationLoader) : locationLoader()).then(
async (plugin) => {
try {
// In case of ts-node, we have a bit different import structure, so let's check for it.
if (typeof plugin.default === 'object') {
@ -89,7 +95,8 @@ async function loadPlugin (id: Plugin): Promise<Resources> {
console.error(err)
throw err
}
})
}
)
loading.set(id, pluginLoader)
}
return await pluginLoader

3
packages/query/src/__tests__/connection.ts
View File

@ -96,6 +96,7 @@ FulltextStorage & {
searchFulltext: async (query: SearchQuery, options: SearchOptions): Promise<SearchResult> => {
return { docs: [] }
},
measure: async () => async () => ({ time: 0, serverTime: 0 })
measure: async () => async () => ({ time: 0, serverTime: 0 }),
sendForceClose: async () => {}
}
}

9
packages/ui/src/popups.ts
View File

@ -30,6 +30,9 @@ export interface CompAndProps {
refId?: string
}
dock?: boolean
// Internal
closing?: boolean
}
export interface PopupResult {
@ -116,7 +119,13 @@ export function closePopup (category?: string): void {
} else {
for (let i = popups.length - 1; i >= 0; i--) {
if (popups[i].options.fixed !== true) {
const isClosing = popups[i].closing ?? false
popups[i].closing = true
if (!isClosing) {
// To prevent possible recursion, we need to check if we call some code from popup close, to do close.
popups[i].onClose?.(undefined)
}
popups[i].closing = false
popups.splice(i, 1)
break
}

14
plugins/client-resources/src/connection.ts
View File

@ -134,22 +134,12 @@ class Connection implements ClientConnection {
async close (): Promise<void> {
this.closed = true
clearInterval(this.interval)
const closeEvt = serialize(
{
method: 'close',
params: [],
id: -1
},
false
)
if (this.websocket !== null) {
if (this.websocket instanceof Promise) {
await this.websocket.then((ws) => {
ws.send(closeEvt)
ws.close(1000)
})
} else {
this.websocket.send(closeEvt)
this.websocket.close(1000)
}
this.websocket = null
@ -547,6 +537,10 @@ class Connection implements ClientConnection {
searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
return this.sendRequest({ method: 'searchFulltext', params: [query, options] })
}
sendForceClose (): Promise<void> {
return this.sendRequest({ method: 'forceClose', params: [] })
}
}
/**

11
plugins/document-resources/src/components/teamspace/CreateTeamspace.svelte
View File

@ -29,7 +29,7 @@
} from '@hcengineering/core'
import document, { Teamspace } from '@hcengineering/document'
import { Asset } from '@hcengineering/platform'
import presentation, { Card, getClient } from '@hcengineering/presentation'
import presentation, { Card, getClient, reduceCalls } from '@hcengineering/presentation'
import {
Button,
EditBox,
@ -72,7 +72,7 @@
let spaceType: WithLookup<SpaceType> | undefined
$: void loadSpaceType(typeId)
async function loadSpaceType (id: typeof typeId): Promise<void> {
const loadSpaceType = reduceCalls(async (id: typeof typeId): Promise<void> => {
spaceType =
id !== undefined
? await client
@ -85,7 +85,7 @@
}
rolesAssignment = getRolesAssignment()
}
})
function getRolesAssignment (): RolesAssignment {
if (teamspace === undefined || spaceType?.targetClass === undefined || spaceType?.$lookup?.roles === undefined) {
@ -243,7 +243,10 @@
label={isNew ? documentRes.string.NewTeamspace : documentRes.string.EditTeamspace}
okLabel={isNew ? presentation.string.Create : presentation.string.Save}
okAction={handleSave}
canSave={name.trim().length > 0 && !(members.length === 0 && isPrivate) && typeId !== undefined}
canSave={name.trim().length > 0 &&
!(members.length === 0 && isPrivate) &&
typeId !== undefined &&
spaceType?.targetClass !== undefined}
accentHeader
width={'medium'}
gap={'gapV-6'}

48
plugins/workbench-resources/src/components/ServerManager.svelte
View File

@ -3,7 +3,7 @@
import { Metrics, systemAccountEmail } from '@hcengineering/core'
import login from '@hcengineering/login'
import { getEmbeddedLabel, getMetadata } from '@hcengineering/platform'
import presentation, { createQuery } from '@hcengineering/presentation'
import presentation, { createQuery, isAdminUser } from '@hcengineering/presentation'
import {
Button,
CheckBox,
@ -88,13 +88,20 @@
$: activeSessions =
(data?.statistics?.activeSessions as Record<
string,
Array<{
{
sessions: Array<{
userId: string
data?: Record<string, any>
total: StatisticsElement
mins5: StatisticsElement
current: StatisticsElement
}>
name: string
wsId: string
sessionsTotal: number
upgrading: boolean
closing: boolean
}
>) ?? {}
const employeeQuery = createQuery()
@ -116,8 +123,8 @@
$: totalStats = Array.from(Object.entries(activeSessions).values()).reduce(
(cur, it) => {
const totalFind = it[1].reduce((it, itm) => itm.current.find + it, 0)
const totalTx = it[1].reduce((it, itm) => itm.current.tx + it, 0)
const totalFind = it[1].sessions.reduce((it, itm) => itm.current.find + it, 0)
const totalTx = it[1].sessions.reduce((it, itm) => itm.current.tx + it, 0)
return {
find: cur.find + totalFind,
tx: cur.tx + totalTx
@ -197,26 +204,49 @@
<div class="flex-column p-3 h-full" style:overflow="auto">
{#each Object.entries(activeSessions) as act}
{@const wsInstance = $workspacesStore.find((it) => it.workspaceId === act[0])}
{@const totalFind = act[1].reduce((it, itm) => itm.current.find + it, 0)}
{@const totalTx = act[1].reduce((it, itm) => itm.current.tx + it, 0)}
{@const employeeGroups = Array.from(new Set(act[1].map((it) => it.userId))).filter(
{@const totalFind = act[1].sessions.reduce((it, itm) => itm.current.find + it, 0)}
{@const totalTx = act[1].sessions.reduce((it, itm) => itm.current.tx + it, 0)}
{@const employeeGroups = Array.from(new Set(act[1].sessions.map((it) => it.userId))).filter(
(it) => systemAccountEmail !== it || !realUsers
)}
{@const realGroup = Array.from(new Set(act[1].map((it) => it.userId))).filter(
{@const realGroup = Array.from(new Set(act[1].sessions.map((it) => it.userId))).filter(
(it) => systemAccountEmail !== it
)}
{#if employeeGroups.length > 0}
<span class="flex-col">
<Expandable contentColor expanded={false} expandable={true} bordered>
<svelte:fragment slot="title">
<div class="flex flex-row-center flex-between flex-grow p-1">
<div class="fs-title" class:greyed={realGroup.length === 0}>
Workspace: {wsInstance?.workspaceName ?? act[0]}: {employeeGroups.length} current 5 mins => {totalFind}/{totalTx}
{#if act[1].upgrading}
(Upgrading)
{/if}
{#if act[1].closing}
(Closing)
{/if}
</div>
{#if isAdminUser()}
<Button
label={getEmbeddedLabel('Force close')}
size={'small'}
kind={'ghost'}
on:click={() => {
void fetch(
endpoint + `/api/v1/manage?token=${token}&operation=force-close&wsId=${act[1].wsId}`,
{
method: 'PUT'
}
)
}}
/>
{/if}
</div>
</svelte:fragment>
<div class="flex-col">
{#each employeeGroups as employeeId}
{@const employee = employees.get(employeeId)}
{@const connections = act[1].filter((it) => it.userId === employeeId)}
{@const connections = act[1].sessions.filter((it) => it.userId === employeeId)}
{@const find = connections.reduce((it, itm) => itm.current.find + it, 0)}
{@const txes = connections.reduce((it, itm) => itm.current.tx + it, 0)}

2
plugins/workbench-resources/src/components/WorkbenchApp.svelte
View File

@ -96,6 +96,7 @@
<style lang="scss">
.version-wrapper {
height: 100%;
width: 100%;
display: flex;
align-items: center;
justify-content: center;
@ -108,5 +109,6 @@
align-items: center;
justify-content: center;
padding: 2rem;
flex-grow: 1;
}
</style>

69
server/account/src/operations.ts
View File

@ -839,8 +839,9 @@ export async function createWorkspace (
email: string,
workspaceName: string,
workspace?: string,
notifyHandler?: (workspace: Workspace) => void
): Promise<{ workspaceInfo: Workspace, err?: any, client?: Client }> {
notifyHandler?: (workspace: Workspace) => void,
postInitHandler?: (workspace: Workspace, model: Tx[]) => Promise<void>
): Promise<{ workspaceInfo: Workspace, err?: any, model?: Tx[] }> {
return await rateLimiter.exec(async () => {
// We need to search for duplicate workspaceUrl
await searchPromise
@ -861,7 +862,6 @@ export async function createWorkspace (
await updateInfo({ createProgress: 10 })
let client: Client | undefined
const childLogger = ctx.newChild('createWorkspace', { workspace: workspaceInfo.workspace })
const ctxModellogger: ModelLogger = {
log: (msg, data) => {
@ -871,6 +871,7 @@ export async function createWorkspace (
void childLogger.error(msg, data)
}
}
let model: Tx[] = []
try {
const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
const wsId = getWorkspaceId(workspaceInfo.workspace, productId)
@ -882,11 +883,10 @@ export async function createWorkspace (
initWS !== workspaceInfo.workspace
) {
// Just any valid model for transactor to be able to function
await (
await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger, async (value) => {
await updateInfo({ createProgress: Math.round((Math.min(value, 100) / 100) * 20) })
})
).close()
await updateInfo({ createProgress: 20 })
// Clone init workspace.
await cloneWorkspace(
@ -899,7 +899,7 @@ export async function createWorkspace (
}
)
await updateInfo({ createProgress: 50 })
client = await upgradeModel(
model = await upgradeModel(
ctx,
getTransactor(),
wsId,
@ -913,26 +913,23 @@ export async function createWorkspace (
)
await updateInfo({ createProgress: 90 })
} else {
client = await initModel(
ctx,
getTransactor(),
wsId,
txes,
migrationOperation,
ctxModellogger,
async (value) => {
await initModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger, async (value) => {
await updateInfo({ createProgress: Math.round(Math.min(value, 100)) })
}
)
})
}
} catch (err: any) {
Analytics.handleError(err)
return { workspaceInfo, err, client: null as any }
}
if (postInitHandler !== undefined) {
await postInitHandler?.(workspaceInfo, model)
}
childLogger.end()
// Workspace is created, we need to clear disabled flag.
await updateInfo({ createProgress: 100, disabled: false, creating: false })
return { workspaceInfo, client }
return { workspaceInfo, model }
})
}
@ -970,7 +967,6 @@ export async function upgradeWorkspace (
toVersion: versionStr,
workspace: ws.workspace
})
await (
await upgradeModel(
ctx,
getTransactor(),
@ -981,7 +977,6 @@ export async function upgradeWorkspace (
false,
async (value) => {}
)
).close()
await db.collection(WORKSPACE_COLLECTION).updateOne(
{ _id: ws._id },
@ -1020,7 +1015,7 @@ export const createUserWorkspace =
}
async function doCreate (info: Account, notifyHandler: (workspace: Workspace) => void): Promise<void> {
const { workspaceInfo, err, client } = await createWorkspace(
const { workspaceInfo, err } = await createWorkspace(
ctx,
version,
txes,
@ -1030,7 +1025,29 @@ export const createUserWorkspace =
email,
workspaceName,
undefined,
notifyHandler
notifyHandler,
async (workspace, model) => {
const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
const shouldUpdateAccount = initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null
const client = await connect(
getTransactor(),
getWorkspaceId(workspace.workspace, productId),
undefined,
{
admin: 'true'
},
model
)
try {
await assignWorkspace(ctx, db, productId, email, workspace.workspace, shouldUpdateAccount, client)
await setRole(email, workspace.workspace, productId, AccountRole.Owner, client)
await ctx.info('Creating server side done', { workspaceName, email })
} catch (err: any) {
Analytics.handleError(err)
} finally {
await client.close()
}
}
)
if (err != null) {
@ -1045,20 +1062,10 @@ export const createUserWorkspace =
)
throw err
}
try {
info.lastWorkspace = Date.now()
// Update last workspace time.
await db.collection(ACCOUNT_COLLECTION).updateOne({ _id: info._id }, { $set: { lastWorkspace: Date.now() } })
const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
const shouldUpdateAccount = initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null
await assignWorkspace(ctx, db, productId, email, workspaceInfo.workspace, shouldUpdateAccount, client)
await setRole(email, workspaceInfo.workspace, productId, AccountRole.Owner, client)
await ctx.info('Creating server side done', { workspaceName, email })
} finally {
await client?.close()
}
}
const workspaceInfo = await new Promise<Workspace>((resolve) => {

6
server/backup/src/backup.ts
View File

@ -215,7 +215,9 @@ export async function cloneWorkspace (
mode: 'backup'
})) as unknown as CoreClient & BackupClient
const targetConnection = (await connect(transactorUrl, targetWorkspaceId, undefined, {
mode: 'backup'
mode: 'backup',
model: 'upgrade',
admin: 'true'
})) as unknown as CoreClient & BackupClient
try {
const domains = sourceConnection
@ -338,6 +340,7 @@ export async function cloneWorkspace (
} finally {
console.log('end clone')
await sourceConnection.close()
await targetConnection.sendForceClose()
await targetConnection.close()
}
}
@ -997,6 +1000,7 @@ export async function restore (
}
}
} finally {
await connection.sendForceClose()
await connection.close()
}
}

6
server/core/src/fulltext.ts
View File

@ -37,10 +37,10 @@ import core, {
type TxResult,
type WorkspaceId,
docKey,
isClassIndexable,
isFullTextAttribute,
isIndexedAttribute,
toFindResult,
isClassIndexable
toFindResult
} from '@hcengineering/core'
import { type FullTextIndexPipeline } from './indexer'
import { createStateDoc } from './indexer/utils'
@ -53,7 +53,6 @@ import type { FullTextAdapter, IndexedDoc, WithFind } from './types'
*/
export class FullTextIndex implements WithFind {
txFactory = new TxFactory(core.account.System, true)
consistency: Promise<void> | undefined
constructor (
private readonly hierarchy: Hierarchy,
@ -72,7 +71,6 @@ export class FullTextIndex implements WithFind {
async close (): Promise<void> {
await this.indexer.cancel()
await this.consistency
}
async tx (ctx: MeasureContext, txes: Tx[]): Promise<TxResult> {

3
server/core/src/server/storage.ts
View File

@ -52,6 +52,7 @@ import core, {
type TxResult,
type TxUpdateDoc,
type WorkspaceIdWithUrl,
cutObjectArray,
toFindResult
} from '@hcengineering/core'
import { type Metadata, getResource } from '@hcengineering/platform'
@ -393,7 +394,7 @@ export class TServerStorage implements ServerStorage {
{ clazz, query, options }
)
if (Date.now() - st > 1000) {
await ctx.error('FindAll', { time: Date.now() - st, clazz, query, options })
await ctx.error('FindAll', { time: Date.now() - st, clazz, query: cutObjectArray(query), options })
}
return result
}

3
server/mongo/src/__tests__/storage.test.ts
View File

@ -183,7 +183,8 @@ describe('mongo operations', () => {
clean: async (domain: Domain, docs: Ref<Doc>[]) => {},
loadModel: async () => txes,
getAccount: async () => ({}) as any,
measure: async () => async () => ({ time: 0, serverTime: 0 })
measure: async () => async () => ({ time: 0, serverTime: 0 }),
sendForceClose: async () => {}
}
return st
})

29
server/tool/src/index.ts
View File

@ -117,7 +117,7 @@ export async function initModel (
migrateOperations: [string, MigrateOperation][],
logger: ModelLogger = consoleModelLogger,
progress: (value: number) => Promise<void>
): Promise<CoreClient> {
): Promise<void> {
const { mongodbUri, storageAdapter: minio, txes } = prepareTools(rawTxes)
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
throw Error('Model txes must target only core.space.Model')
@ -125,7 +125,7 @@ export async function initModel (
const _client = getMongoClient(mongodbUri)
const client = await _client.getClient()
let connection: CoreClient & BackupClient
let connection: (CoreClient & BackupClient) | undefined
try {
const db = getWorkspaceDB(client, workspaceId)
@ -156,12 +156,8 @@ export async function initModel (
)) as unknown as CoreClient & BackupClient
const states = await connection.findAll<MigrationState>(core.class.MigrationState, {})
const migrateState = new Map(
Array.from(groupByArray(states, (it) => it.plugin).entries()).map((it) => [
it[0],
new Set(it[1].map((q) => q.state))
])
)
const sts = Array.from(groupByArray(states, (it) => it.plugin).entries())
const migrateState = new Map(sts.map((it) => [it[0], new Set(it[1].map((q) => q.state))]))
;(connection as any).migrateState = migrateState
try {
@ -183,8 +179,9 @@ export async function initModel (
logger.error('error', { error: e })
throw e
}
return connection
} finally {
await connection?.sendForceClose()
await connection?.close()
_client.close()
}
}
@ -201,7 +198,7 @@ export async function upgradeModel (
logger: ModelLogger = consoleModelLogger,
skipTxUpdate: boolean = false,
progress: (value: number) => Promise<void>
): Promise<CoreClient> {
): Promise<Tx[]> {
const { mongodbUri, txes } = prepareTools(rawTxes)
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
@ -266,7 +263,7 @@ export async function upgradeModel (
})
logger.log('Apply upgrade operations', { workspaceId: workspaceId.name })
const connection = await ctx.with(
const connection = (await ctx.with(
'connect-platform',
{},
async (ctx) =>
@ -281,8 +278,8 @@ export async function upgradeModel (
},
model
)
)
)) as CoreClient & BackupClient
try {
await ctx.with('upgrade', {}, async () => {
let i = 0
for (const op of migrateOperations) {
@ -303,7 +300,11 @@ export async function upgradeModel (
})
})
}
return connection
} finally {
await connection.sendForceClose()
await connection.close()
}
return model
} finally {
_client.close()
}

22
server/ws/src/client.ts
View File

@ -14,14 +14,9 @@
//
import core, {
type Account,
AccountRole,
type BulkUpdateEvent,
TxFactory,
TxProcessor,
type TxWorkspaceEvent,
WorkspaceEvent,
generateId,
type Account,
type Class,
type Doc,
type DocumentQuery,
@ -38,7 +33,12 @@ import core, {
type TxApplyIf,
type TxApplyResult,
type TxCUD,
type TxResult
TxFactory,
TxProcessor,
type TxResult,
type TxWorkspaceEvent,
WorkspaceEvent,
generateId
} from '@hcengineering/core'
import { type Pipeline, type SessionContext } from '@hcengineering/server-core'
import { type Token } from '@hcengineering/server-token'
@ -71,6 +71,14 @@ export class ClientSession implements Session {
return this.token.email
}
isUpgradeClient (): boolean {
return this.token.extra?.model === 'upgrade'
}
getMode (): string {
return this.token.extra?.mode ?? 'normal'
}
pipeline (): Pipeline {
return this._pipeline
}

168
server/ws/src/server.ts
View File

@ -45,13 +45,14 @@ import {
type Workspace
} from './types'
interface WorkspaceLoginInfo extends BaseWorkspaceInfo {
interface WorkspaceLoginInfo extends Omit<BaseWorkspaceInfo, 'workspace'> {
upgrade?: {
toProcess: number
total: number
elapsed: number
eta: number
}
workspaceId: string
}
function timeoutPromise (time: number): Promise<void> {
@ -66,7 +67,6 @@ function timeoutPromise (time: number): Promise<void> {
export interface Timeouts {
// Timeout preferences
pingTimeout: number // Default 1 second
shutdownWarmTimeout: number // Default 1 minute
reconnectTimeout: number // Default 3 seconds
}
@ -145,8 +145,8 @@ class TSessionManager implements SessionManager {
ticks = 0
handleInterval (): void {
for (const h of this.workspaces.entries()) {
for (const s of h[1].sessions) {
for (const [wsId, workspace] of this.workspaces.entries()) {
for (const s of workspace.sessions) {
if (this.ticks % (5 * 60) === 0) {
s[1].session.mins5.find = s[1].session.current.find
s[1].session.mins5.tx = s[1].session.current.tx
@ -162,13 +162,15 @@ class TSessionManager implements SessionManager {
}
if (diff > timeout && this.ticks % 10 === 0) {
void this.ctx.error('session hang, closing...', { sessionId: h[0], user: s[1].session.getUser() })
void this.close(s[1].socket, h[1].workspaceId, 1001, 'CLIENT_HANGOUT')
void this.ctx.error('session hang, closing...', { wsId, user: s[1].session.getUser() })
// Force close workspace if only one client and it hang.
void this.close(s[1].socket, workspace.workspaceId)
continue
}
if (diff > 20000 && diff < 60000 && this.ticks % 10 === 0) {
void s[1].socket.send(
h[1].context,
workspace.context,
{ result: 'ping' },
s[1].session.binaryResponseMode,
s[1].session.useCompression
@ -178,13 +180,29 @@ class TSessionManager implements SessionManager {
for (const r of s[1].session.requests.values()) {
if (now - r.start > 30000) {
void this.ctx.info('request hang found, 30sec', {
sessionId: h[0],
wsId,
user: s[1].session.getUser(),
...r.params
})
}
}
}
// Wait some time for new client to appear before closing workspace.
if (workspace.sessions.size === 0 && workspace.closing === undefined) {
workspace.softShutdown--
if (workspace.softShutdown <= 0) {
void this.ctx.info('closing workspace, no users', {
workspace: workspace.workspaceId.name,
wsId,
upgrade: workspace.upgrade,
backup: workspace.backup
})
workspace.closing = this.performWorkspaceCloseCheck(workspace, workspace.workspaceId, wsId)
}
} else {
workspace.softShutdown = 3
}
}
this.ticks++
}
@ -207,6 +225,10 @@ class TSessionManager implements SessionManager {
})
})
).json()
if (userInfo.error !== undefined) {
await this.ctx.error('Error response from account service', { error: JSON.stringify(userInfo) })
throw new Error(JSON.stringify(userInfo.error))
}
return { ...userInfo.result, upgrade: userInfo.upgrade }
}
@ -221,7 +243,7 @@ class TSessionManager implements SessionManager {
sessionId: string | undefined,
accountsUrl: string
): Promise<
| { session: Session, context: MeasureContext, workspaceName: string }
| { session: Session, context: MeasureContext, workspaceId: string }
| { upgrade: true, upgradeInfo?: WorkspaceLoginInfo['upgrade'] }
| { error: any }
> {
@ -258,11 +280,9 @@ class TSessionManager implements SessionManager {
}
let workspace = this.workspaces.get(wsString)
if (workspace?.closeTimeout !== undefined) {
await ctx.info('Cancel workspace warm close', { wsString })
clearTimeout(workspace?.closeTimeout)
}
if (workspace?.closing !== undefined) {
await workspace?.closing
}
workspace = this.workspaces.get(wsString)
if (sessionId !== undefined && workspace?.sessions?.has(sessionId) === true) {
const helloResponse: HelloResponse = {
@ -275,14 +295,20 @@ class TSessionManager implements SessionManager {
await ws.send(ctx, helloResponse, false, false)
return { error: new Error('Session already exists') }
}
const workspaceName = workspaceInfo.workspaceName ?? workspaceInfo.workspaceUrl ?? workspaceInfo.workspace
const workspaceName = workspaceInfo.workspaceName ?? workspaceInfo.workspaceUrl ?? workspaceInfo.workspaceId
if (workspace === undefined) {
await ctx.info('open workspace', {
email: token.email,
workspace: workspaceInfo.workspaceId,
wsUrl: workspaceInfo.workspaceUrl,
...token.extra
})
workspace = this.createWorkspace(
baseCtx,
pipelineFactory,
token,
workspaceInfo.workspaceUrl ?? workspaceInfo.workspace,
workspaceInfo.workspaceUrl ?? workspaceInfo.workspaceId,
workspaceName
)
}
@ -290,9 +316,20 @@ class TSessionManager implements SessionManager {
let pipeline: Pipeline
if (token.extra?.model === 'upgrade') {
if (workspace.upgrade) {
await ctx.info('reconnect workspace in upgrade', {
email: token.email,
workspace: workspaceInfo.workspaceId,
wsUrl: workspaceInfo.workspaceUrl
})
pipeline = await ctx.with('💤 wait', { workspaceName }, async () => await (workspace as Workspace).pipeline)
} else {
pipeline = await this.createUpgradeSession(
await ctx.info('reconnect workspace in upgrade switch', {
email: token.email,
workspace: workspaceInfo.workspaceId,
wsUrl: workspaceInfo.workspaceUrl
})
// We need to wait in case previous upgeade connection is already closing.
pipeline = await this.switchToUpgradeSession(
token,
sessionId,
ctx,
@ -300,7 +337,7 @@ class TSessionManager implements SessionManager {
workspace,
pipelineFactory,
ws,
workspaceInfo.workspaceUrl ?? workspaceInfo.workspace,
workspaceInfo.workspaceUrl ?? workspaceInfo.workspaceId,
workspaceName
)
}
@ -336,13 +373,13 @@ class TSessionManager implements SessionManager {
session.useCompression
)
}
return { session, context: workspace.context, workspaceName }
return { session, context: workspace.context, workspaceId: wsString }
})
}
private wsFromToken (token: Token): WorkspaceLoginInfo {
return {
workspace: token.workspace.name,
workspaceId: token.workspace.name,
workspaceUrl: token.workspace.name,
workspaceName: token.workspace.name,
createdBy: '',
@ -355,7 +392,7 @@ class TSessionManager implements SessionManager {
}
}
private async createUpgradeSession (
private async switchToUpgradeSession (
token: Token,
sessionId: string | undefined,
ctx: MeasureContext,
@ -369,16 +406,20 @@ class TSessionManager implements SessionManager {
if (LOGGING_ENABLED) {
await ctx.info('reloading workspace', { workspaceName, token: JSON.stringify(token) })
}
// Mark as upgrade, to prevent any new clients to connect during close
workspace.upgrade = true
workspace.backup = token.extra?.mode === 'backup'
// If upgrade client is used.
// Drop all existing clients
await this.closeAll(wsString, workspace, 0, 'upgrade')
workspace.closing = this.closeAll(wsString, workspace, 0, 'upgrade')
await workspace.closing
// Wipe workspace and update values.
workspace.workspaceName = workspaceName
if (!workspace.upgrade) {
// This is previous workspace, intended to be closed.
workspace.id = generateId()
workspace.sessions = new Map()
workspace.upgrade = token.extra?.model === 'upgrade'
}
// Re-create pipeline.
workspace.pipeline = pipelineFactory(
@ -432,6 +473,7 @@ class TSessionManager implements SessionManager {
workspaceName: string
): Workspace {
const upgrade = token.extra?.model === 'upgrade'
const backup = token.extra?.mode === 'backup'
const context = ctx.newChild('🧲 session', {})
const pipelineCtx = context.newChild('🧲 pipeline-factory', {})
const workspace: Workspace = {
@ -446,7 +488,9 @@ class TSessionManager implements SessionManager {
}
),
sessions: new Map(),
softShutdown: 3,
upgrade,
backup,
workspaceId: token.workspace,
workspaceName
}
@ -494,7 +538,7 @@ class TSessionManager implements SessionManager {
} catch {}
}
async close (ws: ConnectionSocket, workspaceId: WorkspaceId, code: number, reason: string): Promise<void> {
async close (ws: ConnectionSocket, workspaceId: WorkspaceId): Promise<void> {
const wsid = toWorkspaceString(workspaceId)
const workspace = this.workspaces.get(wsid)
if (workspace === undefined) {
@ -516,25 +560,29 @@ class TSessionManager implements SessionManager {
}
const user = sessionRef.session.getUser()
const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user)
if (another === -1) {
if (another === -1 && !workspace.upgrade) {
await this.trySetStatus(workspace.context, sessionRef.session, false)
}
if (!workspace.upgrade) {
// Wait some time for new client to appear before closing workspace.
if (workspace.sessions.size === 0) {
clearTimeout(workspace.closeTimeout)
void this.ctx.info('schedule warm closing', { workspace: workspace.workspaceName, wsid })
workspace.closeTimeout = setTimeout(() => {
void this.performWorkspaceCloseCheck(workspace, workspaceId, wsid)
}, this.timeouts.shutdownWarmTimeout)
}
} else {
await this.performWorkspaceCloseCheck(workspace, workspaceId, wsid)
}
}
}
async closeAll (wsId: string, workspace: Workspace, code: number, reason: 'upgrade' | 'shutdown'): Promise<void> {
async forceClose (wsId: string, ignoreSocket?: ConnectionSocket): Promise<void> {
const ws = this.workspaces.get(wsId)
if (ws !== undefined) {
ws.upgrade = true // We need to similare upgrade to refresh all clients.
ws.closing = this.closeAll(wsId, ws, 99, 'force-close', ignoreSocket)
await ws.closing
this.workspaces.delete(wsId)
}
}
async closeAll (
wsId: string,
workspace: Workspace,
code: number,
reason: 'upgrade' | 'shutdown' | 'force-close',
ignoreSocket?: ConnectionSocket
): Promise<void> {
if (LOGGING_ENABLED) {
await this.ctx.info('closing workspace', {
workspace: workspace.id,
@ -550,12 +598,11 @@ class TSessionManager implements SessionManager {
const closeS = async (s: Session, webSocket: ConnectionSocket): Promise<void> => {
s.workspaceClosed = true
if (reason === 'upgrade') {
if (reason === 'upgrade' || reason === 'force-close') {
// Override message handler, to wait for upgrading response from clients.
await this.sendUpgrade(workspace.context, webSocket, s.binaryResponseMode)
}
webSocket.close()
await this.trySetStatus(workspace.context, s, false)
}
if (LOGGING_ENABLED) {
@ -565,7 +612,9 @@ class TSessionManager implements SessionManager {
wsName: workspace.workspaceName
})
}
await Promise.all(sessions.map((s) => closeS(s[1].session, s[1].socket)))
await Promise.all(
sessions.filter((it) => it[1].socket.id !== ignoreSocket?.id).map((s) => closeS(s[1].session, s[1].socket))
)
const closePipeline = async (): Promise<void> => {
try {
@ -578,7 +627,7 @@ class TSessionManager implements SessionManager {
}
}
await this.ctx.with('closing', {}, async () => {
await Promise.race([closePipeline(), timeoutPromise(15000)])
await Promise.race([closePipeline(), timeoutPromise(120000)])
})
if (LOGGING_ENABLED) {
await this.ctx.info('Workspace closed...', { workspace: workspace.id, wsId, wsName: workspace.workspaceName })
@ -612,15 +661,12 @@ class TSessionManager implements SessionManager {
workspaceId: WorkspaceId,
wsid: string
): Promise<void> {
if (workspace.sessions.size === 0) {
const wsUID = workspace.id
const logParams = { wsid, workspace: workspace.id, wsName: workspaceId.name }
if (workspace.sessions.size === 0) {
if (LOGGING_ENABLED) {
await this.ctx.info('no sessions for workspace', logParams)
}
if (workspace.closing === undefined) {
const waitAndClose = async (workspace: Workspace): Promise<void> => {
try {
if (workspace.sessions.size === 0) {
const pl = await workspace.pipeline
@ -642,10 +688,13 @@ class TSessionManager implements SessionManager {
await this.ctx.error('failed', { ...logParams, error: err })
}
}
} else {
if (LOGGING_ENABLED) {
await this.ctx.info('few sessions for workspace, close skipped', {
...logParams,
sessions: workspace.sessions.size
})
}
workspace.closing = waitAndClose(workspace)
}
await workspace.closing
}
}
@ -720,13 +769,22 @@ class TSessionManager implements SessionManager {
const backupMode = 'loadChunk' in service
await userCtx.with(`🧭 ${backupMode ? 'handleBackup' : 'handleRequest'}`, {}, async (ctx) => {
const request = await ctx.with('📥 read', {}, async () => readRequest(msg, false))
if (request.id === -1 && request.method === 'close') {
if (request.method === 'forceClose') {
const wsRef = this.workspaces.get(workspace)
let done = false
if (wsRef !== undefined) {
await this.close(ws, wsRef?.workspaceId, 1000, 'client request to close workspace')
} else {
ws.close()
if (wsRef.upgrade) {
done = true
console.log('FORCE CLOSE', workspace)
// In case of upgrade, we need to force close workspace not in interval handler
await this.forceClose(workspace, ws)
}
}
const forceCloseResponse: Response<any> = {
id: request.id,
result: done
}
await ws.send(ctx, forceCloseResponse, service.binaryResponseMode, service.useCompression)
return
}
if (request.id === -1 && request.method === 'hello') {
@ -737,6 +795,7 @@ class TSessionManager implements SessionManager {
if (LOGGING_ENABLED) {
await ctx.info('hello happen', {
workspace,
user: service.getUser(),
binary: service.binaryResponseMode,
compression: service.useCompression,
@ -894,9 +953,8 @@ export function start (
} & Partial<Timeouts>
): () => Promise<void> {
const sessions = new TSessionManager(ctx, opt.sessionFactory, {
pingTimeout: opt.pingTimeout ?? 1000,
shutdownWarmTimeout: opt.shutdownWarmTimeout ?? 60 * 1000,
reconnectTimeout: 3000
pingTimeout: opt.pingTimeout ?? 10000,
reconnectTimeout: 500
})
return opt.serverFactory(
sessions,

17
server/ws/src/server_http.ts
View File

@ -127,6 +127,13 @@ export function startHttpServer (
res.end()
return
}
case 'force-close': {
const wsId = req.query.wsId as string
void sessions.forceClose(wsId)
res.writeHead(200)
res.end()
return
}
case 'reboot': {
process.exit(0)
}
@ -227,11 +234,7 @@ export function startHttpServer (
void ctx.error('error', { error: session.error?.message, stack: session.error?.stack })
}
await cs.send(ctx, { id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } }, false, false)
// Wait 1 second before closing the connection
setTimeout(() => {
cs.close()
}, 10000)
return
}
// eslint-disable-next-line @typescript-eslint/no-misused-promises
@ -244,7 +247,7 @@ export function startHttpServer (
buff = Buffer.concat(msg).toString()
}
if (buff !== undefined) {
void handleRequest(session.context, session.session, cs, buff, session.workspaceName)
void handleRequest(session.context, session.session, cs, buff, session.workspaceId)
}
} catch (err: any) {
Analytics.handleError(err)
@ -259,12 +262,12 @@ export function startHttpServer (
return
}
// remove session after 1seconds, give a time to reconnect.
void sessions.close(cs, token.workspace, code, reason.toString())
void sessions.close(cs, token.workspace)
})
const b = buffer
buffer = undefined
for (const msg of b) {
await handleRequest(session.context, session.session, cs, msg, session.workspaceName)
await handleRequest(session.context, session.session, cs, msg, session.workspaceId)
}
}
wss.on('connection', handleConnection as any)

19
server/ws/src/stats.ts
View File

@ -3,7 +3,8 @@ import {
MeasureMetricsContext,
type Metrics,
metricsAggregate,
type MetricsData
type MetricsData,
toWorkspaceString
} from '@hcengineering/core'
import os from 'os'
import { type SessionManager } from './types'
@ -20,14 +21,22 @@ export function getStatistics (ctx: MeasureContext, sessions: SessionManager, ad
}
data.statistics.totalClients = sessions.sessions.size
if (admin) {
for (const [k, v] of sessions.workspaces) {
data.statistics.activeSessions[k] = Array.from(v.sessions.entries()).map(([k, v]) => ({
for (const [k, vv] of sessions.workspaces) {
data.statistics.activeSessions[k] = {
sessions: Array.from(vv.sessions.entries()).map(([k, v]) => ({
userId: v.session.getUser(),
data: v.socket.data(),
mins5: v.session.mins5,
total: v.session.total,
current: v.session.current
}))
current: v.session.current,
upgrade: v.session.isUpgradeClient()
})),
name: vv.workspaceName,
wsId: toWorkspaceString(vv.workspaceId),
sessionsTotal: vv.sessions.size,
upgrading: vv.upgrade,
closing: vv.closing !== undefined
}
}
}

29
server/ws/src/types.ts
View File

@ -1,5 +1,4 @@
import {
type WorkspaceIdWithUrl,
type Class,
type Doc,
type DocumentQuery,
@ -9,7 +8,8 @@ import {
type Ref,
type Tx,
type TxResult,
type WorkspaceId
type WorkspaceId,
type WorkspaceIdWithUrl
} from '@hcengineering/core'
import { type Response } from '@hcengineering/rpc'
import { type BroadcastFunc, type Pipeline } from '@hcengineering/server-core'
@ -65,6 +65,10 @@ export interface Session {
measureCtx?: { ctx: MeasureContext, time: number }
lastRequest: number
isUpgradeClient: () => boolean
getMode: () => string
}
/**
@ -118,9 +122,10 @@ export interface Workspace {
pipeline: Promise<Pipeline>
sessions: Map<string, { session: Session, socket: ConnectionSocket }>
upgrade: boolean
backup: boolean
closing?: Promise<void>
closeTimeout?: any
softShutdown: number
workspaceId: WorkspaceId
workspaceName: string
@ -144,15 +149,21 @@ export interface SessionManager {
productId: string,
sessionId: string | undefined,
accountsUrl: string
) => Promise<
{ session: Session, context: MeasureContext, workspaceName: string } | { upgrade: true } | { error: any }
>
) => Promise<{ session: Session, context: MeasureContext, workspaceId: string } | { upgrade: true } | { error: any }>
broadcastAll: (workspace: Workspace, tx: Tx[], targets?: string[]) => void
close: (ws: ConnectionSocket, workspaceId: WorkspaceId, code: number, reason: string) => Promise<void>
close: (ws: ConnectionSocket, workspaceId: WorkspaceId) => Promise<void>
closeAll: (wsId: string, workspace: Workspace, code: number, reason: 'upgrade' | 'shutdown') => Promise<void>
closeAll: (
wsId: string,
workspace: Workspace,
code: number,
reason: 'upgrade' | 'shutdown',
ignoreSocket?: ConnectionSocket
) => Promise<void>
forceClose: (wsId: string, ignoreSocket?: ConnectionSocket) => Promise<void>
closeWorkspaces: (ctx: MeasureContext) => Promise<void>
@ -169,7 +180,7 @@ export type HandleRequestFunction = <S extends Session>(
service: S,
ws: ConnectionSocket,
msg: Buffer,
workspace: string
workspaceId: string
) => Promise<void>
/**