twentyhq/twenty
1
1
Fork 0
mirror of https://github.com/twentyhq/twenty.git synced 2024-11-22 21:50:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove performance logs (#6709)

Browse Source 
We have found the root cause of the issue:
- when using a datasource (including the cached ones), we are fetching
ObjectMetadataCollection from cache (700kB). Datasource usage is
happening any time we are using twentyORM, which is everywhere in the
jobs and in some resolvers (including the GetCurrentUser one). This is
leading to a high load on redis and leading to the performance issues we
are seeing.
- we actually don't need to fetch this objectMetadataCollection while
using a cached datasource, only when we instantiate a new one
This commit is contained in:
Charles Bochet 2024-08-21 14:17:11 +02:00 committed by GitHub
parent aa4ae53fb4
commit 612a875c7b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 25 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
packages/twenty-server/src/engine/core-modules/user/services/user.service.ts
View File

@ -40,24 +40,18 @@ export class UserService extends TypeOrmQueryService<User> {
return null;
}
console.time('loadWorkspaceMember repo');
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
user.defaultWorkspaceId,
'workspaceMember',
);
console.timeEnd('loadWorkspaceMember repo');
console.time('loadWorkspaceMember find');
const workspaceMember = await workspaceMemberRepository.findOne({
where: {
userId: user.id,
},
});
console.timeEnd('loadWorkspaceMember find');
return workspaceMember;
}
@ -66,21 +60,14 @@ export class UserService extends TypeOrmQueryService<User> {
return [];
}
console.time('loadWorkspaceMembers repo');
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspace.id,
'workspaceMember',
);
console.timeEnd('loadWorkspaceMembers repo');
console.time('loadWorkspaceMembers find');
const workspaceMembers = workspaceMemberRepository.find();
console.timeEnd('loadWorkspaceMembers find');
return workspaceMembers;
}

5
packages/twenty-server/src/engine/core-modules/user/user.resolver.ts
View File

@ -98,7 +98,6 @@ export class UserResolver {
nullable: true,
})
async workspaceMember(@Parent() user: User): Promise<WorkspaceMember | null> {
console.time('resolver workspaceMember');
const workspaceMember = await this.userService.loadWorkspaceMember(user);
if (workspaceMember && workspaceMember.avatarUrl) {
@ -109,7 +108,6 @@ export class UserResolver {
workspaceMember.avatarUrl = `${workspaceMember.avatarUrl}?token=${avatarUrlToken}`;
}
console.timeEnd('resolver workspaceMember');
// TODO: Fix typing disrepency between Entity and DTO
return workspaceMember as WorkspaceMember | null;
@ -119,7 +117,6 @@ export class UserResolver {
nullable: true,
})
async workspaceMembers(@Parent() user: User): Promise<WorkspaceMember[]> {
console.time('resolver workspaceMembers');
const workspaceMembers = await this.userService.loadWorkspaceMembers(
user.defaultWorkspace,
);
@ -135,8 +132,6 @@ export class UserResolver {
}
}
console.timeEnd('resolver workspaceMembers');
// TODO: Fix typing disrepency between Entity and DTO
return workspaceMembers as WorkspaceMember[];
}

67
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { EntitySchema, Repository } from 'typeorm';
import { v4 } from 'uuid';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@ -29,17 +28,11 @@ export class WorkspaceDatasourceFactory {
workspaceId: string,
workspaceMetadataVersion: string | null,
): Promise<WorkspaceDataSource> {
const logId = v4();
console.time(`fetch in datasource factory ${logId}`);
const latestWorkspaceMetadataVersion =
await this.workspaceMetadataVersionService.getMetadataVersion(
workspaceId,
);
console.timeEnd(`fetch in datasource factory ${logId}`);
const desiredWorkspaceMetadataVersion =
workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
@ -55,38 +48,35 @@ export class WorkspaceDatasourceFactory {
);
}
let cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
freshObjectMetadataCollection,
);
cachedObjectMetadataCollection = freshObjectMetadataCollection;
}
const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${latestWorkspaceMetadataVersion}`,
async () => {
const logId = v4();
let cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
freshObjectMetadataCollection,
);
cachedObjectMetadataCollection = freshObjectMetadataCollection;
}
console.log('Creating workspace fresh data source...' + logId);
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
@ -104,7 +94,6 @@ export class WorkspaceDatasourceFactory {
);
}
console.time('create entity schema' + logId);
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
@ -130,9 +119,7 @@ export class WorkspaceDatasourceFactory {
cachedEntitySchemas = entitySchemas;
}
console.timeEnd('create entity schema' + logId);
console.time('create workspace data source' + logId);
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
@ -156,11 +143,7 @@ export class WorkspaceDatasourceFactory {
},
);
console.timeEnd('create workspace data source' + logId);
console.time('initialize workspace data source' + logId);
await workspaceDataSource.initialize();
console.timeEnd('initialize workspace data source' + logId);
return workspaceDataSource;
},

2
packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts
View File

@ -16,7 +16,6 @@ export class CacheManager<T> {
return this.cache.get(cacheKey)!;
}
// Remove old entries with the same workspaceId
for (const key of this.cache.keys()) {
if (key.startsWith(`${workspaceId}-`)) {
await onDelete?.(this.cache.get(key)!);
@ -24,7 +23,6 @@ export class CacheManager<T> {
}
}
// Create a new value using the factory callback
const value = await factory();
if (!value) {

6
packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
View File

@ -1,7 +1,6 @@
import { Injectable, Type } from '@nestjs/common';
import { ObjectLiteral } from 'typeorm';
import { v4 } from 'uuid';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
@ -37,16 +36,11 @@ export class TwentyORMGlobalManager {
);
}
const logId = v4();
console.time(`createDataSource in orm ${logId}`);
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
null,
);
console.timeEnd(`createDataSource in orm ${logId}`);
const repository = workspaceDataSource.getRepository<T>(objectMetadataName);
return repository;