[backend] add cache storage module (#4320)

* [backend] add cache storage module

* update docs

* update default TTL to a week

packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx

@@ -41,6 +41,8 @@ import TabItem from '@theme/TabItem';
     ['FRONT_BASE_URL', 'http://localhost:3001', 'Url to the hosted frontend'],
     ['SERVER_URL', 'http://localhost:3000', 'Url to the hosted server'],
     ['PORT', '3000', 'Port'],
+    ['CACHE_STORAGE_TYPE', 'memory', 'Cache type (memory, redis...)'],
+    ['CACHE_STORAGE_TTL', '3600 * 24 * 7', 'Cache TTL in seconds']
     ]}></OptionTable>
 
 ### Security

packages/twenty-server/package.json

@@ -32,8 +32,11 @@
   },
   "dependencies": {
     "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch",
+    "@nestjs/cache-manager": "^2.2.1",
     "@nestjs/graphql": "patch:@nestjs/graphql@12.0.8#./patches/@nestjs+graphql+12.0.8.patch",
     "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch",
+    "cache-manager": "^5.4.0",
+    "cache-manager-redis-yet": "^4.1.2",
     "class-validator": "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch",
     "graphql-middleware": "^6.1.35",
     "passport": "^0.7.0",

packages/twenty-server/src/integrations/cache-storage/__tests__/cache-storage.service.spec.ts

@@ -0,0 +1,78 @@
+import { Cache } from '@nestjs/cache-manager';
+
+import { CacheStorageService } from 'src/integrations/cache-storage/cache-storage.service';
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+const cacheStorageNamespace = CacheStorageNamespace.Messaging;
+
+describe('CacheStorageService', () => {
+  let cacheStorageService: CacheStorageService;
+  let cacheManagerMock: Partial<Cache>;
+
+  beforeEach(() => {
+    cacheManagerMock = {
+      get: jest.fn(),
+      set: jest.fn(),
+    };
+
+    cacheStorageService = new CacheStorageService(
+      cacheManagerMock as Cache,
+      cacheStorageNamespace,
+    );
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('get', () => {
+    it('should call cacheManager.get with the correct namespaced key', async () => {
+      const key = 'testKey';
+      const namespacedKey = `${cacheStorageNamespace}:${key}`;
+
+      await cacheStorageService.get(key);
+
+      expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
+    });
+
+    it('should return the value returned by cacheManager.get', async () => {
+      const key = 'testKey';
+      const value = 'testValue';
+
+      jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
+
+      const result = await cacheStorageService.get(key);
+
+      expect(result).toBe(value);
+    });
+  });
+
+  describe('set', () => {
+    it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
+      const key = 'testKey';
+      const value = 'testValue';
+      const ttl = 60;
+      const namespacedKey = `${cacheStorageNamespace}:${key}`;
+
+      await cacheStorageService.set(key, value, ttl);
+
+      expect(cacheManagerMock.set).toHaveBeenCalledWith(
+        namespacedKey,
+        value,
+        ttl,
+      );
+    });
+
+    it('should not throw if cacheManager.set resolves successfully', async () => {
+      const key = 'testKey';
+      const value = 'testValue';
+      const ttl = 60;
+
+      jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
+
+      await expect(
+        cacheStorageService.set(key, value, ttl),
+      ).resolves.not.toThrow();
+    });
+  });
+});

packages/twenty-server/src/integrations/cache-storage/cache-storage.module-factory.ts

@@ -0,0 +1,46 @@
+import { CacheModuleOptions } from '@nestjs/common';
+
+import { redisStore } from 'cache-manager-redis-yet';
+
+import { CacheStorageType } from 'src/integrations/cache-storage/types/cache-storage-type.enum';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+export const cacheStorageModuleFactory = (
+  environmentService: EnvironmentService,
+): CacheModuleOptions => {
+  const cacheStorageType = environmentService.getCacheStorageType();
+  const cacheStorageTtl = environmentService.getCacheStorageTtl();
+  const cacheModuleOptions: CacheModuleOptions = {
+    isGlobal: true,
+    ttl: cacheStorageTtl * 1000,
+  };
+
+  switch (cacheStorageType) {
+    case CacheStorageType.Memory: {
+      return cacheModuleOptions;
+    }
+    case CacheStorageType.Redis: {
+      const host = environmentService.getRedisHost();
+      const port = environmentService.getRedisPort();
+
+      if (!(host && port)) {
+        throw new Error(
+          `${cacheStorageType} cache storage requires host: ${host} and port: ${port} to be defined, check your .env file`,
+        );
+      }
+
+      return {
+        ...cacheModuleOptions,
+        store: redisStore,
+        socket: {
+          host,
+          port,
+        },
+      };
+    }
+    default:
+      throw new Error(
+        `Invalid cache-storage (${cacheStorageType}), check your .env file`,
+      );
+  }
+};

packages/twenty-server/src/integrations/cache-storage/cache-storage.module.ts

@@ -0,0 +1,31 @@
+import { Module, Global } from '@nestjs/common';
+import { CacheModule, CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+import { ConfigModule } from '@nestjs/config';
+
+import { CacheStorageService } from 'src/integrations/cache-storage/cache-storage.service';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import { cacheStorageModuleFactory } from 'src/integrations/cache-storage/cache-storage.module-factory';
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+@Global()
+@Module({
+  imports: [
+    CacheModule.registerAsync({
+      isGlobal: true,
+      imports: [ConfigModule],
+      useFactory: cacheStorageModuleFactory,
+      inject: [EnvironmentService],
+    }),
+  ],
+  providers: [
+    ...Object.values(CacheStorageNamespace).map((cacheStorageNamespace) => ({
+      provide: cacheStorageNamespace,
+      useFactory: (cacheManager: Cache) => {
+        return new CacheStorageService(cacheManager, cacheStorageNamespace);
+      },
+      inject: [CACHE_MANAGER],
+    })),
+  ],
+  exports: [...Object.values(CacheStorageNamespace)],
+})
+export class CacheStorageModule {}

packages/twenty-server/src/integrations/cache-storage/cache-storage.service.ts

@@ -0,0 +1,21 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+@Injectable()
+export class CacheStorageService {
+  constructor(
+    @Inject(CACHE_MANAGER)
+    private readonly cacheManager: Cache,
+    private readonly namespace: CacheStorageNamespace,
+  ) {}
+
+  async get<T>(key: string): Promise<T | undefined> {
+    return this.cacheManager.get(`${this.namespace}:${key}`);
+  }
+
+  async set<T>(key: string, value: T, ttl?: number) {
+    return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
+  }
+}

packages/twenty-server/src/integrations/cache-storage/types/cache-storage-namespace.enum.ts

@@ -0,0 +1,3 @@
+export enum CacheStorageNamespace {
+  Messaging = 'messaging',
+}

packages/twenty-server/src/integrations/cache-storage/types/cache-storage-type.enum.ts

@@ -0,0 +1,4 @@
+export enum CacheStorageType {
+  Memory = 'memory',
+  Redis = 'redis',
+}

packages/twenty-server/src/integrations/environment/environment.service.ts

@@ -328,4 +328,12 @@ export class EnvironmentService {
       this.configService.get<number>('MUTATION_MAXIMUM_RECORD_AFFECTED') ?? 100
     );
   }
+
+  getCacheStorageType(): string {
+    return this.configService.get<string>('CACHE_STORAGE_TYPE') ?? 'memory';
+  }
+
+  getCacheStorageTtl(): number {
+    return this.configService.get<number>('CACHE_STORAGE_TTL') ?? 3600 * 24 * 7;
+  }
 }

packages/twenty-server/src/integrations/integrations.module.ts

@@ -9,6 +9,7 @@ import { loggerModuleFactory } from 'src/integrations/logger/logger.module-facto
 import { messageQueueModuleFactory } from 'src/integrations/message-queue/message-queue.module-factory';
 import { EmailModule } from 'src/integrations/email/email.module';
 import { emailModuleFactory } from 'src/integrations/email/email.module-factory';
+import { CacheStorageModule } from 'src/integrations/cache-storage/cache-storage.module';
 
 import { EnvironmentModule } from './environment/environment.module';
 import { EnvironmentService } from './environment/environment.service';
@@ -40,6 +41,7 @@ import { MessageQueueModule } from './message-queue/message-queue.module';
       inject: [EnvironmentService],
     }),
     EventEmitterModule.forRoot(),
+    CacheStorageModule,
   ],
   exports: [],
   providers: [],

packages/twenty-server/tsconfig.json

@@ -31,5 +31,6 @@
   "ts-node": {
     "files": true,
     "require": ["tsconfig-paths/register"]
-  }
+  },
+  "exclude": ["dist"]
 }

yarn.lock

@@ -7726,6 +7726,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@nestjs/cache-manager@npm:^2.2.1":
+  version: 2.2.1
+  resolution: "@nestjs/cache-manager@npm:2.2.1"
+  peerDependencies:
+    "@nestjs/common": ^9.0.0 || ^10.0.0
+    "@nestjs/core": ^9.0.0 || ^10.0.0
+    cache-manager: <=5
+    rxjs: ^7.0.0
+  checksum: d6be3ff686ef7a3c43175e9f3ddb08993e23a0f30b51e0390581ee5c894f8bfe212d4f8be000b0712acd35b8fe5ab7ffbb65145ae3b29801f295596459577648
+  languageName: node
+  linkType: hard
+
 "@nestjs/cli@npm:10.3.0":
   version: 10.3.0
   resolution: "@nestjs/cli@npm:10.3.0"
@@ -11048,6 +11060,62 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@redis/bloom@npm:1.2.0, @redis/bloom@npm:^1.2.0":
+  version: 1.2.0
+  resolution: "@redis/bloom@npm:1.2.0"
+  peerDependencies:
+    "@redis/client": ^1.0.0
+  checksum: 7dde8e67188164e96226c8a5c78ebd2801f1662947371e78fb95fb180c1e9ddff8d237012eb5e9182775be61cb546f67f759927cdaee0d178d863ee290e1fb27
+  languageName: node
+  linkType: hard
+
+"@redis/client@npm:1.5.14, @redis/client@npm:^1.5.8":
+  version: 1.5.14
+  resolution: "@redis/client@npm:1.5.14"
+  dependencies:
+    cluster-key-slot: "npm:1.1.2"
+    generic-pool: "npm:3.9.0"
+    yallist: "npm:4.0.0"
+  checksum: e8036ef1bce676891a492c198251238b3eb19eb7120ff974291f2a4e8cea97ac0f4aabbc0c59f40a923dcb43456e4e2a29b7287bd6a91535330e5e4631d9b176
+  languageName: node
+  linkType: hard
+
+"@redis/graph@npm:1.1.1, @redis/graph@npm:^1.1.0":
+  version: 1.1.1
+  resolution: "@redis/graph@npm:1.1.1"
+  peerDependencies:
+    "@redis/client": ^1.0.0
+  checksum: 64199db2cb3669c4911af8aba3b7116c4c2c1df37ca74b2a65555e62c863935a0cea74bc41bd92acf2e551074eb2a30c75f54a9f439b40e0f9bb67fc5fb66614
+  languageName: node
+  linkType: hard
+
+"@redis/json@npm:1.0.6, @redis/json@npm:^1.0.4":
+  version: 1.0.6
+  resolution: "@redis/json@npm:1.0.6"
+  peerDependencies:
+    "@redis/client": ^1.0.0
+  checksum: ac6072c33ac4552cf4748b6b2dc5fdc63f7a9396e6453b59ee03831cdde8d495caa90786e04036633d058c39cdf5c6fce903272c43ff942941b15c157ac34498
+  languageName: node
+  linkType: hard
+
+"@redis/search@npm:1.1.6, @redis/search@npm:^1.1.3":
+  version: 1.1.6
+  resolution: "@redis/search@npm:1.1.6"
+  peerDependencies:
+    "@redis/client": ^1.0.0
+  checksum: 690b30dc914f013c10c03899ddc5585194e891323c14f4d974d51d912944e50b5f21208e0fc5eed958dde87b730254846e9ffe5caf0b54ff1ff2c64a051df057
+  languageName: node
+  linkType: hard
+
+"@redis/time-series@npm:1.0.5, @redis/time-series@npm:^1.0.4":
+  version: 1.0.5
+  resolution: "@redis/time-series@npm:1.0.5"
+  peerDependencies:
+    "@redis/client": ^1.0.0
+  checksum: 3c7f31f64a5f215534db6f0a10845be046ffee2928972037713acdd72cdb9ccc4a476ecce70d896333346a8f4081bd2139a4d50da4d19b9d61a6836066188d68
+  languageName: node
+  linkType: hard
+
 "@rehooks/component-size@npm:^1.0.3":
   version: 1.0.3
   resolution: "@rehooks/component-size@npm:1.0.3"
@@ -20213,6 +20281,33 @@ __metadata:
   languageName: node
   linkType: hard
 
+"cache-manager-redis-yet@npm:^4.1.2":
+  version: 4.1.2
+  resolution: "cache-manager-redis-yet@npm:4.1.2"
+  dependencies:
+    "@redis/bloom": "npm:^1.2.0"
+    "@redis/client": "npm:^1.5.8"
+    "@redis/graph": "npm:^1.1.0"
+    "@redis/json": "npm:^1.0.4"
+    "@redis/search": "npm:^1.1.3"
+    "@redis/time-series": "npm:^1.0.4"
+    cache-manager: "npm:^5.2.2"
+    redis: "npm:^4.6.7"
+  checksum: dfa74d7de775cf89b570f19e97e3c8498f9c9441a0a049567fcec7ff2782967c6e61bb43dd038c0bcf940468e6f64eaa98d5b180a3589a7ebe0759595902876a
+  languageName: node
+  linkType: hard
+
+"cache-manager@npm:^5.2.2, cache-manager@npm:^5.4.0":
+  version: 5.4.0
+  resolution: "cache-manager@npm:5.4.0"
+  dependencies:
+    lodash.clonedeep: "npm:^4.5.0"
+    lru-cache: "npm:^10.1.0"
+    promise-coalesce: "npm:^1.1.2"
+  checksum: b89178fdb422625a40576a5db1bd11328381505cec84a10794b73239b36636ce36020a1b28bea2c96b5e96d30577d0ce1e28b63cd96258a6cae9c0df6f03f433
+  languageName: node
+  linkType: hard
+
 "cacheable-lookup@npm:^5.0.3":
   version: 5.0.4
   resolution: "cacheable-lookup@npm:5.0.4"
@@ -21163,7 +21258,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cluster-key-slot@npm:^1.1.0":
+"cluster-key-slot@npm:1.1.2, cluster-key-slot@npm:^1.1.0":
   version: 1.1.2
   resolution: "cluster-key-slot@npm:1.1.2"
   checksum: d7d39ca28a8786e9e801eeb8c770e3c3236a566625d7299a47bb71113fb2298ce1039596acb82590e598c52dbc9b1f088c8f587803e697cb58e1867a95ff94d3
@@ -26614,6 +26709,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"generic-pool@npm:3.9.0":
+  version: 3.9.0
+  resolution: "generic-pool@npm:3.9.0"
+  checksum: 6b314d0d71170d5cbaf7162c423f53f8d6556b2135626a65bcdc03c089840b0a2f59eeb2d907939b8200e945eaf71ceb6630426f22d2128a1d242aec4b232aa7
+  languageName: node
+  linkType: hard
+
 "gensync@npm:^1.0.0-beta.2":
   version: 1.0.0-beta.2
   resolution: "gensync@npm:1.0.0-beta.2"
@@ -32039,6 +32141,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lodash.clonedeep@npm:^4.5.0":
+  version: 4.5.0
+  resolution: "lodash.clonedeep@npm:4.5.0"
+  checksum: 2caf0e4808f319d761d2939ee0642fa6867a4bbf2cfce43276698828380756b99d4c4fa226d881655e6ac298dd453fe12a5ec8ba49861777759494c534936985
+  languageName: node
+  linkType: hard
+
 "lodash.debounce@npm:^4.0.8":
   version: 4.0.8
   resolution: "lodash.debounce@npm:4.0.8"
@@ -32447,6 +32556,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lru-cache@npm:^10.1.0":
+  version: 10.2.0
+  resolution: "lru-cache@npm:10.2.0"
+  checksum: c9847612aa2daaef102d30542a8d6d9b2c2bb36581c1bf0dc3ebf5e5f3352c772a749e604afae2e46873b930a9e9523743faac4e5b937c576ab29196774712ee
+  languageName: node
+  linkType: hard
+
 "lru-cache@npm:^4.0.1":
   version: 4.1.5
   resolution: "lru-cache@npm:4.1.5"
@@ -38550,6 +38666,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"promise-coalesce@npm:^1.1.2":
+  version: 1.1.2
+  resolution: "promise-coalesce@npm:1.1.2"
+  checksum: 9bcdc954a645b30ccb094ae76e36212af2ee7ecc6b9fc67634d7d4f490d082c1efc39590737d957a32f6df93e7c5f51ca232549095e9e2d7291c0482d1bb1557
+  languageName: node
+  linkType: hard
+
 "promise-inflight@npm:^1.0.1":
   version: 1.0.1
   resolution: "promise-inflight@npm:1.0.1"
@@ -40088,6 +40211,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"redis@npm:^4.6.7":
+  version: 4.6.13
+  resolution: "redis@npm:4.6.13"
+  dependencies:
+    "@redis/bloom": "npm:1.2.0"
+    "@redis/client": "npm:1.5.14"
+    "@redis/graph": "npm:1.1.1"
+    "@redis/json": "npm:1.0.6"
+    "@redis/search": "npm:1.1.6"
+    "@redis/time-series": "npm:1.0.5"
+  checksum: 5fbbf61fc244ca0d0eeca648e470de92d30a494f724f17fac434a7b2713425fc67be31736e17dc53798eba7d236cdfb66e330034ccced51f0faa1a3df1711d5d
+  languageName: node
+  linkType: hard
+
 "redux@npm:^4.2.1":
   version: 4.2.1
   resolution: "redux@npm:4.2.1"
@@ -44284,6 +44421,7 @@ __metadata:
   resolution: "twenty-server@workspace:packages/twenty-server"
   dependencies:
     "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch"
+    "@nestjs/cache-manager": "npm:^2.2.1"
     "@nestjs/cli": "npm:10.3.0"
     "@nestjs/graphql": "patch:@nestjs/graphql@12.0.8#./patches/@nestjs+graphql+12.0.8.patch"
     "@nx/js": "npm:17.2.8"
@@ -44295,6 +44433,8 @@ __metadata:
     "@types/lodash.snakecase": "npm:^4.1.7"
     "@types/lodash.upperfirst": "npm:^4.3.7"
     "@types/react": "npm:^18.2.39"
+    cache-manager: "npm:^5.4.0"
+    cache-manager-redis-yet: "npm:^4.1.2"
     class-validator: "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch"
     graphql-middleware: "npm:^6.1.35"
     passport: "npm:^0.7.0"
@@ -47256,6 +47396,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"yallist@npm:4.0.0, yallist@npm:^4.0.0":
+  version: 4.0.0
+  resolution: "yallist@npm:4.0.0"
+  checksum: 2286b5e8dbfe22204ab66e2ef5cc9bbb1e55dfc873bbe0d568aa943eb255d131890dfd5bf243637273d31119b870f49c18fcde2c6ffbb7a7a092b870dc90625a
+  languageName: node
+  linkType: hard
+
 "yallist@npm:^2.1.2":
   version: 2.1.2
   resolution: "yallist@npm:2.1.2"
@@ -47270,13 +47417,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"yallist@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "yallist@npm:4.0.0"
-  checksum: 2286b5e8dbfe22204ab66e2ef5cc9bbb1e55dfc873bbe0d568aa943eb255d131890dfd5bf243637273d31119b870f49c18fcde2c6ffbb7a7a092b870dc90625a
-  languageName: node
-  linkType: hard
-
 "yaml-ast-parser@npm:^0.0.43":
   version: 0.0.43
   resolution: "yaml-ast-parser@npm:0.0.43"