Uniformize datasources (#5196)

## Context

We recently enabled the option to bypass SSL certificate authority
validation when establishing a connection to PostgreSQL. Previously, if
this validation failed, the server would revert to unencrypted traffic.
Now, it maintains encryption even if the SSL certificate check fails. In
the process, we overlooked a few DataSource setups, prompting a review
of DataSource creation within our code.

## Current State

Our DataSource initialization is distributed as follows:
- **Database folder**: Contains 'core', 'metadata', and 'raw'
DataSources. The 'core' and 'metadata' DataSources manage migrations and
static resolver calls to the database. The 'raw' DataSource is utilized
in scripts and commands that require handling both aspects.
- **typeorm.service.ts script**: These DataSources facilitate
multi-schema connections.

## Vision for Discussion
- **SystemSchema (formerly core) DataSource**: Manages system schema
migrations and system resolvers/repos. The 'core' schema will be renamed
to 'system' as the Core API will include parts of the system and
workspace schemas.
- **MetadataSchema DataSource**: Handles metadata schema migrations and
metadata API resolvers/repos.
- **(Dynamic) WorkspaceSchema DataSource**: Will be used in the Twenty
ORM to access a specific workspace schema.

We currently do not support cross-schema joins, so maintaining these
DataSources separately should be feasible. Core API resolvers will
select the appropriate DataSource based on the field context.
- **To be discussed**: The potential need for an AdminDataSource (akin
to 'Raw'), which would be used in commands, setup scripts, and the admin
panel to connect to any database schema without loading any model. This
DataSource should be reserved for cases where utilizing metadata,
system, or workspace entities is impractical.

## In This PR
- Ensuring all existing DataSources are compliant with the SSL update.
- Introducing RawDataSource to eliminate the need for declaring new
DataSource() instances in commands.

packages/twenty-server/scripts/setup-db.ts

@@ -1,8 +1,10 @@
 import console from 'console';
 
-import { camelToSnakeCase, connectionSource, performQuery } from './utils';
+import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
 
-connectionSource
+import { camelToSnakeCase, performQuery } from './utils';
+
+rawDataSource
   .initialize()
   .then(async () => {
     await performQuery(

packages/twenty-server/scripts/truncate-db.ts

@@ -1,10 +1,12 @@
 import console from 'console';
 
-import { connectionSource, performQuery } from './utils';
+import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
+
+import { performQuery } from './utils';
 
 async function dropSchemasSequentially() {
   try {
-    await connectionSource.initialize();
+    await rawDataSource.initialize();
 
     // Fetch all schemas
     const schemas = await performQuery(

packages/twenty-server/scripts/utils.ts

@@ -1,12 +1,6 @@
 import console from 'console';
 
-import { DataSource } from 'typeorm';
+import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
-
-export const connectionSource = new DataSource({
-  type: 'postgres',
-  logging: false,
-  url: process.env.PG_DATABASE_URL,
-});
 
 export const camelToSnakeCase = (str) =>
   str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
@@ -18,7 +12,7 @@ export const performQuery = async (
   ignoreAlreadyExistsError = false,
 ) => {
   try {
-    const result = await connectionSource.query(query);
+    const result = await rawDataSource.query(query);
 
     withLog && console.log(`Performed '${consoleDescription}' successfully`);
 

packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts

@@ -1,13 +1,12 @@
 import { Injectable } from '@nestjs/common';
 
-import { DataSource } from 'typeorm';
-
 import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
 import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
 import {
   deleteCoreSchema,
   seedCoreSchema,
 } from 'src/database/typeorm-seeds/core/demo';
+import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
 
 @Injectable()
 export class DataSeedDemoWorkspaceService {
@@ -18,14 +17,7 @@ export class DataSeedDemoWorkspaceService {
 
   async seedDemo(): Promise<void> {
     try {
-      const dataSource = new DataSource({
+      await rawDataSource.initialize();
-        url: this.environmentService.get('PG_DATABASE_URL'),
-        type: 'postgres',
-        logging: true,
-        schema: 'public',
-      });
-
-      await dataSource.initialize();
       const demoWorkspaceIds =
         this.environmentService.get('DEMO_WORKSPACE_IDS');
 
@@ -35,10 +27,10 @@ export class DataSeedDemoWorkspaceService {
         );
       }
       for (const workspaceId of demoWorkspaceIds) {
-        await deleteCoreSchema(dataSource, workspaceId);
+        await deleteCoreSchema(rawDataSource, workspaceId);
         await this.workspaceManagerService.delete(workspaceId);
 
-        await seedCoreSchema(dataSource, workspaceId);
+        await seedCoreSchema(rawDataSource, workspaceId);
         await this.workspaceManagerService.initDemo(workspaceId);
       }
     } catch (error) {

packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts

@@ -1,5 +1,5 @@
 import { Command, CommandRunner } from 'nest-commander';
-import { DataSource, EntityManager } from 'typeorm';
+import { EntityManager } from 'typeorm';
 
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { seedCompanies } from 'src/database/typeorm-seeds/workspace/companies';
@@ -27,6 +27,7 @@ import { seedCalendarEvents } from 'src/database/typeorm-seeds/workspace/calenda
 import { seedCalendarChannels } from 'src/database/typeorm-seeds/workspace/calendar-channel';
 import { seedCalendarChannelEventAssociations } from 'src/database/typeorm-seeds/workspace/calendar-channel-event-association';
 import { seedCalendarEventParticipants } from 'src/database/typeorm-seeds/workspace/calendar-event-participants';
+import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
 
 // TODO: implement dry-run
 @Command({
@@ -50,19 +51,12 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
 
   async run(): Promise<void> {
     try {
-      const dataSource = new DataSource({
-        url: this.environmentService.get('PG_DATABASE_URL'),
-        type: 'postgres',
-        logging: true,
-        schema: 'core',
-      });
-
       for (const workspaceId of this.workspaceIds) {
-        await dataSource.initialize();
+        await rawDataSource.initialize();
 
-        await seedCoreSchema(dataSource, workspaceId);
+        await seedCoreSchema(rawDataSource, workspaceId);
 
-        await dataSource.destroy();
+        await rawDataSource.destroy();
 
         const schemaName =
           await this.workspaceDataSourceService.createWorkspaceDBSchema(

packages/twenty-server/src/database/typeorm/core/core.datasource.ts

@@ -21,6 +21,7 @@ export const typeORMCoreModuleOptions: TypeOrmModuleOptions = {
         }
       : undefined,
 };
+
 export const connectionSource = new DataSource(
   typeORMCoreModuleOptions as DataSourceOptions,
 );

packages/twenty-server/src/database/typeorm/raw/raw.datasource.ts

@@ -0,0 +1,17 @@
+import { config } from 'dotenv';
+import { DataSource, DataSourceOptions } from 'typeorm';
+config();
+
+const typeORMRawModuleOptions: DataSourceOptions = {
+  url: process.env.PG_DATABASE_URL,
+  type: 'postgres',
+  logging: ['error'],
+  ssl:
+    process.env.PG_SSL_ALLOW_SELF_SIGNED === 'true'
+      ? {
+          rejectUnauthorized: false,
+        }
+      : undefined,
+};
+
+export const rawDataSource = new DataSource(typeORMRawModuleOptions);

packages/twenty-server/src/database/typeorm/typeorm.service.ts

@@ -89,6 +89,11 @@ export class TypeORMService implements OnModuleInit, OnModuleDestroy {
         ? ['query', 'error']
         : ['error'],
       schema,
+      ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
+        ? {
+            rejectUnauthorized: false,
+          }
+        : undefined,
     });
 
     await workspaceDataSource.initialize();