refactor(shared): split node-only shared utils in a separate subpath of the package

docker-compose.dev.yml

@@ -3,7 +3,7 @@ version: '3.7'
 services:
   tipi-docker-proxy:
     container_name: tipi-docker-proxy
-    image: tecnativa/docker-socket-proxy
+    image: zoeyvid/docker-socket-proxy
     restart: unless-stopped
     networks:
       - socket_proxy
@@ -61,7 +61,7 @@ services:
     container_name: tipi-redis
     image: redis:7.2.0
     restart: unless-stopped
-    command: redis-server --requirepass ${REDIS_PASSWORD}
+    command: redis-server --requirepass ${REDIS_PASSWORD} --stop-writes-on-bgsave-error no
     ports:
       - 6379:6379
     volumes:

docker-compose.prod.yml

@@ -3,7 +3,7 @@ version: '3.7'
 services:
   tipi-docker-proxy:
     container_name: tipi-docker-proxy
-    image: tecnativa/docker-socket-proxy
+    image: zoeyvid/docker-socket-proxy
     restart: unless-stopped
     networks:
       - socket_proxy
@@ -61,7 +61,7 @@ services:
     container_name: tipi-redis
     image: redis:7.2.0
     restart: unless-stopped
-    command: redis-server --requirepass ${REDIS_PASSWORD}
+    command: redis-server --requirepass ${REDIS_PASSWORD} --stop-writes-on-bgsave-error no
     ports:
       - 6379:6379
     volumes:

e2e/helpers/settings.ts

@@ -1,6 +1,7 @@
 import { promises } from 'fs';
 import { z } from 'zod';
-import { settingsSchema, pathExists } from '@runtipi/shared';
+import { settingsSchema } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import { execRemoteCommand } from './write-remote-file';
 
 export const setSettings = async (settings: z.infer<typeof settingsSchema>) => {

packages/cli/src/executors/system/system.executors.ts

@@ -9,7 +9,7 @@ import path from 'path';
 import { spawn } from 'child_process';
 import { Stream } from 'stream';
 import dotenv from 'dotenv';
-import { pathExists } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import { AppExecutors } from '../app/app.executors';
 import { copySystemFiles, generateSystemEnvFile } from './system.helpers';
 import { TerminalSpinner } from '@/utils/logger/terminal-spinner';

packages/cli/src/executors/system/system.helpers.ts

@@ -2,7 +2,8 @@ import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
-import { envMapToString, envStringToMap, pathExists, settingsSchema } from '@runtipi/shared';
+import { envMapToString, envStringToMap, settingsSchema } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import { logger } from '@/utils/logger/logger';
 
 type EnvKeys =

packages/cli/src/utils/logger/logger.ts

@@ -1,4 +1,4 @@
-import { FileLogger } from '@runtipi/shared';
+import { FileLogger } from '@runtipi/shared/node';
 import path from 'node:path';
 
 export const logger = new FileLogger('cli', path.join(process.cwd(), 'logs'));

packages/shared/node/package.json

@@ -0,0 +1,6 @@
+{
+  "browser": null,
+  "main": "../src/node/index.ts",
+  "module": "../src/node/index.ts",
+  "types": "../src/node/index.ts"
+}

packages/shared/package.json

@@ -2,7 +2,22 @@
   "name": "@runtipi/shared",
   "version": "1.0.0",
   "description": "",
-  "main": "src/index.ts",
+  "main": "./src/index.ts",
+  "exports": {
+    ".": {
+      "import": "./src/index.ts"
+    },
+    "./node": {
+      "browser": null,
+      "import": "./src/node/index.ts"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "src",
+    "node",
+    "dist"
+  ],
   "scripts": {
     "lint": "eslint --ext .ts src",
     "tsc": "tsc --noEmit"

packages/shared/src/helpers/error-helpers.ts

@@ -6,6 +6,10 @@ const IgnoreErrors = [
   // Innocuous browser errors
   /ResizeObserver loop limit exceeded/,
   /ResizeObserver loop completed with undelivered notifications/,
+  // Error on user's side
+  /no space left on device/,
+  // Dark reader extension
+  /WeakMap key undefined must be an object or an unregistered symbol/,
 ];
 
 const cleanseUrl = (url: string) => {
@@ -60,7 +64,7 @@ export const cleanseErrorData = (event: ErrorEvent, hint: EventHint) => {
   }
 
   // IF error message starts with 'Command failed: docker-compose' then grab only the 200 last characters
-  if (error.message.startsWith('Command failed: docker-compose')) {
+  if (error.message?.startsWith('Command failed: docker-compose')) {
     // Command failed: docker-compose --env-file /storage/app-data/<app-name>/app.env
     const appName = error.message.split('/')[3];
     const message = error.message.slice(-200);

packages/shared/src/helpers/fs-helpers/index.ts

@@ -1 +0,0 @@
-export * from './fs-helpers';

packages/shared/src/helpers/index.ts

@@ -1,2 +0,0 @@
-export * from './env-helpers';
-export * from './fs-helpers';

packages/shared/src/index.ts

@@ -1,5 +1,10 @@
-export * from './schemas';
-export * from './helpers';
-export { createLogger } from './utils/logger';
-export { FileLogger } from './lib/FileLogger';
-export { execAsync } from './lib/exec-async';
+// Schemas
+export { appInfoSchema, formFieldSchema, FIELD_TYPES, APP_CATEGORIES, type AppInfo, type FormField, type AppCategory } from './schemas/app-schemas';
+export { envSchema, settingsSchema, ARCHITECTURES, type Architecture } from './schemas/env-schemas';
+export { eventSchema, eventResultSchema, EVENT_TYPES, type EventType, type SystemEvent } from './schemas/queue-schemas';
+export { linkSchema, type LinkInfo, type LinkInfoInput } from './schemas/link-schemas';
+export { systemInfoSchema, socketEventSchema, type SocketEvent } from './schemas/socket-schemas';
+
+// Helpers
+export { envMapToString, envStringToMap } from './helpers/env-helpers';
+export { cleanseErrorData } from './helpers/error-helpers';

packages/shared/src/lib/FileLogger/index.ts

@@ -1 +0,0 @@
-export { FileLogger } from './FileLogger';

packages/shared/src/lib/exec-async/index.ts

@@ -1 +0,0 @@
-export { execAsync } from './execAsync';

packages/shared/src/node/index.ts

@@ -0,0 +1,4 @@
+export { execAsync } from './helpers/exec-async';
+export { pathExists } from './helpers/fs-helpers';
+
+export { FileLogger } from './logger/FileLogger';

packages/shared/src/node/logger/FileLogger.ts

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import path from 'path';
-import { createLogger } from '../../utils/logger';
+import { newLogger as createLogger } from './Logger';
 
 function streamLogToHistory(logsFolder: string, logFile: string) {
   return new Promise((resolve, reject) => {

packages/shared/src/schemas/index.ts

@@ -1,4 +0,0 @@
-export * from './app-schemas';
-export * from './env-schemas';
-export * from './queue-schemas';
-export * from './link-schemas';

packages/shared/src/utils/logger/index.ts

@@ -1 +0,0 @@
-export { newLogger as createLogger } from './Logger';

packages/worker/build.js

@@ -18,6 +18,7 @@ async function bundle() {
     plugins: [
       sentryEsbuildPlugin({
         authToken: process.env.SENTRY_AUTH_TOKEN,
+        release: process.env.TIPI_VERSION,
         org: 'runtipi',
         project: 'runtipi-worker',
       }),

packages/worker/src/index.ts

@@ -43,6 +43,7 @@ const main = async () => {
   try {
     await logger.flush();
 
+    logger.info(`Running tipi-worker version: ${process.env.TIPI_VERSION}`);
     logger.info('Generating system env file...');
     const envMap = await generateSystemEnvFile();
 

packages/worker/src/lib/db/db.ts

@@ -1,18 +1,57 @@
 import pg from 'pg';
+import * as Sentry from '@sentry/node';
 import { getEnv } from '../environment';
+import { logger } from '../logger';
+
+class DbClientSingleton {
+  private client: pg.Client | null;
+
+  constructor() {
+    this.client = null;
+  }
+
+  async connect() {
+    if (!this.client) {
+      try {
+        const { postgresHost, postgresDatabase, postgresUsername, postgresPassword, postgresPort } = getEnv();
+
+        this.client = new pg.Client({
+          host: postgresHost,
+          database: postgresDatabase,
+          user: postgresUsername,
+          password: postgresPassword,
+          port: Number(postgresPort),
+        });
+
+        await this.client.connect();
+        logger.info('Database connection successfully established.');
+      } catch (error) {
+        logger.error('Failed to connect to the database:', error);
+        this.client = null; // Ensure client is null to retry connection on next call
+        throw error; // Rethrow or handle error as needed
+      }
+    }
+
+    this.client.on('error', (error) => {
+      Sentry.captureException(error);
+      logger.error('Database connection error:', error);
+      this.client = null;
+    });
+
+    return this.client;
+  }
+
+  async getClient() {
+    if (!this.client) {
+      await this.connect();
+    }
+
+    return this.client;
+  }
+}
+
+const dbClientSingleton = new DbClientSingleton();
 
 export const getDbClient = async () => {
-  const { postgresHost, postgresDatabase, postgresUsername, postgresPassword, postgresPort } = getEnv();
-
-  const client = new pg.Client({
-    host: postgresHost,
-    database: postgresDatabase,
-    user: postgresUsername,
-    password: postgresPassword,
-    port: Number(postgresPort),
-  });
-
-  await client.connect();
-
-  return client;
+  return dbClientSingleton.getClient();
 };

packages/worker/src/lib/docker/docker-helpers.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { execAsync, pathExists } from '@runtipi/shared';
+import { execAsync, pathExists } from '@runtipi/shared/node';
 import { logger } from '@/lib/logger';
 import { getEnv } from '@/lib/environment';
 import { ROOT_FOLDER, STORAGE_FOLDER } from '@/config/constants';

packages/worker/src/lib/logger/logger.ts

@@ -1,4 +1,4 @@
-import { FileLogger } from '@runtipi/shared';
+import { FileLogger } from '@runtipi/shared/node';
 import path from 'node:path';
 
 export const logger = new FileLogger('worker', path.join('/app', 'logs'), true);

packages/worker/src/lib/socket/SocketManager.ts

@@ -1,4 +1,4 @@
-import { SocketEvent } from '@runtipi/shared/src/schemas/socket';
+import { SocketEvent } from '@runtipi/shared';
 import { Server } from 'socket.io';
 import { logger } from '../logger';
 

packages/worker/src/lib/system/system.helpers.ts

@@ -4,7 +4,8 @@ import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
-import { envMapToString, envStringToMap, execAsync, pathExists, settingsSchema } from '@runtipi/shared';
+import { envMapToString, envStringToMap, settingsSchema } from '@runtipi/shared';
+import { execAsync, pathExists } from '@runtipi/shared/node';
 import { logger } from '../logger/logger';
 import { getRepoHash } from '../../services/repo/repo.helpers';
 import { ROOT_FOLDER } from '@/config/constants';

packages/worker/src/services/app/__tests__/app.executors.test.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { describe, it, expect, vi } from 'vitest';
 import path from 'path';
 import { faker } from '@faker-js/faker';
-import { pathExists } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import { AppExecutors } from '../app.executors';
 import { createAppConfig } from '@/tests/apps.factory';
 import * as dockerHelpers from '@/lib/docker';

packages/worker/src/services/app/__tests__/app.helpers.test.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import { describe, it, expect } from 'vitest';
 import { faker } from '@faker-js/faker';
-import { pathExists } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import { copyDataDir, generateEnvFile } from '../app.helpers';
 import { createAppConfig } from '@/tests/apps.factory';
 import { getAppEnvMap } from '../env.helpers';

packages/worker/src/services/app/app.executors.ts

@@ -3,8 +3,8 @@
 import fs from 'fs';
 import path from 'path';
 import * as Sentry from '@sentry/node';
-import { execAsync, pathExists } from '@runtipi/shared';
-import { SocketEvent } from '@runtipi/shared/src/schemas/socket';
+import { execAsync, pathExists } from '@runtipi/shared/node';
+import { SocketEvent } from '@runtipi/shared';
 import { copyDataDir, generateEnvFile } from './app.helpers';
 import { logger } from '@/lib/logger';
 import { compose } from '@/lib/docker';
@@ -138,7 +138,7 @@ export class AppExecutors {
 
       // run docker-compose up
       this.logger.info(`Running docker-compose up for app ${appId}`);
-      await compose(appId, 'up -d');
+      await compose(appId, 'up --detach --force-recreate --remove-orphans --pull always');
 
       this.logger.info(`Docker-compose up for app ${appId} finished`);
 
@@ -156,7 +156,6 @@ export class AppExecutors {
    * @param {Record<string, unknown>} config - The config of the app
    */
   public stopApp = async (appId: string, config: Record<string, unknown>, skipEnvGeneration = false) => {
-    const client = await getDbClient();
     try {
       const { appDirPath } = this.getAppPaths(appId);
       const configJsonPath = path.join(appDirPath, 'config.json');
@@ -181,17 +180,15 @@ export class AppExecutors {
 
       SocketManager.emit({ type: 'app', event: 'stop_success', data: { appId } });
 
-      await client.query('UPDATE app SET status = $1 WHERE id = $2', ['stopped', appId]);
+      const client = await getDbClient();
+      await client?.query('UPDATE app SET status = $1 WHERE id = $2', ['stopped', appId]);
       return { success: true, message: `App ${appId} stopped successfully` };
     } catch (err) {
       return this.handleAppError(err, appId, 'stop_error');
-    } finally {
-      await client.end();
     }
   };
 
   public startApp = async (appId: string, config: Record<string, unknown>, skipEnvGeneration = false) => {
-    const client = await getDbClient();
     try {
       SocketManager.emit({ type: 'app', event: 'status_change', data: { appId } });
 
@@ -217,17 +214,15 @@ export class AppExecutors {
 
       SocketManager.emit({ type: 'app', event: 'start_success', data: { appId } });
 
-      await client.query('UPDATE app SET status = $1 WHERE id = $2', ['running', appId]);
+      const client = await getDbClient();
+      await client?.query('UPDATE app SET status = $1 WHERE id = $2', ['running', appId]);
       return { success: true, message: `App ${appId} started successfully` };
     } catch (err) {
       return this.handleAppError(err, appId, 'start_error');
-    } finally {
-      await client.end();
     }
   };
 
   public uninstallApp = async (appId: string, config: Record<string, unknown>) => {
-    const client = await getDbClient();
     try {
       SocketManager.emit({ type: 'app', event: 'status_change', data: { appId } });
 
@@ -261,17 +256,15 @@ export class AppExecutors {
 
       SocketManager.emit({ type: 'app', event: 'uninstall_success', data: { appId } });
 
-      await client.query(`DELETE FROM app WHERE id = $1`, [appId]);
+      const client = await getDbClient();
+      await client?.query(`DELETE FROM app WHERE id = $1`, [appId]);
       return { success: true, message: `App ${appId} uninstalled successfully` };
     } catch (err) {
       return this.handleAppError(err, appId, 'uninstall_error');
-    } finally {
-      await client.end();
     }
   };
 
   public resetApp = async (appId: string, config: Record<string, unknown>) => {
-    const client = await getDbClient();
     try {
       SocketManager.emit({ type: 'app', event: 'status_change', data: { appId } });
 
@@ -318,12 +311,11 @@ export class AppExecutors {
 
       SocketManager.emit({ type: 'app', event: 'reset_success', data: { appId } });
 
-      await client.query(`UPDATE app SET status = $1 WHERE id = $2`, ['running', appId]);
+      const client = await getDbClient();
+      await client?.query(`UPDATE app SET status = $1 WHERE id = $2`, ['running', appId]);
       return { success: true, message: `App ${appId} reset successfully` };
     } catch (err) {
       return this.handleAppError(err, appId, 'reset_error');
-    } finally {
-      await client.end();
     }
   };
 
@@ -363,23 +355,22 @@ export class AppExecutors {
    * Start all apps with status running
    */
   public startAllApps = async (forceStartAll = false) => {
-    const client = await getDbClient();
-
     try {
+      const client = await getDbClient();
       let rows: { id: string; config: Record<string, unknown> }[] = [];
 
       if (!forceStartAll) {
         // Get all apps with status running
-        const result = await client.query(`SELECT * FROM app WHERE status = 'running'`);
-        rows = result.rows;
+        const result = await client?.query(`SELECT * FROM app WHERE status = 'running'`);
+        rows = result?.rows || [];
       } else {
         // Get all apps
-        const result = await client.query(`SELECT * FROM app`);
-        rows = result.rows;
+        const result = await client?.query(`SELECT * FROM app`);
+        rows = result?.rows || [];
       }
 
       // Update all apps with status different than running or stopped to stopped
-      await client.query(`UPDATE app SET status = 'stopped' WHERE status != 'stopped' AND status != 'running' AND status != 'missing'`);
+      await client?.query(`UPDATE app SET status = 'stopped' WHERE status != 'stopped' AND status != 'running' AND status != 'missing'`);
 
       // Start all apps
       for (const row of rows) {
@@ -389,15 +380,13 @@ export class AppExecutors {
 
         if (!success) {
           this.logger.error(`Error starting app ${id}`);
-          await client.query(`UPDATE app SET status = $1 WHERE id = $2`, ['stopped', id]);
+          await client?.query(`UPDATE app SET status = $1 WHERE id = $2`, ['stopped', id]);
         } else {
-          await client.query(`UPDATE app SET status = $1 WHERE id = $2`, ['running', id]);
+          await client?.query(`UPDATE app SET status = $1 WHERE id = $2`, ['running', id]);
         }
       }
     } catch (err) {
       this.logger.error(`Error starting apps: ${err}`);
-    } finally {
-      await client.end();
     }
   };
 }

packages/worker/src/services/app/app.helpers.ts

@@ -1,7 +1,8 @@
 import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
-import { appInfoSchema, envMapToString, envStringToMap, execAsync, pathExists } from '@runtipi/shared';
+import { appInfoSchema, envMapToString, envStringToMap } from '@runtipi/shared';
+import { pathExists, execAsync } from '@runtipi/shared/node';
 import { generateVapidKeys, getAppEnvMap } from './env.helpers';
 import { getEnv } from '@/lib/environment';
 import { ROOT_FOLDER, STORAGE_FOLDER } from '@/config/constants';

packages/worker/src/services/repo/repo.executors.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { execAsync, pathExists } from '@runtipi/shared';
+import { execAsync, pathExists } from '@runtipi/shared/node';
 import * as Sentry from '@sentry/node';
 import { getRepoHash, getRepoBaseUrlAndBranch } from './repo.helpers';
 import { logger } from '@/lib/logger';

pnpm-lock.yaml

@@ -441,6 +441,8 @@ importers:
         specifier: ^4.5.9
         version: 4.5.9
 
+  packages/shared/node: {}
+
   packages/worker:
     dependencies:
       '@hono/node-server':

src/app/api/app-image/route.ts

@@ -1,6 +1,6 @@
 import * as Sentry from '@sentry/nextjs';
 import { TipiConfig } from '@/server/core/TipiConfig/TipiConfig';
-import { pathExists } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import fs from 'fs-extra';
 import path from 'path';
 

src/lib/socket/useSocket.ts

@@ -1,4 +1,4 @@
-import { SocketEvent, socketEventSchema } from '@runtipi/shared/src/schemas/socket';
+import { SocketEvent, socketEventSchema } from '@runtipi/shared';
 import { useEffect, useState } from 'react';
 import io from 'socket.io-client';
 

src/server/services/apps/apps.helpers.ts

@@ -1,5 +1,6 @@
 import { App } from '@/server/db/schema';
-import { appInfoSchema, pathExists } from '@runtipi/shared';
+import { appInfoSchema } from '@runtipi/shared';
+import { pathExists } from '@runtipi/shared/node';
 import { fileExists, readdirSync, readFile, readJsonFile } from '../../common/fs.helpers';
 import { TipiConfig } from '../../core/TipiConfig';
 import { Logger } from '../../core/Logger';

src/server/services/system/system.service.ts

@@ -1,7 +1,7 @@
 import { promises } from 'fs';
 import axios from 'redaxios';
 import { TipiCache } from '@/server/core/TipiCache';
-import { systemInfoSchema } from '@runtipi/shared/src/schemas/socket';
+import { systemInfoSchema } from '@runtipi/shared';
 import { fileExists, readJsonFile } from '../../common/fs.helpers';
 import { Logger } from '../../core/Logger';
 import { TipiConfig } from '../../core/TipiConfig';