Recovery plan for failed migrations

docker-compose.rc.yml

@@ -29,7 +29,7 @@ services:
       POSTGRES_USER: tipi
       POSTGRES_DB: tipi
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -d tipi -U tipi" ]
+      test: ["CMD-SHELL", "pg_isready -d tipi -U tipi"]
       interval: 5s
       timeout: 10s
       retries: 120
@@ -59,6 +59,7 @@ services:
       POSTGRES_USERNAME: tipi
       POSTGRES_DBNAME: tipi
       POSTGRES_HOST: tipi-db
+      NODE_ENV: production
     networks:
       - tipi_main_network
 
@@ -71,7 +72,8 @@ services:
     networks:
       - tipi_main_network
     environment:
-      - INTERNAL_IP=${INTERNAL_IP}
+      INTERNAL_IP: ${INTERNAL_IP}
+      NODE_ENV: production
     labels:
       traefik.enable: true
       traefik.http.routers.dashboard.rule: PathPrefix("/") # Host(`tipi.local`) &&

docker-compose.yml

@@ -29,7 +29,7 @@ services:
       POSTGRES_USER: tipi
       POSTGRES_DB: tipi
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -d tipi -U tipi" ]
+      test: ["CMD-SHELL", "pg_isready -d tipi -U tipi"]
       interval: 5s
       timeout: 10s
       retries: 120
@@ -60,6 +60,7 @@ services:
       POSTGRES_USERNAME: tipi
       POSTGRES_DBNAME: tipi
       POSTGRES_HOST: tipi-db
+      NODE_ENV: production
     networks:
       - tipi_main_network
 
@@ -73,7 +74,8 @@ services:
     networks:
       - tipi_main_network
     environment:
-      - INTERNAL_IP=${INTERNAL_IP}
+      INTERNAL_IP: ${INTERNAL_IP}
+      NODE_ENV: production
     labels:
       traefik.enable: true
       traefik.http.routers.dashboard.rule: PathPrefix("/") # Host(`tipi.local`) &&

packages/system-api/src/core/updates/recover-migrations.ts

@@ -0,0 +1,38 @@
+import datasource from '../../config/datasource';
+import App from '../../modules/apps/app.entity';
+import User from '../../modules/auth/user.entity';
+import Update from '../../modules/system/update.entity';
+
+const recover = async () => {
+  console.log('Recovering broken database');
+  const apps = await App.find();
+  const users = await User.find();
+  const updated = await Update.find();
+
+  // drop database
+  await datasource.dropDatabase();
+
+  console.log('running migrations');
+  await datasource.runMigrations();
+
+  // create users
+  for (const user of users) {
+    await User.create(user).save();
+  }
+
+  // create apps
+  for (const app of apps) {
+    await App.create(app).save();
+  }
+
+  // create updates
+  for (const update of updated) {
+    await Update.create(update).save();
+  }
+
+  console.log('Users recovered', users.length);
+  console.log('Apps recovered', apps.length);
+  console.log('Database recovered');
+};
+
+export default recover;

packages/system-api/src/server.ts

@@ -14,6 +14,7 @@ import cors from 'cors';
 import datasource from './config/datasource';
 import appsService from './modules/apps/apps.service';
 import { runUpdates } from './core/updates/run';
+import recover from './core/updates/recover-migrations';
 
 let corsOptions = __prod__
   ? {
@@ -42,10 +43,6 @@ const main = async () => {
 
     await datasource.initialize();
 
-    if (__prod__) {
-      await datasource.runMigrations();
-    }
-
     const schema = await createSchema();
     const httpServer = createServer(app);
     const plugins = [ApolloLogs];
@@ -63,13 +60,22 @@ const main = async () => {
     await apolloServer.start();
     apolloServer.applyMiddleware({ app, cors: corsOptions });
 
+    if (__prod__) {
+      try {
+        await datasource.runMigrations();
+      } catch (e) {
+        logger.error(e);
+        await recover();
+      }
+    }
+
     // Run migrations
     await runUpdates();
 
     httpServer.listen(port, () => {
       // Start apps
       appsService.startAllApps();
-      logger.info(`Server running on port ${port} 🚀 Production => ${__prod__}`);
+      console.info(`Server running on port ${port} 🚀 Production => ${__prod__}`);
     });
   } catch (error) {
     console.log(error);