uberf-8480: configurable liveness condition for upgrade (#6943)

Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>

.vscode/launch.json

@@ -95,6 +95,7 @@
         "ACCOUNT_PORT": "3000",
         "FRONT_URL": "http://localhost:8080",
         "SES_URL": "",
+        // "WS_LIVENESS_DAYS": "1",
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost",

dev/docker-compose.yaml

@@ -83,6 +83,7 @@ services:
       - RESERVED_DB_NAMES=telegram,gmail,github
       - MODEL_ENABLED=*
       - LAST_NAME_FIRST=true
+      # - WS_LIVENESS_DAYS=1
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
       # - DISABLE_SIGNUP=true

server/account-service/src/index.ts

@@ -65,6 +65,17 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap
   const productName = process.env.PRODUCT_NAME
   const lang = process.env.LANGUAGE ?? 'en'
 
+  const wsLivenessDaysRaw = process.env.WS_LIVENESS_DAYS
+  let wsLivenessDays: number | undefined
+
+  if (wsLivenessDaysRaw !== undefined) {
+    try {
+      wsLivenessDays = parseInt(wsLivenessDaysRaw)
+    } catch (err: any) {
+      // DO NOTHING
+    }
+  }
+
   setMetadata(account.metadata.Transactors, transactorUri)
   setMetadata(platform.metadata.locale, lang)
   setMetadata(account.metadata.ProductName, productName)
@@ -72,6 +83,7 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap
   setMetadata(account.metadata.OtpRetryDelaySec, parseInt(process.env.OTP_RETRY_DELAY ?? '60'))
   setMetadata(account.metadata.SES_URL, ses)
   setMetadata(account.metadata.FrontURL, frontURL)
+  setMetadata(account.metadata.WsLivenessDays, wsLivenessDays)
 
   setMetadata(serverToken.metadata.Secret, serverSecret)
 

server/account/src/collections/mongo.ts

@@ -212,7 +212,8 @@ export class WorkspaceMongoDbCollection extends MongoDbCollection<Workspace> imp
     region: string,
     version: Data<Version>,
     operation: WorkspaceOperation,
-    processingTimeoutMs: number
+    processingTimeoutMs: number,
+    wsLivenessMs?: number
   ): Promise<WorkspaceInfo | undefined> {
     const pendingCreationQuery: Filter<Workspace>['$or'] = [{ mode: { $in: ['pending-creation', 'creating'] } }]
 
@@ -233,9 +234,13 @@ export class WorkspaceMongoDbCollection extends MongoDbCollection<Workspace> imp
             $or: [{ mode: 'active' }, { mode: { $exists: false } }]
           },
           versionQuery,
-          {
-            lastVisit: { $gt: Date.now() - 24 * 60 * 60 * 1000 }
-          }
+          ...(wsLivenessMs !== undefined
+            ? [
+                {
+                  lastVisit: { $gt: Date.now() - wsLivenessMs }
+                }
+              ]
+            : [])
         ]
       },
       {

server/account/src/collections/postgres.ts

@@ -358,7 +358,8 @@ export class WorkspacePostgresDbCollection extends PostgresDbCollection<Workspac
     region: string,
     version: Data<Version>,
     operation: WorkspaceOperation,
-    processingTimeoutMs: number
+    processingTimeoutMs: number,
+    wsLivenessMs?: number
   ): Promise<WorkspaceInfo | undefined> {
     const sqlChunks: string[] = [`SELECT * FROM ${this.name}`]
     const whereChunks: string[] = []
@@ -367,7 +368,7 @@ export class WorkspacePostgresDbCollection extends PostgresDbCollection<Workspac
     const pendingCreationSql = "mode IN ('pending-creation', 'creating')"
     const versionSql =
       '("versionMajor" < $1) OR ("versionMajor" = $1 AND "versionMinor" < $2) OR ("versionMajor" = $1 AND "versionMinor" = $2 AND "versionPatch" < $3)'
-    const pendingUpgradeSql = `(((disabled = FALSE OR disabled IS NULL) AND (mode = 'active' OR mode IS NULL) AND ${versionSql} AND "lastVisit" > $4) OR ((disabled = FALSE OR disabled IS NULL) AND mode = 'upgrading'))`
+    const pendingUpgradeSql = `(((disabled = FALSE OR disabled IS NULL) AND (mode = 'active' OR mode IS NULL) AND ${versionSql} ${wsLivenessMs !== undefined ? 'AND "lastVisit" > $4' : ''}) OR ((disabled = FALSE OR disabled IS NULL) AND mode = 'upgrading'))`
     const operationSql =
       operation === 'create'
         ? pendingCreationSql
@@ -375,7 +376,11 @@ export class WorkspacePostgresDbCollection extends PostgresDbCollection<Workspac
           ? pendingUpgradeSql
           : `(${pendingCreationSql} OR ${pendingUpgradeSql})`
     if (operation === 'upgrade' || operation === 'all') {
-      values.push(version.major, version.minor, version.patch, Date.now() - 24 * 60 * 60 * 1000)
+      values.push(version.major, version.minor, version.patch)
+
+      if (wsLivenessMs !== undefined) {
+        values.push(Date.now() - wsLivenessMs)
+      }
     }
     whereChunks.push(operationSql)
 
@@ -383,11 +388,11 @@ export class WorkspacePostgresDbCollection extends PostgresDbCollection<Workspac
     // to clear them with the worker.
 
     whereChunks.push('(attempts IS NULL OR attempts <= 3)')
-    whereChunks.push('("lastProcessingTime" IS NULL OR "lastProcessingTime" < $5)')
+    whereChunks.push(`("lastProcessingTime" IS NULL OR "lastProcessingTime" < $${values.length + 1})`)
     values.push(Date.now() - processingTimeoutMs)
 
     if (region !== '') {
-      whereChunks.push('region = $6')
+      whereChunks.push(`region = $${values.length + 1}`)
       values.push(region)
     } else {
       whereChunks.push("(region IS NULL OR region = '')")

server/account/src/operations.ts

@@ -1278,8 +1278,10 @@ export async function getPendingWorkspace (
   }
   // Move to config?
   const processingTimeoutMs = 30 * 1000
+  const wsLivenessDays = getMetadata(accountPlugin.metadata.WsLivenessDays)
+  const wsLivenessMs = wsLivenessDays !== undefined ? wsLivenessDays * 24 * 60 * 60 * 1000 : undefined
 
-  const result = await db.workspace.getPendingWorkspace(region, version, operation, processingTimeoutMs)
+  const result = await db.workspace.getPendingWorkspace(region, version, operation, processingTimeoutMs, wsLivenessMs)
 
   if (result != null) {
     ctx.info('getPendingWorkspace', {

server/account/src/plugin.ts

@@ -15,7 +15,8 @@ export const accountPlugin = plugin(accountId, {
     ProductName: '' as Metadata<string>,
     Transactors: '' as Metadata<string>,
     OtpTimeToLiveSec: '' as Metadata<number>,
-    OtpRetryDelaySec: '' as Metadata<number>
+    OtpRetryDelaySec: '' as Metadata<number>,
+    WsLivenessDays: '' as Metadata<number>
   },
   string: {
     ConfirmationText: '' as IntlString,

server/account/src/types.ts

@@ -195,6 +195,7 @@ export interface WorkspaceDbCollection extends DbCollection<Workspace> {
     region: string,
     version: Data<Version>,
     operation: WorkspaceOperation,
-    processingTimeoutMs: number
+    processingTimeoutMs: number,
+    wsLivenessMs?: number
   ) => Promise<WorkspaceInfo | undefined>
 }