Add retries to GraalVM updater commands (#7079)

The change adds logic that will attempt a few retries when executing `gu` (GraalVM updater) commands. Previously, if it failed, it failed. Retries should help with the most common case - occassional network hiccups.

Closes #6880.

# Important Notes
Note that I don't use an external library for retries on purpose. Didn't want to introduce a yet another dependency for this tiny functionality.

CHANGELOG.md

@@ -839,8 +839,9 @@
 - [Improve and colorize compiler's diagnostic messages][6931]
 - [Execute some runtime commands synchronously to avoid race conditions][6998]
 - [Scala 2.13.11 update][7010]
-- [Improve parallel execution of commands and jobs in Language Server][7042]
 - [Add special handling for static method calls on Any][7033]
+- [Improve parallel execution of commands and jobs in Language Server][7042]
+- [Added retries when executing GraalVM updater][7079]
 
 [3227]: https://github.com/enso-org/enso/pull/3227
 [3248]: https://github.com/enso-org/enso/pull/3248
@@ -959,8 +960,9 @@
 [6931]: https://github.com/enso-org/enso/pull/6931
 [6998]: https://github.com/enso-org/enso/pull/6998
 [7010]: https://github.com/enso-org/enso/pull/7010
-[7042]: https://github.com/enso-org/enso/pull/7042
 [7033]: https://github.com/enso-org/enso/pull/7033
+[7042]: https://github.com/enso-org/enso/pull/7042
+[7079]: https://github.com/enso-org/enso/pull/7079
 
 # Enso 2.0.0-alpha.18 (2021-10-12)
 

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdater.scala

@@ -1,11 +1,10 @@
 package org.enso.runtimeversionmanager.components
 
 import java.nio.file.Path
-
 import com.typesafe.scalalogging.Logger
 
 import scala.sys.process._
-import scala.util.{Success, Try}
+import scala.util.{Failure, Success, Try}
 
 /** Module that manages components of the GraalVM distribution.
   *
@@ -19,18 +18,19 @@ class GraalVMComponentUpdater(runtime: GraalRuntime)
   private val logger = Logger[GraalVMComponentUpdater]
   private val gu     = runtime.findExecutable("gu")
 
+  /** Path to the GraalVM's updater.
+    *
+    * @return path that will be executed to call the updater
+    */
+  protected def updaterExec: Path = gu
+
   /** List the installed GraalVM components.
     *
     * @return the list of installed GraalVM components
     */
   override def list(): Try[Seq[GraalVMComponent]] = {
     val command = Seq("list", "-v")
-    val process = Process(
-      gu.toAbsolutePath.toString +: command,
-      Some(runtime.javaHome.toFile),
-      ("JAVA_HOME", runtime.javaHome),
-      ("GRAALVM_HOME", runtime.javaHome)
-    )
+
     logger.trace("{} {}", gu, Properties(gu))
     logger.debug(
       "Executing: JAVA_HOME={} GRRAALVM_HOME={} {} {}",
@@ -40,10 +40,23 @@ class GraalVMComponentUpdater(runtime: GraalRuntime)
       command.mkString(" ")
     )
 
-    for {
-      stdout <- Try(process.lazyLines(stderrLogger))
-      _ = logger.trace(stdout.mkString(System.lineSeparator()))
-    } yield ListOut.parse(stdout.toVector)
+    val executor = new ExponentialBackoffRetry(5, logger) {
+      override def cmd: String = "list"
+      override def executeProcess(
+        logger: ProcessLogger
+      ): Try[LazyList[String]] = {
+        val process = Process(
+          updaterExec.toAbsolutePath.toString +: command,
+          Some(runtime.javaHome.toFile),
+          ("JAVA_HOME", runtime.javaHome),
+          ("GRAALVM_HOME", runtime.javaHome)
+        )
+        Try(process.lazyLines(logger))
+      }
+    }
+    executor
+      .execute()
+      .map(stdout => if (stdout.isEmpty) Seq() else ListOut.parse(stdout))
   }
 
   /** Install the provided GraalVM components.
@@ -53,12 +66,6 @@ class GraalVMComponentUpdater(runtime: GraalRuntime)
   override def install(components: Seq[GraalVMComponent]): Try[Unit] = {
     if (components.nonEmpty) {
       val command = "install" +: components.map(_.id)
-      val process = Process(
-        gu.toAbsolutePath.toString +: command,
-        Some(runtime.path.toFile),
-        ("JAVA_HOME", runtime.javaHome),
-        ("GRAALVM_HOME", runtime.javaHome)
-      )
       logger.trace("{} {}", gu, Properties(gu))
       logger.debug(
         "Executing: JAVA_HOME={} GRRAALVM_HOME={} {} {}",
@@ -67,20 +74,78 @@ class GraalVMComponentUpdater(runtime: GraalRuntime)
         gu,
         command.mkString(" ")
       )
-      for {
-        stdout <- Try(process.lazyLines(stderrLogger))
-        _ = logger.trace(stdout.mkString(System.lineSeparator()))
-      } yield ()
+      val executor = new ExponentialBackoffRetry(5, logger) {
+        override def cmd: String = "install"
+        override def executeProcess(
+          logger: ProcessLogger
+        ): Try[LazyList[String]] = {
+          val process = Process(
+            updaterExec.toAbsolutePath.toString +: command,
+            Some(runtime.path.toFile),
+            ("JAVA_HOME", runtime.javaHome),
+            ("GRAALVM_HOME", runtime.javaHome)
+          )
+          Try(process.lazyLines(logger))
+        }
+      }
+      executor.execute().map { stdout =>
+        stdout.foreach(logger.trace(_))
+        ()
+      }
     } else {
       Success(())
     }
   }
 
-  private def stderrLogger =
-    ProcessLogger(err => logger.trace("[stderr] {}", err))
 }
 object GraalVMComponentUpdater {
 
+  abstract class ProcessWithRetries(maxRetries: Int, logger: Logger) {
+    def executeProcess(logger: ProcessLogger): Try[LazyList[String]]
+
+    def cmd: String
+
+    def execute(): Try[List[String]] = execute(0)
+
+    protected def retryWait(retry: Int): Long
+
+    private def execute(retry: Int): Try[List[String]] = {
+      val errors        = scala.collection.mutable.ListBuffer[String]()
+      val processLogger = ProcessLogger(err => errors.addOne(err))
+      executeProcess(processLogger) match {
+        case Success(stdout) =>
+          Try(stdout.toList).recoverWith({
+            case _ if retry < maxRetries =>
+              try {
+                Thread.sleep(retryWait(retry))
+              } catch {
+                case _: InterruptedException =>
+              }
+              execute(retry + 1)
+          })
+        case Failure(exception) if retry < maxRetries =>
+          logger.warn("{} failed: {}. Retrying...", cmd, exception.getMessage)
+          try {
+            Thread.sleep(retryWait(retry))
+          } catch {
+            case _: InterruptedException =>
+          }
+          execute(retry + 1)
+        case Failure(exception) =>
+          errors.foreach(logger.trace("[stderr] {}", _))
+          Failure(exception)
+      }
+    }
+  }
+
+  abstract class ExponentialBackoffRetry(maxRetries: Int, logger: Logger)
+      extends ProcessWithRetries(maxRetries, logger) {
+    override def retryWait(retry: Int): Long = {
+      200 * 2.toLong ^ retry
+    }
+
+  }
+
   implicit private def pathToString(path: Path): String =
     path.toAbsolutePath.toString
 

lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala

@@ -65,10 +65,54 @@ class GraalVMComponentUpdaterSpec extends AnyWordSpec with Matchers {
 
       ru.list() match {
         case Success(components) =>
-          components should not be empty
+          val componentIds = components.map(_.id)
+          componentIds should (contain("graalvm") and contain("js"))
         case Failure(cause) =>
           fail(cause)
       }
+
+      var maxFailures = 3
+      val ruSometimesFailing = new GraalVMComponentUpdater(graal) {
+        override def updaterExec: Path = if (maxFailures == 0) super.updaterExec
+        else {
+          maxFailures = maxFailures - 1
+          OS.operatingSystem match {
+            case OS.Linux   => Path.of("/bin/false")
+            case OS.MacOS   => Path.of("/bin/false")
+            case OS.Windows => Path.of("foobar")
+          }
+        }
+      }
+
+      ruSometimesFailing.list() match {
+        case Success(components) =>
+          val componentIds = components.map(_.id)
+          componentIds should (contain("graalvm") and contain("js"))
+        case Failure(_) =>
+      }
+
+      var attempted = 0
+      val ruAlwaysFailing = new GraalVMComponentUpdater(graal) {
+        override def updaterExec: Path = {
+          attempted = attempted + 1
+          OS.operatingSystem match {
+            case OS.Linux   => Path.of("/bin/false")
+            case OS.MacOS   => Path.of("/bin/false")
+            case OS.Windows => Path.of("foobar")
+          }
+        }
+      }
+
+      val expectedRetries = 5
+      ruAlwaysFailing.list() match {
+        case Success(_) =>
+          fail("expected `gu list` to always fail")
+        case Failure(_) =>
+          if (attempted != (expectedRetries + 1))
+            fail(
+              s"should have retried ${expectedRetries + 1} times, got $attempted"
+            )
+      }
     }
   }