Fix ensoup launcher upgrade mechanism (#11833)

- Closes #11821 by updating the upgrade logic to the new executable name
- Re-enables the long disabled `UpgradeSpec` to make sure this remains tested.
- If the tests were enabled we would have caught the regression in #10535
- The tests have been heavily outdated due to being disabled, many small details changed and had to be amended.
- The tests are still marked as Flaky - they were known to be problematic on CI so their failures will not stop CI for now. But at least they are run and we can see if they succeed or not. Plus when running tests locally they will fail (as all tests marked as Flaky - the failure is only ignored on CI).
- Fixes another issue with an infinite cycle when no upgrade path can be found and adds a test for this case.
- If running a development build, the minimum version check can be ignored, as the check does not really make sense for `0.0.0-dev` build.
- Thus it closes #11831 also.
- Makes sure that `GithubAPI` caches the list of releases as fetching it can take time.

engine/launcher/src/main/scala/org/enso/launcher/InfoLogger.scala

@@ -1,6 +1,5 @@
 package org.enso.launcher
 
-import com.typesafe.scalalogging.Logger
 import org.enso.cli.CLIOutput
 
 /** Handles displaying of user-facing information.
@@ -11,23 +10,12 @@ import org.enso.cli.CLIOutput
   */
 object InfoLogger {
 
-  private val logger = Logger("launcher")
-
   /** Prints an info level message.
     *
-    * If the default logger is set-up to display info-messages, they are send to
+    * Currently, the message is always printed to standard output. But this may be changed by changing this method.
-    * the logger, otherwise they are printed to stdout.
-    *
-    * It is important to note that these messages should always be displayed to
-    * the user, so unless run in debug mode, all launcher settings should ensure
-    * that info-level logs are printed to the console output.
     */
   def info(msg: => String): Unit = {
-    if (logger.underlying.isInfoEnabled) {
+    CLIOutput.println(msg)
-      logger.info(msg)
-    } else {
-      CLIOutput.println(msg)
-    }
   }
 
 }

engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala

@@ -33,6 +33,9 @@ object LauncherRepository {
   private val launcherFallbackProviderHostname =
     "launcherfallback.release.enso.org"
 
+  /** URL to the repo that could be displayed to the user. */
+  def websiteUrl: String = "https://github.com/enso-org/enso"
+
   /** Defines a part of the URL scheme of the fallback mechanism - the name of
     * the directory that holds the releases.
     *

engine/launcher/src/main/scala/org/enso/launcher/releases/launcher/LauncherReleaseProvider.scala

@@ -29,8 +29,7 @@ class LauncherReleaseProvider(releaseProvider: SimpleReleaseProvider)
           .find(_.fileName == LauncherManifest.assetName)
           .toRight(
             ReleaseProviderException(
-              s"${LauncherManifest.assetName} file is missing from release " +
+              s"${LauncherManifest.assetName} file is missing from $tag release assets."
-              s"assets."
             )
           )
           .toTry

engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala

@@ -1,6 +1,6 @@
 package org.enso.launcher.upgrade
 
-import java.nio.file.{Files, Path}
+import java.nio.file.{AccessDeniedException, Files, Path}
 import com.typesafe.scalalogging.Logger
 import org.enso.semver.SemVer
 import org.enso.semver.SemVerOrdering._
@@ -24,7 +24,7 @@ import org.enso.launcher.cli.{
 import org.enso.launcher.releases.launcher.LauncherRelease
 import org.enso.runtimeversionmanager.releases.ReleaseProvider
 import org.enso.launcher.releases.LauncherRepository
-import org.enso.launcher.InfoLogger
+import org.enso.launcher.{Constants, InfoLogger}
 import org.enso.launcher.distribution.DefaultManagers
 import org.enso.runtimeversionmanager.locking.Resources
 import org.slf4j.LoggerFactory
@@ -97,12 +97,45 @@ class LauncherUpgrader(
           }
         }
       }
-      if (release.canPerformUpgradeFromCurrentVersion)
+
+      val canPerformDirectUpgrade: Boolean =
+        if (release.canPerformUpgradeFromCurrentVersion) true
+        else if (CurrentVersion.isDevVersion) {
+          logger.warn(
+            s"Cannot upgrade to version ${release.version} directly, because " +
+            s"it requires at least version " +
+            s"${release.minimumVersionToPerformUpgrade}."
+          )
+          if (globalCLIOptions.autoConfirm) {
+            logger.warn(
+              s"However, the current version (${CurrentVersion.version}) is " +
+              s"a development version, so the minimum version check can be " +
+              s"ignored. Since `auto-confirm` is set, the upgrade will " +
+              s"continue. But please be warned that it may fail due to " +
+              s"incompatibility."
+            )
+            true
+          } else {
+            logger.warn(
+              s"Since the current version (${CurrentVersion.version}) is " +
+              s"a development version, the minimum version check can be " +
+              s"ignored. However, please be warned that the upgrade " +
+              s"may fail due to incompatibility."
+            )
+            CLIOutput.askConfirmation(
+              "Do you want to continue upgrading to this version " +
+              "despite the warning?"
+            )
+          }
+        } else false
+
+      if (canPerformDirectUpgrade)
         performUpgradeTo(release)
       else
         performStepByStepUpgrade(release)
 
       runCleanup()
+      logger.debug("Upgrade completed successfully.")
     }
   }
 
@@ -120,11 +153,13 @@ class LauncherUpgrader(
     val temporaryFiles =
       FileSystem.listDirectory(binRoot).filter(isTemporaryExecutable)
     if (temporaryFiles.nonEmpty && isStartup) {
-      logger.debug("Cleaning temporary files from a previous upgrade.")
+      logger.debug(
+        s"Cleaning ${temporaryFiles.size} temporary files from a previous upgrade."
+      )
     }
     for (file <- temporaryFiles) {
       try {
-        Files.delete(file)
+        tryHardToDelete(file)
         logger.debug(s"Upgrade cleanup: removed `$file`.")
       } catch {
         case NonFatal(e) =>
@@ -133,6 +168,21 @@ class LauncherUpgrader(
     }
   }
 
+  /** On Windows, deleting an executable immediately after it has exited may fail
+    * and the process may need to wait a few millisecond. This method detects
+    * this kind of failure and retries a few times.
+    */
+  private def tryHardToDelete(file: Path, attempts: Int = 30): Unit = {
+    try {
+      Files.delete(file)
+    } catch {
+      case _: AccessDeniedException if attempts > 0 =>
+        logger.trace(s"Failed to delete file `$file`. Retrying.")
+        Thread.sleep(100)
+        tryHardToDelete(file, attempts - 1)
+    }
+  }
+
   /** Continues a multi-step upgrade.
     *
     * Called by [[InternalOpts]] when the upgrade continuation is requested by
@@ -217,28 +267,53 @@ class LauncherUpgrader(
 
   @scala.annotation.tailrec
   private def nextVersionToUpgradeTo(
-    release: LauncherRelease,
+    currentTargetRelease: LauncherRelease,
     availableVersions: Seq[SemVer]
   ): LauncherRelease = {
-    val recentEnoughVersions =
+    assert(
-      availableVersions.filter(
+      currentTargetRelease.minimumVersionToPerformUpgrade.isGreaterThan(
-        _.isGreaterThanOrEqual(release.minimumVersionToPerformUpgrade)
+        CurrentVersion.version
       )
+    )
+
+    // We look at older versions that are satisfying the minimum version
+    // required to upgrade to currentTargetRelease.
+    val recentEnoughVersions =
+      availableVersions.filter { possibleVersion =>
+        val canUpgradeToTarget = possibleVersion.isGreaterThanOrEqual(
+          currentTargetRelease.minimumVersionToPerformUpgrade
+        )
+        val isEarlierThanTarget =
+          possibleVersion.isLessThan(currentTargetRelease.version)
+        canUpgradeToTarget && isEarlierThanTarget
+      }
+
+    // We take the oldest of these, hoping that it will yield the shortest
+    // upgrade path (perhaps it will be possible to upgrade directly from
+    // current version)
     val minimumValidVersion = recentEnoughVersions.sorted.headOption.getOrElse {
       throw UpgradeError(
-        s"Upgrade failed: To continue upgrade, a version at least " +
+        s"Upgrade failed: To continue upgrade, at least version " +
-        s"${release.minimumVersionToPerformUpgrade} is required, but no " +
+        s"${currentTargetRelease.minimumVersionToPerformUpgrade} is required, " +
-        s"valid version satisfying this requirement could be found."
+        s"but no upgrade path has been found from the current version " +
+        s"${CurrentVersion.version}. " +
+        s"Please manually download a newer release from " +
+        s"${LauncherRepository.websiteUrl}"
       )
     }
-    val nextRelease = releaseProvider.fetchRelease(minimumValidVersion).get
+
+    val newTargetRelease = releaseProvider.fetchRelease(minimumValidVersion).get
+    assert(newTargetRelease.version != currentTargetRelease.version)
     logger.debug(
-      s"To upgrade to ${release.version}, " +
+      s"To upgrade to ${currentTargetRelease.version}, " +
-      s"the launcher will have to upgrade to ${nextRelease.version} first."
+      s"the launcher will have to upgrade to ${newTargetRelease.version} first."
     )
-    if (nextRelease.canPerformUpgradeFromCurrentVersion)
+
-      nextRelease
+    // If the current version cannot upgrade directly to the new target version,
-    else nextVersionToUpgradeTo(nextRelease, availableVersions)
+    // we continue the search looking for an even earlier version that we could
+    // upgrade to.
+    if (newTargetRelease.canPerformUpgradeFromCurrentVersion) newTargetRelease
+    else nextVersionToUpgradeTo(newTargetRelease, availableVersions)
   }
 
   /** Extracts just the launcher executable from the archive.
@@ -331,7 +406,7 @@ class LauncherUpgrader(
 
       val temporaryExecutable = temporaryExecutablePath("new")
       FileSystem.copyFile(
-        extractedRoot / "bin" / OS.executableName("enso"),
+        extractedRoot / "bin" / OS.executableName(Constants.name),
         temporaryExecutable
       )
 

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.1/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.2
+minimum-version-for-upgrade: 1.0.0
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.2/launcher-manifest.yaml

@@ -1,4 +1,4 @@
-minimum-version-for-upgrade: 0.0.0
+minimum-version-for-upgrade: 1.0.1
 files-to-copy:
   - README.md
 directories-to-copy:

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.3/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.1
+minimum-version-for-upgrade: 1.0.2
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.4/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.2
+minimum-version-for-upgrade: 1.0.2
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-99.9999.0/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.2
+minimum-version-for-upgrade: 0.0.0
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala

@@ -58,7 +58,7 @@ trait NativeTest
     args: Seq[String],
     extraEnv: Map[String, String]      = Map.empty,
     extraJVMProps: Map[String, String] = Map.empty,
-    timeoutSeconds: Long               = 15
+    timeoutSeconds: Long               = defaultTimeoutSeconds
   ): RunResult = {
     if (extraEnv.contains("PATH")) {
       throw new IllegalArgumentException(
@@ -89,7 +89,7 @@ trait NativeTest
     args: Seq[String],
     extraEnv: Map[String, String]      = Map.empty,
     extraJVMProps: Map[String, String] = Map.empty,
-    timeoutSeconds: Long               = 15
+    timeoutSeconds: Long               = defaultTimeoutSeconds
   ): RunResult = {
     if (extraEnv.contains("PATH")) {
       throw new IllegalArgumentException(
@@ -146,7 +146,7 @@ trait NativeTest
     args: Seq[String],
     pathOverride: String,
     extraJVMProps: Map[String, String] = Map.empty,
-    timeoutSeconds: Long               = 15
+    timeoutSeconds: Long               = defaultTimeoutSeconds
   ): RunResult = {
     runCommand(
       Seq(baseLauncherLocation.toAbsolutePath.toString) ++ args,
@@ -155,6 +155,8 @@ trait NativeTest
       timeoutSeconds = timeoutSeconds
     )
   }
+
+  private val defaultTimeoutSeconds: Long = 30
 }
 
 object NativeTest {

engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala

@@ -1,24 +1,23 @@
 package org.enso.launcher.upgrade
 
-import java.nio.file.{Files, Path, StandardCopyOption}
 import io.circe.parser
-import org.enso.semver.SemVer
-import org.enso.distribution.FileSystem
-import org.enso.distribution.locking.{FileLockManager, LockType}
-import FileSystem.PathSyntax
 import org.enso.cli.OS
+import org.enso.distribution.FileSystem
+import org.enso.distribution.FileSystem.PathSyntax
+import org.enso.distribution.locking.{FileLockManager, LockType}
 import org.enso.launcher._
-import org.enso.testkit.{FlakySpec, WithTemporaryDirectory}
 import org.enso.process.{RunResult, WrappedProcess}
-import org.enso.version.BuildVersion
+import org.enso.runtimeversionmanager.releases.testing.FakeAsset
+import org.enso.semver.SemVer
+import org.enso.testkit.{FlakySpec, WithTemporaryDirectory}
 import org.scalatest.exceptions.TestFailedException
-import org.scalatest.{BeforeAndAfterAll, Ignore, OptionValues}
+import org.scalatest.{BeforeAndAfterAll, OptionValues}
 
+import java.nio.file.{Files, Path, StandardCopyOption}
 import scala.concurrent.TimeoutException
 
 // TODO [DB] The suite became quite flaky and frequently fails the
 // Windows CI. Disabled until #3183 is implemented.
-@Ignore
 class UpgradeSpec
     extends NativeTest
     with WithTemporaryDirectory
@@ -32,13 +31,14 @@ class UpgradeSpec
 
   /** Location of built Rust artifacts.
     */
-  private val rustBuildRoot = Path.of("./target/rust/debug/")
+  private val rustBuildRoot =
+    Path.of("../../target/rust/debug/").toAbsolutePath.normalize()
 
   /** Location of the actual launcher executable that is wrapped by the shims.
     */
   private val realLauncherLocation =
     Path
-      .of(".")
+      .of("../../")
       .resolve(OS.executableName(Constants.name))
       .toAbsolutePath
       .normalize
@@ -70,10 +70,13 @@ class UpgradeSpec
   override def beforeAll(): Unit = {
     super.beforeAll()
     prepareLauncherBinary(SemVer.of(0, 0, 0))
-    prepareLauncherBinary(SemVer.of(0, 0, 1))
+    prepareLauncherBinary(SemVer.of(1, 0, 1))
-    prepareLauncherBinary(SemVer.of(0, 0, 2))
+    prepareLauncherBinary(SemVer.of(1, 0, 2))
-    prepareLauncherBinary(SemVer.of(0, 0, 3))
+    prepareLauncherBinary(SemVer.of(1, 0, 3))
-    prepareLauncherBinary(SemVer.of(0, 0, 4))
+    prepareLauncherBinary(SemVer.of(1, 0, 4))
+
+    // The 99.9999.0 version is marked as broken so it should not be considered for upgrades.
+    prepareLauncherBinary(SemVer.of(99, 9999, 0))
   }
 
   /** Prepares a launcher distribution in the temporary test location.
@@ -88,7 +91,7 @@ class UpgradeSpec
     */
   private def prepareDistribution(
     portable: Boolean,
-    launcherVersion: Option[SemVer] = None
+    launcherVersion: Option[SemVer]
   ): Unit = {
     val sourceLauncherLocation =
       launcherVersion.map(builtLauncherBinary).getOrElse(baseLauncherLocation)
@@ -172,41 +175,41 @@ class UpgradeSpec
     "upgrade to latest version (excluding broken)" taggedAs Flaky in {
       prepareDistribution(
         portable        = true,
-        launcherVersion = Some(SemVer.of(0, 0, 2))
+        launcherVersion = Some(SemVer.of(1, 0, 2))
       )
       run(Seq("upgrade")) should returnSuccess
 
-      checkVersion() shouldEqual SemVer.of(0, 0, 4)
+      checkVersion() shouldEqual SemVer.of(1, 0, 4)
     }
 
     "not downgrade without being explicitly asked to do so" taggedAs Flaky in {
-      // precondition for the test to make sense
-      SemVer
-        .parse(BuildVersion.ensoVersion)
-        .get
-        .isGreaterThan(SemVer.of(0, 0, 4)) shouldBe true
-
       prepareDistribution(
-        portable = true
+        portable        = false,
+        launcherVersion = Some(SemVer.of(99, 9999, 0))
       )
-      run(Seq("upgrade")).exitCode shouldEqual 1
+
+      // precondition for the test to make sense
+      checkVersion().isGreaterThan(SemVer.of(1, 0, 4)) shouldBe true
+      val result = run(Seq("upgrade"))
+      withClue(result) {
+        result.exitCode shouldEqual 1
+        result.stdout should include("If you really want to downgrade")
+      }
     }
 
     "upgrade/downgrade to a specific version " +
     "(and update necessary files)" taggedAs Flaky in {
-      // precondition for the test to make sense
-      SemVer
-        .parse(BuildVersion.ensoVersion)
-        .get
-        .isGreaterThan(SemVer.of(0, 0, 4)) shouldBe true
-
       prepareDistribution(
-        portable = true
+        portable        = true,
+        launcherVersion = Some(SemVer.of(99, 9999, 0))
       )
+      // precondition for the test to make sense
+      checkVersion().isGreaterThan(SemVer.of(1, 0, 4)) shouldBe true
+
       val root = launcherPath.getParent.getParent
       FileSystem.writeTextFile(root / "README.md", "Old readme")
-      run(Seq("upgrade", "0.0.1")) should returnSuccess
+      run(Seq("upgrade", "1.0.2")) should returnSuccess
-      checkVersion() shouldEqual SemVer.of(0, 0, 1)
+      checkVersion() shouldEqual SemVer.of(1, 0, 2)
       TestHelpers.readFileContent(root / "README.md").trim shouldEqual "Content"
       TestHelpers
         .readFileContent(root / "THIRD-PARTY" / "test-license.txt")
@@ -216,19 +219,21 @@ class UpgradeSpec
     "upgrade also in installed mode" taggedAs Flaky in {
       prepareDistribution(
         portable        = false,
-        launcherVersion = Some(SemVer.of(0, 0, 0))
+        launcherVersion = Some(SemVer.of(1, 0, 1))
       )
       val dataRoot   = getTestDirectory / "data"
       val configRoot = getTestDirectory / "config"
-      checkVersion() shouldEqual SemVer.of(0, 0, 0)
+      checkVersion() shouldEqual SemVer.of(1, 0, 1)
       val env = Map(
         "ENSO_DATA_DIRECTORY"    -> dataRoot.toString,
         "ENSO_CONFIG_DIRECTORY"  -> configRoot.toString,
         "ENSO_RUNTIME_DIRECTORY" -> (getTestDirectory / "run").toString
       )
 
-      run(Seq("upgrade", "0.0.1"), extraEnv = env) should returnSuccess
+      run(Seq("upgrade", "1.0.2"), extraEnv = env) should returnSuccess
-      checkVersion() shouldEqual SemVer.of(0, 0, 1)
+      checkVersion() shouldEqual SemVer.of(1, 0, 2)
+
+      // Make sure that files were added
       TestHelpers
         .readFileContent(dataRoot / "README.md")
         .trim shouldEqual "Content"
@@ -238,27 +243,25 @@ class UpgradeSpec
     }
 
     "perform a multi-step upgrade if necessary" taggedAs Flaky in {
-      // 0.0.3 can only be upgraded from 0.0.2 which can only be upgraded from
+      // 1.0.4 can only be upgraded from 1.0.2 which can only be upgraded from
-      // 0.0.1, so the upgrade path should be following:
+      // 1.0.1, so the upgrade path should be following: 1.0.1 -> 1.0.2 -> 1.0.4
-      // 0.0.0 -> 0.0.1 -> 0.0.2 -> 0.0.3
       prepareDistribution(
         portable        = true,
-        launcherVersion = Some(SemVer.of(0, 0, 0))
+        launcherVersion = Some(SemVer.of(1, 0, 1))
       )
 
-      checkVersion() shouldEqual SemVer.of(0, 0, 0)
+      checkVersion() shouldEqual SemVer.of(1, 0, 1)
-      val process = startLauncher(Seq("upgrade", "0.0.3"))
+      val process = startLauncher(Seq("upgrade", "1.0.4"))
       try {
-        process.join(timeoutSeconds = 30) should returnSuccess
+        process.join(timeoutSeconds = 60) should returnSuccess
 
-        checkVersion() shouldEqual SemVer.of(0, 0, 3)
+        checkVersion() shouldEqual SemVer.of(1, 0, 4)
 
         val launchedVersions = Seq(
-          "0.0.0",
+          "1.0.1",
-          "0.0.0",
+          "1.0.1",
-          "0.0.1",
+          "1.0.2",
-          "0.0.2",
+          "1.0.4"
-          "0.0.3"
         )
 
         val reportedLaunchLog = TestHelpers
@@ -294,7 +297,7 @@ class UpgradeSpec
     "that action with the upgraded launcher" ignore {
       prepareDistribution(
         portable        = true,
-        launcherVersion = Some(SemVer.of(0, 0, 2))
+        launcherVersion = Some(SemVer.of(1, 0, 2))
       )
       val enginesPath = getTestDirectory / "enso" / "dist"
       Files.createDirectories(enginesPath)
@@ -303,7 +306,7 @@ class UpgradeSpec
       //  engine distribution can be used in the test
 //      FileSystem.copyDirectory(
 //        Path.of("target/distribution/"),
-//        enginesPath / "0.1.0"
+//        enginesPath / "1.0.2"
 //      )
       val script  = getTestDirectory / "script.enso"
       val message = "Hello from test"
@@ -321,17 +324,20 @@ class UpgradeSpec
           script.toAbsolutePath.toString,
           "--use-system-jvm",
           "--use-enso-version",
-          "0.1.0"
+          "1.0.2"
         )
       )
-      result should returnSuccess
+
-      result.stdout should include(message)
+      withClue(result) {
+        result should returnSuccess
+        result.stdout should include(message)
+      }
     }
 
     "fail if another upgrade is running in parallel" taggedAs Flaky in {
       prepareDistribution(
         portable        = true,
-        launcherVersion = Some(SemVer.of(0, 0, 1))
+        launcherVersion = Some(SemVer.of(1, 0, 1))
       )
 
       val syncLocker = new FileLockManager(getTestDirectory / "enso" / "lock")
@@ -341,14 +347,14 @@ class UpgradeSpec
       // so acquiring this exclusive lock will stall access to that file until
       // the exclusive lock is released
       val lock = syncLocker.acquireLock(
-        "testasset-" + launcherManifestAssetName,
+        FakeAsset.lockNameForAsset(launcherManifestAssetName),
         LockType.Exclusive
       )
 
       val firstSuspended = startLauncher(
         Seq(
           "upgrade",
-          "0.0.2",
+          "1.0.2",
           "--internal-emulate-repository-wait",
           "--launcher-log-level=trace"
         )
@@ -361,7 +367,7 @@ class UpgradeSpec
 
         val secondFailed = run(Seq("upgrade", "0.0.0"))
 
-        secondFailed.stderr should include("Another upgrade is in progress")
+        secondFailed.stdout should include("Another upgrade is in progress")
         secondFailed.exitCode shouldEqual 1
       } catch {
         case e: TimeoutException =>
@@ -376,8 +382,37 @@ class UpgradeSpec
         lock.release()
       }
 
-      firstSuspended.join(timeoutSeconds = 20) should returnSuccess
+      firstSuspended.join(timeoutSeconds = 60) should returnSuccess
-      checkVersion() shouldEqual SemVer.of(0, 0, 2)
+      checkVersion() shouldEqual SemVer.of(1, 0, 2)
+    }
+
+    "should return a useful error message if upgrade cannot be performed because no upgrade path exists" taggedAs Flaky in {
+      prepareDistribution(
+        portable        = true,
+        launcherVersion = Some(SemVer.of(0, 0, 0))
+      )
+
+      val result = run(Seq("upgrade", "1.0.4"))
+      withClue(result) {
+        result.exitCode shouldEqual 1
+        result.stdout should include(
+          "no upgrade path has been found from the current version 0.0.0."
+        )
+      }
+    }
+
+    "should allow to upgrade the development version ignoring the version check, but warn about it" taggedAs Flaky in {
+      prepareDistribution(
+        portable        = true,
+        launcherVersion = Some(SemVer.of(0, 0, 0, "dev"))
+      )
+
+      val result = run(Seq("upgrade", "1.0.4"))
+      result should returnSuccess
+      withClue(result) {
+        result.stdout should include("development version")
+        result.stdout should include("minimum version check can be ignored")
+      }
     }
   }
 }

lib/rust/launcher-shims/src/bin/launcher_000dev.rs

@@ -0,0 +1,12 @@
+use launcher_shims::wrap_launcher;
+
+
+
+// ===========================
+// === EntryPoint0.0.0-dev ===
+// ===========================
+
+/// Runs the launcher wrapper overriding the version to 0.0.0-dev.
+fn main() {
+    wrap_launcher("0.0.0-dev")
+}

lib/rust/launcher-shims/src/bin/launcher_001.rs

@@ -1,12 +0,0 @@
-use launcher_shims::wrap_launcher;
-
-
-
-// =======================
-// === EntryPoint0.0.1 ===
-// =======================
-
-/// Runs the launcher wrapper overriding the version to 0.0.1.
-fn main() {
-    wrap_launcher("0.0.1")
-}

lib/rust/launcher-shims/src/bin/launcher_002.rs

@@ -1,12 +0,0 @@
-use launcher_shims::wrap_launcher;
-
-
-
-// =======================
-// === EntryPoint0.0.2 ===
-// =======================
-
-/// Runs the launcher wrapper overriding the version to 0.0.2.
-fn main() {
-    wrap_launcher("0.0.2")
-}

lib/rust/launcher-shims/src/bin/launcher_003.rs

@@ -1,12 +0,0 @@
-use launcher_shims::wrap_launcher;
-
-
-
-// =======================
-// === EntryPoint0.0.3 ===
-// =======================
-
-/// Runs the launcher wrapper overriding the version to 0.0.3.
-fn main() {
-    wrap_launcher("0.0.3")
-}

lib/rust/launcher-shims/src/bin/launcher_004.rs

@@ -1,12 +0,0 @@
-use launcher_shims::wrap_launcher;
-
-
-
-// =======================
-// === EntryPoint0.0.4 ===
-// =======================
-
-/// Runs the launcher wrapper overriding the version to 0.0.4.
-fn main() {
-    wrap_launcher("0.0.4")
-}

lib/rust/launcher-shims/src/bin/launcher_101.rs

@@ -0,0 +1,12 @@
+use launcher_shims::wrap_launcher;
+
+
+
+// =======================
+// === EntryPoint1.0.1 ===
+// =======================
+
+/// Runs the launcher wrapper overriding the version to 1.0.1.
+fn main() {
+    wrap_launcher("1.0.1")
+}

lib/rust/launcher-shims/src/bin/launcher_102.rs

@@ -0,0 +1,12 @@
+use launcher_shims::wrap_launcher;
+
+
+
+// =======================
+// === EntryPoint1.0.2 ===
+// =======================
+
+/// Runs the launcher wrapper overriding the version to 1.0.2.
+fn main() {
+    wrap_launcher("1.0.2")
+}

lib/rust/launcher-shims/src/bin/launcher_103.rs

@@ -0,0 +1,12 @@
+use launcher_shims::wrap_launcher;
+
+
+
+// =======================
+// === EntryPoint1.0.3 ===
+// =======================
+
+/// Runs the launcher wrapper overriding the version to 1.0.3.
+fn main() {
+    wrap_launcher("1.0.3")
+}

lib/rust/launcher-shims/src/bin/launcher_104.rs

@@ -0,0 +1,12 @@
+use launcher_shims::wrap_launcher;
+
+
+
+// =======================
+// === EntryPoint1.0.4 ===
+// =======================
+
+/// Runs the launcher wrapper overriding the version to 1.0.4.
+fn main() {
+    wrap_launcher("1.0.4")
+}

lib/rust/launcher-shims/src/bin/launcher_9999990.rs

@@ -0,0 +1,12 @@
+use launcher_shims::wrap_launcher;
+
+
+
+// ===========================
+// === EntryPoint99.9999.0 ===
+// ===========================
+
+/// Runs the launcher wrapper overriding the version to 99.9999.0.
+fn main() {
+    wrap_launcher("99.9999.0")
+}

lib/rust/launcher-shims/src/lib.rs

@@ -74,7 +74,7 @@ pub fn wrap_launcher(version: impl AsRef<str>) {
 
 /// Appends a line to the file located at the provided path.
 pub fn append_to_log(path: PathBuf, line: impl AsRef<str>) -> io::Result<()> {
-    let mut log_file = OpenOptions::new().append(true).open(path)?;
+    let mut log_file = OpenOptions::new().create(true).append(true).open(path)?;
     writeln!(log_file, "{}", line.as_ref())?;
     Ok(())
 }

lib/scala/cli/src/main/scala/org/enso/cli/task/MappedTask.scala

@@ -6,7 +6,7 @@ import scala.util.Try
   *
   * Used internally by [[TaskProgress.flatMap]].
   */
-private class MappedTask[A, B](source: TaskProgress[A], f: A => Try[B])
+private class MappedTask[A, B](source: TaskProgress[A], f: Try[A] => Try[B])
     extends TaskProgress[B] { self =>
 
   var listeners: List[ProgressListener[B]] = Nil
@@ -17,7 +17,7 @@ private class MappedTask[A, B](source: TaskProgress[A], f: A => Try[B])
       listeners.foreach(_.progressUpdate(done, total))
 
     override def done(result: Try[A]): Unit = self.synchronized {
-      val mapped = result.flatMap(f)
+      val mapped = f(result)
       savedResult = Some(mapped)
       listeners.foreach(_.done(mapped))
     }

lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala

@@ -50,7 +50,19 @@ trait TaskProgress[A] {
     *         succeeded and the transformation succeeded too
     */
   def flatMap[B](f: A => Try[B]): TaskProgress[B] =
-    new MappedTask(this, f)
+    new MappedTask(this, (t: Try[A]) => t.flatMap(f))
+
+  /** Alters the task by transforming its failure case with a partial function
+    * `pf`.
+    *
+    * @param pf the partial function that can transform the original error
+    * @tparam B resulting type of `pf`
+    * @return a new [[TaskProgress]] with update failure behaviours
+    */
+  def recoverWith[B >: A](
+    pf: PartialFunction[Throwable, Try[B]]
+  ): TaskProgress[B] =
+    new MappedTask(this, (t: Try[A]) => t.recoverWith(pf))
 
   /** Alters the task by transforming its result with a function `f`.
     *

lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala

@@ -44,7 +44,7 @@ object HTTPDownload {
     sizeHint: Option[Long] = None,
     encoding: Charset      = StandardCharsets.UTF_8
   ): TaskProgress[APIResponse] = {
-    logger.debug("Fetching [{}].", request.requestImpl.uri())
+    logger.debug("Fetching [{} {}].", request.method, request.uri)
     val taskProgress =
       new TaskProgressImplementation[APIResponse](ProgressUnit.Bytes)
     val total: java.lang.Long = if (sizeHint.isDefined) sizeHint.get else null

lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala

@@ -5,4 +5,11 @@ import java.net.http.HttpRequest
 /** Wraps an underlying HTTP request implementation to make the outside API
   * independent of the internal implementation.
   */
-case class HTTPRequest(requestImpl: HttpRequest)
+case class HTTPRequest(requestImpl: HttpRequest) {
+
+  /** Returns the method of this request. */
+  def method: String = requestImpl.method()
+
+  /** Returns the URI of this request as string, used e.g. for logging. */
+  def uri: String = requestImpl.uri().toString
+}

lib/scala/runtime-version-manager/src/main/java/org/enso/runtimeversionmanager/releases/testing/CompressZipArchive.java

@@ -0,0 +1,30 @@
+package org.enso.runtimeversionmanager.releases.testing;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+class CompressZipArchive {
+  static void compress(Path source, Path destination) throws IOException {
+    try (var zip = Files.newOutputStream(destination);
+        var zos = new ZipOutputStream(zip);
+        var files = Files.walk(source)) {
+      files
+          .filter(path -> !Files.isDirectory(path))
+          .forEach(
+              path -> {
+                var zipEntry = new ZipEntry(source.relativize(path).toString());
+                try {
+                  zos.putNextEntry(zipEntry);
+                  Files.copy(path, zos);
+                  zos.closeEntry();
+                } catch (IOException e) {
+                  throw new UncheckedIOException(e);
+                }
+              });
+    }
+  }
+}

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala

@@ -1,5 +1,7 @@
 package org.enso.runtimeversionmanager.releases
 
+import org.enso.downloader.http.ResourceNotFound
+
 /** Indicates a release provider failure. */
 sealed class ReleaseProviderException(message: String, cause: Throwable)
     extends RuntimeException(message, cause) {
@@ -30,6 +32,20 @@ case class ReleaseNotFound(
   message: Option[String] = None,
   cause: Throwable        = null
 ) extends ReleaseProviderException(
-      message.getOrElse(s"Cannot find release `$tag`."),
+      ReleaseNotFound.constructMessage(tag, message, cause),
       cause
     )
+
+object ReleaseNotFound {
+  private def constructMessage(
+    tag: String,
+    message: Option[String],
+    cause: Throwable
+  ): String =
+    message.getOrElse {
+      val isCauseInteresting = cause != null && cause != ResourceNotFound()
+      val suffix =
+        if (isCauseInteresting) s" Caused by: ${cause.getMessage}" else ""
+      s"Cannot find release `$tag`.$suffix"
+    }
+}

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala

@@ -1,5 +1,7 @@
 package org.enso.runtimeversionmanager.releases.github
 
+import com.typesafe.scalalogging.Logger
+
 import java.nio.file.Path
 import io.circe._
 import io.circe.parser._
@@ -7,6 +9,7 @@ import org.enso.cli.task.TaskProgress
 import org.enso.downloader.http.{
   APIResponse,
   HTTPDownload,
+  HTTPException,
   HTTPRequestBuilder,
   Header,
   URIBuilder
@@ -16,7 +19,7 @@ import org.enso.runtimeversionmanager.releases.{
   ReleaseProviderException
 }
 
-import scala.util.{Success, Try}
+import scala.util.{Failure, Success, Try}
 
 /** Contains functions used to query the GitHubAPI endpoints.
   */
@@ -61,13 +64,31 @@ object GithubAPI {
     * bar for this task.
     */
   def listReleases(repository: Repository): Try[Seq[Release]] = {
+    releaseListCache.get(repository) match {
+      case Some(cachedList) =>
+        logger.debug("Using cached release list for repository {}", repository)
+        Success(cachedList)
+      case None =>
+        makeListReleasesRequest(repository).map { releases =>
+          releaseListCache.put(repository, releases)
+          releases
+        }
+    }
+  }
+
+  private def makeListReleasesRequest(
+    repository: Repository
+  ): Try[Seq[Release]] = {
     val perPage = 100
     def listPage(page: Int): Try[Seq[Release]] = {
-      val uri = (projectURI(repository) / "releases") ?
+      val uri =
-        ("per_page" -> perPage.toString) ? ("page" -> page.toString)
+        ((projectURI(
+          repository
+        ) / "releases") ? ("per_page" -> perPage.toString) ? ("page" -> page.toString))
+          .build()
 
       val downloadTask = HTTPDownload
-        .fetchString(HTTPRequestBuilder.fromURI(uri.build()).GET)
+        .fetchString(HTTPRequestBuilder.fromURI(uri).GET)
         .flatMap(response =>
           parse(response.content)
             .flatMap(
@@ -95,13 +116,47 @@ object GithubAPI {
     listAllPages(1)
   }
 
+  /** Fetching the list of releases may involve making multiple requests with big payloads, so it is good to cache the relevant information extracted from it to avoid unnecessary re-fetching.
+    *
+    * The runtime version manager assumes that the list of known releases will not change during the runtime of the program, so the cache is never invalidated as long as the program is running.
+    */
+  private val releaseListCache
+    : collection.concurrent.TrieMap[Repository, Seq[Release]] =
+    collection.concurrent.TrieMap.empty
+
   /** Fetches release metadata for the release associated with the given tag.
     */
   def getRelease(repo: Repository, tag: String): TaskProgress[Release] = {
-    val uri = projectURI(repo) / "releases" / "tags" / tag
+    findCachedRelease(repo, tag) match {
+      case Some(release) =>
+        logger.debug(
+          "Using cached release for tag {} in repository {}",
+          tag,
+          repo
+        )
+        TaskProgress.runImmediately(release)
+      case None =>
+        makeGetSingleReleaseRequest(repo, tag)
+    }
+  }
+
+  private def findCachedRelease(
+    repo: Repository,
+    tag: String
+  ): Option[Release] =
+    for {
+      cachedList <- releaseListCache.get(repo)
+      release    <- cachedList.find(_.tag == tag)
+    } yield release
+
+  private def makeGetSingleReleaseRequest(
+    repo: Repository,
+    tag: String
+  ): TaskProgress[Release] = {
+    val uri = (projectURI(repo) / "releases" / "tags" / tag).build()
 
     HTTPDownload
-      .fetchString(HTTPRequestBuilder.fromURI(uri.build()).GET)
+      .fetchString(HTTPRequestBuilder.fromURI(uri).GET)
       .flatMap(response =>
         parse(response.content)
           .flatMap(_.as[Release])
@@ -114,6 +169,9 @@ object GithubAPI {
           )
           .toTry
       )
+      .recoverWith { case httpException: HTTPException =>
+        Failure(ReleaseNotFound(tag, cause = httpException))
+      }
   }
 
   /** A helper function that detecte a rate-limit error and tries to make a more
@@ -165,6 +223,7 @@ object GithubAPI {
       .addHeader("Accept", "application/octet-stream")
       .GET
 
+    System.err.println(s"Preparing to download asset ${asset.url}")
     HTTPDownload
       .download(request, destination, Some(asset.size))
       .map(_ => ())
@@ -189,4 +248,6 @@ object GithubAPI {
       assets <- json.get[Seq[Asset]]("assets")
     } yield Release(tag, assets)
   }
+
+  private val logger: Logger = Logger[GithubAPI.type]
 }

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/FakeReleaseProvider.scala

@@ -118,7 +118,7 @@ case class FakeAsset(
     */
   private def maybeWaitForAsset(): Unit =
     lockManagerForAssets.foreach { lockManager =>
-      val name     = "testasset-" + fileName
+      val name     = FakeAsset.lockNameForAsset(fileName)
       val lockType = LockType.Shared
       val lock = lockManager.tryAcquireLock(
         name,
@@ -186,3 +186,9 @@ case class FakeAsset(
       }
     }
 }
+
+object FakeAsset {
+
+  /** Name for the lock used for synchronizing access to asset, used when instrumenting tests. */
+  def lockNameForAsset(assetName: String): String = s"testasset-$assetName"
+}

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala

@@ -53,22 +53,6 @@ object TestArchivePackager {
   }
 
   private def packZip(source: Path, destination: Path): Unit = {
-    val files = FileSystem.listDirectory(source)
+    CompressZipArchive.compress(source, destination)
-    val exitCode = Process(
-      Seq(
-        "powershell",
-        "Compress-Archive",
-        "-Path",
-        files.map(_.getFileName.toString).mkString(","),
-        "-DestinationPath",
-        destination.toAbsolutePath.toString
-      ),
-      source.toFile
-    ).!
-    if (exitCode != 0) {
-      throw new RuntimeException(
-        s"tar failed. Cannot create fake-archive for $source"
-      )
-    }
   }
 }

project/NativeImage.scala

@@ -97,7 +97,7 @@ object NativeImage {
   ): Def.Initialize[Task[Unit]] = Def
     .task {
       val log       = state.value.log
-      val targetLoc = artifactFile(targetDir, name, false)
+      val targetLoc = artifactFile(targetDir, name, withExtension = false)
 
       def nativeImagePath(prefix: Path)(path: Path): Path = {
         val base = path.resolve(prefix)
@@ -253,7 +253,7 @@ object NativeImage {
         s"Started building $targetLoc native image. The output is captured."
       )
       val retCode    = process.!(processLogger)
-      val targetFile = artifactFile(targetDir, name, true)
+      val targetFile = artifactFile(targetDir, name)
       if (retCode != 0 || !targetFile.exists()) {
         log.error(s"Native Image build of $targetFile failed, with output: ")
         println(sb.toString())
@@ -318,7 +318,7 @@ object NativeImage {
   def artifactFile(
     targetDir: File,
     name: String,
-    withExtension: Boolean = false
+    withExtension: Boolean = true
   ): File = {
     val artifactName =
       if (withExtension && Platform.isWindows) name + ".exe"