Fix ensoup launcher upgrade mechanism (#11833)

- Closes #11821 by updating the upgrade logic to the new executable name
- Re-enables the long disabled `UpgradeSpec` to make sure this remains tested.
- If the tests were enabled we would have caught the regression in #10535
- The tests have been heavily outdated due to being disabled, many small details changed and had to be amended.
- The tests are still marked as Flaky - they were known to be problematic on CI so their failures will not stop CI for now. But at least they are run and we can see if they succeed or not. Plus when running tests locally they will fail (as all tests marked as Flaky - the failure is only ignored on CI).
- Fixes another issue with an infinite cycle when no upgrade path can be found and adds a test for this case.
- If running a development build, the minimum version check can be ignored, as the check does not really make sense for `0.0.0-dev` build.
- Thus it closes #11831 also.
- Makes sure that `GithubAPI` caches the list of releases as fetching it can take time.
This commit is contained in:
Radosław Waśko 2024-12-12 18:06:01 +01:00 committed by GitHub
parent fe5e13456a
commit e9b0ba95b0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
39 changed files with 430 additions and 188 deletions

View File

@ -1,6 +1,5 @@
package org.enso.launcher
import com.typesafe.scalalogging.Logger
import org.enso.cli.CLIOutput
/** Handles displaying of user-facing information.
@ -11,23 +10,12 @@ import org.enso.cli.CLIOutput
*/
object InfoLogger {
private val logger = Logger("launcher")
/** Prints an info level message.
*
* If the default logger is set-up to display info-messages, they are send to
* the logger, otherwise they are printed to stdout.
*
* It is important to note that these messages should always be displayed to
* the user, so unless run in debug mode, all launcher settings should ensure
* that info-level logs are printed to the console output.
* Currently, the message is always printed to standard output. But this may be changed by changing this method.
*/
def info(msg: => String): Unit = {
if (logger.underlying.isInfoEnabled) {
logger.info(msg)
} else {
CLIOutput.println(msg)
}
CLIOutput.println(msg)
}
}

View File

@ -33,6 +33,9 @@ object LauncherRepository {
private val launcherFallbackProviderHostname =
"launcherfallback.release.enso.org"
/** URL to the repo that could be displayed to the user. */
def websiteUrl: String = "https://github.com/enso-org/enso"
/** Defines a part of the URL scheme of the fallback mechanism - the name of
* the directory that holds the releases.
*

View File

@ -29,8 +29,7 @@ class LauncherReleaseProvider(releaseProvider: SimpleReleaseProvider)
.find(_.fileName == LauncherManifest.assetName)
.toRight(
ReleaseProviderException(
s"${LauncherManifest.assetName} file is missing from release " +
s"assets."
s"${LauncherManifest.assetName} file is missing from $tag release assets."
)
)
.toTry

View File

@ -1,6 +1,6 @@
package org.enso.launcher.upgrade
import java.nio.file.{Files, Path}
import java.nio.file.{AccessDeniedException, Files, Path}
import com.typesafe.scalalogging.Logger
import org.enso.semver.SemVer
import org.enso.semver.SemVerOrdering._
@ -24,7 +24,7 @@ import org.enso.launcher.cli.{
import org.enso.launcher.releases.launcher.LauncherRelease
import org.enso.runtimeversionmanager.releases.ReleaseProvider
import org.enso.launcher.releases.LauncherRepository
import org.enso.launcher.InfoLogger
import org.enso.launcher.{Constants, InfoLogger}
import org.enso.launcher.distribution.DefaultManagers
import org.enso.runtimeversionmanager.locking.Resources
import org.slf4j.LoggerFactory
@ -97,12 +97,45 @@ class LauncherUpgrader(
}
}
}
if (release.canPerformUpgradeFromCurrentVersion)
val canPerformDirectUpgrade: Boolean =
if (release.canPerformUpgradeFromCurrentVersion) true
else if (CurrentVersion.isDevVersion) {
logger.warn(
s"Cannot upgrade to version ${release.version} directly, because " +
s"it requires at least version " +
s"${release.minimumVersionToPerformUpgrade}."
)
if (globalCLIOptions.autoConfirm) {
logger.warn(
s"However, the current version (${CurrentVersion.version}) is " +
s"a development version, so the minimum version check can be " +
s"ignored. Since `auto-confirm` is set, the upgrade will " +
s"continue. But please be warned that it may fail due to " +
s"incompatibility."
)
true
} else {
logger.warn(
s"Since the current version (${CurrentVersion.version}) is " +
s"a development version, the minimum version check can be " +
s"ignored. However, please be warned that the upgrade " +
s"may fail due to incompatibility."
)
CLIOutput.askConfirmation(
"Do you want to continue upgrading to this version " +
"despite the warning?"
)
}
} else false
if (canPerformDirectUpgrade)
performUpgradeTo(release)
else
performStepByStepUpgrade(release)
runCleanup()
logger.debug("Upgrade completed successfully.")
}
}
@ -120,11 +153,13 @@ class LauncherUpgrader(
val temporaryFiles =
FileSystem.listDirectory(binRoot).filter(isTemporaryExecutable)
if (temporaryFiles.nonEmpty && isStartup) {
logger.debug("Cleaning temporary files from a previous upgrade.")
logger.debug(
s"Cleaning ${temporaryFiles.size} temporary files from a previous upgrade."
)
}
for (file <- temporaryFiles) {
try {
Files.delete(file)
tryHardToDelete(file)
logger.debug(s"Upgrade cleanup: removed `$file`.")
} catch {
case NonFatal(e) =>
@ -133,6 +168,21 @@ class LauncherUpgrader(
}
}
/** On Windows, deleting an executable immediately after it has exited may fail
* and the process may need to wait a few millisecond. This method detects
* this kind of failure and retries a few times.
*/
private def tryHardToDelete(file: Path, attempts: Int = 30): Unit = {
try {
Files.delete(file)
} catch {
case _: AccessDeniedException if attempts > 0 =>
logger.trace(s"Failed to delete file `$file`. Retrying.")
Thread.sleep(100)
tryHardToDelete(file, attempts - 1)
}
}
/** Continues a multi-step upgrade.
*
* Called by [[InternalOpts]] when the upgrade continuation is requested by
@ -217,28 +267,53 @@ class LauncherUpgrader(
@scala.annotation.tailrec
private def nextVersionToUpgradeTo(
release: LauncherRelease,
currentTargetRelease: LauncherRelease,
availableVersions: Seq[SemVer]
): LauncherRelease = {
val recentEnoughVersions =
availableVersions.filter(
_.isGreaterThanOrEqual(release.minimumVersionToPerformUpgrade)
assert(
currentTargetRelease.minimumVersionToPerformUpgrade.isGreaterThan(
CurrentVersion.version
)
)
// We look at older versions that are satisfying the minimum version
// required to upgrade to currentTargetRelease.
val recentEnoughVersions =
availableVersions.filter { possibleVersion =>
val canUpgradeToTarget = possibleVersion.isGreaterThanOrEqual(
currentTargetRelease.minimumVersionToPerformUpgrade
)
val isEarlierThanTarget =
possibleVersion.isLessThan(currentTargetRelease.version)
canUpgradeToTarget && isEarlierThanTarget
}
// We take the oldest of these, hoping that it will yield the shortest
// upgrade path (perhaps it will be possible to upgrade directly from
// current version)
val minimumValidVersion = recentEnoughVersions.sorted.headOption.getOrElse {
throw UpgradeError(
s"Upgrade failed: To continue upgrade, a version at least " +
s"${release.minimumVersionToPerformUpgrade} is required, but no " +
s"valid version satisfying this requirement could be found."
s"Upgrade failed: To continue upgrade, at least version " +
s"${currentTargetRelease.minimumVersionToPerformUpgrade} is required, " +
s"but no upgrade path has been found from the current version " +
s"${CurrentVersion.version}. " +
s"Please manually download a newer release from " +
s"${LauncherRepository.websiteUrl}"
)
}
val nextRelease = releaseProvider.fetchRelease(minimumValidVersion).get
val newTargetRelease = releaseProvider.fetchRelease(minimumValidVersion).get
assert(newTargetRelease.version != currentTargetRelease.version)
logger.debug(
s"To upgrade to ${release.version}, " +
s"the launcher will have to upgrade to ${nextRelease.version} first."
s"To upgrade to ${currentTargetRelease.version}, " +
s"the launcher will have to upgrade to ${newTargetRelease.version} first."
)
if (nextRelease.canPerformUpgradeFromCurrentVersion)
nextRelease
else nextVersionToUpgradeTo(nextRelease, availableVersions)
// If the current version cannot upgrade directly to the new target version,
// we continue the search looking for an even earlier version that we could
// upgrade to.
if (newTargetRelease.canPerformUpgradeFromCurrentVersion) newTargetRelease
else nextVersionToUpgradeTo(newTargetRelease, availableVersions)
}
/** Extracts just the launcher executable from the archive.
@ -331,7 +406,7 @@ class LauncherUpgrader(
val temporaryExecutable = temporaryExecutablePath("new")
FileSystem.copyFile(
extractedRoot / "bin" / OS.executableName("enso"),
extractedRoot / "bin" / OS.executableName(Constants.name),
temporaryExecutable
)

View File

@ -58,7 +58,7 @@ trait NativeTest
args: Seq[String],
extraEnv: Map[String, String] = Map.empty,
extraJVMProps: Map[String, String] = Map.empty,
timeoutSeconds: Long = 15
timeoutSeconds: Long = defaultTimeoutSeconds
): RunResult = {
if (extraEnv.contains("PATH")) {
throw new IllegalArgumentException(
@ -89,7 +89,7 @@ trait NativeTest
args: Seq[String],
extraEnv: Map[String, String] = Map.empty,
extraJVMProps: Map[String, String] = Map.empty,
timeoutSeconds: Long = 15
timeoutSeconds: Long = defaultTimeoutSeconds
): RunResult = {
if (extraEnv.contains("PATH")) {
throw new IllegalArgumentException(
@ -146,7 +146,7 @@ trait NativeTest
args: Seq[String],
pathOverride: String,
extraJVMProps: Map[String, String] = Map.empty,
timeoutSeconds: Long = 15
timeoutSeconds: Long = defaultTimeoutSeconds
): RunResult = {
runCommand(
Seq(baseLauncherLocation.toAbsolutePath.toString) ++ args,
@ -155,6 +155,8 @@ trait NativeTest
timeoutSeconds = timeoutSeconds
)
}
private val defaultTimeoutSeconds: Long = 30
}
object NativeTest {

View File

@ -1,24 +1,23 @@
package org.enso.launcher.upgrade
import java.nio.file.{Files, Path, StandardCopyOption}
import io.circe.parser
import org.enso.semver.SemVer
import org.enso.distribution.FileSystem
import org.enso.distribution.locking.{FileLockManager, LockType}
import FileSystem.PathSyntax
import org.enso.cli.OS
import org.enso.distribution.FileSystem
import org.enso.distribution.FileSystem.PathSyntax
import org.enso.distribution.locking.{FileLockManager, LockType}
import org.enso.launcher._
import org.enso.testkit.{FlakySpec, WithTemporaryDirectory}
import org.enso.process.{RunResult, WrappedProcess}
import org.enso.version.BuildVersion
import org.enso.runtimeversionmanager.releases.testing.FakeAsset
import org.enso.semver.SemVer
import org.enso.testkit.{FlakySpec, WithTemporaryDirectory}
import org.scalatest.exceptions.TestFailedException
import org.scalatest.{BeforeAndAfterAll, Ignore, OptionValues}
import org.scalatest.{BeforeAndAfterAll, OptionValues}
import java.nio.file.{Files, Path, StandardCopyOption}
import scala.concurrent.TimeoutException
// TODO [DB] The suite became quite flaky and frequently fails the
// Windows CI. Disabled until #3183 is implemented.
@Ignore
class UpgradeSpec
extends NativeTest
with WithTemporaryDirectory
@ -32,13 +31,14 @@ class UpgradeSpec
/** Location of built Rust artifacts.
*/
private val rustBuildRoot = Path.of("./target/rust/debug/")
private val rustBuildRoot =
Path.of("../../target/rust/debug/").toAbsolutePath.normalize()
/** Location of the actual launcher executable that is wrapped by the shims.
*/
private val realLauncherLocation =
Path
.of(".")
.of("../../")
.resolve(OS.executableName(Constants.name))
.toAbsolutePath
.normalize
@ -70,10 +70,13 @@ class UpgradeSpec
override def beforeAll(): Unit = {
super.beforeAll()
prepareLauncherBinary(SemVer.of(0, 0, 0))
prepareLauncherBinary(SemVer.of(0, 0, 1))
prepareLauncherBinary(SemVer.of(0, 0, 2))
prepareLauncherBinary(SemVer.of(0, 0, 3))
prepareLauncherBinary(SemVer.of(0, 0, 4))
prepareLauncherBinary(SemVer.of(1, 0, 1))
prepareLauncherBinary(SemVer.of(1, 0, 2))
prepareLauncherBinary(SemVer.of(1, 0, 3))
prepareLauncherBinary(SemVer.of(1, 0, 4))
// The 99.9999.0 version is marked as broken so it should not be considered for upgrades.
prepareLauncherBinary(SemVer.of(99, 9999, 0))
}
/** Prepares a launcher distribution in the temporary test location.
@ -88,7 +91,7 @@ class UpgradeSpec
*/
private def prepareDistribution(
portable: Boolean,
launcherVersion: Option[SemVer] = None
launcherVersion: Option[SemVer]
): Unit = {
val sourceLauncherLocation =
launcherVersion.map(builtLauncherBinary).getOrElse(baseLauncherLocation)
@ -172,41 +175,41 @@ class UpgradeSpec
"upgrade to latest version (excluding broken)" taggedAs Flaky in {
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer.of(0, 0, 2))
launcherVersion = Some(SemVer.of(1, 0, 2))
)
run(Seq("upgrade")) should returnSuccess
checkVersion() shouldEqual SemVer.of(0, 0, 4)
checkVersion() shouldEqual SemVer.of(1, 0, 4)
}
"not downgrade without being explicitly asked to do so" taggedAs Flaky in {
// precondition for the test to make sense
SemVer
.parse(BuildVersion.ensoVersion)
.get
.isGreaterThan(SemVer.of(0, 0, 4)) shouldBe true
prepareDistribution(
portable = true
portable = false,
launcherVersion = Some(SemVer.of(99, 9999, 0))
)
run(Seq("upgrade")).exitCode shouldEqual 1
// precondition for the test to make sense
checkVersion().isGreaterThan(SemVer.of(1, 0, 4)) shouldBe true
val result = run(Seq("upgrade"))
withClue(result) {
result.exitCode shouldEqual 1
result.stdout should include("If you really want to downgrade")
}
}
"upgrade/downgrade to a specific version " +
"(and update necessary files)" taggedAs Flaky in {
// precondition for the test to make sense
SemVer
.parse(BuildVersion.ensoVersion)
.get
.isGreaterThan(SemVer.of(0, 0, 4)) shouldBe true
prepareDistribution(
portable = true
portable = true,
launcherVersion = Some(SemVer.of(99, 9999, 0))
)
// precondition for the test to make sense
checkVersion().isGreaterThan(SemVer.of(1, 0, 4)) shouldBe true
val root = launcherPath.getParent.getParent
FileSystem.writeTextFile(root / "README.md", "Old readme")
run(Seq("upgrade", "0.0.1")) should returnSuccess
checkVersion() shouldEqual SemVer.of(0, 0, 1)
run(Seq("upgrade", "1.0.2")) should returnSuccess
checkVersion() shouldEqual SemVer.of(1, 0, 2)
TestHelpers.readFileContent(root / "README.md").trim shouldEqual "Content"
TestHelpers
.readFileContent(root / "THIRD-PARTY" / "test-license.txt")
@ -216,19 +219,21 @@ class UpgradeSpec
"upgrade also in installed mode" taggedAs Flaky in {
prepareDistribution(
portable = false,
launcherVersion = Some(SemVer.of(0, 0, 0))
launcherVersion = Some(SemVer.of(1, 0, 1))
)
val dataRoot = getTestDirectory / "data"
val configRoot = getTestDirectory / "config"
checkVersion() shouldEqual SemVer.of(0, 0, 0)
checkVersion() shouldEqual SemVer.of(1, 0, 1)
val env = Map(
"ENSO_DATA_DIRECTORY" -> dataRoot.toString,
"ENSO_CONFIG_DIRECTORY" -> configRoot.toString,
"ENSO_RUNTIME_DIRECTORY" -> (getTestDirectory / "run").toString
)
run(Seq("upgrade", "0.0.1"), extraEnv = env) should returnSuccess
checkVersion() shouldEqual SemVer.of(0, 0, 1)
run(Seq("upgrade", "1.0.2"), extraEnv = env) should returnSuccess
checkVersion() shouldEqual SemVer.of(1, 0, 2)
// Make sure that files were added
TestHelpers
.readFileContent(dataRoot / "README.md")
.trim shouldEqual "Content"
@ -238,27 +243,25 @@ class UpgradeSpec
}
"perform a multi-step upgrade if necessary" taggedAs Flaky in {
// 0.0.3 can only be upgraded from 0.0.2 which can only be upgraded from
// 0.0.1, so the upgrade path should be following:
// 0.0.0 -> 0.0.1 -> 0.0.2 -> 0.0.3
// 1.0.4 can only be upgraded from 1.0.2 which can only be upgraded from
// 1.0.1, so the upgrade path should be following: 1.0.1 -> 1.0.2 -> 1.0.4
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer.of(0, 0, 0))
launcherVersion = Some(SemVer.of(1, 0, 1))
)
checkVersion() shouldEqual SemVer.of(0, 0, 0)
val process = startLauncher(Seq("upgrade", "0.0.3"))
checkVersion() shouldEqual SemVer.of(1, 0, 1)
val process = startLauncher(Seq("upgrade", "1.0.4"))
try {
process.join(timeoutSeconds = 30) should returnSuccess
process.join(timeoutSeconds = 60) should returnSuccess
checkVersion() shouldEqual SemVer.of(0, 0, 3)
checkVersion() shouldEqual SemVer.of(1, 0, 4)
val launchedVersions = Seq(
"0.0.0",
"0.0.0",
"0.0.1",
"0.0.2",
"0.0.3"
"1.0.1",
"1.0.1",
"1.0.2",
"1.0.4"
)
val reportedLaunchLog = TestHelpers
@ -294,7 +297,7 @@ class UpgradeSpec
"that action with the upgraded launcher" ignore {
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer.of(0, 0, 2))
launcherVersion = Some(SemVer.of(1, 0, 2))
)
val enginesPath = getTestDirectory / "enso" / "dist"
Files.createDirectories(enginesPath)
@ -303,7 +306,7 @@ class UpgradeSpec
// engine distribution can be used in the test
// FileSystem.copyDirectory(
// Path.of("target/distribution/"),
// enginesPath / "0.1.0"
// enginesPath / "1.0.2"
// )
val script = getTestDirectory / "script.enso"
val message = "Hello from test"
@ -321,17 +324,20 @@ class UpgradeSpec
script.toAbsolutePath.toString,
"--use-system-jvm",
"--use-enso-version",
"0.1.0"
"1.0.2"
)
)
result should returnSuccess
result.stdout should include(message)
withClue(result) {
result should returnSuccess
result.stdout should include(message)
}
}
"fail if another upgrade is running in parallel" taggedAs Flaky in {
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer.of(0, 0, 1))
launcherVersion = Some(SemVer.of(1, 0, 1))
)
val syncLocker = new FileLockManager(getTestDirectory / "enso" / "lock")
@ -341,14 +347,14 @@ class UpgradeSpec
// so acquiring this exclusive lock will stall access to that file until
// the exclusive lock is released
val lock = syncLocker.acquireLock(
"testasset-" + launcherManifestAssetName,
FakeAsset.lockNameForAsset(launcherManifestAssetName),
LockType.Exclusive
)
val firstSuspended = startLauncher(
Seq(
"upgrade",
"0.0.2",
"1.0.2",
"--internal-emulate-repository-wait",
"--launcher-log-level=trace"
)
@ -361,7 +367,7 @@ class UpgradeSpec
val secondFailed = run(Seq("upgrade", "0.0.0"))
secondFailed.stderr should include("Another upgrade is in progress")
secondFailed.stdout should include("Another upgrade is in progress")
secondFailed.exitCode shouldEqual 1
} catch {
case e: TimeoutException =>
@ -376,8 +382,37 @@ class UpgradeSpec
lock.release()
}
firstSuspended.join(timeoutSeconds = 20) should returnSuccess
checkVersion() shouldEqual SemVer.of(0, 0, 2)
firstSuspended.join(timeoutSeconds = 60) should returnSuccess
checkVersion() shouldEqual SemVer.of(1, 0, 2)
}
"should return a useful error message if upgrade cannot be performed because no upgrade path exists" taggedAs Flaky in {
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer.of(0, 0, 0))
)
val result = run(Seq("upgrade", "1.0.4"))
withClue(result) {
result.exitCode shouldEqual 1
result.stdout should include(
"no upgrade path has been found from the current version 0.0.0."
)
}
}
"should allow to upgrade the development version ignoring the version check, but warn about it" taggedAs Flaky in {
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer.of(0, 0, 0, "dev"))
)
val result = run(Seq("upgrade", "1.0.4"))
result should returnSuccess
withClue(result) {
result.stdout should include("development version")
result.stdout should include("minimum version check can be ignored")
}
}
}
}

View File

@ -0,0 +1,12 @@
use launcher_shims::wrap_launcher;
// ===========================
// === EntryPoint0.0.0-dev ===
// ===========================
/// Runs the launcher wrapper overriding the version to 0.0.0-dev.
fn main() {
wrap_launcher("0.0.0-dev")
}

View File

@ -1,12 +0,0 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint0.0.1 ===
// =======================
/// Runs the launcher wrapper overriding the version to 0.0.1.
fn main() {
wrap_launcher("0.0.1")
}

View File

@ -1,12 +0,0 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint0.0.2 ===
// =======================
/// Runs the launcher wrapper overriding the version to 0.0.2.
fn main() {
wrap_launcher("0.0.2")
}

View File

@ -1,12 +0,0 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint0.0.3 ===
// =======================
/// Runs the launcher wrapper overriding the version to 0.0.3.
fn main() {
wrap_launcher("0.0.3")
}

View File

@ -1,12 +0,0 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint0.0.4 ===
// =======================
/// Runs the launcher wrapper overriding the version to 0.0.4.
fn main() {
wrap_launcher("0.0.4")
}

View File

@ -0,0 +1,12 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint1.0.1 ===
// =======================
/// Runs the launcher wrapper overriding the version to 1.0.1.
fn main() {
wrap_launcher("1.0.1")
}

View File

@ -0,0 +1,12 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint1.0.2 ===
// =======================
/// Runs the launcher wrapper overriding the version to 1.0.2.
fn main() {
wrap_launcher("1.0.2")
}

View File

@ -0,0 +1,12 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint1.0.3 ===
// =======================
/// Runs the launcher wrapper overriding the version to 1.0.3.
fn main() {
wrap_launcher("1.0.3")
}

View File

@ -0,0 +1,12 @@
use launcher_shims::wrap_launcher;
// =======================
// === EntryPoint1.0.4 ===
// =======================
/// Runs the launcher wrapper overriding the version to 1.0.4.
fn main() {
wrap_launcher("1.0.4")
}

View File

@ -0,0 +1,12 @@
use launcher_shims::wrap_launcher;
// ===========================
// === EntryPoint99.9999.0 ===
// ===========================
/// Runs the launcher wrapper overriding the version to 99.9999.0.
fn main() {
wrap_launcher("99.9999.0")
}

View File

@ -74,7 +74,7 @@ pub fn wrap_launcher(version: impl AsRef<str>) {
/// Appends a line to the file located at the provided path.
pub fn append_to_log(path: PathBuf, line: impl AsRef<str>) -> io::Result<()> {
let mut log_file = OpenOptions::new().append(true).open(path)?;
let mut log_file = OpenOptions::new().create(true).append(true).open(path)?;
writeln!(log_file, "{}", line.as_ref())?;
Ok(())
}

View File

@ -6,7 +6,7 @@ import scala.util.Try
*
* Used internally by [[TaskProgress.flatMap]].
*/
private class MappedTask[A, B](source: TaskProgress[A], f: A => Try[B])
private class MappedTask[A, B](source: TaskProgress[A], f: Try[A] => Try[B])
extends TaskProgress[B] { self =>
var listeners: List[ProgressListener[B]] = Nil
@ -17,7 +17,7 @@ private class MappedTask[A, B](source: TaskProgress[A], f: A => Try[B])
listeners.foreach(_.progressUpdate(done, total))
override def done(result: Try[A]): Unit = self.synchronized {
val mapped = result.flatMap(f)
val mapped = f(result)
savedResult = Some(mapped)
listeners.foreach(_.done(mapped))
}

View File

@ -50,7 +50,19 @@ trait TaskProgress[A] {
* succeeded and the transformation succeeded too
*/
def flatMap[B](f: A => Try[B]): TaskProgress[B] =
new MappedTask(this, f)
new MappedTask(this, (t: Try[A]) => t.flatMap(f))
/** Alters the task by transforming its failure case with a partial function
* `pf`.
*
* @param pf the partial function that can transform the original error
* @tparam B resulting type of `pf`
* @return a new [[TaskProgress]] with update failure behaviours
*/
def recoverWith[B >: A](
pf: PartialFunction[Throwable, Try[B]]
): TaskProgress[B] =
new MappedTask(this, (t: Try[A]) => t.recoverWith(pf))
/** Alters the task by transforming its result with a function `f`.
*

View File

@ -44,7 +44,7 @@ object HTTPDownload {
sizeHint: Option[Long] = None,
encoding: Charset = StandardCharsets.UTF_8
): TaskProgress[APIResponse] = {
logger.debug("Fetching [{}].", request.requestImpl.uri())
logger.debug("Fetching [{} {}].", request.method, request.uri)
val taskProgress =
new TaskProgressImplementation[APIResponse](ProgressUnit.Bytes)
val total: java.lang.Long = if (sizeHint.isDefined) sizeHint.get else null

View File

@ -5,4 +5,11 @@ import java.net.http.HttpRequest
/** Wraps an underlying HTTP request implementation to make the outside API
* independent of the internal implementation.
*/
case class HTTPRequest(requestImpl: HttpRequest)
case class HTTPRequest(requestImpl: HttpRequest) {
/** Returns the method of this request. */
def method: String = requestImpl.method()
/** Returns the URI of this request as string, used e.g. for logging. */
def uri: String = requestImpl.uri().toString
}

View File

@ -0,0 +1,30 @@
package org.enso.runtimeversionmanager.releases.testing;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
class CompressZipArchive {
static void compress(Path source, Path destination) throws IOException {
try (var zip = Files.newOutputStream(destination);
var zos = new ZipOutputStream(zip);
var files = Files.walk(source)) {
files
.filter(path -> !Files.isDirectory(path))
.forEach(
path -> {
var zipEntry = new ZipEntry(source.relativize(path).toString());
try {
zos.putNextEntry(zipEntry);
Files.copy(path, zos);
zos.closeEntry();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
}
}
}

View File

@ -1,5 +1,7 @@
package org.enso.runtimeversionmanager.releases
import org.enso.downloader.http.ResourceNotFound
/** Indicates a release provider failure. */
sealed class ReleaseProviderException(message: String, cause: Throwable)
extends RuntimeException(message, cause) {
@ -30,6 +32,20 @@ case class ReleaseNotFound(
message: Option[String] = None,
cause: Throwable = null
) extends ReleaseProviderException(
message.getOrElse(s"Cannot find release `$tag`."),
ReleaseNotFound.constructMessage(tag, message, cause),
cause
)
object ReleaseNotFound {
private def constructMessage(
tag: String,
message: Option[String],
cause: Throwable
): String =
message.getOrElse {
val isCauseInteresting = cause != null && cause != ResourceNotFound()
val suffix =
if (isCauseInteresting) s" Caused by: ${cause.getMessage}" else ""
s"Cannot find release `$tag`.$suffix"
}
}

View File

@ -1,5 +1,7 @@
package org.enso.runtimeversionmanager.releases.github
import com.typesafe.scalalogging.Logger
import java.nio.file.Path
import io.circe._
import io.circe.parser._
@ -7,6 +9,7 @@ import org.enso.cli.task.TaskProgress
import org.enso.downloader.http.{
APIResponse,
HTTPDownload,
HTTPException,
HTTPRequestBuilder,
Header,
URIBuilder
@ -16,7 +19,7 @@ import org.enso.runtimeversionmanager.releases.{
ReleaseProviderException
}
import scala.util.{Success, Try}
import scala.util.{Failure, Success, Try}
/** Contains functions used to query the GitHubAPI endpoints.
*/
@ -61,13 +64,31 @@ object GithubAPI {
* bar for this task.
*/
def listReleases(repository: Repository): Try[Seq[Release]] = {
releaseListCache.get(repository) match {
case Some(cachedList) =>
logger.debug("Using cached release list for repository {}", repository)
Success(cachedList)
case None =>
makeListReleasesRequest(repository).map { releases =>
releaseListCache.put(repository, releases)
releases
}
}
}
private def makeListReleasesRequest(
repository: Repository
): Try[Seq[Release]] = {
val perPage = 100
def listPage(page: Int): Try[Seq[Release]] = {
val uri = (projectURI(repository) / "releases") ?
("per_page" -> perPage.toString) ? ("page" -> page.toString)
val uri =
((projectURI(
repository
) / "releases") ? ("per_page" -> perPage.toString) ? ("page" -> page.toString))
.build()
val downloadTask = HTTPDownload
.fetchString(HTTPRequestBuilder.fromURI(uri.build()).GET)
.fetchString(HTTPRequestBuilder.fromURI(uri).GET)
.flatMap(response =>
parse(response.content)
.flatMap(
@ -95,13 +116,47 @@ object GithubAPI {
listAllPages(1)
}
/** Fetching the list of releases may involve making multiple requests with big payloads, so it is good to cache the relevant information extracted from it to avoid unnecessary re-fetching.
*
* The runtime version manager assumes that the list of known releases will not change during the runtime of the program, so the cache is never invalidated as long as the program is running.
*/
private val releaseListCache
: collection.concurrent.TrieMap[Repository, Seq[Release]] =
collection.concurrent.TrieMap.empty
/** Fetches release metadata for the release associated with the given tag.
*/
def getRelease(repo: Repository, tag: String): TaskProgress[Release] = {
val uri = projectURI(repo) / "releases" / "tags" / tag
findCachedRelease(repo, tag) match {
case Some(release) =>
logger.debug(
"Using cached release for tag {} in repository {}",
tag,
repo
)
TaskProgress.runImmediately(release)
case None =>
makeGetSingleReleaseRequest(repo, tag)
}
}
private def findCachedRelease(
repo: Repository,
tag: String
): Option[Release] =
for {
cachedList <- releaseListCache.get(repo)
release <- cachedList.find(_.tag == tag)
} yield release
private def makeGetSingleReleaseRequest(
repo: Repository,
tag: String
): TaskProgress[Release] = {
val uri = (projectURI(repo) / "releases" / "tags" / tag).build()
HTTPDownload
.fetchString(HTTPRequestBuilder.fromURI(uri.build()).GET)
.fetchString(HTTPRequestBuilder.fromURI(uri).GET)
.flatMap(response =>
parse(response.content)
.flatMap(_.as[Release])
@ -114,6 +169,9 @@ object GithubAPI {
)
.toTry
)
.recoverWith { case httpException: HTTPException =>
Failure(ReleaseNotFound(tag, cause = httpException))
}
}
/** A helper function that detecte a rate-limit error and tries to make a more
@ -165,6 +223,7 @@ object GithubAPI {
.addHeader("Accept", "application/octet-stream")
.GET
System.err.println(s"Preparing to download asset ${asset.url}")
HTTPDownload
.download(request, destination, Some(asset.size))
.map(_ => ())
@ -189,4 +248,6 @@ object GithubAPI {
assets <- json.get[Seq[Asset]]("assets")
} yield Release(tag, assets)
}
private val logger: Logger = Logger[GithubAPI.type]
}

View File

@ -118,7 +118,7 @@ case class FakeAsset(
*/
private def maybeWaitForAsset(): Unit =
lockManagerForAssets.foreach { lockManager =>
val name = "testasset-" + fileName
val name = FakeAsset.lockNameForAsset(fileName)
val lockType = LockType.Shared
val lock = lockManager.tryAcquireLock(
name,
@ -186,3 +186,9 @@ case class FakeAsset(
}
}
}
object FakeAsset {
/** Name for the lock used for synchronizing access to asset, used when instrumenting tests. */
def lockNameForAsset(assetName: String): String = s"testasset-$assetName"
}

View File

@ -53,22 +53,6 @@ object TestArchivePackager {
}
private def packZip(source: Path, destination: Path): Unit = {
val files = FileSystem.listDirectory(source)
val exitCode = Process(
Seq(
"powershell",
"Compress-Archive",
"-Path",
files.map(_.getFileName.toString).mkString(","),
"-DestinationPath",
destination.toAbsolutePath.toString
),
source.toFile
).!
if (exitCode != 0) {
throw new RuntimeException(
s"tar failed. Cannot create fake-archive for $source"
)
}
CompressZipArchive.compress(source, destination)
}
}

View File

@ -97,7 +97,7 @@ object NativeImage {
): Def.Initialize[Task[Unit]] = Def
.task {
val log = state.value.log
val targetLoc = artifactFile(targetDir, name, false)
val targetLoc = artifactFile(targetDir, name, withExtension = false)
def nativeImagePath(prefix: Path)(path: Path): Path = {
val base = path.resolve(prefix)
@ -253,7 +253,7 @@ object NativeImage {
s"Started building $targetLoc native image. The output is captured."
)
val retCode = process.!(processLogger)
val targetFile = artifactFile(targetDir, name, true)
val targetFile = artifactFile(targetDir, name)
if (retCode != 0 || !targetFile.exists()) {
log.error(s"Native Image build of $targetFile failed, with output: ")
println(sb.toString())
@ -318,7 +318,7 @@ object NativeImage {
def artifactFile(
targetDir: File,
name: String,
withExtension: Boolean = false
withExtension: Boolean = true
): File = {
val artifactName =
if (withExtension && Platform.isWindows) name + ".exe"