enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-09-11 21:27:15 +03:00
Code Issues Projects Releases Wiki Activity

Add Locks to the Launcher (#1147)

Browse Source 
Adds file-based locks that synchronize access and modification of
components by various launcher instances.
This commit is contained in:
Radosław Waśko 2020-09-18 17:37:22 +02:00 committed by GitHub
parent 6fe54c5034
commit 5cd977e904
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 2586 additions and 502 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.sbt
View File

@ -1053,7 +1053,6 @@ lazy val launcher = project
assemblyOutputPath in assembly := file("launcher.jar")
)
.settings(
parallelExecution in Test := false,
(Test / test) := (Test / test)
.dependsOn(
NativeImage.incrementalNativeImageBuild(
@ -1064,7 +1063,8 @@ lazy val launcher = project
.dependsOn(
LauncherShimsForTest.prepare(rustcVersion = rustVersion)
)
.value
.value,
parallelExecution in Test := false
)
.settings(licenseSettings)
.dependsOn(cli)

143
engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala
View File

@ -13,7 +13,12 @@ import org.enso.launcher.components.runner.{
WhichEngine
}
import org.enso.launcher.config.{DefaultVersion, GlobalConfigurationManager}
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.installation.DistributionInstaller.BundleAction
import org.enso.launcher.installation.{
DistributionInstaller,
DistributionManager,
DistributionUninstaller
}
import org.enso.launcher.project.ProjectManager
import org.enso.launcher.upgrade.LauncherUpgrader
import org.enso.version.{VersionDescription, VersionDescriptionParameter}
@ -25,7 +30,7 @@ import org.enso.version.{VersionDescription, VersionDescriptionParameter}
* @param cliOptions the global CLI options to use for the commands
*/
case class Launcher(cliOptions: GlobalCLIOptions) {
private lazy val componentsManager = ComponentsManager.makeDefault(cliOptions)
private lazy val componentsManager = ComponentsManager.default(cliOptions)
private lazy val configurationManager =
new GlobalConfigurationManager(componentsManager, DistributionManager)
private lazy val projectManager = new ProjectManager(configurationManager)
@ -36,7 +41,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
componentsManager,
Environment
)
private lazy val upgrader = LauncherUpgrader.makeDefault(cliOptions)
private lazy val upgrader = LauncherUpgrader.default(cliOptions)
upgrader.runCleanup(isStartup = true)
/**
@ -56,14 +61,14 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
useSystemJVM: Boolean,
jvmOpts: Seq[(String, String)],
additionalArguments: Seq[String]
): Unit = {
): Int = {
val actualPath = path.getOrElse(Launcher.workingDirectory.resolve(name))
val version =
versionOverride.getOrElse(configurationManager.defaultVersion)
val globalConfig = configurationManager.getConfig
val exitCode = runner
.createCommand(
.withCommand(
runner
.newProject(
path = actualPath,
@ -75,32 +80,34 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
)
.get,
JVMSettings(useSystemJVM, jvmOpts)
)
.run()
.get
) { command =>
command.run().get
}
if (exitCode == 0) {
Logger.info(s"Project created in `$actualPath` using version $version.")
} else {
Logger.error("Project creation failed.")
sys.exit(exitCode)
}
exitCode
}
/**
* Prints a list of installed engines.
*/
def listEngines(): Unit = {
def listEngines(): Int = {
for (engine <- componentsManager.listInstalledEngines()) {
val broken = if (engine.isMarkedBroken) " (broken)" else ""
println(engine.version.toString + broken)
}
0
}
/**
* Prints a list of installed runtimes.
*/
def listRuntimes(): Unit = {
def listRuntimes(): Int = {
for (runtime <- componentsManager.listInstalledRuntimes()) {
val engines = componentsManager.findEnginesUsingRuntime(runtime)
val usedBy = {
@ -111,12 +118,13 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
}
println(s"$runtime $usedBy")
}
0
}
/**
* Prints a summary of installed components and their dependencies.
*/
def listSummary(): Unit = {
def listSummary(): Int = {
for (engine <- componentsManager.listInstalledEngines()) {
val runtime = componentsManager.findRuntime(engine)
val runtimeName = runtime
@ -125,6 +133,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
val broken = if (engine.isMarkedBroken) " (broken)" else ""
println(s"Enso ${engine.version}$broken -> $runtimeName")
}
0
}
/**
@ -132,13 +141,14 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
*
* Also installs the required runtime if it wasn't already installed.
*/
def installEngine(version: SemVer): Unit = {
def installEngine(version: SemVer): Int = {
val existing = componentsManager.findEngine(version)
if (existing.isDefined) {
Logger.info(s"Engine $version is already installed.")
} else {
componentsManager.findOrInstallEngine(version, complain = false)
}
0
}
/**
@ -146,7 +156,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
*
* Also installs the required runtime if it wasn't already installed.
*/
def installLatestEngine(): Unit = {
def installLatestEngine(): Int = {
val latest = componentsManager.fetchLatestEngineVersion()
Logger.info(s"Installing Enso engine $latest.")
installEngine(latest)
@ -157,8 +167,11 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
*
* If a runtime is not used by any engines anymore, it is also removed.
*/
def uninstallEngine(version: SemVer): Unit =
def uninstallEngine(version: SemVer): Int = {
componentsManager.uninstallEngine(version)
DistributionManager.tryCleaningUnusedLockfiles()
0
}
/**
* Runs the Enso REPL.
@ -174,6 +187,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
* instead of the JVM associated with the engine version
* @param jvmOpts additional options to pass to the launched JVM
* @param additionalArguments additional arguments to pass to the runner
* @return exit code of the launched program
*/
def runRepl(
projectPath: Option[Path],
@ -181,15 +195,15 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
useSystemJVM: Boolean,
jvmOpts: Seq[(String, String)],
additionalArguments: Seq[String]
): Unit = {
): Int = {
val exitCode = runner
.createCommand(
.withCommand(
runner.repl(projectPath, versionOverride, additionalArguments).get,
JVMSettings(useSystemJVM, jvmOpts)
)
.run()
.get
sys.exit(exitCode)
) { command =>
command.run().get
}
exitCode
}
/**
@ -208,6 +222,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
* instead of the JVM associated with the engine version
* @param jvmOpts additional options to pass to the launched JVM
* @param additionalArguments additional arguments to pass to the runner
* @return exit code of the launched program
*/
def runRun(
path: Option[Path],
@ -215,15 +230,15 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
useSystemJVM: Boolean,
jvmOpts: Seq[(String, String)],
additionalArguments: Seq[String]
): Unit = {
): Int = {
val exitCode = runner
.createCommand(
.withCommand(
runner.run(path, versionOverride, additionalArguments).get,
JVMSettings(useSystemJVM, jvmOpts)
)
.run()
.get
sys.exit(exitCode)
) { command =>
command.run().get
}
exitCode
}
/**
@ -239,6 +254,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
* instead of the JVM associated with the engine version
* @param jvmOpts additional options to pass to the launched JVM
* @param additionalArguments additional arguments to pass to the runner
* @return exit code of the launched program
*/
def runLanguageServer(
options: LanguageServerOptions,
@ -246,17 +262,17 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
useSystemJVM: Boolean,
jvmOpts: Seq[(String, String)],
additionalArguments: Seq[String]
): Unit = {
): Int = {
val exitCode = runner
.createCommand(
.withCommand(
runner
.languageServer(options, versionOverride, additionalArguments)
.get,
JVMSettings(useSystemJVM, jvmOpts)
)
.run()
.get
sys.exit(exitCode)
) { command =>
command.run().get
}
exitCode
}
/**
@ -267,7 +283,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
* config. Any updates that set a known key to an invalid value which would
* prevent from loading the config are cancelled.
*/
def updateConfig(key: String, value: String): Unit = {
def updateConfig(key: String, value: String): Int = {
if (value.isEmpty) {
configurationManager.removeFromConfig(key)
Logger.info(
@ -281,6 +297,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
s"(${configurationManager.configLocation.toAbsolutePath})."
)
}
0
}
/**
@ -289,26 +306,26 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
* If the `key` is not set in the config, sets exit code to 1 and prints a
* warning.
*/
def printConfig(key: String): Unit = {
def printConfig(key: String): Int = {
configurationManager.getConfig.original.apply(key) match {
case Some(value) =>
println(value)
sys.exit()
0
case None =>
Logger.warn(s"Key $key is not set in the global config.")
sys.exit(1)
1
}
}
/**
* Sets the default Enso version.
*/
def setDefaultVersion(version: DefaultVersion): Unit = {
def setDefaultVersion(defaultVersion: DefaultVersion): Int = {
configurationManager.updateConfig { config =>
config.copy(defaultVersion = version)
config.copy(defaultVersion = defaultVersion)
}
version match {
defaultVersion match {
case DefaultVersion.LatestInstalled =>
Logger.info(
s"Default Enso version set to the latest installed version, " +
@ -317,13 +334,41 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
case DefaultVersion.Exact(version) =>
Logger.info(s"Default Enso version set to $version.")
}
0
}
/**
* Prints the default Enso version.
*/
def printDefaultVersion(): Unit = {
def printDefaultVersion(): Int = {
println(configurationManager.defaultVersion)
0
}
/**
* Installs the Enso distribution.
*/
def installDistribution(
doNotRemoveOldLauncher: Boolean,
bundleAction: Option[BundleAction]
): Int = {
DistributionInstaller
.default(
globalCLIOptions = cliOptions,
removeOldLauncher = !doNotRemoveOldLauncher,
bundleActionOption = bundleAction
)
.install()
0
}
/**
* Uninstalls the Enso distribution.
*/
def uninstallDistribution(): Int = {
DistributionUninstaller.default(cliOptions).uninstall()
0
}
/**
@ -335,7 +380,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
*/
def displayVersion(
hideEngineVersion: Boolean = false
): Unit = {
): Int = {
val useJSON = cliOptions.useJSON
val runtimeVersionParameter =
if (hideEngineVersion) None else Some(getEngineVersion(useJSON))
@ -349,6 +394,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
)
println(versionDescription.asString(useJSON))
0
}
private def getEngineVersion(
@ -359,12 +405,13 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
val isEngineInstalled =
componentsManager.findEngine(runtimeVersionRunSettings.version).isDefined
val runtimeVersionString = if (isEngineInstalled) {
val runtimeVersionCommand = runner.createCommand(
val output = runner.withCommand(
runtimeVersionRunSettings,
JVMSettings(useSystemJVM = false, jvmOptions = Seq.empty)
)
) { runtimeVersionCommand =>
runtimeVersionCommand.captureOutput().get
}
val output = runtimeVersionCommand.captureOutput().get
if (useJSON) output else "\n" + output.stripTrailing()
} else {
if (useJSON) "null"
@ -390,11 +437,12 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
* specified, the latest available version is chosen, unless it is older than
* the current one.
*/
def upgrade(version: Option[SemVer]): Unit = {
def upgrade(version: Option[SemVer]): Int = {
val targetVersion = version.getOrElse(upgrader.latestVersion().get)
val isManuallyRequested = version.isDefined
if (targetVersion == CurrentVersion.version) {
Logger.info("Already up-to-date.")
0
} else if (targetVersion < CurrentVersion.version && !isManuallyRequested) {
Logger.warn(
s"The latest available version is $targetVersion, but you are " +
@ -404,9 +452,10 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
s"If you really want to downgrade, please run " +
s"`enso upgrade $targetVersion`."
)
sys.exit(1)
1
} else {
upgrader.upgrade(targetVersion)
0
}
}
}

56
engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala
View File

@ -70,11 +70,13 @@ object InternalOpts {
private val CONTINUE_UPGRADE = "internal-continue-upgrade"
private val UPGRADE_ORIGINAL_PATH = "internal-upgrade-original-path"
private val EMULATE_VERSION = "internal-emulate-version"
private val EMULATE_LOCATION = "internal-emulate-location"
private val EMULATE_REPOSITORY = "internal-emulate-repository"
private val EMULATE_VERSION = "internal-emulate-version"
private val EMULATE_LOCATION = "internal-emulate-location"
private val EMULATE_REPOSITORY = "internal-emulate-repository"
private val EMULATE_REPOSITORY_WAIT = "internal-emulate-repository-wait"
private var inheritEmulateRepository: Option[Path] = None
private var inheritShouldWaitForAssets: Boolean = false
/**
* Removes internal testing options that should not be preserved in the called executable.
@ -180,7 +182,7 @@ object InternalOpts {
continueUpgrade.foreach { version =>
LauncherUpgrader
.makeDefault(config, originalExecutablePath = originalPath)
.default(config, originalExecutablePath = originalPath)
.internalContinueUpgrade(version)
sys.exit(0)
}
@ -201,9 +203,11 @@ object InternalOpts {
Opts.optionalParameter[Path](EMULATE_LOCATION, "PATH", "").hidden
val emulateRepository =
Opts.optionalParameter[Path](EMULATE_REPOSITORY, "PATH", "").hidden
val waitForAssets =
Opts.flag(EMULATE_REPOSITORY_WAIT, "", showInUsage = false).hidden
(emulateVersion, emulateLocation, emulateRepository) mapN {
(emulateVersion, emulateLocation, emulateRepository) =>
(emulateVersion, emulateLocation, emulateRepository, waitForAssets) mapN {
(emulateVersion, emulateLocation, emulateRepository, waitForAssets) =>
emulateVersion.foreach { version =>
CurrentVersion.internalOverrideVersion(version)
}
@ -212,20 +216,36 @@ object InternalOpts {
Environment.internalOverrideExecutableLocation(location)
}
if (waitForAssets) {
inheritShouldWaitForAssets = true
}
emulateRepository.foreach { repositoryPath =>
inheritEmulateRepository = Some(repositoryPath)
EnsoRepository.internalUseFakeRepository(repositoryPath)
EnsoRepository.internalUseFakeRepository(
repositoryPath,
waitForAssets
)
}
}
}
private def optionsToInherit: Seq[String] =
inheritEmulateRepository
/**
* Specifies options that are inherited by the process that is launched when
* continuing the upgrade.
*/
private def optionsToInherit: Seq[String] = {
val repositoryPath = inheritEmulateRepository
.map { path =>
Seq(s"--$EMULATE_REPOSITORY", path.toAbsolutePath.toString)
}
.getOrElse(Seq())
val waitForAssets =
if (inheritShouldWaitForAssets) Seq(s"--$EMULATE_REPOSITORY_WAIT")
else Seq()
repositoryPath ++ waitForAssets
}
/**
* Returns a helper class that allows to run the launcher located at the
@ -245,7 +265,10 @@ object InternalOpts {
* executable.
*
* It retries for a few seconds to give the process running the old
* launcher to terminate and release the lock on its file.
* launcher to terminate and release the lock on its file. It overrides the
* ENSO_RUNTIME_DIRECTORY for the launched executable to the temporary
* directory it resides in, so that its main lock does not block removing
* the original directory.
*/
def removeOldExecutableAndExit(oldExecutablePath: Path): Nothing = {
val command = Seq(
@ -253,7 +276,12 @@ object InternalOpts {
s"--$REMOVE_OLD_EXECUTABLE",
oldExecutablePath.toAbsolutePath.toString
)
runDetachedAndExit(command)
val temporaryRuntimeDirectory =
pathToNewLauncher.getParent.toAbsolutePath.normalize
runDetachedAndExit(
command,
"ENSO_RUNTIME_DIRECTORY" -> temporaryRuntimeDirectory.toString
)
}
/**
@ -375,7 +403,10 @@ object InternalOpts {
pb.start().waitFor()
}
private def runDetachedAndExit(command: Seq[String]): Nothing = {
private def runDetachedAndExit(
command: Seq[String],
extraEnv: (String, String)*
): Nothing = {
if (!OS.isWindows) {
throw new IllegalStateException(
"Internal error: Detached process workarounds are only available on " +
@ -384,6 +415,7 @@ object InternalOpts {
}
val pb = new java.lang.ProcessBuilder(command: _*)
extraEnv.foreach(envPair => pb.environment().put(envPair._1, envPair._2))
pb.inheritIO()
pb.start()
sys.exit()

122
engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala
View File

@ -16,17 +16,21 @@ import org.enso.launcher.config.DefaultVersion
import org.enso.launcher.installation.DistributionInstaller.BundleAction
import org.enso.launcher.installation.{
DistributionInstaller,
DistributionManager,
DistributionUninstaller
DistributionManager
}
import org.enso.launcher.locking.DefaultResourceManager
/**
* Defines the CLI commands and options for the program.
*
* Each command is parametrized with a config that describes global CLI options
* set at the top-level and returns an integer which determines the programs
* exit code.
*/
object LauncherApplication {
type Config = GlobalCLIOptions
private def versionCommand: Command[Config => Unit] =
private def versionCommand: Command[Config => Int] =
Command(
"version",
"Print version of the launcher and currently selected Enso distribution."
@ -45,7 +49,7 @@ object LauncherApplication {
}
}
private def newCommand: Command[Config => Unit] =
private def newCommand: Command[Config => Int] =
Command("new", "Create a new Enso project.", related = Seq("create")) {
val nameOpt = Opts.positionalArgument[String]("PROJECT-NAME")
val pathOpt = Opts.optionalArgument[Path](
@ -102,7 +106,7 @@ object LauncherApplication {
"Override the Enso version that would normally be used."
)
private def runCommand: Command[Config => Unit] =
private def runCommand: Command[Config => Int] =
Command(
"run",
"Run an Enso project or script. " +
@ -138,7 +142,7 @@ object LauncherApplication {
}
}
private def languageServerCommand: Command[Config => Unit] =
private def languageServerCommand: Command[Config => Int] =
Command(
"language-server",
"Launch the Language Server for a given project. " +
@ -207,7 +211,7 @@ object LauncherApplication {
}
}
private def replCommand: Command[Config => Unit] =
private def replCommand: Command[Config => Int] =
Command(
"repl",
"Launch an Enso REPL. " +
@ -235,7 +239,7 @@ object LauncherApplication {
}
}
private def defaultCommand: Command[Config => Unit] =
private def defaultCommand: Command[Config => Int] =
Command("default", "Print or change the default Enso version.") {
val version = Opts.optionalArgument[DefaultVersion](
"VERSION",
@ -254,7 +258,7 @@ object LauncherApplication {
}
}
private def upgradeCommand: Command[Config => Unit] =
private def upgradeCommand: Command[Config => Int] =
Command("upgrade", "Upgrade the launcher.") {
val version = Opts.optionalArgument[SemVer](
"VERSION",
@ -266,7 +270,7 @@ object LauncherApplication {
}
}
private def installEngineCommand: Command[Config => Unit] =
private def installEngineCommand: Command[Config => Int] =
Command(
"engine",
"Install the specified engine VERSION, defaulting to the latest if " +
@ -274,6 +278,7 @@ object LauncherApplication {
) {
val version = Opts.optionalArgument[SemVer]("VERSION")
version map { version => (config: Config) =>
DistributionManager.tryCleaningTemporaryDirectory()
version match {
case Some(value) =>
Launcher(config).installEngine(value)
@ -283,7 +288,7 @@ object LauncherApplication {
}
}
private def installDistributionCommand: Command[Config => Unit] =
private def installDistributionCommand: Command[Config => Int] =
Command(
"distribution",
"Install Enso on the system, deactivating portable mode."
@ -311,25 +316,21 @@ object LauncherApplication {
"no-remove-old-launcher",
"If `auto-confirm` is set, the default behavior is to remove the old " +
"launcher after installing the distribution. Setting this flag may " +
"override this behavior to keep the original launcher.",
"override this behavior to keep the original launcher. Applies only " +
"if `auto-confirm` is set.",
showInUsage = true
)
(bundleAction, doNotRemoveOldLauncher) mapN {
(bundleAction, doNotRemoveOldLauncher) => (config: Config) =>
new DistributionInstaller(
DistributionManager,
config.autoConfirm,
removeOldLauncher = !doNotRemoveOldLauncher,
bundleActionOption =
if (config.autoConfirm)
Some(bundleAction.getOrElse(DistributionInstaller.MoveBundles))
else bundleAction
).install()
Launcher(config).installDistribution(
doNotRemoveOldLauncher,
bundleAction
)
}
}
private def installCommand: Command[Config => Unit] =
private def installCommand: Command[Config => Int] =
Command(
"install",
"Install a new version of engine or install the distribution locally."
@ -337,7 +338,7 @@ object LauncherApplication {
Opts.subcommands(installEngineCommand, installDistributionCommand)
}
private def uninstallEngineCommand: Command[Config => Unit] =
private def uninstallEngineCommand: Command[Config => Int] =
Command(
"engine",
"Uninstall the provided engine version. If the corresponding runtime " +
@ -349,7 +350,7 @@ object LauncherApplication {
}
}
private def uninstallDistributionCommand: Command[Config => Unit] =
private def uninstallDistributionCommand: Command[Config => Int] =
Command(
"distribution",
"Uninstall whole Enso distribution and all components managed by " +
@ -358,14 +359,12 @@ object LauncherApplication {
"unexpected files."
) {
Opts.pure(()) map { (_: Unit) => (config: Config) =>
new DistributionUninstaller(
DistributionManager,
autoConfirm = config.autoConfirm
).uninstall()
DistributionManager.tryCleaningTemporaryDirectory()
Launcher(config).uninstallDistribution()
}
}
private def uninstallCommand: Command[Config => Unit] =
private def uninstallCommand: Command[Config => Int] =
Command(
"uninstall",
"Uninstall an Enso component."
@ -373,7 +372,7 @@ object LauncherApplication {
Opts.subcommands(uninstallEngineCommand, uninstallDistributionCommand)
}
private def listCommand: Command[Config => Unit] =
private def listCommand: Command[Config => Int] =
Command("list", "List installed components.") {
sealed trait Components
case object EnsoComponents extends Components
@ -402,7 +401,7 @@ object LauncherApplication {
}
}
private def configCommand: Command[Config => Unit] =
private def configCommand: Command[Config => Int] =
Command("config", "Modify global user configuration.") {
val key = Opts.positionalArgument[String](
"KEY",
@ -423,9 +422,9 @@ object LauncherApplication {
}
}
private def helpCommand: Command[Config => Unit] =
private def helpCommand: Command[Config => Int] =
Command("help", "Display summary of available commands.") {
Opts.pure(()) map { _ => (_: Config) => printTopLevelHelp() }
Opts.pure(()) map { _ => (_: Config) => printTopLevelHelp(); 0 }
}
private def topLevelOpts: Opts[() => TopLevelBehavior[Config]] = {
@ -483,35 +482,48 @@ object LauncherApplication {
)
internalOptsCallback(globalCLIOptions)
initializeApp()
if (version) {
Launcher(globalCLIOptions).displayVersion(useJSON)
TopLevelBehavior.Halt
TopLevelBehavior.Halt(0)
} else
TopLevelBehavior.Continue(globalCLIOptions)
}
}
/**
* Application initializer that is run after handling of the internal
* options.
*/
private def initializeApp(): Unit = {
// Note [Main Lock Initialization]
DefaultResourceManager.initializeMainLock()
}
val commands: NonEmptyList[Command[Config => Int]] = NonEmptyList
.of(
versionCommand,
helpCommand,
newCommand,
replCommand,
runCommand,
languageServerCommand,
defaultCommand,
installCommand,
uninstallCommand,
upgradeCommand,
listCommand,
configCommand
)
val application: Application[Config] =
Application(
"enso",
"Enso",
"Enso Launcher",
topLevelOpts,
NonEmptyList.of(
versionCommand,
helpCommand,
newCommand,
replCommand,
runCommand,
languageServerCommand,
defaultCommand,
installCommand,
uninstallCommand,
upgradeCommand,
listCommand,
configCommand
),
commands,
PluginManager
)
@ -519,3 +531,17 @@ object LauncherApplication {
CLIOutput.println(application.renderHelp())
}
}
/* Note [Main Lock Initialization]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* The main program lock is used by the distribution installer/uninstaller to
* ensure that no other launcher instances are running when the distribution is
* being installed or uninstalled.
*
* That lock should be acquired (in shared mode) as soon as possible, but it
* must be acquired *after* handling the internal options. That is because,
* acquiring any locks will initialize the DistributionManager's paths, but in
* test-mode, the internal options may need to override the
* DistributionManager's executable path and that must be done before their
* initialization for it to take effect.
*/

6
engine/launcher/src/main/scala/org/enso/launcher/cli/Main.scala
View File

@ -2,6 +2,7 @@ package org.enso.launcher.cli
import org.enso.cli.CLIOutput
import org.enso.launcher.Logger
import org.enso.launcher.locking.DefaultResourceManager
import org.enso.launcher.upgrade.LauncherUpgrader
/**
@ -19,8 +20,8 @@ object Main {
case Left(errors) =>
CLIOutput.println(errors.mkString("\n"))
1
case Right(()) =>
0
case Right(exitCode) =>
exitCode
}
def main(args: Array[String]): Unit = {
@ -36,6 +37,7 @@ object Main {
1
}
DefaultResourceManager.releaseMainLock()
sys.exit(exitCode)
}
}

6
engine/launcher/src/main/scala/org/enso/launcher/cli/PluginManager.scala
View File

@ -20,15 +20,15 @@ class PluginManager(env: Environment) extends arguments.PluginManager {
*
* @param name name of the plugin
* @param args arguments that should be passed to it
* @return exit code of the launched plugin
*/
override def runPlugin(
name: String,
args: Seq[String]
): Nothing =
): Int =
findPlugin(name) match {
case Some(PluginDescription(commandName, _)) =>
val exitCode = (Seq(commandName) ++ args).!
sys.exit(exitCode)
(Seq(commandName) ++ args).!
case None =>
throw new RuntimeException(
"Internal error: Could not find the plugin. " +

323
engine/launcher/src/main/scala/org/enso/launcher/components/ComponentsManager.scala
View File

@ -8,6 +8,12 @@ import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher.archive.Archive
import org.enso.launcher.cli.GlobalCLIOptions
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.locking.{
DefaultResourceManager,
LockType,
Resource,
ResourceManager
}
import org.enso.launcher.releases.engine.EngineRelease
import org.enso.launcher.releases.runtime.{
GraalCEReleaseProvider,
@ -24,6 +30,8 @@ import scala.util.{Failure, Success, Try, Using}
*
* Allows to find, list, install and uninstall components.
*
* See Note [Components Manager Concurrency Model]
*
* @param cliOptions options from the CLI setting verbosity of the executed
* actions
* @param distributionManager the [[DistributionManager]] to use
@ -33,6 +41,7 @@ import scala.util.{Failure, Success, Try, Using}
class ComponentsManager(
cliOptions: GlobalCLIOptions,
distributionManager: DistributionManager,
resourceManager: ResourceManager,
engineReleaseProvider: ReleaseProvider[EngineRelease],
runtimeReleaseProvider: RuntimeReleaseProvider
) {
@ -76,6 +85,86 @@ class ComponentsManager(
} else None
}
/**
* Executes the provided action with a requested engine version.
*
* The engine is locked with a shared lock, so it is guaranteed that it will
* not be uninstalled while the action is being executed.
*
* The engine will be installed if needed.
*/
def withEngine[R](engineVersion: SemVer)(
action: Engine => R
): R = {
val engine = findOrInstallEngine(version = engineVersion)
resourceManager.withResources(
(Resource.Engine(engineVersion), LockType.Shared)
) {
engine
.ensureValid()
.recoverWith { error =>
Failure(
ComponentMissingError(
"The engine has been removed before the command could be " +
"started.",
error
)
)
}
.get
action(engine)
}
}
/**
* Executes the provided action with a requested engine version and its
* corresponding runtime.
*
* The components are locked with a shared lock, so it is guaranteed that
* they will not be uninstalled while the action is being executed.
*
* The components will be installed if needed.
*/
def withEngineAndRuntime[R](engineVersion: SemVer)(
action: (Engine, Runtime) => R
): R = {
val engine = findOrInstallEngine(version = engineVersion)
val runtime = findOrInstallRuntime(engine)
resourceManager.withResources(
(Resource.Engine(engineVersion), LockType.Shared),
(Resource.Runtime(runtime.version), LockType.Shared)
) {
engine
.ensureValid()
.recoverWith { error =>
Failure(
ComponentMissingError(
"The engine has been removed before the command could be " +
"started.",
error
)
)
}
.get
runtime
.ensureValid()
.recoverWith { error =>
Failure(
ComponentMissingError(
"The runtime has been removed before the command could be " +
"started.",
error
)
)
}
.get
action(engine, runtime)
}
}
/**
* Returns the runtime needed for the given engine, trying to install it if
* it is missing.
@ -86,7 +175,7 @@ class ComponentsManager(
* [[cliOptions.autoConfirm]] is set, in which case it
* installs it without asking)
*/
def findOrInstallRuntime(
private def findOrInstallRuntime(
engine: Engine,
complain: Boolean = true
): Runtime =
@ -104,7 +193,13 @@ class ComponentsManager(
)
}
if (!complain || complainAndAsk()) {
installRuntime(engine.manifest.runtimeVersion)
val version = engine.manifest.runtimeVersion
resourceManager.withResources(
(Resource.AddOrRemoveComponents, LockType.Shared),
(Resource.Runtime(version), LockType.Exclusive)
) {
installRuntime(version)
}
} else {
throw ComponentMissingError(
s"No runtime for engine $engine. Cannot continue."
@ -115,7 +210,7 @@ class ComponentsManager(
/**
* Finds an installed engine with the given `version` and reports any errors.
*/
def getEngine(version: SemVer): Try[Engine] = {
private def getEngine(version: SemVer): Try[Engine] = {
val name = engineNameForVersion(version)
val path = distributionManager.paths.engines / name
if (Files.exists(path)) {
@ -183,7 +278,21 @@ class ComponentsManager(
}
if (!complain || complainAndAsk()) {
installEngine(version)
resourceManager.withResources(
(Resource.AddOrRemoveComponents, LockType.Shared),
(Resource.Engine(version), LockType.Exclusive)
) {
findEngine(version) match {
case Some(engine) =>
Logger.info(
"The engine has already been installed by a different " +
"process."
)
engine
case None =>
installEngine(version)
}
}
} else {
throw ComponentMissingError(s"No engine $version. Cannot continue.")
}
@ -244,16 +353,20 @@ class ComponentsManager(
/**
* Uninstalls the engine with the provided `version` (if it was installed).
*/
def uninstallEngine(version: SemVer): Unit = {
val engine = getEngine(version).getOrElse {
Logger.warn(s"Enso Engine $version is not installed.")
sys.exit(1)
}
def uninstallEngine(version: SemVer): Unit =
resourceManager.withResources(
(Resource.AddOrRemoveComponents, LockType.Exclusive),
(Resource.Engine(version), LockType.Exclusive)
) {
val engine = getEngine(version).getOrElse {
Logger.warn(s"Enso Engine $version is not installed.")
throw ComponentMissingError(s"Enso Engine $version is not installed.")
}
safelyRemoveComponent(engine.path)
Logger.info(s"Uninstalled $engine.")
cleanupRuntimes()
}
safelyRemoveComponent(engine.path)
Logger.info(s"Uninstalled $engine.")
cleanupRuntimes()
}
/**
* Installs the engine with the provided version.
@ -265,6 +378,10 @@ class ComponentsManager(
* package is extracted to a temporary directory next to the `engines`
* directory (to ensure that they are on the same filesystem) and is moved to
* the actual directory after doing simple sanity checks.
*
* The caller should hold a shared lock on [[Resource.AddOrRemoveComponents]]
* and an exclusive lock on [[Resource.Engine]]. The function itself acquires
* [[Resource.Runtime]], but it is released before it returns.
*/
private def installEngine(version: SemVer): Engine = {
val engineRelease = engineReleaseProvider.fetchRelease(version).get
@ -371,24 +488,73 @@ class ComponentsManager(
)
}
findOrInstallRuntime(temporaryEngine, complain = false)
/**
* Finalizes the installation.
*
* Has to be called with an acquired lock for the runtime. If
* `wasJustInstalled` is true, the lock must be exclusive and it the
* runtime may be removed if the installation fails.
*/
def finishInstallation(
runtime: Runtime,
wasJustInstalled: Boolean
): Engine = {
val enginePath =
distributionManager.paths.engines / engineDirectoryName
FileSystem.atomicMove(engineTemporaryPath, enginePath)
val engine = getEngine(version).getOrElse {
Logger.error(
"fatal: Could not load the installed engine." +
"Reverting the installation."
)
FileSystem.removeDirectory(enginePath)
if (wasJustInstalled && findEnginesUsingRuntime(runtime).isEmpty) {
safelyRemoveComponent(runtime.path)
}
val enginePath = distributionManager.paths.engines / engineDirectoryName
FileSystem.atomicMove(engineTemporaryPath, enginePath)
val engine = getEngine(version).getOrElse {
Logger.error(
"fatal: Could not load the installed engine." +
"Reverting the installation."
)
FileSystem.removeDirectory(enginePath)
cleanupRuntimes()
throw InstallationError(
"fatal: Could not load the installed engine"
)
throw InstallationError(
"fatal: Could not load the installed engine"
)
}
Logger.info(s"Installed $engine.")
engine
}
Logger.info(s"Installed $engine.")
engine
val runtimeVersion = temporaryEngine.graalRuntimeVersion
/**
* Tries to finalize the installation assuming that the runtime was
* installed and without acquiring an unnecessary exclusive lock.
*/
def getEngineIfRuntimeIsInstalled: Option[Engine] =
resourceManager.withResource(
Resource.Runtime(runtimeVersion),
LockType.Shared
) {
findRuntime(runtimeVersion).map { runtime =>
finishInstallation(runtime, wasJustInstalled = false)
}
}
/**
* Finalizes the installation, installing the runtime if necessary.
* This variant acquires an exclusive lock on the runtime (but it
* should generally be called only if the runtime was not installed).
*/
def getEngineOtherwise: Engine =
resourceManager.withResource(
Resource.Runtime(runtimeVersion),
LockType.Exclusive
) {
val (runtime, wasJustInstalled) = findRuntime(runtimeVersion)
.map((_, false))
.getOrElse((installRuntime(runtimeVersion), true))
finishInstallation(runtime, wasJustInstalled = wasJustInstalled)
}
getEngineIfRuntimeIsInstalled.getOrElse(getEngineOtherwise)
} catch {
case e: Exception =>
undoTemporaryEngine()
@ -414,13 +580,6 @@ class ComponentsManager(
*/
private def loadGraalRuntime(path: Path): Try[Runtime] = {
val name = path.getFileName.toString
def verifyRuntime(runtime: Runtime): Try[Unit] =
if (runtime.isValid) {
Success(())
} else {
Failure(CorruptedComponentError(s"Runtime $runtime is corrupted."))
}
for {
version <- parseGraalRuntimeVersionString(name)
.toRight(
@ -428,7 +587,7 @@ class ComponentsManager(
)
.toTry
runtime = Runtime(version, path)
_ <- verifyRuntime(runtime)
_ <- runtime.ensureValid()
} yield runtime
}
@ -461,23 +620,13 @@ class ComponentsManager(
/**
* Loads the engine definition.
*/
private def loadEngine(path: Path): Try[Engine] = {
def verifyEngine(engine: Engine): Try[Unit] =
if (!engine.isValid) {
Failure(
CorruptedComponentError(s"Engine ${engine.version} is corrupted.")
)
} else {
Success(())
}
private def loadEngine(path: Path): Try[Engine] =
for {
version <- parseEngineVersion(path)
manifest <- loadAndCheckEngineManifest(path)
engine = Engine(version, path, manifest)
_ <- verifyEngine(engine)
_ <- engine.ensureValid()
} yield engine
}
/**
* Gets the engine version from its path.
@ -513,13 +662,16 @@ class ComponentsManager(
/**
* Installs the runtime with the provided version.
*
* Used internally by [[findOrInstallRuntime]]. Does not check if the runtime
* is already installed.
* Does not check if the runtime is already installed.
*
* The installation tries as much as possible to be robust - the downloaded
* package is extracted to a temporary directory next to the `runtimes`
* directory (to ensure that they are on the same filesystem) and is moved to
* the actual directory after doing simple sanity checks.
*
* The caller should hold a shared lock for
* [[Resource.AddOrRemoveComponents]] and an exclusive lock for
* [[Resource.Runtime]].
*/
private def installRuntime(runtimeVersion: RuntimeVersion): Runtime =
FileSystem.withTemporaryDirectory("enso-install-runtime") { directory =>
@ -588,8 +740,10 @@ class ComponentsManager(
/**
* Removes runtimes that are not used by any installed engines.
*
* The caller must hold [[Resource.AddOrRemoveComponents]] exclusively.
*/
def cleanupRuntimes(): Unit = {
private def cleanupRuntimes(): Unit = {
for (runtime <- listInstalledRuntimes()) {
if (findEnginesUsingRuntime(runtime).isEmpty) {
Logger.info(
@ -612,6 +766,14 @@ class ComponentsManager(
* removed, but it will already be in the temporary directory, so it will be
* unreachable. The temporary directory is cleaned when doing
* installation-related operations.
*
* The caller should hold an exclusive lock for
* [[Resource.AddOrRemoveComponents]] or otherwise guarantee that this
* component can be safely removed. The latter is an exception that only
* happens when uninstalling a just-installed runtime when an engine
* installation failed. In that case the installer has an exclusive lock on
* that runtime and it was installed by it, so nobody else could be using it.
* In all other situations the lock is strictly required.
*/
private def safelyRemoveComponent(path: Path): Unit = {
val temporaryPath =
@ -621,6 +783,62 @@ class ComponentsManager(
}
}
/* Note [Components Manager Concurrency Model]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* The [[ComponentsManager]] itself is not synchronized as it should be used in
* the launcher from a single thread. However, multiple launcher instances may
* run at the same time and their operations have to be synchronized to some
* extent, especially to avoid two processes modifying the same engine version.
*
* The concurrency is managed as follows:
*
* 1. `withEngine` and `withEngineAndRuntime` acquire shared locks for the
* engine (and in the second case also for the runtime) and can be used to
* safely run a process using the engine ensuring that the engine will not be
* uninstalled while that process is running. As the locks are shared,
* multiple processes can use the same engine or runtime for running. These
* functions may install the missing engine or runtime as described below.
* The installation acquires an exclusive lock for the component, but to
* avoid keeping that lock when launching a possibly long-running process, it
* is released and a shared lock is re-acquired, so that other processes can
* start using the newly installed engine in parallel. As the underlying
* mechanism does not support lock downgrade, the lock is released for a very
* short time. Theoretically, it is possible for an uninstall command to
* start in this short time-window and remove the just-installed component.
* To avoid errors, after re-acquiring the shared lock, the engine and
* runtime components are re-checked to ensure that they have not been
* uninstalled in the meantime. In the unlikely case that one of them was
* removed, an error is reported and execution is terminated.
* 2. `uninstallEngine` acquires an exclusive lock on the affected engine (so no
* other process can be using it) and removes it. Additionally it runs a
* runtime cleanup, removing any runtimes that are not used anymore. For that
* cleanup to be safe, add-remove-components lock is acquired which
* guarantees that no other installations are run in parallel.
* 2. `findOrInstallEngine` and `findOrInstallRuntime` do not normally use
* locking, so their result is immediately invalidated - if the caller wants
* to have any guarantees they have to acquire a lock *after* getting the
* result and re-check its validity, as described in (1). In case the
* requested component is missing, an exclusive lock for it is acquired along
* with a shared lock for add-remove-components for the time of the
* installation, but all locks are released before returning from this
* function.
* 3. `installRuntime` does not acquire any locks, it relies on its caller to
* acquire an exclusive lock for add-remove-components and the affected
* runtime.
* 4. `installEngine` relies on the caller to acquire an exclusive lock for
* add-remove-components and the affected engine, but it may itself acquire a
* a shared or exclusive lock for the runtime (the exclusive lock is acquired
* if the runtime is missing and has to be installed).
* 5. Other functions do not acquire or require any locks. Their results are
* immediately invalidated, so they should only be used for non-safety
* critical operations, like listing available engines.
*
* To summarize, components management (install and uninstall) is synchronized
* for safety. Other operations are mostly unsynchronized. If a guarantee is
* expected that the engine (and possibly its runtime) are not modified when
* running a command, `withEngine` or `withEngineAndRuntime` should be used.
*/
object ComponentsManager {
/**
@ -630,10 +848,11 @@ object ComponentsManager {
* @param globalCLIOptions options from the CLI setting verbosity of the
* executed actions
*/
def makeDefault(globalCLIOptions: GlobalCLIOptions): ComponentsManager =
def default(globalCLIOptions: GlobalCLIOptions): ComponentsManager =
new ComponentsManager(
globalCLIOptions,
DistributionManager,
DefaultResourceManager,
EnsoRepository.defaultEngineReleaseProvider,
GraalCEReleaseProvider
)

23
engine/launcher/src/main/scala/org/enso/launcher/components/Engine.scala
View File

@ -5,6 +5,8 @@ import java.nio.file.{Files, Path}
import nl.gn0s1s.bump.SemVer
import org.enso.launcher.FileSystem.PathSyntax
import scala.util.{Failure, Success, Try}
/**
* Represents an engine component.
*
@ -44,10 +46,25 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) {
def runtimePath: Path = path / "component" / "runtime.jar"
/**
* Returns if the installation seems not-corrupted.
* Checks if the installation is not corrupted and reports any issues as
* failures.
*/
def isValid: Boolean =
Files.exists(runnerPath) && Files.exists(runtimePath)
def ensureValid(): Try[Unit] =
if (!Files.exists(runnerPath))
Failure(
CorruptedComponentError(
s"Engine's runner.jar (expected at " +
s"${runnerPath.toAbsolutePath.normalize} is missing."
)
)
else if (!Files.exists(runtimePath))
Failure(
CorruptedComponentError(
s"Engine's runtime.jar (expected at " +
s"${runtimePath.toAbsolutePath.normalize} is missing."
)
)
else Success(())
/**
* Returns if the engine release was marked as broken when it was being

22
engine/launcher/src/main/scala/org/enso/launcher/components/Runtime.scala
View File

@ -5,6 +5,8 @@ import java.nio.file.{Files, Path}
import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher.OS
import scala.util.{Failure, Success, Try}
/**
* Represents a runtime component.
*
@ -38,8 +40,22 @@ case class Runtime(version: RuntimeVersion, path: Path) {
}
/**
* Returns if the installation seems not-corrupted.
* Checks if the installation is not corrupted and reports any issues as
* failures.
*/
def isValid: Boolean =
Files.exists(javaExecutable) && Files.isExecutable(javaExecutable)
def ensureValid(): Try[Unit] =
if (!Files.exists(javaExecutable))
Failure(
CorruptedComponentError(
s"Runtime's java executable (expected at " +
s"${javaExecutable.toAbsolutePath.normalize}) is missing."
)
)
else if (!Files.isExecutable(javaExecutable))
Failure(
CorruptedComponentError(
"Runtime's java executable is not marked as executable."
)
)
else Success(())
}

103
engine/launcher/src/main/scala/org/enso/launcher/components/runner/Runner.scala
View File

@ -3,7 +3,12 @@ package org.enso.launcher.components.runner
import java.nio.file.{Files, Path}
import nl.gn0s1s.bump.SemVer
import org.enso.launcher.components.{ComponentsManager, Manifest, Runtime}
import org.enso.launcher.components.{
ComponentsManager,
Engine,
Manifest,
Runtime
}
import org.enso.launcher.config.GlobalConfigurationManager
import org.enso.launcher.project.ProjectManager
import org.enso.launcher.{Environment, Logger}
@ -208,58 +213,68 @@ class Runner(
final private val JVM_OPTIONS_ENV_VAR = "ENSO_JVM_OPTS"
/**
* Creates a command that can be used to launch the component.
* Runs an action giving it a command that can be used to launch the
* component.
*
* While the action is executed, it is guaranteed that the component
* referenced by the command is available. The command is considered invalid
* after the `action` exits.
*
* Combines the [[RunSettings]] for the runner with the [[JVMSettings]] for
* the underlying JVM to get the full command for launching the component.
*/
def createCommand(
runSettings: RunSettings,
jvmSettings: JVMSettings
): Command = {
val engine = componentsManager.findOrInstallEngine(runSettings.version)
val javaCommand =
if (jvmSettings.useSystemJVM) systemJavaCommand
else {
val runtime = componentsManager.findOrInstallRuntime(engine)
javaCommandForRuntime(runtime)
def withCommand[R](runSettings: RunSettings, jvmSettings: JVMSettings)(
action: Command => R
): R = {
def prepareAndRunCommand(engine: Engine, javaCommand: JavaCommand): R = {
val jvmOptsFromEnvironment = environment.getEnvVar(JVM_OPTIONS_ENV_VAR)
jvmOptsFromEnvironment.foreach { opts =>
Logger.debug(
s"Picking up additional JVM options ($opts) from the " +
s"$JVM_OPTIONS_ENV_VAR environment variable."
)
}
val jvmOptsFromEnvironment = environment.getEnvVar(JVM_OPTIONS_ENV_VAR)
jvmOptsFromEnvironment.foreach { opts =>
Logger.debug(
s"Picking up additional JVM options ($opts) from the " +
s"$JVM_OPTIONS_ENV_VAR environment variable."
)
val runnerJar = engine.runnerPath.toAbsolutePath.normalize.toString
def translateJVMOption(option: (String, String)): String = {
val name = option._1
val value = option._2
s"-D$name=$value"
}
val context = Manifest.JVMOptionsContext(enginePackagePath = engine.path)
val manifestOptions =
engine.defaultJVMOptions.filter(_.isRelevant).map(_.substitute(context))
val environmentOptions =
jvmOptsFromEnvironment.map(_.split(' ').toIndexedSeq).getOrElse(Seq())
val commandLineOptions = jvmSettings.jvmOptions.map(translateJVMOption)
val jvmArguments =
manifestOptions ++ environmentOptions ++ commandLineOptions ++
Seq("-jar", runnerJar)
val command = Seq(javaCommand.executableName) ++
jvmArguments ++ runSettings.runnerArguments
val extraEnvironmentOverrides =
javaCommand.javaHomeOverride.map("JAVA_HOME" -> _).toSeq
action(Command(command, extraEnvironmentOverrides))
}
val runnerJar = engine.runnerPath.toAbsolutePath.normalize.toString
def translateJVMOption(option: (String, String)): String = {
val name = option._1
val value = option._2
s"-D$name=$value"
val engineVersion = runSettings.version
if (jvmSettings.useSystemJVM) {
componentsManager.withEngine(engineVersion) { engine =>
prepareAndRunCommand(engine, systemJavaCommand)
}
} else {
componentsManager.withEngineAndRuntime(engineVersion) {
(engine, runtime) =>
prepareAndRunCommand(engine, javaCommandForRuntime(runtime))
}
}
val context = Manifest.JVMOptionsContext(enginePackagePath = engine.path)
val manifestOptions =
engine.defaultJVMOptions.filter(_.isRelevant).map(_.substitute(context))
val environmentOptions =
jvmOptsFromEnvironment.map(_.split(' ').toIndexedSeq).getOrElse(Seq())
val commandLineOptions = jvmSettings.jvmOptions.map(translateJVMOption)
val jvmArguments =
manifestOptions ++ environmentOptions ++ commandLineOptions ++
Seq("-jar", runnerJar)
val command = Seq(javaCommand.executableName) ++
jvmArguments ++ runSettings.runnerArguments
val extraEnvironmentOverrides =
javaCommand.javaHomeOverride.map("JAVA_HOME" -> _).toSeq
Command(command, extraEnvironmentOverrides)
}
/**

212
engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala
View File

@ -1,11 +1,17 @@
package org.enso.launcher.installation
import java.nio.file.Files
import java.nio.file.{Files, Path}
import org.enso.cli.CLIOutput
import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher.cli.InternalOpts
import org.enso.launcher.cli.{GlobalCLIOptions, InternalOpts}
import org.enso.launcher.config.GlobalConfigurationManager
import org.enso.launcher.installation.DistributionInstaller.{
BundleAction,
IgnoreBundles,
MoveBundles
}
import org.enso.launcher.locking.{DefaultResourceManager, ResourceManager}
import org.enso.launcher.{FileSystem, Logger, OS}
import scala.util.control.NonFatal
@ -27,6 +33,7 @@ import scala.util.control.NonFatal
*/
class DistributionInstaller(
manager: DistributionManager,
resourceManager: ResourceManager,
autoConfirm: Boolean,
removeOldLauncher: Boolean,
bundleActionOption: Option[DistributionInstaller.BundleAction]
@ -56,12 +63,22 @@ class DistributionInstaller(
*/
def install(): Unit = {
try {
prepare()
val settings = prepare()
resourceManager.acquireExclusiveMainLock(waitAction = () => {
Logger.warn(
"No other Enso processes associated with this distribution can be " +
"running during the installation. The installer will wait until " +
"other Enso processes are terminated."
)
})
installBinary()
createDirectoryStructure()
installBundles()
installBundles(settings.bundleAction)
Logger.info("Installation succeeded.")
maybeRemoveInstaller()
if (settings.removeInstaller) {
removeInstaller()
}
} catch {
case NonFatal(e) =>
val message = s"Installation failed with error: $e."
@ -69,9 +86,13 @@ class DistributionInstaller(
CLIOutput.println(message)
sys.exit(1)
}
}
private case class InstallationSettings(
bundleAction: BundleAction,
removeInstaller: Boolean
)
private val currentLauncherPath = env.getPathToRunningExecutable
private val installedLauncherPath = installed.binaryExecutable
@ -82,7 +103,7 @@ class DistributionInstaller(
* proceed (unless [[autoConfirm]] is set, in which case it only reports
* conflicts).
*/
private def prepare(): Unit = {
private def prepare(): InstallationSettings = {
if (installedLauncherPath == currentLauncherPath) {
Logger.error(
"The installation source and destination are the same. Nothing to " +
@ -92,15 +113,13 @@ class DistributionInstaller(
}
if (Files.exists(installed.dataDirectory)) {
Logger.warn(s"${installed.dataDirectory} already exists.")
if (!Files.isDirectory(installed.dataDirectory)) {
Logger.error(
s"${installed.dataDirectory} already exists but is not a " +
s"directory. Please remove it or change the installation " +
s"location by setting `${installed.ENSO_DATA_DIRECTORY}`."
)
sys.exit(1)
}
Logger.error(
s"${installed.dataDirectory} already exists. " +
s"Please uninstall the already existing distribution. " +
s"If no distribution is installed, please remove the directory " +
s"${installed.dataDirectory} before installing."
)
sys.exit(1)
}
if (
@ -159,6 +178,9 @@ class DistributionInstaller(
Logger.info(message)
additionalMessages.foreach(msg => Logger.warn(msg))
val removeInstaller = decideIfInstallerShouldBeRemoved()
val bundleAction = decideBundleAction()
if (!autoConfirm) {
CLIOutput.println(message + additionalMessages.map("\n" + _).mkString)
val proceed = CLIOutput.askConfirmation(
@ -170,6 +192,8 @@ class DistributionInstaller(
sys.exit(1)
}
}
InstallationSettings(bundleAction, removeInstaller)
}
/**
@ -255,34 +279,70 @@ class DistributionInstaller(
}
}
/**
* Finds bundles included in the portable package.
*
* @return a tuple containing sequences of runtime and engine bundles
*/
private def findBundles(): (Seq[Path], Seq[Path]) = {
val runtimes =
if (runtimesDirectory != manager.paths.runtimes)
FileSystem.listDirectory(manager.paths.runtimes)
else Seq()
val engines =
if (enginesDirectory != manager.paths.engines)
FileSystem.listDirectory(manager.paths.engines)
else Seq()
(runtimes, engines)
}
/**
* Checks if any bundles are available and depending on selected settings,
* decides how to proceed with the bundles.
*
* May ask the user interactively, unless this is prohibited.
*/
private def decideBundleAction(): BundleAction =
if (manager.isRunningPortable) {
val (runtimes, engines) = findBundles()
if (runtimes.length + engines.length > 0)
bundleActionOption.getOrElse {
if (autoConfirm) MoveBundles
else
CLIOutput.askQuestion(
"Found engine/runtime components bundled with the installation. " +
"How do you want to proceed?",
Seq(
DistributionInstaller.MoveBundles,
DistributionInstaller.CopyBundles,
DistributionInstaller.IgnoreBundles
)
)
}
else IgnoreBundles
} else {
bundleActionOption match {
case Some(value) if value != DistributionInstaller.IgnoreBundles =>
Logger.warn(
s"Installer was asked to ${value.description}, but it seems to " +
s"not be running from a bundle package."
)
case None =>
}
IgnoreBundles
}
/**
* Copies (and possibly removes the originals) bundled engine and runtime
* components.
*/
private def installBundles(): Unit = {
private def installBundles(bundleAction: BundleAction): Unit = {
if (manager.isRunningPortable) {
val runtimes =
if (runtimesDirectory != manager.paths.runtimes)
FileSystem.listDirectory(manager.paths.runtimes)
else Seq()
val engines =
if (enginesDirectory != manager.paths.engines)
FileSystem.listDirectory(manager.paths.engines)
else Seq()
val (runtimes, engines) = findBundles()
if (runtimes.length + engines.length > 0) {
val bundleAction = bundleActionOption.getOrElse {
CLIOutput.askQuestion(
"Found engine/runtime components bundled with the installation. " +
"How do you want to proceed?",
Seq(
DistributionInstaller.MoveBundles,
DistributionInstaller.CopyBundles,
DistributionInstaller.IgnoreBundles
)
)
}
if (bundleAction.copy) {
for (engine <- engines) {
Logger.info(s"Copying bundled Enso engine ${engine.getFileName}.")
@ -309,44 +369,42 @@ class DistributionInstaller(
)
}
}
} else {
bundleActionOption match {
case Some(value) if value != DistributionInstaller.IgnoreBundles =>
Logger.warn(
s"Installer was asked to ${value.description}, but it seems to " +
s"not be running from a bundle package."
)
case None =>
}
} else if (bundleAction != IgnoreBundles) {
throw new IllegalStateException(
s"Internal error: The runner is not run in portable mode, " +
s"but the final bundle action was not Ignore, but $bundleAction."
)
}
}
/**
* Decides if the installer should be removed if the installation succeeds.
*/
private def decideIfInstallerShouldBeRemoved(): Boolean = {
def askForRemoval(): Boolean =
CLIOutput.askConfirmation(
s"Do you want to remove the original launcher after " +
s"the installation? (It will not be needed anymore, as the launcher " +
s"will be copied to `${installed.binaryExecutable}`)",
yesDefault = true
)
if (installedLauncherPath != currentLauncherPath) {
if (autoConfirm) removeOldLauncher
else askForRemoval()
} else false
}
/**
* If the user wants to, removes the installer.
*/
private def maybeRemoveInstaller(): Nothing = {
def askForRemoval(): Boolean =
CLIOutput.askConfirmation(
s"Do you want to remove the original launcher that was used for " +
s"installation? (It is not needed anymore, as the launcher has been " +
s"copied to `${installed.binaryExecutable}`)",
yesDefault = true
)
if (installedLauncherPath != currentLauncherPath) {
def shouldRemove(): Boolean =
if (autoConfirm) removeOldLauncher
else askForRemoval()
if (shouldRemove()) {
if (OS.isWindows) {
InternalOpts
.runWithNewLauncher(installedLauncherPath)
.removeOldExecutableAndExit(currentLauncherPath)
} else {
Files.delete(currentLauncherPath)
}
}
private def removeInstaller(): Nothing = {
if (OS.isWindows) {
InternalOpts
.runWithNewLauncher(installedLauncherPath)
.removeOldExecutableAndExit(currentLauncherPath)
} else {
Files.delete(currentLauncherPath)
}
sys.exit()
}
@ -355,6 +413,22 @@ class DistributionInstaller(
object DistributionInstaller {
/**
* Creates a [[DistributionInstaller]] using the default managers.
*/
def default(
globalCLIOptions: GlobalCLIOptions,
removeOldLauncher: Boolean,
bundleActionOption: Option[BundleAction]
): DistributionInstaller =
new DistributionInstaller(
DistributionManager,
DefaultResourceManager,
globalCLIOptions.autoConfirm,
removeOldLauncher = removeOldLauncher,
bundleActionOption = bundleActionOption
)
/**
* Defines the set of possible actions to take when installing the bundled
* components.

116
engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionManager.scala
View File

@ -3,9 +3,11 @@ package org.enso.launcher.installation
import java.nio.file.{Files, Path}
import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher.locking.{DefaultResourceManager, ResourceManager}
import org.enso.launcher.{Environment, FileSystem, Logger, OS}
import scala.util.Try
import scala.util.control.NonFatal
/**
* Gathers filesystem paths used by the launcher.
@ -17,6 +19,8 @@ import scala.util.Try
* @param engines location of engine versions, corresponding to `dist`
* directory
* @param config location of configuration
* @param locks a directory for storing lockfiles that are used to synchronize
* access to the various components
* @param tmp a directory for storing temporary files that is located on the
* same filesystem as `runtimes` and `engines`, used during
* installation to decrease the possibility of getting a broken
@ -25,13 +29,17 @@ import scala.util.Try
* requested for the first time) and is removed if the application
* exits normally (as long as it is empty, but normal termination of
* the installation process should ensure that).
* @param resourceManager reference to the resource manager used for
* synchronizing access to the temporary files
*/
case class DistributionPaths(
dataRoot: Path,
runtimes: Path,
engines: Path,
config: Path,
private val tmp: Path
locks: Path,
tmp: Path,
resourceManager: ResourceManager
) {
/**
@ -43,31 +51,24 @@ case class DistributionPaths(
| runtimes = $runtimes,
| engines = $engines,
| config = $config,
| locks = $locks,
| tmp = $tmp
|)""".stripMargin
lazy val temporaryDirectory: Path = {
runCleanup()
resourceManager.startUsingTemporaryDirectory()
tmp
}
private def runCleanup(): Unit = {
if (Files.exists(tmp)) {
if (!FileSystem.isDirectoryEmpty(tmp)) {
Logger.info("Cleaning up temporary files from a previous installation.")
}
FileSystem.removeDirectory(tmp)
Files.createDirectories(tmp)
FileSystem.removeEmptyDirectoryOnExit(tmp)
}
}
}
/**
* A helper class that detects if a portable or installed distribution is run
* and encapsulates management of paths to components of the distribution.
*/
class DistributionManager(val env: Environment) {
class DistributionManager(
val env: Environment,
resourceManager: ResourceManager
) {
/**
* Specifies whether the launcher has been run as a portable distribution or
@ -108,7 +109,6 @@ class DistributionManager(val env: Environment) {
lazy val paths: DistributionPaths = {
val paths = detectPaths()
Logger.debug(s"Detected paths are: $paths")
paths
}
@ -117,7 +117,8 @@ class DistributionManager(val env: Environment) {
val RUNTIMES_DIRECTORY = "runtime"
val CONFIG_DIRECTORY = "config"
val BIN_DIRECTORY = "bin"
private val TMP_DIRECTORY = "tmp"
val LOCK_DIRECTORY = "lock"
val TMP_DIRECTORY = "tmp"
private def detectPortable(): Boolean = Files.exists(portableMarkFilePath)
private def possiblePortableRoot: Path =
@ -134,20 +135,65 @@ class DistributionManager(val env: Environment) {
runtimes = root / RUNTIMES_DIRECTORY,
engines = root / ENGINES_DIRECTORY,
config = root / CONFIG_DIRECTORY,
tmp = root / TMP_DIRECTORY
locks = root / LOCK_DIRECTORY,
tmp = root / TMP_DIRECTORY,
resourceManager
)
} else {
val dataRoot = LocallyInstalledDirectories.dataDirectory
val configRoot = LocallyInstalledDirectories.configDirectory
val runRoot = LocallyInstalledDirectories.runtimeDirectory
DistributionPaths(
dataRoot = dataRoot,
runtimes = dataRoot / RUNTIMES_DIRECTORY,
engines = dataRoot / ENGINES_DIRECTORY,
config = configRoot,
tmp = dataRoot / TMP_DIRECTORY
locks = runRoot / LOCK_DIRECTORY,
tmp = dataRoot / TMP_DIRECTORY,
resourceManager
)
}
/**
* Tries to clean the temporary files directory.
*
* It should be run at startup whenever the program wants to run clean-up.
* Currently it is run when installation-related operations are taking place.
* It may not proceed if another process is using it. It has to be run before
* the first access to the temporaryDirectory, as after that the directory is
* marked as in-use and will not be cleaned.
*/
def tryCleaningTemporaryDirectory(): Unit = {
val tmp = paths.tmp
if (Files.exists(tmp)) {
resourceManager.tryWithExclusiveTemporaryDirectory {
if (!FileSystem.isDirectoryEmpty(tmp)) {
Logger.info(
"Cleaning up temporary files from a previous installation."
)
}
FileSystem.removeDirectory(tmp)
Files.createDirectories(tmp)
FileSystem.removeEmptyDirectoryOnExit(tmp)
}
}
}
/**
* Removes unused lockfiles.
*/
def tryCleaningUnusedLockfiles(): Unit = {
val lockfiles = FileSystem.listDirectory(paths.locks)
for (lockfile <- lockfiles) {
try {
Files.delete(lockfile)
Logger.debug(s"Removed unused lockfile ${lockfile.getFileName}.")
} catch {
case NonFatal(_) =>
}
}
}
/**
* A helper for managing directories of the non-portable installation.
*
@ -156,13 +202,15 @@ class DistributionManager(val env: Environment) {
* to determine destination for installed files.
*/
object LocallyInstalledDirectories {
val ENSO_DATA_DIRECTORY = "ENSO_DATA_DIRECTORY"
val ENSO_CONFIG_DIRECTORY = "ENSO_CONFIG_DIRECTORY"
val ENSO_BIN_DIRECTORY = "ENSO_BIN_DIRECTORY"
val ENSO_DATA_DIRECTORY = "ENSO_DATA_DIRECTORY"
val ENSO_CONFIG_DIRECTORY = "ENSO_CONFIG_DIRECTORY"
val ENSO_BIN_DIRECTORY = "ENSO_BIN_DIRECTORY"
val ENSO_RUNTIME_DIRECTORY = "ENSO_RUNTIME_DIRECTORY"
private val XDG_DATA_DIRECTORY = "XDG_DATA_DIRECTORY"
private val XDG_CONFIG_DIRECTORY = "XDG_CONFIG_DIRECTORY"
private val XDG_BIN_DIRECTORY = "XDG_BIN_DIRECTORY"
private val XDG_DATA_DIRECTORY = "XDG_DATA_HOME"
private val XDG_CONFIG_DIRECTORY = "XDG_CONFIG_HOME"
private val XDG_BIN_DIRECTORY = "XDG_BIN_HOME"
private val XDG_RUN_DIRECTORY = "XDG_RUNTIME_DIR"
private val LINUX_ENSO_DIRECTORY = "enso"
private val MACOS_ENSO_DIRECTORY = "org.enso"
@ -237,6 +285,23 @@ class DistributionManager(val env: Environment) {
}
.toAbsolutePath
/**
* The directory where runtime-synchronization files are stored.
*/
def runtimeDirectory: Path =
env
.getEnvPath(ENSO_RUNTIME_DIRECTORY)
.getOrElse {
OS.operatingSystem match {
case OS.Linux =>
env
.getEnvPath(XDG_RUN_DIRECTORY)
.map(_ / LINUX_ENSO_DIRECTORY)
.getOrElse(dataDirectory)
case _ => dataDirectory
}
}
private def executableName: String =
OS.executableName("enso")
@ -270,4 +335,5 @@ class DistributionManager(val env: Environment) {
/**
* A default DistributionManager using the default environment.
*/
object DistributionManager extends DistributionManager(Environment)
object DistributionManager
extends DistributionManager(Environment, DefaultResourceManager)

82
engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala
View File

@ -5,10 +5,13 @@ import java.nio.file.{Files, Path}
import org.apache.commons.io.FileUtils
import org.enso.cli.CLIOutput
import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher.cli.InternalOpts
import org.enso.launcher.cli.{GlobalCLIOptions, InternalOpts}
import org.enso.launcher.config.GlobalConfigurationManager
import org.enso.launcher.locking.{DefaultResourceManager, ResourceManager}
import org.enso.launcher.{FileSystem, Logger, OS}
import scala.util.control.NonFatal
/**
* Allows to [[uninstall]] an installed distribution.
*
@ -19,6 +22,7 @@ import org.enso.launcher.{FileSystem, Logger, OS}
*/
class DistributionUninstaller(
manager: DistributionManager,
resourceManager: ResourceManager,
autoConfirm: Boolean
) {
@ -35,6 +39,13 @@ class DistributionUninstaller(
def uninstall(): Unit = {
checkPortable()
askConfirmation()
resourceManager.acquireExclusiveMainLock(waitAction = () => {
Logger.warn(
"Please ensure that no other Enso processes are using this " +
"distribution before uninstalling. The uninstaller will resume once " +
"all related Enso processes exit."
)
})
if (OS.isWindows) uninstallWindows()
else uninstallUNIX()
}
@ -65,13 +76,15 @@ class DistributionUninstaller(
private def uninstallWindows(): Unit = {
val deferRootRemoval = isBinaryInsideData
uninstallConfig()
val newPath = partiallyUninstallExecutableWindows()
uninstallDataContents(deferRootRemoval)
Logger.info(
"Successfully uninstalled the distribution but for the launcher " +
"executable. It will be removed in a moment after this program " +
"terminates."
)
uninstallExecutableWindows(
finishUninstallExecutableWindows(
newPath,
if (deferRootRemoval) Some(manager.paths.dataRoot) else None
)
}
@ -162,7 +175,11 @@ class DistributionUninstaller(
/**
* Directories that are expected to be inside of the data root.
*/
private val knownDataDirectories = Seq("tmp", "components-licences", "config")
private val knownDataDirectories = Seq(
manager.TMP_DIRECTORY,
manager.CONFIG_DIRECTORY,
"components-licences"
)
/**
* Removes all files contained in the ENSO_DATA_DIRECTORY and possibly the
@ -185,6 +202,26 @@ class DistributionUninstaller(
FileSystem.removeFileIfExists(dataRoot / fileName)
}
resourceManager.unlockTemporaryDirectory()
resourceManager.releaseMainLock()
val lockDirectory = dataRoot / manager.LOCK_DIRECTORY
if (Files.isDirectory(lockDirectory)) {
for (lock <- FileSystem.listDirectory(lockDirectory)) {
try {
Files.delete(lock)
} catch {
case NonFatal(exception) =>
Logger.error(
s"Cannot remove lockfile ${lock.getFileName}.",
exception
)
throw exception
}
}
FileSystem.removeDirectory(lockDirectory)
}
if (!deferDataRootRemoval) {
val nestedBinDirectory = dataRoot / "bin"
if (Files.exists(nestedBinDirectory))
@ -261,22 +298,57 @@ class DistributionUninstaller(
FileSystem.removeFileIfExists(manager.env.getPathToRunningExecutable)
}
/**
* Moves the current launcher executable, so other processes cannot start it
* while uninstallation is in progress.
*
* It will be removed at the last stage of the uninstallation.
*
* @return new path of the executable
*/
private def partiallyUninstallExecutableWindows(): Path = {
val currentPath = manager.env.getPathToRunningExecutable
val newPath = currentPath.getParent.resolve(OS.executableName("enso.old"))
Files.move(currentPath, newPath)
newPath
}
/**
* Uninstalls the executable on Windows where it is impossible to remove an
* executable that is running.
*
* Uses a workaround implemented in [[InternalOpts]]. Has to be run at the
* very end as it has to terminate the current executable.
*
* @param myNewPath path to the current (possibly moved) executable
* @param parentToRemove optional path to the parent directory that should be
* removed alongside the executable
*/
private def uninstallExecutableWindows(
private def finishUninstallExecutableWindows(
myNewPath: Path,
parentToRemove: Option[Path]
): Nothing = {
val temporaryLauncher =
Files.createTempDirectory("enso-uninstall") / OS.executableName("enso")
val oldLauncher = manager.env.getPathToRunningExecutable
val oldLauncher = myNewPath
Files.copy(oldLauncher, temporaryLauncher)
InternalOpts
.runWithNewLauncher(temporaryLauncher)
.finishUninstall(oldLauncher, parentToRemove)
}
}
object DistributionUninstaller {
/**
* Creates a default [[DistributionUninstaller]] using the default managers
* and the provided CLI options.
*/
def default(globalCLIOptions: GlobalCLIOptions): DistributionUninstaller =
new DistributionUninstaller(
DistributionManager,
DefaultResourceManager,
globalCLIOptions.autoConfirm
)
}

16
engine/launcher/src/main/scala/org/enso/launcher/locking/DefaultFileLockManager.scala Normal file
View File

@ -0,0 +1,16 @@
package org.enso.launcher.locking
import java.nio.file.Path
import org.enso.launcher.installation.DistributionManager
/**
* Default [[FileLockManager]] storing lock files in a directory defined by the
* [[DistributionManager]].
*/
object DefaultFileLockManager extends FileLockManager {
/**
* @inheritdoc
*/
override def locksRoot: Path = DistributionManager.paths.locks
}

6
engine/launcher/src/main/scala/org/enso/launcher/locking/DefaultResourceManager.scala Normal file
View File

@ -0,0 +1,6 @@
package org.enso.launcher.locking
/**
* Default [[ResourceManager]] using the [[DefaultFileLockManager]].
*/
object DefaultResourceManager extends ResourceManager(DefaultFileLockManager)

123
engine/launcher/src/main/scala/org/enso/launcher/locking/FileLockManager.scala Normal file
View File

@ -0,0 +1,123 @@
package org.enso.launcher.locking
import java.nio.channels.{FileChannel, FileLock}
import java.nio.file.{Files, Path, StandardOpenOption}
import org.enso.launcher.Logger
import scala.util.control.NonFatal
/**
* [[LockManager]] using file-based locks.
*
* This is the [[LockManager]] that should be used in production as it is able
* to synchronize locks between different processes.
*
* Under the hood, it uses the [[FileLock]] mechanism. This mechanism specifies
* that on some platforms shared locks may not be supported and all shared
* locks are treated as exclusive locks there. That would be harmful to the
* user experience by allowing only a single Enso process to be running at the
* same time as we use shared locks extensively. However, all the platforms we
* currently support (Linux, macOS, Windows), do support shared locks, so this
* should not be an issue for us. However, if a shared lock request returns an
* exclusive lock, a warning is issued, just in case.
*/
abstract class FileLockManager extends LockManager {
/**
* Specifies the directory in which lockfiles should be created.
*/
def locksRoot: Path
/**
* @inheritdoc
*/
override def acquireLock(resourceName: String, lockType: LockType): Lock = {
val channel = openChannel(resourceName)
try {
lockChannel(channel, lockType)
} catch {
case NonFatal(e) =>
channel.close()
throw e
}
}
/**
* @inheritdoc
*/
override def tryAcquireLock(
resourceName: String,
lockType: LockType
): Option[Lock] = {
val channel = openChannel(resourceName)
try {
tryLockChannel(channel, lockType)
} catch {
case NonFatal(e) =>
channel.close()
throw e
}
}
private def isShared(lockType: LockType): Boolean =
lockType match {
case LockType.Exclusive => false
case LockType.Shared => true
}
private def lockPath(resourceName: String): Path =
locksRoot.resolve(resourceName + ".lock")
private def openChannel(resourceName: String): FileChannel = {
val path = lockPath(resourceName)
val parent = path.getParent
if (!Files.exists(parent)) {
try Files.createDirectories(parent)
catch { case NonFatal(_) => }
}
FileChannel.open(
path,
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.WRITE
)
}
private def lockChannel(channel: FileChannel, lockType: LockType): Lock = {
WrapLock(
channel.lock(0L, Long.MaxValue, isShared(lockType)),
channel,
lockType
)
}
private def tryLockChannel(
channel: FileChannel,
lockType: LockType
): Option[Lock] =
Option(channel.tryLock(0L, Long.MaxValue, isShared(lockType)))
.map(WrapLock(_, channel, lockType))
private case class WrapLock(
fileLock: FileLock,
channel: FileChannel,
lockType: LockType
) extends Lock {
if (isShared(lockType) && !fileLock.isShared) {
Logger.warn(
"A request for a shared lock returned an exclusive lock. " +
"The platform that you are running on may not support shared locks, " +
"this may result in only a single Enso instance being able to run at " +
"any time."
)
}
override def release(): Unit = {
fileLock.release()
channel.close()
}
}
}

25
engine/launcher/src/main/scala/org/enso/launcher/locking/Lock.scala Normal file
View File

@ -0,0 +1,25 @@
package org.enso.launcher.locking
import scala.util.Using.Releasable
/**
* A lock synchronizing access to some resource.
*/
trait Lock {
/**
* Releases the lock and any resources (file channels etc.) that are
* associated with it.
*/
def release(): Unit
}
object Lock {
/**
* [[Releasable]] instance for [[Lock]] making it usable with the `Using`
* construct.
*/
implicit val releasable: Releasable[Lock] = (resource: Lock) =>
resource.release()
}

46
engine/launcher/src/main/scala/org/enso/launcher/locking/LockManager.scala Normal file
View File

@ -0,0 +1,46 @@
package org.enso.launcher.locking
/**
* Manages locks that can be used to synchronize different launcher processes
* running in parallel to avoid components corruption caused by simultaneous
* modification of components.
*/
trait LockManager {
/**
* Acquires the lock with the given name and type.
*
* Will wait if the lock cannot be acquired immediately.
*/
def acquireLock(resourceName: String, lockType: LockType): Lock
/**
* Tries to immediately acquire the lock.
*
* Returns immediately with None if the lock cannot be acquired without
* waiting.
*/
def tryAcquireLock(resourceName: String, lockType: LockType): Option[Lock]
/**
* Acquires the lock with the given name and type.
*
* If the lock cannot be acquired immediately, it will execute the
* `waitingAction` and start waiting to acquire the lock. The waiting action
* may be used to notify the user that the program will have to wait and that
* some programs must be closed to continue.
*/
def acquireLockWithWaitingAction(
resourceName: String,
lockType: LockType,
waitingAction: () => Unit
): Lock = {
val immediate = tryAcquireLock(resourceName, lockType)
immediate match {
case Some(lock) => lock
case None =>
waitingAction()
acquireLock(resourceName, lockType)
}
}
}

18
engine/launcher/src/main/scala/org/enso/launcher/locking/LockType.scala Normal file
View File

@ -0,0 +1,18 @@
package org.enso.launcher.locking
/**
* Defines the lock type.
*/
sealed trait LockType
object LockType {
/**
* An exclusive lock can be held by only one user at a time.
*/
case object Exclusive extends LockType
/**
* A shared lock may be held by multiple users in a given moment.
*/
case object Shared extends LockType
}

73
engine/launcher/src/main/scala/org/enso/launcher/locking/Resource.scala Normal file
View File

@ -0,0 +1,73 @@
package org.enso.launcher.locking
import nl.gn0s1s.bump.SemVer
import org.enso.launcher.components.RuntimeVersion
/**
* Represents a resource that can be locked.
*/
trait Resource {
/**
* Name of the resource.
*
* Must be a valid filename part.
*/
def name: String
/**
* A message that is displayed by default if the lock on that resource cannot
* be acquired immediately.
*/
def waitMessage: String
}
object Resource {
/**
* Synchronizes launcher upgrades.
*/
case object LauncherExecutable extends Resource {
override def name: String = "launcher-executable"
override def waitMessage: String =
"Another upgrade is in progress, " +
"the current process must wait until it is completed."
}
/**
* This resource is held when adding or removing any components.
*/
case object AddOrRemoveComponents extends Resource {
override def name: String = "add-remove-components"
override def waitMessage: String =
"Another process is adding or removing components, " +
"the current process must wait until it finishes."
}
/**
* This resource should be held (shared) by any process that is using the
* engine to ensure that it will stay available for the duration of the
* action.
*
* It is acquired exclusively when the engine is installed or uninstalled.
*/
case class Engine(version: SemVer) extends Resource {
override def name: String = s"engine-$version"
override def waitMessage: String =
s"Another process is using engine $version, " +
"the current process must wait until other processes complete."
}
/**
* This resource should be held (shared) by any process that is using the
* runtime to ensure that it will stay available for the duration of the
* action.
*
* It is acquired exclusively when the runtime is installed or uninstalled.
*/
case class Runtime(version: RuntimeVersion) extends Resource {
override def name: String = s"runtime-${version.graal}-${version.java}"
override def waitMessage: String =
s"Another process is using $version, " +
"the current process must wait until other processes complete."
}
}

193
engine/launcher/src/main/scala/org/enso/launcher/locking/ResourceManager.scala Normal file
View File

@ -0,0 +1,193 @@
package org.enso.launcher.locking
import org.enso.launcher.Logger
import scala.util.Using
/**
* Uses a [[LockManager]] implementation to synchronize access to [[Resource]].
*/
class ResourceManager(lockManager: LockManager) {
/**
* Runs the `action` while holding a lock (of `lockType`) for the `resource`.
*
* If the lock cannot be acquired immediately, the `waitingAction` is
* executed. It can be used to notify the user.
*/
def withResource[R](
resource: Resource,
lockType: LockType,
waitingAction: Option[Resource => Unit] = None
)(
action: => R
): R =
Using {
lockManager.acquireLockWithWaitingAction(
resource.name,
lockType = lockType,
() =>
waitingAction
.map(_.apply(resource))
.getOrElse(Logger.warn(resource.waitMessage))
)
} { _ => action }.get
/**
* Runs the `action` while holding multiple locks for a sequence of
* resources.
*/
def withResources[R](resources: (Resource, LockType)*)(action: => R): R =
resources match {
case Seq((head, exclusive), tail @ _*) =>
withResource(head, exclusive) { withResources(tail: _*)(action) }
case Seq() =>
action
}
var mainLock: Option[Lock] = None
/**
* Initializes the [[MainLock]].
*/
def initializeMainLock(): Unit = {
val lock =
lockManager
.tryAcquireLock(MainLock.name, LockType.Shared)
.getOrElse {
throw DistributionIsModifiedError(MainLock.waitMessage)
}
mainLock = Some(lock)
}
/**
* Exception that is thrown when the main lock is held exclusively.
*
* This situation means that the current distribution is being installed or
* uninstalled, so it should not be used in the meantime and the application
* has to terminate immediately.
*/
case class DistributionIsModifiedError(message: String)
extends RuntimeException(message)
/**
* Acquires an exclusive main lock (first releasing the shared lock),
* ensuring that no other processes using this distribution can be running in
* parallel.
*
* @param waitAction function that is executed if the lock cannot be acquired
* immediately
*/
def acquireExclusiveMainLock(waitAction: () => Unit): Unit = {
mainLock match {
case Some(oldLock) =>
oldLock.release()
mainLock = None
case None =>
}
val lock = lockManager.acquireLockWithWaitingAction(
MainLock.name,
LockType.Exclusive,
waitAction
)
mainLock = Some(lock)
}
/**
* Releases the main lock.
*
* Should be called just before the program terminates. It is not an error to
* skip it, as the operating system should unlock all resources after the
* program terminates, but on some platforms this automatic 'garbage
* collection for locks' may take some time, so it is better to release it
* manually.
*/
def releaseMainLock(): Unit =
mainLock match {
case Some(lock) =>
lock.release()
mainLock = None
case None =>
}
/**
* Runs the provided `action` if an exclusive lock can be immediately
* acquired for the temporary directory, i.e. the directory is not used by
* anyone.
*
* If the lock cannot be acquired immediately, the action is not executed and
* None is returned.
*/
def tryWithExclusiveTemporaryDirectory[R](action: => R): Option[R] = {
lockManager.tryAcquireLock(
TemporaryDirectory.name,
LockType.Exclusive
) match {
case Some(lock) =>
try Some(action)
finally lock.release()
case None => None
}
}
private var temporaryDirectoryLock: Option[Lock] = None
/**
* Marks the temporary directory as in use.
*
* This lock does not have to be released as it will be automatically
* released when the program terminates.
*/
def startUsingTemporaryDirectory(): Unit = {
val lock = lockManager.acquireLockWithWaitingAction(
TemporaryDirectory.name,
LockType.Shared,
() => Logger.warn(TemporaryDirectory.waitMessage)
)
temporaryDirectoryLock = Some(lock)
}
/**
* Releases the lock for the temporary directory.
*
* Used by the uninstaller as part of releasing all locks to ensure that the
* locks directory can be removed.
*/
def unlockTemporaryDirectory(): Unit =
temporaryDirectoryLock match {
case Some(lock) =>
lock.release()
temporaryDirectoryLock = None
case None =>
}
/**
* This resource is acquired whenever the temporary directory is first
* accessed.
*
* It ensures that installations running in parallel will not remove each
* other's files. The temporary directory is only cleaned when no other
* processes use it.
*/
private case object TemporaryDirectory extends Resource {
override def name: String = "temporary-files"
override def waitMessage: String =
"Another process is cleaning temporary files, " +
"the installation has to wait until that is complete to avoid " +
"conflicts. It should not take a long time."
}
/**
* The main lock that is held by all launcher processes.
*
* It is used to ensure that no other processes are running when the
* distribution is being installed or uninstalled.
*/
private case object MainLock extends Resource {
override def name: String = "launcher-main"
override def waitMessage: String =
"Another process is installing or uninstalling the current " +
"distribution. Please wait until that finishes."
}
}

16
engine/launcher/src/main/scala/org/enso/launcher/releases/EnsoRepository.scala
View File

@ -72,7 +72,10 @@ object EnsoRepository {
*
* Internal method used for testing.
*/
def internalUseFakeRepository(fakeRepositoryRoot: Path): Unit =
def internalUseFakeRepository(
fakeRepositoryRoot: Path,
shouldWaitForAssets: Boolean
): Unit =
if (buildinfo.Info.isRelease)
throw new IllegalStateException(
"Internal testing function internalUseFakeRepository used in a " +
@ -80,7 +83,8 @@ object EnsoRepository {
)
else {
Logger.debug(s"[TEST] Using a fake repository at $fakeRepositoryRoot.")
launcherRepository = makeFakeRepository(fakeRepositoryRoot)
launcherRepository =
makeFakeRepository(fakeRepositoryRoot, shouldWaitForAssets)
}
private val defaultEngineRepository = githubRepository
@ -92,7 +96,11 @@ object EnsoRepository {
defaultLauncherRepository
private def makeFakeRepository(
fakeRepositoryRoot: Path
fakeRepositoryRoot: Path,
shouldWaitForAssets: Boolean
): SimpleReleaseProvider =
FakeReleaseProvider(fakeRepositoryRoot)
FakeReleaseProvider(
fakeRepositoryRoot,
shouldWaitForAssets = shouldWaitForAssets
)
}

46
engine/launcher/src/main/scala/org/enso/launcher/releases/testing/FakeReleaseProvider.scala
View File

@ -3,6 +3,7 @@ package org.enso.launcher.releases.testing
import java.nio.file.{Files, Path, StandardCopyOption}
import org.enso.cli.{ProgressListener, TaskProgress}
import org.enso.launcher.locking.{DefaultFileLockManager, LockType}
import org.enso.launcher.releases.{
Asset,
Release,
@ -25,12 +26,13 @@ import scala.util.{Success, Try, Using}
*/
case class FakeReleaseProvider(
releasesRoot: Path,
copyIntoArchiveRoot: Seq[String] = Seq.empty
copyIntoArchiveRoot: Seq[String] = Seq.empty,
shouldWaitForAssets: Boolean = false
) extends SimpleReleaseProvider {
private val releases =
FileSystem
.listDirectory(releasesRoot)
.map(FakeRelease(_, copyIntoArchiveRoot))
.map(FakeRelease(_, copyIntoArchiveRoot, shouldWaitForAssets))
/**
* @inheritdoc
@ -54,8 +56,11 @@ case class FakeReleaseProvider(
* represents a [[FakeAsset]]
* @param copyIntoArchiveRoot list of
*/
case class FakeRelease(path: Path, copyIntoArchiveRoot: Seq[String] = Seq.empty)
extends Release {
case class FakeRelease(
path: Path,
copyIntoArchiveRoot: Seq[String] = Seq.empty,
shouldWaitForAssets: Boolean
) extends Release {
/**
* @inheritdoc
@ -67,7 +72,9 @@ case class FakeRelease(path: Path, copyIntoArchiveRoot: Seq[String] = Seq.empty)
*/
override def assets: Seq[Asset] = {
val pathsToCopy = copyIntoArchiveRoot.map(path.resolve)
FileSystem.listDirectory(path).map(FakeAsset(_, pathsToCopy))
FileSystem
.listDirectory(path)
.map(FakeAsset(_, pathsToCopy, shouldWaitForAssets))
}
}
@ -83,8 +90,11 @@ case class FakeRelease(path: Path, copyIntoArchiveRoot: Seq[String] = Seq.empty)
* root of that created archive. This allows to avoid maintaining additional
* copies of shared files like the manifest.
*/
case class FakeAsset(source: Path, copyIntoArchiveRoot: Seq[Path] = Seq.empty)
extends Asset {
case class FakeAsset(
source: Path,
copyIntoArchiveRoot: Seq[Path] = Seq.empty,
shouldWaitForAssets: Boolean
) extends Asset {
/**
* @inheritdoc
@ -95,6 +105,7 @@ case class FakeAsset(source: Path, copyIntoArchiveRoot: Seq[Path] = Seq.empty)
* @inheritdoc
*/
override def downloadTo(path: Path): TaskProgress[Unit] = {
maybeWaitForAsset()
val result = Try(copyFakeAsset(path))
new TaskProgress[Unit] {
override def addProgressListener(
@ -105,6 +116,26 @@ case class FakeAsset(source: Path, copyIntoArchiveRoot: Seq[Path] = Seq.empty)
}
}
/**
* If [[shouldWaitForAssets]] is set, acquires a shared lock on the asset.
*
* The test runner may grab an exclusive lock on an asset as a way to
* synchronize actions (this download will wait until such exclusive lock is
* released).
*/
private def maybeWaitForAsset(): Unit =
if (shouldWaitForAssets) {
val name = "testasset-" + fileName
val lock = DefaultFileLockManager.acquireLockWithWaitingAction(
name,
LockType.Shared,
waitingAction = () => {
System.err.println("INTERNAL-TEST-ACQUIRING-LOCK")
}
)
lock.release()
}
private def copyFakeAsset(destination: Path): Unit =
if (Files.isDirectory(source))
copyArchive(destination)
@ -155,6 +186,7 @@ case class FakeAsset(source: Path, copyIntoArchiveRoot: Seq[Path] = Seq.empty)
"Cannot fetch a fake archive (a directory) as text."
)
else {
maybeWaitForAsset()
val txt = Using(Source.fromFile(source.toFile)) { src =>
src.getLines().mkString("\n")
}

16
engine/launcher/src/main/scala/org/enso/launcher/upgrade/AnotherUpgradeInProgressError.scala Normal file
View File

@ -0,0 +1,16 @@
package org.enso.launcher.upgrade
/**
* Error thrown when trying to upgrade, but another upgrade is in progress and
* this one cannot continue.
*/
case class AnotherUpgradeInProgressError()
extends RuntimeException(
"Another upgrade is in progress. Please wait for it to finish."
) {
/**
* @inheritdoc
*/
override def toString: String = getMessage
}

184
engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala
View File

@ -9,6 +9,12 @@ import org.enso.launcher.archive.Archive
import org.enso.launcher.cli.{GlobalCLIOptions, InternalOpts}
import org.enso.launcher.components.LauncherUpgradeRequiredError
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.locking.{
DefaultResourceManager,
LockType,
Resource,
ResourceManager
}
import org.enso.launcher.releases.launcher.LauncherRelease
import org.enso.launcher.releases.{EnsoRepository, ReleaseProvider}
import org.enso.launcher.{CurrentVersion, FileSystem, Logger, OS}
@ -20,6 +26,7 @@ class LauncherUpgrader(
globalCLIOptions: GlobalCLIOptions,
distributionManager: DistributionManager,
releaseProvider: ReleaseProvider[LauncherRelease],
resourceManager: ResourceManager,
originalExecutablePath: Option[Path]
) {
@ -36,41 +43,50 @@ class LauncherUpgrader(
*
* The upgrade may first temporarily install versions older than the target
* if the upgrade cannot be performed directly from the current version.
*
* If another upgrade is in progress, [[AnotherUpgradeInProgressError]] is
* thrown.
*/
def upgrade(targetVersion: SemVer): Unit = {
runCleanup(isStartup = true)
val release = releaseProvider.fetchRelease(targetVersion).get
if (release.isMarkedBroken) {
if (globalCLIOptions.autoConfirm) {
Logger.warn(
s"The launcher release $targetVersion is marked as broken and it " +
s"should not be used. Since `auto-confirm` is set, the upgrade " +
s"will continue, but you may want to reconsider upgrading to a " +
s"stable release."
)
} else {
Logger.warn(
s"The launcher release $targetVersion is marked as broken and it " +
s"should not be used."
)
val continue = CLIOutput.askConfirmation(
"Are you sure you still want to continue upgrading to this version " +
"despite the warning?"
)
if (!continue) {
throw UpgradeError(
"Upgrade has been cancelled by the user because the requested " +
"version is marked as broken."
resourceManager.withResource(
Resource.LauncherExecutable,
LockType.Exclusive,
waitingAction = Some(_ => throw AnotherUpgradeInProgressError())
) {
runCleanup(isStartup = true)
val release = releaseProvider.fetchRelease(targetVersion).get
if (release.isMarkedBroken) {
if (globalCLIOptions.autoConfirm) {
Logger.warn(
s"The launcher release $targetVersion is marked as broken and it " +
s"should not be used. Since `auto-confirm` is set, the upgrade " +
s"will continue, but you may want to reconsider upgrading to a " +
s"stable release."
)
} else {
Logger.warn(
s"The launcher release $targetVersion is marked as broken and it " +
s"should not be used."
)
val continue = CLIOutput.askConfirmation(
"Are you sure you still want to continue upgrading to this " +
"version despite the warning?"
)
if (!continue) {
throw UpgradeError(
"Upgrade has been cancelled by the user because the requested " +
"version is marked as broken."
)
}
}
}
}
if (release.canPerformUpgradeFromCurrentVersion)
performUpgradeTo(release)
else
performStepByStepUpgrade(release)
if (release.canPerformUpgradeFromCurrentVersion)
performUpgradeTo(release)
else
performStepByStepUpgrade(release)
runCleanup()
runCleanup()
}
}
/**
@ -358,7 +374,7 @@ object LauncherUpgrader {
* executable that will be replaced in the last
* step of the upgrade
*/
def makeDefault(
def default(
globalCLIOptions: GlobalCLIOptions,
originalExecutablePath: Option[Path] = None
): LauncherUpgrader =
@ -366,57 +382,87 @@ object LauncherUpgrader {
globalCLIOptions,
DistributionManager,
EnsoRepository.defaultLauncherReleaseProvider,
DefaultResourceManager,
originalExecutablePath
)
/**
* Wraps an action and intercepts the [[LauncherUpgradeRequiredError]]
* offering to upgrade the launcher and re-run the command with the newer
* version.
*
* @param originalArguments original CLI arguments, needed to be able to
* re-run the command
* @param action action that is executed and may throw the exception; it
* should return the desired exit code
* @return if `action` succeeds, its exit code is returned; otherwise if the
* [[LauncherUpgradeRequiredError]] is intercepted and an upgrade is
* performed, the exit code of the command that has been re-executed
* is returned
*/
def recoverUpgradeRequiredErrors(originalArguments: Array[String])(
action: => Int
): Int = {
try {
action
} catch {
case e: LauncherUpgradeRequiredError =>
val autoConfirm = e.globalCLIOptions.autoConfirm
def shouldProceed: Boolean =
if (autoConfirm) {
Logger.warn(
"A more recent launcher version is required. Since " +
"`auto-confirm` is set, the launcher upgrade will be peformed " +
"automatically."
)
true
} else {
Logger.warn(
s"A more recent launcher version (at least " +
s"${e.expectedLauncherVersion}) is required to continue."
)
CLIOutput.askConfirmation(
"Do you want to upgrade the launcher and continue?",
yesDefault = true
)
}
case upgradeRequiredError: LauncherUpgradeRequiredError =>
askToUpgrade(upgradeRequiredError, originalArguments)
}
}
if (!shouldProceed) {
throw e
}
val upgrader = makeDefault(e.globalCLIOptions)
val targetVersion = upgrader.latestVersion().get
val launcherExecutable = upgrader.originalExecutable
upgrader.upgrade(targetVersion)
Logger.info(
"Re-running the current command with the upgraded launcher."
private def askToUpgrade(
upgradeRequiredError: LauncherUpgradeRequiredError,
originalArguments: Array[String]
): Int = {
val autoConfirm = upgradeRequiredError.globalCLIOptions.autoConfirm
def shouldProceed: Boolean =
if (autoConfirm) {
Logger.warn(
"A more recent launcher version is required. Since `auto-confirm` " +
"is set, the launcher upgrade will be peformed automatically."
)
true
} else {
Logger.warn(
s"A more recent launcher version (at least " +
s"${upgradeRequiredError.expectedLauncherVersion}) is required to " +
s"continue."
)
CLIOutput.askConfirmation(
"Do you want to upgrade the launcher and continue?",
yesDefault = true
)
}
val arguments =
InternalOpts.removeInternalTestOptions(originalArguments.toIndexedSeq)
val rerunCommand =
Seq(launcherExecutable.toAbsolutePath.normalize.toString) ++ arguments
Logger.debug(s"Running `${rerunCommand.mkString(" ")}`.")
val processBuilder = new ProcessBuilder(rerunCommand: _*)
val process = processBuilder.inheritIO().start()
process.waitFor()
if (!shouldProceed) {
throw upgradeRequiredError
}
val upgrader = default(upgradeRequiredError.globalCLIOptions)
val targetVersion = upgrader.latestVersion().get
val launcherExecutable = upgrader.originalExecutable
try {
upgrader.upgrade(targetVersion)
Logger.info("Re-running the current command with the upgraded launcher.")
val arguments =
InternalOpts.removeInternalTestOptions(originalArguments.toIndexedSeq)
val rerunCommand =
Seq(launcherExecutable.toAbsolutePath.normalize.toString) ++ arguments
Logger.debug(s"Running `${rerunCommand.mkString(" ")}`.")
val processBuilder = new ProcessBuilder(rerunCommand: _*)
val process = processBuilder.inheritIO().start()
process.waitFor()
} catch {
case _: AnotherUpgradeInProgressError =>
Logger.error(
"Another upgrade is in progress." +
"Please wait for it to finish and manually re-run the requested " +
"command."
)
1
}
}
}

2
engine/launcher/src/test/scala/org/enso/launcher/FakeEnvironment.scala
View File

@ -42,10 +42,12 @@ trait FakeEnvironment { self: WithTemporaryDirectory =>
val dataDir = getTestDirectory / "test_data"
val configDir = getTestDirectory / "test_config"
val binDir = getTestDirectory / "test_bin"
val runDir = getTestDirectory / "test_run"
val env = extraOverrides
.updated("ENSO_DATA_DIRECTORY", dataDir.toString)
.updated("ENSO_CONFIG_DIRECTORY", configDir.toString)
.updated("ENSO_BIN_DIRECTORY", binDir.toString)
.updated("ENSO_RUNTIME_DIRECTORY", runDir.toString)
val fakeEnvironment = new Environment {
override def getPathToRunningExecutable: Path = executable

186
engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala
View File

@ -1,14 +1,15 @@
package org.enso.launcher
import java.io.{BufferedReader, InputStream, InputStreamReader}
import java.nio.file.{Files, Path}
import java.lang.{ProcessBuilder => JProcessBuilder}
import java.util.concurrent.{Semaphore, TimeUnit, TimeoutException}
import org.scalatest.concurrent.{Signaler, TimeLimitedTests}
import org.scalatest.matchers.should.Matchers
import org.scalatest.matchers.{MatchResult, Matcher}
import org.scalatest.time.Span
import org.scalatest.wordspec.AnyWordSpec
import org.scalatest.time.SpanSugar._
import scala.collection.Factory
@ -21,7 +22,7 @@ import scala.jdk.StreamConverters._
*/
trait NativeTest extends AnyWordSpec with Matchers with TimeLimitedTests {
override val timeLimit: Span = 15 seconds
override val timeLimit: Span = 30 seconds
override val defaultTestSignaler: Signaler = _.interrupt()
/**
@ -150,7 +151,7 @@ trait NativeTest extends AnyWordSpec with Matchers with TimeLimitedTests {
}
/**
* Runs the provided `command`.
* Starts the provided `command`.
*
* `extraEnv` may be provided to extend the environment. Care must be taken
* on Windows where environment variables are (mostly) case-insensitive.
@ -159,11 +160,10 @@ trait NativeTest extends AnyWordSpec with Matchers with TimeLimitedTests {
* launched process to finish too. Especially important on Windows where
* child processes may run after the launcher parent has been terminated.
*/
private def run(
def start(
command: Seq[String],
extraEnv: Seq[(String, String)],
waitForDescendants: Boolean = true
): RunResult = {
extraEnv: Seq[(String, String)]
): WrappedProcess = {
val builder = new JProcessBuilder(command: _*)
val newKeys = extraEnv.map(_._1.toLowerCase)
if (newKeys.distinct.size < newKeys.size) {
@ -193,29 +193,165 @@ trait NativeTest extends AnyWordSpec with Matchers with TimeLimitedTests {
try {
val process = builder.start()
try {
val exitCode = process.waitFor()
if (waitForDescendants) {
val descendants = process.descendants().toScala(Factory.arrayFactory)
descendants.foreach(_.onExit().join())
}
val stdout = new String(process.getInputStream.readAllBytes())
val stderr = new String(process.getErrorStream.readAllBytes())
RunResult(exitCode, stdout, stderr)
} catch {
case e: InterruptedException =>
if (process.isAlive) {
println(s"Killing the timed-out process: ${command.mkString(" ")}")
process.destroy()
}
throw e
}
new WrappedProcess(command, process)
} catch {
case e: Exception =>
throw new RuntimeException("Cannot run the Native Image binary", e)
}
}
class WrappedProcess(command: Seq[String], process: Process) {
private val outQueue =
new java.util.concurrent.LinkedTransferQueue[String]()
private val errQueue =
new java.util.concurrent.LinkedTransferQueue[String]()
sealed trait StreamType
case object StdErr extends StreamType
case object StdOut extends StreamType
@volatile private var ioHandlers: Seq[(String, StreamType) => Unit] = Seq()
def watchStream(
stream: InputStream,
streamType: StreamType
): Unit = {
val reader = new BufferedReader(new InputStreamReader(stream))
var line: String = null
val queue = streamType match {
case StdErr => errQueue
case StdOut => outQueue
}
while ({ line = reader.readLine(); line != null }) {
queue.add(line)
ioHandlers.foreach(f => f(line, streamType))
}
}
private val outThread = new Thread(() =>
watchStream(process.getInputStream, StdOut)
)
private val errThread = new Thread(() =>
watchStream(process.getErrorStream, StdErr)
)
outThread.start()
errThread.start()
/**
* Waits for a message on the stderr to appear.
*/
def waitForMessageOnErrorStream(
message: String,
timeoutSeconds: Long = 10
): Unit = {
val semaphore = new Semaphore(0)
def handler(line: String, streamType: StreamType): Unit = {
if (streamType == StdErr && line.contains(message)) {
semaphore.release()
}
}
this.synchronized {
ioHandlers ++= Seq(handler _)
}
val acquired = semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS)
if (!acquired) {
throw new RuntimeException(s"Waiting for `$message` timed out.")
}
}
/**
* Starts printing the stdout and stderr of the started process to the
* stdout with prefixes to indicate that these messages come from another
* process.
*
* It also prints lines that were printed before invoking this method.
* Thus, it is possible that a line may be printed twice (once as
* 'before-printIO' and once normally).
*/
def printIO(): Unit = {
def handler(line: String, streamType: StreamType): Unit = {
val prefix = streamType match {
case StdErr => "stderr> "
case StdOut => "stdout> "
}
println(prefix + line)
}
this.synchronized {
ioHandlers ++= Seq(handler _)
}
outQueue.asScala.toSeq.foreach(line =>
println(s"stdout-before-printIO> $line")
)
errQueue.asScala.toSeq.foreach(line =>
println(s"stderr-before-printIO> $line")
)
}
/**
* Waits for the process to finish and returns its [[RunResult]].
*
* If `waitForDescendants` is set, tries to wait for descendants of the
* launched process to finish too. Especially important on Windows where
* child processes may run after the launcher parent has been terminated.
*
* It will timeout after `timeoutSeconds` and try to kill the process (or
* its descendants), although it may not always be able to.
*/
def join(
waitForDescendants: Boolean = true,
timeoutSeconds: Long = 10
): RunResult = {
var descendants: Seq[ProcessHandle] = Seq()
try {
val exitCode =
if (process.waitFor(timeoutSeconds, TimeUnit.SECONDS))
process.exitValue()
else throw new TimeoutException("Process timed out")
if (waitForDescendants) {
descendants =
process.descendants().toScala(Factory.arrayFactory).toSeq
descendants.foreach(_.onExit().get(timeoutSeconds, TimeUnit.SECONDS))
}
errThread.join(1000)
outThread.join(1000)
if (errThread.isAlive) {
errThread.interrupt()
}
if (outThread.isAlive) {
outThread.interrupt()
}
val stdout = outQueue.asScala.toSeq.mkString("\n")
val stderr = errQueue.asScala.toSeq.mkString("\n")
RunResult(exitCode, stdout, stderr)
} catch {
case e @ (_: InterruptedException | _: TimeoutException) =>
if (process.isAlive) {
println(s"Killing the timed-out process: ${command.mkString(" ")}")
process.destroyForcibly()
}
for (processHandle <- descendants) {
if (processHandle.isAlive) {
processHandle.destroyForcibly()
}
}
throw e
}
}
}
/**
* Runs the provided `command`.
*
* `extraEnv` may be provided to extend the environment. Care must be taken
* on Windows where environment variables are (mostly) case-insensitive.
*/
private def run(
command: Seq[String],
extraEnv: Seq[(String, String)],
waitForDescendants: Boolean = true
): RunResult = start(command, extraEnv).join(waitForDescendants)
}
/* Note [Windows Path]

2
engine/launcher/src/test/scala/org/enso/launcher/WithTemporaryDirectory.scala
View File

@ -31,7 +31,7 @@ trait WithTemporaryDirectory extends Suite with BeforeAndAfterEach {
/**
* Returns the temporary directory for this test.
*/
def getTestDirectory: Path = testDirectory.toAbsolutePath
def getTestDirectory: Path = testDirectory.toAbsolutePath.normalize
/**
* Tries to remove the directory, retrying every 100ms for 3 seconds.

7
engine/launcher/src/test/scala/org/enso/launcher/components/ComponentsManagerTest.scala
View File

@ -5,6 +5,7 @@ import java.nio.file.Path
import nl.gn0s1s.bump.SemVer
import org.enso.launcher.cli.GlobalCLIOptions
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.locking.TestLocalResourceManager
import org.enso.launcher.releases.engine.EngineReleaseProvider
import org.enso.launcher.releases.runtime.GraalCEReleaseProvider
import org.enso.launcher.releases.testing.FakeReleaseProvider
@ -34,8 +35,9 @@ class ComponentsManagerTest
def makeManagers(
environmentOverrides: Map[String, String] = Map.empty
): (DistributionManager, ComponentsManager, Environment) = {
val env = fakeInstalledEnvironment(environmentOverrides)
val distributionManager = new DistributionManager(env)
val env = fakeInstalledEnvironment(environmentOverrides)
val distributionManager =
new DistributionManager(env, TestLocalResourceManager.create())
val fakeReleasesRoot =
Path.of(
getClass
@ -58,6 +60,7 @@ class ComponentsManagerTest
useJSON = false
),
distributionManager,
TestLocalResourceManager.create(),
engineProvider,
runtimeProvider
)

35
engine/launcher/src/test/scala/org/enso/launcher/components/runner/RunnerSpec.scala
View File

@ -40,21 +40,6 @@ class RunnerSpec extends ComponentsManagerTest {
val runSettings = RunSettings(SemVer(0, 0, 0), Seq("arg1", "--flag2"))
val jvmOptions = Seq(("locally-added-options", "value1"))
val systemCommand = runner.createCommand(
runSettings,
JVMSettings(useSystemJVM = true, jvmOptions = jvmOptions)
)
systemCommand.command.head shouldEqual "java"
val managedCommand = runner.createCommand(
runSettings,
JVMSettings(useSystemJVM = false, jvmOptions = jvmOptions)
)
managedCommand.command.head should include("java")
managedCommand.extraEnv.find(_._1 == "JAVA_HOME").value._2 should
include("graalvm-ce")
val enginePath =
getTestDirectory / "test_data" / "dist" / "0.0.0"
@ -63,7 +48,7 @@ class RunnerSpec extends ComponentsManagerTest {
val runnerPath =
(enginePath / "component" / "runner.jar").toAbsolutePath.normalize
for (command <- Seq(systemCommand, managedCommand)) {
def checkCommandLine(command: Command): Unit = {
val commandLine = command.command.mkString(" ")
val arguments = command.command.tail
arguments should contain("-Xfrom-env")
@ -81,6 +66,24 @@ class RunnerSpec extends ComponentsManagerTest {
commandLine should include(s"-jar $runnerPath")
}
runner.withCommand(
runSettings,
JVMSettings(useSystemJVM = true, jvmOptions = jvmOptions)
) { systemCommand =>
systemCommand.command.head shouldEqual "java"
checkCommandLine(systemCommand)
}
runner.withCommand(
runSettings,
JVMSettings(useSystemJVM = false, jvmOptions = jvmOptions)
) { managedCommand =>
managedCommand.command.head should include("java")
val javaHome =
managedCommand.extraEnv.find(_._1 == "JAVA_HOME").value._2
javaHome should include("graalvm-ce")
}
}
}

6
engine/launcher/src/test/scala/org/enso/launcher/config/GlobalConfigurationManagerSpec.scala
View File

@ -4,6 +4,7 @@ import io.circe.Json
import nl.gn0s1s.bump.SemVer
import org.enso.launcher.{FakeEnvironment, WithTemporaryDirectory}
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.locking.TestLocalResourceManager
import org.scalatest.OptionValues
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
@ -15,8 +16,9 @@ class GlobalConfigurationManagerSpec
with FakeEnvironment
with OptionValues {
def makeConfigManager(): GlobalConfigurationManager = {
val env = fakeInstalledEnvironment()
val distributionManager = new DistributionManager(env)
val env = fakeInstalledEnvironment()
val distributionManager =
new DistributionManager(env, TestLocalResourceManager.create())
new GlobalConfigurationManager(null, distributionManager) {
override def defaultVersion: SemVer = SemVer(0, 0, 0)
}

18
engine/launcher/src/test/scala/org/enso/launcher/installation/DistributionManagerSpec.scala
View File

@ -3,6 +3,7 @@ package org.enso.launcher.installation
import java.nio.file.Path
import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher.locking.TestLocalResourceManager
import org.enso.launcher.{Environment, FakeEnvironment, WithTemporaryDirectory}
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
@ -20,7 +21,11 @@ class DistributionManagerSpec
override def getPathToRunningExecutable: Path = executable
}
val distributionManager = new DistributionManager(fakeEnvironment)
val distributionManager =
new DistributionManager(
fakeEnvironment,
TestLocalResourceManager.create()
)
distributionManager.isRunningPortable shouldEqual true
distributionManager.paths.dataRoot shouldEqual getTestDirectory
distributionManager.paths.config shouldEqual getTestDirectory / "config"
@ -35,7 +40,11 @@ class DistributionManagerSpec
override def getPathToRunningExecutable: Path = executable
}
val distributionManager = new DistributionManager(fakeEnvironment)
val distributionManager =
new DistributionManager(
fakeEnvironment,
TestLocalResourceManager.create()
)
distributionManager.isRunningPortable shouldEqual false
}
@ -46,7 +55,10 @@ class DistributionManagerSpec
val binDir = getTestDirectory / "test_bin"
val distributionManager =
new DistributionManager(fakeInstalledEnvironment())
new DistributionManager(
fakeInstalledEnvironment(),
TestLocalResourceManager.create()
)
distributionManager.paths.dataRoot shouldEqual dataDir
distributionManager.paths.config shouldEqual configDir
distributionManager.LocallyInstalledDirectories.binDirectory shouldEqual

7
engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala
View File

@ -23,9 +23,10 @@ class InstallerSpec extends NativeTest with WithTemporaryDirectory {
def installedRoot = getTestDirectory / "installed"
def env =
Map(
"ENSO_DATA_DIRECTORY" -> (installedRoot / "data").toString,
"ENSO_CONFIG_DIRECTORY" -> (installedRoot / "config").toString,
"ENSO_BIN_DIRECTORY" -> (installedRoot / "bin").toString
"ENSO_DATA_DIRECTORY" -> (installedRoot / "data").toString,
"ENSO_CONFIG_DIRECTORY" -> (installedRoot / "config").toString,
"ENSO_BIN_DIRECTORY" -> (installedRoot / "bin").toString,
"ENSO_RUNTIME_DIRECTORY" -> (installedRoot / "run").toString
)
def prepareBundles(): Unit = {

8
engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala
View File

@ -27,6 +27,7 @@ class UninstallerSpec extends NativeTest with WithTemporaryDirectory {
if (everythingInsideData) installedRoot / "config"
else getTestDirectory / "enso-config"
val dataDirectory = installedRoot
val runDirectory = installedRoot
val portableLauncher = binDirectory / OS.executableName("enso")
copyLauncherTo(portableLauncher)
Files.createDirectories(dataDirectory / "dist")
@ -39,9 +40,10 @@ class UninstallerSpec extends NativeTest with WithTemporaryDirectory {
Files.createDirectories(dataDirectory / "tmp")
val env = Map(
"ENSO_DATA_DIRECTORY" -> dataDirectory.toAbsolutePath.normalize.toString,
"ENSO_BIN_DIRECTORY" -> binDirectory.toAbsolutePath.normalize.toString,
"ENSO_CONFIG_DIRECTORY" -> configDirectory.toAbsolutePath.normalize.toString
"ENSO_DATA_DIRECTORY" -> dataDirectory.toString,
"ENSO_BIN_DIRECTORY" -> binDirectory.toString,
"ENSO_CONFIG_DIRECTORY" -> configDirectory.toString,
"ENSO_RUNTIME_DIRECTORY" -> runDirectory.toString
)
(portableLauncher, env)
}

419
engine/launcher/src/test/scala/org/enso/launcher/locking/ConcurrencyTest.scala Normal file
View File

@ -0,0 +1,419 @@
package org.enso.launcher.locking
import java.nio.file.{Files, Path}
import nl.gn0s1s.bump.SemVer
import org.enso.cli.TaskProgress
import org.enso.launcher.cli.GlobalCLIOptions
import org.enso.launcher.components.{ComponentsManager, RuntimeVersion}
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.releases.engine.{EngineRelease, EngineReleaseProvider}
import org.enso.launcher.releases.runtime.GraalCEReleaseProvider
import org.enso.launcher.releases.testing.FakeReleaseProvider
import org.enso.launcher.{
components,
FakeEnvironment,
FileSystem,
WithTemporaryDirectory
}
import org.enso.launcher.FileSystem.PathSyntax
import org.scalatest.BeforeAndAfterEach
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import scala.util.Try
class ConcurrencyTest
extends AnyWordSpec
with Matchers
with WithTemporaryDirectory
with FakeEnvironment
with BeforeAndAfterEach {
case class WrapEngineRelease(
originalRelease: EngineRelease,
callback: String => Unit
) extends EngineRelease {
override def version: SemVer = originalRelease.version
override def manifest: components.Manifest =
originalRelease.manifest
override def isBroken: Boolean = originalRelease.isBroken
override def packageFileName: String =
originalRelease.packageFileName
override def downloadPackage(
destination: Path
): TaskProgress[Unit] = {
callback(packageFileName)
originalRelease.downloadPackage(destination)
}
}
var testLocalLockManager: Option[TestLocalLockManager] = None
override def beforeEach(): Unit = {
super.beforeEach()
testLocalLockManager = Some(new TestLocalLockManager)
}
/**
* A separate [[LockManager]] for each test case.
* @return
*/
def lockManager: LockManager = testLocalLockManager.get
/**
* Each call creates a distinct [[ResourceManager]], but all resource
* managers created within one test case share the same [[lockManager]], so
* that they see each other's locks.
*/
def makeNewResourceManager(): ResourceManager =
new ResourceManager(lockManager)
/**
* Creates a [[DistributionManager]] and [[ComponentsManager]] that can be
* used in a test.
*
* @param releaseCallback called when a release asset is fetched
* @param lockWaitsCallback called when a lock with the given name is not
* acquired immediately
* @return a tuple of [[DistributionManager]] and [[ComponentsManager]]
*/
def makeManagers(
releaseCallback: String => Unit,
lockWaitsCallback: String => Unit
): (DistributionManager, ComponentsManager) = {
val env = fakeInstalledEnvironment()
val resourceManager = new ResourceManager(lockManager) {
override def withResource[R](
resource: Resource,
lockType: LockType,
waitingAction: Option[Resource => Unit]
)(action: => R): R = {
val overriddenWaitingAction = (resource: Resource) => {
lockWaitsCallback(resource.name)
waitingAction.foreach(_.apply(resource))
}
super.withResource(resource, lockType, Some(overriddenWaitingAction))(
action
)
}
}
val distributionManager = new DistributionManager(env, resourceManager)
val fakeReleasesRoot =
Path.of(
getClass
.getResource("/org/enso/launcher/components/fake-releases")
.toURI
)
val engineProvider = new EngineReleaseProvider(
FakeReleaseProvider(
fakeReleasesRoot.resolve("enso"),
copyIntoArchiveRoot = Seq("manifest.yaml")
)
) {
override def fetchRelease(version: SemVer): Try[EngineRelease] =
super.fetchRelease(version).map(WrapEngineRelease(_, releaseCallback))
}
val runtimeProvider = new GraalCEReleaseProvider(
FakeReleaseProvider(fakeReleasesRoot.resolve("graalvm"))
) {
override def downloadPackage(
version: RuntimeVersion,
destination: Path
): TaskProgress[Unit] = {
releaseCallback(packageFileName(version))
super.downloadPackage(version, destination)
}
}
val componentsManager = new ComponentsManager(
GlobalCLIOptions(
autoConfirm = true,
hideProgress = true,
useJSON = false
),
distributionManager,
resourceManager,
engineProvider,
runtimeProvider
)
(distributionManager, componentsManager)
}
/**
* Helper function, acts as [[makeManagers]] but returns only the
* [[ComponentsManager]].
*/
def makeComponentsManager(
releaseCallback: String => Unit,
lockWaitsCallback: String => Unit
): ComponentsManager =
makeManagers(
releaseCallback = releaseCallback,
lockWaitsCallback = lockWaitsCallback
)._2
"locks" should {
"synchronize parallel installations with the same runtime" in {
/**
* Two threads start installing different engines in parallel, but these
* engines use the same runtime. The second thread is stalled on
* downloading its engine package, so that the first one can start
* downloading the runtime. When it starts downloading, it is suspended
* and the second thread is resumed so it also wants to download the
* runtime. It should however wait on the runtime lock, when we see that
* it indeed waits, the first is resumed to finish downloading the
* runtime. Then the second thread should automatically resume and not
* install the runtime a second time, seeing changes from first thread.
*
* This test also checks that the temporary directory is cleaned at
* startup, but only if it is safe to do so (so other processes are not
* using it).
*/
val sync = new TestSynchronizer
val engine1 = SemVer(0, 0, 1)
val engine2 = engine1.withPreRelease("pre")
val tmpRoot = getTestDirectory / "test_data" / "tmp"
Files.createDirectories(tmpRoot)
val garbage = tmpRoot / "garbage.txt"
FileSystem.writeTextFile(garbage, "Garbage")
sync.startThread("t1") {
val (distributionManager, componentsManager) = makeManagers(
releaseCallback = { asset =>
if (asset.startsWith("graalvm-")) {
sync.signal("t1-downloads-runtime")
sync.waitFor("t2-waits-for-runtime")
}
},
lockWaitsCallback = resource =>
throw new IllegalStateException(
s"t1 should not be waiting on $resource."
)
)
distributionManager.tryCleaningTemporaryDirectory()
componentsManager.findOrInstallEngine(engine1, complain = false)
}
sync.waitFor("t1-downloads-runtime")
sync.startThread("t2") {
val (distributionManager, componentsManager) = makeManagers(
releaseCallback = { asset =>
if (asset.startsWith("graalvm-")) {
throw new IllegalStateException(
"t2 should not download runtime, " +
"as it should have been done by t1 already."
)
}
},
lockWaitsCallback = {
case resource if resource.startsWith("runtime") =>
sync.signal("t2-waits-for-runtime")
case resource =>
throw new IllegalStateException(s"Unexpected wait on $resource.")
}
)
distributionManager.tryCleaningTemporaryDirectory()
componentsManager.findOrInstallEngine(engine2, complain = false)
}
sync.join()
assert(
Files.notExists(garbage),
"The temporary directory should have been cleaned."
)
}
"synchronize installation and usage" in {
/**
* The first thread starts installing the engine, but is suspended when
* downloading the package. The second thread then tries to use it, but
* it should wait until the installation is finished.
*/
val sync = new TestSynchronizer
val engineVersion = SemVer(0, 0, 1)
sync.startThread("t1") {
val componentsManager = makeComponentsManager(
releaseCallback = { asset =>
if (asset.startsWith("enso-engine-")) {
sync.signal("t1-downloads-engine")
sync.waitFor("t2-waits-for-engine")
} else if (asset.startsWith("graal")) {
sync.report("installation-continues")
}
},
lockWaitsCallback = resource =>
throw new IllegalStateException(
s"t1 should not be waiting on $resource."
)
)
componentsManager.findOrInstallEngine(engineVersion, complain = false)
}
sync.waitFor("t1-downloads-engine")
sync.startThread("t2") {
val componentsManager = makeComponentsManager(
releaseCallback = { asset =>
throw new IllegalStateException(
s"t2 should not be downloading $asset."
)
},
lockWaitsCallback = {
case resource if resource.startsWith("engine") =>
sync.signal("t2-waits-for-engine")
case resource =>
throw new IllegalStateException(s"Unexpected wait on $resource.")
}
)
componentsManager.withEngineAndRuntime(engineVersion) { (_, _) =>
sync.report("using-engine")
}
}
sync.join()
sync.summarizeReports() shouldEqual Seq(
"installation-continues",
"using-engine"
)
}
"synchronize uninstallation and usage" in {
/**
* The first thread starts using the engine, while in the meantime
* another thread starts uninstalling it. The second thread has to wait
* with uninstalling until the first one finishes using it.
*/
val sync = new TestSynchronizer
val engineVersion = SemVer(0, 0, 1)
sync.startThread("t1") {
val componentsManager = makeComponentsManager(
releaseCallback = _ => (),
lockWaitsCallback = resource =>
throw new IllegalStateException(
s"t1 should not be waiting on $resource."
)
)
componentsManager.withEngine(engineVersion) { _ =>
sync.report("t1-start-using")
sync.signal("t1-start-using")
sync.waitFor("t2-waits-with-uninstall")
sync.report("t1-end-using")
}
}
sync.waitFor("t1-start-using")
sync.startThread("t2") {
val componentsManager = makeComponentsManager(
releaseCallback = asset =>
throw new IllegalStateException(s"t2 should not download $asset."),
lockWaitsCallback = {
case resource if resource.startsWith("engine") =>
sync.report("t2-start-uninstall-and-wait")
sync.signal("t2-waits-with-uninstall")
case resource =>
throw new IllegalStateException(s"Unexpected wait on $resource.")
}
)
componentsManager.uninstallEngine(engineVersion)
sync.report("t2-uninstalled")
}
sync.join()
sync.summarizeReports() shouldEqual Seq(
"t1-start-using",
"t2-start-uninstall-and-wait",
"t1-end-using",
"t2-uninstalled"
)
}
"synchronize main lock" in {
/**
* First two threads start and acquire the shared lock, than the third
* thread tries to acquire an exclusive lock (in practice that will be our
* (un)installer), it should wait for the other threads to finish. When
* the threads see that it started waiting (the waiting notification is
* normally used to tell the user what the application is waiting for),
* the two threads finish and after that the third one is able to acquire
* the exclusive lock.
*/
val sync = new TestSynchronizer
sync.startThread("t1") {
val resourceManager = makeNewResourceManager()
resourceManager.initializeMainLock()
sync.report("shared-start")
sync.signal("started-1")
sync.waitFor("finish-1")
sync.report("shared-end")
resourceManager.releaseMainLock()
}
sync.startThread("t2") {
val resourceManager = makeNewResourceManager()
resourceManager.initializeMainLock()
sync.report("shared-start")
sync.signal("started-2")
sync.waitFor("finish-2")
sync.report("shared-end")
resourceManager.releaseMainLock()
}
sync.waitFor("started-1")
sync.waitFor("started-2")
sync.startThread("t3") {
val resourceManager = makeNewResourceManager()
resourceManager.initializeMainLock()
sync.report("t3-start")
resourceManager.acquireExclusiveMainLock(() => {
sync.report("t3-wait")
sync.signal("waiting")
})
sync.report("t3-end")
sync.signal("finish-all")
resourceManager.releaseMainLock()
}
sync.waitFor("waiting")
Thread.sleep(1000)
sync.signal("finish-1")
sync.signal("finish-2")
sync.waitFor("finish-all")
sync.join()
sync.summarizeReports() shouldEqual Seq(
"shared-start",
"shared-start",
"t3-start",
"t3-wait",
"shared-end",
"shared-end",
"t3-end"
)
}
}
}

64
engine/launcher/src/test/scala/org/enso/launcher/locking/TestLocalLockManager.scala Normal file
View File

@ -0,0 +1,64 @@
package org.enso.launcher.locking
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.{
ReadWriteLock,
ReentrantReadWriteLock,
Lock => JLock
}
/**
* A [[LockManager]] that creates process-local locks.
*
* The locks are not visible by other processes, so this manager is not useful
* for synchronizing multiple processes. It can be used to test concurrency
* implementation using threads within the same JVM.
*
* Normally [[acquireLock]] would wait forever until the lock can be acquired
* or the thread is interrupted. To aid with testing, this implementation times
* out after 30 seconds.
*/
class TestLocalLockManager extends LockManager {
/**
* @inheritdoc
*/
override def acquireLock(resourceName: String, lockType: LockType): Lock = {
val lock = getLock(resourceName, lockType)
val locked = lock.tryLock(30, TimeUnit.SECONDS)
if (!locked) {
throw new RuntimeException(
"Acquiring the lock took more than 30s, perhaps a deadlock?"
)
}
WrapLock(lock)
}
/**
* @inheritdoc
*/
override def tryAcquireLock(
resourceName: String,
lockType: LockType
): Option[Lock] = {
val lock = getLock(resourceName, lockType)
if (lock.tryLock()) Some(WrapLock(lock))
else None
}
case class WrapLock(lock: JLock) extends Lock {
override def release(): Unit = lock.unlock()
}
private val locks: collection.concurrent.Map[String, ReadWriteLock] =
collection.concurrent.TrieMap()
private def getLock(resourceName: String, lockType: LockType): JLock = {
val rwLock =
locks.getOrElseUpdate(resourceName, new ReentrantReadWriteLock(true))
lockType match {
case LockType.Exclusive => rwLock.writeLock()
case LockType.Shared => rwLock.readLock()
}
}
}

22
engine/launcher/src/test/scala/org/enso/launcher/locking/TestLocalResourceManager.scala Normal file
View File

@ -0,0 +1,22 @@
package org.enso.launcher.locking
object TestLocalResourceManager {
/**
* Creates a [[ResourceManager]] that manages resource access between threads
* of a single process.
*
* The resource locks are not visible by other processes, so this manager is
* not useful for synchronizing multiple processes. It can be used to test
* concurrency implementation using threads within the same JVM, in contrary
* to the default [[FileLockManager]] which can be only used for
* inter-process synchronization.
*
* The [[ResourceManager]] created using that method uses an independent lock
* manager that is not shared with other resource managers. It is meant to be
* used for non-concurrent tests. In concurrent tests, each used
* [[ResourceManager]] should share their [[TestLocalLockManager]] so that
* all the threads see each other's locks.
*/
def create(): ResourceManager = new ResourceManager(new TestLocalLockManager)
}

152
engine/launcher/src/test/scala/org/enso/launcher/locking/TestSynchronizer.scala Normal file
View File

@ -0,0 +1,152 @@
package org.enso.launcher.locking
import java.util.concurrent.{Semaphore, TimeUnit}
import org.scalatest.exceptions.TestFailedException
import scala.jdk.CollectionConverters._
/**
* A helper class that can be used to synchronize actions between multiple
* threads to create interleaving scenarios useful in testing concurrent
* interaction.
*
* To avoid stalling the tests, all blocking actions can time out after a
* specified time.
*/
class TestSynchronizer {
/**
* If enabled, prints additional debug messages describing when events are
* signalled or waited for.
*
* Can be enabled to aid with debugging, but should be disabled by default.
*/
val enableDebugOutput: Boolean = false
/**
* Executes the `action` in a separate thread with the given `name`
*/
def startThread(name: String)(action: => Unit): Unit = {
val thread = new Thread(() => action, name)
threads ::= thread
thread.setUncaughtExceptionHandler(reportException)
thread.start()
}
/**
* Waits for a signal indicating that an event with the specified name has
* happened.
*
* Will return immediately if the event has happened prior to calling that
* function.
*
* Each `waitFor` call corresponds to a single [[signal]] call, so waiting
* twice on the same event name will need two signals to wake up.
*/
def waitFor(event: String): Unit = {
if (enableDebugOutput) {
System.err.println(s"$threadName waiting for $event.")
System.err.flush()
}
val acquired =
getSemaphore(event).tryAcquire(timeOutSeconds, TimeUnit.SECONDS)
if (!acquired) {
throw new RuntimeException(s"$threadName waitFor($event) has timed out.")
}
if (enableDebugOutput) {
System.err.println(s"$threadName has been woken up by $event.")
System.err.flush()
}
}
/**
* Signals that an event has happened.
*
* Will wake up the thread waiting for it. Only one waiting thread is woken
* up, so if multiple threads wait for the same event, it must be signalled
* multiple times. It is advised to not make multiple threads wait on a
* single event.
*/
def signal(event: String): Unit = {
if (enableDebugOutput) {
System.err.println(s"$threadName signalling $event.")
System.err.flush()
}
getSemaphore(event).release()
}
/**
* Timeout used by [[waitFor]].
*/
val timeOutSeconds: Long = 10
/**
* Reports that the event has happened now.
*
* Can be used to test for order of events. Independent of [[waitFor]] and
* [[signal]].
*/
def report(event: String): Unit = {
if (enableDebugOutput) {
System.err.println(s"$threadName reporting $event.")
System.err.flush()
}
reports.add(event)
}
/**
* Returns names of events reported using [[report]] in the order that they
* were reported.
*
* Should only be called *after* [[join]].
*/
def summarizeReports(): Seq[String] = reports.asScala.toSeq
private def reportException(thread: Thread, throwable: Throwable): Unit = {
System.err.println(s"${thread.getName} got an exception: $throwable")
throwable.printStackTrace()
hadException = true
}
/**
* Waits for all threads started with [[startThread]] to finish execution.
*
* Reports any exceptions thrown inside of the threads or if any of the
* threads has timed out.
*
* @param joinTimeOutSeconds timeout used when waiting for each thread
*/
def join(joinTimeOutSeconds: Long = timeOutSeconds): Unit = {
var hadTimeout = false
for (thread <- threads.reverse) {
thread.join(joinTimeOutSeconds * 1000)
if (thread.isAlive) {
System.err.println(s"Thread ${thread.getName} timed out.")
thread.interrupt()
hadTimeout = true
}
}
if (hadException) {
throw new TestFailedException(
"One of the threads caught an exception.",
1
)
}
if (hadTimeout) {
throw new TestFailedException("One of the threads has timed out.", 1)
}
}
private def threadName: String = Thread.currentThread.getName
private val reports = new java.util.concurrent.LinkedTransferQueue[String]
private var threads: List[Thread] = Nil
@volatile private var hadException: Boolean = false
private val semaphores = collection.concurrent.TrieMap[String, Semaphore]()
private def getSemaphore(event: String): Semaphore =
semaphores.getOrElseUpdate(event, new Semaphore(0, true))
}

6
engine/launcher/src/test/scala/org/enso/launcher/project/ProjectManagerSpec.scala
View File

@ -4,6 +4,7 @@ import nl.gn0s1s.bump.SemVer
import org.enso.launcher.components.ComponentsManagerTest
import org.enso.launcher.config.GlobalConfigurationManager
import org.enso.launcher.installation.DistributionManager
import org.enso.launcher.locking.TestLocalResourceManager
import org.enso.pkg.Contact
import org.scalatest.{Inside, OptionValues}
@ -13,8 +14,9 @@ class ProjectManagerSpec
with OptionValues {
private val defaultEnsoVersion = SemVer(0, 0, 0, Some("default"))
def makeProjectManager(): (GlobalConfigurationManager, ProjectManager) = {
val env = fakeInstalledEnvironment()
val distributionManager = new DistributionManager(env)
val env = fakeInstalledEnvironment()
val distributionManager =
new DistributionManager(env, TestLocalResourceManager.create())
val fakeConfigurationManager =
new GlobalConfigurationManager(null, distributionManager) {
override def defaultVersion: SemVer = defaultEnsoVersion

90
engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala
View File

@ -6,6 +6,7 @@ import io.circe.parser
import nl.gn0s1s.bump.SemVer
import org.enso.launcher.FileSystem.PathSyntax
import org.enso.launcher._
import org.enso.launcher.locking.{FileLockManager, LockType}
import org.scalatest.exceptions.TestFailedException
import org.scalatest.{BeforeAndAfterAll, OptionValues}
@ -129,10 +130,22 @@ class UpgradeSpec
* @param extraEnv environment variable overrides
* @return result of the run
*/
private def run(
def run(
args: Seq[String],
extraEnv: Map[String, String] = Map.empty
): RunResult = {
): RunResult = startLauncher(args, extraEnv).join(timeoutSeconds = 20)
/**
* Starts the launcher in the temporary distribution.
*
* @param args arguments for the launcher
* @param extraEnv environment variable overrides
* @return wrapped process
*/
def startLauncher(
args: Seq[String],
extraEnv: Map[String, String] = Map.empty
): WrappedProcess = {
val testArgs = Seq(
"--internal-emulate-repository",
fakeReleaseRoot.toAbsolutePath.toString,
@ -141,7 +154,10 @@ class UpgradeSpec
)
val env =
extraEnv.updated("ENSO_LAUNCHER_LOCATION", realLauncherLocation.toString)
runLauncherAt(launcherPath, testArgs ++ args, env)
start(
Seq(launcherPath.toAbsolutePath.toString) ++ testArgs ++ args,
env.toSeq
)
}
"upgrade" should {
@ -192,13 +208,12 @@ class UpgradeSpec
val configRoot = getTestDirectory / "config"
checkVersion() shouldEqual SemVer(0, 0, 0)
val env = Map(
"ENSO_DATA_DIRECTORY" -> dataRoot.toString,
"ENSO_CONFIG_DIRECTORY" -> configRoot.toString
"ENSO_DATA_DIRECTORY" -> dataRoot.toString,
"ENSO_CONFIG_DIRECTORY" -> configRoot.toString,
"ENSO_RUNTIME_DIRECTORY" -> (getTestDirectory / "run").toString
)
run(
Seq("upgrade", "0.0.1"),
extraEnv = env
) should returnSuccess
run(Seq("upgrade", "0.0.1"), extraEnv = env) should returnSuccess
checkVersion() shouldEqual SemVer(0, 0, 1)
TestHelpers
.readFileContent(dataRoot / "README.md")
@ -218,7 +233,11 @@ class UpgradeSpec
)
checkVersion() shouldEqual SemVer(0, 0, 0)
run(Seq("upgrade", "0.0.3")) should returnSuccess
// run(Seq("upgrade", "0.0.3")) should returnSuccess
val proc = startLauncher(Seq("upgrade", "0.0.3"))
proc.printIO()
proc.join(timeoutSeconds = 20) should returnSuccess
checkVersion() shouldEqual SemVer(0, 0, 3)
val launchedVersions = Seq(
@ -236,6 +255,19 @@ class UpgradeSpec
.toSeq
reportedLaunchLog shouldEqual launchedVersions
withClue(
"After the update we run the version check, running the launcher " +
"after the update should ensure no leftover temporary executables " +
"are left in the bin directory."
) {
val binDirectory = launcherPath.getParent
val leftOverExecutables = FileSystem
.listDirectory(binDirectory)
.map(_.getFileName.toString)
.filter(_.startsWith("enso"))
leftOverExecutables shouldEqual Seq(OS.executableName("enso"))
}
}
"automatically trigger if an action requires a newer version and re-run " +
@ -275,5 +307,43 @@ class UpgradeSpec
result should returnSuccess
result.stdout should include(message)
}
"fail if another upgrade is running in parallel" in {
prepareDistribution(
portable = true,
launcherVersion = Some(SemVer(0, 0, 1))
)
val syncLocker = new FileLockManager {
override def locksRoot: Path = getTestDirectory / "enso" / "lock"
}
val launcherManifestAssetName = "launcher-manifest.yaml"
// The fake release tries to acquire a shared lock on each accessed file,
// so acquiring this exclusive lock will stall access to that file until
// the exclusive lock is released
val lock = syncLocker.acquireLock(
"testasset-" + launcherManifestAssetName,
LockType.Exclusive
)
val firstSuspended = startLauncher(
Seq("upgrade", "0.0.2", "--internal-emulate-repository-wait")
)
try {
firstSuspended.waitForMessageOnErrorStream(
"INTERNAL-TEST-ACQUIRING-LOCK"
)
val secondFailed = run(Seq("upgrade", "0.0.0"))
secondFailed.stderr should include("Another upgrade is in progress")
secondFailed.exitCode shouldEqual 1
} finally {
lock.release()
}
firstSuspended.join(timeoutSeconds = 20) should returnSuccess
checkVersion() shouldEqual SemVer(0, 0, 2)
}
}
}

20
lib/scala/cli/src/main/scala/org/enso/cli/arguments/Application.scala
View File

@ -46,7 +46,7 @@ class Application[Config](
val prettyName: String,
val helpHeader: String,
val topLevelOpts: Opts[() => TopLevelBehavior[Config]],
val commands: NonEmptyList[Command[Config => Unit]],
val commands: NonEmptyList[Command[Config => Int]],
val pluginManager: Option[PluginManager]
) {
@ -63,7 +63,7 @@ class Application[Config](
*/
def run(
args: Array[String]
): Either[List[String], Unit] = run(args.toSeq)
): Either[List[String], Int] = run(args.toSeq)
/**
* Runs the application logic. Parses the top level options and depending on
@ -73,11 +73,11 @@ class Application[Config](
* with the exit code returned by the plugin).
*
* @return either a list of errors encountered when parsing the options or
* [[Unit]] if it succeeded
* the exit code if succeeded
*/
def run(
args: Seq[String]
): Either[List[String], Unit] = {
): Either[List[String], Int] = {
val (tokens, additionalArguments) = Parser.tokenize(args)
val parseResult =
Parser.parseOpts(
@ -90,12 +90,12 @@ class Application[Config](
case ((topLevelAction, commandResult), pluginIntercepted) =>
pluginIntercepted match {
case Some(pluginHandler) =>
pluginHandler()
Right(pluginHandler())
case None =>
val topLevelBehavior = topLevelAction()
topLevelBehavior match {
case TopLevelBehavior.Halt =>
Right(())
case TopLevelBehavior.Halt(exitCode) =>
Right(exitCode)
case TopLevelBehavior.Continue(config) =>
commandResult match {
case Some(action) =>
@ -138,7 +138,7 @@ object Application {
prettyName: String,
helpHeader: String,
topLevelOpts: Opts[() => TopLevelBehavior[Config]],
commands: NonEmptyList[Command[Config => Unit]],
commands: NonEmptyList[Command[Config => Int]],
pluginManager: PluginManager
): Application[Config] =
new Application(
@ -170,7 +170,7 @@ object Application {
prettyName: String,
helpHeader: String,
topLevelOpts: Opts[() => TopLevelBehavior[Config]],
commands: NonEmptyList[Command[Config => Unit]]
commands: NonEmptyList[Command[Config => Int]]
): Application[Config] =
new Application(
commandName,
@ -197,7 +197,7 @@ object Application {
commandName: String,
prettyName: String,
helpHeader: String,
commands: NonEmptyList[Command[Unit => Unit]]
commands: NonEmptyList[Command[Unit => Int]]
): Application[()] =
new Application(
commandName,

3
lib/scala/cli/src/main/scala/org/enso/cli/arguments/PluginManager.scala
View File

@ -13,8 +13,9 @@ trait PluginManager {
*
* @param name name of the plugin
* @param args arguments that should be passed to it
* @return exit code of the launched plugin
*/
def runPlugin(name: String, args: Seq[String]): Nothing
def runPlugin(name: String, args: Seq[String]): Int
/**
* Returns whether the plugin of the given `name` is available in the system.

4
lib/scala/cli/src/main/scala/org/enso/cli/arguments/TopLevelBehavior.scala
View File

@ -25,6 +25,8 @@ object TopLevelBehavior {
* top-level options have handled the execution and commands should not be
* parsed further. This can be useful to implement top-level options like
* `--version`.
*
* @param exitCode exit code that should be returned when halting
*/
case object Halt extends TopLevelBehavior[Nothing]
case class Halt(exitCode: Int) extends TopLevelBehavior[Nothing]
}

6
lib/scala/cli/src/main/scala/org/enso/cli/internal/Parser.scala
View File

@ -22,7 +22,7 @@ object Parser {
tokens: Seq[Token],
additionalArguments: Seq[String],
applicationName: String
): Either[OptsParseError, (A, Option[() => Nothing])] = {
): Either[OptsParseError, (A, Option[() => Int])] = {
var parseErrors: List[String] = Nil
def addError(error: String): Unit = {
parseErrors = error :: parseErrors
@ -82,8 +82,8 @@ object Parser {
opts.reset()
val tokenProvider = new TokenStream(tokens, addError)
var escapeParsing: Option[(Seq[Token], Seq[String]) => Nothing] = None
var parsingStopped: Boolean = false
var escapeParsing: Option[(Seq[Token], Seq[String]) => Int] = None
var parsingStopped: Boolean = false
while (!parsingStopped && tokenProvider.hasTokens) {
tokenProvider.consumeToken() match {

2
lib/scala/cli/src/main/scala/org/enso/cli/internal/ParserContinuation.scala
View File

@ -37,6 +37,6 @@ object ParserContinuation {
* be invoked.
*/
case class Escape(
continuation: (Seq[Token], Seq[String]) => Nothing
continuation: (Seq[Token], Seq[String]) => Int
) extends ParserContinuation
}

10
lib/scala/cli/src/main/scala/org/enso/cli/internal/opts/TopLevelCommandsOpt.scala
View File

@ -21,10 +21,10 @@ import org.enso.cli.internal.{Parser, ParserContinuation}
*/
class TopLevelCommandsOpt[A, B](
toplevelOpts: Opts[A],
commands: NonEmptyList[Command[B => Unit]],
commands: NonEmptyList[Command[B => Int]],
pluginManager: Option[PluginManager],
helpHeader: String
) extends BaseSubcommandOpt[(A, Option[B => Unit]), B => Unit] {
) extends BaseSubcommandOpt[(A, Option[B => Int]), B => Int] {
private def helpOpt: Opts[Boolean] =
Opts.flag("help", 'h', "Print this help message.", showInUsage = true)
@ -38,7 +38,7 @@ class TopLevelCommandsOpt[A, B](
/**
* @inheritdoc
*/
override def availableSubcommands: NonEmptyList[Command[B => Unit]] = commands
override def availableSubcommands: NonEmptyList[Command[B => Int]] = commands
/**
* Handles an unknown command.
@ -64,7 +64,7 @@ class TopLevelCommandsOpt[A, B](
override private[cli] def result(
commandPrefix: Seq[String]
): Either[OptsParseError, (A, Option[B => Unit])] = {
): Either[OptsParseError, (A, Option[B => Int])] = {
val prefix = extendPrefix(commandPrefix)
val topLevelResultWithHelp = toplevelWithHelp.result(prefix)
val topLevelResult = topLevelResultWithHelp.map(_._1)
@ -76,7 +76,7 @@ class TopLevelCommandsOpt[A, B](
CLIOutput.println(helpText)
}
Right((result, Some((_: B) => displayHelp())))
Right((result, Some((_: B) => { displayHelp(); 0 })))
case (_, Some(value)) =>
OptsParseError.product(topLevelResult, value.map(Some(_)))
case (_, None) =>

39
lib/scala/cli/src/test/scala/org/enso/cli/ApplicationSpec.scala
View File

@ -38,11 +38,13 @@ class ApplicationSpec
Command("cmd1", "cmd1") {
Opts.pure { _ =>
ranCommand = Some("cmd1")
0
}
},
Command("cmd2", "cmd2") {
Opts.positionalArgument[String]("arg") map { arg => _ =>
ranCommand = Some(arg)
0
}
}
)
@ -61,7 +63,7 @@ class ApplicationSpec
val plugins = Seq("plugin1", "plugin2")
val pluginManager = new PluginManager {
override def runPlugin(name: String, args: Seq[String]): Nothing =
override def runPlugin(name: String, args: Seq[String]): Int =
if (plugins.contains(name))
throw PluginRanException(name, args)
else throw new RuntimeException("Plugin not found.")
@ -82,8 +84,8 @@ class ApplicationSpec
TopLevelBehavior.Continue(())
},
NonEmptyList.of(
Command[Unit => Unit]("cmd", "cmd") {
Opts.pure { _ => () }
Command[Unit => Int]("cmd", "cmd") {
Opts.pure { _ => 0 }
}
),
pluginManager
@ -113,34 +115,31 @@ class ApplicationSpec
Opts.optionalParameter[String]("setting", "setting", "setting")
(halt, setting) mapN { (halt, setting) => () =>
if (halt)
TopLevelBehavior.Halt
TopLevelBehavior.Halt(10)
else TopLevelBehavior.Continue(setting.getOrElse("none"))
}
},
NonEmptyList.of(
Command[String => Unit]("cmd1", "cmd1") {
Command[String => Int]("cmd1", "cmd1") {
Opts.pure { setting =>
ranCommand = Some(setting)
42
}
}
)
)
assert(
app.run(Seq("--halt", "cmd1")).isRight,
"Should parse successfully."
)
app.run(Seq("--halt", "cmd1")) shouldEqual Right(10)
app.run(Seq("cmd1", "--halt")) shouldEqual Right(10)
ranCommand should not be defined
assert(app.run(Seq("cmd1")).isRight, "Should parse successfully.")
app.run(Seq("cmd1")) shouldEqual Right(42)
ranCommand.value shouldEqual "none"
withClue("top-level option before command:") {
ranCommand = None
assert(
app.run(Seq("--setting=SET", "cmd1")).isRight,
"Should parse successfully."
)
app.run(Seq("--setting=SET", "cmd1")) shouldEqual Right(42)
ranCommand.value shouldEqual "SET"
}
@ -161,7 +160,7 @@ class ApplicationSpec
"Test app.",
NonEmptyList.of(
Command("cmd", "cmd", related = Seq("related")) {
Opts.pure { _ => }
Opts.pure { _ => 0 }
}
)
)
@ -181,11 +180,13 @@ class ApplicationSpec
Command("cmd1", "cmd1") {
Opts.pure { _ =>
ranCommand = Some("cmd1")
0
}
},
Command("cmd2", "cmd2") {
Opts.positionalArgument[String]("arg") map { arg => _ =>
ranCommand = Some(arg)
0
}
}
)
@ -197,13 +198,13 @@ class ApplicationSpec
}
def appWithSubcommands(): Application[_] = {
val sub1 = Command[Boolean => Unit]("sub1", "Sub1.") {
val sub1 = Command[Boolean => Int]("sub1", "Sub1.") {
val flag = Opts.flag("inner-flag", "Inner.", showInUsage = true)
flag map { _ => _ => () }
flag map { _ => _ => 0 }
}
val sub2 = Command[Boolean => Unit]("sub2", "Sub2.") {
val sub2 = Command[Boolean => Int]("sub2", "Sub2.") {
val arg = Opts.optionalArgument[String]("ARG")
arg map { _ => _ => () }
arg map { _ => _ => 0 }
}
val topLevelOpts =
Opts.flag("toplevel-flag", "Top.", showInUsage = true) map {