From 6a6d7dbff364af9546dce96590056a663887c55f Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 17 Jul 2023 18:38:54 +0100 Subject: [PATCH] Simplify onboarding instructions for engine & project-manager developers (#7181) The current instructions to _build, use and debug_ `project-manager` and its engine/ls process are complicated and require a lot of symlinks to properly point to each other. This pull requests simplifies all of that by introduction of `ENSO_ENGINE_PATH` and `ENSO_JVM_PATH` environment variables. Then it hides all the complexity behind a simple _sbt command_: `runProjectManagerDistribution --debug`. # Important Notes I decided to tackle this problem as I have three repositories with different branches of Enso and switching between them requires me to mangle the symlinks. I hope I will not need to do that anymore with the introduction of the `runProjectManagerDistribution` command. --- build.sbt | 18 ++- docs/CONTRIBUTING.md | 137 ++++++------------ .../distribution/DefaultManagers.scala | 1 + .../DefaultDistributionConfiguration.scala | 1 + .../TestDistributionConfiguration.scala | 1 + .../test/RuntimeVersionManagerTest.scala | 1 + .../locking/ConcurrencyTest.scala | 1 + .../components/RuntimeVersionManager.scala | 40 ++++- .../runtimeversionmanager/runner/Runner.scala | 16 +- project/DistributionPackage.scala | 30 ++++ 10 files changed, 146 insertions(+), 100 deletions(-) diff --git a/build.sbt b/build.sbt index 3f7f8f9d322..6db9a293188 100644 --- a/build.sbt +++ b/build.sbt @@ -2223,7 +2223,7 @@ buildEngineDistributionNoIndex := { } lazy val runEngineDistribution = - inputKey[Unit]("Run the engine distribution with arguments") + inputKey[Unit]("Run or --debug the engine distribution with arguments") runEngineDistribution := { buildEngineDistribution.value val args: Seq[String] = spaceDelimited("").parsed @@ -2234,6 +2234,22 @@ runEngineDistribution := { ) } +lazy val runProjectManagerDistribution = + inputKey[Unit]( + "Run or --debug the project manager distribution with arguments" + ) +runProjectManagerDistribution := { + buildEngineDistribution.value + buildProjectManagerDistribution.value + val args: Seq[String] = spaceDelimited("").parsed + DistributionPackage.runProjectManagerPackage( + engineDistributionRoot.value, + projectManagerDistributionRoot.value, + args, + streams.value.log + ) +} + val allStdBitsSuffix = List("All", "AllWithIndex") val stdBitsProjects = List( diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 2cab62235ff..57459aef273 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -669,125 +669,82 @@ Hello, World! You can start [IDE](https://github.com/enso-org/enso/tree/develop/gui) with a development version of the language server. IDE executable has `--external-backend` flag that switches off the bundled backend. That requires -you to run the project manager process yourself. You can either get a project -manager from one of the latest releases on -[GitHub](https://github.com/enso-org/enso/releases), or build one using SBT -`buildProjectManagerDistribution` command. - -Running development version of the IDE is possible via the `./run` script in the -root of the repository: +you to run the project manager process yourself. Running development version of +the IDE is also possible via the `./run` script in the root of the repository: ```bash -$ ./run ide start --wasm-profile dev --external-backend +enso$ ./run gui watch --skip-wasm-opt ``` -##### Bash +To build the `project-manager` one needs to launch `sbt` - one way to do it is +to execute `./run backend sbt`. When in the _sbt prompt_ one can request +compilation of the `project-manager`: ```bash -sbt buildProjectManagerDistribution -``` - -##### PowerShell - -```powershell -sbt.bat buildProjectManagerDistribution +sbt:enso> buildProjectManagerDistribution ``` When the command is completed, a development version of the project manager will have appeared in the `built-distribution` directory. -The IDE will connect to the running project manager to look up the project and -start the language server. The required version of the language server is -specified in the `edition` field of the `package.yaml` project description. Enso -projects are located in the `~/enso` directory on Unix and `%userprofile%\enso` -on Windows systems by default. +Project manager is there to wait for the IDE to connect to it and then launch +the engine with its embedded language server. To build the engine issue +following command in the _sbt prompt_: ```bash -cat ~/enso/projects/Unnamed/package.yaml +sbt:enso> buildEngineDistribution ``` -```yaml -name: Unnamed -namespace: local -version: 0.0.1 -license: "" -authors: [] -maintainers: [] -edition: "2021.20-SNAPSHOT" -prefer-local-libraries: true -``` +Once all the components are assembled, it is time to execute them in +orchestration. One can pass following environment variables to +`project-manager`: -We need to set `edition` to a value that will represent the development version. -It should be different from any Enso versions that have already been released. -In this case, we chose the `2021.20-SNAPSHOT` (the current development edition). -The project manager will look for the appropriate subdirectory in the _engines_ -directory of the distribution folder. Distribution paths are printed when you -run project manager with `-v` verbose logging. +- `ENSO_JVM_OPTS` to for example turn + [debugging of the Engine runtime](debugger/README.md) on +- `ENSO_JVM_PATH` to force a fixed GraalVM to execute the engine/language server + process on +- `ENSO_ENGINE_PATH` the path to engine/language server as created by + `buildEngineDistribution`, usually + `/built-distribution/enso-engine-0.0.0-dev--/enso-0.0.0-dev/` -Btw. you can specify `ENSO_JVM_OPTS` to turn -[debugging of the Engine runtime](debugger/README.md) on: +One doesn't need to deal with these options directly, there is an _sbt command_ +to orchestrate them all: ```bash -$ export ENSO_JVM_OPTS=-agentlib:jdwp=transport=dt_socket,address=5005 -$ ./built-distribution/enso-project-manager-0.0.0-dev-linux-amd64/enso/bin/project-manager --no-log-masking -v -[info] [2021-06-16T11:49:33.639Z] [org.enso.projectmanager.boot.ProjectManager$] Starting Project Manager... -[debug] [2021-06-16T11:49:33.639Z] [org.enso.runtimeversionmanager.distribution.DistributionManager] Detected paths: DistributionPaths( - dataRoot = /home/dbv/.local/share/enso, - runtimes = /home/dbv/.local/share/enso/runtime, - engines = /home/dbv/.local/share/enso/dist, - bundle = None, - config = /home/dbv/.config/enso, - locks = /run/user/1000/enso/lock, - tmp = /home/dbv/.local/share/enso/tmp -) +sbt:enso> runProjectManagerDistribution ``` -On Linux it looks for the `~/.local/share/enso/dist/0.2.32-SNAPSHOT/` directory. +The above command invokes `buildProjectManagerDistribution`, +`buildEngineDistribution` and then defines `ENSO_ENGINE_PATH` to connect them +together and also specifies the `ENSO_JVM_PATH` to the JVM `sbt` process runs +on. -We can build an engine distribution using the `buildEngineDistribution` command -in SBT. - -##### Bash +There also is a simple way to [debug](debugger/README.md). When adding `--debug` +option to the _sbt command_: ```bash -sbt buildEngineDistribution +sbt:enso> runProjectManagerDistribution --debug ``` -##### PowerShell - -```powershell -sbt.bat buildEngineDistribution -``` - -And copy the result to the `0.2.32-SNAPSHOT` engines directory of the -distribution folder. - -##### Bash - -```bash -cp -r built-distribution/enso-engine-0.2.32-SNAPSHOT-linux-amd64/enso-0.2.32-SNAPSHOT ~/.local/share/enso/dist/0.2.32-SNAPSHOT -``` - -##### PowerShell - -```powershell -cp -r built-distribution/enso-engine-0.2.32-SNAPSHOT-linux-amd64/enso-0.2.32-SNAPSHOT ~/.local/share/enso/dist/0.2.32-SNAPSHOT -``` - -Now, when the project manager is running and the engines directory contains the -required engine version, you can start IDE with the `--no-backend` flag. It will -pick up the development version of the language server we just prepared. +the system also sets +`ENSO_JVM_OPTS=-agentlib:jdwp=transport=dt_socket,address=5005`. Just +[configure your Java IDE](debugger/README.md) to listen on port 5005 before +invoking the command and you'll be able to debug the engine launched by the +project manager. To summarize, these are the steps required to run IDE with the development -version of the language server. +version of the language server: -1. Run the project manager process. -2. Copy or symlink the development version of the engine created with SBT's - `buildEnginedistribution` command to the engines directory of the Enso - distribution folder. -3. Set the `edition` field of the `package.yaml` project definition to the - version that you created in the previous step. -4. Run the IDE with `--no-backend` flag. +```bash +enso$ ./run gui watch --skip-wasm-opt +``` + +together with that also (after launching `./run backend sbt`) following _sbt +command_: + +```bash +sbt:enso> runProjectManagerDistribution +``` #### Language Server Mode diff --git a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala index 1c4ca678eab..d23063c813b 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala @@ -49,6 +49,7 @@ object DefaultManagers { alwaysInstallMissing: Boolean ): RuntimeVersionManager = new RuntimeVersionManager( + LauncherEnvironment, new CLIRuntimeVersionManagementUserInterface( globalCLIOptions, alwaysInstallMissing diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala index 66377815573..b4fd9c79e0f 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala @@ -67,6 +67,7 @@ object DefaultDistributionConfiguration userInterface: RuntimeVersionManagementUserInterface ): RuntimeVersionManager = new RuntimeVersionManager( + environment = this.environment, userInterface = userInterface, distributionManager = distributionManager, temporaryDirectoryManager = temporaryDirectoryManager, diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala index 258f5f87987..d47fcc7bf9e 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala @@ -76,6 +76,7 @@ class TestDistributionConfiguration( override def makeRuntimeVersionManager( userInterface: RuntimeVersionManagementUserInterface ): RuntimeVersionManager = new RuntimeVersionManager( + environment = environment, userInterface = userInterface, distributionManager = distributionManager, temporaryDirectoryManager = temporaryDirectoryManager, diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala index 25a8214fa5f..d2527854bd0 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala @@ -58,6 +58,7 @@ class RuntimeVersionManagerTest val componentConfig = new GraalVMComponentConfiguration val runtimeVersionManager = new RuntimeVersionManager( + env, userInterface, distributionManager, temporaryDirectoryManager, diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala index aa59dbf12d4..bdb4076539c 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala @@ -144,6 +144,7 @@ class ConcurrencyTest TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val componentsManager = new RuntimeVersionManager( + env, TestRuntimeVersionManagementUserInterface.default, distributionManager, temporaryDirectoryManager, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala index 5aaa2654eee..35caeadb570 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala @@ -6,6 +6,7 @@ import nl.gn0s1s.bump.SemVer import org.enso.cli.OS import org.enso.distribution.{ DistributionManager, + Environment, FileSystem, TemporaryDirectoryManager } @@ -38,6 +39,7 @@ import scala.util.{Failure, Success, Try, Using} * @param componentUpdaterFactory the runtime component updater factory */ class RuntimeVersionManager( + environment: Environment, userInterface: RuntimeVersionManagementUserInterface, distributionManager: DistributionManager, temporaryDirectoryManager: TemporaryDirectoryManager, @@ -63,10 +65,16 @@ class RuntimeVersionManager( * Returns None if that version is not installed. */ def findGraalRuntime(version: GraalVMVersion): Option[GraalRuntime] = { - val name = graalRuntimeNameForVersion(version) - val graalRuntimeOpt = - firstExisting(distributionManager.paths.runtimeSearchPaths.map(_ / name)) - .map { path => + val explicitPathOpt = this.environment.getEnvPath("ENSO_JVM_PATH") + val graalRuntimeOpt = explicitPathOpt + .map(path => { + val runtime = GraalRuntime(version, path) + runtime.ensureValid() + runtime + }) + .orElse { + val pathOpt = findGraalRuntimeOnSearchPath(version) + pathOpt.map { path => // TODO [RW] for now an exception is thrown if the installation is // corrupted, in #1052 offer to repair the broken installation loadGraalRuntime(path).recoverWith { case e: Exception => @@ -82,6 +90,7 @@ class RuntimeVersionManager( ) }.get } + } graalRuntimeOpt match { case Some(graalRuntime) => logger.info("Found GraalVM runtime [{}].", graalRuntime) @@ -91,6 +100,13 @@ class RuntimeVersionManager( graalRuntimeOpt } + private def findGraalRuntimeOnSearchPath( + version: GraalVMVersion + ): Option[Path] = { + val name = graalRuntimeNameForVersion(version) + firstExisting(distributionManager.paths.runtimeSearchPaths.map(_ / name)) + } + /** Executes the provided action with a requested engine version. * * The engine is locked with a shared lock, so it is guaranteed that it will @@ -206,8 +222,20 @@ class RuntimeVersionManager( */ private def getEngine(version: SemVer): Try[Engine] = { val name = engineNameForVersion(version) - firstExisting(distributionManager.paths.engineSearchPaths.map(_ / name)) - .map(loadEngine) + this.environment + .getEnvPath("ENSO_ENGINE_PATH") + .map { p => + logger.info("Using explicit ENSO_ENGINE_PATH: " + p) + val manifest = loadAndCheckEngineManifest(p) + val engine = Engine(version, p, manifest.get) + Success(engine) + } + .orElse { + val f = firstExisting( + distributionManager.paths.engineSearchPaths.map(_ / name) + ) + f.map(loadEngine) + } .getOrElse { Failure(ComponentMissingError(s"Engine $version is not installed.")) } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala index fc775a08ee6..56423ed81e2 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala @@ -130,6 +130,7 @@ class Runner( ) } + final private val JVM_PATH_ENV_VAR = "ENSO_JVM_PATH" final private val JVM_OPTIONS_ENV_VAR = "ENSO_JVM_OPTS" /** Runs an action giving it a command that can be used to launch the @@ -186,10 +187,19 @@ class Runner( val distributionSettings = distributionManager.getEnvironmentToInheritSettings + + val javaHome: Option[String] = environment + .getEnvPath(JVM_PATH_ENV_VAR) + .map { p => + Logger[Runner].info( + "Using explicit " + JVM_PATH_ENV_VAR + " JVM: " + p + ) + p.toString() + } + .orElse(javaCommand.javaHomeOverride) + val extraEnvironmentOverrides = - javaCommand.javaHomeOverride - .map("JAVA_HOME" -> _) - .toSeq ++ distributionSettings.toSeq + javaHome.map("JAVA_HOME" -> _).toSeq ++ distributionSettings.toSeq action(Command(command, extraEnvironmentOverrides)) } diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index 079adbc12eb..ce26e3f2e59 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -278,6 +278,36 @@ object DistributionPackage { exitCode == 0 } + def runProjectManagerPackage( + engineRoot: File, + distributionRoot: File, + args: Seq[String], + log: Logger + ): Boolean = { + import scala.collection.JavaConverters._ + + val enso = distributionRoot / "bin" / "project-manager" + log.info(s"Executing $enso ${args.mkString(" ")}") + val pb = new java.lang.ProcessBuilder() + val all = new java.util.ArrayList[String]() + all.add(enso.getAbsolutePath()) + all.addAll(args.asJava) + pb.command(all) + pb.environment().put("ENSO_ENGINE_PATH", engineRoot.toString()) + pb.environment().put("ENSO_JVM_PATH", System.getProperty("java.home")) + if (args.contains("--debug")) { + all.remove("--debug") + pb.environment().put("ENSO_JVM_OPTS", WithDebugCommand.DEBUG_OPTION) + } + pb.inheritIO() + val p = pb.start() + val exitCode = p.waitFor() + if (exitCode != 0) { + log.warn(enso + " finished with exit code " + exitCode) + } + exitCode == 0 + } + def fixLibraryManifest( packageRoot: File, targetVersion: String,