diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 170bb8f663..faa586e127 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -55,9 +55,9 @@ For example: ``` Enso Compiler and Runtime Version: 0.0.1 -Built with: scala-2.13.1 for GraalVM 20.0.0 -Built from: main @ ac5a9eb639fd8850b1886e4ea2183977ce2fc1fa -Running on: Java HotSpot(TM) 64-Bit Server VM GraalVM EE 20.0.0, JDK 1.8.0_241-b07 - Mac OS X 10.15.3 (x86_64) +Built with: scala-2.13.2 for GraalVM 20.1.0 +Built from: wip/rw/java11-migration* @ 7e161560423c68e3f0fb8337c043156d9d724aac +Running on: OpenJDK 64-Bit Server VM, GraalVM Community, JDK 11.0.7+10-jvmci-20.1-b02 + Linux 4.15.0-108-generic (amd64) ``` --> diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index cc26b56bb2..4052d0cad6 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -9,7 +9,7 @@ on: env: # Please ensure that this is in sync with graalAPIVersion in build.sbt graalVersion: 20.1.0 - javaVersion: java8 + javaVersion: java11 # Please ensure that this is in sync with project/build.properties sbtVersion: 1.3.13 excludedPaths: | @@ -73,6 +73,8 @@ jobs: restore-keys: ${{ runner.os }}-sbt- # Build + - name: Bootstrap Enso project + run: sbt --no-colors bootstrap - name: Build Enso run: sbt --no-colors compile @@ -83,6 +85,8 @@ jobs: run: sbt -no-colors syntax/bench - name: Check Runtime Benchmark Compilation run: sbt -no-colors runtime/Benchmark/compile + - name: Check Language Server Benchmark Compilation + run: sbt -no-colors language-server/Benchmark/compile - name: Build the Uberjar run: sbt -no-colors runner/assembly - name: Test the Uberjar @@ -134,6 +138,8 @@ jobs: restore-keys: ${{ runner.os }}-sbt- # Build Artifacts + - name: Bootstrap the project + run: sbt --no-colors bootstrap - name: Build the Runtime Uberjar run: sbt --no-colors runtime/assembly - name: Build the Runner Uberjar diff --git a/.jvmopts b/.jvmopts index b3ab951984..142c561258 100644 --- a/.jvmopts +++ b/.jvmopts @@ -1,3 +1,4 @@ -Xss16M -Xmx3G --XX:+UseCompressedOops \ No newline at end of file +-XX:+UseCompressedOops +--upgrade-module-path=engine/runtime/build-cache/truffle-api.jar \ No newline at end of file diff --git a/build.sbt b/build.sbt index 3fe57cda59..031ffb7bf0 100644 --- a/build.sbt +++ b/build.sbt @@ -15,6 +15,7 @@ import scala.sys.process._ val scalacVersion = "2.13.3" val graalVersion = "20.1.0" +val javaVersion = "11" val ensoVersion = "0.0.1" organization in ThisBuild := "org.enso" scalaVersion in ThisBuild := scalacVersion @@ -87,6 +88,11 @@ lazy val Benchmark = config("bench") extend sbt.Test lazy val buildNativeImage = taskKey[Unit]("Build native image for the Enso executable") +// Bootstrap task +lazy val bootstrap = + taskKey[Unit]("Prepares Truffle JARs that are required by the sbt JVM") +bootstrap := {} + // ============================================================================ // === Global Project ========================================================= // ============================================================================ @@ -140,12 +146,13 @@ lazy val enso = (project in file(".")) // === Akka =================================================================== -def akkaPkg(name: String) = akkaURL %% s"akka-$name" % akkaVersion -def akkaHTTPPkg(name: String) = akkaURL %% s"akka-$name" % akkaHTTPVersion +def akkaPkg(name: String) = akkaURL %% s"akka-$name" % akkaVersion +def akkaHTTPPkg(name: String) = akkaURL %% s"akka-$name" % akkaHTTPVersion val akkaURL = "com.typesafe.akka" val akkaVersion = "2.6.6" val akkaHTTPVersion = "10.2.0-RC1" val akkaMockSchedulerVersion = "0.5.5" +val slf4jVersion = "1.7.30" val akkaActor = akkaPkg("actor") val akkaStream = akkaPkg("stream") val akkaTyped = akkaPkg("actor-typed") @@ -154,8 +161,16 @@ val akkaSLF4J = akkaPkg("slf4j") val akkaTestkitTyped = akkaPkg("actor-testkit-typed") % Test val akkaHttp = akkaHTTPPkg("http") val akkaSpray = akkaHTTPPkg("http-spray-json") +val slf4jImplementation = "org.slf4j" % "slf4j-simple" % slf4jVersion % Test +val akkaTest = Seq(slf4jImplementation) val akka = - Seq(akkaActor, akkaStream, akkaHttp, akkaSpray, akkaTyped) + Seq( + akkaActor, + akkaStream, + akkaHttp, + akkaSpray, + akkaTyped + ) ++ akkaTest // === Cats =================================================================== @@ -207,6 +222,14 @@ val jackson = Seq( "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion ) +// === JAXB ================================================================ + +val jaxbVersion = "2.3.3" +val jaxb = Seq( + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion % Benchmark, + "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion % Benchmark +) + // === JMH ==================================================================== val jmhVersion = "1.23" @@ -348,17 +371,14 @@ lazy val syntax = crossProject(JVMPlatform, JSPlatform) "io.circe" %%% "circe-generic" % circeVersion, "io.circe" %%% "circe-parser" % circeVersion ), - compile := (Compile / compile) - .dependsOn(Def.taskDyn { - val parserCompile = - (`syntax-definition`.jvm / Compile / compileIncremental).value - if (parserCompile.hasModified) { - Def.task { - streams.value.log.info("Parser changed, forcing recompilation.") - clean.value - } - } else Def.task {} - }) + (Compile / compile) := (Compile / compile) + .dependsOn(RecompileParser.run(`syntax-definition`)) + .value, + (Test / compile) := (Test / compile) + .dependsOn(RecompileParser.run(`syntax-definition`)) + .value, + (Benchmark / compile) := (Benchmark / compile) + .dependsOn(RecompileParser.run(`syntax-definition`)) .value ) .jvmSettings( @@ -581,12 +601,13 @@ lazy val searcher = project .in(file("lib/searcher")) .configs(Test) .settings( - libraryDependencies ++= Seq( + libraryDependencies ++= akkaTest ++ Seq( "com.typesafe.slick" %% "slick" % slickVersion, "org.xerial" % "sqlite-jdbc" % sqliteVersion, "org.scalatest" %% "scalatest" % scalatestVersion % Test ) ) + .dependsOn(`json-rpc-server-test` % Test) // ============================================================================ // === Sub-Projects =========================================================== @@ -662,6 +683,7 @@ lazy val `language-server` = (project in file("engine/language-server")) .settings( inConfig(Benchmark)(Defaults.testSettings), bench := (test in Benchmark).value, + libraryDependencies ++= akkaTest, libraryDependencies += "com.storm-enroute" %% "scalameter" % scalameterVersion % "bench", testFrameworks ++= List( new TestFramework("org.scalameter.ScalaMeterFramework") @@ -684,7 +706,7 @@ lazy val runtime = (project in file("engine/runtime")) logBuffered in Test := false, scalacOptions += "-Ymacro-annotations", scalacOptions ++= Seq("-Ypatmat-exhaust-depth", "off"), - libraryDependencies ++= circe ++ jmh ++ Seq( + libraryDependencies ++= circe ++ jmh ++ jaxb ++ Seq( "com.chuusai" %% "shapeless" % shapelessVersion, "org.apache.commons" % "commons-lang3" % commonsLangVersion, "org.apache.tika" % "tika-core" % tikaVersion, @@ -704,15 +726,23 @@ lazy val runtime = (project in file("engine/runtime")) // Note [Unmanaged Classpath] Compile / unmanagedClasspath += (`core-definition` / Compile / packageBin).value, Test / unmanagedClasspath += (`core-definition` / Compile / packageBin).value, + Compile / compile / compileInputs := (Compile / compile / compileInputs) + .dependsOn(CopyTruffleJAR.preCompileTask) + .value, Compile / compile := FixInstrumentsGeneration.patchedCompile - .dependsOn(`core-definition` / Compile / packageBin) .dependsOn(FixInstrumentsGeneration.preCompileTask) + .dependsOn(`core-definition` / Compile / packageBin) .value, // Note [Classpath Separation] Test / javaOptions ++= Seq( - "-XX:-UseJVMCIClassLoader", "-Dgraalvm.locatorDisabled=true" - ) + ), + bootstrap := CopyTruffleJAR.bootstrapJARs.value, + Global / onLoad := EnvironmentCheck.addVersionCheck( + graalVersion, + javaVersion, + flatbuffersVersion + )((Global / onLoad).value) ) .settings( (Compile / javacOptions) ++= Seq( diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 3561494c3e..0826354a28 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -22,7 +22,8 @@ sections of this document are linked below: - [Design Documentation](#design-documentation) - [System Requirements](#system-requirements) - [Getting the Sources](#getting-the-sources) - - [Getting Set Up \(Rust\)](#getting-set-up-rust) + - [Getting Set Up (Rust)](#getting-set-up-rust) + - [Getting Set Up (JVM)](#getting-set-up-jvm) - [Building Enso](#building-enso) - [Running Enso](#running-enso) - [Pull Requests](#pull-requests) @@ -31,6 +32,7 @@ sections of this document are linked below: - [Out-of-Tree Contributions](#out-of-tree-contributions) - [Helpful Documentation and Links](#helpful-documentation-and-links) + All contributions to Enso should be in keeping with our @@ -117,9 +119,10 @@ In order to build and run Enso you will need the following tools: - [sbt](https://www.scala-sbt.org/) with the same version as specified in [`project/build.properties`](../project/build.properties). -- [GraalVM](https://www.graalvm.org/) with version at least that described in - the [`build.sbt`](../build.sbt) file, and Java 8, configured as your default - JVM. +- [GraalVM](https://www.graalvm.org/) with the same version as described in the + [`build.sbt`](../build.sbt) file, configured as your default JVM. GraalVM is + distributed for different Java versions, so you need a GraalVM distribution + for the same Java version as specified in [`build.sbt`](../build.sbt). - [Flatbuffers Compiler](https://google.github.io/flatbuffers) with version 1.12.0. - [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html), @@ -171,6 +174,23 @@ rustup component add clippy Please note that once the parser is integrated into the SBT build, the rust-related commands will be automatically performed for you. +### Getting Set Up (JVM) +In order to properly build the `runtime` component, the JVM running SBT needs +to have some dependency JARs available in its module path at startup. To ensure +they are available, before running any compilation or other tasks, these +dependencies should be prepared. To do so, run the following command in the +repository root directory: + +```bash +sbt bootstrap +``` + +It is preferred to not run this command from the sbt shell, but in batch mode, +because SBT has to be launched again anyway to pick up these JARs at startup. + +Bootstrap has to be run only when building the project for the first time +**and** after each change of Graal version. + ### Building Enso There are multiple projects in this repository, but all can be built, run and tested using `sbt`. As long as your configuration is correct, with the correct @@ -270,8 +290,8 @@ assembly. It can be acquired for MacOS and Linux please choose one best suited for you. Once you have a copy of the dynamic library, it needs to be placed in -`$JVM_HOME/jre/lib/server` on a JDK-8 install, or in `$JVM_HOME/lib/server` for -a JDK-10 or later install (once this is supported). +`$JVM_HOME/lib/server`. + #### Native Image Native image is a capability provided alongside GraalVM that allows the @@ -307,8 +327,8 @@ getting the project into a working state in IntelliJ. both 'for imports' and 'for builds'. 8. Disallow the overriding of the sbt version. 9. Under the 'Project JDK' setting, please ensure that it is set up to use a - GraalVM version as described in [Requirements](#requirements). You may need - to add it using the 'New' button if it isn't already set up. + GraalVM version as described in [System requirements](#system-requirements). + You may need to add it using the 'New' button if it isn't already set up. 10. Click 'Finish'. This will prompt you as to whether you want to overwrite the `project` folder. Select 'Yes' to continue. The Enso project will load up with an open SBT shell, which can be interacted with as described above. You diff --git a/docs/debugger/protocol.md b/docs/debugger/protocol.md index d2f99c372f..3daf8efb62 100644 --- a/docs/debugger/protocol.md +++ b/docs/debugger/protocol.md @@ -19,10 +19,10 @@ implement a simple REPL or add debugging capabilities to the editor. - [`Exception`](#exception) - [`Binding`](#binding) - [Messages](#messages) - - [Session start](#session-start) + - [Session Start](#session-start) - [Evaluation](#evaluation) - - [List bindings](#list-bindings) - - [Session exit](#session-exit) + - [List Bindings](#list-bindings) + - [Session Exit](#session-exit) @@ -108,7 +108,7 @@ table Response { } ``` -### Session start +### Session Start When a breakpoint is reached, the debugger sends a notification to the client indicating that a REPL session should be started. @@ -153,7 +153,7 @@ table ReplEvaluationFailure { } ``` -### List bindings +### List Bindings Lists all the bindings available in the current execution scope and sends them back. @@ -173,7 +173,7 @@ table ReplListBindingsResult { } ``` -### Session exit +### Session Exit Terminates this REPL session (and resumes normal program execution). The last result of Evaluation will be returned from the instrumented node or if diff --git a/docs/getting-enso.md b/docs/getting-enso.md index c18e919ca7..52a7a306da 100644 --- a/docs/getting-enso.md +++ b/docs/getting-enso.md @@ -26,7 +26,7 @@ GraalVM. You can get the Community Edition pre-built distributions from It is important to run Enso with exactly the version specified here. Given that Graal is still a relatively young project, even the minor version changes introduce breaking API changes. The current version of GraalVM required for -Enso is `20.1.0`, and it must be the Java 8 build. +Enso is `20.1.0`, and it must be the Java 11 build. Before running the Enso packages, make sure that the `JAVA_HOME` environment variable points to the correct home location of the Graal distribution. @@ -55,8 +55,8 @@ This section lists the most common failures and their probable causes. `bin/enso --version`. Take note of the version displayed in the `Running on` section. It should be similar to: ``` - Running on: OpenJDK 64-Bit Server VM GraalVM CE 20.1.0, JDK 1.8.0_252-b09 - Mac OS X 10.15.3 (x86_64) + Running on: OpenJDK 64-Bit Server VM, GraalVM Community, JDK 11.0.7+10-jvmci-20.1-b02 + Linux 4.15.0-106-generic (amd64) ``` It could also be caused by not using the launcher scripts and trying to run the component `.jar` files via `java -jar` without setting the necessary diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index 2a3b5ee061..7ee846e711 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -7,10 +7,10 @@ order: 0 --- # Infrastructure -The Enso runtime runs on the GraalVM which is a version of the JVM. It currently -uses JDK 8, but there are plans to migrate to JDK 11. This folder contains all -documentation pertaining to Enso's infrastructure, which is broken up as -follows: +The Enso runtime runs on the GraalVM which is a version of the JVM. This folder +contains all documentation pertaining to Enso's infrastructure, which is broken +up as follows: - [**sbt:**](sbt.md) The build tools that are used for building the project. -- [**Java 11:**](java-11.md) Current status of migration to Java 11. +- [**Java 11:**](java-11.md) Description of changes related to the Java 11 + migration. diff --git a/docs/infrastructure/java-11.md b/docs/infrastructure/java-11.md index d26c96c620..ece6f8591a 100644 --- a/docs/infrastructure/java-11.md +++ b/docs/infrastructure/java-11.md @@ -10,12 +10,12 @@ order: 2 JDK 11 will be supported longer than JDK 8 that we currently use and it adds new features that could improve performance. Moreover, we want to be compliant to the Java Platform Module System, as all future versions of the JDK will rely on -it. Thus, we want to move to using Graal builds for Java 11. +it. Thus, we have moved to using Graal builds for Java 11. -- [Migration progress](#migration-progress) - - [Build configuration](#build-configuration) +- [Migration Progress](#migration-progress) + - [Build Configuration](#build-configuration) - [Testing](#testing) - [Benchmarks](#benchmarks) - [Problems](#problems) @@ -23,23 +23,32 @@ it. Thus, we want to move to using Graal builds for Java 11. -## Migration progress +## Migration Progress The overall steps of the migration and their status are outlined in this -section. The task is tracked as issue -[#671](https://github.com/enso-org/enso/issues/671). +section. -### Build configuration -Currently, the only modification was the removal of the option -`-XX:-UseJVMCIClassLoader` that is deprecated in Java 11. Some other -modifications may be necessary however, to fix the issues described in -[Problems](#problems). +### Build Configuration +The option `-XX:-UseJVMCIClassLoader` is deprecated in Java 11 and has been +removed from the test configuration. + +The JVM running sbt must have `--upgrade-module-path=lib/truffle-api.jar` added +as an option and the build tool must ensure that the `truffle-api.jar` is copied +from the Maven repository to the `lib/` directory before the `runtime` project +is compiled. Section [IllegalAccessError](#illegalaccesserror) explains why +this is necessary and [Bootstrapping](./sbt.md#bootstrapping) explains the tasks +that help with this process. ### Testing -After making the build succeed, all runtime tests are passing. -This will have to be revisited after fixing a final build configuration. +All tests are passing. ### Benchmarks -Benchmarks have not yet been compared. +Initially there were some regressions found in the benchmarks, but further +investigation revealed this was caused by some issues in the methodology of how +the JMH benchmarks were implemented. There are plans to rewrite these +benchmarks. + +Benchmarks in pure Enso are currently more meaningful. They yield comparable +results with Java 11 being slightly faster. ## Problems The problems that were encountered when doing the migration. @@ -64,15 +73,30 @@ java.lang.IllegalAccessError: superinterface check failed: class com.oracle.truf The Truffle API does not export its packages for security reasons and it uses some custom mechanisms when loading the language runtime. However zinc is not -aware of these mechanisms and just directly loads the classfiles, resulting in -errors. +aware of these mechanisms and just directly loads the `.class` files, resulting +in errors. -Some of these errors are expected and instead of failing, simply a warning is +Some of these errors are caught and instead of failing, simply a warning is printed. Others are not detected where zinc expects them, but they fail later, -crashing the compilation process. +crashing the compilation process. All of them are problematic though, because +they mean that zinc is not able to read dependencies of the affected files. This +harms incremental compilation (as some dependencies are not detected it might be +necessary to do a clean and full recompilation when changing these files). -It may be possible to detect and catch those latter by modifying zinc, but this -would likely negatively impact its ability to detect dependencies for -incremental compilation. +#### Solution +We want to make the ClassLoader read these class files without errors. For that +we need to ensure it has permissions to load the Truffle modules. -We are currently looking into what can be done to avoid these errors. +Truffle API is distributed in two ways. The distribution included in the Graal +runtime (the one that is picked up by sbt by default) exports the required APIs +only to its own modules, so they are not available for us (thus the errors). +This is to ensure better security (to disallow language users introspecting the +VM internals). However, there is a second Truffle distribution on Maven that is +to be used for development only and that version exports the necessary APIs to +all packages. + +We need to ensure that the sbt process doing the compilation uses the Maven +version of Truffle, so that it does not complain about the illegal accesses. To +achieve that, we need to add the option +`--upgrade-module-path=lib/truffle-api.jar` to the JVM running sbt and ensure +that the Truffle JAR from maven is copied to the `lib/` directory. diff --git a/docs/infrastructure/sbt.md b/docs/infrastructure/sbt.md index 6e492225b1..a9c9866311 100644 --- a/docs/infrastructure/sbt.md +++ b/docs/infrastructure/sbt.md @@ -14,17 +14,21 @@ compilation. The build configuration is defined in -- [Incremental compilation](#incremental-compilation) -- [Helper tasks](#helper-tasks) +- [Incremental Compilation](#incremental-compilation) +- [Bootstrapping](#bootstrapping) +- [Compile Hooks](#compile-hooks) +- [Helper Tasks](#helper-tasks) + - [Graal and Flatc Version Check](#graal-and-flatc-version-check) - [Benchmarks](#benchmarks) - - [Build information](#build-information) - - [Instruments generation](#instruments-generation) - - [Flatbuffers generation](#flatbuffers-generation) - - [Debugging command](#debugging-command) + - [Build Information](#build-information) + - [Instruments Generation](#instruments-generation) + - [Flatbuffers Generation](#flatbuffers-generation) + - [Ensuring JARs Were Loaded](#ensuring-jars-were-loaded) + - [Debugging Command](#debugging-command) -## Incremental compilation +## Incremental Compilation To help wit build times, we do not want to rebuild the whole project with every change, but to only recompile the files that have been affected by the change. This is handled by sbt which under the hood uses @@ -32,43 +36,140 @@ This is handled by sbt which under the hood uses analyses the compiled files and detects dependencies between them to determine which files have to be recompiled when something has been changed. -## Helper tasks +## Bootstrapping +As described in [Java 11 Migration](./java-11.md#illegalaccesserror) to +successfully compile the `runtime` project, the JVM running sbt must use the +overridden JAR for Truffle API. This JAR has to be present during startup of the +JVM, but it has to be downloaded from the Maven repository. + +To fix this chicken-and-egg problem, we have a special `bootstrap` task, that +has to be ran when setting-up the project (and after a version change of Graal). +It makes sure the JAR is downloaded and copied to our directory and terminates +the sbt process to ensure that the user restarts it. Without the full restart, +the JAR would not be seen by the JVM. So when setting up the project or after +changing the version of Graal, before launching the sbt shell, you should first +run `sbt bootstrap`, to make sure the environment is properly prepared. + +The logic for copying the JAR is implemented in the `bootstrapJARs` task in +[`CopyTruffleJAR`](../../project/CopyTruffleJAR.scala). + +## Compile Hooks +There are some invariants that are specific to our project, so they are not +tracked by sbt, but we want to ensure that they hold to avoid cryptic errors at +compilation or runtime. + +To check some state before compilation, we add our tasks as dependencies of +`Compile / compile / compileInputs` by adding the following to the settings of a +particular project. + +``` +Compile / compile / compileInputs := (Compile / compile / compileInputs) + .dependsOn(preCompileHookTask) + .value +``` + +Tasks that should be run before compilation, should be attached to the +`compileInputs` task. That is because the actual compilation process is ran in +the task `compileIncremental`. `Compile / compile` depends on +`compileIncremental` but if we add our dependency to `Compile / compile`, it is +considered as independent with `compileIncremental`, so sbt may schedule it to +run in parallel with the actual compilation process. To guarantee that our +pre-flight checks complete *before* the actual compilation, we add them as a +dependency of `compileInputs` which runs *strictly before* actual compilation. + +To check some invariants *after* compilation, we can replace the original +`Compile / compile` task with a custom one which does its post-compile checks +and returns the result of `(Compile / compile).value`. An example of such a +'patched' compile task is implemented in +[`FixInstrumentsGeneration`](../../project/FixInstrumentsGeneration.scala). + +## Helper Tasks There are additional tasks defined in the [`project`](../../project) directory. They are used by [`build.sbt`](../../build.sbt) to provide some additional functionality. +### Graal and Flatc Version Check +[`EnvironmentCheck`](../../project/EnvironmentCheck.scala) defines a helper +function that can be attached to the default `Global / onLoad` state transition +to run a version check when loading the sbt project. This helper function +compares the version of JVM running sbt with GraalVM version defined in +[`build.sbt`](../../build.sbt) and the version of `flatc` installed in the +system with the Flatbuffers library version defined in +[`build.sbt`](../../build.sbt). If the versions do not match it reports an error +telling the user to change to the correct version. + ### Benchmarks [`BenchTasks`](../../project/BenchTasks.scala) defines configuration keys for benchmarking. -### Build information +### Build Information [`BenchTasks`](../../project/BuildInfo.scala) records version information including what git commit has been used for compiling the project. This information is used by `enso --version`. -### Instruments generation +### Instruments Generation Truffle annotation processor generates a file that registers instruments provided by the runtime. Unfortunately, with incremental compilation, only the changed instruments are recompiled and the annotation processor does not detect this, so un-changed instruments get un-registered. -To fix this, the task defined in +To fix this, the pre-compile task defined in [`FixInstrumentsGeneration`](../../project/FixInstrumentsGeneration.scala) detects changes to instruments and if only one of them should be recompiled, it forces recompilation of all of them, to ensure consistency. -Sometimes it is unable to detect the need for recompilation before it takes -place. As it also cannot restart compilation, to preserve consistency it stops -the compilation and asks the user to restart it, to allow it to force -recompilation of instruments. +For unclear reasons, if this task is attached to +`Compile / compile / compileInputs`, while it runs strictly *before* +compilation, the deleted class files are not always all recompiled. So instead, +it is attached directly to `Compile / compile`. This technically could allow for +a data race between this task and the actual compilation that happens in +`compileIncremental`, but in practice it seems to be a stable solution. -### Flatbuffers generation +Sometimes it is unable to detect the need for recompilation before it takes +place. To help that, there is another task that replaces the default `compile` +task, which executes the default compilation task and after it, verifies the +consistency of instruments files. As it cannot restart compilation, to preserve +consistency it ensures the instruments will be recompiled the next time and +stops the current compilation task, asking the user to restart it. + +### Flatbuffers Generation [`GenerateFlatbuffers`](../../project/GenerateFlatbuffers.scala) defines the task that runs the Flatbuffer compiler `flatc` whenever the flatbuffer definitions have been changed. It also makes sure that `flatc` is available on PATH and that its version matches the version of the library. It reports any errors. -### Debugging command +### Ensuring JARs Were Loaded +As described in [Bootstrapping](#bootstrapping), to successfully compile the +`runtime` subproject, the JVM running sbt must load some JARs at startup. The +user should run `sbt bootstrap` to ensure that. + +If the compilation proceeds without the bootstrapped JARs it may lead to +inconsistent state with some dependencies being undetected and weird errors. To +avoid such situations, [`CopyTruffleJAR`](../../project/CopyTruffleJAR.scala) +defines is a pre-compile task that is executed before compiling the `runtime` +subproject which makes sure that the Truffle JAR is up-to-date. Even if the +developer forgets about `bootstrap`, this pre-compile task will update the +Truffle JARs. However, as the JARs are loaded at startup, the JVM has to be +restarted. To force the user to restart it, the current sbt process is +terminated. + +This pre-compile task runs before compiling `runtime` sources, but some +dependencies may have started compiling in the meantime (as sbt schedules +independent tasks in parallel). The fact that these dependencies are up-to-date +may not be registered by sbt due to the abrupt termination. This does not affect +consistency of the build state though - as a result these dependencies may be +recompiled again when the compilation is restarted. This is acceptable as this +restart is just an additional check to ensure correctness and improve user +experience. In normal operation, the restart should never be triggered, as the +user should remember to run `bootstrap` when necessary. + +### Debugging Command [`WithDebugCommand`](../../project/WithDebugCommand.scala) defines a command -that allows to run a task with additional JVM-level flags. \ No newline at end of file +that allows to run a task with additional JVM-level flags. + +### Recompile Parser +[`RecompileParser`](../../project/RecompileParser.scala) defines a task that can +be attached to the `compile` task in configurations of the `syntax` project. +This task ensures that the `syntax` project is recompiled whenever +`syntax-definition` changes. \ No newline at end of file diff --git a/engine/language-server/src/bench/scala/org/enso/languageserver/text/RopeBench.scala b/engine/language-server/src/bench/scala/org/enso/languageserver/text/RopeBench.scala index a7ff4a628c..39c991a40b 100644 --- a/engine/language-server/src/bench/scala/org/enso/languageserver/text/RopeBench.scala +++ b/engine/language-server/src/bench/scala/org/enso/languageserver/text/RopeBench.scala @@ -1,6 +1,5 @@ package org.enso.languageserver.text -import org.enso.languageserver.data.buffer.Rope -import org.scalacheck.Gen.Parameters +import org.enso.text.buffer.Rope import org.scalameter.{Bench, Gen} object RopeBench extends Bench.LocalTime { diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/FileManagerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/FileManagerTest.scala index 3f5a51e5f0..39121c4d4f 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/FileManagerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/FileManagerTest.scala @@ -6,11 +6,12 @@ import java.util.UUID import io.circe.literal._ import org.apache.commons.io.FileUtils +import org.enso.jsonrpc.test.RetrySpec -class FileManagerTest extends BaseServerTest { +class FileManagerTest extends BaseServerTest with RetrySpec { "File Server" must { - "write textual content to a file" in { + "write textual content to a file" taggedAs Retry() in { val client = getInitialisedWsClient() client.send(json""" { "jsonrpc": "2.0", diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/TextOperationsTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/TextOperationsTest.scala index c3886944d2..62cf95630f 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/TextOperationsTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/TextOperationsTest.scala @@ -2,13 +2,14 @@ package org.enso.languageserver.websocket.json import akka.testkit.TestProbe import io.circe.literal._ +import org.enso.jsonrpc.test.RetrySpec import org.enso.languageserver.event.BufferClosed import org.enso.languageserver.filemanager.Path -class TextOperationsTest extends BaseServerTest { +class TextOperationsTest extends BaseServerTest with RetrySpec { "text/openFile" must { - "fail opening a file if it does not exist" in { + "fail opening a file if it does not exist" taggedAs Retry() in { // Interaction: // 1. Client tries to open a non-existent file. // 2. Client receives an error message. diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index 62181ef969..474f8d286f 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -283,6 +283,7 @@ object Main { def displayVersion(useJson: Boolean): Unit = { // Running platform information val vmName = System.getProperty("java.vm.name") + val vmVendor = System.getProperty("java.vm.vendor") val jreVersion = System.getProperty("java.runtime.version") val osArch = System.getProperty("os.arch") val osName = System.getProperty("os.name") @@ -297,6 +298,7 @@ object Main { | "dirty": ${Info.isDirty}, | "commit": "${Info.commit}", | "vmName": "$vmName", + | "vmVendor": "$vmVendor", | "jreVersion": "$jreVersion", | "osName": "$osName", | "osVersion": "$osVersion", @@ -313,7 +315,7 @@ object Main { |Version: ${Info.ensoVersion} |Built with: scala-${Info.scalacVersion} for GraalVM ${Info.graalVersion} |Built from: ${Info.branch}$dirtyStr @ ${Info.commit} - |Running on: $vmName, JDK $jreVersion + |Running on: $vmName, $vmVendor, JDK $jreVersion | $osName $osVersion ($osArch) |""".stripMargin } diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/DocumentationCommentsTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/DocumentationCommentsTest.scala index 0510968db4..06d7544823 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/DocumentationCommentsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/pass/resolve/DocumentationCommentsTest.scala @@ -223,7 +223,6 @@ class DocumentationCommentsTest extends CompilerTest with Inside { | _ -> 50 |""".stripMargin.preprocessModule - println(ir) val t1 = ir.bindings(0) getDoc(t1) shouldEqual " the constructor Bar" inside(ir.bindings(1)) { diff --git a/lib/interpreter-dsl/src/main/java/org/enso/interpreter/dsl/MethodProcessor.java b/lib/interpreter-dsl/src/main/java/org/enso/interpreter/dsl/MethodProcessor.java index c25230c415..0f7d7ba6de 100644 --- a/lib/interpreter-dsl/src/main/java/org/enso/interpreter/dsl/MethodProcessor.java +++ b/lib/interpreter-dsl/src/main/java/org/enso/interpreter/dsl/MethodProcessor.java @@ -17,7 +17,7 @@ import java.util.*; /** The processor used to generate code from the {@link BuiltinMethod} annotation. */ @SupportedAnnotationTypes("org.enso.interpreter.dsl.BuiltinMethod") -@SupportedSourceVersion(SourceVersion.RELEASE_8) +@SupportedSourceVersion(SourceVersion.RELEASE_11) @AutoService(Processor.class) public class MethodProcessor extends AbstractProcessor { diff --git a/lib/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala b/lib/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala index d82a96c2f7..46f412ac75 100644 --- a/lib/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala +++ b/lib/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala @@ -1,5 +1,6 @@ package org.enso.searcher.sql +import org.enso.jsonrpc.test.RetrySpec import org.enso.searcher.Suggestion import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers @@ -12,6 +13,7 @@ import scala.concurrent.duration._ class SuggestionsRepoTest extends AnyWordSpec + with RetrySpec with Matchers with BeforeAndAfterAll { @@ -43,7 +45,7 @@ class SuggestionsRepoTest Await.result(action, Timeout) shouldEqual Some(suggestion.atom) } - "find suggestion by returnType" in { + "find suggestion by returnType" taggedAs Retry() in { val action = for { _ <- db.run(repo.insert(suggestion.local)) diff --git a/lib/text-buffer/src/test/scala/org/enso/text/editing/RopeTextEditorSpec.scala b/lib/text-buffer/src/test/scala/org/enso/text/editing/RopeTextEditorSpec.scala index fb454c10b9..ad69bbe6f7 100644 --- a/lib/text-buffer/src/test/scala/org/enso/text/editing/RopeTextEditorSpec.scala +++ b/lib/text-buffer/src/test/scala/org/enso/text/editing/RopeTextEditorSpec.scala @@ -42,8 +42,10 @@ class RopeTextEditorSpec extends AnyFlatSpec with Matchers { it should "replace a multiline substring" in { //given - val resultPosition = Range(Position(5, 4), Position(6, 10)) - val change = s"sum = plusOne 5${System.lineSeparator()} sum" + val resultPosition = Range(Position(5, 4), Position(6, 10)) + val change = + """sum = plusOne 5 + | sum""".stripMargin val resultReplacement = TextEdit(resultPosition, change) //when val result = RopeTextEditor.edit(testSnippet, resultReplacement) @@ -59,8 +61,10 @@ class RopeTextEditorSpec extends AnyFlatSpec with Matchers { it should "be able to insert change at the end of file" in { //given - val eof = Range(Position(6, 10), Position(6, 10)) - val insertion = TextEdit(eof, s"${System.lineSeparator()} return result") + val eof = Range(Position(6, 10), Position(6, 10)) + val insertedText = """ + | return result""".stripMargin + val insertion = TextEdit(eof, insertedText) //when val result = RopeTextEditor.edit(testSnippet, insertion) //then diff --git a/project/CopyTruffleJAR.scala b/project/CopyTruffleJAR.scala new file mode 100644 index 0000000000..35be533d74 --- /dev/null +++ b/project/CopyTruffleJAR.scala @@ -0,0 +1,131 @@ +import sbt.Keys._ +import sbt._ +import sbt.internal.util.ManagedLogger + +object CopyTruffleJAR { + + /** + * The task that is used for copying the Truffle JAR file to our target + * directory. It should be ran when setting-up the project and after each + * update of Graal version. + * + * As the JARs have to be available to the sbt JVM on startup, to ensure the + * JARs are available, the whole sbt process must be restarted (otherwise the + * compilation would fail with cryptic errors). To make sure that the user + * restarts the process, it is terminated. + */ + lazy val bootstrapJARs = Def.task { + val log = streams.value.log + if ( + ensureTruffleJARUpToDate( + baseDirectory.value, + (Compile / update).value, + log + ) + ) { + log.info("Truffle JARs have been updated.") + System.err.println( + "You have to restart the sbt JVM for the changes to take effect." + ) + System.out.flush() + System.err.flush() + System.exit(0) + } else { + log.info("Truffle JARs are up to date.") + } + } + + /** + * This task should be added as a dependency of compileInputs in the runtime + * subproject. It ensures that the compilation will not proceed unless the + * JARs have not been bootstrapped. If the JARs were out of date, they are + * updated within this task, so bootstrap does not have to be re-run. + * However, for the same reasons as for [[bootstrapJARs]], the sbt process is + * terminated, because a restart is required. + */ + lazy val preCompileTask = Def.task { + val log = streams.value.log + if ( + ensureTruffleJARUpToDate( + baseDirectory.value, + (Compile / update).value, + log + ) + ) { + log.error( + "JARs that have to be loaded by the sbt JVM at startup have been" + + " modified or did not exist.\n" + + "Did you run bootstrap?" + ) + log.warn( + "Bootstrap has been triggered automatically, but the sbt JVM must be" + + " restarted to apply the changes, so the compilation had to be stopped." + ) + log.warn( + "To avoid disrupting compilation, remember to run bootstrap when " + + "setting up the project and after each version change of Graal." + ) + System.err.println( + "The sbt JVM has to be restarted to apply the changes.\n" + + "Please re-launch sbt and re-run the last command." + ) + System.out.flush() + System.err.flush() + System.exit(0) + } + } + + /** + * Checks the Truffle JARs and updates them if necessary. + * + * @param libraryUpdates the value of Compile / updates + * @return true if an update has been performed and the JVM needs a restart + */ + private def ensureTruffleJARUpToDate( + baseDirectory: File, + libraryUpdates: UpdateReport, + log: ManagedLogger + ): Boolean = { + var truffleInstancesFound = 0 + var restartRequired = false + libraryUpdates.allFiles.foreach { f => + if (f.getName.contains("truffle-api")) { + truffleInstancesFound += 1 + val dest = baseDirectory / "build-cache" / "truffle-api.jar" + val needsUpdate = if (!dest.exists()) { + log.debug("truffle-api.jar does not exist in target/") + true + } else if (dest.lastModified() < f.lastModified()) { + log.debug("truffle-api.jar in target/ is out of date") + true + } else false + + if (needsUpdate) { + IO.copyFile(f, dest) + restartRequired = true + } + } + } + + if (truffleInstancesFound == 0) { + throw new IllegalStateException( + "Truffle API has not been found in the dependencies!\n" + + "If dependencies have been changed in build.sbt, make sure " + + "the algorithm locating truffle-api.jar in" + + " project/CopyTruffleJAR.scala has been updated accordingly.\n" + + "Please report this as a bug." + ) + } else if (truffleInstancesFound > 1) { + throw new IllegalStateException( + "More than one version of Truffle API has been found in the " + + "dependencies.\n" + + "If dependencies have been changed in build.sbt, make sure " + + "the algorithm locating truffle-api.jar in" + + " project/CopyTruffleJAR.scala has been updated accordingly.\n" + + "Please report this as a bug." + ) + } + + restartRequired + } +} diff --git a/project/EnvironmentCheck.scala b/project/EnvironmentCheck.scala new file mode 100644 index 0000000000..97443d80d3 --- /dev/null +++ b/project/EnvironmentCheck.scala @@ -0,0 +1,94 @@ +import sbt._ +import sbt.internal.util.ManagedLogger + +object EnvironmentCheck { + + /** Compares the version of JVM running sbt with the GraalVM versions defined + * in project configuration and reports errors if the versions do not match. + * + * @param expectedGraalVersion the GraalVM version that should be used for + * building this project + * @param expectedJavaVersion the Java version of the used GraalVM + * distribution + * @param log a logger used to report errors if the versions are mismatched + */ + def checkVersions( + expectedGraalVersion: String, + expectedJavaVersion: String, + expectedFlatbuffersVersion: String, + log: ManagedLogger + ): Unit = { + val javaSpecificationVersion = + System.getProperty("java.vm.specification.version") + val graalVersion = + System.getProperty("org.graalvm.version") + + val graalOk = + if (graalVersion == null) { + log.error( + "Property org.graalvm.version is not defined. " + + s"Make sure your current JVM is set to " + + s"GraalVM $expectedGraalVersion Java $expectedJavaVersion" + ) + false + } else if (graalVersion != expectedGraalVersion) { + log.error( + s"GraalVM version mismatch - you are running $graalVersion but " + + s"$expectedGraalVersion is expected" + ) + false + } else true + + val javaOk = + if (javaSpecificationVersion != expectedJavaVersion) { + log.error( + s"Java version mismatch - you are running " + + s"Java $javaSpecificationVersion " + + s"but Java $expectedJavaVersion is expected" + ) + false + } else true + + val versionsOk = graalOk && javaOk + if (!versionsOk) { + log.error( + "=== Please make sure to change to a correct version of" + + " GraalVM before attempting compilation ===" + ) + } + + GenerateFlatbuffers.verifyFlatcVersion(expectedFlatbuffersVersion) match { + case Left(explanation) => + log.error(explanation) + log.error( + "=== Please make sure to install a correct version of" + + " flatc before attempting compilation ===" + ) + case Right(_) => + } + } + + /** + * Augments a state transition to do a GraalVM version check. + * + * @param graalVersion the GraalVM version that should be used for + * building this project + * @param javaVersion the Java version of the used GraalVM distribution + * @param flatbuffersVersion the Flatbuffers library version + * @param oldTransition the state transition to be augmented + * @return an augmented state transition that does all the state changes of + * oldTransition but also runs the GraalVM version check + */ + def addVersionCheck( + graalVersion: String, + javaVersion: String, + flatbuffersVersion: String + )( + oldTransition: State => State + ): State => State = + (state: State) => { + val newState = oldTransition(state) + checkVersions(graalVersion, javaVersion, flatbuffersVersion, newState.log) + newState + } +} diff --git a/project/FixInstrumentsGeneration.scala b/project/FixInstrumentsGeneration.scala index 257bbaa99a..c005306b2c 100644 --- a/project/FixInstrumentsGeneration.scala +++ b/project/FixInstrumentsGeneration.scala @@ -11,9 +11,10 @@ object FixInstrumentsGeneration { * Without that fix, incremental compilation would not register unchanged * instruments, leading to runtime errors. * - * It should be added as a dependency of compilation. + * It should be added as a dependency of Compile / compile / compileInputs. */ lazy val preCompileTask = Def.task { + val log = streams.value.log val root = baseDirectory.value val classFilesDirectory = (Compile / classDirectory).value val FragileFiles(fragileSources, fragileClassFiles) = @@ -22,23 +23,24 @@ object FixInstrumentsGeneration { val fragileSourcesStore = streams.value.cacheStoreFactory.make("instruments_fixer") - Tracked.diffInputs(fragileSourcesStore, FileInfo.hash)(fragileSources.toSet) { - sourcesDiff: ChangeReport[File] => - if (sourcesDiff.modified.nonEmpty) { - val others = - if (sourcesDiff.modified.size >= 2) - s" and ${sourcesDiff.modified.size - 1} others" - else "" - val firstInstrument = sourcesDiff.modified.head - val sourcesMessage = firstInstrument.toString + others - println( - s"Instruments sources ($sourcesMessage) have been changed.\n" + - s"Forcing recompilation of all instruments to maintain " + - s"consistency of generated services files." - ) + Tracked.diffInputs(fragileSourcesStore, FileInfo.hash)( + fragileSources.toSet + ) { sourcesDiff: ChangeReport[File] => + if (sourcesDiff.modified.nonEmpty && sourcesDiff.unmodified.nonEmpty) { + val others = + if (sourcesDiff.modified.size >= 2) + s" and ${sourcesDiff.modified.size - 1} others" + else "" + val firstInstrument = sourcesDiff.modified.head + val sourcesMessage = firstInstrument.toString + others + log.warn( + s"Instruments sources ($sourcesMessage) have been changed.\n" + + s"Forcing recompilation of all instruments to maintain " + + s"consistency of generated services files." + ) - fragileClassFiles.foreach(_.delete()) - } + fragileClassFiles.foreach(_.delete()) + } } } @@ -57,6 +59,7 @@ object FixInstrumentsGeneration { lazy val patchedCompile = Def.task { val compilationResult = (Compile / compile).value + val log = streams.value.log val root = baseDirectory.value val classFilesDirectory = (Compile / classDirectory).value val FragileFiles(_, fragileClassFiles) = @@ -72,7 +75,7 @@ object FixInstrumentsGeneration { fragileClassFiles.foreach(_.delete()) val projectName = name.value - println( + log.error( "Truffle Instrumentation is not up to date, " + "which will lead to runtime errors\n" + "Fixes have been applied to ensure consistent Instrumentation state, " + diff --git a/project/GenerateFlatbuffers.scala b/project/GenerateFlatbuffers.scala index 7d2b4c081e..9a296091de 100644 --- a/project/GenerateFlatbuffers.scala +++ b/project/GenerateFlatbuffers.scala @@ -2,6 +2,7 @@ import java.io.IOException import sbt.Keys._ import sbt._ +import sbt.internal.util.ManagedLogger import sbt.util.FilesInfo import scala.sys.process._ @@ -12,7 +13,13 @@ object GenerateFlatbuffers { private val flatcCmd = "flatc" lazy val task = Def.task { - verifyFlatcVersion(flatcVersion.value) + val log = state.value.log + verifyFlatcVersion(flatcVersion.value) match { + case Left(explanation) => + log.error(explanation) + throw new RuntimeException("flatc version check failed.") + case Right(_) => + } val root = baseDirectory.value val schemas = @@ -23,7 +30,7 @@ object GenerateFlatbuffers { val schemaSourcesStore = streams.value.cacheStoreFactory.make("flatc_schemas") val out = (sourceManaged in Compile).value - val generatedSources = gatherGeneratedSources(schemas, out) + val generatedSources = gatherGeneratedSources(schemas, out, log) Tracked.diffOutputs(generatedSourcesStore, FileInfo.exists)( generatedSources @@ -31,21 +38,23 @@ object GenerateFlatbuffers { generatedDiff.removed foreach { removedFile => removedFile.delete() } } - Tracked.diffInputs(schemaSourcesStore, FileInfo.lastModified)(schemas.toSet) { - schemasDiff: ChangeReport[File] => - val allGeneratedSourcesExist = generatedSources.forall(_.exists()) - if (schemasDiff.modified.nonEmpty || !allGeneratedSourcesExist) { - schemas foreach { schema => - val cmdGenerate = - s"$flatcCmd --java -o ${out.getAbsolutePath} $schema" - cmdGenerate.!! // Note [flatc Error Reporting] - } - - val projectName = name.value - println( - f"*** Flatbuffers code generation generated ${generatedSources.size} files in project $projectName" - ) + Tracked.diffInputs(schemaSourcesStore, FileInfo.lastModified)( + schemas.toSet + ) { schemasDiff: ChangeReport[File] => + val allGeneratedSourcesExist = generatedSources.forall(_.exists()) + if (schemasDiff.modified.nonEmpty || !allGeneratedSourcesExist) { + schemas foreach { schema => + val cmdGenerate = + s"$flatcCmd --java -o ${out.getAbsolutePath} $schema" + cmdGenerate.!! // Note [flatc Error Reporting] } + + val projectName = name.value + log.info( + "*** Flatbuffers code generation generated " + + s"${generatedSources.size} files in project $projectName" + ) + } } generatedSources.toSeq @@ -72,25 +81,25 @@ object GenerateFlatbuffers { * * @param expectedVersion flatc version that is expected to be installed, * should be based on project settings + * @return either an error message explaining what is wrong with the flatc + * version or Unit meaning it is correct */ - private def verifyFlatcVersion(expectedVersion: String): Unit = { + def verifyFlatcVersion(expectedVersion: String): Either[String, Unit] = { val cmd = f"$flatcCmd --version" - val versionStr = - try { - cmd.!!.trim - } catch { - case ex @ (_: RuntimeException | _: IOException) => - println("flatc version check failed. Make sure flatc is in your PATH") - throw new RuntimeException("Could not check flatc version", ex) - } - - val expectedVersionStr = s"flatc version $expectedVersion" - if (expectedVersionStr != versionStr) { - println("flatc version mismatch.") - println( - s"$expectedVersionStr is expected, but it seems $versionStr is installed" - ) - throw new RuntimeException("flatc version mismatch") + try { + val versionStr = cmd.!!.trim + val expectedVersionStr = s"flatc version $expectedVersion" + if (expectedVersionStr != versionStr) { + Left( + s"flatc version mismatch. $expectedVersionStr is expected, " + + s"but it seems $versionStr is installed" + ) + } else Right(()) + } catch { + case _ @(_: RuntimeException | _: IOException) => + Left( + "flatc version check failed. Make sure flatc is in your PATH" + ) } } @@ -148,7 +157,8 @@ object GenerateFlatbuffers { */ private def gatherGeneratedSources( schemas: Seq[File], - out: File + out: File, + log: ManagedLogger ): Set[File] = { val affectedSources = schemas.flatMap { schema => @@ -160,7 +170,7 @@ object GenerateFlatbuffers { } catch { case ex: RuntimeException => val exitCode = cmdMakeRules.! // Note [flatc Error Reporting] - println( + log.error( s"flatc on ${schema.getAbsolutePath} failed with exit code $exitCode" ) throw ex @@ -170,8 +180,8 @@ object GenerateFlatbuffers { case Left(errorMessage) => val exceptionMessage = s"Cannot parse flatc Make rules, flatc command: $cmdMakeRules" - println(exceptionMessage) - println(errorMessage) + log.error(exceptionMessage) + log.error(errorMessage) throw new RuntimeException(exceptionMessage) case Right(affectedSources) => affectedSources } diff --git a/project/RecompileParser.scala b/project/RecompileParser.scala new file mode 100644 index 0000000000..355de28516 --- /dev/null +++ b/project/RecompileParser.scala @@ -0,0 +1,24 @@ +import sbt._ +import sbt.Keys._ +import sbtcrossproject.CrossProject +import sbtcrossproject.CrossPlugin.autoImport._ + +object RecompileParser { + + /** + * Ensures that the project is recompiled whenever the project from + * `syntaxDefinition` is changed. Should be attached to the `compile` task as + * a dependency. + */ + def run(syntaxDefinition: CrossProject) = + Def.taskDyn { + val parserCompile = + (syntaxDefinition.jvm / Compile / compileIncremental).value + if (parserCompile.hasModified) { + Def.task { + streams.value.log.info("Parser changed, forcing recompilation.") + clean.value + } + } else Def.task {} + } +}