diff --git a/build.sbt b/build.sbt index 45e14561ed..1129dfac4c 100644 --- a/build.sbt +++ b/build.sbt @@ -254,6 +254,8 @@ lazy val buildNativeImage = lazy val enso = (project in file(".")) .settings(version := "0.1") .aggregate( + `persistance-dsl`, + `persistance`, `interpreter-dsl`, `interpreter-dsl-test`, `json-rpc-server-test`, @@ -1074,6 +1076,35 @@ lazy val searcher = project .dependsOn(testkit % Test) .dependsOn(`polyglot-api`) +lazy val `persistance` = (project in file("lib/java/persistance")) + .settings( + version := "0.1", + frgaalJavaCompilerSetting, + Compile / javacOptions := ((Compile / javacOptions).value), + libraryDependencies ++= Seq( + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test + ) + ) + .dependsOn(`persistance-dsl` % Test) + +lazy val `persistance-dsl` = (project in file("lib/java/persistance-dsl")) + .settings( + version := "0.1", + frgaalJavaCompilerSetting, + Compile / javacOptions := ((Compile / javacOptions).value ++ + // Only run ServiceProvider processor and ignore those defined in META-INF, thus + // fixing incremental compilation setup + Seq( + "-processor", + "org.netbeans.modules.openide.util.ServiceProviderProcessor" + )), + libraryDependencies ++= Seq( + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided" + ) + ) + lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl")) .settings( version := "0.1", @@ -1087,7 +1118,7 @@ lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl")) )), libraryDependencies ++= Seq( "org.apache.commons" % "commons-lang3" % commonsLangVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", "com.google.guava" % "guava" % guavaVersion exclude ("com.google.code.findbugs", "jsr305") ) ) @@ -1373,23 +1404,24 @@ lazy val runtime = (project in file("engine/runtime")) scalacOptions += "-Ymacro-annotations", scalacOptions ++= Seq("-Ypatmat-exhaust-depth", "off"), libraryDependencies ++= jmh ++ jaxb ++ circe ++ GraalVM.langsPkgs ++ Seq( - "org.apache.commons" % "commons-lang3" % commonsLangVersion, - "org.apache.tika" % "tika-core" % tikaVersion, - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", - "org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % "provided", - "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % "provided", - "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided", - "org.graalvm.truffle" % "truffle-tck" % graalMavenPackagesVersion % "provided", - "org.graalvm.truffle" % "truffle-tck-common" % graalMavenPackagesVersion % "provided", - "org.scalacheck" %% "scalacheck" % scalacheckVersion % Test, - "org.scalactic" %% "scalactic" % scalacticVersion % Test, - "org.scalatest" %% "scalatest" % scalatestVersion % Test, - "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark, - "org.typelevel" %% "cats-core" % catsVersion, - "junit" % "junit" % junitVersion % Test, - "com.github.sbt" % "junit-interface" % junitIfVersion % Test, - "org.hamcrest" % "hamcrest-all" % hamcrestVersion % Test, - "org.slf4j" % "slf4j-nop" % slf4jVersion % Benchmark + "org.apache.commons" % "commons-lang3" % commonsLangVersion, + "org.apache.tika" % "tika-core" % tikaVersion, + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", + "org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-tck" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-tck-common" % graalMavenPackagesVersion % "provided", + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "org.scalacheck" %% "scalacheck" % scalacheckVersion % Test, + "org.scalactic" %% "scalactic" % scalacticVersion % Test, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark, + "org.typelevel" %% "cats-core" % catsVersion, + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.hamcrest" % "hamcrest-all" % hamcrestVersion % Test, + "org.slf4j" % "slf4j-nop" % slf4jVersion % Benchmark ), // Add all GraalVM packages with Runtime scope - we don't need them for compilation, // just provide them at runtime (in module-path). @@ -1454,6 +1486,7 @@ lazy val runtime = (project in file("engine/runtime")) .dependsOn(`common-polyglot-core-utils`) .dependsOn(`edition-updater`) .dependsOn(`interpreter-dsl`) + .dependsOn(`persistance-dsl` % "provided") .dependsOn(`library-manager`) .dependsOn(`logging-truffle-connector`) .dependsOn(`polyglot-api`) @@ -1468,14 +1501,23 @@ lazy val `runtime-parser` = .settings( frgaalJavaCompilerSetting, instrumentationSettings, + commands += WithDebugCommand.withDebug, + fork := true, + Test / javaOptions ++= Seq( + "-Dgraalvm.locatorDisabled=true", + s"--upgrade-module-path=${file("engine/runtime/build-cache/truffle-api.jar").absolutePath}" + ), libraryDependencies ++= Seq( - "junit" % "junit" % junitVersion % Test, - "com.github.sbt" % "junit-interface" % junitIfVersion % Test, - "org.scalatest" %% "scalatest" % scalatestVersion % Test + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided" ) ) .dependsOn(syntax) .dependsOn(`syntax-rust-definition`) + .dependsOn(`persistance`) + .dependsOn(`persistance-dsl` % "provided") lazy val `runtime-compiler` = (project in file("engine/runtime-compiler")) @@ -1483,16 +1525,18 @@ lazy val `runtime-compiler` = frgaalJavaCompilerSetting, instrumentationSettings, libraryDependencies ++= Seq( - "junit" % "junit" % junitVersion % Test, - "com.github.sbt" % "junit-interface" % junitIfVersion % Test, - "org.scalatest" %% "scalatest" % scalatestVersion % Test, - "com.lihaoyi" %% "fansi" % fansiVersion + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "com.lihaoyi" %% "fansi" % fansiVersion ) ) .dependsOn(`runtime-parser`) .dependsOn(pkg) .dependsOn(`polyglot-api`) .dependsOn(editions) + .dependsOn(`persistance-dsl` % "provided") lazy val `runtime-instrument-common` = (project in file("engine/runtime-instrument-common")) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Expand_Objects_Helpers.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Expand_Objects_Helpers.enso index 806cf155ee..548f575060 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Expand_Objects_Helpers.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Expand_Objects_Helpers.enso @@ -13,6 +13,7 @@ import project.Extensions.Prefix_Name.Prefix_Name import project.Internal.Fan_Out import project.Internal.Java_Exports import project.Internal.Java_Problems +import project.Internal.Widget_Helpers from project.Internal.Java_Exports import make_inferred_builder ## PRIVATE diff --git a/docs/runtime/ir-caching.md b/docs/runtime/ir-caching.md index aee3cd66cc..92a50560db 100644 --- a/docs/runtime/ir-caching.md +++ b/docs/runtime/ir-caching.md @@ -12,84 +12,82 @@ One of the largest pain points for users of Enso at the moment is the fact that it has to precompile the entire standard library on every project load. This is, in essence, due to the fact that the current parser is abysmally slow, and incredibly demanding. The obvious solution to improve this is to take the parser -out of the equation in its entirety, by serialising the parser's output. +out of the equation in its entirety, by serializing the parser's output. -To that end, we want to serialise the Enso IR to a format that can later be read +To that end, we want to serialize the Enso IR to a format that can later be read back in, bypassing the parser entirely. Furthermore, we can move the boundary at -which this serialisation takes place to the end of the compiler pipeline, +which this serialization takes place to the end of the compiler pipeline, thereby bypassing doing most of the compilation work, and further improving startup performance. -- [Serialising the IR](#serialising-the-ir) +- [Serializing the IR](#serializing-the-ir) - [Breaking Links](#breaking-links) - [Storing the IR](#storing-the-ir) - [Metadata Format](#metadata-format) - - [Portability Guarantees](#portability-guarantees) + - [Portability and Versioning](#portability-and-versioning) - [Loading the IR](#loading-the-ir) - [Integrity Checking](#integrity-checking) - [Error Handling](#error-handling) - [Imports](#imports) -- [Testing the Serialisation](#testing-the-serialisation) +- [Testing the Serialization](#testing-the-serialization) - [Future Directions](#future-directions) -## Serialising the IR +## Serializing the IR -As the serialised IR doesn't need to be read by anything other than Enso, we -need not use a representation that is portable between platforms. As a result, -we have picked the `Serializable` infrastructure that is _already present_ on -the JVM. It has the following benefits: +Using classical Java Serialization turned out to be unsuitably slow. Rather than +switching to other serialization framework that does the same, but faster we +desided in [PR-8207](https://github.com/enso-org/enso/pull/8207) to create _own +persistance framework_ that radically changes the way we can read the caches. +Rather than loading all the megabytes of stored data, it reads them _lazily on +demand_. -- It is able to serialise arbitrary object graphs while maintaining object - identity and tracking references. This cannot be disabled for `Serializable`, - but that is fine as we want it. -- It is built into the JVM and is hence guaranteed to be portable between - instances of the same JVM. -- It copes fine with highly-nested scala types, like our IR. +Use following command to generate the Javadoc for the `org.enso.persist` +package: -In order to maximise the benefits of this process, we want to serialise the IR -as _late_ in the compiler pipeline as possible. This means serialising it just +```bash +enso$ find lib/java/persistance/src/main/java/ | grep java$ | xargs ~/bin/graalvm-21/bin/javadoc -d target/javadoc/ --snippet-path lib/java/persistance/src/test/java/ +enso$ links target/javadoc/index.html +``` + +In order to maximize the benefits of this process, we want to serialize the IR +as _late_ in the compiler pipeline as possible. This means serializing it just before the code generation step that generates Truffle nodes (before the `RuntimeStubsGenerator` and `IrToTruffle` run). -This serialisation should take place in an _offloaded thread_ so that it doesn't +This serialization should take place in an _offloaded thread_ so that it doesn't block the compiler from continuing. ### Breaking Links -Doing this naïvely, however, means that we can inadvertently end up serialising +Doing this naïvely, however, means that we can inadvertently end up serializing the entire module graph. This is due to the `BindingsMap`, which contains a reference to the associated `runtime.Module`, from which there is a reference to the `ModuleScope`. The `ModuleScope` may then reference other `runtime.Module`s which all contain `IR.Module`s. Therefore, done in a silly fashion, we end up -serialising the entire reachable module graph. This is not what we want. +serializing the entire reachable module graph. This is not what we want. -While the ideal way of solving this problem would be to customise the -serialisation and deserialisation process for the `BindingsMap`, the JVM's -`Serializable` does not provide the ability to customise it enough to solve this -problem. Instead, we solve it using a preprocessing step: +The `Persistance.write` method contains additional `writeReplace` function which +our cache system uses to perform following modification just before +`ProcessingPass.Metadata` are stored down: -- We can modify `BindingsMap` and its child types to be able to contain an - unlinked module pointer - `case class ModulePointer(qualifiedName: List[String])` in place of a - `Module`. -- As the `MetadataStorage` type that holds the `BindingsMap` is mutable it can - be updated in place without having to reassemble the entire IR graph. -- Hence, we can traverse all the nodes in the `ir.preorder` that have metadata - consisting of either the `BindingsMap` or `ResolvedName` types (provided by - the following passes: `BindingAnalysis`, `MethodDefinitions`, `GlobalNames`, - `VectorLiterals`, `Patterns`), and perform a replacement. +- modify `BindingsMap` and its child types to be able to contain an unlinked + module pointer `case class ModulePointer(qualifiedName: List[String])` in + place of a `Module`. +- As the `MetadataStorage` type that holds the `BindingsMap` is mutable it might + be tempting to update it in place, but relying on `writeReplace` mechanism is + safer as it only changes the format of object being written down, rather than + modifying objects of live `IR` - potentially shared with other parts of the + system. Having done this, we have broken any links that the IR may hold between modules, -and can serialise each module individually. +and can serialize each module individually. -This serialisation must take place _after_ codegen has happened as it modifies -the IR in place. The compiler can handle giving it to the offloaded -serialisation thread. It _may_ be necessary to `duplicate` the IR before handing -it to this thread, but this should be checked during development. +It _may_ be safer to `duplicate` the IR before handing it to serialization, but +it shouldn't be necessary if the `writeReplace` function is written correctly. ## Storing the IR @@ -146,12 +144,28 @@ All hashes are encoded in SHA1 format, for performance reasons. The engine version is encoded in the cache path, and hence does not need to be explicitly specified in the metadata. -### Portability Guarantees +### Portability and Versioning -As part of this design we provide only the following portability guarantees: +These are two static methods in `Persistance` class to help creating a `byte[]` +from a single object and then read it back. The array is identified with +following header: -- The serialised IR must be able to be deserialised by _the same version of - Enso_ that wrote the original blob. +- 4 bytes fixed header +- 4 bytes describing the version +- 4 bytes to locate the beginning of the object (the objects aren't written + linearly) + +E.g. 12 bytes overhead before the actual data start. Following versioning is +recommended when making a change: + +- when you change something really core in the `Persitance` implementation - + change the builtin header first four bytes +- when you add or remove a Persistance implementation the version changes (as it + is computed from all the IDs present in the system) +- when you change format of some `Persitance.writeObject` method - change its ID + +That way the same version of Enso will recognize its `.ir` files. Different +versions of Enso will realize that the files aren't in suitable form. ## Loading the IR @@ -165,10 +179,12 @@ checking on the loaded cache. It works as follows. 2. **Check Integrity:** Check the module's [metadata](#metadata-format) for validity according to the [integrity rules](#integrity-checking). 3. **Load:** If the cache passes the integrity check, load the `.ir` file. If - deserialisation fails in any way, immediately fall back to parsing the source + deserialization fails in any way, immediately fall back to parsing the source file. -4. **Re-Link:** If loading completed successfully, re-link the `BindingsMap` - metadata to the proper modules in question. +4. **Re-Link:** Relinking is part of **Load**. When using `Persistance.read` + provide own `readResolve` function. Such a function gets a chance to change + and replace each object read-in with appropriate variant respecting the whole + compiler environment. The main subtlety here is handling the dependencies between modules. We need to ensure that, when loading multiple cached libraries, we properly handle them @@ -177,8 +193,8 @@ setting `AFTER_STATIC_PASSES` as the compilation state after loading the module. This will tie into the current `ImportsResolver` and `ExportsResolver` which are run in an un-gated fashion in `Compiler::run`. -In order to prevent the execution of malicious code when deserialising we should -employ a deserialisation filter as built into the JDK. +Unlike classical Java deserialization nly registered `Persistance` subclasses +may participate in deserialization making it much safer and less vulnerable. ### Integrity Checking @@ -195,8 +211,8 @@ ignored if it is in a read-only location. It is important, as part of this, that we fail under all circumstances into a working state. This means that: -- If serialisation fails, we report a low-priority error message and continue. -- If deserialisation fails, we fall back to loading and parsing the original +- If serialization fails, we report a low-priority error message and continue. +- If deserialization fails, we fall back to loading and parsing the original source file. At no point should this mechanism be exposed to the user in any visible way, @@ -215,12 +231,15 @@ until a complete cache invalidation was forced. Therefore, the compiler performs an additional check by invalidating module's cache if any of its imported modules have been invalidated. -## Testing the Serialisation +## Testing the Serialization There are two main elements that need to be tested as part of this feature. -- Firstly, we need to test the serialisation and deserialisation process, - including the rewrite of `BindingsMap` to work properly. +- `persistance` project comes with its own unit tests +- `runtime-parser` project adds tests of various core classes used during `IR` + serialization - like Scala `List` or checks of the _laziness_ of Scala `Seq` +- We need to test the serialization and deserialization process, including the + rewrite of `BindingsMap` to work properly. - We also need to test the discovery of cache locations on the filesystem and cache eviction strategies. The best way to do this is to set `$ENSO_DATA` to a temporary directory and then directly interact with the filesystem. Caching @@ -243,16 +262,23 @@ library in a single pass. The bindings are serialized along with the library caches in a file with a `.bindings` suffix. +Further more the storage of `.ir` files contains usage of _lazy_ `Seq` +references to separate the general part of the `IR` tree from elements +representing method bodies. As such the compiler can process the structure of +`.ir` files, but avoid loading in `IR` for methods that aren't being executed. + ## Future Directions -Due to the less than ideal platform situation we're in, we're limited to using -Java's `Serializable`. It is not as performant as other options. +The `Persistance` framework gives us _laziness_ opportunities and we should use +them more: -- [FST](https://github.com/RuedigerMoeller/fast-serialization) is around 10x - faster than the JVM's serialization, and is a drop-in replacement. -- However, the version that supports Java 11 utilises reflection that trips - warnings that will be disallowed with Java 17 (the next LTS version for - GraalVM). -- The version that fixes this relies on the foreign memory API which is - available in Java 17. I recommend that once we're on Java 17 builds the - serialization is updated to work using FST. +- have a single _blob_ with all `IR`s per a library and read only the parts that + are needed + +- experiement with GC - being able to release parts of unused `IR` once they + were used (for code generation or co.) + +- make the `.ir` files smaller where possible + +The use of `Persistance` has already sped up the execution time of simple +`IO.println "Hello!"` by 16% - let's use it to speed things up even more. diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index 71497fc28e..159b1d3b9d 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -572,7 +572,8 @@ object Main { topScope.compile(shouldCompileDependencies) exitSuccess() } catch { - case _: Throwable => + case t: Throwable => + logger.error("Unexpected internal error", t) exitFail() } finally { context.context.close() diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java index c6a6122855..dea58aaf38 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java @@ -81,8 +81,6 @@ public interface CompilerContext extends CompilerStub { boolean wasLoadedFromCache(Module module); - boolean hasCrossModuleLinks(Module module); - org.enso.compiler.core.ir.Module getIr(Module module); CompilationStage getCompilationStage(Module module); @@ -108,8 +106,6 @@ public interface CompilerContext extends CompilerStub { void loadedFromCache(boolean b); - void hasCrossModuleLinks(boolean b); - void resetScope(); void invalidateCache(); @@ -134,8 +130,6 @@ public interface CompilerContext extends CompilerStub { public abstract boolean isSynthetic(); - public abstract boolean hasCrossModuleLinks(); - public abstract org.enso.compiler.core.ir.Module getIr(); public abstract boolean isPrivate(); diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/pass/analyse/PassPersistance.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/pass/analyse/PassPersistance.java new file mode 100644 index 0000000000..e4d66da2c0 --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/pass/analyse/PassPersistance.java @@ -0,0 +1,155 @@ +package org.enso.compiler.pass.analyse; + +import java.io.IOException; +import org.enso.compiler.pass.analyse.AliasAnalysis.Graph; +import org.enso.compiler.pass.resolve.DocumentationComments; +import org.enso.compiler.pass.resolve.FullyQualifiedNames; +import org.enso.compiler.pass.resolve.IgnoredBindings; +import org.enso.compiler.pass.resolve.ModuleAnnotations; +import org.enso.compiler.pass.resolve.TypeSignatures; +import org.enso.persist.Persistable; +import org.enso.persist.Persistance; +import org.openide.util.lookup.ServiceProvider; +import scala.Option; + +@Persistable(clazz = CachePreferenceAnalysis.WeightInfo.class, id = 1111) +@Persistable(clazz = DataflowAnalysis.DependencyInfo.class, id = 1112) +@Persistable(clazz = DataflowAnalysis.DependencyMapping.class, id = 1113) +@Persistable(clazz = GatherDiagnostics.DiagnosticsMeta.class, id = 1114) +@Persistable(clazz = DocumentationComments.Doc.class, id = 1115) +@Persistable(clazz = AliasAnalysis$Info$Occurrence.class, id = 1116) +@Persistable(clazz = TypeSignatures.Signature.class, id = 1117) +@Persistable(clazz = ModuleAnnotations.Annotations.class, id = 1118) +@Persistable(clazz = AliasAnalysis$Info$Scope$Root.class, id = 1120) +@Persistable(clazz = DataflowAnalysis$DependencyInfo$Type$Static.class, id = 1121) +@Persistable(clazz = DataflowAnalysis$DependencyInfo$Type$Dynamic.class, id = 1122) +@Persistable(clazz = AliasAnalysis$Info$Scope$Child.class, id = 1123) +@Persistable(clazz = AliasAnalysis$Graph$Occurrence$Use.class, id = 1125) +@Persistable(clazz = AliasAnalysis$Graph$Occurrence$Def.class, id = 1126) +@Persistable(clazz = AliasAnalysis$Graph$Link.class, id = 1127) +@Persistable(clazz = FullyQualifiedNames.FQNResolution.class, id = 1128) +@Persistable(clazz = FullyQualifiedNames.ResolvedLibrary.class, id = 1129) +@Persistable(clazz = FullyQualifiedNames.ResolvedModule.class, id = 1130) +public final class PassPersistance { + private PassPersistance() {} + + @ServiceProvider(service = Persistance.class) + public static final class PersistState extends Persistance { + public PersistState() { + super(IgnoredBindings.State.class, true, 1101); + } + + @Override + protected void writeObject(IgnoredBindings.State obj, Output out) throws IOException { + out.writeBoolean(obj.isIgnored()); + } + + @Override + protected IgnoredBindings.State readObject(Input in) + throws IOException, ClassNotFoundException { + var b = in.readBoolean(); + return b + ? org.enso.compiler.pass.resolve.IgnoredBindings$State$Ignored$.MODULE$ + : org.enso.compiler.pass.resolve.IgnoredBindings$State$NotIgnored$.MODULE$; + } + } + + @ServiceProvider(service = Persistance.class) + public static final class PersistTail extends Persistance { + public PersistTail() { + super(TailCall.TailPosition.class, true, 1102); + } + + @Override + protected void writeObject(TailCall.TailPosition obj, Output out) throws IOException { + out.writeBoolean(obj.isTail()); + } + + @Override + protected TailCall.TailPosition readObject(Input in) + throws IOException, ClassNotFoundException { + var b = in.readBoolean(); + return b + ? org.enso.compiler.pass.analyse.TailCall$TailPosition$Tail$.MODULE$ + : org.enso.compiler.pass.analyse.TailCall$TailPosition$NotTail$.MODULE$; + } + } + + @org.openide.util.lookup.ServiceProvider(service = Persistance.class) + public static final class PersistAliasAnalysisGraphScope + extends Persistance { + public PersistAliasAnalysisGraphScope() { + super(org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope.class, false, 1124); + } + + @Override + @SuppressWarnings("unchecked") + protected org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope readObject(Input in) + throws IOException { + var childScopes = in.readInline(scala.collection.immutable.List.class); + var occurrences = (scala.collection.immutable.Set) in.readObject(); + var allDefinitions = in.readInline(scala.collection.immutable.List.class); + var parent = + new org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope( + childScopes, occurrences, allDefinitions); + var optionParent = Option.apply(parent); + childScopes.forall( + (object) -> { + var ch = (org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope) object; + ch.parent_$eq(optionParent); + return null; + }); + return parent; + } + + @Override + @SuppressWarnings("unchecked") + protected void writeObject( + org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope obj, Output out) + throws IOException { + out.writeInline(scala.collection.immutable.List.class, obj.childScopes()); + out.writeObject(obj.occurrences()); + out.writeInline(scala.collection.immutable.List.class, obj.allDefinitions()); + } + } + + @org.openide.util.lookup.ServiceProvider(service = Persistance.class) + public static final class PersistAliasAnalysisGraph extends Persistance { + public PersistAliasAnalysisGraph() { + super(Graph.class, false, 1119); + } + + @SuppressWarnings("unchecked") + protected Graph readObject(Input in) throws IOException { + var g = new Graph(); + + var rootScope = (AliasAnalysis$Graph$Scope) in.readObject(); + assignParents(rootScope); + g.rootScope_$eq(rootScope); + + var links = + (scala.collection.immutable.Set) in.readInline(scala.collection.immutable.Set.class); + g.links_$eq(links); + return g; + } + + @SuppressWarnings("unchecked") + @Override + protected void writeObject(Graph obj, Output out) throws IOException { + out.writeObject(obj.rootScope()); + out.writeInline(scala.collection.immutable.Set.class, obj.links()); + } + + private static void assignParents(AliasAnalysis$Graph$Scope scope) { + var option = Option.apply(scope); + scope + .childScopes() + .foreach( + (ch) -> { + assignParents(ch); + ch.parent_$eq(option); + return null; + }); + } + } +} diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala index 523ef8d3af..3560667f54 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala @@ -257,7 +257,7 @@ class Compiler( } ) - var requiredModules = modules.flatMap { module => + val requiredModules = modules.flatMap { module => val importedModules = runImportsAndExportsResolution(module, generateCode) val isLoadedFromSource = (m: Module) => !context.wasLoadedFromCache(m) && !context.isSynthetic(m) @@ -286,47 +286,11 @@ class Compiler( } }.distinct - var hasInvalidModuleRelink = false if (irCachingEnabled) { requiredModules.foreach { module => ensureParsed(module) - if (!context.hasCrossModuleLinks(module)) { - val flags = - context - .getIr(module) - .preorder - .map(_.passData.restoreFromSerialization(this.context)) - - if (!flags.contains(false)) { - context.log( - Compiler.defaultLogLevel, - "Restored links (late phase) for module [{0}].", - context.getModuleName(module) - ) - } else { - hasInvalidModuleRelink = true - context.log( - Compiler.defaultLogLevel, - "Failed to restore links (late phase) for module [{0}].", - context.getModuleName(module) - ) - uncachedParseModule(module, isGenDocs = false) - } - } } } - - if (hasInvalidModuleRelink) { - context.log( - Compiler.defaultLogLevel, - s"Some modules failed to relink. Re-running import and " + - s"export resolution." - ) - - requiredModules = - modules.flatMap(runImportsAndExportsResolution(_, generateCode)) - } - requiredModules.foreach { module => if ( !context @@ -629,7 +593,6 @@ class Compiler( u.ir(discoveredModule) u.compilationStage(CompilationStage.AFTER_PARSING) u.loadedFromCache(false) - u.hasCrossModuleLinks(true) } ) } @@ -892,7 +855,9 @@ class Compiler( ): Unit = { if (config.isStrictErrors) { val diagnostics = modules.flatMap { module => - val errors = gatherDiagnostics(module) + val errors = + if (context.wasLoadedFromCache(module)) List() + else gatherDiagnostics(module) List((module, errors)) } if (reportDiagnostics(diagnostics)) { diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/LocalScope.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/LocalScope.scala index 099a3beb04..885c114fad 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/LocalScope.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/LocalScope.scala @@ -84,7 +84,10 @@ class LocalScope( * @return the frame slot index for `id`. */ def getVarSlotIdx(id: Graph.Id): Int = { - assert(localFrameSlotIdxs.contains(id)) + assert( + localFrameSlotIdxs.contains(id), + "Cannot find " + id + " in " + localFrameSlotIdxs + ) localFrameSlotIdxs(id) } diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala index 4dc17bca26..b797458a4a 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala @@ -11,6 +11,7 @@ import org.enso.compiler.core.ir.{ Function, IdentifiedLocation, Literal, + Location, Name, Type } @@ -28,7 +29,6 @@ import org.enso.compiler.pass.resolve.{ import org.enso.pkg.QualifiedName import org.enso.polyglot.Suggestion import org.enso.polyglot.data.{Tree, TypeGraph} -import org.enso.syntax.text.Location import org.enso.text.editing.IndexedSource import java.util.UUID diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/DemandAnalysis.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/DemandAnalysis.scala index 0d87234fce..87828149e6 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/DemandAnalysis.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/DemandAnalysis.scala @@ -8,6 +8,7 @@ import org.enso.compiler.core.ir.{ Empty, Expression, Function, + IdentifiedLocation, Literal, Module, Name, @@ -185,9 +186,10 @@ case object DemandAnalysis extends IRPass { } else { name match { case lit: Name.Literal if isDefined(lit) => - val forceLocation = name.location - val newNameLocation = name.location.map(l => l.copy(id = None)) - val newName = lit.copy(location = newNameLocation) + val forceLocation = name.location + val newNameLocation = + name.location.map(l => new IdentifiedLocation(l.location())) + val newName = lit.copy(location = newNameLocation) Application.Force(newName, forceLocation) case _ => name } diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/FunctionBinding.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/FunctionBinding.scala index 08dbda3cfc..916f95f5c7 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/FunctionBinding.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/FunctionBinding.scala @@ -111,7 +111,7 @@ case object FunctionBinding extends IRPass { val lambda = args .map(_.mapExpressions(desugarExpression)) .foldRight(desugarExpression(body))((arg, body) => - Function.Lambda(List(arg), body, None) + new Function.Lambda(List(arg), body, None) ) .asInstanceOf[Function.Lambda] .copy(canBeTCO = canBeTCO, location = location) @@ -155,10 +155,10 @@ case object FunctionBinding extends IRPass { val newBody = args .map(_.mapExpressions(desugarExpression)) .foldRight(desugarExpression(body))((arg, body) => - Function.Lambda(List(arg), body, None) + new Function.Lambda(List(arg), body, None) ) - definition.Method.Explicit( + new definition.Method.Explicit( methRef, newBody, loc, @@ -245,7 +245,7 @@ case object FunctionBinding extends IRPass { val newBody = (requiredArgs ::: remainingArgs) .map(_.mapExpressions(desugarExpression)) .foldRight(desugarExpression(body))((arg, body) => - Function.Lambda(List(arg), body, None) + new Function.Lambda(List(arg), body, None) ) Right( definition.Method.Conversion( diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/GenerateMethodBodies.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/GenerateMethodBodies.scala index 6a9a922cd4..6592cc3c6a 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/GenerateMethodBodies.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/GenerateMethodBodies.scala @@ -244,7 +244,7 @@ case object GenerateMethodBodies extends IRPass { expr: Expression, funName: Name ): Expression = { - Function.Lambda( + new Function.Lambda( arguments = if (funName.name == MAIN_FUNCTION_NAME) Nil else genSyntheticSelf() :: Nil, diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/LambdaShorthandToLambda.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/LambdaShorthandToLambda.scala index 86bf4a9f76..3fcf31efe9 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/LambdaShorthandToLambda.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/LambdaShorthandToLambda.scala @@ -7,6 +7,7 @@ import org.enso.compiler.core.ir.{ DefinitionArgument, Expression, Function, + IdentifiedLocation, Module, Name, Type @@ -142,7 +143,7 @@ case object LambdaShorthandToLambda extends IRPass { case blank: Name.Blank => val newName = supply.newName() - Function.Lambda( + new Function.Lambda( List( DefinitionArgument.Specified( name = Name.Literal( @@ -221,12 +222,12 @@ case object LambdaShorthandToLambda extends IRPass { // arg val appResult = actualDefArgs.foldRight(processedApp: Expression)((arg, body) => - Function.Lambda(List(arg), body, None) + new Function.Lambda(List(arg), body, None) ) // If the function is shorthand, do the same val resultExpr = if (functionIsShorthand) { - Function.Lambda( + new Function.Lambda( List( DefinitionArgument.Specified( Name @@ -267,8 +268,9 @@ case object LambdaShorthandToLambda extends IRPass { name case it => desugarExpression(it, freshNameSupply) } - val newVec = vector.copy(newItems) - val locWithoutId = newVec.location.map(_.copy(id = None)) + val newVec = vector.copy(newItems) + val locWithoutId = + newVec.location.map(l => new IdentifiedLocation(l.location())) bindings.foldLeft(newVec: Expression) { (body, bindingName) => val defArg = DefinitionArgument.Specified( bindingName, @@ -277,7 +279,7 @@ case object LambdaShorthandToLambda extends IRPass { suspended = false, location = None ) - Function.Lambda(List(defArg), body, locWithoutId) + new Function.Lambda(List(defArg), body, locWithoutId) } case tSet @ Application.Typeset(expr, _, _, _) => tSet.copy(expression = expr.map(desugarExpression(_, freshNameSupply))) @@ -422,7 +424,7 @@ case object LambdaShorthandToLambda extends IRPass { branches = newBranches ) - Function.Lambda( + new Function.Lambda( List(lambdaArg), newCaseExpr, caseExpr.location, diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/SectionsToBinOp.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/SectionsToBinOp.scala index c29141f1f3..3c1189c281 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/SectionsToBinOp.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/desugar/SectionsToBinOp.scala @@ -132,13 +132,13 @@ case object SectionsToBinOp extends IRPass { diagnostics ) - val rightLam = Function.Lambda( + val rightLam = new Function.Lambda( List(rightDefArg), opCall, None ) - Function.Lambda( + new Function.Lambda( List(leftDefArg), rightLam, loc @@ -187,13 +187,13 @@ case object SectionsToBinOp extends IRPass { diagnostics ) - val rightLambda = Function.Lambda( + val rightLambda = new Function.Lambda( List(rightDefArg), opCall, None ) - Function.Lambda( + new Function.Lambda( List(leftDefArg), rightLambda, loc @@ -253,13 +253,13 @@ case object SectionsToBinOp extends IRPass { diagnostics ) - val leftLam = Function.Lambda( + val leftLam = new Function.Lambda( List(leftDefArg), opCall, None ) - Function.Lambda( + new Function.Lambda( List(rightDefArg), leftLam, loc @@ -274,7 +274,7 @@ case object SectionsToBinOp extends IRPass { diagnostics ) - Function.Lambda( + new Function.Lambda( List(leftDefArg), opCall, loc diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/LambdaConsolidate.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/LambdaConsolidate.scala index 5f6c9df252..2d75cebef8 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/LambdaConsolidate.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/LambdaConsolidate.scala @@ -9,6 +9,7 @@ import org.enso.compiler.core.ir.{ Expression, Function, IdentifiedLocation, + Location, Module, Name } @@ -23,7 +24,6 @@ import org.enso.compiler.pass.analyse.{ } import org.enso.compiler.pass.desugar._ import org.enso.compiler.pass.resolve.IgnoredBindings -import org.enso.syntax.text.Location import java.util.UUID @@ -171,8 +171,8 @@ case object LambdaConsolidate extends IRPass { val newLocation = chainedLambdas.head.location match { case Some(location) => Some( - IdentifiedLocation( - Location( + IdentifiedLocation.create( + new Location( location.start, chainedLambdas.last.location.getOrElse(location).location.end ), diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/UnreachableMatchBranches.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/UnreachableMatchBranches.scala index 9b6c01bd3d..d7d02c7ae8 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/UnreachableMatchBranches.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/optimise/UnreachableMatchBranches.scala @@ -5,6 +5,7 @@ import org.enso.compiler.core.Implicits.AsDiagnostics import org.enso.compiler.core.ir.{ Expression, IdentifiedLocation, + Location, Module, Pattern } @@ -19,7 +20,6 @@ import org.enso.compiler.pass.analyse.{ } import org.enso.compiler.pass.desugar._ import org.enso.compiler.pass.resolve.{DocumentationComments, IgnoredBindings} -import org.enso.syntax.text.Location import scala.annotation.unused @@ -146,8 +146,8 @@ case object UnreachableMatchBranches extends IRPass { branch.location match { case Some(branchLoc) => Some( - IdentifiedLocation( - Location(loc.start, branchLoc.end), + IdentifiedLocation.create( + new Location(loc.start, branchLoc.end), loc.id ) ) diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/MethodDefinitions.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/MethodDefinitions.scala index c4a8ae26b1..04c1765b4b 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/MethodDefinitions.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/MethodDefinitions.scala @@ -101,7 +101,7 @@ case object MethodDefinitions extends IRPass { if canGenerateStaticWrappers(tp) => val dup = method.duplicate() val static = dup.copy(body = - Function.Lambda( + new Function.Lambda( List( DefinitionArgument .Specified( diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/ModuleAnnotations.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/ModuleAnnotations.scala index bfaf7e548d..c88158f193 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/ModuleAnnotations.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/resolve/ModuleAnnotations.scala @@ -128,9 +128,6 @@ case object ModuleAnnotations extends IRPass { /** @inheritdoc */ override def prepareForSerialization(compiler: Compiler): Annotations = { - annotations.foreach(ir => - ir.preorder.foreach(_.passData.prepareForSerialization(compiler)) - ) this } @@ -138,14 +135,6 @@ case object ModuleAnnotations extends IRPass { override def restoreFromSerialization( compiler: Compiler ): Option[IRPass.IRMetadata] = { - annotations.foreach { ann => - ann.preorder.foreach { ir => - if (!ir.passData.restoreFromSerialization(compiler)) { - return None - } - } - } - Some(this) } } diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala index 0fc3518040..12b061d2a5 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala @@ -58,7 +58,7 @@ class ImportResolver(compiler: Compiler) { .getCompilationStage(current) .isBefore( CompilationStage.AFTER_IMPORT_RESOLUTION - ) || !context.hasCrossModuleLinks(current) + ) ) { val importedModules: List[ (Import, Option[BindingsMap.ResolvedImport]) diff --git a/engine/runtime-instrument-common/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala b/engine/runtime-instrument-common/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala index 8b650e2df7..997b4f1238 100644 --- a/engine/runtime-instrument-common/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala +++ b/engine/runtime-instrument-common/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala @@ -12,11 +12,11 @@ import org.enso.compiler.core.{ } import org.enso.compiler.core.Implicits.AsMetadata import org.enso.compiler.core.ir.Literal +import org.enso.compiler.core.ir.Location import org.enso.compiler.core.ir.Name import org.enso.compiler.core.ir.module.scope.definition import org.enso.compiler.pass.analyse.DataflowAnalysis import org.enso.interpreter.instrument.execution.model.PendingEdit -import org.enso.syntax.text.Location import org.enso.text.editing.model.TextEdit import org.enso.text.editing.{IndexedSource, TextEditor} @@ -288,7 +288,7 @@ object ChangesetBuilder { * @return the node with a new location */ def shift(offset: Int): Node = { - val newLocation = location.copy( + val newLocation = new Location( start = location.start + offset, end = location.end + offset ) @@ -359,7 +359,7 @@ object ChangesetBuilder { val nodeBetweenPreviousPositionAndNextNode = Node( NodeId(currentIr), - Location(previousPosition, nextNode.location.start), + new Location(previousPosition, nextNode.location.start), false ) acc += nodeBetweenPreviousPositionAndNextNode @@ -378,7 +378,7 @@ object ChangesetBuilder { if (hasRemainingTextAfterLastChild) { val nodeAfterLastChild = Node( NodeId(currentIr), - Location(lastCoveredPosition, endOfNonLeafIr), + new Location(lastCoveredPosition, endOfNonLeafIr), false ) acc += nodeAfterLastChild @@ -491,7 +491,7 @@ object ChangesetBuilder { edit: TextEdit, source: A ): Location = { - Location( + new Location( IndexedSource[A].toIndex(edit.range.start, source), IndexedSource[A].toIndex(edit.range.end, source) ) diff --git a/engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/execution/LocationResolver.scala b/engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/execution/LocationResolver.scala index 80297ff2d4..5d3ac17fda 100644 --- a/engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/execution/LocationResolver.scala +++ b/engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/execution/LocationResolver.scala @@ -2,9 +2,9 @@ package org.enso.interpreter.instrument.execution import com.oracle.truffle.api.source.SourceSection import org.enso.compiler.core.{ExternalID, IR, Identifier} +import org.enso.compiler.core.ir.Location import org.enso.compiler.core.ir.IdentifiedLocation import org.enso.interpreter.runtime.Module -import org.enso.syntax.text.Location import org.enso.text.editing.{model, IndexedSource} import java.util.UUID @@ -121,7 +121,7 @@ object LocationResolver { source: A ): Location = { val range = sectionToRange(section) - Location( + new Location( IndexedSource[A].toIndex(range.start, source), IndexedSource[A].toIndex(range.end, source) ) diff --git a/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java b/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java index 5f314f703f..4f7a1a81f3 100644 --- a/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java +++ b/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java @@ -12,6 +12,7 @@ import org.enso.compiler.core.ir.Empty; import org.enso.compiler.core.ir.Expression; import org.enso.compiler.core.ir.Function; import org.enso.compiler.core.ir.Literal; +import org.enso.compiler.core.ir.Location; import org.enso.compiler.core.ir.Name; import org.enso.compiler.core.ir.MetadataStorage; import org.enso.compiler.core.ir.Module; @@ -29,7 +30,6 @@ import org.enso.compiler.core.ir.module.scope.definition.Method; import org.enso.compiler.core.ir.module.scope.Export; import org.enso.compiler.core.ir.module.scope.Import; import org.enso.compiler.core.ir.module.scope.imports.Polyglot; -import org.enso.syntax.text.Location; import org.enso.syntax2.ArgumentDefinition; import org.enso.syntax2.Base; import org.enso.syntax2.DocComment; @@ -41,6 +41,7 @@ import org.enso.syntax2.Tree; import org.enso.syntax2.Tree.Invalid; import org.enso.syntax2.Tree.Private; + import scala.Option; import scala.collection.immutable.LinearSeq; import scala.collection.immutable.List; @@ -121,7 +122,7 @@ final class TreeToIr { locations.get(1).start(), locations.get(locations.size() - 1).end() ), - Option.empty() + null ) ); } @@ -842,7 +843,7 @@ final class TreeToIr { var locationWithANewLine = getIdentifiedLocation(body, 0, 0, null); if (last != null && last.location().isDefined() && last.location().get().end() != locationWithANewLine.get().end()) { var patched = new Location(last.location().get().start(), locationWithANewLine.get().end() - 1); - var id = new IdentifiedLocation(patched, last.location().get().id()); + var id = IdentifiedLocation.create(patched, last.location().get().id()); last = last.setLocation(Option.apply(id)); } yield new Expression.Block(list, last, locationWithANewLine, false, meta(), diag()); @@ -1693,7 +1694,7 @@ final class TreeToIr { Math.min(en.start(), in.start()), Math.max(en.end(), in.end()) ); - return Option.apply(new IdentifiedLocation(loc, en.id())); + return Option.apply(IdentifiedLocation.create(loc, en.id())); } else { return encapsulating; } @@ -1716,7 +1717,7 @@ final class TreeToIr { default -> { var begin = castToInt(ast.getStartCode()) + b; var end = castToInt(ast.getEndCode()) + e; - yield new IdentifiedLocation(new Location(begin, end), someId); + yield IdentifiedLocation.create(new Location(begin, end), someId); } }); } @@ -1751,7 +1752,7 @@ final class TreeToIr { end = ast.getPattern().getEndCode(); } int end_ = castToInt(end); - return Option.apply(new IdentifiedLocation(new Location(begin_, end_), Option.empty())); + return Option.apply(IdentifiedLocation.create(new Location(begin_, end_), Option.empty())); } private Option getIdentifiedLocation(Token ast) { @@ -1763,7 +1764,7 @@ final class TreeToIr { default -> { int begin = castToInt(ast.getStartCode()); int end = castToInt(ast.getEndCode()); - var id = Option.apply(generateId ? UUID.randomUUID() : null); + var id = generateId ? UUID.randomUUID() : null; yield new IdentifiedLocation(new Location(begin, end), id); } }); diff --git a/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IdentifiedLocation.java b/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IdentifiedLocation.java new file mode 100644 index 0000000000..30d6ce3c9e --- /dev/null +++ b/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IdentifiedLocation.java @@ -0,0 +1,45 @@ +package org.enso.compiler.core.ir; + +import java.util.UUID; +import scala.Option; + +public record IdentifiedLocation(Location location, UUID uuid) { + public IdentifiedLocation(Location location) { + this(location, (UUID)null); + } + + /** + * Creates new location from an optional UUID. + */ + public static IdentifiedLocation create(Location location, Option uuid) { + return new IdentifiedLocation(location, uuid.isEmpty() ? null : uuid.get()); + } + + /** @return the character index of the start of this source location. + */ + public int start() { + return location().start(); + } + + /** @return the character index of the end of this source location. + */ + public int end() { + return location().end(); + } + + /** @return the length in characters of this location. + */ + public int length() { + return location().length(); + } + + /** @return option with/out UUID */ + public Option id() { + return Option.apply(uuid()); + } + + @Override + public String toString() { + return "IdentifiedLocation[location=" + this.location() + ", uuid="+ id() + "]"; + } +} diff --git a/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrLazySeq.java b/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrLazySeq.java new file mode 100644 index 0000000000..637030eaae --- /dev/null +++ b/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrLazySeq.java @@ -0,0 +1,69 @@ +package org.enso.compiler.core.ir; + +import java.util.NoSuchElementException; +import org.enso.persist.Persistance; +import scala.collection.Iterator; +import scala.collection.SeqFactory; +import scala.collection.immutable.AbstractSeq; + +final class IrLazySeq extends AbstractSeq { + private final Persistance.Reference[] arr; + private final int size; + + IrLazySeq(Persistance.Reference[] arr, int size) { + this.arr = arr; + this.size = size; + } + + @Override + public Object apply(int i) throws IndexOutOfBoundsException { + return arr[i].get(Object.class); + } + + @Override + public int length() { + return size; + } + + @Override + public boolean isDefinedAt(int idx) { + return 0 <= idx && idx < size; + } + + @Override + public boolean isDefinedAt(Object idx) { + throw new IllegalStateException(); + } + + @Override + public Object apply(Object i) throws IndexOutOfBoundsException { + throw new IllegalStateException(); + } + + @Override + public SeqFactory iterableFactory() { + return super.iterableFactory(); + } + + @Override + public Iterator iterator() { + return new IrIter(); + } + + private final class IrIter implements Iterator { + private int at; + + @Override + public boolean hasNext() { + return at < size; + } + + @Override + public Object next() throws NoSuchElementException { + if (at >= size) { + throw new NoSuchElementException(); + } + return apply(at++); + } + } +} diff --git a/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrPersistance.java b/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrPersistance.java new file mode 100644 index 0000000000..4a7a986ac8 --- /dev/null +++ b/engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrPersistance.java @@ -0,0 +1,439 @@ +package org.enso.compiler.core.ir; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.enso.compiler.core.ir.expression.Application; +import org.enso.compiler.core.ir.expression.Case; +import org.enso.compiler.core.ir.expression.Foreign; +import org.enso.compiler.core.ir.expression.warnings.Unused; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.Export; +import org.enso.compiler.core.ir.module.scope.Import; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.compiler.core.ir.module.scope.imports.Polyglot; +import org.enso.compiler.core.ir.type.Set; +import org.enso.persist.Persistable; +import org.enso.persist.Persistance; +import org.openide.util.lookup.ServiceProvider; +import scala.Option; +import scala.Tuple2; +import scala.collection.immutable.List; +import scala.collection.immutable.Seq; + +@Persistable(clazz = Module.class, id = 201) +@Persistable(clazz = Name.Literal.class, id = 351) +@Persistable(clazz = Import.Module.class, id = 342) +@Persistable(clazz = Polyglot.class, id = 343) +@Persistable(clazz = Export.Module.class, id = 344) +@Persistable(clazz = Name.Qualified.class, id = 352) +@Persistable(clazz = Method.Explicit.class, id = 361) +@Persistable(clazz = Name.MethodReference.class, id = 362) +@Persistable(clazz = Function.Lambda.class, id = 363) +@Persistable(clazz = Polyglot.Java.class, id = 703) +@Persistable(clazz = DefinitionArgument.Specified.class, id = 704) +@Persistable(clazz = Name.Self.class, id = 705) +@Persistable(clazz = Literal.Number.class, id = 706) +@Persistable(clazz = Literal.Text.class, id = 707) +@Persistable(clazz = CallArgument.Specified.class, id = 708) +@Persistable(clazz = Definition.Type.class, id = 709) +@Persistable(clazz = Definition.Data.class, id = 710) +@Persistable(clazz = Name.Blank.class, id = 711) +@Persistable(clazz = Name.GenericAnnotation.class, id = 712) +@Persistable(clazz = Name.SelfType.class, id = 713) +@Persistable(clazz = Expression.Block.class, id = 751) +@Persistable(clazz = Expression.Binding.class, id = 752) +@Persistable(clazz = Application.Prefix.class, id = 753) +@Persistable(clazz = Application.Force.class, id = 754) +@Persistable(clazz = Application.Sequence.class, id = 755) +@Persistable(clazz = Case.Expr.class, id = 761) +@Persistable(clazz = Case.Branch.class, id = 762) +@Persistable(clazz = Pattern.Constructor.class, id = 763) +@Persistable(clazz = Pattern.Name.class, id = 764) +@Persistable(clazz = Pattern.Literal.class, id = 765) +@Persistable(clazz = Pattern.Type.class, id = 766) +@Persistable(clazz = Method.Conversion.class, id = 771) +@Persistable(clazz = Set.Union.class, id = 772) +@Persistable(clazz = Set.Intersection.class, id = 773) +@Persistable(clazz = Foreign.Definition.class, id = 781) +@Persistable(clazz = Type.Function.class, id = 782) +@Persistable(clazz = Name.BuiltinAnnotation.class, id = 783) +@Persistable(clazz = Type.Error.class, id = 784) +@Persistable(clazz = Unused.Binding.class, id = 785) +public final class IrPersistance { + private IrPersistance() {} + + @ServiceProvider(service = Persistance.class) + public static final class PersistIdentifiedLocation extends Persistance { + public PersistIdentifiedLocation() { + super(IdentifiedLocation.class, false, 2); + } + + @Override + protected void writeObject(IdentifiedLocation obj, Output out) throws IOException { + out.writeInline(Location.class, obj.location()); + out.writeInline(Option.class, obj.id()); + } + + @Override + @SuppressWarnings("unchecked") + protected IdentifiedLocation readObject(Input in) throws IOException, ClassNotFoundException { + var obj = in.readInline(Location.class); + var id = in.readInline(Option.class); + return IdentifiedLocation.create((Location) obj, id); + } + } + + @ServiceProvider(service = Persistance.class) + public static final class PersistUUID extends Persistance { + public PersistUUID() { + super(UUID.class, false, 73); + } + + @Override + protected void writeObject(UUID obj, Output out) throws IOException { + out.writeLong(obj.getLeastSignificantBits()); + out.writeLong(obj.getMostSignificantBits()); + } + + @Override + protected UUID readObject(Input in) throws IOException, ClassNotFoundException { + var least = in.readLong(); + var most = in.readLong(); + return new UUID(most, least); + } + } + + @ServiceProvider(service = Persistance.class) + public static final class PersistScalaOption extends Persistance