From 9397a6ec2ffda16baa0e5de3c1df55500caeec2b Mon Sep 17 00:00:00 2001 From: Dmitry Bushev Date: Wed, 8 Mar 2023 15:37:48 +0300 Subject: [PATCH] Pre compute suggestion db during build time (#5698) Close #5068 Cache suggestions during the `buildEngineDistribution` command, and read them from the disk when the library is loaded. Initial graph coloring takes ~20 seconds vs ~25 seconds on the develop branch. [peek-develop-branch.webm](https://user-images.githubusercontent.com/357683/223504462-e7d48262-4f5e-4724-b2b0-2cb97fc05140.webm) [peek-suggestions-branch.webm](https://user-images.githubusercontent.com/357683/223504464-0fe86c04-8c4b-443c-ba96-6c5e2fb1e396.webm) --- build.sbt | 1 + .../boot/resource/RepoInitialization.scala | 2 +- .../event/InitializedEvent.scala | 10 +- .../search/SuggestionsHandler.scala | 136 ++++++++++----- .../languageserver/text/BufferRegistry.scala | 4 +- .../resource/RepoInitializationSpec.scala | 10 +- .../languageserver/search/Suggestions.scala | 1 + .../search/SuggestionsHandlerSpec.scala | 104 ++--------- .../websocket/json/BaseServerTest.scala | 5 - .../org/enso/polyglot/runtime/Runtime.scala | 59 ++++--- .../org/enso/polyglot/data/TreeTest.scala | 24 +++ .../instrument/RuntimeComponentsTest.scala | 33 ---- .../test/instrument/RuntimeErrorsTest.scala | 31 ++-- .../instrument/RuntimeInstrumentTest.scala | 64 +++---- .../test/instrument/RuntimeServerTest.scala | 126 +++++++------- .../test/instrument/RuntimeStdlibTest.scala | 34 ---- .../RuntimeSuggestionUpdatesTest.scala | 38 ++-- .../main/java/org/enso/compiler/Cache.java | 29 +++- .../org/enso/compiler/ImportExportCache.java | 21 +-- .../java/org/enso/compiler/ModuleCache.java | 2 +- .../org/enso/compiler/SuggestionsCache.java | 157 +++++++++++++++++ .../org/enso/interpreter/runtime/Module.java | 4 +- .../scala/org/enso/compiler/Compiler.scala | 10 +- .../enso/compiler/SerializationManager.scala | 164 +++++++++++++++--- .../compiler/context/SuggestionBuilder.scala | 9 + .../compiler/context/SuggestionDiff.scala | 1 + .../enso/interpreter/instrument/Handler.scala | 13 +- .../instrument/NotificationHandler.scala | 21 ++- .../instrument/command/CommandFactory.scala | 6 +- .../DeserializeLibrarySuggestionsCmd.scala | 26 +++ .../command/VerifyModulesIndexCmd.scala | 39 ----- .../job/AnalyzeModuleInScopeJob.scala | 44 +---- .../DeserializeLibrarySuggestionsJob.scala | 44 +++++ .../instrument/job/EnsureCompiledJob.scala | 26 ++- .../org/enso/compiler/SerializerTest.java | 4 +- .../compiler/test/context/JacksonTest.java | 92 ++++++++++ .../docs/sections}/DocSectionsBuilder.scala | 3 +- .../sections}/DocSectionsBuilderTest.scala | 2 +- .../src/main/scala/org/enso/pkg/Package.scala | 24 +++ .../scala/org/enso/pkg/QualifiedName.scala | 3 +- project/DistributionPackage.scala | 22 +-- .../lib/Standard/Base/0.0.0-dev/src/Main.enso | 10 +- 42 files changed, 910 insertions(+), 548 deletions(-) create mode 100644 engine/runtime/src/main/java/org/enso/compiler/SuggestionsCache.java create mode 100644 engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/DeserializeLibrarySuggestionsCmd.scala delete mode 100644 engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/VerifyModulesIndexCmd.scala create mode 100644 engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala create mode 100644 engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java rename {engine/language-server/src/main/scala/org/enso/languageserver/search => lib/scala/docs-generator/src/main/scala/org/enso/docs/sections}/DocSectionsBuilder.scala (95%) rename {engine/language-server/src/test/scala/org/enso/languageserver/search => lib/scala/docs-generator/src/test/scala/org/enso/docs/sections}/DocSectionsBuilderTest.scala (99%) diff --git a/build.sbt b/build.sbt index 9fcfa2e3195..f84e1f3e0ae 100644 --- a/build.sbt +++ b/build.sbt @@ -643,6 +643,7 @@ lazy val `docs-generator` = (project in file("lib/scala/docs-generator")) .dependsOn(syntax.jvm) .dependsOn(cli) .dependsOn(`version-output`) + .dependsOn(`polyglot-api`) .configs(Benchmark) .settings( frgaalJavaCompilerSetting, diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala b/engine/language-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala index 73b569242d7..47e91525739 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala @@ -89,7 +89,7 @@ class RepoInitialization( } yield () initAction.onComplete { case Success(()) => - eventStream.publish(InitializedEvent.FileVersionsRepoInitialized) + eventStream.publish(InitializedEvent.VersionsRepoInitialized) case Failure(ex) => logger.error( "Failed to initialize SQL versions repo [{}]. {}", diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala b/engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala index 2e8f6049b41..2926463df22 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala @@ -5,9 +5,9 @@ sealed trait InitializedEvent extends Event object InitializedEvent { - case object SuggestionsRepoInitialized extends InitializedEvent - case object FileVersionsRepoInitialized extends InitializedEvent - case object TruffleContextInitialized extends InitializedEvent - case object InitializationFinished extends InitializedEvent - case object InitializationFailed extends InitializedEvent + case object SuggestionsRepoInitialized extends InitializedEvent + case object VersionsRepoInitialized extends InitializedEvent + case object TruffleContextInitialized extends InitializedEvent + case object InitializationFinished extends InitializedEvent + case object InitializationFailed extends InitializedEvent } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala index 7a9a0d39988..2677f6e57f0 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala @@ -5,6 +5,7 @@ import java.util.concurrent.Executors import akka.actor.{Actor, ActorRef, Props, Stash, Status} import akka.pattern.pipe import com.typesafe.scalalogging.LazyLogging +import org.enso.docs.sections.DocSectionsBuilder import org.enso.languageserver.capability.CapabilityProtocol.{ AcquireCapability, CapabilityAcquired, @@ -111,6 +112,12 @@ final class SuggestionsHandler( .subscribe(self, classOf[Api.ExpressionUpdates]) context.system.eventStream .subscribe(self, classOf[Api.SuggestionsDatabaseModuleUpdateNotification]) + context.system.eventStream.subscribe( + self, + classOf[Api.SuggestionsDatabaseSuggestionsLoadedNotification] + ) + context.system.eventStream + .subscribe(self, classOf[Api.LibraryLoaded]) context.system.eventStream.subscribe(self, classOf[ProjectNameChangedEvent]) context.system.eventStream.subscribe(self, classOf[FileDeletedEvent]) context.system.eventStream @@ -122,7 +129,7 @@ final class SuggestionsHandler( override def receive: Receive = initializing(SuggestionsHandler.Initialization()) - def initializing(init: SuggestionsHandler.Initialization): Receive = { + private def initializing(init: SuggestionsHandler.Initialization): Receive = { case ProjectNameChangedEvent(oldName, newName) => logger.info( "Initializing: project name changed from [{}] to [{}].", @@ -178,35 +185,7 @@ final class SuggestionsHandler( case _ => stash() } - def verifying( - projectName: String, - graph: TypeGraph - ): Receive = { - case Api.Response(_, Api.VerifyModulesIndexResponse(toRemove)) => - logger.info("Verifying: got verification response.") - val removeAction = for { - _ <- versionsRepo.remove(toRemove) - _ <- suggestionsRepo.removeModules(toRemove) - } yield SuggestionsHandler.Verified - removeAction.pipeTo(self) - - case SuggestionsHandler.Verified => - logger.info("Verified.") - context.become(initialized(projectName, graph, Set(), State())) - unstashAll() - - case Status.Failure(ex) => - logger.error( - "Database verification failure [{}]. {}", - ex.getClass, - ex.getMessage - ) - - case _ => - stash() - } - - def initialized( + private def initialized( projectName: String, graph: TypeGraph, clients: Set[ClientId], @@ -230,12 +209,37 @@ final class SuggestionsHandler( initialized(projectName, graph, clients - client.clientId, state) ) + case msg: Api.SuggestionsDatabaseSuggestionsLoadedNotification => + logger.debug( + "Starting loading suggestions for library [{}].", + msg.libraryName + ) + applyLoadedSuggestions(msg.suggestions) + .onComplete { + case Success(notification) => + logger.debug( + "Complete loading suggestions for library [{}].", + msg.libraryName + ) + if (notification.updates.nonEmpty) { + clients.foreach { clientId => + sessionRouter ! DeliverToJsonController(clientId, notification) + } + } + case Failure(ex) => + logger.error( + "Error applying suggestion updates for loaded library [{}] ({})", + msg.libraryName, + ex.getMessage + ) + } + case msg: Api.SuggestionsDatabaseModuleUpdateNotification if state.isSuggestionUpdatesRunning => state.suggestionUpdatesQueue.enqueue(msg) case SuggestionUpdatesBatch(updates) if state.isSuggestionUpdatesRunning => - state.suggestionUpdatesQueue.enqueueAll(updates) + state.suggestionUpdatesQueue.prependAll(updates) case SuggestionUpdatesBatch(updates) => val modules = updates.map(_.module) @@ -493,6 +497,13 @@ final class SuggestionsHandler( updates.foreach(sessionRouter ! _) context.become(initialized(name, graph, clients, state)) + case libraryLoaded: Api.LibraryLoaded => + logger.debug( + "Loaded Library [{}.{}]", + libraryLoaded.namespace, + libraryLoaded.name + ) + case SuggestionUpdatesCompleted => if (state.suggestionUpdatesQueue.nonEmpty) { self ! SuggestionUpdatesBatch(state.suggestionUpdatesQueue.removeAll()) @@ -515,17 +526,9 @@ final class SuggestionsHandler( private def tryInitialize(state: SuggestionsHandler.Initialization): Unit = { logger.debug("Trying to initialize with state [{}]", state) state.initialized.fold(context.become(initializing(state))) { - case (name, graph) => + case (projectName, graph) => logger.debug("Initialized with state [{}].", state) - val requestId = UUID.randomUUID() - suggestionsRepo.getAllModules - .map { modules => - runtimeConnector ! Api.Request( - requestId, - Api.VerifyModulesIndexRequest(modules) - ) - } - context.become(verifying(name, graph)) + context.become(initialized(projectName, graph, Set(), State())) unstashAll() } } @@ -543,6 +546,48 @@ final class SuggestionsHandler( } } + /** Handle the suggestions of the loaded library. + * + * Adds the new suggestions to the suggestions database and sends the + * appropriate notification to the client. + * + * @param suggestions the loaded suggestions + * @return the API suggestions database update notification + */ + private def applyLoadedSuggestions( + suggestions: Vector[Suggestion] + ): Future[SuggestionsDatabaseUpdateNotification] = { + for { + treeResults <- suggestionsRepo.applyTree( + suggestions.map(Api.SuggestionUpdate(_, Api.SuggestionAction.Add())) + ) + version <- suggestionsRepo.currentVersion + } yield { + val treeUpdates = treeResults.flatMap { + case QueryResult(ids, Api.SuggestionUpdate(suggestion, action)) => + action match { + case Api.SuggestionAction.Add() => + if (ids.isEmpty) { + val verb = action.getClass.getSimpleName + logger.error("Cannot {} [{}].", verb, suggestion) + } + ids.map( + SuggestionsDatabaseUpdate.Add( + _, + generateDocumentation(suggestion) + ) + ) + case action => + logger.error( + s"Invalid action during suggestions loading [$action]." + ) + Seq() + } + } + SuggestionsDatabaseUpdateNotification(version, treeUpdates) + } + } + /** Handle the suggestions database update. * * Function applies notification updates on the suggestions database and @@ -783,9 +828,6 @@ object SuggestionsHandler { new ProjectNameUpdated(projectName, Seq()) } - /** The notification that the suggestions database has been verified. */ - case object Verified - /** The notification that the suggestion updates are processed. */ case object SuggestionUpdatesCompleted @@ -843,7 +885,7 @@ object SuggestionsHandler { * @param config the server configuration * @param contentRootManager the content root manager * @param suggestionsRepo the suggestions repo - * @param fileVersionsRepo the file versions repo + * @param versionsRepo the versions repo * @param sessionRouter the session router * @param runtimeConnector the runtime connector * @param docSectionsBuilder the builder of documentation sections @@ -852,7 +894,7 @@ object SuggestionsHandler { config: Config, contentRootManager: ContentRootManager, suggestionsRepo: SuggestionsRepo[Future], - fileVersionsRepo: VersionsRepo[Future], + versionsRepo: VersionsRepo[Future], sessionRouter: ActorRef, runtimeConnector: ActorRef, docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder() @@ -862,7 +904,7 @@ object SuggestionsHandler { config, contentRootManager, suggestionsRepo, - fileVersionsRepo, + versionsRepo, sessionRouter, runtimeConnector, docSectionsBuilder diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala b/engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala index 811a42bb055..a4e026d5585 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala @@ -102,13 +102,13 @@ class BufferRegistry( override def preStart(): Unit = { logger.info("Starting initialization.") context.system.eventStream - .subscribe(self, InitializedEvent.FileVersionsRepoInitialized.getClass) + .subscribe(self, InitializedEvent.VersionsRepoInitialized.getClass) } override def receive: Receive = initializing private def initializing: Receive = { - case InitializedEvent.FileVersionsRepoInitialized => + case InitializedEvent.VersionsRepoInitialized => logger.info("Initiaized.") context.become(running(Map.empty)) unstashAll() diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala index 7b8f18544a6..cf06a387301 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala @@ -66,7 +66,7 @@ class RepoInitializationSpec expectMsgAllOf( InitializedEvent.SuggestionsRepoInitialized, - InitializedEvent.FileVersionsRepoInitialized + InitializedEvent.VersionsRepoInitialized ) } @@ -96,7 +96,7 @@ class RepoInitializationSpec expectMsgAllOf( InitializedEvent.SuggestionsRepoInitialized, - InitializedEvent.FileVersionsRepoInitialized + InitializedEvent.VersionsRepoInitialized ) } @@ -123,7 +123,7 @@ class RepoInitializationSpec version1 shouldEqual SchemaVersion.CurrentVersion expectMsgAllOf( InitializedEvent.SuggestionsRepoInitialized, - InitializedEvent.FileVersionsRepoInitialized + InitializedEvent.VersionsRepoInitialized ) // remove schema and re-initialize @@ -138,7 +138,7 @@ class RepoInitializationSpec version2 shouldEqual SchemaVersion.CurrentVersion expectMsgAllOf( InitializedEvent.SuggestionsRepoInitialized, - InitializedEvent.FileVersionsRepoInitialized + InitializedEvent.VersionsRepoInitialized ) } @@ -199,7 +199,7 @@ class RepoInitializationSpec version2 shouldEqual SchemaVersion.CurrentVersion expectMsgAllOf( InitializedEvent.SuggestionsRepoInitialized, - InitializedEvent.FileVersionsRepoInitialized + InitializedEvent.VersionsRepoInitialized ) } } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala index 88a5b97d1f8..9fcc9843762 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala @@ -1,6 +1,7 @@ package org.enso.languageserver.search import org.enso.docs.generator.DocsGenerator +import org.enso.docs.sections.DocSectionsBuilder import org.enso.polyglot.Suggestion import java.util.UUID diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala index e1520bb1dbe..6f235df00a5 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala @@ -4,6 +4,7 @@ import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{ImplicitSender, TestKit, TestProbe} import org.apache.commons.io.FileUtils import org.enso.docs.generator.DocsGenerator +import org.enso.docs.sections.DocSectionsBuilder import org.enso.languageserver.boot.ProfilingConfig import org.enso.languageserver.capability.CapabilityProtocol.{ AcquireCapability, @@ -57,61 +58,7 @@ class SuggestionsHandlerSpec "SuggestionsHandler" should { - "prune stale modules" in withDbs { (config, suggestions, versions) => - // setup database - val version1 = Array[Byte](1, 2, 3) - val version2 = Array[Byte](2, 3, 4) - val setupAction = for { - _ <- suggestions.insert(TestSuggestion.atom) - _ <- suggestions.insert(TestSuggestion.method) - _ <- versions.setVersion(TestSuggestion.atom.module, version1) - _ <- versions.setVersion(TestSuggestion.method.module, version2) - } yield () - Await.ready(setupAction, Timeout) - - withHandler(config, suggestions, versions) { (_, connector, handler) => - // initialize - handler ! SuggestionsHandler.ProjectNameUpdated("Test") - handler ! InitializedEvent.TruffleContextInitialized - handler ! InitializedEvent.SuggestionsRepoInitialized - handler ! InitializedEvent.FileVersionsRepoInitialized - connector.receiveN(1) - handler ! Api.Response( - UUID.randomUUID(), - Api.GetTypeGraphResponse(buildTestTypeGraph) - ) - connector.receiveN(1) - // prune atom module - handler ! Api.Response( - UUID.randomUUID(), - Api.VerifyModulesIndexResponse(Seq(TestSuggestion.atom.module)) - ) - // wait for initialization - handler ! AcquireCapability( - newJsonSession(UUID.randomUUID()), - CapabilityRegistration(ReceivesSuggestionsDatabaseUpdates()) - ) - expectMsg(CapabilityAcquired) - // check - val (_, entries) = Await.result(suggestions.getAll, Timeout) - entries.map(_.suggestion) should contain theSameElementsAs Seq( - TestSuggestion.method - ) - val module1Version = Await.result( - versions.getVersion(TestSuggestion.atom.module), - Timeout - ) - module1Version.isEmpty shouldBe true - val module2Version = Await.result( - versions.getVersion(TestSuggestion.method.module), - Timeout - ) - module2Version.isEmpty shouldBe false - module2Version.get should contain theSameElementsInOrderAs version2 - } - } - - "subscribe to notification updates" taggedAs Retry in withDb { + "subscribe to notification updates" /*taggedAs Retry*/ in withDb { (_, _, _, _, handler) => val clientId = UUID.randomUUID() @@ -1032,7 +979,7 @@ class SuggestionsHandlerSpec sessionRouter: TestProbe, runtimeConnector: TestProbe, suggestionsRepo: SuggestionsRepo[Future], - fileVersionsRepo: VersionsRepo[Future] + versionsRepo: VersionsRepo[Future] ): ActorRef = { val contentRootManagerActor = system.actorOf(ContentRootManagerActor.props(config)) @@ -1043,7 +990,7 @@ class SuggestionsHandlerSpec config, contentRootManagerWrapper, suggestionsRepo, - fileVersionsRepo, + versionsRepo, sessionRouter.ref, runtimeConnector.ref ) @@ -1055,7 +1002,7 @@ class SuggestionsHandlerSpec sessionRouter: TestProbe, runtimeConnector: TestProbe, suggestionsRepo: SuggestionsRepo[Future], - fileVersionsRepo: VersionsRepo[Future] + versionsRepo: VersionsRepo[Future] ): ActorRef = { val handler = newSuggestionsHandler( @@ -1063,11 +1010,12 @@ class SuggestionsHandlerSpec sessionRouter, runtimeConnector, suggestionsRepo, - fileVersionsRepo + versionsRepo ) handler ! SuggestionsHandler.ProjectNameUpdated("Test") handler ! InitializedEvent.TruffleContextInitialized + // GetTypeGraphRequest runtimeConnector.receiveN(1) handler ! Api.Response( UUID.randomUUID(), @@ -1075,7 +1023,7 @@ class SuggestionsHandlerSpec ) val suggestionsInit = suggestionsRepo.init - val versionsInit = fileVersionsRepo.init + val versionsInit = versionsRepo.init suggestionsInit.onComplete { case Success(()) => system.eventStream.publish(InitializedEvent.SuggestionsRepoInitialized) @@ -1084,16 +1032,13 @@ class SuggestionsHandlerSpec } versionsInit.onComplete { case Success(()) => - system.eventStream.publish(InitializedEvent.FileVersionsRepoInitialized) + system.eventStream.publish(InitializedEvent.VersionsRepoInitialized) case Failure(ex) => system.log.error(ex, "Failed to initialize FileVersions repo") } - runtimeConnector.receiveN(1) - handler ! Api.Response( - UUID.randomUUID(), - Api.VerifyModulesIndexResponse(Seq()) - ) + Await.ready(suggestionsInit, Timeout) + Await.ready(versionsInit, Timeout) handler } @@ -1150,7 +1095,7 @@ class SuggestionsHandlerSpec } versionsInit.onComplete { case Success(()) => - system.eventStream.publish(InitializedEvent.FileVersionsRepoInitialized) + system.eventStream.publish(InitializedEvent.VersionsRepoInitialized) case Failure(ex) => system.log.error(ex, "Failed to initialize FileVersions repo") } @@ -1165,29 +1110,6 @@ class SuggestionsHandlerSpec } } - def withHandler( - config: Config, - suggestionsRepo: SuggestionsRepo[Future], - versionsRepo: VersionsRepo[Future] - )( - test: (TestProbe, TestProbe, ActorRef) => Any - ): Unit = { - val router = TestProbe("session-router") - val connector = TestProbe("runtime-connector") - val handler = newSuggestionsHandler( - config, - router, - connector, - suggestionsRepo, - versionsRepo - ) - - try test(router, connector, handler) - finally { - system.stop(handler) - } - } - def withDb( test: ( Config, @@ -1207,7 +1129,7 @@ class SuggestionsHandlerSpec ) val router = TestProbe("session-router") val connector = TestProbe("runtime-connector") - val sqlDatabase = SqlDatabase(config.directories.suggestionsDatabaseFile) + val sqlDatabase = SqlDatabase.inmem("testdb") val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase) val versionsRepo = new SqlVersionsRepo(sqlDatabase) val handler = newInitializedSuggestionsHandler( diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala index 5f650d03dc7..e31b6ca10e9 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala @@ -248,11 +248,6 @@ class BaseServerTest ) Await.ready(initializationComponent.init(), timeout) system.eventStream.publish(ProjectNameChangedEvent("Test", "Test")) - runtimeConnectorProbe.receiveN(1) - suggestionsHandler ! Api.Response( - UUID.randomUUID(), - Api.VerifyModulesIndexResponse(Seq()) - ) val environment = fakeInstalledEnvironment() val languageHome = LanguageHome.detectFromExecutableLocation(environment) diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala index c31a3f3e60a..cf2731f2ebc 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala @@ -205,6 +205,10 @@ object Runtime { value = classOf[Api.SuggestionsDatabaseModuleUpdateNotification], name = "suggestionsDatabaseModuleUpdateNotification" ), + new JsonSubTypes.Type( + value = classOf[Api.SuggestionsDatabaseSuggestionsLoadedNotification], + name = "suggestionsDatabaseSuggestionsLoadedNotification" + ), new JsonSubTypes.Type( value = classOf[Api.AnalyzeModuleInScopeJobFinished], name = "analyzeModuleInScopeJobFinished" @@ -217,14 +221,6 @@ object Runtime { value = classOf[Api.InvalidateModulesIndexResponse], name = "invalidateModulesIndexResponse" ), - new JsonSubTypes.Type( - value = classOf[Api.VerifyModulesIndexRequest], - name = "verifyModulesIndexRequest" - ), - new JsonSubTypes.Type( - value = classOf[Api.VerifyModulesIndexResponse], - name = "verifyModulesIndexResponse" - ), new JsonSubTypes.Type( value = classOf[Api.GetTypeGraphRequest], name = "getTypeGraphRequest" @@ -268,6 +264,10 @@ object Runtime { new JsonSubTypes.Type( value = classOf[Api.LockReleaseFailed], name = "lockReleaseFailed" + ), + new JsonSubTypes.Type( + value = classOf[Api.DeserializeLibrarySuggestions], + name = "deserializeLibrarySuggestions" ) ) ) @@ -1514,6 +1514,25 @@ object Runtime { ")" } + /** A notification about the suggestions of the loaded library. + * + * @param libraryName the name of the loaded library + * @param suggestions the loaded suggestions + */ + final case class SuggestionsDatabaseSuggestionsLoadedNotification( + libraryName: LibraryName, + suggestions: Vector[Suggestion] + ) extends ApiNotification + with ToLogString { + + /** @inheritdoc */ + override def toLogString(shouldMask: Boolean): String = + "SuggestionsDatabaseSuggestionsLoadedNotification(" + + s"libraryName=$libraryName," + + s"suggestions=${suggestions.map(_.toLogString(shouldMask))}" + + ")" + } + /** A notification about the finished background analyze job. */ final case class AnalyzeModuleInScopeJobFinished() extends ApiNotification @@ -1523,20 +1542,6 @@ object Runtime { /** Signals that the module indexes has been invalidated. */ final case class InvalidateModulesIndexResponse() extends ApiResponse - /** A request to verify the modules in the suggestions database. - * - * @param modules the list of modules - */ - final case class VerifyModulesIndexRequest(modules: Seq[String]) - extends ApiRequest - - /** A response to the module verification request. - * - * @param remove the list of modules to remove from suggestions database. - */ - final case class VerifyModulesIndexResponse(remove: Seq[String]) - extends ApiResponse - /** A request for the type hierarchy graph. */ final case class GetTypeGraphRequest() extends ApiRequest @@ -1650,6 +1655,16 @@ object Runtime { */ final case class LockReleaseFailed(errorMessage: String) extends ApiResponse + /** A request to deserialize the library suggestions. + * + * Does not have a companion response message. The response will be + * delivered asynchronously as a notification. + * + * @param libraryName the name of the loaded library. + */ + final case class DeserializeLibrarySuggestions(libraryName: LibraryName) + extends ApiRequest + private lazy val mapper = { val factory = new CBORFactory() val mapper = new ObjectMapper(factory) with ClassTagExtensions diff --git a/engine/polyglot-api/src/test/scala/org/enso/polyglot/data/TreeTest.scala b/engine/polyglot-api/src/test/scala/org/enso/polyglot/data/TreeTest.scala index e675e0f78a4..bb0ba3b46e7 100644 --- a/engine/polyglot-api/src/test/scala/org/enso/polyglot/data/TreeTest.scala +++ b/engine/polyglot-api/src/test/scala/org/enso/polyglot/data/TreeTest.scala @@ -3,6 +3,8 @@ package org.enso.polyglot.data import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.collection.mutable.Builder + class TreeTest extends AnyWordSpec with Matchers { val tree: Tree.Root[Long] = Tree.Root( @@ -189,4 +191,26 @@ class TreeTest extends AnyWordSpec with Matchers { Tree.zip(tree1, tree2) shouldEqual expected } + "building the tree with mutable builder" in { + val b: Builder[Tree.Node[String], Vector[ + Tree.Node[String] + ]] = Vector.newBuilder + b += Tree.Node("ahoj", Vector()) + b += Tree.Node("cześć", Vector()) + b += Tree.Node("hi", Vector()) + b += Tree.Node("приве́т", Vector()) + b += Tree.Node("ciao", Vector()) + val v = b.result() + val t = Tree.Root(v) + + val expected = Tree.Root( + Vector( + Tree.Node("ahoj", Vector()), + Tree.Node("hi", Vector()), + Tree.Node("ciao", Vector()) + ) + ) + + t.filter(_.forall(ch => 'a' <= ch && ch <= 'z')) shouldEqual expected + } } diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala index fa700678deb..a3ee7c8cfa6 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala @@ -244,7 +244,6 @@ class RuntimeComponentsTest context.receiveAllUntil( Seq( context.executionComplete(contextId), - context.analyzeJobFinished, context.analyzeJobFinished ), timeout = 180 @@ -338,7 +337,6 @@ class RuntimeComponentsTest context.receiveAllUntil( Seq( context.executionComplete(contextId), - context.analyzeJobFinished, context.analyzeJobFinished ), timeout = 180 @@ -400,37 +398,6 @@ class RuntimeComponentsTest GroupReference(LibraryName("Standard", "Base"), GroupName("Output")) ) - // check that component group symbols can be resolved - val suggestionSymbols = responses - .collect { - case Api.Response( - None, - msg: Api.SuggestionsDatabaseModuleUpdateNotification - ) => - msg.updates.toVector.map(_.suggestion) - } - .flatten - .flatMap { suggestion => - for { - selfType <- Suggestion.SelfType(suggestion) - } yield s"$selfType.${suggestion.name}" - } - .toSet - - val componentSymbols = components - .flatMap { case (_, componentGroups) => - val newComponents = componentGroups.newGroups.flatMap(_.exports) - val extendedComponents = - componentGroups.extendedGroups.flatMap(_.exports) - newComponents ++ extendedComponents - } - .map(_.name) - - componentSymbols should not be empty - componentSymbols.foreach { component => - suggestionSymbols should contain(component) - } - context.consumeOut shouldEqual List() } diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala index cebbf9daa4d..086ff36736a 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala @@ -352,12 +352,12 @@ class RuntimeErrorsTest val requestId = UUID.randomUUID() val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val xId = metadata.addItem(40, 9) - val yId = metadata.addItem(58, 2) - val mainResId = metadata.addItem(65, 12) + val xId = metadata.addItem(46, 9) + val yId = metadata.addItem(64, 2) + val mainResId = metadata.addItem(71, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | x = undefined @@ -958,12 +958,12 @@ class RuntimeErrorsTest val requestId = UUID.randomUUID() val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val xId = metadata.addItem(40, 7) - val yId = metadata.addItem(56, 5) - val mainResId = metadata.addItem(66, 12) + val xId = metadata.addItem(46, 7) + val yId = metadata.addItem(62, 5) + val mainResId = metadata.addItem(72, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | x = 1 + foo @@ -1386,13 +1386,12 @@ class RuntimeErrorsTest val requestId = UUID.randomUUID() val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val xId = metadata.addItem(98, 3) - val yId = metadata.addItem(110, 5) - val mainResId = metadata.addItem(120, 12) + val xId = metadata.addItem(71, 3) + val yId = metadata.addItem(83, 5) + val mainResId = metadata.addItem(93, 12) val code = - """import Standard.Base.IO - |import Standard.Base.Error.Error + """from Standard.Base import all | |foo = | Error.throw 9 @@ -1462,7 +1461,7 @@ class RuntimeErrorsTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 4), model.Position(4, 17)), + model.Range(model.Position(3, 4), model.Position(3, 17)), "10002 - 10000" ) ), @@ -1589,8 +1588,8 @@ class RuntimeErrorsTest context.receiveNIgnorePendingExpressionUpdates( 3 ) should contain theSameElementsAs Seq( - TestMessages.update(contextId, x1Id, ConstantsGen.NOTHING), - TestMessages.update(contextId, mainRes1Id, ConstantsGen.NOTHING), + TestMessages.update(contextId, x1Id, ConstantsGen.NOTHING_BUILTIN), + TestMessages.update(contextId, mainRes1Id, ConstantsGen.NOTHING_BUILTIN), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("MyError") diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala index 78f69042766..8a849afe577 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala @@ -255,14 +255,14 @@ class RuntimeInstrumentTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val mainBody = metadata.addItem(31, 52) - val xExpr = metadata.addItem(40, 2) - val yExpr = metadata.addItem(51, 5) - val zExpr = metadata.addItem(65, 1) - val mainResExpr = metadata.addItem(71, 12) + val mainBody = metadata.addItem(37, 52) + val xExpr = metadata.addItem(46, 2) + val yExpr = metadata.addItem(57, 5) + val zExpr = metadata.addItem(71, 1) + val mainResExpr = metadata.addItem(77, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | x = 42 @@ -318,12 +318,12 @@ class RuntimeInstrumentTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val mainBody = metadata.addItem(72, 39) - val xExpr = metadata.addItem(81, 13) - val mainResExpr = metadata.addItem(99, 12) + val mainBody = metadata.addItem(78, 39) + val xExpr = metadata.addItem(87, 13) + val mainResExpr = metadata.addItem(105, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all |polyglot java import java.time.LocalDate | |main = @@ -376,14 +376,14 @@ class RuntimeInstrumentTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val mainBody = metadata.addItem(31, 42) - val xExpr = metadata.addItem(40, 2) - val yExpr = metadata.addItem(51, 5) - val mainResExpr = metadata.addItem(61, 12) - val mainRes1Expr = metadata.addItem(72, 1) + val mainBody = metadata.addItem(37, 42) + val xExpr = metadata.addItem(46, 2) + val yExpr = metadata.addItem(57, 5) + val mainResExpr = metadata.addItem(67, 12) + val mainRes1Expr = metadata.addItem(78, 1) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | x = 42 @@ -618,13 +618,13 @@ class RuntimeInstrumentTest // f expression metadata.addItem(42, 5) - val aExpr = metadata.addItem(56, 1) - val fApp = metadata.addItem(74, 3) - val mainRes = metadata.addItem(62, 16) - val mainExpr = metadata.addItem(31, 47) + val aExpr = metadata.addItem(62, 1) + val fApp = metadata.addItem(80, 3) + val mainRes = metadata.addItem(68, 16) + val mainExpr = metadata.addItem(37, 47) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | f x = x + 1 @@ -678,16 +678,16 @@ class RuntimeInstrumentTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val aExpr = metadata.addItem(40, 14) + val aExpr = metadata.addItem(46, 14) // lambda - metadata.addItem(41, 10) + metadata.addItem(47, 10) // lambda expression - metadata.addItem(46, 5) - val lamArg = metadata.addItem(53, 1) - val mainRes = metadata.addItem(59, 12) + metadata.addItem(52, 5) + val lamArg = metadata.addItem(59, 1) + val mainRes = metadata.addItem(65, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | a = (x -> x + 1) 1 @@ -739,14 +739,14 @@ class RuntimeInstrumentTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val aExpr = metadata.addItem(40, 9) + val aExpr = metadata.addItem(46, 9) // lambda - metadata.addItem(41, 5) - val lamArg = metadata.addItem(48, 1) - val mainRes = metadata.addItem(54, 12) + metadata.addItem(47, 5) + val lamArg = metadata.addItem(54, 1) + val mainRes = metadata.addItem(60, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | a = (_ + 1) 1 diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala index ff5eb61142b..8cc521a7c43 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala @@ -252,12 +252,11 @@ class RuntimeServerTest object Main2 { val metadata = new Metadata - val idMainY = metadata.addItem(173, 5) - val idMainZ = metadata.addItem(187, 5) + val idMainY = metadata.addItem(178, 5) + val idMainZ = metadata.addItem(192, 5) val code = metadata.appendToCode( - """ - |import Standard.Base.IO + """from Standard.Base import all | |foo = arg -> | IO.println "I'm expensive!" @@ -432,10 +431,10 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idFoo = metadata.addItem(35, 6) + val idFoo = metadata.addItem(41, 6) val code = - """import Standard.Base.IO + """from Standard.Base import all | |foo x=0 = x + 42 | @@ -485,11 +484,11 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idMain = metadata.addItem(48, 19) - val idMainFoo = metadata.addItem(64, 3) + val idMain = metadata.addItem(54, 19) + val idMainFoo = metadata.addItem(70, 3) val code = - """import Standard.Base.IO + """from Standard.Base import all | |foo a=0 = a + 1 | @@ -558,16 +557,16 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idMain = metadata.addItem(99, 120) - val idMainX = metadata.addItem(126, 9) - val idMainY = metadata.addItem(144, 3) - val idMainM = metadata.addItem(156, 8) - val idMainP = metadata.addItem(173, 5) - val idMainQ = metadata.addItem(187, 5) - val idMainF = metadata.addItem(209, 9) + val idMain = metadata.addItem(105, 120) + val idMainX = metadata.addItem(132, 9) + val idMainY = metadata.addItem(150, 3) + val idMainM = metadata.addItem(162, 8) + val idMainP = metadata.addItem(179, 5) + val idMainQ = metadata.addItem(193, 5) + val idMainF = metadata.addItem(215, 9) val code = - """import Standard.Base.IO + """from Standard.Base import all |import Enso_Test.Test.A | |type QuuxT @@ -808,11 +807,11 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idMain = metadata.addItem(48, 25) - val idMainFoo = metadata.addItem(65, 7) + val idMain = metadata.addItem(54, 25) + val idMainFoo = metadata.addItem(71, 7) val code = - """import Standard.Base.IO + """from Standard.Base import all | |foo a b = a + b | @@ -868,12 +867,11 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idMain = metadata.addItem(113, 36) - val idMainBar = metadata.addItem(145, 3) + val idMain = metadata.addItem(73, 36) + val idMainBar = metadata.addItem(105, 3) val code = - """from Standard.Base.Data.Numbers import Number - |import Standard.Base.IO + """from Standard.Base import all |import Standard.Base.Runtime.State | |main = IO.println (State.run Number 42 bar) @@ -929,12 +927,11 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idMain = metadata.addItem(113, 35) - val idMainBar = metadata.addItem(144, 3) + val idMain = metadata.addItem(73, 35) + val idMainBar = metadata.addItem(104, 3) val code = - """from Standard.Base.Data.Numbers import Number - |import Standard.Base.IO + """from Standard.Base import all |import Standard.Base.Runtime.State | |main = IO.println (State.run Number 0 bar) @@ -1054,13 +1051,13 @@ class RuntimeServerTest metadata.addItem(25, 22) // foo name metadata.addItem(25, 3) - val fooX = metadata.addItem(39, 1, "aa") - val fooRes = metadata.addItem(45, 1, "ab") - val mainFoo = metadata.addItem(63, 3, "ac") - val mainRes = metadata.addItem(71, 12, "ad") + val fooX = metadata.addItem(45, 1, "aa") + val fooRes = metadata.addItem(51, 1, "ab") + val mainFoo = metadata.addItem(69, 3, "ac") + val mainRes = metadata.addItem(77, 12, "ad") val code = - """import Standard.Base.IO + """from Standard.Base import all | |foo = | x = 4 @@ -1174,14 +1171,14 @@ class RuntimeServerTest val metadata = new Metadata // foo definition - metadata.addItem(25, 22) + metadata.addItem(31, 22) // foo name - metadata.addItem(25, 3) - val mainFoo = metadata.addItem(63, 3) - val mainRes = metadata.addItem(71, 12) + metadata.addItem(31, 3) + val mainFoo = metadata.addItem(69, 3) + val mainRes = metadata.addItem(77, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |foo = | x = 4 @@ -1356,11 +1353,11 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val idResult = metadata.addItem(45, 4, "aae") - val idPrintln = metadata.addItem(54, 17, "aaf") - val idMain = metadata.addItem(31, 40, "aaa") + val idResult = metadata.addItem(51, 4, "aae") + val idPrintln = metadata.addItem(60, 17, "aaf") + val idMain = metadata.addItem(37, 40, "aaa") val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | result = 1337 @@ -1436,20 +1433,19 @@ class RuntimeServerTest val numberTypeName = "Standard.Base.Data.Numbers.Number" val metadata = new Metadata - val idMain = metadata.addItem(77, 34, "aaaa") - val idMainA = metadata.addItem(86, 8, "aabb") - val idMainP = metadata.addItem(99, 12, "aacc") + val idMain = metadata.addItem(37, 34, "aaaa") + val idMainA = metadata.addItem(46, 8, "aabb") + val idMainP = metadata.addItem(59, 12, "aacc") // pie id - metadata.addItem(119, 1, "eee") + metadata.addItem(89, 1, "eee") // uwu id - metadata.addItem(127, 1, "bbb") + metadata.addItem(87, 1, "bbb") // hie id - metadata.addItem(135, 6, "fff") + metadata.addItem(95, 6, "fff") // Number.x id - metadata.addItem(155, 1, "999") + metadata.addItem(115, 1, "999") val code = - """from Standard.Base.Data.Numbers import Number - |import Standard.Base.IO + """from Standard.Base import all | |main = | a = 123 + 21 @@ -1505,7 +1501,7 @@ class RuntimeServerTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 8), model.Position(4, 16)), + model.Range(model.Position(3, 8), model.Position(3, 16)), "1234.x 4" ) ), @@ -1534,7 +1530,7 @@ class RuntimeServerTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 8), model.Position(4, 16)), + model.Range(model.Position(3, 8), model.Position(3, 16)), "1000.x 5" ) ), @@ -1563,7 +1559,7 @@ class RuntimeServerTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 8), model.Position(4, 16)), + model.Range(model.Position(3, 8), model.Position(3, 16)), "Main.pie" ) ), @@ -1592,7 +1588,7 @@ class RuntimeServerTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 8), model.Position(4, 16)), + model.Range(model.Position(3, 8), model.Position(3, 16)), "Main.uwu" ) ), @@ -1621,7 +1617,7 @@ class RuntimeServerTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 8), model.Position(4, 16)), + model.Range(model.Position(3, 8), model.Position(3, 16)), "Main.hie" ) ), @@ -1650,7 +1646,7 @@ class RuntimeServerTest mainFile, Seq( TextEdit( - model.Range(model.Position(4, 8), model.Position(4, 16)), + model.Range(model.Position(3, 8), model.Position(3, 16)), "\"Hello!\"" ) ), @@ -1899,11 +1895,11 @@ class RuntimeServerTest val moduleName = "Enso_Test.Test.Main" val metadata = new Metadata - val xId = metadata.addItem(40, 10) - val mainRes = metadata.addItem(55, 12) + val xId = metadata.addItem(46, 10) + val mainRes = metadata.addItem(61, 12) val code = - """import Standard.Base.IO + """from Standard.Base import all | |main = | x = a -> a + 1 @@ -2372,7 +2368,7 @@ class RuntimeServerTest context.consumeOut shouldEqual List() } - it should "send expression updates when file is restoredzzz" in { + it should "send expression updates when file is restored" in { val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() val moduleName = "Enso_Test.Test.Main" @@ -2383,12 +2379,12 @@ class RuntimeServerTest ) val metadata = new Metadata - val idText = metadata.addItem(43, 12, "aa") - val idRes = metadata.addItem(60, 15, "ab") + val idText = metadata.addItem(49, 12, "aa") + val idRes = metadata.addItem(66, 15, "ab") def template(text: String) = metadata.appendToCode( - s"""import Standard.Base.IO + s"""from Standard.Base import all | |main = | text = "$text" @@ -2661,7 +2657,7 @@ class RuntimeServerTest ) ) ) - context.receiveN(3) should contain theSameElementsAs Seq( + context.receiveNIgnoreStdLib(3) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), Api.Response( Api.ExecutionFailed( diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala index a68481331c5..0f01a89ef3f 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala @@ -240,7 +240,6 @@ class RuntimeStdlibTest context.receiveAllUntil( Seq( context.executionComplete(contextId), - context.analyzeJobFinished, context.analyzeJobFinished ), timeout = 180 @@ -299,39 +298,6 @@ class RuntimeStdlibTest ) } - // check that the Standard.Base library is indexed - val stdlibSuggestions = responses.collect { - case Api.Response( - None, - Api.SuggestionsDatabaseModuleUpdateNotification( - module, - _, - as, - _, - xs - ) - ) if module.contains("Vector") => - (xs.nonEmpty || as.nonEmpty) shouldBe true - xs.toVector.map(_.suggestion.module) - } - stdlibSuggestions.nonEmpty shouldBe true - - // check that builtins are indexed - val builtinsSuggestions = responses.collect { - case Api.Response( - None, - Api.SuggestionsDatabaseModuleUpdateNotification( - module, - _, - as, - _, - xs - ) - ) if module.contains("Builtins") => - (xs.nonEmpty || as.nonEmpty) shouldBe true - } - builtinsSuggestions.length shouldBe 1 - // check LibraryLoaded notifications val contentRootNotifications = responses.collect { case Api.Response( diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala index e51727e7248..802791e41e9 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala @@ -138,7 +138,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 3 + 4 ) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), Api.Response( @@ -183,6 +183,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("Hello World!") @@ -203,7 +204,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( @@ -260,6 +261,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("42") @@ -284,7 +286,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( @@ -366,6 +368,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("51") @@ -386,7 +389,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( @@ -477,6 +480,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("51") @@ -497,7 +501,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( @@ -618,6 +622,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("51") @@ -638,7 +643,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( @@ -708,6 +713,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("51") @@ -765,7 +771,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 3 + 4 ) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), Api.Response( @@ -884,6 +890,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) } @@ -894,7 +901,7 @@ class RuntimeSuggestionUpdatesTest val moduleName = "Enso_Test.Test.Main" val mainCode = - """import Standard.Base.IO + """from Standard.Base import all | |import Enso_Test.Test.A |from Enso_Test.Test.A export all @@ -904,7 +911,7 @@ class RuntimeSuggestionUpdatesTest |main = IO.println "Hello World!" |""".stripMargin.linesIterator.mkString("\n") val aCode = - """from Standard.Base.Data.Numbers import Integer + """from Standard.Base import all | |type MyType | MkA a @@ -950,7 +957,7 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 4 + 5 ) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), Api.Response( @@ -1141,6 +1148,7 @@ class RuntimeSuggestionUpdatesTest ) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("Hello World!") @@ -1161,13 +1169,13 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnoreExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( module = moduleName, version = contentsVersion( - """import Standard.Base.IO + """from Standard.Base import all | |import Enso_Test.Test.A |from Enso_Test.Test.A export all hiding hello @@ -1190,6 +1198,7 @@ class RuntimeSuggestionUpdatesTest updates = Tree.Root(Vector()) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("Hello World!") @@ -1210,13 +1219,13 @@ class RuntimeSuggestionUpdatesTest ) ) context.receiveNIgnorePendingExpressionUpdates( - 2 + 3 ) should contain theSameElementsAs Seq( Api.Response( Api.SuggestionsDatabaseModuleUpdateNotification( module = moduleName, version = contentsVersion( - """import Standard.Base.IO + """from Standard.Base import all | |main = IO.println "Hello World!" |""".stripMargin.linesIterator.mkString("\n") @@ -1234,6 +1243,7 @@ class RuntimeSuggestionUpdatesTest updates = Tree.Root(Vector()) ) ), + Api.Response(Api.AnalyzeModuleInScopeJobFinished()), context.executionComplete(contextId) ) context.consumeOut shouldEqual List("Hello World!") diff --git a/engine/runtime/src/main/java/org/enso/compiler/Cache.java b/engine/runtime/src/main/java/org/enso/compiler/Cache.java index 61042c78bbe..b97d7905574 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/Cache.java +++ b/engine/runtime/src/main/java/org/enso/compiler/Cache.java @@ -6,6 +6,7 @@ import org.bouncycastle.jcajce.provider.digest.SHA3; import org.bouncycastle.util.encoders.Hex; import org.enso.interpreter.runtime.EnsoContext; import org.enso.logger.masking.MaskedPath; +import org.enso.pkg.SourceFile; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -19,6 +20,8 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.security.MessageDigest; +import java.util.Comparator; +import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -326,10 +329,34 @@ public abstract class Cache { * @param bytes bytes for which hash will be computed * @return string representation of bytes' hash */ - protected String computeDigestFromBytes(byte[] bytes) { + protected final String computeDigestFromBytes(byte[] bytes) { return Hex.toHexString(messageDigest().digest(bytes)); } + /** + * Computes digest from package sources using a default hashing algorithm. + * + * @param pkgSources the list of package sources + * @param logger the truffle logger + * @return string representation of bytes' hash + */ + protected final String computeDigestOfLibrarySources( + List> pkgSources, TruffleLogger logger) { + pkgSources.sort(Comparator.comparing(o -> o.qualifiedName().toString())); + + var digest = messageDigest(); + pkgSources.forEach( + source -> { + try { + digest.update(source.file().readAllBytes()); + } catch (IOException e) { + logger.log( + logLevel, "failed to compute digest for " + source.qualifiedName().toString(), e); + } + }); + return Hex.toHexString(digest.digest()); + } + /** * Returns a default hashing algorithm used for Enso caches. * diff --git a/engine/runtime/src/main/java/org/enso/compiler/ImportExportCache.java b/engine/runtime/src/main/java/org/enso/compiler/ImportExportCache.java index 3fe88112b4f..dd7450819c5 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/ImportExportCache.java +++ b/engine/runtime/src/main/java/org/enso/compiler/ImportExportCache.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.oracle.truffle.api.TruffleFile; import com.oracle.truffle.api.TruffleLogger; -import org.bouncycastle.util.encoders.Hex; import org.enso.compiler.data.BindingsMap; import org.enso.editions.LibraryName; import org.enso.interpreter.runtime.EnsoContext; @@ -15,15 +14,13 @@ import org.enso.pkg.SourceFile; import scala.collection.immutable.Map; import scala.jdk.CollectionConverters; -import java.io.IOException; import java.io.Serializable; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.logging.Level; -public class ImportExportCache extends Cache { +public final class ImportExportCache extends Cache { private final LibraryName libraryName; @@ -79,22 +76,6 @@ public class ImportExportCache extends Cache computeDigestOfLibrarySources(pkg.listSourcesJava(), logger)); } - private String computeDigestOfLibrarySources(List> pkgSources, TruffleLogger logger) { - pkgSources.sort(Comparator.comparing(o -> o.qualifiedName().toString())); - - var digest = messageDigest(); - pkgSources.forEach(source -> - { - try { - digest.update(source.file().readAllBytes()); - } catch (IOException e) { - logger.log(logLevel, "failed to compute digest for " + source.qualifiedName().toString(), e); - } - } - ); - return Hex.toHexString(digest.digest()); - } - @Override @SuppressWarnings("unchecked") protected Optional getCacheRoots(EnsoContext context) { diff --git a/engine/runtime/src/main/java/org/enso/compiler/ModuleCache.java b/engine/runtime/src/main/java/org/enso/compiler/ModuleCache.java index 06e3c803d8d..90766fb4170 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/ModuleCache.java +++ b/engine/runtime/src/main/java/org/enso/compiler/ModuleCache.java @@ -18,7 +18,7 @@ import java.util.Arrays; import java.util.Optional; import java.util.logging.Level; -public class ModuleCache extends Cache { +public final class ModuleCache extends Cache { private final Module module; diff --git a/engine/runtime/src/main/java/org/enso/compiler/SuggestionsCache.java b/engine/runtime/src/main/java/org/enso/compiler/SuggestionsCache.java new file mode 100644 index 00000000000..33025087ac5 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/compiler/SuggestionsCache.java @@ -0,0 +1,157 @@ +package org.enso.compiler; + +import buildinfo.Info; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.oracle.truffle.api.TruffleFile; +import com.oracle.truffle.api.TruffleLogger; +import org.enso.editions.LibraryName; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.pkg.SourceFile; +import org.enso.polyglot.Suggestion; +import scala.jdk.CollectionConverters; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; + +public final class SuggestionsCache + extends Cache { + + private static final String SUGGESTIONS_CACHE_DATA_EXTENSION = ".suggestions"; + private static final String SUGGESTIONS_CACHE_METADATA_EXTENSION =".suggestions.meta"; + + private final static ObjectMapper objectMapper = new ObjectMapper(); + + final LibraryName libraryName; + + public SuggestionsCache(LibraryName libraryName) { + this.libraryName = libraryName; + this.logLevel = Level.FINEST; + this.stringRepr = libraryName.toString(); + this.entryName = libraryName.name(); + this.dataSuffix = SUGGESTIONS_CACHE_DATA_EXTENSION; + this.metadataSuffix = SUGGESTIONS_CACHE_METADATA_EXTENSION; + } + + @Override + protected byte[] metadata(String sourceDigest, String blobDigest, CachedSuggestions entry) { + try { + return objectMapper.writeValueAsString(new Metadata(sourceDigest, blobDigest)).getBytes(metadataCharset); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + @Override + protected CachedSuggestions validateReadObject(Object obj, Metadata meta, TruffleLogger logger) + throws CacheException { + if (obj instanceof Suggestions suggestions) { + return new CachedSuggestions(libraryName, suggestions, Optional.empty()); + } else { + throw new CacheException("Expected SuggestionsCache.Suggestions, got " + obj.getClass()); + } + } + + @Override + protected Optional metadataFromBytes(byte[] bytes) { + var maybeJsonString = new String(bytes, Cache.metadataCharset); + try { + return Optional.of(objectMapper.readValue(maybeJsonString, SuggestionsCache.Metadata.class)); + } catch (JsonProcessingException e) { + return Optional.empty(); + } + } + + @Override + protected Optional computeDigest(CachedSuggestions entry, TruffleLogger logger) { + return entry.getSources().map(sources -> computeDigestOfLibrarySources(sources, logger)); + } + + @Override + protected Optional computeDigestFromSource(EnsoContext context, TruffleLogger logger) { + return context + .getPackageRepository() + .getPackageForLibraryJava(libraryName) + .map(pkg -> computeDigestOfLibrarySources(pkg.listSourcesJava(), logger)); + } + + + @Override + protected Optional getCacheRoots(EnsoContext context) { + return context.getPackageRepository().getPackageForLibraryJava(libraryName).map(pkg -> { + var bindingsCacheRoot = pkg.getSuggestionsCacheRootForPackage(Info.ensoVersion()); + var localCacheRoot = bindingsCacheRoot.resolve(libraryName.namespace()); + var distribution = context.getDistributionManager(); + var pathSegments = CollectionConverters.ListHasAsScala(Arrays.asList( + pkg.namespace(), + pkg.name(), + pkg.config().version(), + Info.ensoVersion(), + libraryName.namespace()) + ).asScala(); + var path = distribution.LocallyInstalledDirectories().irCacheDirectory() + .resolve(pathSegments.mkString("/")); + var globalCacheRoot = context.getTruffleFile(path.toFile()); + return new Cache.Roots(localCacheRoot, globalCacheRoot); + }); + } + + @Override + protected Object extractObjectToSerialize(CachedSuggestions entry) { + return entry.getSuggestionsObjectToSerialize(); + } + + // Suggestions class is not a record because of a Frgaal bug leading to invalid compilation error. + final static class Suggestions implements Serializable { + + private final List suggestions; + + public Suggestions(List suggestions) { + this.suggestions = suggestions; + } + + public List getSuggestions() { + return suggestions; + } + } + + // CachedSuggestions class is not a record because of a Frgaal bug leading to invalid compilation error. + public final static class CachedSuggestions { + + private final LibraryName libraryName; + private final Suggestions suggestions; + + private final Optional>> sources; + + public CachedSuggestions(LibraryName libraryName, Suggestions suggestions, Optional>> sources) { + this.libraryName = libraryName; + this.suggestions = suggestions; + this.sources = sources; + } + + public LibraryName getLibraryName() { + return libraryName; + } + + public Optional>> getSources() { + return sources; + } + + public Suggestions getSuggestionsObjectToSerialize() { + return suggestions; + } + + public List getSuggestions() { + return suggestions.getSuggestions(); + } + } + + record Metadata( + @JsonProperty("source_hash") String sourceHash, + @JsonProperty("blob_hash") String blobHash + ) implements Cache.Metadata { } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java index 7e1ea762ac6..04c7f00d5bb 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java @@ -88,7 +88,7 @@ public final class Module implements TruffleObject { private ModuleScope scope; private ModuleSources sources; private PatchedModuleValues patchedValues; - private Map allSources = new WeakHashMap<>(); + private final Map allSources = new WeakHashMap<>(); private final Package pkg; private CompilationStage compilationStage = CompilationStage.INITIAL; private boolean isIndexed = false; @@ -97,7 +97,7 @@ public final class Module implements TruffleObject { private final ModuleCache cache; private boolean wasLoadedFromCache; private boolean hasCrossModuleLinks; - private boolean synthetic; + private final boolean synthetic; private List directModulesRefs; /** diff --git a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala index 01b63aed513..dcd0d84eb39 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala @@ -36,6 +36,7 @@ import java.util.concurrent.{ TimeUnit } import java.util.logging.Level + import scala.jdk.OptionConverters._ /** This class encapsulates the static transformation processes that take place @@ -102,7 +103,7 @@ class Compiler( } /** Lazy-initializes the IR for the builtins module. */ - def initializeBuiltinsIr(): Unit = { + private def initializeBuiltinsIr(): Unit = { if (!builtins.isIrInitialized) { logger.log( Compiler.defaultLogLevel, @@ -137,6 +138,10 @@ class Compiler( } } + /** @return the serialization manager instance. */ + def getSerializationManager: SerializationManager = + serializationManager + /** Processes the provided language sources, registering any bindings in the * given scope. * @@ -195,7 +200,8 @@ class Compiler( generateCode = false, shouldCompileDependencies ) - serializationManager.serializeLibraryBindings( + + serializationManager.serializeLibrary( pkg.libraryName, useGlobalCacheLocations = true ) diff --git a/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala b/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala index 19fc944d85e..69471260cc2 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala @@ -2,13 +2,17 @@ package org.enso.compiler import com.oracle.truffle.api.TruffleLogger import com.oracle.truffle.api.source.Source +import org.enso.compiler.context.SuggestionBuilder import org.enso.compiler.core.IR import org.enso.compiler.pass.analyse.BindingAnalysis +import org.enso.docs.sections.DocSectionsBuilder import org.enso.editions.LibraryName import org.enso.interpreter.runtime.Module import org.enso.pkg.QualifiedName +import org.enso.polyglot.Suggestion import java.io.NotSerializableException +import java.util import java.util.concurrent.{ Callable, CompletableFuture, @@ -19,10 +23,15 @@ import java.util.concurrent.{ TimeUnit } import java.util.logging.Level + import scala.collection.mutable import scala.jdk.OptionConverters.RichOptional -class SerializationManager(compiler: Compiler) { +final class SerializationManager( + compiler: Compiler, + docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder() +) { + import SerializationManager._ /** The debug logging level. */ @@ -133,13 +142,13 @@ class SerializationManager(compiler: Compiler) { } } - def serializeLibraryBindings( + def serializeLibrary( libraryName: LibraryName, useGlobalCacheLocations: Boolean ): Future[Boolean] = { logger.log( Level.INFO, - s"Requesting serialization for library [$libraryName] bindings." + s"Requesting serialization for library [$libraryName]." ) val task: Callable[Boolean] = @@ -157,7 +166,7 @@ class SerializationManager(compiler: Compiler) { case e: Throwable => logger.log( debugLogLevel, - s"Serialization task failed for library [${libraryName}].", + s"Serialization task failed for library [$libraryName].", e ) CompletableFuture.completedFuture(false) @@ -165,7 +174,7 @@ class SerializationManager(compiler: Compiler) { } } - def doSerializeLibrary( + private def doSerializeLibrary( libraryName: LibraryName, useGlobalCacheLocations: Boolean ): Callable[Boolean] = () => { @@ -195,29 +204,100 @@ class SerializationManager(compiler: Compiler) { .map(_.listSourcesJava()) ) try { - new ImportExportCache(libraryName) - .save(bindingsCache, compiler.context, useGlobalCacheLocations) - .isPresent() - } catch { - case e: NotSerializableException => - logger.log( - Level.SEVERE, - s"Could not serialize bindings [$libraryName].", - e - ) - throw e - case e: Throwable => - logger.log( - Level.SEVERE, - s"Serialization of bindings `$libraryName` failed: ${e.getMessage}`", - e - ) - throw e + val result = + try { + new ImportExportCache(libraryName) + .save(bindingsCache, compiler.context, useGlobalCacheLocations) + .isPresent + } catch { + case e: NotSerializableException => + logger.log( + Level.SEVERE, + s"Could not serialize bindings [$libraryName].", + e + ) + throw e + case e: Throwable => + logger.log( + Level.SEVERE, + s"Serialization of bindings `$libraryName` failed: ${e.getMessage}`", + e + ) + throw e + } + + try { + val suggestions = new util.ArrayList[Suggestion]() + compiler.packageRepository + .getModulesForLibrary(libraryName) + .flatMap { module => + SuggestionBuilder(module) + .build(module.getName, module.getIr) + .toVector + } + .map(generateDocumentation) + .foreach(suggestions.add) + val cachedSuggestions = + new SuggestionsCache.CachedSuggestions( + libraryName, + new SuggestionsCache.Suggestions(suggestions), + compiler.packageRepository + .getPackageForLibraryJava(libraryName) + .map(_.listSourcesJava()) + ) + new SuggestionsCache(libraryName) + .save(cachedSuggestions, compiler.context, false) + .isPresent + } catch { + case e: NotSerializableException => + logger.log( + Level.SEVERE, + s"Could not serialize suggestions [$libraryName].", + e + ) + throw e + case e: Throwable => + logger.log( + Level.SEVERE, + s"Serialization of suggestions `$libraryName` failed: ${e.getMessage}`", + e + ) + throw e + } + + result } finally { finishSerializing(libraryName.toQualifiedName) } } + def deserializeSuggestions( + libraryName: LibraryName + ): Option[SuggestionsCache.CachedSuggestions] = { + if (isWaitingForSerialization(libraryName)) { + abort(libraryName) + None + } else { + while (isSerializingLibrary(libraryName)) { + Thread.sleep(100) + } + new SuggestionsCache(libraryName).load(compiler.context).toScala match { + case result @ Some(_: SuggestionsCache.CachedSuggestions) => + logger.log( + Level.FINE, + s"Restored suggestions for library [$libraryName]." + ) + result + case _ => + logger.log( + Level.FINEST, + s"Unable to load suggestions for library [$libraryName]." + ) + None + } + } + } + def deserializeLibraryBindings( libraryName: LibraryName ): Option[ImportExportCache.CachedBindings] = { @@ -524,7 +604,45 @@ class SerializationManager(compiler: Compiler) { ) .asScala } + + /** Generate the documentation for the given suggestion. + * + * @param suggestion the initial suggestion + * @return the suggestion with documentation fields set + */ + private def generateDocumentation(suggestion: Suggestion): Suggestion = + suggestion match { + case module: Suggestion.Module => + val docSections = module.documentation.map(docSectionsBuilder.build) + module.copy(documentationSections = docSections) + + case constructor: Suggestion.Constructor => + val docSections = + constructor.documentation.map(docSectionsBuilder.build) + constructor.copy(documentationSections = docSections) + + case tpe: Suggestion.Type => + val docSections = tpe.documentation.map(docSectionsBuilder.build) + tpe.copy(documentationSections = docSections) + + case method: Suggestion.Method => + val docSections = method.documentation.map(docSectionsBuilder.build) + method.copy(documentationSections = docSections) + + case conversion: Suggestion.Conversion => + val docSections = conversion.documentation.map(docSectionsBuilder.build) + conversion.copy(documentationSections = docSections) + + case function: Suggestion.Function => + val docSections = function.documentation.map(docSectionsBuilder.build) + function.copy(documentationSections = docSections) + + case local: Suggestion.Local => + val docSections = local.documentation.map(docSectionsBuilder.build) + local.copy(documentationSections = docSections) + } } + object SerializationManager { /** The maximum number of serialization threads allowed. */ diff --git a/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala b/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala index 333e203d2c6..e4fe4095ed1 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala @@ -8,6 +8,7 @@ import org.enso.compiler.pass.resolve.{ TypeNames, TypeSignatures } +import org.enso.interpreter.runtime.Module import org.enso.interpreter.runtime.`type`.Types import org.enso.pkg.QualifiedName import org.enso.polyglot.Suggestion @@ -655,6 +656,14 @@ object SuggestionBuilder { /** TODO[DB] enable conversions when they get the runtime support. */ private val ConversionsEnabled: Boolean = false + /** Creates the suggestion builder for a module. + * + * @param module the module to index + * @return the suggestions builder for the module + */ + def apply(module: Module): SuggestionBuilder[CharSequence] = + SuggestionBuilder(module.getSource.getCharacters) + /** Create the suggestion builder. * * @param source the text source diff --git a/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionDiff.scala b/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionDiff.scala index 40275447863..5c46cfc3d10 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionDiff.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionDiff.scala @@ -1,4 +1,5 @@ package org.enso.compiler.context + import org.enso.polyglot.Suggestion import org.enso.polyglot.data.{These, Tree} import org.enso.polyglot.runtime.Runtime.Api diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala index 2fe3d22d3af..26031762fcc 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala @@ -17,9 +17,7 @@ import org.graalvm.polyglot.io.MessageEndpoint import java.nio.ByteBuffer import scala.concurrent.Future -/** A message endpoint implementation used by the - * [[org.enso.interpreter.instrument.RuntimeServerInstrument]]. - */ +/** A message endpoint implementation. */ class Endpoint(handler: Handler) extends MessageEndpoint with RuntimeServerConnectionEndpoint { @@ -47,6 +45,15 @@ class Endpoint(handler: Handler) def sendToClient(msg: Api.Response): Unit = client.sendBinary(Api.serialize(msg)) + /** Sends a notification to the runtime. + * + * Can be used to start a command processing in the background. + * + * @param msg the message to send. + */ + def sendToSelf(msg: Api.Request): Unit = + handler.onMessage(msg) + /** Sends a request to the connected client and expects a reply. */ override def sendRequest(msg: ApiRequest): Future[ApiResponse] = reverseRequestEndpoint.sendRequest(msg) diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala index 5a831c490a9..a1198927447 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala @@ -115,14 +115,21 @@ object NotificationHandler { libraryName: LibraryName, libraryVersion: LibraryVersion, location: Path - ): Unit = sendMessage( - Api.LibraryLoaded( - namespace = libraryName.namespace, - name = libraryName.name, - version = libraryVersion.toString, - location = location.toFile + ): Unit = { + sendMessage( + Api.LibraryLoaded( + namespace = libraryName.namespace, + name = libraryName.name, + version = libraryVersion.toString, + location = location.toFile + ) ) - ) + endpoint.sendToSelf( + Api.Request( + Api.DeserializeLibrarySuggestions(libraryName) + ) + ) + } /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/CommandFactory.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/CommandFactory.scala index 7166881331a..a7d9590eba5 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/CommandFactory.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/CommandFactory.scala @@ -56,12 +56,12 @@ object CommandFactory { case payload: Api.InvalidateModulesIndexRequest => new InvalidateModulesIndexCmd(request.requestId, payload) - case payload: Api.VerifyModulesIndexRequest => - new VerifyModulesIndexCmd(request.requestId, payload) - case _: Api.GetTypeGraphRequest => new GetTypeGraphCommand(request.requestId) + case payload: Api.DeserializeLibrarySuggestions => + new DeserializeLibrarySuggestionsCmd(request.requestId, payload) + case Api.ShutDownRuntimeServer() => throw new IllegalArgumentException( "ShutDownRuntimeServer request is not convertible to command object" diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/DeserializeLibrarySuggestionsCmd.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/DeserializeLibrarySuggestionsCmd.scala new file mode 100644 index 00000000000..348a65d154d --- /dev/null +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/DeserializeLibrarySuggestionsCmd.scala @@ -0,0 +1,26 @@ +package org.enso.interpreter.instrument.command + +import org.enso.interpreter.instrument.execution.RuntimeContext +import org.enso.interpreter.instrument.job.DeserializeLibrarySuggestionsJob +import org.enso.polyglot.runtime.Runtime.Api + +import scala.concurrent.{ExecutionContext, Future} + +/** A command that initiates the deserialization of suggestions. + * + * @param maybeRequestId an option with request id + */ +class DeserializeLibrarySuggestionsCmd( + maybeRequestId: Option[Api.RequestId], + request: Api.DeserializeLibrarySuggestions +) extends Command(maybeRequestId) { + + /** @inheritdoc */ + override def execute(implicit + ctx: RuntimeContext, + ec: ExecutionContext + ): Future[Unit] = + ctx.jobProcessor.runBackground( + new DeserializeLibrarySuggestionsJob(request.libraryName) + ) +} diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/VerifyModulesIndexCmd.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/VerifyModulesIndexCmd.scala deleted file mode 100644 index cc6e15f2e58..00000000000 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/VerifyModulesIndexCmd.scala +++ /dev/null @@ -1,39 +0,0 @@ -package org.enso.interpreter.instrument.command - -import org.enso.interpreter.instrument.execution.RuntimeContext -import org.enso.polyglot.runtime.Runtime.Api - -import scala.collection.mutable -import scala.concurrent.{ExecutionContext, Future} - -/** A command that verifies the modules index. - * - * @param maybeRequestId an option with request id - * @param request a verification request - */ -class VerifyModulesIndexCmd( - maybeRequestId: Option[Api.RequestId], - val request: Api.VerifyModulesIndexRequest -) extends Command(maybeRequestId) { - - /** Executes a request. - * - * @param ctx contains suppliers of services to perform a request - */ - override def execute(implicit - ctx: RuntimeContext, - ec: ExecutionContext - ): Future[Unit] = { - ctx.locking.acquireReadCompilationLock() - try { - val builder = mutable.Set(request.modules: _*) - ctx.executionService.getContext.getTopScope.getModules.forEach { module => - builder -= module.getName.toString - } - Future(reply(Api.VerifyModulesIndexResponse(builder.toVector))) - } finally { - ctx.locking.releaseReadCompilationLock() - } - } - -} diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala index 50041f9c6c3..02e3dcfeb81 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala @@ -8,7 +8,6 @@ import org.enso.compiler.context.{ } import org.enso.interpreter.instrument.execution.RuntimeContext import org.enso.interpreter.runtime.Module -import org.enso.pkg.QualifiedName import org.enso.polyglot.data.Tree import org.enso.polyglot.runtime.Runtime.Api import org.enso.polyglot.{ModuleExports, Suggestion} @@ -16,7 +15,6 @@ import org.enso.polyglot.{ModuleExports, Suggestion} import java.util.logging.Level final class AnalyzeModuleInScopeJob( - moduleName: Option[QualifiedName], modules: Iterable[Module] ) extends Job[Unit]( List(AnalyzeModuleJob.backgroundContextId), @@ -33,27 +31,10 @@ final class AnalyzeModuleInScopeJob( // disable the suggestion updates and reduce the number of messages that // runtime sends. if (ctx.executionService.getContext.isProjectSuggestionsEnabled) { - // When the project module is compiled it can involve compilation of - // global (library) modules, so we need to check if the global - // suggestions are enabled as well. - if (ctx.executionService.getContext.isGlobalSuggestionsEnabled) { - modules.foreach(analyzeModuleInScope) - ctx.endpoint.sendToClient( - Api.Response(Api.AnalyzeModuleInScopeJobFinished()) - ) - } else { - // When the global suggestions are disabled, we will skip indexing - // of external libraries, but still want to index the modules that - // belongs to the project. - val projectModules = - moduleName match { - case Some(name) => - modules.filter(m => rootName(m.getName) == rootName(name)) - case None => - Seq() - } - projectModules.foreach(analyzeModuleInScope) - } + modules.foreach(analyzeModuleInScope) + ctx.endpoint.sendToClient( + Api.Response(Api.AnalyzeModuleInScopeJobFinished()) + ) } } @@ -94,9 +75,6 @@ final class AnalyzeModuleInScopeJob( case _: Suggestion.Local => false } - private def rootName(name: QualifiedName): String = - name.path.headOption.getOrElse(name.item) - /** Send notification about module updates. * * @param payload the module update @@ -116,23 +94,11 @@ final class AnalyzeModuleInScopeJob( object AnalyzeModuleInScopeJob { - /** Create an instance of [[AnalyzeModuleInScopeJob]]. - * - * @param project the project module name - * @param modules the list of modules to analyze - * @return the [[AnalyzeModuleInScopeJob]] - */ - def apply( - project: QualifiedName, - modules: Iterable[Module] - ): AnalyzeModuleInScopeJob = - new AnalyzeModuleInScopeJob(Some(project), modules) - /** Create an instance of [[AnalyzeModuleInScopeJob]]. * * @param modules the list of modules to analyze * @return the [[AnalyzeModuleInScopeJob]] */ def apply(modules: Iterable[Module]): AnalyzeModuleInScopeJob = - new AnalyzeModuleInScopeJob(None, modules) + new AnalyzeModuleInScopeJob(modules) } diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala new file mode 100644 index 00000000000..5b73a033f0e --- /dev/null +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala @@ -0,0 +1,44 @@ +package org.enso.interpreter.instrument.job + +import org.enso.editions.LibraryName +import org.enso.interpreter.instrument.execution.RuntimeContext +import org.enso.polyglot.runtime.Runtime.Api + +import java.util.logging.Level + +import scala.jdk.CollectionConverters._ + +/** A job responsible for deserializing suggestions of loaded library. + * + * @param libraryName the name of loaded library + */ +final class DeserializeLibrarySuggestionsJob( + libraryName: LibraryName +) extends Job[Unit]( + List(), + isCancellable = false, + mayInterruptIfRunning = false + ) { + + /** @inheritdoc */ + override def run(implicit ctx: RuntimeContext): Unit = { + ctx.executionService.getLogger.log( + Level.FINE, + s"Deserializing suggestions for library [$libraryName]." + ) + val serializationManager = + ctx.executionService.getContext.getCompiler.getSerializationManager + serializationManager + .deserializeSuggestions(libraryName) + .foreach { cachedSuggestions => + ctx.endpoint.sendToClient( + Api.Response( + Api.SuggestionsDatabaseSuggestionsLoadedNotification( + libraryName, + cachedSuggestions.getSuggestions.asScala.toVector + ) + ) + ) + } + } +} diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala index 0f173e9e55d..694f1351000 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala @@ -27,7 +27,6 @@ import org.enso.text.buffer.Rope import java.io.File import java.util.logging.Level -import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ /** A job that ensures that specified files are compiled. @@ -58,7 +57,7 @@ final class EnsureCompiledJob(protected val files: Iterable[File]) * @param files the list of files to compile. * @param ctx the runtime context */ - protected def ensureCompiledFiles( + private def ensureCompiledFiles( files: Iterable[File] )(implicit ctx: RuntimeContext): CompilationStatus = { val modules = files.flatMap { file => @@ -66,7 +65,7 @@ final class EnsureCompiledJob(protected val files: Iterable[File]) } val moduleCompilationStatus = modules.map(ensureCompiledModule) val modulesInScope = - getModulesInScope.filterNot(m => modules.exists(_ == m)) + getProjectModulesInScope.filterNot(m => modules.exists(_ == m)) val scopeCompilationStatus = ensureCompiledScope(modulesInScope) (moduleCompilationStatus.flatten ++ scopeCompilationStatus).maxOption .getOrElse(CompilationStatus.Success) @@ -84,14 +83,8 @@ final class EnsureCompiledJob(protected val files: Iterable[File]) compile(module) applyEdits(new File(module.getPath)).map { changeset => compile(module) - .map { compilerResult => + .map { _ => invalidateCaches(module, changeset) - ctx.jobProcessor.runBackground( - AnalyzeModuleInScopeJob( - module.getName, - compilerResult.compiledModules - ) - ) ctx.jobProcessor.runBackground(AnalyzeModuleJob(module, changeset)) runCompilationDiagnostics(module) } @@ -433,11 +426,16 @@ final class EnsureCompiledJob(protected val files: Iterable[File]) module.getIr.getMetadata(CachePreferenceAnalysis) } - /** Get all modules in the current compiler scope. */ - private def getModulesInScope(implicit + /** Get all project modules in the current compiler scope. */ + private def getProjectModulesInScope(implicit ctx: RuntimeContext - ): Iterable[Module] = - ctx.executionService.getContext.getTopScope.getModules.asScala + ): Iterable[Module] = { + val packageRepository = + ctx.executionService.getContext.getCompiler.packageRepository + packageRepository.getMainProjectPackage + .map(pkg => packageRepository.getModulesForLibrary(pkg.libraryName)) + .getOrElse(Seq()) + } /** Check if stack belongs to the provided module. * diff --git a/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java b/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java index c6ce59260ca..a775af6587e 100644 --- a/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java +++ b/engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java @@ -1,5 +1,6 @@ package org.enso.compiler; +import org.enso.docs.sections.DocSectionsBuilder; import org.enso.interpreter.runtime.EnsoContext; import org.enso.pkg.PackageManager; import org.enso.polyglot.LanguageInfo; @@ -55,7 +56,8 @@ public class SerializerTest { ctx.enter(); var result = compiler.run(module); assertEquals(result.compiledModules().exists(m -> m == module), true); - var serializationManager = new SerializationManager(ensoContext.getCompiler()); + var serializationManager = + new SerializationManager(ensoContext.getCompiler(), DocSectionsBuilder.apply()); var future = serializationManager.serializeModule(module, true); var serialized = future.get(5, TimeUnit.SECONDS); assertEquals(serialized, true); diff --git a/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java b/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java new file mode 100644 index 00000000000..95ae60230ae --- /dev/null +++ b/engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java @@ -0,0 +1,92 @@ +package org.enso.compiler.test.context; + +import java.util.List; + +import org.enso.polyglot.Suggestion; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import org.junit.Test; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.scala.DefaultScalaModule; + +import scala.Option; + +public class JacksonTest { + + @Test + public void testSerdeOfSuggestion() throws Exception { + Object shape = new Suggestion.Module( + "SampleModule", + Option.apply("doc"), + Option.apply("html"), + Option.empty(), + Option.empty() + ); + final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule()); + String result = m + .writerWithDefaultPrettyPrinter() + .writeValueAsString(shape); + + Suggestion suggestion = m.readerFor(Suggestion.class).readValue(result); + assertEquals("SampleModule", suggestion.name()); + assertEquals("doc", suggestion.documentation().get()); + assertEquals(Suggestion.Module.class, suggestion.getClass()); + } + + @Test + public void testArraySerdeOfSuggestion() throws Exception { + Object shape = new Suggestion[]{new Suggestion.Module( + "SampleModule", + Option.apply("doc"), + Option.apply("html"), + Option.empty(), + Option.empty() + )}; + final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule()); + String result = m + .writerWithDefaultPrettyPrinter() + .writeValueAsString(shape); + + var it = m.readerFor(Suggestion.class).readValues(result); + var suggestion = it.nextValue(); + assertEquals(Suggestion.Module.class, suggestion.getClass()); + if (suggestion instanceof Suggestion.Module module) { + assertEquals("SampleModule", module.name()); + assertEquals("doc", module.documentation().get()); + } else { + fail("Expecting Suggestion.Module: " + suggestion); + } + } + + @Test + public void testRecordSerdeOfSuggestion() throws Exception { + Object shape = new SuggestionCache(11, List.of(new Suggestion.Module( + "SampleModule", + Option.apply("doc"), + Option.apply("html"), + Option.empty(), + Option.empty() + ))); + final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule()); + String result = m + .writerWithDefaultPrettyPrinter() + .writeValueAsString(shape); + + var cache = (SuggestionCache) m.readerFor(SuggestionCache.class).readValue(result); + assertEquals("One suggestion", 1, cache.suggestions.size()); + if (cache.suggestions().get(0) instanceof Suggestion.Module module) { + assertEquals("SampleModule", module.name()); + assertEquals("doc", module.documentation().get()); + } else { + fail("Expecting Suggestion.Module: " + cache); + } + } + + public record SuggestionCache( + @JsonProperty("version") int version, + @JsonProperty("suggestions") List suggestions + ) { + } +} diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/search/DocSectionsBuilder.scala b/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala similarity index 95% rename from engine/language-server/src/main/scala/org/enso/languageserver/search/DocSectionsBuilder.scala rename to lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala index 1fa6369c907..1400d9dc04c 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/search/DocSectionsBuilder.scala +++ b/lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala @@ -1,6 +1,5 @@ -package org.enso.languageserver.search +package org.enso.docs.sections -import org.enso.docs.sections.{HtmlRepr, ParsedSectionsBuilder, Section} import org.enso.polyglot.DocSection import org.enso.syntax.text.DocParser import org.enso.syntax.text.ast.Doc diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/DocSectionsBuilderTest.scala b/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala similarity index 99% rename from engine/language-server/src/test/scala/org/enso/languageserver/search/DocSectionsBuilderTest.scala rename to lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala index 50eeaad19a7..8dc47e2fb7f 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/DocSectionsBuilderTest.scala +++ b/lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala @@ -1,4 +1,4 @@ -package org.enso.languageserver.search +package org.enso.docs.sections import org.enso.polyglot.DocSection import org.scalatest.matchers.should.Matchers diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala index b8f689390b2..cb9194a372d 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala @@ -43,6 +43,9 @@ case class Package[F]( val bindingsCacheDirectory: F = internalDirectory .getChild(Package.cacheDirName) .getChild(Package.bindingsCacheDirName) + val suggestionsCacheDirectory: F = internalDirectory + .getChild(Package.cacheDirName) + .getChild(Package.suggestionsCacheDirName) /** Sets the package name. * @@ -82,10 +85,30 @@ case class Package[F]( irCacheDirectory.getChild(ensoVersion) } + /** Gets the bindings cache root location within this package for a given Enso + * version. + * + * This will create the location if it does not exist. + * + * @param ensoVersion the enso version to get the cache root for + * @return the cache root location + */ def getBindingsCacheRootForPackage(ensoVersion: String): F = { bindingsCacheDirectory.getChild(ensoVersion) } + /** Gets the suggestions cache root location within this package for a given + * Enso version. + * + * This will create the location if it does not exist. + * + * @param ensoVersion the enso version to get the cache root for + * @return the cache root location + */ + def getSuggestionsCacheRootForPackage(ensoVersion: String): F = { + suggestionsCacheDirectory.getChild(ensoVersion) + } + /** Changes the package name. * * @param newName the new package name @@ -495,4 +518,5 @@ object Package { val cacheDirName = "cache" val irCacheDirName = "ir" val bindingsCacheDirName = "bindings" + val suggestionsCacheDirName = "suggestions" } diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/QualifiedName.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/QualifiedName.scala index b68f1d7cc13..8950db65ec3 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/QualifiedName.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/QualifiedName.scala @@ -1,8 +1,9 @@ package org.enso.pkg -import scala.jdk.CollectionConverters._; import com.oracle.truffle.api.CompilerDirectives +import scala.jdk.CollectionConverters._ + /** Represents a qualified name of a source item. * * @param path the names of the package and directories the item is diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index 7d4d644e7ae..a789f6c68ae 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -38,8 +38,7 @@ object DistributionPackage { } } - /** - * Conditional copying, based on the contents of cache and timestamps of files. + /** Conditional copying, based on the contents of cache and timestamps of files. * * @param source source directory * @param destination target directory @@ -123,7 +122,6 @@ object DistributionPackage { targetStdlibVersion: String, targetDir: File ): Unit = { - copyDirectoryIncremental( file("distribution/engine/THIRD-PARTY"), distributionRoot / "THIRD-PARTY", @@ -135,15 +133,7 @@ object DistributionPackage { distributionRoot / "component", cacheFactory.make("engine-jars") ) - val os = System.getProperty("os.name") - val isMac = os.startsWith("Mac") - val parser = targetDir / (if (isMac) { - "libenso_parser.dylib" - } else if (os.startsWith("Windows")) { - "enso_parser.dll" - } else { - "libenso_parser.so" - }) + val parser = targetDir / Platform.dynamicLibraryFileName("enso_parser") copyFilesIncremental( Seq(parser), distributionRoot / "component", @@ -583,7 +573,7 @@ object DistributionPackage { arguments: String* ): String = { val shallowFile = graalDir / "bin" / "gu" - val deepFile = graalDir / "Contents" / "Home" / "bin" / "gu" + val deepFile = graalDir / "Contents" / "Home" / "bin" / "gu" val executableFile = os match { case OS.Linux => shallowFile @@ -597,11 +587,13 @@ object DistributionPackage { graalDir / "bin" / "gu.cmd" } val javaHomeFile = executableFile.getParentFile.getParentFile - val javaHome = javaHomeFile.toPath.toAbsolutePath + val javaHome = javaHomeFile.toPath.toAbsolutePath val command = executableFile.toPath.toAbsolutePath.toString +: arguments - log.debug(s"Running $command in $graalDir with JAVA_HOME=${javaHome.toString}") + log.debug( + s"Running $command in $graalDir with JAVA_HOME=${javaHome.toString}" + ) try { Process( diff --git a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso index 2aac48c15f6..11b7d9dd6a4 100644 --- a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso +++ b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso @@ -16,11 +16,17 @@ export project.Data.Array.Array import project.Data.Boolean from project.Data.Boolean export Boolean, True, False +import project.Data.Text.Text +export project.Data.Text.Text + +import project.Data.Time.Date.Date +export project.Data.Time.Date.Date + import project.Data.List.List export project.Data.List.List -import project.Data.Numbers.Number -export project.Data.Numbers.Number +import project.Data.Numbers +from project.Data.Numbers export Number, Integer import project.Data.Vector.Vector export project.Data.Vector.Vector