From 5ea7615bb4b7f2b47bc52ce6987425c3be34ec62 Mon Sep 17 00:00:00 2001 From: Dmitry Bushev Date: Mon, 21 Sep 2020 15:05:58 +0300 Subject: [PATCH] Populate the Suggestions Database with Imported Modules (#1155) During the compilation, the runtime will analyze all modules in scope and send the appropriate suggestion updates to the server. --- build.sbt | 4 - .../src/main/resources/application.conf | 2 +- .../enso/languageserver/boot/MainModule.scala | 4 + .../search/SuggestionsHandler.scala | 78 ++++- .../enso/languageserver/util/Logging.scala | 10 + .../search/SuggestionsHandlerSpec.scala | 5 +- .../websocket/json/BaseServerTest.scala | 2 + .../scala/org/enso/polyglot/Suggestion.scala | 7 +- .../org/enso/polyglot/runtime/Runtime.scala | 38 ++- .../org/enso/interpreter/OptionsHelper.java | 1 - .../org/enso/interpreter/runtime/Context.java | 1 - .../org/enso/interpreter/runtime/Module.java | 2 + .../interpreter/service/ExecutionService.java | 35 ++- .../error/FailedToApplyEditsException.java | 16 + .../error/ModuleNotFoundForFileException.java | 17 ++ .../error/SourceNotFoundException.java | 12 +- ...Changeset.scala => ChangesetBuilder.scala} | 62 +++- .../enso/interpreter/instrument/Handler.scala | 2 +- .../instrument/command/EditFileCmd.scala | 2 +- .../command/InvalidateModulesIndexCmd.scala | 3 +- .../instrument/job/EnsureCompiledJob.scala | 280 +++++++++++++---- .../job/EnsureCompiledStackJob.scala | 19 +- ...tTest.scala => ChangesetBuilderTest.scala} | 6 +- .../test/instrument/RuntimeServerTest.scala | 288 ++++++++++++++---- .../instrument/StdlibRuntimeServerTest.scala | 192 ++++++++++++ .../src/main/resources/reference.conf | 3 +- .../org/enso/searcher/FileVersionsRepo.scala | 8 + .../org/enso/searcher/SuggestionsRepo.scala | 7 + .../searcher/sql/SqlSuggestionsRepo.scala | 39 ++- .../enso/searcher/sql/SqlVersionsRepo.scala | 26 ++ .../src/test/resources/application.conf | 1 - .../searcher/sql/FileVersionsRepoTest.scala | 28 +- .../searcher/sql/SuggestionsRepoTest.scala | 88 +++++- .../main/scala/org/enso/testkit/OsSpec.scala | 12 +- 34 files changed, 1087 insertions(+), 213 deletions(-) create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/service/error/FailedToApplyEditsException.java create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/service/error/ModuleNotFoundForFileException.java rename engine/runtime/src/main/scala/org/enso/compiler/context/{Changeset.scala => ChangesetBuilder.scala} (83%) rename engine/runtime/src/test/scala/org/enso/compiler/test/context/{ChangesetTest.scala => ChangesetBuilderTest.scala} (97%) create mode 100644 engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/StdlibRuntimeServerTest.scala delete mode 100644 lib/scala/searcher/src/test/resources/application.conf diff --git a/build.sbt b/build.sbt index 50a778b77d..5bec4bbee6 100644 --- a/build.sbt +++ b/build.sbt @@ -553,7 +553,6 @@ lazy val `project-manager` = (project in file("lib/scala/project-manager")) "commons-cli" % "commons-cli" % commonsCliVersion, "commons-io" % "commons-io" % commonsIoVersion, "com.beachape" %% "enumeratum-circe" % enumeratumCirceVersion, - "com.typesafe.slick" %% "slick-hikaricp" % slickVersion % Runtime, "com.miguno.akka" %% "akka-mock-scheduler" % akkaMockSchedulerVersion % Test, "org.mockito" %% "mockito-scala" % mockitoScalaVersion % Test ), @@ -691,7 +690,6 @@ lazy val searcher = project libraryDependencies ++= jmh ++ Seq( "com.typesafe.slick" %% "slick" % slickVersion, "org.xerial" % "sqlite-jdbc" % sqliteVersion, - "com.typesafe.slick" %% "slick-hikaricp" % slickVersion % Runtime, "ch.qos.logback" % "logback-classic" % logbackClassicVersion % Test, "org.scalatest" %% "scalatest" % scalatestVersion % Test ) @@ -773,7 +771,6 @@ lazy val `language-server` = (project in file("engine/language-server")) "com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion, "commons-io" % "commons-io" % commonsIoVersion, akkaTestkit % Test, - "com.typesafe.slick" %% "slick-hikaricp" % slickVersion % Runtime, "org.scalatest" %% "scalatest" % scalatestVersion % Test, "org.scalacheck" %% "scalacheck" % scalacheckVersion % Test, "org.graalvm.sdk" % "polyglot-tck" % graalVersion % "provided" @@ -998,7 +995,6 @@ lazy val runner = project "com.monovore" %% "decline" % declineVersion, "org.jline" % "jline" % jlineVersion, "org.typelevel" %% "cats-core" % catsVersion, - "com.typesafe.slick" %% "slick-hikaricp" % slickVersion % Runtime ), connectInput in run := true ) diff --git a/engine/language-server/src/main/resources/application.conf b/engine/language-server/src/main/resources/application.conf index 5ad1127ec9..b1f656e4e4 100644 --- a/engine/language-server/src/main/resources/application.conf +++ b/engine/language-server/src/main/resources/application.conf @@ -1,3 +1,3 @@ akka.http.server.idle-timeout = infinite akka.http.server.remote-address-header = on -akka.http.server.websocket.periodic-keep-alive-max-idle = 1 second \ No newline at end of file +akka.http.server.websocket.periodic-keep-alive-max-idle = 1 second diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala index 69c89dfaea..606b1f8cd1 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala @@ -187,6 +187,10 @@ class MainModule(serverConfig: LanguageServerConfig) { context.initialize(LanguageInfo.ID) log.trace("Runtime context initialized") + val logLevel = Logging.LogLevel.fromJava(logHandler.getLevel) + system.eventStream.setLogLevel(Logging.LogLevel.toAkka(logLevel)) + log.trace(s"Set akka log level to $logLevel") + val runtimeKiller = system.actorOf( RuntimeKiller.props(runtimeConnector, context), diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala index c17c0a175a..fdcfad350b 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala @@ -12,6 +12,7 @@ import org.enso.languageserver.data.{ CapabilityRegistration, ClientId, Config, + ContentBasedVersioning, ReceivesSuggestionsDatabaseUpdates } import org.enso.languageserver.event.InitializedEvent @@ -74,7 +75,8 @@ final class SuggestionsHandler( fileVersionsRepo: FileVersionsRepo[Future], sessionRouter: ActorRef, runtimeConnector: ActorRef -) extends Actor +)(implicit versionCalculator: ContentBasedVersioning) + extends Actor with Stash with ActorLogging with UnhandledLogging { @@ -91,6 +93,8 @@ final class SuggestionsHandler( .subscribe(self, classOf[Api.SuggestionsDatabaseUpdateNotification]) context.system.eventStream .subscribe(self, classOf[Api.SuggestionsDatabaseReIndexNotification]) + context.system.eventStream + .subscribe(self, classOf[Api.SuggestionsDatabaseIndexUpdateNotification]) context.system.eventStream.subscribe(self, classOf[ProjectNameChangedEvent]) context.system.eventStream.subscribe(self, classOf[FileDeletedEvent]) context.system.eventStream @@ -139,6 +143,23 @@ final class SuggestionsHandler( sender() ! CapabilityReleased context.become(initialized(projectName, clients - client.clientId)) + case msg: Api.SuggestionsDatabaseIndexUpdateNotification => + applyIndexedModuleUpdates(msg.updates.toSeq) + .onComplete { + case Success(notification) => + if (notification.updates.nonEmpty) { + clients.foreach { clientId => + sessionRouter ! DeliverToJsonController(clientId, notification) + } + } + case Failure(ex) => + log.error( + ex, + "Error applying suggestion database updates: {}", + msg.updates.map(_.file) + ) + } + case msg: Api.SuggestionsDatabaseUpdateNotification => applyDatabaseUpdates(msg) .onComplete { @@ -157,7 +178,8 @@ final class SuggestionsHandler( } case msg: Api.SuggestionsDatabaseReIndexNotification => - applyReIndexUpdates(msg.moduleName, msg.updates) + log.debug(s"ReIndex ${msg.moduleName} ${msg.updates.map(_.suggestion)}") + applyReIndexUpdates(msg.updates) .onComplete { case Success(notification) => if (notification.updates.nonEmpty) { @@ -174,9 +196,11 @@ final class SuggestionsHandler( } case Api.ExpressionValuesComputed(_, updates) => - val types = updates.flatMap(update => - update.expressionType.map(update.expressionId -> _) + log.debug( + s"ExpressionValuesComputed ${updates.map(u => (u.expressionId, u.expressionType))}" ) + val types = updates + .flatMap(update => update.expressionType.map(update.expressionId -> _)) suggestionsRepo .updateAll(types) .map { @@ -290,6 +314,33 @@ final class SuggestionsHandler( } } + private def applyIndexedModuleUpdates( + updates: Seq[Api.IndexedModule] + ): Future[SuggestionsDatabaseUpdateNotification] = { + def createIndexedModuleUpdatesBatch( + contents: String, + file: java.io.File, + updates: Seq[Api.SuggestionsDatabaseUpdate.Add] + ): Future[Seq[Api.SuggestionsDatabaseUpdate.Add]] = + fileVersionsRepo + .updateVersion(file, versionCalculator.evalDigest(contents)) + .map(versionChanged => if (versionChanged) updates else Seq()) + def getBatches = + Future + .traverse(updates) { indexed => + createIndexedModuleUpdatesBatch( + indexed.contents, + indexed.file, + indexed.updates + ) + } + .map(_.flatten) + for { + batch <- getBatches + update <- applyReIndexUpdates(batch) + } yield update + } + /** * Handle the suggestions database re-index update. * @@ -297,17 +348,17 @@ final class SuggestionsHandler( * suggestions and builds the notification containing combined removed and * added suggestions. * - * @param moduleName the module name * @param updates the list of updates after the full module re-index * @return the API suggestions database update notification */ private def applyReIndexUpdates( - moduleName: String, updates: Seq[Api.SuggestionsDatabaseUpdate.Add] ): Future[SuggestionsDatabaseUpdateNotification] = { - val added = updates.map(_.suggestion) + val added = updates.map(_.suggestion) + val modules = updates.map(_.suggestion.module).distinct + log.debug(s"Applying re-index updates; modules=$modules") for { - (_, removedIds) <- suggestionsRepo.removeByModule(moduleName) + (_, removedIds) <- suggestionsRepo.removeAllByModule(modules) (version, addedIds) <- suggestionsRepo.insertAll(added) } yield { val updatesRemoved = removedIds.map(SuggestionsDatabaseUpdate.Remove) @@ -315,7 +366,7 @@ final class SuggestionsHandler( case (Some(id), suggestion) => Some(SuggestionsDatabaseUpdate.Add(id, suggestion)) case (None, suggestion) => - log.error("failed to insert suggestion: {}", suggestion) + log.error("Failed to insert re-index suggestion: {}", suggestion) None } SuggestionsDatabaseUpdateNotification( @@ -344,7 +395,10 @@ final class SuggestionsHandler( case ((add, remove), msg: Api.SuggestionsDatabaseUpdate.Remove) => (add, remove :+ msg.suggestion) } - + log.debug( + s"Applying suggestion updates; added=${added + .map(_.name)}; removed=${removed.map(_.name)}" + ) for { (_, removedIds) <- suggestionsRepo.removeAll(removed) (version, addedIds) <- suggestionsRepo.insertAll(added) @@ -357,7 +411,7 @@ final class SuggestionsHandler( case (Some(id), suggestion) => Some(SuggestionsDatabaseUpdate.Add(id, suggestion)) case (None, suggestion) => - log.error("failed to insert suggestion: {}", suggestion) + log.error("Failed to insert suggestion: {}", suggestion) None } SuggestionsDatabaseUpdateNotification( @@ -437,7 +491,7 @@ object SuggestionsHandler { fileVersionsRepo: FileVersionsRepo[Future], sessionRouter: ActorRef, runtimeConnector: ActorRef - ): Props = + )(implicit versionCalculator: ContentBasedVersioning): Props = Props( new SuggestionsHandler( config, diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/util/Logging.scala b/engine/language-server/src/main/scala/org/enso/languageserver/util/Logging.scala index bca674a7b5..2a1dd754e9 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/util/Logging.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/util/Logging.scala @@ -70,6 +70,16 @@ object Logging { case Debug => event.Level.DEBUG case Trace => event.Level.TRACE } + + /** Convert to akka logging level. */ + def toAkka(level: LogLevel): akka.event.Logging.LogLevel = + level match { + case Error => akka.event.Logging.ErrorLevel + case Warning => akka.event.Logging.WarningLevel + case Info => akka.event.Logging.InfoLevel + case Debug => akka.event.Logging.DebugLevel + case Trace => akka.event.Logging.DebugLevel + } } private val ROOT_LOGGER = "org.enso" diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala index ffe7d04001..43a9e2ac90 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala @@ -242,7 +242,7 @@ class SuggestionsHandlerSpec runtimeConnector: TestProbe, suggestionsRepo: SuggestionsRepo[Future], fileVersionsRepo: FileVersionsRepo[Future] - ): ActorRef = { + )(implicit versionCalculator: ContentBasedVersioning): ActorRef = { val handler = system.actorOf( SuggestionsHandler.props( @@ -284,7 +284,8 @@ class SuggestionsHandlerSpec ActorRef ) => Any ): Unit = { - val testContentRoot = Files.createTempDirectory(null).toRealPath() + implicit val versionCalc = Sha3_224VersionCalculator + val testContentRoot = Files.createTempDirectory(null).toRealPath() sys.addShutdownHook(FileUtils.deleteQuietly(testContentRoot.toFile)) val config = newConfig(testContentRoot.toFile) val router = TestProbe("session-router") diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala index d26cd842b7..dfb14c8c2a 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala @@ -80,6 +80,8 @@ class BaseServerTest extends JsonRpcServerTestKit { ) override def clientControllerFactory: ClientControllerFactory = { + implicit val versionCalculator = Sha3_224VersionCalculator + val zioExec = ZioExec(zio.Runtime.default) val sqlDatabase = SqlDatabase(config.directories.suggestionsDatabaseFile) val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase)(system.dispatcher) diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala index cfd7b8c805..88253fc4ac 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala @@ -26,7 +26,12 @@ import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} ) ) ) -sealed trait Suggestion +sealed trait Suggestion { + + def name: String + def module: String +} + object Suggestion { type ExternalId = UUID diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala index c27f0de9a5..0e9397a41e 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala @@ -179,6 +179,10 @@ object Runtime { new JsonSubTypes.Type( value = classOf[Api.InvalidateModulesIndexResponse], name = "invalidateModulesIndexResponse" + ), + new JsonSubTypes.Type( + value = classOf[Api.SuggestionsDatabaseIndexUpdateNotification], + name = "suggestionsDatabaseIndexUpdateNotification" ) ) ) @@ -726,18 +730,40 @@ object Runtime { updates: Seq[SuggestionsDatabaseUpdate.Add] ) extends ApiNotification - private lazy val mapper = { - val factory = new CBORFactory() - val mapper = new ObjectMapper(factory) with ScalaObjectMapper - mapper.registerModule(DefaultScalaModule) - } - /** A request to invalidate the indexed flag of the modules. */ case class InvalidateModulesIndexRequest() extends ApiRequest /** Signals that the module indexes has been invalidated. */ case class InvalidateModulesIndexResponse() extends ApiResponse + /** + * An indexed module. + * + * @param file the module file path + * @param contents the module source + * @param updates the list of suggestions extracted from module + */ + case class IndexedModule( + file: File, + contents: String, + updates: Seq[SuggestionsDatabaseUpdate.Add] + ) extends ApiNotification + + /** + * A notification about new indexed modules. + * + * @param updates the list of suggestions database updates + */ + case class SuggestionsDatabaseIndexUpdateNotification( + updates: Iterable[IndexedModule] + ) extends ApiNotification + + private lazy val mapper = { + val factory = new CBORFactory() + val mapper = new ObjectMapper(factory) with ScalaObjectMapper + mapper.registerModule(DefaultScalaModule) + } + /** * Serializes a Request into a byte buffer. * diff --git a/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java b/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java index 328b4beec9..9f2ab1c1e3 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java @@ -2,7 +2,6 @@ package org.enso.interpreter; import com.oracle.truffle.api.TruffleFile; import com.oracle.truffle.api.TruffleLanguage; -import java.io.File; import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java index 5a9edfa765..cb38110114 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java @@ -9,7 +9,6 @@ import org.enso.interpreter.Language; import org.enso.interpreter.OptionsHelper; import org.enso.interpreter.runtime.builtin.Builtins; import org.enso.interpreter.runtime.callable.atom.AtomConstructor; -import org.enso.interpreter.runtime.scope.ModuleScope; import org.enso.interpreter.runtime.scope.TopLevelScope; import org.enso.interpreter.runtime.util.TruffleFileSystem; import org.enso.interpreter.util.ScalaConversions; diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java index 049406b357..437eab3a51 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java @@ -229,6 +229,7 @@ public class Module implements TruffleObject { Source.newBuilder(LanguageInfo.ID, literalSource.characters(), name.toString()).build(); } else if (sourceFile != null) { cachedSource = Source.newBuilder(LanguageInfo.ID, sourceFile).build(); + literalSource = Rope.apply(cachedSource.getCharacters().toString()); } return cachedSource; } @@ -345,6 +346,7 @@ public class Module implements TruffleObject { throws ArityException { Types.extractArguments(args); module.cachedSource = null; + module.literalSource = null; try { module.compile(context); } catch (IOException ignored) { diff --git a/engine/runtime/src/main/java/org/enso/interpreter/service/ExecutionService.java b/engine/runtime/src/main/java/org/enso/interpreter/service/ExecutionService.java index dc77528ccb..0c3f257cea 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/service/ExecutionService.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/service/ExecutionService.java @@ -5,7 +5,7 @@ import com.oracle.truffle.api.instrumentation.EventBinding; import com.oracle.truffle.api.instrumentation.ExecutionEventListener; import com.oracle.truffle.api.interop.*; import com.oracle.truffle.api.source.SourceSection; -import org.enso.compiler.context.Changeset; +import org.enso.compiler.context.ChangesetBuilder; import org.enso.interpreter.instrument.IdExecutionInstrument; import org.enso.interpreter.instrument.MethodCallsCache; import org.enso.interpreter.instrument.RuntimeCache; @@ -16,10 +16,7 @@ import org.enso.interpreter.runtime.callable.atom.AtomConstructor; import org.enso.interpreter.runtime.callable.function.Function; import org.enso.interpreter.runtime.scope.ModuleScope; import org.enso.interpreter.runtime.state.data.EmptyMap; -import org.enso.interpreter.service.error.ConstructorNotFoundException; -import org.enso.interpreter.service.error.MethodNotFoundException; -import org.enso.interpreter.service.error.ModuleNotFoundException; -import org.enso.interpreter.service.error.SourceNotFoundException; +import org.enso.interpreter.service.error.*; import org.enso.polyglot.LanguageInfo; import org.enso.polyglot.MethodNames; import org.enso.text.buffer.Rope; @@ -29,6 +26,7 @@ import org.enso.text.editing.TextEditor; import org.enso.text.editing.model; import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -204,7 +202,7 @@ public class ExecutionService { */ public void setModuleSources(File path, String contents, boolean isIndexed) { Optional module = context.getModuleForFile(path); - if (!module.isPresent()) { + if (module.isEmpty()) { module = context.createModuleForFile(path); } module.ifPresent( @@ -241,23 +239,30 @@ public class ExecutionService { * @param edits the edits to apply. * @return an object for computing the changed IR nodes. */ - public Optional> modifyModuleSources(File path, List edits) { + public ChangesetBuilder modifyModuleSources( + File path, List edits) { Optional moduleMay = context.getModuleForFile(path); - if (!moduleMay.isPresent()) { - return Optional.empty(); + if (moduleMay.isEmpty()) { + throw new ModuleNotFoundForFileException(path); } Module module = moduleMay.get(); - if (module.getLiteralSource() == null) { - return Optional.empty(); + try { + module.getSource(); + } catch (IOException e) { + throw new SourceNotFoundException(path, e); } - Changeset changeset = - new Changeset<>( + ChangesetBuilder changesetBuilder = + new ChangesetBuilder<>( module.getLiteralSource(), module.getIr(), TextEditor.ropeTextEditor(), IndexedSource.RopeIndexedSource()); Optional editedSource = JavaEditorAdapter.applyEdits(module.getLiteralSource(), edits); - editedSource.ifPresent(module::setLiteralSource); - return Optional.of(changeset); + editedSource.ifPresentOrElse( + module::setLiteralSource, + () -> { + throw new FailedToApplyEditsException(path); + }); + return changesetBuilder; } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/service/error/FailedToApplyEditsException.java b/engine/runtime/src/main/java/org/enso/interpreter/service/error/FailedToApplyEditsException.java new file mode 100644 index 0000000000..16d0d04da6 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/service/error/FailedToApplyEditsException.java @@ -0,0 +1,16 @@ +package org.enso.interpreter.service.error; + +import java.io.File; + +/** Thrown when the edits can not be applied to the source. */ +public class FailedToApplyEditsException extends RuntimeException implements ServiceException { + + /** + * Create new instance of this error. + * + * @param path the source file path. + */ + public FailedToApplyEditsException(File path) { + super("Filed to apply edits for file " + path + "."); + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/service/error/ModuleNotFoundForFileException.java b/engine/runtime/src/main/java/org/enso/interpreter/service/error/ModuleNotFoundForFileException.java new file mode 100644 index 0000000000..5041737d56 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/service/error/ModuleNotFoundForFileException.java @@ -0,0 +1,17 @@ +package org.enso.interpreter.service.error; + +import java.io.File; + +/** Thrown when a module for given file was requested but could not be found. */ +public class ModuleNotFoundForFileException extends ModuleNotFoundException { + + /** + * Create new instance of this error. + * + * @param path the path of the module file. + */ + public ModuleNotFoundForFileException(File path) { + super("Module not found for file " + path + "."); + } + +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/service/error/SourceNotFoundException.java b/engine/runtime/src/main/java/org/enso/interpreter/service/error/SourceNotFoundException.java index 059abfbedc..23210008a2 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/service/error/SourceNotFoundException.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/service/error/SourceNotFoundException.java @@ -8,7 +8,17 @@ public class SourceNotFoundException extends RuntimeException implements Service * * @param item the item which source code is missing. */ - public SourceNotFoundException(String item) { + public SourceNotFoundException(Object item) { super("The " + item + " source not found."); } + + /** + * Create new instance of this error. + * + * @param item the item which source code is missing. + * @param cause the cause of the exception. + */ + public SourceNotFoundException(Object item, Throwable cause) { + super("The " + item + " source not found.", cause); + } } diff --git a/engine/runtime/src/main/scala/org/enso/compiler/context/Changeset.scala b/engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala similarity index 83% rename from engine/runtime/src/main/scala/org/enso/compiler/context/Changeset.scala rename to engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala index 7357a85894..7ee132a240 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/context/Changeset.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala @@ -11,13 +11,39 @@ import org.enso.text.editing.{IndexedSource, TextEditor} import scala.collection.mutable +/** The changeset of a module containing the computed list of invalidated + * expressions. + * + * @param source the module source + * @param ir the IR node of the module + * @param invalidated the list of invalidated expressions + * @tparam A the source type + */ +case class Changeset[A]( + source: A, + ir: IR, + invalidated: Set[IR.ExternalId] +) + /** Compute invalidated expressions. * * @param source the text source * @param ir the IR node * @tparam A the source type */ -final class Changeset[A: TextEditor: IndexedSource](val source: A, val ir: IR) { +final class ChangesetBuilder[A: TextEditor: IndexedSource]( + val source: A, + val ir: IR +) { + + /** Build the changeset containing the nodes invalidated by the edits. + * + * @param edits the edits applied to the source + * @return the computed changeset + */ + @throws[CompilerError] + def build(edits: Seq[TextEdit]): Changeset[A] = + Changeset(source, ir, compute(edits)) /** Traverses the IR and returns a list of all IR nodes affected by the edit * using the [[DataflowAnalysis]] information. @@ -35,7 +61,7 @@ final class Changeset[A: TextEditor: IndexedSource](val source: A, val ir: IR) { ) val direct = invalidated(edits) val transitive = direct - .map(Changeset.toDataflowDependencyType) + .map(ChangesetBuilder.toDataflowDependencyType) .flatMap(metadata.getExternal) .flatten direct.flatMap(_.externalId) ++ transitive @@ -47,25 +73,26 @@ final class Changeset[A: TextEditor: IndexedSource](val source: A, val ir: IR) { * @param edits the text edits * @return the set of IR nodes directly affected by the edit */ - def invalidated(edits: Seq[TextEdit]): Set[Changeset.NodeId] = { + def invalidated(edits: Seq[TextEdit]): Set[ChangesetBuilder.NodeId] = { @scala.annotation.tailrec def go( - tree: Changeset.Tree, + tree: ChangesetBuilder.Tree, source: A, edits: mutable.Queue[TextEdit], - ids: mutable.Set[Changeset.NodeId] - ): Set[Changeset.NodeId] = { + ids: mutable.Set[ChangesetBuilder.NodeId] + ): Set[ChangesetBuilder.NodeId] = { if (edits.isEmpty) ids.toSet else { - val edit = edits.dequeue() - val locationEdit = Changeset.toLocationEdit(edit, source) - val invalidatedSet = Changeset.invalidated(tree, locationEdit.location) - val newTree = Changeset.updateLocations(tree, locationEdit) - val newSource = TextEditor[A].edit(source, edit) + val edit = edits.dequeue() + val locationEdit = ChangesetBuilder.toLocationEdit(edit, source) + val invalidatedSet = + ChangesetBuilder.invalidated(tree, locationEdit.location) + val newTree = ChangesetBuilder.updateLocations(tree, locationEdit) + val newSource = TextEditor[A].edit(source, edit) go(newTree, newSource, edits, ids ++= invalidatedSet.map(_.id)) } } - val tree = Changeset.buildTree(ir) + val tree = ChangesetBuilder.buildTree(ir) go(tree, source, mutable.Queue.from(edits), mutable.HashSet()) } @@ -79,7 +106,7 @@ final class Changeset[A: TextEditor: IndexedSource](val source: A, val ir: IR) { } -object Changeset { +object ChangesetBuilder { /** An identifier of IR node. * @@ -226,7 +253,7 @@ object Changeset { * @return the invalidated nodes of the tree */ private def invalidated(tree: Tree, edit: Location): Tree = { - val invalidated = mutable.TreeSet[Changeset.Node]() + val invalidated = mutable.TreeSet[ChangesetBuilder.Node]() tree.iterator.foreach { node => if (intersect(edit, node)) { invalidated += node @@ -242,7 +269,10 @@ object Changeset { * @param node the node * @return true if the node and edit locations are intersecting */ - private def intersect(edit: Location, node: Changeset.Node): Boolean = { + private def intersect( + edit: Location, + node: ChangesetBuilder.Node + ): Boolean = { intersect(edit, node.location) } @@ -268,7 +298,7 @@ object Changeset { private def inside(index: Int, location: Location): Boolean = index >= location.start && index <= location.end - /** Convert [[TextEdit]] to [[Changeset.LocationEdit]] edit in the provided + /** Convert [[TextEdit]] to [[ChangesetBuilder.LocationEdit]] edit in the provided * source. * * @param edit the text edit diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala index 56bedfae61..dc54cb3d13 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala @@ -72,7 +72,6 @@ final class Handler { ): Unit = { executionService = service truffleContext = context - endpoint.sendToClient(Api.Response(Api.InitializedNotification())) val interpreterCtx = InterpreterContext( executionService, @@ -81,6 +80,7 @@ final class Handler { truffleContext ) commandProcessor = new CommandExecutionEngine(interpreterCtx) + endpoint.sendToClient(Api.Response(Api.InitializedNotification())) } /** diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/EditFileCmd.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/EditFileCmd.scala index dda7fa0070..9c80950876 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/EditFileCmd.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/EditFileCmd.scala @@ -25,7 +25,7 @@ class EditFileCmd(request: Api.EditFileNotification) extends Command(None) { for { _ <- Future { ctx.jobControlPlane.abortAllJobs() } _ <- ctx.jobProcessor.run( - new EnsureCompiledJob(request.path, request.edits) + EnsureCompiledJob(request.path, request.edits) ) _ <- Future.sequence(executeJobs.map(ctx.jobProcessor.run)) } yield () diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/InvalidateModulesIndexCmd.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/InvalidateModulesIndexCmd.scala index 66a3f64819..aee36fe84e 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/InvalidateModulesIndexCmd.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/InvalidateModulesIndexCmd.scala @@ -2,7 +2,6 @@ package org.enso.interpreter.instrument.command import org.enso.interpreter.instrument.execution.RuntimeContext import org.enso.polyglot.runtime.Runtime.Api -import org.enso.polyglot.runtime.Runtime.Api.RequestId import scala.concurrent.{ExecutionContext, Future} @@ -13,7 +12,7 @@ import scala.concurrent.{ExecutionContext, Future} * @param request a request for invalidation */ class InvalidateModulesIndexCmd( - maybeRequestId: Option[RequestId], + maybeRequestId: Option[Api.RequestId], val request: Api.InvalidateModulesIndexRequest ) extends Command(maybeRequestId) { diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala index 2cd09bc30f..a5cb07701d 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala @@ -1,9 +1,10 @@ package org.enso.interpreter.instrument.job -import java.io.File -import java.util.Optional +import java.io.{File, IOException} +import java.util.logging.Level import org.enso.compiler.context.{Changeset, SuggestionBuilder} +import org.enso.compiler.phase.ImportResolver import org.enso.interpreter.instrument.CacheInvalidation import org.enso.interpreter.instrument.execution.RuntimeContext import org.enso.interpreter.runtime.Module @@ -24,63 +25,158 @@ import scala.jdk.OptionConverters._ class EnsureCompiledJob(protected val files: Iterable[File]) extends Job[Unit](List.empty, true, false) { - /** - * Create a job ensuring that files are compiled after applying the edits. - * - * @param file a file to compile - */ - def this(file: File, edits: Seq[TextEdit]) = { - this(List(file)) - EnsureCompiledJob.enqueueEdits(file, edits) - } - /** @inheritdoc */ override def run(implicit ctx: RuntimeContext): Unit = { ctx.locking.acquireWriteCompilationLock() try { - ensureCompiled(files) + val modules = ensureCompiledFiles(files) + ensureCompiledScope() + ensureCompiledImports(modules) } finally { ctx.locking.releaseWriteCompilationLock() } } /** - * Run the compilation and invalidation logic. + * Run the scheduled compilation and invalidation logic. * - * @param files the list of files to compile + * @param files the list of files to compile. * @param ctx the runtime context */ - protected def ensureCompiled( + protected def ensureCompiledFiles( files: Iterable[File] + )(implicit ctx: RuntimeContext): Iterable[Module] = { + for { + file <- files + module <- compile(file) + } yield { + val changeset = applyEdits(file) + compile(module) + runInvalidationCommands( + buildCacheInvalidationCommands(changeset, module.getLiteralSource) + ) + analyzeModule(module, changeset) + module + } + } + + /** + * Compile the imported modules and send the suggestion updates. + * + * @param modules the list of modules to analyze. + * @param ctx the runtime context + */ + protected def ensureCompiledImports( + modules: Iterable[Module] )(implicit ctx: RuntimeContext): Unit = { - files.foreach { file => - compile(file).foreach { module => - applyEdits(file).ifPresent { - case (changeset, edits) => - val moduleName = module.getName.toString - runInvalidationCommands( - buildCacheInvalidationCommands(changeset, edits) - ) - if (module.isIndexed) { - val removedSuggestions = SuggestionBuilder(changeset.source) - .build(moduleName, module.getIr) - compile(module) - val addedSuggestions = - SuggestionBuilder(changeset.applyEdits(edits)) - .build(moduleName, module.getIr) - sendSuggestionsUpdateNotification( - removedSuggestions diff addedSuggestions, - addedSuggestions diff removedSuggestions - ) - } else { - val addedSuggestions = - SuggestionBuilder(changeset.applyEdits(edits)) - .build(moduleName, module.getIr) - sendReIndexNotification(moduleName, addedSuggestions) - module.setIndexed(true) - } - } - } + modules.foreach { module => + compile(module) + val importedModules = + new ImportResolver(ctx.executionService.getContext.getCompiler) + .mapImports(module) + .filter(_.getName != module.getName) + ctx.executionService.getLogger.finest( + s"Module ${module.getName} imports ${importedModules.map(_.getName)}" + ) + importedModules.foreach(analyzeImport) + } + } + + /** + * Compile all modules in the scope and send the extracted suggestions. + * + * @param ctx the runtime context + */ + protected def ensureCompiledScope()(implicit ctx: RuntimeContext): Unit = { + val modulesInScope = + ctx.executionService.getContext.getTopScope.getModules.asScala + ctx.executionService.getLogger + .finest(s"Modules in scope: ${modulesInScope.map(_.getName)}") + val updates = modulesInScope.flatMap { module => + compile(module) + analyzeModuleInScope(module) + } + sendIndexUpdateNotification( + Api.SuggestionsDatabaseIndexUpdateNotification(updates) + ) + } + + private def analyzeImport( + module: Module + )(implicit ctx: RuntimeContext): Unit = { + if (!module.isIndexed && module.getLiteralSource != null) { + ctx.executionService.getLogger + .finest(s"Analyzing imported ${module.getName}") + val moduleName = module.getName.toString + val addedSuggestions = SuggestionBuilder(module.getLiteralSource) + .build(module.getName.toString, module.getIr) + .filter(isSuggestionGlobal) + sendReIndexNotification(moduleName, addedSuggestions) + module.setIndexed(true) + } + } + + private def analyzeModuleInScope(module: Module)(implicit + ctx: RuntimeContext + ): Option[Api.IndexedModule] = { + try module.getSource + catch { + case e: IOException => + ctx.executionService.getLogger.log( + Level.SEVERE, + s"Failed to get module source to analyze ${module.getName}", + e + ) + } + if ( + !module.isIndexed && + module.getLiteralSource != null && + module.getPath != null + ) { + ctx.executionService.getLogger + .finest(s"Analyzing module in scope ${module.getName}") + val moduleName = module.getName.toString + val addedSuggestions = SuggestionBuilder(module.getLiteralSource) + .build(moduleName, module.getIr) + .filter(isSuggestionGlobal) + module.setIndexed(true) + Some( + Api.IndexedModule( + new File(module.getPath), + module.getSource.getCharacters.toString, + addedSuggestions.map(Api.SuggestionsDatabaseUpdate.Add) + ) + ) + } else { + None + } + } + + private def analyzeModule( + module: Module, + changeset: Changeset[Rope] + )(implicit ctx: RuntimeContext): Unit = { + val moduleName = module.getName.toString + if (module.isIndexed) { + ctx.executionService.getLogger + .finest(s"Analyzing indexed module ${module.getName}") + val removedSuggestions = SuggestionBuilder(changeset.source) + .build(moduleName, changeset.ir) + val addedSuggestions = + SuggestionBuilder(module.getLiteralSource) + .build(moduleName, module.getIr) + sendSuggestionsUpdateNotification( + removedSuggestions diff addedSuggestions, + addedSuggestions diff removedSuggestions + ) + } else { + ctx.executionService.getLogger + .finest(s"Analyzing not-indexed module ${module.getName}") + val addedSuggestions = + SuggestionBuilder(module.getLiteralSource) + .build(moduleName, module.getIr) + sendReIndexNotification(moduleName, addedSuggestions) + module.setIndexed(true) } } @@ -107,28 +203,36 @@ class EnsureCompiledJob(protected val files: Iterable[File]) * @param ctx the runtime context * @return the compiled module */ - private def compile(module: Module)(implicit ctx: RuntimeContext): Module = + private def compile( + module: Module + )(implicit ctx: RuntimeContext): Module = { + val prevStage = module.getCompilationStage module.compileScope(ctx.executionService.getContext).getModule + if (prevStage != module.getCompilationStage) { + ctx.executionService.getLogger.finer( + s"Compiled ${module.getName} $prevStage->${module.getCompilationStage}" + ) + } + module + } /** * Apply pending edits to the file. * * @param file the file to apply edits to * @param ctx the runtime context - * @return the [[Changeset]] object and the list of applied edits + * @return the [[Changeset]] after applying the edits to the source */ private def applyEdits( file: File - )(implicit - ctx: RuntimeContext - ): Optional[(Changeset[Rope], Seq[TextEdit])] = { + )(implicit ctx: RuntimeContext): Changeset[Rope] = { ctx.locking.acquireFileLock(file) ctx.locking.acquireReadCompilationLock() try { val edits = EnsureCompiledJob.dequeueEdits(file) - ctx.executionService + val suggestionBuilder = ctx.executionService .modifyModuleSources(file, edits.asJava) - .map(_ -> edits) + suggestionBuilder.build(edits) } finally { ctx.locking.releaseReadCompilationLock() ctx.locking.releaseFileLock(file) @@ -138,20 +242,19 @@ class EnsureCompiledJob(protected val files: Iterable[File]) /** * Create cache invalidation commands after applying the edits. * - * @param changeset the [[Changeset]] object capturing the previous version - * of IR - * @param edits the list of applied edits + * @param changeset the [[Changeset]] object capturing the previous + * version of IR * @param ctx the runtime context * @return the list of cache invalidation commands */ private def buildCacheInvalidationCommands( changeset: Changeset[Rope], - edits: Seq[TextEdit] + source: Rope )(implicit ctx: RuntimeContext): Seq[CacheInvalidation] = { val invalidateExpressionsCommand = - CacheInvalidation.Command.InvalidateKeys(changeset.compute(edits)) + CacheInvalidation.Command.InvalidateKeys(changeset.invalidated) val scopeIds = ctx.executionService.getContext.getCompiler - .parseMeta(changeset.source.toString) + .parseMeta(source.toString) .map(_._2) val invalidateStaleCommand = CacheInvalidation.Command.InvalidateStale(scopeIds) @@ -217,7 +320,7 @@ class EnsureCompiledJob(protected val files: Iterable[File]) private def sendReIndexNotification( moduleName: String, added: Seq[Suggestion] - )(implicit ctx: RuntimeContext): Unit = + )(implicit ctx: RuntimeContext): Unit = { ctx.endpoint.sendToClient( Api.Response( Api.SuggestionsDatabaseReIndexNotification( @@ -226,6 +329,25 @@ class EnsureCompiledJob(protected val files: Iterable[File]) ) ) ) + } + + private def sendIndexUpdateNotification( + msg: Api.SuggestionsDatabaseIndexUpdateNotification + )(implicit + ctx: RuntimeContext + ): Unit = { + if (msg.updates.nonEmpty) { + ctx.endpoint.sendToClient(Api.Response(msg)) + } + } + + private def isSuggestionGlobal(suggestion: Suggestion): Boolean = + suggestion match { + case _: Suggestion.Atom => true + case _: Suggestion.Method => true + case _: Suggestion.Function => false + case _: Suggestion.Local => false + } } object EnsureCompiledJob { @@ -241,4 +363,46 @@ object EnsureCompiledJob { case Some(v) => Some(v :++ edits) case None => Some(edits) } + + /** + * Create a job ensuring that files are compiled. + * + * @param files the list of files to compile + * @return a new job + */ + def apply(files: Iterable[File]): EnsureCompiledJob = { + new EnsureCompiledJob(files) + } + + /** + * Create a job ensuring that files are compiled after applying the edits. + * + * @param file a file to compile + * @param edits the list of edits to apply + * @return a new job + */ + def apply(file: File, edits: Seq[TextEdit]): EnsureCompiledJob = { + EnsureCompiledJob.enqueueEdits(file, edits) + EnsureCompiledJob(List(file)) + } + + /** + * Create a job ensuring that modules are compiled. + * + * @param modules a list of modules to compile + * @return a new job + */ + def apply( + modules: Iterable[Module] + )(implicit ctx: RuntimeContext): EnsureCompiledJob = { + val files = modules.flatMap { module => + val file = Option(module.getPath).map(new File(_)) + if (file.isEmpty) { + ctx.executionService.getLogger + .severe(s"Failed to get file for module ${module.getName}") + } + file + } + new EnsureCompiledJob(files) + } } diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledStackJob.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledStackJob.scala index 3d4a126630..34304eb12a 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledStackJob.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledStackJob.scala @@ -5,6 +5,7 @@ import java.io.File import org.enso.compiler.pass.analyse.CachePreferenceAnalysis import org.enso.interpreter.instrument.{CacheInvalidation, InstrumentFrame} import org.enso.interpreter.instrument.execution.RuntimeContext +import org.enso.interpreter.runtime.Module import org.enso.polyglot.runtime.Runtime.Api import scala.jdk.OptionConverters._ @@ -19,10 +20,10 @@ class EnsureCompiledStackJob(stack: Iterable[InstrumentFrame])(implicit ) extends EnsureCompiledJob(EnsureCompiledStackJob.extractFiles(stack)) { /** @inheritdoc */ - override def ensureCompiled( + override protected def ensureCompiledFiles( files: Iterable[File] - )(implicit ctx: RuntimeContext): Unit = { - super.ensureCompiled(files) + )(implicit ctx: RuntimeContext): Iterable[Module] = { + val modules = super.ensureCompiledFiles(files) getCacheMetadata(stack).foreach { metadata => CacheInvalidation.run( stack, @@ -32,6 +33,7 @@ class EnsureCompiledStackJob(stack: Iterable[InstrumentFrame])(implicit ) ) } + modules } private def getCacheMetadata( @@ -44,7 +46,7 @@ class EnsureCompiledStackJob(stack: Iterable[InstrumentFrame])(implicit module.getIr .unsafeGetMetadata( CachePreferenceAnalysis, - "Empty cache preference metadata" + s"Empty cache preference metadata ${module.getName}" ) } case _ => None @@ -68,7 +70,14 @@ object EnsureCompiledStackJob { case Api.StackItem.ExplicitCall(methodPointer, _, _) => ctx.executionService.getContext .findModule(methodPointer.module) - .flatMap(module => java.util.Optional.ofNullable(module.getPath)) + .flatMap { module => + val path = java.util.Optional.ofNullable(module.getPath) + if (path.isEmpty) { + ctx.executionService.getLogger + .severe(s"${module.getName} module path is empty") + } + path + } .map(path => new File(path)) .toScala case _ => diff --git a/engine/runtime/src/test/scala/org/enso/compiler/test/context/ChangesetTest.scala b/engine/runtime/src/test/scala/org/enso/compiler/test/context/ChangesetBuilderTest.scala similarity index 97% rename from engine/runtime/src/test/scala/org/enso/compiler/test/context/ChangesetTest.scala rename to engine/runtime/src/test/scala/org/enso/compiler/test/context/ChangesetBuilderTest.scala index f2df960264..2262862b9e 100644 --- a/engine/runtime/src/test/scala/org/enso/compiler/test/context/ChangesetTest.scala +++ b/engine/runtime/src/test/scala/org/enso/compiler/test/context/ChangesetBuilderTest.scala @@ -1,12 +1,12 @@ package org.enso.compiler.test.context -import org.enso.compiler.context.Changeset +import org.enso.compiler.context.ChangesetBuilder import org.enso.compiler.core.IR import org.enso.compiler.test.CompilerTest import org.enso.text.buffer.Rope import org.enso.text.editing.model.{Position, Range, TextEdit} -class ChangesetTest extends CompilerTest { +class ChangesetBuilderTest extends CompilerTest { "DiffChangeset" should { @@ -213,5 +213,5 @@ class ChangesetTest extends CompilerTest { } def invalidated(ir: IR, code: String, edits: TextEdit*): Set[IR.Identifier] = - new Changeset(Rope(code), ir).invalidated(edits).map(_.internalId) + new ChangesetBuilder(Rope(code), ir).invalidated(edits).map(_.internalId) } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala index f77b76c74a..f95eca5be8 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala @@ -283,7 +283,8 @@ class RuntimeServerTest } "RuntimeServer" should "push and pop functions on the stack" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -293,6 +294,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // open file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push local item on top of the empty stack val invalidLocalItem = Api.StackItem.LocalCall(context.Main.idMainY) context.send( @@ -1861,8 +1868,7 @@ class RuntimeServerTest ) } - it should "support file modification operations" in { - val fooFile = new File(context.pkg.sourceDir, "Foo.enso") + it should "support file modification operations without attached ids" in { val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -1878,18 +1884,10 @@ class RuntimeServerTest |""".stripMargin // Create a new file - context.writeFile(fooFile, code) + val mainFile = context.writeMain(code) // Open the new file - context.send( - Api.Request( - Api.OpenFileNotification( - fooFile, - code, - false - ) - ) - ) + context.send(Api.Request(Api.OpenFileNotification(mainFile, code, false))) context.receiveNone shouldEqual None context.consumeOut shouldEqual List() @@ -1901,7 +1899,7 @@ class RuntimeServerTest contextId, Api.StackItem .ExplicitCall( - Api.MethodPointer("Test.Foo", "Foo", "main"), + Api.MethodPointer("Test.Main", "Main", "main"), None, Vector() ) @@ -1912,15 +1910,15 @@ class RuntimeServerTest Api.Response(requestId, Api.PushContextResponse(contextId)), Api.Response( Api.SuggestionsDatabaseReIndexNotification( - "Test.Foo", + "Test.Main", Seq( Api.SuggestionsDatabaseUpdate.Add( Suggestion.Method( None, - "Test.Foo", + "Test.Main", "main", Seq(Suggestion.Argument("this", "Any", false, false, None)), - "Foo", + "Main", "Any", None ) @@ -1936,7 +1934,7 @@ class RuntimeServerTest context.send( Api.Request( Api.EditFileNotification( - fooFile, + mainFile, Seq( TextEdit( model.Range(model.Position(2, 25), model.Position(2, 29)), @@ -1950,12 +1948,11 @@ class RuntimeServerTest context.consumeOut shouldEqual List("I'm a modified!") // Close the file - context.send(Api.Request(Api.CloseFileNotification(fooFile))) + context.send(Api.Request(Api.CloseFileNotification(mainFile))) context.consumeOut shouldEqual List() } it should "support file modification operations with attached ids" in { - val fooFile = new File(context.pkg.sourceDir, "Foo.enso") val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() val metadata = new Metadata @@ -1968,13 +1965,13 @@ class RuntimeServerTest ) // Create a new file - context.writeFile(fooFile, code) + val mainFile = context.writeMain(code) // Open the new file context.send( Api.Request( Api.OpenFileNotification( - fooFile, + mainFile, code, false ) @@ -1990,7 +1987,7 @@ class RuntimeServerTest contextId, Api.StackItem .ExplicitCall( - Api.MethodPointer("Test.Foo", "Foo", "main"), + Api.MethodPointer("Test.Main", "Main", "main"), None, Vector() ) @@ -2009,15 +2006,15 @@ class RuntimeServerTest ), Api.Response( Api.SuggestionsDatabaseReIndexNotification( - "Test.Foo", + "Test.Main", Seq( Api.SuggestionsDatabaseUpdate.Add( Suggestion.Method( Some(idMain), - "Test.Foo", + "Test.Main", "main", Seq(Suggestion.Argument("this", "Any", false, false, None)), - "Foo", + "Main", "Any", None ) @@ -2032,7 +2029,7 @@ class RuntimeServerTest context.send( Api.Request( Api.EditFileNotification( - fooFile, + mainFile, Seq( TextEdit( model.Range(model.Position(0, 0), model.Position(0, 9)), @@ -2222,7 +2219,6 @@ class RuntimeServerTest } it should "send suggestion notifications when file is modified" in { - val fooFile = new File(context.pkg.sourceDir, "Foo.enso") val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2238,13 +2234,13 @@ class RuntimeServerTest |""".stripMargin // Create a new file - context.writeFile(fooFile, code) + val mainFile = context.writeMain(code) // Open the new file context.send( Api.Request( Api.OpenFileNotification( - fooFile, + mainFile, code, false ) @@ -2261,7 +2257,7 @@ class RuntimeServerTest contextId, Api.StackItem .ExplicitCall( - Api.MethodPointer("Test.Foo", "Foo", "main"), + Api.MethodPointer("Test.Main", "Main", "main"), None, Vector() ) @@ -2272,15 +2268,15 @@ class RuntimeServerTest Api.Response(requestId, Api.PushContextResponse(contextId)), Api.Response( Api.SuggestionsDatabaseReIndexNotification( - "Test.Foo", + "Test.Main", Seq( Api.SuggestionsDatabaseUpdate.Add( Suggestion.Method( None, - "Test.Foo", + "Test.Main", "main", Seq(Suggestion.Argument("this", "Any", false, false, None)), - "Foo", + "Main", "Any", None ) @@ -2296,7 +2292,7 @@ class RuntimeServerTest context.send( Api.Request( Api.EditFileNotification( - fooFile, + mainFile, Seq( TextEdit( model.Range(model.Position(2, 25), model.Position(2, 29)), @@ -2317,7 +2313,7 @@ class RuntimeServerTest Api.SuggestionsDatabaseUpdate.Add( Suggestion.Method( None, - "Test.Foo", + "Test.Main", "lucky", Seq(Suggestion.Argument("this", "Any", false, false, None)), "Number", @@ -2333,13 +2329,14 @@ class RuntimeServerTest context.consumeOut shouldEqual List("I'm a modified!") // Close the file - context.send(Api.Request(Api.CloseFileNotification(fooFile))) + context.send(Api.Request(Api.CloseFileNotification(mainFile))) context.receiveNone shouldEqual None context.consumeOut shouldEqual List() } it should "recompute expressions without invalidation" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2349,6 +2346,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main val item1 = Api.StackItem.ExplicitCall( Api.MethodPointer("Test.Main", "Main", "main"), @@ -2377,7 +2380,8 @@ class RuntimeServerTest } it should "recompute expressions invalidating all" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2387,6 +2391,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main val item1 = Api.StackItem.ExplicitCall( Api.MethodPointer("Test.Main", "Main", "main"), @@ -2421,7 +2431,8 @@ class RuntimeServerTest } it should "recompute expressions invalidating some" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2431,6 +2442,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + + context.receiveNone shouldEqual None // push main val item1 = Api.StackItem.ExplicitCall( Api.MethodPointer("Test.Main", "Main", "main"), @@ -2467,7 +2484,8 @@ class RuntimeServerTest } it should "return error when module not found" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(context.Main.code) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2477,6 +2495,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + + context.receiveNone shouldEqual None // push main context.send( Api.Request( @@ -2500,7 +2524,8 @@ class RuntimeServerTest } it should "return error when constructor not found" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2510,6 +2535,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( @@ -2536,7 +2567,8 @@ class RuntimeServerTest } it should "return error when method not found" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2546,6 +2578,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( @@ -2582,7 +2620,7 @@ class RuntimeServerTest |bar x y = x + y |""".stripMargin.linesIterator.mkString("\n") val contents = metadata.appendToCode(code) - context.writeMain(contents) + val mainFile = context.writeMain(contents) // create context context.send(Api.Request(requestId, Api.CreateContextRequest(contextId))) @@ -2590,6 +2628,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( @@ -2623,7 +2667,7 @@ class RuntimeServerTest |bar x y = x + y |""".stripMargin.linesIterator.mkString("\n") val contents = metadata.appendToCode(code) - context.writeMain(contents) + val mainFile = context.writeMain(contents) // create context context.send(Api.Request(requestId, Api.CreateContextRequest(contextId))) @@ -2631,6 +2675,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( @@ -2662,7 +2712,7 @@ class RuntimeServerTest |bar x y = x + y |""".stripMargin.linesIterator.mkString("\n") val contents = metadata.appendToCode(code) - context.writeMain(contents) + val mainFile = context.writeMain(contents) // create context context.send(Api.Request(requestId, Api.CreateContextRequest(contextId))) @@ -2670,6 +2720,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( @@ -2707,7 +2763,7 @@ class RuntimeServerTest |main = Number.pi |""".stripMargin val contents = metadata.appendToCode(code) - context.writeMain(contents) + val mainFile = context.writeMain(contents) // create context context.send(Api.Request(requestId, Api.CreateContextRequest(contextId))) @@ -2715,6 +2771,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( @@ -2741,7 +2803,8 @@ class RuntimeServerTest } it should "skip side effects when evaluating cached expression" in { - context.writeMain(context.Main2.code) + val contents = context.Main2.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -2751,6 +2814,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main val item1 = Api.StackItem.ExplicitCall( Api.MethodPointer("Test.Main", "Main", "main"), @@ -2780,7 +2849,8 @@ class RuntimeServerTest } it should "emit visualisation update when expression is evaluated" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(context.Main.code) val visualisationFile = context.writeInSrcDir("Visualisation", context.Visualisation.code) @@ -2804,6 +2874,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // Open the new file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main val item1 = Api.StackItem.ExplicitCall( Api.MethodPointer("Test.Main", "Main", "main"), @@ -2813,11 +2889,51 @@ class RuntimeServerTest context.send( Api.Request(requestId, Api.PushContextRequest(contextId, item1)) ) - context.receive(5) should contain theSameElementsAs Seq( + context.receive(6) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), context.Main.Update.mainX(contextId), context.Main.Update.mainY(contextId), context.Main.Update.mainZ(contextId), + Api.Response( + Api.SuggestionsDatabaseIndexUpdateNotification( + Seq( + Api.IndexedModule( + visualisationFile, + context.Visualisation.code, + Seq( + Api.SuggestionsDatabaseUpdate.Add( + Suggestion.Method( + None, + "Test.Visualisation", + "encode", + List( + Suggestion.Argument("this", "Any", false, false, None), + Suggestion.Argument("x", "Any", false, false, None) + ), + "Visualisation", + "Any", + None + ) + ), + Api.SuggestionsDatabaseUpdate.Add( + Suggestion.Method( + None, + "Test.Visualisation", + "incAndEncode", + List( + Suggestion.Argument("this", "Any", false, false, None), + Suggestion.Argument("x", "Any", false, false, None) + ), + "Visualisation", + "Any", + None + ) + ) + ) + ) + ) + ) + ), context.executionSuccessful(contextId) ) @@ -2922,6 +3038,7 @@ class RuntimeServerTest ) ) ) + context.receiveNone shouldEqual None context.send( Api.Request(Api.OpenFileNotification(mainFile, contents, false)) ) @@ -2943,11 +3060,51 @@ class RuntimeServerTest Api.Request(requestId, Api.PushContextRequest(contextId, item1)) ) - context.receive(6) should contain theSameElementsAs Seq( + context.receive(7) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), context.Main.Update.mainX(contextId), context.Main.Update.mainY(contextId), context.Main.Update.mainZ(contextId), + Api.Response( + Api.SuggestionsDatabaseIndexUpdateNotification( + Seq( + Api.IndexedModule( + visualisationFile, + context.Visualisation.code, + Seq( + Api.SuggestionsDatabaseUpdate.Add( + Suggestion.Method( + None, + "Test.Visualisation", + "encode", + List( + Suggestion.Argument("this", "Any", false, false, None), + Suggestion.Argument("x", "Any", false, false, None) + ), + "Visualisation", + "Any", + None + ) + ), + Api.SuggestionsDatabaseUpdate.Add( + Suggestion.Method( + None, + "Test.Visualisation", + "incAndEncode", + List( + Suggestion.Argument("this", "Any", false, false, None), + Suggestion.Argument("x", "Any", false, false, None) + ), + "Visualisation", + "Any", + None + ) + ) + ) + ) + ) + ) + ), Api.Response( Api.SuggestionsDatabaseReIndexNotification( moduleName, @@ -3105,16 +3262,22 @@ class RuntimeServerTest } it should "be able to modify visualisations" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val visualisationFile = context.writeInSrcDir("Visualisation", context.Visualisation.code) + // open files + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None context.send( Api.Request( Api.OpenFileNotification( visualisationFile, context.Visualisation.code, - false + true ) ) ) @@ -3223,16 +3386,22 @@ class RuntimeServerTest } it should "not emit visualisation updates when visualisation is detached" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val visualisationFile = context.writeInSrcDir("Visualisation", context.Visualisation.code) + // open files + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None context.send( Api.Request( Api.OpenFileNotification( visualisationFile, context.Visualisation.code, - false + true ) ) ) @@ -3345,7 +3514,8 @@ class RuntimeServerTest } it should "rename a project" in { - context.writeMain(context.Main.code) + val contents = context.Main.code + val mainFile = context.writeMain(contents) val contextId = UUID.randomUUID() val requestId = UUID.randomUUID() @@ -3355,6 +3525,12 @@ class RuntimeServerTest Api.Response(requestId, Api.CreateContextResponse(contextId)) ) + // open file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + // push main context.send( Api.Request( diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/StdlibRuntimeServerTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/StdlibRuntimeServerTest.scala new file mode 100644 index 0000000000..bc9f417932 --- /dev/null +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/StdlibRuntimeServerTest.scala @@ -0,0 +1,192 @@ +package org.enso.interpreter.test.instrument + +import java.io.{ByteArrayOutputStream, File} +import java.nio.ByteBuffer +import java.nio.file.{Files, Paths} +import java.util.UUID +import java.util.concurrent.{LinkedBlockingQueue, TimeUnit} + +import org.enso.interpreter.test.Metadata +import org.enso.pkg.{Package, PackageManager} +import org.enso.polyglot._ +import org.enso.polyglot.runtime.Runtime.Api +import org.enso.testkit.OsSpec +import org.graalvm.polyglot.Context +import org.graalvm.polyglot.io.MessageEndpoint +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +@scala.annotation.nowarn("msg=multiarg infix syntax") +class StdlibRuntimeServerTest + extends AnyFlatSpec + with Matchers + with BeforeAndAfterEach + with OsSpec { + + final val ContextPathSeparator: String = + if (isWindows) ";" else ":" + + var context: TestContext = _ + + class TestContext(packageName: String) { + + var endPoint: MessageEndpoint = _ + val messageQueue: LinkedBlockingQueue[Api.Response] = + new LinkedBlockingQueue() + + val tmpDir: File = Files.createTempDirectory("enso-test-packages").toFile + val stdlib: File = + Paths.get("../../distribution/std-lib/Base").toFile.getAbsoluteFile + + val pkg: Package[File] = + PackageManager.Default.create(tmpDir, packageName, "0.0.1") + val out: ByteArrayOutputStream = new ByteArrayOutputStream() + val executionContext = new PolyglotContext( + Context + .newBuilder(LanguageInfo.ID) + .allowExperimentalOptions(true) + .allowAllAccess(true) + .option( + RuntimeOptions.PACKAGES_PATH, + toPackagesPath(pkg.root.getAbsolutePath, stdlib.toString) + ) + .option(RuntimeOptions.LOG_LEVEL, "WARNING") + .option(RuntimeOptions.INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION, "true") + .option(RuntimeServerInfo.ENABLE_OPTION, "true") + .out(out) + .serverTransport { (uri, peer) => + if (uri.toString == RuntimeServerInfo.URI) { + endPoint = peer + new MessageEndpoint { + override def sendText(text: String): Unit = {} + + override def sendBinary(data: ByteBuffer): Unit = + Api.deserializeResponse(data).foreach(messageQueue.add) + + override def sendPing(data: ByteBuffer): Unit = {} + + override def sendPong(data: ByteBuffer): Unit = {} + + override def sendClose(): Unit = {} + } + } else null + } + .build() + ) + executionContext.context.initialize(LanguageInfo.ID) + + def toPackagesPath(paths: String*): String = + paths.mkString(ContextPathSeparator) + + def writeMain(contents: String): File = + Files.write(pkg.mainFile.toPath, contents.getBytes).toFile + + def writeFile(file: File, contents: String): File = + Files.write(file.toPath, contents.getBytes).toFile + + def writeInSrcDir(moduleName: String, contents: String): File = { + val file = new File(pkg.sourceDir, s"$moduleName.enso") + Files.write(file.toPath, contents.getBytes).toFile + } + + def send(msg: Api.Request): Unit = endPoint.sendBinary(Api.serialize(msg)) + + def receiveNone: Option[Api.Response] = { + Option(messageQueue.poll()) + } + + def receive: Option[Api.Response] = { + Option(messageQueue.poll(3, TimeUnit.SECONDS)) + } + + def receive(timeout: Long): Option[Api.Response] = { + Option(messageQueue.poll(timeout, TimeUnit.SECONDS)) + } + + def receiveN(n: Int): List[Api.Response] = { + Iterator.continually(receive).take(n).flatten.toList + } + + def receiveN(n: Int, timeout: Long): List[Api.Response] = { + Iterator.continually(receive(timeout)).take(n).flatten.toList + } + + def consumeOut: List[String] = { + val result = out.toString + out.reset() + result.linesIterator.toList + } + + def executionSuccessful(contextId: UUID): Api.Response = + Api.Response(Api.ExecutionSuccessful(contextId)) + + } + + override protected def beforeEach(): Unit = { + context = new TestContext("Test") + val Some(Api.Response(_, Api.InitializedNotification())) = context.receive + } + + it should "import Base modules" in { + val contextId = UUID.randomUUID() + val requestId = UUID.randomUUID() + val moduleName = "Test.Main" + + val metadata = new Metadata + + val code = + """from Base import all + | + |main = IO.println "Hello World!" + |""".stripMargin.linesIterator.mkString("\n") + val contents = metadata.appendToCode(code) + val mainFile = context.writeMain(contents) + + // create context + context.send(Api.Request(requestId, Api.CreateContextRequest(contextId))) + context.receive shouldEqual Some( + Api.Response(requestId, Api.CreateContextResponse(contextId)) + ) + + // open file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents, true)) + ) + context.receiveNone shouldEqual None + + // push main + context.send( + Api.Request( + requestId, + Api.PushContextRequest( + contextId, + Api.StackItem.ExplicitCall( + Api.MethodPointer(moduleName, "Main", "main"), + None, + Vector() + ) + ) + ) + ) + val response = context.receiveN(3, timeout = 30) + response.length shouldEqual 3 + response should contain allOf ( + Api.Response(requestId, Api.PushContextResponse(contextId)), + context.executionSuccessful(contextId) + ) + response.collect { + case Api.Response( + None, + Api.SuggestionsDatabaseIndexUpdateNotification(xs) + ) => + xs.nonEmpty shouldBe true + xs.flatMap( + _.updates.headOption.map(_.suggestion.module) + ) should not contain "Test.Main" + } should have length 1 + + context.consumeOut shouldEqual List("Hello World!") + } + +} diff --git a/lib/scala/searcher/src/main/resources/reference.conf b/lib/scala/searcher/src/main/resources/reference.conf index 9648872fd7..21a3d1ec3c 100644 --- a/lib/scala/searcher/src/main/resources/reference.conf +++ b/lib/scala/searcher/src/main/resources/reference.conf @@ -2,7 +2,8 @@ searcher { db { url = "jdbc:sqlite::memory:" driver = "org.sqlite.JDBC" - connectionPool = "HikariCP" + connectionPool = disabled properties.journal_mode = "wal" + numThreads = 1 } } diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/FileVersionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/FileVersionsRepo.scala index 0fb0902c8d..a7b15572b1 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/FileVersionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/FileVersionsRepo.scala @@ -20,6 +20,14 @@ trait FileVersionsRepo[F[_]] { */ def setVersion(file: File, digest: Array[Byte]): F[Option[Array[Byte]]] + /** Update the version if it differs from the recorded version. + * + * @param file the file path + * @param digest the version digest + * @return `true` if the version has been updated + */ + def updateVersion(file: File, digest: Array[Byte]): F[Boolean] + /** Remove the version record. * * @param file the file path diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala index e6e6011ef7..154f0aea05 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala @@ -73,6 +73,13 @@ trait SuggestionsRepo[F[_]] { */ def removeByModule(name: String): F[(Long, Seq[Long])] + /** Remove all suggestions by module names. + * + * @param modules the list of modules to remove + * @return the current database version and a list of removed suggestion ids + */ + def removeAllByModule(modules: Seq[String]): F[(Long, Seq[Long])] + /** Remove a list of suggestions. * * @param suggestions the suggestions to remove diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala index 4b6b1a8c93..a82a2e1342 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala @@ -77,6 +77,12 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) override def removeByModule(name: String): Future[(Long, Seq[Long])] = db.run(removeByModuleQuery(name)) + /** @inheritdoc */ + override def removeAllByModule( + modules: Seq[String] + ): Future[(Long, Seq[Long])] = + db.run(removeAllByModuleQuery(modules)) + /** @inheritdoc */ override def removeAll( suggestions: Seq[Suggestion] @@ -137,7 +143,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) suggestions <- joined.result.map(joinedToSuggestionEntries) version <- currentVersionQuery } yield (version, suggestions) - query.transactionally + query } /** The query to get the suggestions by the method call info. @@ -204,7 +210,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) results <- searchAction version <- currentVersionQuery } yield (version, results) - query.transactionally + query } /** The query to select the suggestion by id. @@ -234,7 +240,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) } _ <- incrementVersionQuery } yield id - query.transactionally.asTry.map { + query.asTry.map { case Failure(_) => None case Success(id) => Some(id) } @@ -252,7 +258,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) ids <- DBIO.sequence(suggestions.map(insertQuery)) version <- currentVersionQuery } yield (version, ids) - query.transactionally + query } /** The query to remove the suggestion. @@ -274,7 +280,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) n <- selectQuery.delete _ <- if (n > 0) incrementVersionQuery else DBIO.successful(()) } yield rows.flatMap(_.id).headOption - deleteQuery.transactionally + deleteQuery } /** The query to remove the suggestions by module name @@ -289,7 +295,24 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) n <- selectQuery.delete version <- if (n > 0) incrementVersionQuery else currentVersionQuery } yield version -> rows.flatMap(_.id) - deleteQuery.transactionally + deleteQuery + } + + /** The query to remove all suggestions by module names. + * + * @param modules the list of modules to remove + * @return the current database version and a list of removed suggestion ids + */ + private def removeAllByModuleQuery( + modules: Seq[String] + ): DBIO[(Long, Seq[Long])] = { + val selectQuery = Suggestions.filter(_.module inSet modules) + val deleteQuery = for { + rows <- selectQuery.result + n <- selectQuery.delete + version <- if (n > 0) incrementVersionQuery else currentVersionQuery + } yield version -> rows.flatMap(_.id) + deleteQuery } /** The query to remove a list of suggestions. @@ -304,7 +327,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) ids <- DBIO.sequence(suggestions.map(removeQuery)) version <- currentVersionQuery } yield (version, ids) - query.transactionally + query } /** The query to update a suggestion. @@ -342,7 +365,7 @@ final class SqlSuggestionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) if (ids.exists(_.nonEmpty)) incrementVersionQuery else currentVersionQuery } yield (version, ids) - query.transactionally + query } /** The query to update the project name. diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlVersionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlVersionsRepo.scala index 89b464bdc0..4900bb8aaf 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlVersionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlVersionsRepo.scala @@ -1,6 +1,7 @@ package org.enso.searcher.sql import java.io.File +import java.util import org.enso.searcher.FileVersionsRepo import slick.jdbc.SQLiteProfile.api._ @@ -25,6 +26,10 @@ final class SqlVersionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) ): Future[Option[Array[Byte]]] = db.run(setVersionQuery(file, digest)) + /** @inheritdoc */ + override def updateVersion(file: File, digest: Array[Byte]): Future[Boolean] = + db.run(updateVersionQuery(file, digest)) + /** @inheritdoc */ override def remove(file: File): Future[Unit] = db.run(removeQuery(file)) @@ -79,6 +84,24 @@ final class SqlVersionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) query.transactionally } + /** The query to update the version if it differs from the recorded version. + * + * @param file the file path + * @param version the version digest + * @return `true` if the version has been updated + */ + private def updateVersionQuery( + file: File, + version: Array[Byte] + ): DBIO[Boolean] = + for { + repoVersion <- getVersionQuery(file) + versionsEquals = repoVersion.fold(false)(compareVersions(_, version)) + _ <- + if (!versionsEquals) setVersionQuery(file, version) + else DBIO.successful(None) + } yield !versionsEquals + /** The query to remove the version record. * * @param file the file path @@ -91,6 +114,9 @@ final class SqlVersionsRepo(db: SqlDatabase)(implicit ec: ExecutionContext) query.delete >> DBIO.successful(()) } + private def compareVersions(v1: Array[Byte], v2: Array[Byte]): Boolean = + util.Arrays.equals(v1, v2) + } object SqlVersionsRepo { diff --git a/lib/scala/searcher/src/test/resources/application.conf b/lib/scala/searcher/src/test/resources/application.conf deleted file mode 100644 index 341fa925fb..0000000000 --- a/lib/scala/searcher/src/test/resources/application.conf +++ /dev/null @@ -1 +0,0 @@ -searcher.db.numThreads = 1 diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/FileVersionsRepoTest.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/FileVersionsRepoTest.scala index 4442a0dfe1..d0083f0140 100644 --- a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/FileVersionsRepoTest.scala +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/FileVersionsRepoTest.scala @@ -63,7 +63,7 @@ class FileVersionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { util.Arrays.equals(v2.get, digest) shouldBe true } - "update digest" taggedAs Retry in withRepo { repo => + "set digest" taggedAs Retry in withRepo { repo => val file = new File("/foo/bar") val digest1 = nextDigest() val digest2 = nextDigest() @@ -82,6 +82,32 @@ class FileVersionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { util.Arrays.equals(v3.get, digest2) shouldBe true } + "update digest" taggedAs Retry in withRepo { repo => + val file = new File("/foo/bazz") + val digest1 = nextDigest() + val digest2 = nextDigest() + val digest3 = nextDigest() + val action = + for { + b1 <- repo.updateVersion(file, digest1) + v2 <- repo.setVersion(file, digest2) + b2 <- repo.updateVersion(file, digest2) + b3 <- repo.updateVersion(file, digest3) + b4 <- repo.updateVersion(file, digest3) + v3 <- repo.getVersion(file) + } yield (v2, v3, b1, b2, b3, b4) + + val (v2, v3, b1, b2, b3, b4) = Await.result(action, Timeout) + v2 shouldBe a[Some[_]] + v3 shouldBe a[Some[_]] + util.Arrays.equals(v2.get, digest1) shouldBe true + util.Arrays.equals(v3.get, digest3) shouldBe true + b1 shouldBe true + b2 shouldBe false + b3 shouldBe true + b4 shouldBe false + } + "delete digest" taggedAs Retry in withRepo { repo => val file = new File("/foo/bar") val digest = nextDigest() diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala index 129b11834a..5c6a1fedb5 100644 --- a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala @@ -47,10 +47,14 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { "get all suggestions" taggedAs Retry in withRepo { repo => val action = for { - _ <- repo.insert(suggestion.atom) - _ <- repo.insert(suggestion.method) - _ <- repo.insert(suggestion.function) - _ <- repo.insert(suggestion.local) + _ <- repo.insertAll( + Seq( + suggestion.atom, + suggestion.method, + suggestion.function, + suggestion.local + ) + ) all <- repo.getAll } yield all._2 @@ -168,12 +172,33 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { "remove suggestions by module name" taggedAs Retry in withRepo { repo => val action = for { - id1 <- repo.insert(suggestion.atom) - id2 <- repo.insert(suggestion.method) - id3 <- repo.insert(suggestion.function) - id4 <- repo.insert(suggestion.local) - (_, ids) <- repo.removeByModule(suggestion.atom.module) - } yield (Seq(id1, id2, id3, id4).flatten, ids) + (_, idsIns) <- repo.insertAll( + Seq( + suggestion.atom, + suggestion.method, + suggestion.function, + suggestion.local + ) + ) + (_, idsRem) <- repo.removeByModule(suggestion.atom.module) + } yield (idsIns.flatten, idsRem) + + val (inserted, removed) = Await.result(action, Timeout) + inserted should contain theSameElementsAs removed + } + + "remove all suggestions by module name" taggedAs Retry in withRepo { repo => + val action = for { + (_, idsIns) <- repo.insertAll( + Seq( + suggestion.atom, + suggestion.method, + suggestion.function, + suggestion.local + ) + ) + (_, idsRem) <- repo.removeAllByModule(Seq(suggestion.atom.module)) + } yield (idsIns.flatten, idsRem) val (inserted, removed) = Await.result(action, Timeout) inserted should contain theSameElementsAs removed @@ -181,10 +206,14 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { "remove all suggestions" taggedAs Retry in withRepo { repo => val action = for { - id1 <- repo.insert(suggestion.atom) - _ <- repo.insert(suggestion.method) - _ <- repo.insert(suggestion.function) - id4 <- repo.insert(suggestion.local) + (_, Seq(id1, _, _, id4)) <- repo.insertAll( + Seq( + suggestion.atom, + suggestion.method, + suggestion.function, + suggestion.local + ) + ) (_, ids) <- repo.removeAll(Seq(suggestion.atom, suggestion.local)) } yield (Seq(id1, id4), ids) @@ -287,6 +316,37 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { v3 shouldEqual v4 } + "change version after remove all by module name" taggedAs Retry in withRepo { + repo => + val action = for { + v1 <- repo.currentVersion + _ <- repo.insert(suggestion.local) + v2 <- repo.currentVersion + (v3, _) <- repo.removeAllByModule(Seq(suggestion.local.module)) + } yield (v1, v2, v3) + + val (v1, v2, v3) = Await.result(action, Timeout) + v1 should not equal v2 + v2 should not equal v3 + } + + "not change version after failed remove all by module name" taggedAs Retry in withRepo { + repo => + val action = for { + v1 <- repo.currentVersion + _ <- repo.insert(suggestion.local) + v2 <- repo.currentVersion + _ <- repo.removeAllByModule(Seq(suggestion.local.module)) + v3 <- repo.currentVersion + (v4, _) <- repo.removeAllByModule(Seq(suggestion.local.module)) + } yield (v1, v2, v3, v4) + + val (v1, v2, v3, v4) = Await.result(action, Timeout) + v1 should not equal v2 + v2 should not equal v3 + v3 shouldEqual v4 + } + "change version after remove all suggestions" taggedAs Retry in withRepo { repo => val action = for { diff --git a/lib/scala/testkit/src/main/scala/org/enso/testkit/OsSpec.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/OsSpec.scala index d4f6ea25b0..08516288f0 100644 --- a/lib/scala/testkit/src/main/scala/org/enso/testkit/OsSpec.scala +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/OsSpec.scala @@ -14,12 +14,20 @@ trait OsSpec extends TestSuite { override def withFixture(test: NoArgTest): Outcome = { if (test.tags.contains(OsUnix.name)) { - if (SystemUtils.IS_OS_UNIX) super.withFixture(test) else Pending + if (isUnix) super.withFixture(test) else Pending } else if (test.tags.contains(OsWindows.name)) { - if (SystemUtils.IS_OS_WINDOWS) super.withFixture(test) else Pending + if (isWindows) super.withFixture(test) else Pending } else { super.withFixture(test) } } + /** @return `true` if this is a Unix-like system. */ + protected def isUnix: Boolean = + SystemUtils.IS_OS_UNIX + + /** @return `true` if this is a Windows system. */ + protected def isWindows: Boolean = + SystemUtils.IS_OS_WINDOWS + }