enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-12-23 08:21:49 +03:00
Code Issues Projects Releases Wiki Activity

Pre compute suggestion db during build time (#5698)

Browse Source 
Close #5068

Cache suggestions during the `buildEngineDistribution` command, and read them from the disk when the library is loaded. Initial graph coloring takes ~20 seconds vs ~25 seconds on the develop branch.

[peek-develop-branch.webm](https://user-images.githubusercontent.com/357683/223504462-e7d48262-4f5e-4724-b2b0-2cb97fc05140.webm)
[peek-suggestions-branch.webm](https://user-images.githubusercontent.com/357683/223504464-0fe86c04-8c4b-443c-ba96-6c5e2fb1e396.webm)
This commit is contained in:
Dmitry Bushev 2023-03-08 15:37:48 +03:00 committed by GitHub
parent 60d8b8fcea
commit 9397a6ec2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 910 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.sbt
View File

@ -643,6 +643,7 @@ lazy val `docs-generator` = (project in file("lib/scala/docs-generator"))
.dependsOn(syntax.jvm)
.dependsOn(cli)
.dependsOn(`version-output`)
.dependsOn(`polyglot-api`)
.configs(Benchmark)
.settings(
frgaalJavaCompilerSetting,

2
engine/language-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala
View File

@ -89,7 +89,7 @@ class RepoInitialization(
} yield ()
initAction.onComplete {
case Success(()) =>
eventStream.publish(InitializedEvent.FileVersionsRepoInitialized)
eventStream.publish(InitializedEvent.VersionsRepoInitialized)
case Failure(ex) =>
logger.error(
"Failed to initialize SQL versions repo [{}]. {}",

10
engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala
View File

@ -5,9 +5,9 @@ sealed trait InitializedEvent extends Event
object InitializedEvent {
case object SuggestionsRepoInitialized extends InitializedEvent
case object FileVersionsRepoInitialized extends InitializedEvent
case object TruffleContextInitialized extends InitializedEvent
case object InitializationFinished extends InitializedEvent
case object InitializationFailed extends InitializedEvent
case object SuggestionsRepoInitialized extends InitializedEvent
case object VersionsRepoInitialized extends InitializedEvent
case object TruffleContextInitialized extends InitializedEvent
case object InitializationFinished extends InitializedEvent
case object InitializationFailed extends InitializedEvent
}

136
engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala
View File

@ -5,6 +5,7 @@ import java.util.concurrent.Executors
import akka.actor.{Actor, ActorRef, Props, Stash, Status}
import akka.pattern.pipe
import com.typesafe.scalalogging.LazyLogging
import org.enso.docs.sections.DocSectionsBuilder
import org.enso.languageserver.capability.CapabilityProtocol.{
AcquireCapability,
CapabilityAcquired,
@ -111,6 +112,12 @@ final class SuggestionsHandler(
.subscribe(self, classOf[Api.ExpressionUpdates])
context.system.eventStream
.subscribe(self, classOf[Api.SuggestionsDatabaseModuleUpdateNotification])
context.system.eventStream.subscribe(
self,
classOf[Api.SuggestionsDatabaseSuggestionsLoadedNotification]
)
context.system.eventStream
.subscribe(self, classOf[Api.LibraryLoaded])
context.system.eventStream.subscribe(self, classOf[ProjectNameChangedEvent])
context.system.eventStream.subscribe(self, classOf[FileDeletedEvent])
context.system.eventStream
@ -122,7 +129,7 @@ final class SuggestionsHandler(
override def receive: Receive =
initializing(SuggestionsHandler.Initialization())
def initializing(init: SuggestionsHandler.Initialization): Receive = {
private def initializing(init: SuggestionsHandler.Initialization): Receive = {
case ProjectNameChangedEvent(oldName, newName) =>
logger.info(
"Initializing: project name changed from [{}] to [{}].",
@ -178,35 +185,7 @@ final class SuggestionsHandler(
case _ => stash()
}
def verifying(
projectName: String,
graph: TypeGraph
): Receive = {
case Api.Response(_, Api.VerifyModulesIndexResponse(toRemove)) =>
logger.info("Verifying: got verification response.")
val removeAction = for {
_ <- versionsRepo.remove(toRemove)
_ <- suggestionsRepo.removeModules(toRemove)
} yield SuggestionsHandler.Verified
removeAction.pipeTo(self)
case SuggestionsHandler.Verified =>
logger.info("Verified.")
context.become(initialized(projectName, graph, Set(), State()))
unstashAll()
case Status.Failure(ex) =>
logger.error(
"Database verification failure [{}]. {}",
ex.getClass,
ex.getMessage
)
case _ =>
stash()
}
def initialized(
private def initialized(
projectName: String,
graph: TypeGraph,
clients: Set[ClientId],
@ -230,12 +209,37 @@ final class SuggestionsHandler(
initialized(projectName, graph, clients - client.clientId, state)
)
case msg: Api.SuggestionsDatabaseSuggestionsLoadedNotification =>
logger.debug(
"Starting loading suggestions for library [{}].",
msg.libraryName
)
applyLoadedSuggestions(msg.suggestions)
.onComplete {
case Success(notification) =>
logger.debug(
"Complete loading suggestions for library [{}].",
msg.libraryName
)
if (notification.updates.nonEmpty) {
clients.foreach { clientId =>
sessionRouter ! DeliverToJsonController(clientId, notification)
}
}
case Failure(ex) =>
logger.error(
"Error applying suggestion updates for loaded library [{}] ({})",
msg.libraryName,
ex.getMessage
)
}
case msg: Api.SuggestionsDatabaseModuleUpdateNotification
if state.isSuggestionUpdatesRunning =>
state.suggestionUpdatesQueue.enqueue(msg)
case SuggestionUpdatesBatch(updates) if state.isSuggestionUpdatesRunning =>
state.suggestionUpdatesQueue.enqueueAll(updates)
state.suggestionUpdatesQueue.prependAll(updates)
case SuggestionUpdatesBatch(updates) =>
val modules = updates.map(_.module)
@ -493,6 +497,13 @@ final class SuggestionsHandler(
updates.foreach(sessionRouter ! _)
context.become(initialized(name, graph, clients, state))
case libraryLoaded: Api.LibraryLoaded =>
logger.debug(
"Loaded Library [{}.{}]",
libraryLoaded.namespace,
libraryLoaded.name
)
case SuggestionUpdatesCompleted =>
if (state.suggestionUpdatesQueue.nonEmpty) {
self ! SuggestionUpdatesBatch(state.suggestionUpdatesQueue.removeAll())
@ -515,17 +526,9 @@ final class SuggestionsHandler(
private def tryInitialize(state: SuggestionsHandler.Initialization): Unit = {
logger.debug("Trying to initialize with state [{}]", state)
state.initialized.fold(context.become(initializing(state))) {
case (name, graph) =>
case (projectName, graph) =>
logger.debug("Initialized with state [{}].", state)
val requestId = UUID.randomUUID()
suggestionsRepo.getAllModules
.map { modules =>
runtimeConnector ! Api.Request(
requestId,
Api.VerifyModulesIndexRequest(modules)
)
}
context.become(verifying(name, graph))
context.become(initialized(projectName, graph, Set(), State()))
unstashAll()
}
}
@ -543,6 +546,48 @@ final class SuggestionsHandler(
}
}
/** Handle the suggestions of the loaded library.
*
* Adds the new suggestions to the suggestions database and sends the
* appropriate notification to the client.
*
* @param suggestions the loaded suggestions
* @return the API suggestions database update notification
*/
private def applyLoadedSuggestions(
suggestions: Vector[Suggestion]
): Future[SuggestionsDatabaseUpdateNotification] = {
for {
treeResults <- suggestionsRepo.applyTree(
suggestions.map(Api.SuggestionUpdate(_, Api.SuggestionAction.Add()))
)
version <- suggestionsRepo.currentVersion
} yield {
val treeUpdates = treeResults.flatMap {
case QueryResult(ids, Api.SuggestionUpdate(suggestion, action)) =>
action match {
case Api.SuggestionAction.Add() =>
if (ids.isEmpty) {
val verb = action.getClass.getSimpleName
logger.error("Cannot {} [{}].", verb, suggestion)
}
ids.map(
SuggestionsDatabaseUpdate.Add(
_,
generateDocumentation(suggestion)
)
)
case action =>
logger.error(
s"Invalid action during suggestions loading [$action]."
)
Seq()
}
}
SuggestionsDatabaseUpdateNotification(version, treeUpdates)
}
}
/** Handle the suggestions database update.
*
* Function applies notification updates on the suggestions database and
@ -783,9 +828,6 @@ object SuggestionsHandler {
new ProjectNameUpdated(projectName, Seq())
}
/** The notification that the suggestions database has been verified. */
case object Verified
/** The notification that the suggestion updates are processed. */
case object SuggestionUpdatesCompleted
@ -843,7 +885,7 @@ object SuggestionsHandler {
* @param config the server configuration
* @param contentRootManager the content root manager
* @param suggestionsRepo the suggestions repo
* @param fileVersionsRepo the file versions repo
* @param versionsRepo the versions repo
* @param sessionRouter the session router
* @param runtimeConnector the runtime connector
* @param docSectionsBuilder the builder of documentation sections
@ -852,7 +894,7 @@ object SuggestionsHandler {
config: Config,
contentRootManager: ContentRootManager,
suggestionsRepo: SuggestionsRepo[Future],
fileVersionsRepo: VersionsRepo[Future],
versionsRepo: VersionsRepo[Future],
sessionRouter: ActorRef,
runtimeConnector: ActorRef,
docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder()
@ -862,7 +904,7 @@ object SuggestionsHandler {
config,
contentRootManager,
suggestionsRepo,
fileVersionsRepo,
versionsRepo,
sessionRouter,
runtimeConnector,
docSectionsBuilder

4
engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala
View File

@ -102,13 +102,13 @@ class BufferRegistry(
override def preStart(): Unit = {
logger.info("Starting initialization.")
context.system.eventStream
.subscribe(self, InitializedEvent.FileVersionsRepoInitialized.getClass)
.subscribe(self, InitializedEvent.VersionsRepoInitialized.getClass)
}
override def receive: Receive = initializing
private def initializing: Receive = {
case InitializedEvent.FileVersionsRepoInitialized =>
case InitializedEvent.VersionsRepoInitialized =>
logger.info("Initiaized.")
context.become(running(Map.empty))
unstashAll()

10
engine/language-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala
View File

@ -66,7 +66,7 @@ class RepoInitializationSpec
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}
@ -96,7 +96,7 @@ class RepoInitializationSpec
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}
@ -123,7 +123,7 @@ class RepoInitializationSpec
version1 shouldEqual SchemaVersion.CurrentVersion
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
// remove schema and re-initialize
@ -138,7 +138,7 @@ class RepoInitializationSpec
version2 shouldEqual SchemaVersion.CurrentVersion
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}
@ -199,7 +199,7 @@ class RepoInitializationSpec
version2 shouldEqual SchemaVersion.CurrentVersion
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}
}

1
engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala
View File

@ -1,6 +1,7 @@
package org.enso.languageserver.search
import org.enso.docs.generator.DocsGenerator
import org.enso.docs.sections.DocSectionsBuilder
import org.enso.polyglot.Suggestion
import java.util.UUID

104
engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala
View File

@ -4,6 +4,7 @@ import akka.actor.{ActorRef, ActorSystem}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.commons.io.FileUtils
import org.enso.docs.generator.DocsGenerator
import org.enso.docs.sections.DocSectionsBuilder
import org.enso.languageserver.boot.ProfilingConfig
import org.enso.languageserver.capability.CapabilityProtocol.{
AcquireCapability,
@ -57,61 +58,7 @@ class SuggestionsHandlerSpec
"SuggestionsHandler" should {
"prune stale modules" in withDbs { (config, suggestions, versions) =>
// setup database
val version1 = Array[Byte](1, 2, 3)
val version2 = Array[Byte](2, 3, 4)
val setupAction = for {
_ <- suggestions.insert(TestSuggestion.atom)
_ <- suggestions.insert(TestSuggestion.method)
_ <- versions.setVersion(TestSuggestion.atom.module, version1)
_ <- versions.setVersion(TestSuggestion.method.module, version2)
} yield ()
Await.ready(setupAction, Timeout)
withHandler(config, suggestions, versions) { (_, connector, handler) =>
// initialize
handler ! SuggestionsHandler.ProjectNameUpdated("Test")
handler ! InitializedEvent.TruffleContextInitialized
handler ! InitializedEvent.SuggestionsRepoInitialized
handler ! InitializedEvent.FileVersionsRepoInitialized
connector.receiveN(1)
handler ! Api.Response(
UUID.randomUUID(),
Api.GetTypeGraphResponse(buildTestTypeGraph)
)
connector.receiveN(1)
// prune atom module
handler ! Api.Response(
UUID.randomUUID(),
Api.VerifyModulesIndexResponse(Seq(TestSuggestion.atom.module))
)
// wait for initialization
handler ! AcquireCapability(
newJsonSession(UUID.randomUUID()),
CapabilityRegistration(ReceivesSuggestionsDatabaseUpdates())
)
expectMsg(CapabilityAcquired)
// check
val (_, entries) = Await.result(suggestions.getAll, Timeout)
entries.map(_.suggestion) should contain theSameElementsAs Seq(
TestSuggestion.method
)
val module1Version = Await.result(
versions.getVersion(TestSuggestion.atom.module),
Timeout
)
module1Version.isEmpty shouldBe true
val module2Version = Await.result(
versions.getVersion(TestSuggestion.method.module),
Timeout
)
module2Version.isEmpty shouldBe false
module2Version.get should contain theSameElementsInOrderAs version2
}
}
"subscribe to notification updates" taggedAs Retry in withDb {
"subscribe to notification updates" /*taggedAs Retry*/ in withDb {
(_, _, _, _, handler) =>
val clientId = UUID.randomUUID()
@ -1032,7 +979,7 @@ class SuggestionsHandlerSpec
sessionRouter: TestProbe,
runtimeConnector: TestProbe,
suggestionsRepo: SuggestionsRepo[Future],
fileVersionsRepo: VersionsRepo[Future]
versionsRepo: VersionsRepo[Future]
): ActorRef = {
val contentRootManagerActor =
system.actorOf(ContentRootManagerActor.props(config))
@ -1043,7 +990,7 @@ class SuggestionsHandlerSpec
config,
contentRootManagerWrapper,
suggestionsRepo,
fileVersionsRepo,
versionsRepo,
sessionRouter.ref,
runtimeConnector.ref
)
@ -1055,7 +1002,7 @@ class SuggestionsHandlerSpec
sessionRouter: TestProbe,
runtimeConnector: TestProbe,
suggestionsRepo: SuggestionsRepo[Future],
fileVersionsRepo: VersionsRepo[Future]
versionsRepo: VersionsRepo[Future]
): ActorRef = {
val handler =
newSuggestionsHandler(
@ -1063,11 +1010,12 @@ class SuggestionsHandlerSpec
sessionRouter,
runtimeConnector,
suggestionsRepo,
fileVersionsRepo
versionsRepo
)
handler ! SuggestionsHandler.ProjectNameUpdated("Test")
handler ! InitializedEvent.TruffleContextInitialized
// GetTypeGraphRequest
runtimeConnector.receiveN(1)
handler ! Api.Response(
UUID.randomUUID(),
@ -1075,7 +1023,7 @@ class SuggestionsHandlerSpec
)
val suggestionsInit = suggestionsRepo.init
val versionsInit = fileVersionsRepo.init
val versionsInit = versionsRepo.init
suggestionsInit.onComplete {
case Success(()) =>
system.eventStream.publish(InitializedEvent.SuggestionsRepoInitialized)
@ -1084,16 +1032,13 @@ class SuggestionsHandlerSpec
}
versionsInit.onComplete {
case Success(()) =>
system.eventStream.publish(InitializedEvent.FileVersionsRepoInitialized)
system.eventStream.publish(InitializedEvent.VersionsRepoInitialized)
case Failure(ex) =>
system.log.error(ex, "Failed to initialize FileVersions repo")
}
runtimeConnector.receiveN(1)
handler ! Api.Response(
UUID.randomUUID(),
Api.VerifyModulesIndexResponse(Seq())
)
Await.ready(suggestionsInit, Timeout)
Await.ready(versionsInit, Timeout)
handler
}
@ -1150,7 +1095,7 @@ class SuggestionsHandlerSpec
}
versionsInit.onComplete {
case Success(()) =>
system.eventStream.publish(InitializedEvent.FileVersionsRepoInitialized)
system.eventStream.publish(InitializedEvent.VersionsRepoInitialized)
case Failure(ex) =>
system.log.error(ex, "Failed to initialize FileVersions repo")
}
@ -1165,29 +1110,6 @@ class SuggestionsHandlerSpec
}
}
def withHandler(
config: Config,
suggestionsRepo: SuggestionsRepo[Future],
versionsRepo: VersionsRepo[Future]
)(
test: (TestProbe, TestProbe, ActorRef) => Any
): Unit = {
val router = TestProbe("session-router")
val connector = TestProbe("runtime-connector")
val handler = newSuggestionsHandler(
config,
router,
connector,
suggestionsRepo,
versionsRepo
)
try test(router, connector, handler)
finally {
system.stop(handler)
}
}
def withDb(
test: (
Config,
@ -1207,7 +1129,7 @@ class SuggestionsHandlerSpec
)
val router = TestProbe("session-router")
val connector = TestProbe("runtime-connector")
val sqlDatabase = SqlDatabase(config.directories.suggestionsDatabaseFile)
val sqlDatabase = SqlDatabase.inmem("testdb")
val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase)
val versionsRepo = new SqlVersionsRepo(sqlDatabase)
val handler = newInitializedSuggestionsHandler(

5
engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala
View File

@ -248,11 +248,6 @@ class BaseServerTest
)
Await.ready(initializationComponent.init(), timeout)
system.eventStream.publish(ProjectNameChangedEvent("Test", "Test"))
runtimeConnectorProbe.receiveN(1)
suggestionsHandler ! Api.Response(
UUID.randomUUID(),
Api.VerifyModulesIndexResponse(Seq())
)
val environment = fakeInstalledEnvironment()
val languageHome = LanguageHome.detectFromExecutableLocation(environment)

59
engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala
View File

@ -205,6 +205,10 @@ object Runtime {
value = classOf[Api.SuggestionsDatabaseModuleUpdateNotification],
name = "suggestionsDatabaseModuleUpdateNotification"
),
new JsonSubTypes.Type(
value = classOf[Api.SuggestionsDatabaseSuggestionsLoadedNotification],
name = "suggestionsDatabaseSuggestionsLoadedNotification"
),
new JsonSubTypes.Type(
value = classOf[Api.AnalyzeModuleInScopeJobFinished],
name = "analyzeModuleInScopeJobFinished"
@ -217,14 +221,6 @@ object Runtime {
value = classOf[Api.InvalidateModulesIndexResponse],
name = "invalidateModulesIndexResponse"
),
new JsonSubTypes.Type(
value = classOf[Api.VerifyModulesIndexRequest],
name = "verifyModulesIndexRequest"
),
new JsonSubTypes.Type(
value = classOf[Api.VerifyModulesIndexResponse],
name = "verifyModulesIndexResponse"
),
new JsonSubTypes.Type(
value = classOf[Api.GetTypeGraphRequest],
name = "getTypeGraphRequest"
@ -268,6 +264,10 @@ object Runtime {
new JsonSubTypes.Type(
value = classOf[Api.LockReleaseFailed],
name = "lockReleaseFailed"
),
new JsonSubTypes.Type(
value = classOf[Api.DeserializeLibrarySuggestions],
name = "deserializeLibrarySuggestions"
)
)
)
@ -1514,6 +1514,25 @@ object Runtime {
")"
}
/** A notification about the suggestions of the loaded library.
*
* @param libraryName the name of the loaded library
* @param suggestions the loaded suggestions
*/
final case class SuggestionsDatabaseSuggestionsLoadedNotification(
libraryName: LibraryName,
suggestions: Vector[Suggestion]
) extends ApiNotification
with ToLogString {
/** @inheritdoc */
override def toLogString(shouldMask: Boolean): String =
"SuggestionsDatabaseSuggestionsLoadedNotification(" +
s"libraryName=$libraryName," +
s"suggestions=${suggestions.map(_.toLogString(shouldMask))}" +
")"
}
/** A notification about the finished background analyze job. */
final case class AnalyzeModuleInScopeJobFinished() extends ApiNotification
@ -1523,20 +1542,6 @@ object Runtime {
/** Signals that the module indexes has been invalidated. */
final case class InvalidateModulesIndexResponse() extends ApiResponse
/** A request to verify the modules in the suggestions database.
*
* @param modules the list of modules
*/
final case class VerifyModulesIndexRequest(modules: Seq[String])
extends ApiRequest
/** A response to the module verification request.
*
* @param remove the list of modules to remove from suggestions database.
*/
final case class VerifyModulesIndexResponse(remove: Seq[String])
extends ApiResponse
/** A request for the type hierarchy graph. */
final case class GetTypeGraphRequest() extends ApiRequest
@ -1650,6 +1655,16 @@ object Runtime {
*/
final case class LockReleaseFailed(errorMessage: String) extends ApiResponse
/** A request to deserialize the library suggestions.
*
* Does not have a companion response message. The response will be
* delivered asynchronously as a notification.
*
* @param libraryName the name of the loaded library.
*/
final case class DeserializeLibrarySuggestions(libraryName: LibraryName)
extends ApiRequest
private lazy val mapper = {
val factory = new CBORFactory()
val mapper = new ObjectMapper(factory) with ClassTagExtensions

24
engine/polyglot-api/src/test/scala/org/enso/polyglot/data/TreeTest.scala
View File

@ -3,6 +3,8 @@ package org.enso.polyglot.data
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import scala.collection.mutable.Builder
class TreeTest extends AnyWordSpec with Matchers {
val tree: Tree.Root[Long] = Tree.Root(
@ -189,4 +191,26 @@ class TreeTest extends AnyWordSpec with Matchers {
Tree.zip(tree1, tree2) shouldEqual expected
}
"building the tree with mutable builder" in {
val b: Builder[Tree.Node[String], Vector[
Tree.Node[String]
]] = Vector.newBuilder
b += Tree.Node("ahoj", Vector())
b += Tree.Node("cześć", Vector())
b += Tree.Node("hi", Vector())
b += Tree.Node("приве́т", Vector())
b += Tree.Node("ciao", Vector())
val v = b.result()
val t = Tree.Root(v)
val expected = Tree.Root(
Vector(
Tree.Node("ahoj", Vector()),
Tree.Node("hi", Vector()),
Tree.Node("ciao", Vector())
)
)
t.filter(_.forall(ch => 'a' <= ch && ch <= 'z')) shouldEqual expected
}
}

33
engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala
View File

@ -244,7 +244,6 @@ class RuntimeComponentsTest
context.receiveAllUntil(
Seq(
context.executionComplete(contextId),
context.analyzeJobFinished,
context.analyzeJobFinished
),
timeout = 180
@ -338,7 +337,6 @@ class RuntimeComponentsTest
context.receiveAllUntil(
Seq(
context.executionComplete(contextId),
context.analyzeJobFinished,
context.analyzeJobFinished
),
timeout = 180
@ -400,37 +398,6 @@ class RuntimeComponentsTest
GroupReference(LibraryName("Standard", "Base"), GroupName("Output"))
)
// check that component group symbols can be resolved
val suggestionSymbols = responses
.collect {
case Api.Response(
None,
msg: Api.SuggestionsDatabaseModuleUpdateNotification
) =>
msg.updates.toVector.map(_.suggestion)
}
.flatten
.flatMap { suggestion =>
for {
selfType <- Suggestion.SelfType(suggestion)
} yield s"$selfType.${suggestion.name}"
}
.toSet
val componentSymbols = components
.flatMap { case (_, componentGroups) =>
val newComponents = componentGroups.newGroups.flatMap(_.exports)
val extendedComponents =
componentGroups.extendedGroups.flatMap(_.exports)
newComponents ++ extendedComponents
}
.map(_.name)
componentSymbols should not be empty
componentSymbols.foreach { component =>
suggestionSymbols should contain(component)
}
context.consumeOut shouldEqual List()
}

31
engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala
View File

@ -352,12 +352,12 @@ class RuntimeErrorsTest
val requestId = UUID.randomUUID()
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val xId = metadata.addItem(40, 9)
val yId = metadata.addItem(58, 2)
val mainResId = metadata.addItem(65, 12)
val xId = metadata.addItem(46, 9)
val yId = metadata.addItem(64, 2)
val mainResId = metadata.addItem(71, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| x = undefined
@ -958,12 +958,12 @@ class RuntimeErrorsTest
val requestId = UUID.randomUUID()
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val xId = metadata.addItem(40, 7)
val yId = metadata.addItem(56, 5)
val mainResId = metadata.addItem(66, 12)
val xId = metadata.addItem(46, 7)
val yId = metadata.addItem(62, 5)
val mainResId = metadata.addItem(72, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| x = 1 + foo
@ -1386,13 +1386,12 @@ class RuntimeErrorsTest
val requestId = UUID.randomUUID()
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val xId = metadata.addItem(98, 3)
val yId = metadata.addItem(110, 5)
val mainResId = metadata.addItem(120, 12)
val xId = metadata.addItem(71, 3)
val yId = metadata.addItem(83, 5)
val mainResId = metadata.addItem(93, 12)
val code =
"""import Standard.Base.IO
|import Standard.Base.Error.Error
"""from Standard.Base import all
|
|foo =
| Error.throw 9
@ -1462,7 +1461,7 @@ class RuntimeErrorsTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 4), model.Position(4, 17)),
model.Range(model.Position(3, 4), model.Position(3, 17)),
"10002 - 10000"
)
),
@ -1589,8 +1588,8 @@ class RuntimeErrorsTest
context.receiveNIgnorePendingExpressionUpdates(
3
) should contain theSameElementsAs Seq(
TestMessages.update(contextId, x1Id, ConstantsGen.NOTHING),
TestMessages.update(contextId, mainRes1Id, ConstantsGen.NOTHING),
TestMessages.update(contextId, x1Id, ConstantsGen.NOTHING_BUILTIN),
TestMessages.update(contextId, mainRes1Id, ConstantsGen.NOTHING_BUILTIN),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("MyError")

64
engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala
View File

@ -255,14 +255,14 @@ class RuntimeInstrumentTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val mainBody = metadata.addItem(31, 52)
val xExpr = metadata.addItem(40, 2)
val yExpr = metadata.addItem(51, 5)
val zExpr = metadata.addItem(65, 1)
val mainResExpr = metadata.addItem(71, 12)
val mainBody = metadata.addItem(37, 52)
val xExpr = metadata.addItem(46, 2)
val yExpr = metadata.addItem(57, 5)
val zExpr = metadata.addItem(71, 1)
val mainResExpr = metadata.addItem(77, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| x = 42
@ -318,12 +318,12 @@ class RuntimeInstrumentTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val mainBody = metadata.addItem(72, 39)
val xExpr = metadata.addItem(81, 13)
val mainResExpr = metadata.addItem(99, 12)
val mainBody = metadata.addItem(78, 39)
val xExpr = metadata.addItem(87, 13)
val mainResExpr = metadata.addItem(105, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|polyglot java import java.time.LocalDate
|
|main =
@ -376,14 +376,14 @@ class RuntimeInstrumentTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val mainBody = metadata.addItem(31, 42)
val xExpr = metadata.addItem(40, 2)
val yExpr = metadata.addItem(51, 5)
val mainResExpr = metadata.addItem(61, 12)
val mainRes1Expr = metadata.addItem(72, 1)
val mainBody = metadata.addItem(37, 42)
val xExpr = metadata.addItem(46, 2)
val yExpr = metadata.addItem(57, 5)
val mainResExpr = metadata.addItem(67, 12)
val mainRes1Expr = metadata.addItem(78, 1)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| x = 42
@ -618,13 +618,13 @@ class RuntimeInstrumentTest
// f expression
metadata.addItem(42, 5)
val aExpr = metadata.addItem(56, 1)
val fApp = metadata.addItem(74, 3)
val mainRes = metadata.addItem(62, 16)
val mainExpr = metadata.addItem(31, 47)
val aExpr = metadata.addItem(62, 1)
val fApp = metadata.addItem(80, 3)
val mainRes = metadata.addItem(68, 16)
val mainExpr = metadata.addItem(37, 47)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| f x = x + 1
@ -678,16 +678,16 @@ class RuntimeInstrumentTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val aExpr = metadata.addItem(40, 14)
val aExpr = metadata.addItem(46, 14)
// lambda
metadata.addItem(41, 10)
metadata.addItem(47, 10)
// lambda expression
metadata.addItem(46, 5)
val lamArg = metadata.addItem(53, 1)
val mainRes = metadata.addItem(59, 12)
metadata.addItem(52, 5)
val lamArg = metadata.addItem(59, 1)
val mainRes = metadata.addItem(65, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| a = (x -> x + 1) 1
@ -739,14 +739,14 @@ class RuntimeInstrumentTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val aExpr = metadata.addItem(40, 9)
val aExpr = metadata.addItem(46, 9)
// lambda
metadata.addItem(41, 5)
val lamArg = metadata.addItem(48, 1)
val mainRes = metadata.addItem(54, 12)
metadata.addItem(47, 5)
val lamArg = metadata.addItem(54, 1)
val mainRes = metadata.addItem(60, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| a = (_ + 1) 1

126
engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala
View File

@ -252,12 +252,11 @@ class RuntimeServerTest
object Main2 {
val metadata = new Metadata
val idMainY = metadata.addItem(173, 5)
val idMainZ = metadata.addItem(187, 5)
val idMainY = metadata.addItem(178, 5)
val idMainZ = metadata.addItem(192, 5)
val code = metadata.appendToCode(
"""
|import Standard.Base.IO
"""from Standard.Base import all
|
|foo = arg ->
| IO.println "I'm expensive!"
@ -432,10 +431,10 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idFoo = metadata.addItem(35, 6)
val idFoo = metadata.addItem(41, 6)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|foo x=0 = x + 42
|
@ -485,11 +484,11 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idMain = metadata.addItem(48, 19)
val idMainFoo = metadata.addItem(64, 3)
val idMain = metadata.addItem(54, 19)
val idMainFoo = metadata.addItem(70, 3)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|foo a=0 = a + 1
|
@ -558,16 +557,16 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idMain = metadata.addItem(99, 120)
val idMainX = metadata.addItem(126, 9)
val idMainY = metadata.addItem(144, 3)
val idMainM = metadata.addItem(156, 8)
val idMainP = metadata.addItem(173, 5)
val idMainQ = metadata.addItem(187, 5)
val idMainF = metadata.addItem(209, 9)
val idMain = metadata.addItem(105, 120)
val idMainX = metadata.addItem(132, 9)
val idMainY = metadata.addItem(150, 3)
val idMainM = metadata.addItem(162, 8)
val idMainP = metadata.addItem(179, 5)
val idMainQ = metadata.addItem(193, 5)
val idMainF = metadata.addItem(215, 9)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|import Enso_Test.Test.A
|
|type QuuxT
@ -808,11 +807,11 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idMain = metadata.addItem(48, 25)
val idMainFoo = metadata.addItem(65, 7)
val idMain = metadata.addItem(54, 25)
val idMainFoo = metadata.addItem(71, 7)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|foo a b = a + b
|
@ -868,12 +867,11 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idMain = metadata.addItem(113, 36)
val idMainBar = metadata.addItem(145, 3)
val idMain = metadata.addItem(73, 36)
val idMainBar = metadata.addItem(105, 3)
val code =
"""from Standard.Base.Data.Numbers import Number
|import Standard.Base.IO
"""from Standard.Base import all
|import Standard.Base.Runtime.State
|
|main = IO.println (State.run Number 42 bar)
@ -929,12 +927,11 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idMain = metadata.addItem(113, 35)
val idMainBar = metadata.addItem(144, 3)
val idMain = metadata.addItem(73, 35)
val idMainBar = metadata.addItem(104, 3)
val code =
"""from Standard.Base.Data.Numbers import Number
|import Standard.Base.IO
"""from Standard.Base import all
|import Standard.Base.Runtime.State
|
|main = IO.println (State.run Number 0 bar)
@ -1054,13 +1051,13 @@ class RuntimeServerTest
metadata.addItem(25, 22)
// foo name
metadata.addItem(25, 3)
val fooX = metadata.addItem(39, 1, "aa")
val fooRes = metadata.addItem(45, 1, "ab")
val mainFoo = metadata.addItem(63, 3, "ac")
val mainRes = metadata.addItem(71, 12, "ad")
val fooX = metadata.addItem(45, 1, "aa")
val fooRes = metadata.addItem(51, 1, "ab")
val mainFoo = metadata.addItem(69, 3, "ac")
val mainRes = metadata.addItem(77, 12, "ad")
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|foo =
| x = 4
@ -1174,14 +1171,14 @@ class RuntimeServerTest
val metadata = new Metadata
// foo definition
metadata.addItem(25, 22)
metadata.addItem(31, 22)
// foo name
metadata.addItem(25, 3)
val mainFoo = metadata.addItem(63, 3)
val mainRes = metadata.addItem(71, 12)
metadata.addItem(31, 3)
val mainFoo = metadata.addItem(69, 3)
val mainRes = metadata.addItem(77, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|foo =
| x = 4
@ -1356,11 +1353,11 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val idResult = metadata.addItem(45, 4, "aae")
val idPrintln = metadata.addItem(54, 17, "aaf")
val idMain = metadata.addItem(31, 40, "aaa")
val idResult = metadata.addItem(51, 4, "aae")
val idPrintln = metadata.addItem(60, 17, "aaf")
val idMain = metadata.addItem(37, 40, "aaa")
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| result = 1337
@ -1436,20 +1433,19 @@ class RuntimeServerTest
val numberTypeName = "Standard.Base.Data.Numbers.Number"
val metadata = new Metadata
val idMain = metadata.addItem(77, 34, "aaaa")
val idMainA = metadata.addItem(86, 8, "aabb")
val idMainP = metadata.addItem(99, 12, "aacc")
val idMain = metadata.addItem(37, 34, "aaaa")
val idMainA = metadata.addItem(46, 8, "aabb")
val idMainP = metadata.addItem(59, 12, "aacc")
// pie id
metadata.addItem(119, 1, "eee")
metadata.addItem(89, 1, "eee")
// uwu id
metadata.addItem(127, 1, "bbb")
metadata.addItem(87, 1, "bbb")
// hie id
metadata.addItem(135, 6, "fff")
metadata.addItem(95, 6, "fff")
// Number.x id
metadata.addItem(155, 1, "999")
metadata.addItem(115, 1, "999")
val code =
"""from Standard.Base.Data.Numbers import Number
|import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| a = 123 + 21
@ -1505,7 +1501,7 @@ class RuntimeServerTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 8), model.Position(4, 16)),
model.Range(model.Position(3, 8), model.Position(3, 16)),
"1234.x 4"
)
),
@ -1534,7 +1530,7 @@ class RuntimeServerTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 8), model.Position(4, 16)),
model.Range(model.Position(3, 8), model.Position(3, 16)),
"1000.x 5"
)
),
@ -1563,7 +1559,7 @@ class RuntimeServerTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 8), model.Position(4, 16)),
model.Range(model.Position(3, 8), model.Position(3, 16)),
"Main.pie"
)
),
@ -1592,7 +1588,7 @@ class RuntimeServerTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 8), model.Position(4, 16)),
model.Range(model.Position(3, 8), model.Position(3, 16)),
"Main.uwu"
)
),
@ -1621,7 +1617,7 @@ class RuntimeServerTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 8), model.Position(4, 16)),
model.Range(model.Position(3, 8), model.Position(3, 16)),
"Main.hie"
)
),
@ -1650,7 +1646,7 @@ class RuntimeServerTest
mainFile,
Seq(
TextEdit(
model.Range(model.Position(4, 8), model.Position(4, 16)),
model.Range(model.Position(3, 8), model.Position(3, 16)),
"\"Hello!\""
)
),
@ -1899,11 +1895,11 @@ class RuntimeServerTest
val moduleName = "Enso_Test.Test.Main"
val metadata = new Metadata
val xId = metadata.addItem(40, 10)
val mainRes = metadata.addItem(55, 12)
val xId = metadata.addItem(46, 10)
val mainRes = metadata.addItem(61, 12)
val code =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main =
| x = a -> a + 1
@ -2372,7 +2368,7 @@ class RuntimeServerTest
context.consumeOut shouldEqual List()
}
it should "send expression updates when file is restoredzzz" in {
it should "send expression updates when file is restored" in {
val contextId = UUID.randomUUID()
val requestId = UUID.randomUUID()
val moduleName = "Enso_Test.Test.Main"
@ -2383,12 +2379,12 @@ class RuntimeServerTest
)
val metadata = new Metadata
val idText = metadata.addItem(43, 12, "aa")
val idRes = metadata.addItem(60, 15, "ab")
val idText = metadata.addItem(49, 12, "aa")
val idRes = metadata.addItem(66, 15, "ab")
def template(text: String) =
metadata.appendToCode(
s"""import Standard.Base.IO
s"""from Standard.Base import all
|
|main =
| text = "$text"
@ -2661,7 +2657,7 @@ class RuntimeServerTest
)
)
)
context.receiveN(3) should contain theSameElementsAs Seq(
context.receiveNIgnoreStdLib(3) should contain theSameElementsAs Seq(
Api.Response(requestId, Api.PushContextResponse(contextId)),
Api.Response(
Api.ExecutionFailed(

34
engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala
View File

@ -240,7 +240,6 @@ class RuntimeStdlibTest
context.receiveAllUntil(
Seq(
context.executionComplete(contextId),
context.analyzeJobFinished,
context.analyzeJobFinished
),
timeout = 180
@ -299,39 +298,6 @@ class RuntimeStdlibTest
)
}
// check that the Standard.Base library is indexed
val stdlibSuggestions = responses.collect {
case Api.Response(
None,
Api.SuggestionsDatabaseModuleUpdateNotification(
module,
_,
as,
_,
xs
)
) if module.contains("Vector") =>
(xs.nonEmpty || as.nonEmpty) shouldBe true
xs.toVector.map(_.suggestion.module)
}
stdlibSuggestions.nonEmpty shouldBe true
// check that builtins are indexed
val builtinsSuggestions = responses.collect {
case Api.Response(
None,
Api.SuggestionsDatabaseModuleUpdateNotification(
module,
_,
as,
_,
xs
)
) if module.contains("Builtins") =>
(xs.nonEmpty || as.nonEmpty) shouldBe true
}
builtinsSuggestions.length shouldBe 1
// check LibraryLoaded notifications
val contentRootNotifications = responses.collect {
case Api.Response(

38
engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala
View File

@ -138,7 +138,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
3
4
) should contain theSameElementsAs Seq(
Api.Response(requestId, Api.PushContextResponse(contextId)),
Api.Response(
@ -183,6 +183,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("Hello World!")
@ -203,7 +204,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
@ -260,6 +261,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("42")
@ -284,7 +286,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
@ -366,6 +368,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("51")
@ -386,7 +389,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
@ -477,6 +480,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("51")
@ -497,7 +501,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
@ -618,6 +622,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("51")
@ -638,7 +643,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
@ -708,6 +713,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("51")
@ -765,7 +771,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
3
4
) should contain theSameElementsAs Seq(
Api.Response(requestId, Api.PushContextResponse(contextId)),
Api.Response(
@ -884,6 +890,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
}
@ -894,7 +901,7 @@ class RuntimeSuggestionUpdatesTest
val moduleName = "Enso_Test.Test.Main"
val mainCode =
"""import Standard.Base.IO
"""from Standard.Base import all
|
|import Enso_Test.Test.A
|from Enso_Test.Test.A export all
@ -904,7 +911,7 @@ class RuntimeSuggestionUpdatesTest
|main = IO.println "Hello World!"
|""".stripMargin.linesIterator.mkString("\n")
val aCode =
"""from Standard.Base.Data.Numbers import Integer
"""from Standard.Base import all
|
|type MyType
| MkA a
@ -950,7 +957,7 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
4
5
) should contain theSameElementsAs Seq(
Api.Response(requestId, Api.PushContextResponse(contextId)),
Api.Response(
@ -1141,6 +1148,7 @@ class RuntimeSuggestionUpdatesTest
)
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("Hello World!")
@ -1161,13 +1169,13 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnoreExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
module = moduleName,
version = contentsVersion(
"""import Standard.Base.IO
"""from Standard.Base import all
|
|import Enso_Test.Test.A
|from Enso_Test.Test.A export all hiding hello
@ -1190,6 +1198,7 @@ class RuntimeSuggestionUpdatesTest
updates = Tree.Root(Vector())
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("Hello World!")
@ -1210,13 +1219,13 @@ class RuntimeSuggestionUpdatesTest
)
)
context.receiveNIgnorePendingExpressionUpdates(
2
3
) should contain theSameElementsAs Seq(
Api.Response(
Api.SuggestionsDatabaseModuleUpdateNotification(
module = moduleName,
version = contentsVersion(
"""import Standard.Base.IO
"""from Standard.Base import all
|
|main = IO.println "Hello World!"
|""".stripMargin.linesIterator.mkString("\n")
@ -1234,6 +1243,7 @@ class RuntimeSuggestionUpdatesTest
updates = Tree.Root(Vector())
)
),
Api.Response(Api.AnalyzeModuleInScopeJobFinished()),
context.executionComplete(contextId)
)
context.consumeOut shouldEqual List("Hello World!")

29
engine/runtime/src/main/java/org/enso/compiler/Cache.java
View File

@ -6,6 +6,7 @@ import org.bouncycastle.jcajce.provider.digest.SHA3;
import org.bouncycastle.util.encoders.Hex;
import org.enso.interpreter.runtime.EnsoContext;
import org.enso.logger.masking.MaskedPath;
import org.enso.pkg.SourceFile;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@ -19,6 +20,8 @@ import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
@ -326,10 +329,34 @@ public abstract class Cache<T, M extends Cache.Metadata> {
* @param bytes bytes for which hash will be computed
* @return string representation of bytes' hash
*/
protected String computeDigestFromBytes(byte[] bytes) {
protected final String computeDigestFromBytes(byte[] bytes) {
return Hex.toHexString(messageDigest().digest(bytes));
}
/**
* Computes digest from package sources using a default hashing algorithm.
*
* @param pkgSources the list of package sources
* @param logger the truffle logger
* @return string representation of bytes' hash
*/
protected final String computeDigestOfLibrarySources(
List<SourceFile<TruffleFile>> pkgSources, TruffleLogger logger) {
pkgSources.sort(Comparator.comparing(o -> o.qualifiedName().toString()));
var digest = messageDigest();
pkgSources.forEach(
source -> {
try {
digest.update(source.file().readAllBytes());
} catch (IOException e) {
logger.log(
logLevel, "failed to compute digest for " + source.qualifiedName().toString(), e);
}
});
return Hex.toHexString(digest.digest());
}
/**
* Returns a default hashing algorithm used for Enso caches.
*

21
engine/runtime/src/main/java/org/enso/compiler/ImportExportCache.java
View File

@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.oracle.truffle.api.TruffleFile;
import com.oracle.truffle.api.TruffleLogger;
import org.bouncycastle.util.encoders.Hex;
import org.enso.compiler.data.BindingsMap;
import org.enso.editions.LibraryName;
import org.enso.interpreter.runtime.EnsoContext;
@ -15,15 +14,13 @@ import org.enso.pkg.SourceFile;
import scala.collection.immutable.Map;
import scala.jdk.CollectionConverters;
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
public class ImportExportCache extends Cache<ImportExportCache.CachedBindings, ImportExportCache.Metadata> {
public final class ImportExportCache extends Cache<ImportExportCache.CachedBindings, ImportExportCache.Metadata> {
private final LibraryName libraryName;
@ -79,22 +76,6 @@ public class ImportExportCache extends Cache<ImportExportCache.CachedBindings, I
.map(pkg -> computeDigestOfLibrarySources(pkg.listSourcesJava(), logger));
}
private String computeDigestOfLibrarySources(List<SourceFile<TruffleFile>> pkgSources, TruffleLogger logger) {
pkgSources.sort(Comparator.comparing(o -> o.qualifiedName().toString()));
var digest = messageDigest();
pkgSources.forEach(source ->
{
try {
digest.update(source.file().readAllBytes());
} catch (IOException e) {
logger.log(logLevel, "failed to compute digest for " + source.qualifiedName().toString(), e);
}
}
);
return Hex.toHexString(digest.digest());
}
@Override
@SuppressWarnings("unchecked")
protected Optional<Cache.Roots> getCacheRoots(EnsoContext context) {

2
engine/runtime/src/main/java/org/enso/compiler/ModuleCache.java
View File

@ -18,7 +18,7 @@ import java.util.Arrays;
import java.util.Optional;
import java.util.logging.Level;
public class ModuleCache extends Cache<ModuleCache.CachedModule, ModuleCache.Metadata> {
public final class ModuleCache extends Cache<ModuleCache.CachedModule, ModuleCache.Metadata> {
private final Module module;

157
engine/runtime/src/main/java/org/enso/compiler/SuggestionsCache.java Normal file
View File

@ -0,0 +1,157 @@
package org.enso.compiler;
import buildinfo.Info;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.oracle.truffle.api.TruffleFile;
import com.oracle.truffle.api.TruffleLogger;
import org.enso.editions.LibraryName;
import org.enso.interpreter.runtime.EnsoContext;
import org.enso.pkg.SourceFile;
import org.enso.polyglot.Suggestion;
import scala.jdk.CollectionConverters;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
public final class SuggestionsCache
extends Cache<SuggestionsCache.CachedSuggestions, SuggestionsCache.Metadata> {
private static final String SUGGESTIONS_CACHE_DATA_EXTENSION = ".suggestions";
private static final String SUGGESTIONS_CACHE_METADATA_EXTENSION =".suggestions.meta";
private final static ObjectMapper objectMapper = new ObjectMapper();
final LibraryName libraryName;
public SuggestionsCache(LibraryName libraryName) {
this.libraryName = libraryName;
this.logLevel = Level.FINEST;
this.stringRepr = libraryName.toString();
this.entryName = libraryName.name();
this.dataSuffix = SUGGESTIONS_CACHE_DATA_EXTENSION;
this.metadataSuffix = SUGGESTIONS_CACHE_METADATA_EXTENSION;
}
@Override
protected byte[] metadata(String sourceDigest, String blobDigest, CachedSuggestions entry) {
try {
return objectMapper.writeValueAsString(new Metadata(sourceDigest, blobDigest)).getBytes(metadataCharset);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
@Override
protected CachedSuggestions validateReadObject(Object obj, Metadata meta, TruffleLogger logger)
throws CacheException {
if (obj instanceof Suggestions suggestions) {
return new CachedSuggestions(libraryName, suggestions, Optional.empty());
} else {
throw new CacheException("Expected SuggestionsCache.Suggestions, got " + obj.getClass());
}
}
@Override
protected Optional<Metadata> metadataFromBytes(byte[] bytes) {
var maybeJsonString = new String(bytes, Cache.metadataCharset);
try {
return Optional.of(objectMapper.readValue(maybeJsonString, SuggestionsCache.Metadata.class));
} catch (JsonProcessingException e) {
return Optional.empty();
}
}
@Override
protected Optional<String> computeDigest(CachedSuggestions entry, TruffleLogger logger) {
return entry.getSources().map(sources -> computeDigestOfLibrarySources(sources, logger));
}
@Override
protected Optional<String> computeDigestFromSource(EnsoContext context, TruffleLogger logger) {
return context
.getPackageRepository()
.getPackageForLibraryJava(libraryName)
.map(pkg -> computeDigestOfLibrarySources(pkg.listSourcesJava(), logger));
}
@Override
protected Optional<Roots> getCacheRoots(EnsoContext context) {
return context.getPackageRepository().getPackageForLibraryJava(libraryName).map(pkg -> {
var bindingsCacheRoot = pkg.getSuggestionsCacheRootForPackage(Info.ensoVersion());
var localCacheRoot = bindingsCacheRoot.resolve(libraryName.namespace());
var distribution = context.getDistributionManager();
var pathSegments = CollectionConverters.ListHasAsScala(Arrays.asList(
pkg.namespace(),
pkg.name(),
pkg.config().version(),
Info.ensoVersion(),
libraryName.namespace())
).asScala();
var path = distribution.LocallyInstalledDirectories().irCacheDirectory()
.resolve(pathSegments.mkString("/"));
var globalCacheRoot = context.getTruffleFile(path.toFile());
return new Cache.Roots(localCacheRoot, globalCacheRoot);
});
}
@Override
protected Object extractObjectToSerialize(CachedSuggestions entry) {
return entry.getSuggestionsObjectToSerialize();
}
// Suggestions class is not a record because of a Frgaal bug leading to invalid compilation error.
final static class Suggestions implements Serializable {
private final List<Suggestion> suggestions;
public Suggestions(List<Suggestion> suggestions) {
this.suggestions = suggestions;
}
public List<Suggestion> getSuggestions() {
return suggestions;
}
}
// CachedSuggestions class is not a record because of a Frgaal bug leading to invalid compilation error.
public final static class CachedSuggestions {
private final LibraryName libraryName;
private final Suggestions suggestions;
private final Optional<List<SourceFile<TruffleFile>>> sources;
public CachedSuggestions(LibraryName libraryName, Suggestions suggestions, Optional<List<SourceFile<TruffleFile>>> sources) {
this.libraryName = libraryName;
this.suggestions = suggestions;
this.sources = sources;
}
public LibraryName getLibraryName() {
return libraryName;
}
public Optional<List<SourceFile<TruffleFile>>> getSources() {
return sources;
}
public Suggestions getSuggestionsObjectToSerialize() {
return suggestions;
}
public List<Suggestion> getSuggestions() {
return suggestions.getSuggestions();
}
}
record Metadata(
@JsonProperty("source_hash") String sourceHash,
@JsonProperty("blob_hash") String blobHash
) implements Cache.Metadata { }
}

4
engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java
View File

@ -88,7 +88,7 @@ public final class Module implements TruffleObject {
private ModuleScope scope;
private ModuleSources sources;
private PatchedModuleValues patchedValues;
private Map<Source, Module> allSources = new WeakHashMap<>();
private final Map<Source, Module> allSources = new WeakHashMap<>();
private final Package<TruffleFile> pkg;
private CompilationStage compilationStage = CompilationStage.INITIAL;
private boolean isIndexed = false;
@ -97,7 +97,7 @@ public final class Module implements TruffleObject {
private final ModuleCache cache;
private boolean wasLoadedFromCache;
private boolean hasCrossModuleLinks;
private boolean synthetic;
private final boolean synthetic;
private List<QualifiedName> directModulesRefs;
/**

10
engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala
View File

@ -36,6 +36,7 @@ import java.util.concurrent.{
TimeUnit
}
import java.util.logging.Level
import scala.jdk.OptionConverters._
/** This class encapsulates the static transformation processes that take place
@ -102,7 +103,7 @@ class Compiler(
}
/** Lazy-initializes the IR for the builtins module. */
def initializeBuiltinsIr(): Unit = {
private def initializeBuiltinsIr(): Unit = {
if (!builtins.isIrInitialized) {
logger.log(
Compiler.defaultLogLevel,
@ -137,6 +138,10 @@ class Compiler(
}
}
/** @return the serialization manager instance. */
def getSerializationManager: SerializationManager =
serializationManager
/** Processes the provided language sources, registering any bindings in the
* given scope.
*
@ -195,7 +200,8 @@ class Compiler(
generateCode = false,
shouldCompileDependencies
)
serializationManager.serializeLibraryBindings(
serializationManager.serializeLibrary(
pkg.libraryName,
useGlobalCacheLocations = true
)

164
engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala
View File

@ -2,13 +2,17 @@ package org.enso.compiler
import com.oracle.truffle.api.TruffleLogger
import com.oracle.truffle.api.source.Source
import org.enso.compiler.context.SuggestionBuilder
import org.enso.compiler.core.IR
import org.enso.compiler.pass.analyse.BindingAnalysis
import org.enso.docs.sections.DocSectionsBuilder
import org.enso.editions.LibraryName
import org.enso.interpreter.runtime.Module
import org.enso.pkg.QualifiedName
import org.enso.polyglot.Suggestion
import java.io.NotSerializableException
import java.util
import java.util.concurrent.{
Callable,
CompletableFuture,
@ -19,10 +23,15 @@ import java.util.concurrent.{
TimeUnit
}
import java.util.logging.Level
import scala.collection.mutable
import scala.jdk.OptionConverters.RichOptional
class SerializationManager(compiler: Compiler) {
final class SerializationManager(
compiler: Compiler,
docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder()
) {
import SerializationManager._
/** The debug logging level. */
@ -133,13 +142,13 @@ class SerializationManager(compiler: Compiler) {
}
}
def serializeLibraryBindings(
def serializeLibrary(
libraryName: LibraryName,
useGlobalCacheLocations: Boolean
): Future[Boolean] = {
logger.log(
Level.INFO,
s"Requesting serialization for library [$libraryName] bindings."
s"Requesting serialization for library [$libraryName]."
)
val task: Callable[Boolean] =
@ -157,7 +166,7 @@ class SerializationManager(compiler: Compiler) {
case e: Throwable =>
logger.log(
debugLogLevel,
s"Serialization task failed for library [${libraryName}].",
s"Serialization task failed for library [$libraryName].",
e
)
CompletableFuture.completedFuture(false)
@ -165,7 +174,7 @@ class SerializationManager(compiler: Compiler) {
}
}
def doSerializeLibrary(
private def doSerializeLibrary(
libraryName: LibraryName,
useGlobalCacheLocations: Boolean
): Callable[Boolean] = () => {
@ -195,29 +204,100 @@ class SerializationManager(compiler: Compiler) {
.map(_.listSourcesJava())
)
try {
new ImportExportCache(libraryName)
.save(bindingsCache, compiler.context, useGlobalCacheLocations)
.isPresent()
} catch {
case e: NotSerializableException =>
logger.log(
Level.SEVERE,
s"Could not serialize bindings [$libraryName].",
e
)
throw e
case e: Throwable =>
logger.log(
Level.SEVERE,
s"Serialization of bindings `$libraryName` failed: ${e.getMessage}`",
e
)
throw e
val result =
try {
new ImportExportCache(libraryName)
.save(bindingsCache, compiler.context, useGlobalCacheLocations)
.isPresent
} catch {
case e: NotSerializableException =>
logger.log(
Level.SEVERE,
s"Could not serialize bindings [$libraryName].",
e
)
throw e
case e: Throwable =>
logger.log(
Level.SEVERE,
s"Serialization of bindings `$libraryName` failed: ${e.getMessage}`",
e
)
throw e
}
try {
val suggestions = new util.ArrayList[Suggestion]()
compiler.packageRepository
.getModulesForLibrary(libraryName)
.flatMap { module =>
SuggestionBuilder(module)
.build(module.getName, module.getIr)
.toVector
}
.map(generateDocumentation)
.foreach(suggestions.add)
val cachedSuggestions =
new SuggestionsCache.CachedSuggestions(
libraryName,
new SuggestionsCache.Suggestions(suggestions),
compiler.packageRepository
.getPackageForLibraryJava(libraryName)
.map(_.listSourcesJava())
)
new SuggestionsCache(libraryName)
.save(cachedSuggestions, compiler.context, false)
.isPresent
} catch {
case e: NotSerializableException =>
logger.log(
Level.SEVERE,
s"Could not serialize suggestions [$libraryName].",
e
)
throw e
case e: Throwable =>
logger.log(
Level.SEVERE,
s"Serialization of suggestions `$libraryName` failed: ${e.getMessage}`",
e
)
throw e
}
result
} finally {
finishSerializing(libraryName.toQualifiedName)
}
}
def deserializeSuggestions(
libraryName: LibraryName
): Option[SuggestionsCache.CachedSuggestions] = {
if (isWaitingForSerialization(libraryName)) {
abort(libraryName)
None
} else {
while (isSerializingLibrary(libraryName)) {
Thread.sleep(100)
}
new SuggestionsCache(libraryName).load(compiler.context).toScala match {
case result @ Some(_: SuggestionsCache.CachedSuggestions) =>
logger.log(
Level.FINE,
s"Restored suggestions for library [$libraryName]."
)
result
case _ =>
logger.log(
Level.FINEST,
s"Unable to load suggestions for library [$libraryName]."
)
None
}
}
}
def deserializeLibraryBindings(
libraryName: LibraryName
): Option[ImportExportCache.CachedBindings] = {
@ -524,7 +604,45 @@ class SerializationManager(compiler: Compiler) {
)
.asScala
}
/** Generate the documentation for the given suggestion.
*
* @param suggestion the initial suggestion
* @return the suggestion with documentation fields set
*/
private def generateDocumentation(suggestion: Suggestion): Suggestion =
suggestion match {
case module: Suggestion.Module =>
val docSections = module.documentation.map(docSectionsBuilder.build)
module.copy(documentationSections = docSections)
case constructor: Suggestion.Constructor =>
val docSections =
constructor.documentation.map(docSectionsBuilder.build)
constructor.copy(documentationSections = docSections)
case tpe: Suggestion.Type =>
val docSections = tpe.documentation.map(docSectionsBuilder.build)
tpe.copy(documentationSections = docSections)
case method: Suggestion.Method =>
val docSections = method.documentation.map(docSectionsBuilder.build)
method.copy(documentationSections = docSections)
case conversion: Suggestion.Conversion =>
val docSections = conversion.documentation.map(docSectionsBuilder.build)
conversion.copy(documentationSections = docSections)
case function: Suggestion.Function =>
val docSections = function.documentation.map(docSectionsBuilder.build)
function.copy(documentationSections = docSections)
case local: Suggestion.Local =>
val docSections = local.documentation.map(docSectionsBuilder.build)
local.copy(documentationSections = docSections)
}
}
object SerializationManager {
/** The maximum number of serialization threads allowed. */

9
engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala
View File

@ -8,6 +8,7 @@ import org.enso.compiler.pass.resolve.{
TypeNames,
TypeSignatures
}
import org.enso.interpreter.runtime.Module
import org.enso.interpreter.runtime.`type`.Types
import org.enso.pkg.QualifiedName
import org.enso.polyglot.Suggestion
@ -655,6 +656,14 @@ object SuggestionBuilder {
/** TODO[DB] enable conversions when they get the runtime support. */
private val ConversionsEnabled: Boolean = false
/** Creates the suggestion builder for a module.
*
* @param module the module to index
* @return the suggestions builder for the module
*/
def apply(module: Module): SuggestionBuilder[CharSequence] =
SuggestionBuilder(module.getSource.getCharacters)
/** Create the suggestion builder.
*
* @param source the text source

1
engine/runtime/src/main/scala/org/enso/compiler/context/SuggestionDiff.scala
View File

@ -1,4 +1,5 @@
package org.enso.compiler.context
import org.enso.polyglot.Suggestion
import org.enso.polyglot.data.{These, Tree}
import org.enso.polyglot.runtime.Runtime.Api

13
engine/runtime/src/main/scala/org/enso/interpreter/instrument/Handler.scala
View File

@ -17,9 +17,7 @@ import org.graalvm.polyglot.io.MessageEndpoint
import java.nio.ByteBuffer
import scala.concurrent.Future
/** A message endpoint implementation used by the
* [[org.enso.interpreter.instrument.RuntimeServerInstrument]].
*/
/** A message endpoint implementation. */
class Endpoint(handler: Handler)
extends MessageEndpoint
with RuntimeServerConnectionEndpoint {
@ -47,6 +45,15 @@ class Endpoint(handler: Handler)
def sendToClient(msg: Api.Response): Unit =
client.sendBinary(Api.serialize(msg))
/** Sends a notification to the runtime.
*
* Can be used to start a command processing in the background.
*
* @param msg the message to send.
*/
def sendToSelf(msg: Api.Request): Unit =
handler.onMessage(msg)
/** Sends a request to the connected client and expects a reply. */
override def sendRequest(msg: ApiRequest): Future[ApiResponse] =
reverseRequestEndpoint.sendRequest(msg)

21
engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala
View File

@ -115,14 +115,21 @@ object NotificationHandler {
libraryName: LibraryName,
libraryVersion: LibraryVersion,
location: Path
): Unit = sendMessage(
Api.LibraryLoaded(
namespace = libraryName.namespace,
name = libraryName.name,
version = libraryVersion.toString,
location = location.toFile
): Unit = {
sendMessage(
Api.LibraryLoaded(
namespace = libraryName.namespace,
name = libraryName.name,
version = libraryVersion.toString,
location = location.toFile
)
)
)
endpoint.sendToSelf(
Api.Request(
Api.DeserializeLibrarySuggestions(libraryName)
)
)
}
/** @inheritdoc */
override def trackProgress(message: String, task: TaskProgress[_]): Unit = {

6
engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/CommandFactory.scala
View File

@ -56,12 +56,12 @@ object CommandFactory {
case payload: Api.InvalidateModulesIndexRequest =>
new InvalidateModulesIndexCmd(request.requestId, payload)
case payload: Api.VerifyModulesIndexRequest =>
new VerifyModulesIndexCmd(request.requestId, payload)
case _: Api.GetTypeGraphRequest =>
new GetTypeGraphCommand(request.requestId)
case payload: Api.DeserializeLibrarySuggestions =>
new DeserializeLibrarySuggestionsCmd(request.requestId, payload)
case Api.ShutDownRuntimeServer() =>
throw new IllegalArgumentException(
"ShutDownRuntimeServer request is not convertible to command object"

26
engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/DeserializeLibrarySuggestionsCmd.scala Normal file
View File

@ -0,0 +1,26 @@
package org.enso.interpreter.instrument.command
import org.enso.interpreter.instrument.execution.RuntimeContext
import org.enso.interpreter.instrument.job.DeserializeLibrarySuggestionsJob
import org.enso.polyglot.runtime.Runtime.Api
import scala.concurrent.{ExecutionContext, Future}
/** A command that initiates the deserialization of suggestions.
*
* @param maybeRequestId an option with request id
*/
class DeserializeLibrarySuggestionsCmd(
maybeRequestId: Option[Api.RequestId],
request: Api.DeserializeLibrarySuggestions
) extends Command(maybeRequestId) {
/** @inheritdoc */
override def execute(implicit
ctx: RuntimeContext,
ec: ExecutionContext
): Future[Unit] =
ctx.jobProcessor.runBackground(
new DeserializeLibrarySuggestionsJob(request.libraryName)
)
}

39
engine/runtime/src/main/scala/org/enso/interpreter/instrument/command/VerifyModulesIndexCmd.scala
View File

@ -1,39 +0,0 @@
package org.enso.interpreter.instrument.command
import org.enso.interpreter.instrument.execution.RuntimeContext
import org.enso.polyglot.runtime.Runtime.Api
import scala.collection.mutable
import scala.concurrent.{ExecutionContext, Future}
/** A command that verifies the modules index.
*
* @param maybeRequestId an option with request id
* @param request a verification request
*/
class VerifyModulesIndexCmd(
maybeRequestId: Option[Api.RequestId],
val request: Api.VerifyModulesIndexRequest
) extends Command(maybeRequestId) {
/** Executes a request.
*
* @param ctx contains suppliers of services to perform a request
*/
override def execute(implicit
ctx: RuntimeContext,
ec: ExecutionContext
): Future[Unit] = {
ctx.locking.acquireReadCompilationLock()
try {
val builder = mutable.Set(request.modules: _*)
ctx.executionService.getContext.getTopScope.getModules.forEach { module =>
builder -= module.getName.toString
}
Future(reply(Api.VerifyModulesIndexResponse(builder.toVector)))
} finally {
ctx.locking.releaseReadCompilationLock()
}
}
}

44
engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala
View File

@ -8,7 +8,6 @@ import org.enso.compiler.context.{
}
import org.enso.interpreter.instrument.execution.RuntimeContext
import org.enso.interpreter.runtime.Module
import org.enso.pkg.QualifiedName
import org.enso.polyglot.data.Tree
import org.enso.polyglot.runtime.Runtime.Api
import org.enso.polyglot.{ModuleExports, Suggestion}
@ -16,7 +15,6 @@ import org.enso.polyglot.{ModuleExports, Suggestion}
import java.util.logging.Level
final class AnalyzeModuleInScopeJob(
moduleName: Option[QualifiedName],
modules: Iterable[Module]
) extends Job[Unit](
List(AnalyzeModuleJob.backgroundContextId),
@ -33,27 +31,10 @@ final class AnalyzeModuleInScopeJob(
// disable the suggestion updates and reduce the number of messages that
// runtime sends.
if (ctx.executionService.getContext.isProjectSuggestionsEnabled) {
// When the project module is compiled it can involve compilation of
// global (library) modules, so we need to check if the global
// suggestions are enabled as well.
if (ctx.executionService.getContext.isGlobalSuggestionsEnabled) {
modules.foreach(analyzeModuleInScope)
ctx.endpoint.sendToClient(
Api.Response(Api.AnalyzeModuleInScopeJobFinished())
)
} else {
// When the global suggestions are disabled, we will skip indexing
// of external libraries, but still want to index the modules that
// belongs to the project.
val projectModules =
moduleName match {
case Some(name) =>
modules.filter(m => rootName(m.getName) == rootName(name))
case None =>
Seq()
}
projectModules.foreach(analyzeModuleInScope)
}
modules.foreach(analyzeModuleInScope)
ctx.endpoint.sendToClient(
Api.Response(Api.AnalyzeModuleInScopeJobFinished())
)
}
}
@ -94,9 +75,6 @@ final class AnalyzeModuleInScopeJob(
case _: Suggestion.Local => false
}
private def rootName(name: QualifiedName): String =
name.path.headOption.getOrElse(name.item)
/** Send notification about module updates.
*
* @param payload the module update
@ -116,23 +94,11 @@ final class AnalyzeModuleInScopeJob(
object AnalyzeModuleInScopeJob {
/** Create an instance of [[AnalyzeModuleInScopeJob]].
*
* @param project the project module name
* @param modules the list of modules to analyze
* @return the [[AnalyzeModuleInScopeJob]]
*/
def apply(
project: QualifiedName,
modules: Iterable[Module]
): AnalyzeModuleInScopeJob =
new AnalyzeModuleInScopeJob(Some(project), modules)
/** Create an instance of [[AnalyzeModuleInScopeJob]].
*
* @param modules the list of modules to analyze
* @return the [[AnalyzeModuleInScopeJob]]
*/
def apply(modules: Iterable[Module]): AnalyzeModuleInScopeJob =
new AnalyzeModuleInScopeJob(None, modules)
new AnalyzeModuleInScopeJob(modules)
}

44
engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala Normal file
View File

@ -0,0 +1,44 @@
package org.enso.interpreter.instrument.job
import org.enso.editions.LibraryName
import org.enso.interpreter.instrument.execution.RuntimeContext
import org.enso.polyglot.runtime.Runtime.Api
import java.util.logging.Level
import scala.jdk.CollectionConverters._
/** A job responsible for deserializing suggestions of loaded library.
*
* @param libraryName the name of loaded library
*/
final class DeserializeLibrarySuggestionsJob(
libraryName: LibraryName
) extends Job[Unit](
List(),
isCancellable = false,
mayInterruptIfRunning = false
) {
/** @inheritdoc */
override def run(implicit ctx: RuntimeContext): Unit = {
ctx.executionService.getLogger.log(
Level.FINE,
s"Deserializing suggestions for library [$libraryName]."
)
val serializationManager =
ctx.executionService.getContext.getCompiler.getSerializationManager
serializationManager
.deserializeSuggestions(libraryName)
.foreach { cachedSuggestions =>
ctx.endpoint.sendToClient(
Api.Response(
Api.SuggestionsDatabaseSuggestionsLoadedNotification(
libraryName,
cachedSuggestions.getSuggestions.asScala.toVector
)
)
)
}
}
}

26
engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala
View File

@ -27,7 +27,6 @@ import org.enso.text.buffer.Rope
import java.io.File
import java.util.logging.Level
import scala.jdk.CollectionConverters._
import scala.jdk.OptionConverters._
/** A job that ensures that specified files are compiled.
@ -58,7 +57,7 @@ final class EnsureCompiledJob(protected val files: Iterable[File])
* @param files the list of files to compile.
* @param ctx the runtime context
*/
protected def ensureCompiledFiles(
private def ensureCompiledFiles(
files: Iterable[File]
)(implicit ctx: RuntimeContext): CompilationStatus = {
val modules = files.flatMap { file =>
@ -66,7 +65,7 @@ final class EnsureCompiledJob(protected val files: Iterable[File])
}
val moduleCompilationStatus = modules.map(ensureCompiledModule)
val modulesInScope =
getModulesInScope.filterNot(m => modules.exists(_ == m))
getProjectModulesInScope.filterNot(m => modules.exists(_ == m))
val scopeCompilationStatus = ensureCompiledScope(modulesInScope)
(moduleCompilationStatus.flatten ++ scopeCompilationStatus).maxOption
.getOrElse(CompilationStatus.Success)
@ -84,14 +83,8 @@ final class EnsureCompiledJob(protected val files: Iterable[File])
compile(module)
applyEdits(new File(module.getPath)).map { changeset =>
compile(module)
.map { compilerResult =>
.map { _ =>
invalidateCaches(module, changeset)
ctx.jobProcessor.runBackground(
AnalyzeModuleInScopeJob(
module.getName,
compilerResult.compiledModules
)
)
ctx.jobProcessor.runBackground(AnalyzeModuleJob(module, changeset))
runCompilationDiagnostics(module)
}
@ -433,11 +426,16 @@ final class EnsureCompiledJob(protected val files: Iterable[File])
module.getIr.getMetadata(CachePreferenceAnalysis)
}
/** Get all modules in the current compiler scope. */
private def getModulesInScope(implicit
/** Get all project modules in the current compiler scope. */
private def getProjectModulesInScope(implicit
ctx: RuntimeContext
): Iterable[Module] =
ctx.executionService.getContext.getTopScope.getModules.asScala
): Iterable[Module] = {
val packageRepository =
ctx.executionService.getContext.getCompiler.packageRepository
packageRepository.getMainProjectPackage
.map(pkg => packageRepository.getModulesForLibrary(pkg.libraryName))
.getOrElse(Seq())
}
/** Check if stack belongs to the provided module.
*

4
engine/runtime/src/test/java/org/enso/compiler/SerializerTest.java
View File

@ -1,5 +1,6 @@
package org.enso.compiler;
import org.enso.docs.sections.DocSectionsBuilder;
import org.enso.interpreter.runtime.EnsoContext;
import org.enso.pkg.PackageManager;
import org.enso.polyglot.LanguageInfo;
@ -55,7 +56,8 @@ public class SerializerTest {
ctx.enter();
var result = compiler.run(module);
assertEquals(result.compiledModules().exists(m -> m == module), true);
var serializationManager = new SerializationManager(ensoContext.getCompiler());
var serializationManager =
new SerializationManager(ensoContext.getCompiler(), DocSectionsBuilder.apply());
var future = serializationManager.serializeModule(module, true);
var serialized = future.get(5, TimeUnit.SECONDS);
assertEquals(serialized, true);

92
engine/runtime/src/test/java/org/enso/compiler/test/context/JacksonTest.java Normal file
View File

@ -0,0 +1,92 @@
package org.enso.compiler.test.context;
import java.util.List;
import org.enso.polyglot.Suggestion;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import org.junit.Test;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.scala.DefaultScalaModule;
import scala.Option;
public class JacksonTest {
@Test
public void testSerdeOfSuggestion() throws Exception {
Object shape = new Suggestion.Module(
"SampleModule",
Option.apply("doc"),
Option.apply("html"),
Option.empty(),
Option.empty()
);
final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule());
String result = m
.writerWithDefaultPrettyPrinter()
.writeValueAsString(shape);
Suggestion suggestion = m.readerFor(Suggestion.class).readValue(result);
assertEquals("SampleModule", suggestion.name());
assertEquals("doc", suggestion.documentation().get());
assertEquals(Suggestion.Module.class, suggestion.getClass());
}
@Test
public void testArraySerdeOfSuggestion() throws Exception {
Object shape = new Suggestion[]{new Suggestion.Module(
"SampleModule",
Option.apply("doc"),
Option.apply("html"),
Option.empty(),
Option.empty()
)};
final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule());
String result = m
.writerWithDefaultPrettyPrinter()
.writeValueAsString(shape);
var it = m.readerFor(Suggestion.class).readValues(result);
var suggestion = it.nextValue();
assertEquals(Suggestion.Module.class, suggestion.getClass());
if (suggestion instanceof Suggestion.Module module) {
assertEquals("SampleModule", module.name());
assertEquals("doc", module.documentation().get());
} else {
fail("Expecting Suggestion.Module: " + suggestion);
}
}
@Test
public void testRecordSerdeOfSuggestion() throws Exception {
Object shape = new SuggestionCache(11, List.of(new Suggestion.Module(
"SampleModule",
Option.apply("doc"),
Option.apply("html"),
Option.empty(),
Option.empty()
)));
final ObjectMapper m = new ObjectMapper().registerModule(new DefaultScalaModule());
String result = m
.writerWithDefaultPrettyPrinter()
.writeValueAsString(shape);
var cache = (SuggestionCache) m.readerFor(SuggestionCache.class).readValue(result);
assertEquals("One suggestion", 1, cache.suggestions.size());
if (cache.suggestions().get(0) instanceof Suggestion.Module module) {
assertEquals("SampleModule", module.name());
assertEquals("doc", module.documentation().get());
} else {
fail("Expecting Suggestion.Module: " + cache);
}
}
public record SuggestionCache(
@JsonProperty("version") int version,
@JsonProperty("suggestions") List<Suggestion> suggestions
) {
}
}

3
engine/language-server/src/main/scala/org/enso/languageserver/search/DocSectionsBuilder.scala → lib/scala/docs-generator/src/main/scala/org/enso/docs/sections/DocSectionsBuilder.scala
View File

@ -1,6 +1,5 @@
package org.enso.languageserver.search
package org.enso.docs.sections
import org.enso.docs.sections.{HtmlRepr, ParsedSectionsBuilder, Section}
import org.enso.polyglot.DocSection
import org.enso.syntax.text.DocParser
import org.enso.syntax.text.ast.Doc

2
engine/language-server/src/test/scala/org/enso/languageserver/search/DocSectionsBuilderTest.scala → lib/scala/docs-generator/src/test/scala/org/enso/docs/sections/DocSectionsBuilderTest.scala
View File

@ -1,4 +1,4 @@
package org.enso.languageserver.search
package org.enso.docs.sections
import org.enso.polyglot.DocSection
import org.scalatest.matchers.should.Matchers

24
lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala
View File

@ -43,6 +43,9 @@ case class Package[F](
val bindingsCacheDirectory: F = internalDirectory
.getChild(Package.cacheDirName)
.getChild(Package.bindingsCacheDirName)
val suggestionsCacheDirectory: F = internalDirectory
.getChild(Package.cacheDirName)
.getChild(Package.suggestionsCacheDirName)
/** Sets the package name.
*
@ -82,10 +85,30 @@ case class Package[F](
irCacheDirectory.getChild(ensoVersion)
}
/** Gets the bindings cache root location within this package for a given Enso
* version.
*
* This will create the location if it does not exist.
*
* @param ensoVersion the enso version to get the cache root for
* @return the cache root location
*/
def getBindingsCacheRootForPackage(ensoVersion: String): F = {
bindingsCacheDirectory.getChild(ensoVersion)
}
/** Gets the suggestions cache root location within this package for a given
* Enso version.
*
* This will create the location if it does not exist.
*
* @param ensoVersion the enso version to get the cache root for
* @return the cache root location
*/
def getSuggestionsCacheRootForPackage(ensoVersion: String): F = {
suggestionsCacheDirectory.getChild(ensoVersion)
}
/** Changes the package name.
*
* @param newName the new package name
@ -495,4 +518,5 @@ object Package {
val cacheDirName = "cache"
val irCacheDirName = "ir"
val bindingsCacheDirName = "bindings"
val suggestionsCacheDirName = "suggestions"
}

3
lib/scala/pkg/src/main/scala/org/enso/pkg/QualifiedName.scala
View File

@ -1,8 +1,9 @@
package org.enso.pkg
import scala.jdk.CollectionConverters._;
import com.oracle.truffle.api.CompilerDirectives
import scala.jdk.CollectionConverters._
/** Represents a qualified name of a source item.
*
* @param path the names of the package and directories the item is

22
project/DistributionPackage.scala
View File

@ -38,8 +38,7 @@ object DistributionPackage {
}
}
/**
* Conditional copying, based on the contents of cache and timestamps of files.
/** Conditional copying, based on the contents of cache and timestamps of files.
*
* @param source source directory
* @param destination target directory
@ -123,7 +122,6 @@ object DistributionPackage {
targetStdlibVersion: String,
targetDir: File
): Unit = {
copyDirectoryIncremental(
file("distribution/engine/THIRD-PARTY"),
distributionRoot / "THIRD-PARTY",
@ -135,15 +133,7 @@ object DistributionPackage {
distributionRoot / "component",
cacheFactory.make("engine-jars")
)
val os = System.getProperty("os.name")
val isMac = os.startsWith("Mac")
val parser = targetDir / (if (isMac) {
"libenso_parser.dylib"
} else if (os.startsWith("Windows")) {
"enso_parser.dll"
} else {
"libenso_parser.so"
})
val parser = targetDir / Platform.dynamicLibraryFileName("enso_parser")
copyFilesIncremental(
Seq(parser),
distributionRoot / "component",
@ -583,7 +573,7 @@ object DistributionPackage {
arguments: String*
): String = {
val shallowFile = graalDir / "bin" / "gu"
val deepFile = graalDir / "Contents" / "Home" / "bin" / "gu"
val deepFile = graalDir / "Contents" / "Home" / "bin" / "gu"
val executableFile = os match {
case OS.Linux =>
shallowFile
@ -597,11 +587,13 @@ object DistributionPackage {
graalDir / "bin" / "gu.cmd"
}
val javaHomeFile = executableFile.getParentFile.getParentFile
val javaHome = javaHomeFile.toPath.toAbsolutePath
val javaHome = javaHomeFile.toPath.toAbsolutePath
val command =
executableFile.toPath.toAbsolutePath.toString +: arguments
log.debug(s"Running $command in $graalDir with JAVA_HOME=${javaHome.toString}")
log.debug(
s"Running $command in $graalDir with JAVA_HOME=${javaHome.toString}"
)
try {
Process(

10
test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso
View File

@ -16,11 +16,17 @@ export project.Data.Array.Array
import project.Data.Boolean
from project.Data.Boolean export Boolean, True, False
import project.Data.Text.Text
export project.Data.Text.Text
import project.Data.Time.Date.Date
export project.Data.Time.Date.Date
import project.Data.List.List
export project.Data.List.List
import project.Data.Numbers.Number
export project.Data.Numbers.Number
import project.Data.Numbers
from project.Data.Numbers export Number, Integer
import project.Data.Vector.Vector
export project.Data.Vector.Vector