Fix polyglot tree serialization (#9852)

close #9306

Changelog:
- fix: `polyglot.data.Tree` Jackson serialization
- update: report errors during the deserialization messages between the Runtime and the Language Server

engine/language-server/src/main/scala/org/enso/languageserver/runtime/RuntimeConnector.scala

@@ -17,6 +17,8 @@ import org.graalvm.polyglot.io.MessageEndpoint
 
 import java.nio.ByteBuffer
 
+import scala.util.{Failure, Success}
+
 /** An actor managing a connection to Enso's runtime server. */
 final class RuntimeConnector(
   handlers: Map[Class[_], ActorRef],
@@ -199,13 +201,18 @@ object RuntimeConnector {
     * @param peerEndpoint the runtime server's connection end.
     */
   class Endpoint(actor: ActorRef, peerEndpoint: MessageEndpoint)
-      extends MessageEndpoint {
+      extends MessageEndpoint
+      with LazyLogging {
+
     override def sendText(text: String): Unit = {}
 
     override def sendBinary(data: ByteBuffer): Unit =
-      Runtime.Api
-        .deserializeApiEnvelope(data)
-        .foreach(actor ! MessageFromRuntime(_))
+      Runtime.Api.deserializeApiEnvelope(data) match {
+        case Success(msg) =>
+          actor ! MessageFromRuntime(msg)
+        case Failure(ex) =>
+          logger.error("Failed to deserialize runtime API envelope", ex)
+      }
 
     override def sendPing(data: ByteBuffer): Unit = peerEndpoint.sendPong(data)
 

engine/polyglot-api/src/main/scala/org/enso/polyglot/data/Tree.scala

@@ -6,7 +6,11 @@ import scala.collection.mutable
 /** A rose-tree like data structure that distinguishes between root and node
   * elements.
   */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonTypeInfo(
+  use      = JsonTypeInfo.Id.NAME,
+  include  = JsonTypeInfo.As.EXTERNAL_PROPERTY,
+  property = "type"
+)
 @JsonSubTypes(
   Array(
     new JsonSubTypes.Type(

engine/polyglot-api/src/main/scala/org/enso/polyglot/runtime/Runtime.scala

@@ -1975,8 +1975,8 @@ object Runtime {
       * @param bytes the buffer to deserialize
       * @return the deserialized message, if the byte buffer can be deserialized.
       */
-    def deserializeApiEnvelope(bytes: ByteBuffer): Option[ApiEnvelope] =
-      Try(mapper.readValue(bytes.array(), classOf[ApiEnvelope])).toOption
+    def deserializeApiEnvelope(bytes: ByteBuffer): Try[ApiEnvelope] =
+      Try(mapper.readValue(bytes.array(), classOf[ApiEnvelope]))
   }
 
 }

engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerEmulator.scala

@@ -10,6 +10,8 @@ import org.graalvm.polyglot.io.{MessageEndpoint, MessageTransport}
 import java.nio.ByteBuffer
 import java.util.concurrent.LinkedBlockingQueue
 
+import scala.util.{Failure, Success}
+
 /** Emulates the language server for the purposes of testing.
   *
   * Runtime tests are run in the absence of a real language server, which is
@@ -63,12 +65,13 @@ class RuntimeServerEmulator(
 
         override def sendBinary(data: ByteBuffer): Unit = {
           Api.deserializeApiEnvelope(data) match {
-            case Some(request: Api.Request) =>
+            case Success(request: Api.Request) =>
               connector ! request
-            case Some(response: Api.Response) =>
+            case Success(response: Api.Response) =>
               messageQueue.add(response)
-            case None =>
+            case Failure(ex) =>
               println("Failed to deserialize a message.")
+              ex.printStackTrace()
           }
         }
 

engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala

@@ -1416,4 +1416,74 @@ class RuntimeSuggestionUpdatesTest
     indexedModules2 should contain theSameElementsAs Seq(moduleName)
     context.consumeOut shouldEqual List("Hello World!")
   }
+
+  it should "issue 9306" in {
+    val contextId  = UUID.randomUUID()
+    val requestId  = UUID.randomUUID()
+    val moduleName = "Enso_Test.Test.Main"
+
+    val code =
+      """type T
+        |    A
+        |    B
+        |
+        |type K
+        |    C
+        |    D
+        |
+        |type Test
+        |    Value
+        |
+        |    test self (key:(T | K)) = key
+        |
+        |main =
+        |    operator89430 = Test.Value
+        |    operator83086 = operator89430.test
+        |    operator83086
+        |""".stripMargin.linesIterator.mkString("\n")
+    val mainFile = context.writeMain(code)
+
+    // create context
+    context.send(Api.Request(requestId, Api.CreateContextRequest(contextId)))
+    context.receive shouldEqual Some(
+      Api.Response(requestId, Api.CreateContextResponse(contextId))
+    )
+
+    // open file
+    context.send(
+      Api.Request(requestId, Api.OpenFileRequest(mainFile, code))
+    )
+    context.receive shouldEqual Some(
+      Api.Response(Some(requestId), Api.OpenFileResponse)
+    )
+
+    // push main
+    context.send(
+      Api.Request(
+        requestId,
+        Api.PushContextRequest(
+          contextId,
+          Api.StackItem.ExplicitCall(
+            Api.MethodPointer(moduleName, "Enso_Test.Test.Main", "main"),
+            None,
+            Vector()
+          )
+        )
+      )
+    )
+    val updates1 = context.receiveNIgnoreExpressionUpdates(3)
+    updates1.length shouldEqual 3
+    updates1 should contain allOf (
+      Api.Response(requestId, Api.PushContextResponse(contextId)),
+      context.executionComplete(contextId),
+    )
+    val indexedModules = updates1.collect {
+      case Api.Response(
+            None,
+            Api.SuggestionsDatabaseModuleUpdateNotification(moduleName, _, _, _)
+          ) =>
+        moduleName
+    }
+    indexedModules should contain theSameElementsAs Seq(moduleName)
+  }
 }