Wrapping parser for Rust (#325)

* deriving JSON Encoder for Scala AST types
* websocket-based Parser Service
* wrapper for Parser in Rust that includes client for Parser Service

.gitignore

@@ -20,6 +20,12 @@ target/
 *.class
 *.log
 
+##########
+## Rust ##
+##########
+
+Cargo.lock
+
 #############
 ## Haskell ##
 #############

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+
+members = [
+    "common/rust/parser"
+]

build.sbt

@@ -10,6 +10,7 @@ import sbtassembly.AssemblyPlugin.defaultUniversalScript
 
 val scalacVersion = "2.12.10"
 val graalVersion  = "19.2.0.1"
+val circeVersion  = "0.11.1"
 organization in ThisBuild := "org.enso"
 scalaVersion in ThisBuild := scalacVersion
 
@@ -98,6 +99,7 @@ lazy val enso = (project in file("."))
     pkg,
     project_manager,
     runtime,
+    parser_service,
     syntax,
     syntax_definition,
     unused
@@ -131,8 +133,8 @@ val scala_compiler = Seq(
   "org.scala-lang" % "scala-compiler" % scalacVersion
 )
 
-val circe = Seq("circe-core", "circe-generic", "circe-yaml")
-  .map("io.circe" %% _ % "0.10.0")
+val circe = Seq("circe-core", "circe-generic", "circe-parser")
+  .map("io.circe" %% _ % circeVersion)
 
 def akkaPkg(name: String)     = akkaURL %% s"akka-$name" % akkaVersion
 def akkaHTTPPkg(name: String) = akkaURL %% s"akka-$name" % akkaHTTPVersion
@@ -185,7 +187,7 @@ lazy val unused = (project in file("common/scala/unused"))
 lazy val syntax_definition = (project in file("common/scala/syntax/definition"))
   .dependsOn(logger, flexer)
   .settings(
-    libraryDependencies ++= monocle ++ cats ++ scala_compiler ++ Seq(
+    libraryDependencies ++= monocle ++ cats ++ circe ++ scala_compiler ++ Seq(
       "com.lihaoyi" %% "scalatags" % "0.7.0"
     )
   )
@@ -230,7 +232,7 @@ lazy val syntax = (project in file("common/scala/syntax/specialization"))
     inConfig(Benchmark)(Defaults.testSettings),
     bench := (test in Benchmark).tag(Exclusive).value,
     parallelExecution in Benchmark := false,
-    libraryDependencies ++= Seq(
+    libraryDependencies ++= circe ++ Seq(
       "com.storm-enroute" %% "scalameter" % "0.17" % "bench",
       "org.scalatest"     %% "scalatest"  % "3.0.5" % Test,
       "com.lihaoyi"       %% "pprint"     % "0.5.3"
@@ -249,11 +251,21 @@ lazy val syntax = (project in file("common/scala/syntax/specialization"))
       .value
   )
 
+lazy val parser_service = (project in file("common/scala/parser-service"))
+  .dependsOn(syntax)
+  .settings(
+    libraryDependencies ++= akka,
+    mainClass := Some("org.enso.ParserServiceMain")
+  )
+
 lazy val pkg = (project in file("common/scala/pkg"))
   .settings(
     mainClass in (Compile, run) := Some("org.enso.pkg.Main"),
     version := "0.1",
-    libraryDependencies ++= circe ++ Seq("commons-io" % "commons-io" % "2.6")
+    libraryDependencies ++= circe ++ Seq(
+      "io.circe"   %% "circe-yaml" % "0.10.0", // separate from other circe deps because its independent project with its own versioning
+      "commons-io" % "commons-io"  % "2.6"
+    )
   )
 
 lazy val file_manager = (project in file("common/scala/file-manager"))

common/rust/parser/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name    = "parser"
+version = "0.1.0"
+authors = ["Michal Wawrzyniec Urbanczyk <michal.urbanczyk@luna-lang.org>"]
+edition = "2018"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+failure           = "0.1"
+matches           = "0.1"
+serde             = { version = "1.0", features = ["derive"] }
+serde_json        = "1.0"
+shrinkwraprs      = "0.2.1"
+wasm-bindgen      = "0.2"
+wasm-bindgen-test = "0.2"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+websocket = "0.23.0"

common/rust/parser/src/api.rs

@@ -0,0 +1,40 @@
+use failure::Fail;
+
+// ============
+// == Parser ==
+// ============
+
+/// Entity being able to parse Luna programs into Luna's AST.
+pub trait IsParser {
+    fn parse(&mut self, program: String) -> Result<AST>;
+}
+
+// =========
+// == AST ==
+// =========
+
+// TODO: placeholder until we have real AST, see:
+// https://github.com/luna/enso/issues/296
+pub type AST = String;
+
+// ===========
+// == Error ==
+// ===========
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    /// Error due to inner workings of the parser.
+    #[fail(display = "Internal parser error: {:?}", _0)]
+    ParsingError(String),
+    /// Error related to wrapping = communication with the parser service.
+    #[fail(display = "Interop error: {}", _0)]
+    InteropError(#[cause] Box<dyn failure::Fail>),
+}
+
+/// Wraps an arbitrary `std::error::Error` as an `InteropError.`
+pub fn interop_error<T>(error: T) -> Error
+    where T: Fail {
+    Error::InteropError(Box::new(error))
+}

common/rust/parser/src/jsclient.rs

@@ -0,0 +1,30 @@
+use crate::{api, api::IsParser};
+use failure::Fail;
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "JS parser client has not been yet implemented!")]
+    NotImplemented,
+}
+
+/// Wrapper over the JS-compiled parser.
+///
+/// Can only be used when targeting WebAssembly. Not yet implemented.
+pub struct Client {}
+
+impl Client {
+    // avoid warnings when compiling natively and having this usage cfg-ed out
+    #[cfg(not(target_arch = "wasm32"))]
+    #[allow(dead_code)]
+    pub fn new() -> Result<Client> {
+        Err(Error::NotImplemented)
+    }
+}
+
+impl IsParser for Client {
+    fn parse(&mut self, _program: String) -> api::Result<api::AST> {
+        Err(api::interop_error(Error::NotImplemented))
+    }
+}

common/rust/parser/src/lib.rs

@@ -0,0 +1,45 @@
+pub mod api;
+
+mod jsclient;
+mod wsclient;
+
+use std::ops::DerefMut;
+
+/// Handle to a parser implementation.
+///
+/// Currently this component is implemented as a wrapper over parser written
+/// in Scala. Depending on compilation target (native or wasm) it uses either
+/// implementation provided by `wsclient` or `jsclient`.
+#[derive(shrinkwraprs::Shrinkwrap)]
+#[shrinkwrap(mutable)]
+pub struct Parser(pub Box<dyn api::IsParser>);
+
+impl Parser {
+    /// Obtains a default parser implementation.
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn new() -> api::Result<Parser> {
+        let client = wsclient::Client::new()?;
+        let parser = Box::new(client);
+        Ok(Parser(parser))
+    }
+
+    /// Obtains a default parser implementation.
+    #[cfg(target_arch = "wasm32")]
+    pub fn new() -> api::Result<ParserWrapper> {
+        let client = jsclient::Client::new()?;
+        let parser = Box::new(client);
+        Ok(parser)
+    }
+
+    /// Obtains a default parser implementation, panicking in case of failure.
+    pub fn new_or_panic() -> Parser {
+        Parser::new()
+            .unwrap_or_else(|e| panic!("Failed to create a parser: {:?}", e))
+    }
+}
+
+impl api::IsParser for Parser {
+    fn parse(&mut self, program: String) -> api::Result<api::AST> {
+        self.deref_mut().parse(program)
+    }
+}

common/rust/parser/src/main.rs

@@ -0,0 +1,16 @@
+use parser::api::IsParser;
+
+/// Simple interactive tester - calls parser with its argument (or a
+/// hardcoded default) and prints the result
+fn main() {
+    let default_input = String::from("import Foo.Bar\nfoo = a + 2");
+    let program = std::env::args().nth(1).unwrap_or(default_input);
+    println!("Will parse: {}", program);
+
+    let mut parser = parser::Parser::new_or_panic();
+    let     output = parser.parse(program);
+    match output {
+        Ok(result) => println!("Parser responded with: {:?}", result),
+        Err(e)     => println!("Failed to obtain a response: {:?}", e),
+    }
+}

common/rust/parser/src/wsclient.rs

@@ -0,0 +1,208 @@
+#![cfg(not(target_arch = "wasm32"))]
+
+use failure::Fail;
+use std::default::Default;
+use websocket::{
+    stream::sync::TcpStream, ClientBuilder, Message, OwnedMessage,
+};
+
+use crate::api;
+use api::Error::*;
+use Error::*;
+
+type WsTcpClient = websocket::sync::Client<TcpStream>;
+
+// ==========================
+// == Constants & literals ==
+// ==========================
+
+pub const LOCALHOST:        &str = "localhost";
+pub const DEFAULT_PORT:      i32 = 30615;
+pub const DEFAULT_HOSTNAME: &str = LOCALHOST;
+
+pub const HOSTNAME_VAR: &str = "ENSO_PARSER_HOSTNAME";
+pub const PORT_VAR:     &str = "ENSO_PARSER_PORT";
+
+// ===========
+// == Error ==
+// ===========
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "Failed to parse given address url: {}", _0)]
+    WrongUrl(#[cause] websocket::client::ParseError),
+
+    #[fail(display = "Connection error: {}", _0)]
+    ConnectivityError(#[cause] websocket::WebSocketError),
+
+    #[fail(display = "Peer has closed the connection")]
+    PeerClosedConnection,
+
+    #[fail(display = "Received non-text response: {:?}", _0)]
+    NonTextResponse(websocket::OwnedMessage),
+
+    #[fail(display = "JSON (de)serialization failed: {:?}", _0)]
+    JsonSerializationError(#[cause] serde_json::error::Error),
+}
+
+impl From<Error> for api::Error {
+    fn from(e: Error) -> Self {
+        api::interop_error(e)
+    }
+}
+impl From<websocket::client::ParseError> for Error {
+    fn from(error: websocket::client::ParseError) -> Self {
+        WrongUrl(error)
+    }
+}
+impl From<websocket::WebSocketError> for Error {
+    fn from(error: websocket::WebSocketError) -> Self {
+        ConnectivityError(error)
+    }
+}
+impl From<serde_json::error::Error> for Error {
+    fn from(error: serde_json::error::Error) -> Self {
+        JsonSerializationError(error)
+    }
+}
+
+// ==============
+// == Protocol ==
+// ==============
+
+/// All request supported by the Parser Service.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum Request {
+    ParseRequest { program: String },
+}
+
+/// All responses that Parser Service might reply with.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub enum Response {
+    Success { ast:     String },
+    Error   { message: String },
+}
+
+// ============
+// == Config ==
+// ============
+
+/// Describes a WS endpoint.
+pub struct Config {
+    pub host: String,
+    pub port: i32,
+}
+
+impl Config {
+    /// Formats URL String describing a WS endpoint.
+    pub fn address_string(&self) -> String {
+        format!("ws://{}:{}", self.host, self.port)
+    }
+
+    /// Obtains a default WS endpoint to use to connect to parser service
+    /// using environment variables or, if they are not set, hardcoded
+    /// defaults.
+    pub fn from_env() -> Config {
+        let host = env_var_or(HOSTNAME_VAR, DEFAULT_HOSTNAME);
+        let port = env_var_or(PORT_VAR, Default::default())
+            .parse()
+            .unwrap_or(DEFAULT_PORT);
+        Config { host, port }
+    }
+}
+
+pub fn env_var_or(varname: &str, default_value: &str) -> String {
+    std::env::var(varname).unwrap_or_else(|_| default_value.into())
+}
+
+// ============
+// == Client ==
+// ============
+
+/// Client to the Parser Service written in Scala.
+///
+/// Connects through WebSocket to the running service.
+pub struct Client {
+    connection: WsTcpClient,
+}
+
+mod internal {
+    use super::*;
+    impl Client {
+        /// Serializes `Request` to JSON and sends to peer as a text message.
+        pub fn send_request(&mut self, request: Request) -> Result<()> {
+            let request_txt = serde_json::to_string(&request)?;
+            let message     = Message::text(request_txt);
+            self.connection.send_message(&message)?;
+            Ok(())
+        }
+
+        /// Obtains a text message from peer and deserializes it using JSON
+        /// into a `Response`.
+        ///
+        /// Should be called exactly once after each `send_request` invocation.
+        pub fn recv_response(&mut self) -> Result<Response> {
+            let response = self.connection.recv_message()?;
+            match response {
+                OwnedMessage::Text(text) => Ok(serde_json::from_str(&text)?),
+                _                        => Err(NonTextResponse(response)),
+            }
+        }
+
+        /// Sends given `Request` to peer and receives a `Response`.
+        ///
+        /// Both request and response are exchanged in JSON using text messages
+        /// over WebSocket.
+        pub fn rpc_call(&mut self, request: Request) -> Result<Response> {
+            self.send_request(request)?;
+            self.recv_response()
+        }
+    }
+}
+
+impl Client {
+    /// Creates a new `Client` connected to the already running parser service.
+    pub fn from_conf(config: &Config) -> Result<Client> {
+        let address     = config.address_string();
+        let mut builder = ClientBuilder::new(&address)?;
+        let connection  = builder.connect_insecure()?;
+        Ok(Client { connection })
+    }
+
+    /// Creates a `Client` using configuration defined by environment or
+    /// defaults if environment is not set.
+    pub fn new() -> Result<Client> {
+        let config = Config::from_env();
+        println!("Connecting to {}", config.address_string());
+        let client = Client::from_conf(&config)?;
+        println!("Established connection with {}", config.address_string());
+        Ok(client)
+    }
+}
+
+impl api::IsParser for Client {
+    fn parse(&mut self, program: String) -> api::Result<api::AST> {
+        let request  = Request::ParseRequest { program };
+        let response = self.rpc_call(request)?;
+        match response {
+            Response::Success { ast     } => Ok(ast),
+            Response::Error   { message } => Err(ParsingError(message)),
+        }
+    }
+}
+
+// ===========
+// == tests ==
+// ===========
+
+
+#[test]
+fn wrong_url_reported() {
+    let invalid_hostname    = String::from("bgjhkb 7");
+    let wrong_config        = Config { host: invalid_hostname, port: 8080 };
+    let client              = Client::from_conf(&wrong_config);
+    let got_wrong_url_error = matches::matches!(client, Err(WrongUrl(_)));
+    assert!(got_wrong_url_error, "expected WrongUrl error");
+}

common/scala/parser-service/src/main/scala/org/enso/ParserService.scala

@@ -0,0 +1,68 @@
+package org.enso
+
+import io.circe.syntax._
+import io.circe.generic.auto._
+import org.enso.flexer.Reader
+import org.enso.parserservice.Protocol
+import org.enso.parserservice.Server
+import org.enso.syntax.text.AST
+import org.enso.syntax.text.Parser
+
+import scala.util.Try
+
+object ParserService {
+  val HOSTNAME_VAR = "ENSO_PARSER_HOSTNAME"
+  val PORT_VAR     = "ENSO_PARSER_PORT"
+
+  val DEFAULT_PORT     = 30615
+  val DEFAULT_HOSTNAME = "localhost"
+
+  /** Obtains configuration from environment, filling missing values with
+    * defaults.
+    */
+  def configFromEnv(): Server.Config = {
+    val hostname = sys.env.getOrElse(HOSTNAME_VAR, DEFAULT_HOSTNAME)
+    val port = sys.env
+      .get(PORT_VAR)
+      .flatMap(str => Try { str.toInt }.toOption)
+      .getOrElse(DEFAULT_PORT)
+    Server.Config(hostname, port)
+  }
+}
+
+/** Class that allows setting up parser service with given configuration. */
+case class ParserService() extends Server with Protocol {
+  import parserservice._
+  import Protocol._
+
+  def serializeAst(ast: AST.Module): String =
+    ast.asJson.noSpaces
+
+  def runParser(program: String): AST =
+    new Parser().run(new Reader(program))
+
+  def handleRequest(request: Request): Response = {
+    request match {
+      case ParseRequest(program) =>
+        val ast = runParser(program)
+        Protocol.Success(ast.asJson.toString())
+      case _ =>
+        throw new Exception(f"unimplemented request: $request")
+    }
+  }
+}
+
+/** Runs a simple WebSocket server that wraps Parser into a service. */
+object ParserServiceMain extends App {
+  import ParserService._
+  println("Getting configuration from environment...")
+  val config = configFromEnv()
+
+  println(s"Will serve ${config.addressString()}")
+  println(
+    s"To change configuration, restart with $HOSTNAME_VAR or " +
+    s"$PORT_VAR variables set to desired values"
+  )
+  val service = ParserService()
+  service.start(config)
+}

common/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala

@@ -0,0 +1,45 @@
+package org.enso.parserservice
+
+import io.circe.parser._
+import io.circe.generic.auto._
+import io.circe.syntax._
+
+/** Types implementing parser server protocol.
+  *
+  * The protocol always is single request -> single response.
+  */
+object Protocol {
+  sealed trait Request
+  final case class ParseRequest(program: String) extends Request
+
+  sealed trait Response
+  final case class Success(ast: String)   extends Response
+  final case class Error(message: String) extends Response
+}
+
+/** Helper for implementing protocol over text-based transport.
+  *
+  * Requests and responses are marshaled as text using JSON
+  * (and default circe serialzation schema).
+  */
+trait Protocol {
+  import Protocol._
+
+  /** Generate [[Response]] for a given [[Request]].
+    *
+    * Any [[Throwable]] thrown out of implementation will be translated into
+    * [[Protocol.Error]] message.
+    */
+  def handleRequest(request: Request): Response
+
+  def handleMessage(input: String): String = {
+    try {
+      decode[Request](input) match {
+        case Left(err)      => throw err
+        case Right(request) => handleRequest(request).asJson.toString()
+      }
+    } catch {
+      case e: Throwable => (Error(e.toString): Response).asJson.noSpaces
+    }
+  }
+}

common/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala

@@ -0,0 +1,109 @@
+package org.enso.parserservice
+
+import akka.NotUsed
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.HttpMethods.GET
+import akka.http.scaladsl.model.HttpRequest
+import akka.http.scaladsl.model.HttpResponse
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.model.ws.BinaryMessage
+import akka.http.scaladsl.model.ws.Message
+import akka.http.scaladsl.model.ws.TextMessage
+import akka.http.scaladsl.model.ws.UpgradeToWebSocket
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.Flow
+import akka.stream.scaladsl.Sink
+import akka.stream.scaladsl.Source
+
+import scala.concurrent.ExecutionContext
+import scala.util.Failure
+import scala.util.Success
+
+object Server {
+
+  /** Describes endpoint to which [[Server]] can bind. */
+  final case class Config(interface: String, port: Int) {
+    def addressString(): String = s"ws://$interface:$port"
+  }
+}
+
+/** WebSocket server supporting synchronous request-response protocol.
+  *
+  * Server when run binds to endpoint and accepts establishing web socket
+  * connection for any number of peers.
+  *
+  * Server replies to each incoming text message with a single text message.
+  * Server accepts a single Text Message from a peer and responds with
+  * another Text Message.
+  */
+trait Server {
+  implicit val system: ActorSystem             = ActorSystem()
+  implicit val materializer: ActorMaterializer = ActorMaterializer()
+
+  /** Generate text reply for given request text message. */
+  def handleMessage(input: String): String
+
+  /** Akka stream defining server behavior.
+    *
+    * Incoming [[TextMessage]]s are replied to (see [[handleMessage]]).
+    * Incoming binary messages are ignored.
+    */
+  val handlerFlow: Flow[Message, TextMessage.Strict, NotUsed] =
+    Flow[Message]
+      .flatMapConcat {
+        case tm: TextMessage =>
+          val strict = tm.textStream.fold("")(_ + _)
+          strict.map(input => TextMessage(handleMessage(input)))
+        case bm: BinaryMessage =>
+          bm.dataStream.runWith(Sink.ignore)
+          Source.empty
+      }
+
+  /** Server behavior upon receiving HTTP request.
+    *
+    * As server implements websocket-based protocol, this implementation accepts
+    * only GET requests to set up WebSocket connection.
+    *
+    * The request's URI is not checked.
+    */
+  val handleRequest: HttpRequest => HttpResponse = {
+    case req @ HttpRequest(GET, _, _, _, _) =>
+      req.header[UpgradeToWebSocket] match {
+        case Some(upgrade) =>
+          println("Establishing a new connection")
+          upgrade.handleMessages(handlerFlow)
+        case None =>
+          HttpResponse(
+            StatusCodes.BadRequest,
+            entity = "Not a valid websocket request!"
+          )
+      }
+    case r: HttpRequest =>
+      r.discardEntityBytes()
+      HttpResponse(StatusCodes.MethodNotAllowed)
+  }
+
+  /** Starts a HTTP server listening at the given endpoint.
+    *
+    * Function is asynchronous, will return immediately. If the server fails to
+    * start, function will exit the process with a non-zero code.
+    */
+  def start(config: Server.Config): Unit = {
+    val bindingFuture =
+      Http().bindAndHandleSync(
+        handleRequest,
+        interface = config.interface,
+        port      = config.port
+      )
+
+    bindingFuture.onComplete({
+      case Success(_) =>
+        println(s"Server online at ${config.addressString()}")
+      case Failure(exception) =>
+        println(s"Failed to start server: $exception")
+        system.terminate()
+        System.exit(1)
+    })(ExecutionContext.global)
+  }
+}

common/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala

@@ -1,6 +1,9 @@
 package org.enso.data
 
-case class Tree[K, V](value: Option[V], branches: Map[K, Tree[K, V]]) {
+import io.circe.Decoder
+import io.circe.Encoder
+
+final case class Tree[K, V](value: Option[V], branches: Map[K, Tree[K, V]]) {
   def +(item: (List[K], V)): Tree[K, V] = item._1 match {
     case Nil => this.copy(value = Some(item._2))
     case p :: ps => {
@@ -34,4 +37,27 @@ object Tree {
   def apply[K, V](): Tree[K, V] = new Tree(None, Map())
   def apply[K, V](deps: (List[K], V)*): Tree[K, V] =
     deps.foldLeft(Tree[K, V]())(_ + _)
+
+  //////////////////
+  // JSON support //
+  //////////////////
+
+  /* Note [Tree Serialization] */
+  implicit def jsonEncode[K: Encoder, V: Encoder]: Encoder[Tree[K, V]] =
+    Encoder.forProduct2("value", "branches")(
+      tree => tree.value -> tree.branches.toSeq
+    )
+
+  /* Note [Tree Serialization]
+   * We can't directly serialize Map[K,V], as circe tries to use whole K as a
+   * key string in the generated JSON. Thus, we serialize Map[K, V] by
+   * converting it to Seq[(K,V)] first.
+   */
+
+  /* Note [Tree Serialization] */
+  implicit def jsonDecode[K: Decoder, V: Decoder]: Decoder[Tree[K, V]] =
+    Decoder.forProduct2("value", "branches")(
+      (value: Option[V], branches: Seq[(K, Tree[K, V])]) =>
+        Tree(value, branches.toMap)
+    )
 }

common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala

@@ -2,11 +2,14 @@ package org.enso.syntax.text
 
 import java.util.UUID
 
-import shapeless.Id
 import cats.Foldable
 import cats.Functor
 import cats.derived._
 import cats.implicits._
+import io.circe.Encoder
+import io.circe.Json
+import io.circe.generic.AutoDerivation
+import io.circe.generic.auto._
 import org.enso.data.List1._
 import org.enso.data.Index
 import org.enso.data.List1
@@ -275,44 +278,18 @@ object AST {
     def mapWithOff(f: (Index, AST) => AST): ASTOf[T] =
       copy(shape = cls.mapWithOff(shape)(f))
     def zipWithOffset(): T[(Index, AST)] = cls.zipWithOffset(shape)
+    def encodeShape():   Json            = cls.encode(shape)
   }
-  object ASTOf {
+  object ASTOf extends AutoDerivation {
     implicit def repr[T[_]]:                Repr[ASTOf[T]] = _.repr
     implicit def unwrap[T[_]](t: ASTOf[T]): T[AST]         = t.shape
     implicit def wrap[T[_]](t: T[AST])(
       implicit
       ev: ASTClass[T]
     ): ASTOf[T] = ASTOf(t)
-  }
 
-  //// ASTClass ////
-
-  /** [[ASTClass]] implements set of AST operations based on a precise AST
-    * shape. Because the [[T]] parameter in [[ASTOf]] is covariant, we may lose
-    * information about the shape after we construct the AST, thus this instance
-    * is used to cache all necessary operations during AST construction.
-    */
-  trait ASTClass[T[_]] {
-    def repr(t: T[AST]):                               Repr.Builder
-    def map(t: T[AST])(f: AST => AST):                 T[AST]
-    def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST]
-    def zipWithOffset(t: T[AST]):                      T[(Index, AST)]
-  }
-  object ASTClass {
-    def apply[T[_]](implicit cls: ASTClass[T]): ASTClass[T] = cls
-    implicit def instance[T[_]](
-      implicit
-      evRepr: Repr[T[AST]],
-      evFtor: Functor[T],
-      evOzip: OffsetZip[T, AST]
-    ): ASTClass[T] =
-      new ASTClass[T] {
-        def repr(t: T[AST]): Repr.Builder             = evRepr.repr(t)
-        def map(t: T[AST])(f: AST => AST): T[AST]     = Functor[T].map(t)(f)
-        def zipWithOffset(t: T[AST]): T[(Index, AST)] = OffsetZip(t)
-        def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST] =
-          Functor[T].map(zipWithOffset(t))(f.tupled)
-      }
+    implicit def jsonEncoder[T[_]]: Encoder[ASTOf[T]] =
+      Encoder.forProduct2("shape", "id")(ast => ast.encodeShape() -> ast.id)
   }
 
   //// ASTOps ////
@@ -352,7 +329,7 @@ object AST {
     * a type is phantom, then its last type argument is not used and we can
     * safely coerce it to something else.
     */
-  trait Phantom
+  sealed trait Phantom
   implicit class PhantomOps[T[_] <: Phantom](ident: T[_]) {
     def coerce[S]: T[S] = ident.asInstanceOf[T[S]]
   }
@@ -689,15 +666,36 @@ object AST {
           with Phantom
 
       object Line {
-        final case class Raw[T](text: List[Segment._Raw[T]])
+        /* Note [Circe and naming] */
+        val Raw = LineRaw
+        /* Note [Circe and naming] */
+        type Raw[T] = LineRaw[T]
+
+        /* Note [Circe and naming] */
+        val Fmt = LineFmt
+        /* Note [Circe and naming] */
+        type Fmt[T] = LineFmt[T]
+
+        /* Note [Circe and naming] */
+        final case class LineRaw[T](text: List[Segment._Raw[T]])
             extends Line[T]
             with Phantom {
           val quote = '"'
         }
-        final case class Fmt[T](text: List[Segment._Fmt[T]]) extends Line[T] {
+        /* Note [Circe and naming] */
+        final case class LineFmt[T](text: List[Segment._Fmt[T]])
+            extends Line[T] {
           val quote = '\''
         }
 
+        /* Note [Circe and naming]
+         * ~~~~~~~~~~~~~~~~~~~~~~~~
+         * To be able to use Circe automatic derivation for traits, all case
+         * classes in its subtree must bear unique names. So `Line.Fmt` and
+         * `Line.Raw` cannot be used as they would collide with [[Block.Raw]]
+         * and [[Block.Fmt]].
+         */
+
         ////// INSTANCES /////
         import Segment.implicits._
 
@@ -730,7 +728,7 @@ object AST {
             with Phantom {
           val quote = "\"\"\""
         }
-        case class Fmt[T](
+        final case class Fmt[T](
           text: List[Line[Segment._Fmt[T]]],
           spaces: Int,
           offset: Int
@@ -808,7 +806,8 @@ object AST {
         Text(Block.Fmt(line.to[List], spaces, off))
 
       object Raw {
-        def apply(segment: Segment.Raw*): Text = Text(Line.Raw(segment.to[List]))
+        def apply(segment: Segment.Raw*): Text =
+          Text(Line.Raw(segment.to[List]))
         def apply(spaces: Int, off: Int, line: Block.Line[Segment.Raw]*): Text =
           Text(Block.Raw(line.to[List], spaces, off))
       }
@@ -1538,7 +1537,7 @@ object AST {
   //////////////////////////////////////////////////////////////////////////////
 
   type Documented = ASTOf[DocumentedOf]
-  case class DocumentedOf[T](doc: Doc, emptyLinesBetween: Int, ast: T)
+  final case class DocumentedOf[T](doc: Doc, emptyLinesBetween: Int, ast: T)
       extends ShapeOf[T]
   object Documented {
     val any = UnapplyByType[Documented]
@@ -1560,6 +1559,9 @@ object AST {
     }
     implicit def offsetZip[T]: OffsetZip[DocumentedOf, T] =
       _.map(Index.Start -> _)
+
+    implicit def toJson[T]: Encoder[DocumentedOf[T]] =
+      _ => throw new NotImplementedError()
   }
 
   //////////////////////////////////////////////////////////////////////////////
@@ -1688,6 +1690,41 @@ object AST {
     implicit def ozip[T]: OffsetZip[ForeignOf, T] = _.map(Index.Start -> _)
   }
 
+  //// ASTClass ////
+
+  /** [[ASTClass]] implements set of AST operations based on a precise AST
+    * shape. Because the [[T]] parameter in [[ASTOf]] is covariant, we may lose
+    * information about the shape after we construct the AST, thus this instance
+    * is used to cache all necessary operations during AST construction.
+    */
+  sealed trait ASTClass[T[_]] {
+    def repr(t: T[AST]): Repr.Builder
+    def map(t: T[AST])(f: AST => AST): T[AST]
+    def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST]
+    def zipWithOffset(t: T[AST]): T[(Index, AST)]
+    def encode(t: T[AST]): Json
+  }
+  object ASTClass {
+    def apply[T[_]](implicit cls: ASTClass[T]): ASTClass[T] = cls
+    implicit def instance[T[S] <: ShapeOf[S]](
+      implicit
+      evRepr: Repr[T[AST]],
+      evFtor: Functor[T],
+      evOzip: OffsetZip[T, AST]
+    ): ASTClass[T] =
+      new ASTClass[T] {
+        def repr(t: T[AST]):               Repr.Builder    = evRepr.repr(t)
+        def map(t: T[AST])(f: AST => AST): T[AST]          = Functor[T].map(t)(f)
+        def zipWithOffset(t: T[AST]):      T[(Index, AST)] = OffsetZip(t)
+        def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST] =
+          Functor[T].map(zipWithOffset(t))(f.tupled)
+        def encode(t: T[AST]): Json = {
+          val shapeEncoder = implicitly[Encoder[ShapeOf[AST]]]
+          shapeEncoder(t)
+        }
+      }
+  }
+
   /////////////////////////////////////////////////
   /////////////////////////////////////////////////
   /////////////////////////////////////////////////

common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala

@@ -5,7 +5,8 @@ import org.enso.syntax.text.AST.SAST
 import org.enso.syntax.text.prec.Operator
 
 import scala.annotation.tailrec
-import org.enso.data.{Index, Shifted}
+import org.enso.data.Index
+import org.enso.data.Shifted
 import org.enso.syntax.text.ast.Repr
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -13,7 +14,9 @@ import org.enso.syntax.text.ast.Repr
 ////////////////////////////////////////////////////////////////////////////////
 
 object Pattern {
-  import cats.{Foldable, Functor, Traverse}
+  import cats.Foldable
+  import cats.Functor
+  import cats.Traverse
   import cats.derived._
 
   type P      = Pattern
@@ -34,7 +37,7 @@ object Pattern {
     (nStream.reverse, nOff)
   }
 
-  trait Class
+  sealed trait Class
   object Class {
     final case object Normal  extends Class
     final case object Pattern extends Class
@@ -195,7 +198,7 @@ object Pattern {
 
     //// Result ////
 
-    case class Result(elem: Match, stream: AST.Stream) {
+    final case class Result(elem: Match, stream: AST.Stream) {
       def map(fn: Match => Match): Result = copy(elem = fn(elem))
     }
 

common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala

@@ -1,6 +1,7 @@
 package org.enso.syntax.text.ast.text
 
 import org.enso.flexer.ADT
+import org.enso.syntax.text.ast.text.Escape.Slash.toString
 
 sealed trait Escape {
   val repr: String
@@ -8,7 +9,6 @@ sealed trait Escape {
 
 object Escape {
 
-
   final case class Invalid(str: String) extends Escape {
     val repr = str
   }
@@ -21,18 +21,46 @@ object Escape {
   sealed trait Unicode extends Escape
   object Unicode {
 
-    final case class Invalid(unicode: Unicode) extends Unicode {
+    /* Note [Circe and Naming] */
+    type Invalid = InvalidUnicode
+
+    /* Note [Circe and Naming] */
+    val Invalid = InvalidUnicode
+
+    /* Note [Circe and Naming] */
+    final case class InvalidUnicode(unicode: Unicode) extends Unicode {
       val repr = unicode.repr
     }
 
-    abstract class U(pfx: String, sfx: String = "") extends Unicode {
-      val digits: String
+    /* NOTE [Circe and Naming]
+     * Name of the class above cannot be Invalid, as we already have
+     * Escape.Invalid. And to be able to derive JSON serialization with circe
+     * case class names within a trait subtree need to be unique.
+     *
+     * To keep this unpleasant detail hidden from library users, we introduce
+     * aliases for type and object named `Invalid`.
+     */
+
+    type U16 = _U16
+    final case class _U16(digits: String) extends Unicode {
+      val pfx  = "u"
+      val sfx  = ""
       val repr = pfx + digits + sfx
     }
 
-    final case class U16 private (digits: String) extends U("u")
-    final case class U32 private (digits: String) extends U("U")
-    final case class U21 private (digits: String) extends U("u{", "}")
+    type U32 = _U32
+    final case class _U32(digits: String) extends Unicode {
+      val pfx  = "U"
+      val sfx  = ""
+      val repr = pfx + digits + sfx
+    }
+
+    type U21 = _U21
+    final case class _U21(digits: String) extends Unicode {
+      val pfx  = "u{"
+      val sfx  = "}"
+      val repr = pfx + digits + sfx
+    }
 
     object Validator {
       val hexChars =
@@ -43,8 +71,8 @@ object Escape {
 
     object U16 {
       def apply(digits: String): Unicode =
-        if (validate(digits)) U16(digits)
-        else Invalid(U16(digits))
+        if (validate(digits)) _U16(digits)
+        else Invalid(_U16(digits))
       def validate(digits: String) = {
         import Validator._
         val validLength = digits.length == 4
@@ -54,8 +82,8 @@ object Escape {
     }
     object U32 {
       def apply(digits: String): Unicode =
-        if (validate(digits)) U32(digits)
-        else Invalid(U32(digits))
+        if (validate(digits)) _U32(digits)
+        else Invalid(_U32(digits))
       def validate(digits: String) = {
         import Validator._
         val validLength = digits.length == 8
@@ -66,8 +94,8 @@ object Escape {
     }
     object U21 {
       def apply(digits: String): Unicode =
-        if (validate(digits)) U21(digits)
-        else Invalid(U21(digits))
+        if (validate(digits)) _U21(digits)
+        else Invalid(_U21(digits))
       def validate(digits: String) = {
         import Validator._
         val validLength = digits.length >= 1 && digits.length <= 6
@@ -77,66 +105,236 @@ object Escape {
     }
   }
 
-
-  abstract class Simple(val code: Int) extends Escape{
-    def name = toString
-    val repr = name
+  case object Slash extends Escape {
+    val code: Int     = '\\'
+    def name: String  = toString
+    override val repr = "\\"
+  }
+  case object Quote extends Escape {
+    val code: Int     = '\''
+    def name: String  = toString
+    override val repr = "\'"
+  }
+  case object RawQuote extends Escape {
+    val code: Int     = '"'
+    def name: String  = toString
+    override val repr = "\""
   }
 
-  case object Slash    extends Simple('\\') { override val repr = "\\" }
-  case object Quote    extends Simple('\'') { override val repr = "\'" }
-  case object RawQuote extends Simple('"')  { override val repr = "\"" }
-
   // Reference: https://en.wikipedia.org/wiki/String_literal
-  sealed trait Character extends Simple
+  sealed trait Character extends Escape
   object Character {
-    case object a extends Simple('\u0007') with Character
-    case object b extends Simple('\u0008') with Character
-    case object f extends Simple('\u000C') with Character
-    case object n extends Simple('\n') with Character
-    case object r extends Simple('\r') with Character
-    case object t extends Simple('\u0009') with Character
-    case object v extends Simple('\u000B') with Character
-    case object e extends Simple('\u001B') with Character
+    case object a extends Character {
+      val code: Int     = '\u0007'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object b extends Character {
+      val code: Int     = '\u0008'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object f extends Character {
+      val code: Int     = '\u000C'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object n extends Character {
+      val code: Int     = '\n'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object r extends Character {
+      val code: Int     = '\r'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object t extends Character {
+      val code: Int     = '\u0009'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object v extends Character {
+      val code: Int     = '\u000B'
+      def name: String  = toString
+      override val repr = name
+    }
+    case object e extends Character {
+      val code: Int     = '\u001B'
+      def name: String  = toString
+      override val repr = name
+    }
     val codes = ADT.constructors[Character]
   }
 
   // Reference: https://en.wikipedia.org/wiki/Control_character
-  sealed trait Control extends Simple
+  sealed trait Control extends Escape
   object Control {
-    case object NUL extends Simple(0x00) with Control
-    case object SOH extends Simple(0x01) with Control
-    case object STX extends Simple(0x02) with Control
-    case object ETX extends Simple(0x03) with Control
-    case object EOT extends Simple(0x04) with Control
-    case object ENQ extends Simple(0x05) with Control
-    case object ACK extends Simple(0x06) with Control
-    case object BEL extends Simple(0x07) with Control
-    case object BS  extends Simple(0x08) with Control
-    case object TAB extends Simple(0x09) with Control
-    case object LF  extends Simple(0x0A) with Control
-    case object VT  extends Simple(0x0B) with Control
-    case object FF  extends Simple(0x0C) with Control
-    case object CR  extends Simple(0x0D) with Control
-    case object SO  extends Simple(0x0E) with Control
-    case object SI  extends Simple(0x0F) with Control
-    case object DLE extends Simple(0x10) with Control
-    case object DC1 extends Simple(0x11) with Control
-    case object DC2 extends Simple(0x12) with Control
-    case object DC3 extends Simple(0x13) with Control
-    case object DC4 extends Simple(0x14) with Control
-    case object NAK extends Simple(0x15) with Control
-    case object SYN extends Simple(0x16) with Control
-    case object ETB extends Simple(0x17) with Control
-    case object CAN extends Simple(0x18) with Control
-    case object EM  extends Simple(0x19) with Control
-    case object SUB extends Simple(0x1A) with Control
-    case object ESC extends Simple(0x1B) with Control
-    case object FS  extends Simple(0x1C) with Control
-    case object GS  extends Simple(0x1D) with Control
-    case object RS  extends Simple(0x1E) with Control
-    case object US  extends Simple(0x1F) with Control
-    case object DEL extends Simple(0x7F) with Control
+    case object NUL extends Control {
+      val code: Int     = 0x00
+      def name: String  = toString
+      override val repr = name
+    }
+    case object SOH extends Control {
+      val code: Int     = 0x01
+      def name: String  = toString
+      override val repr = name
+    }
+    case object STX extends Control {
+      val code: Int     = 0x02
+      def name: String  = toString
+      override val repr = name
+    }
+    case object ETX extends Control {
+      val code: Int     = 0x03
+      def name: String  = toString
+      override val repr = name
+    }
+    case object EOT extends Control {
+      val code: Int     = 0x04
+      def name: String  = toString
+      override val repr = name
+    }
+    case object ENQ extends Control {
+      val code: Int     = 0x05
+      def name: String  = toString
+      override val repr = name
+    }
+    case object ACK extends Control {
+      val code: Int     = 0x06
+      def name: String  = toString
+      override val repr = name
+    }
+    case object BEL extends Control {
+      val code: Int     = 0x07
+      def name: String  = toString
+      override val repr = name
+    }
+    case object BS extends Control {
+      val code: Int     = 0x08
+      def name: String  = toString
+      override val repr = name
+    }
+    case object TAB extends Control {
+      val code: Int     = 0x09
+      def name: String  = toString
+      override val repr = name
+    }
+    case object LF extends Control {
+      val code: Int     = 0x0A
+      def name: String  = toString
+      override val repr = name
+    }
+    case object VT extends Control {
+      val code: Int     = 0x0B
+      def name: String  = toString
+      override val repr = name
+    }
+    case object FF extends Control {
+      val code: Int     = 0x0C
+      def name: String  = toString
+      override val repr = name
+    }
+    case object CR extends Control {
+      val code: Int     = 0x0D
+      def name: String  = toString
+      override val repr = name
+    }
+    case object SO extends Control {
+      val code: Int     = 0x0E
+      def name: String  = toString
+      override val repr = name
+    }
+    case object SI extends Control {
+      val code: Int     = 0x0F
+      def name: String  = toString
+      override val repr = name
+    }
+    case object DLE extends Control {
+      val code: Int     = 0x10
+      def name: String  = toString
+      override val repr = name
+    }
+    case object DC1 extends Control {
+      val code: Int     = 0x11
+      def name: String  = toString
+      override val repr = name
+    }
+    case object DC2 extends Control {
+      val code: Int     = 0x12
+      def name: String  = toString
+      override val repr = name
+    }
+    case object DC3 extends Control {
+      val code: Int     = 0x13
+      def name: String  = toString
+      override val repr = name
+    }
+    case object DC4 extends Control {
+      val code: Int     = 0x14
+      def name: String  = toString
+      override val repr = name
+    }
+    case object NAK extends Control {
+      val code: Int     = 0x15
+      def name: String  = toString
+      override val repr = name
+    }
+    case object SYN extends Control {
+      val code: Int     = 0x16
+      def name: String  = toString
+      override val repr = name
+    }
+    case object ETB extends Control {
+      val code: Int     = 0x17
+      def name: String  = toString
+      override val repr = name
+    }
+    case object CAN extends Control {
+      val code: Int     = 0x18
+      def name: String  = toString
+      override val repr = name
+    }
+    case object EM extends Control {
+      val code: Int     = 0x19
+      def name: String  = toString
+      override val repr = name
+    }
+    case object SUB extends Control {
+      val code: Int     = 0x1A
+      def name: String  = toString
+      override val repr = name
+    }
+    case object ESC extends Control {
+      val code: Int     = 0x1B
+      def name: String  = toString
+      override val repr = name
+    }
+    case object FS extends Control {
+      val code: Int     = 0x1C
+      def name: String  = toString
+      override val repr = name
+    }
+    case object GS extends Control {
+      val code: Int     = 0x1D
+      def name: String  = toString
+      override val repr = name
+    }
+    case object RS extends Control {
+      val code: Int     = 0x1E
+      def name: String  = toString
+      override val repr = name
+    }
+    case object US extends Control {
+      val code: Int     = 0x1F
+      def name: String  = toString
+      override val repr = name
+    }
+    case object DEL extends Control {
+      val code: Int     = 0x7F
+      def name: String  = toString
+      override val repr = name
+    }
     val codes = ADT.constructors[Control]
   }
 }

common/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala

@@ -1,6 +1,7 @@
 package org.enso.syntax.text
 
-import org.enso.data.{Index, Span}
+import org.enso.data.Index
+import org.enso.data.Span
 import org.enso.flexer
 import org.enso.flexer.Reader
 import org.enso.syntax.text.ast.meta.Builtin