From 6078b54f50a9ccee7b2b2504f8f5399608e8b059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wawrzyniec=20Urba=C5=84czyk?= Date: Mon, 18 Nov 2019 14:12:16 +0100 Subject: [PATCH] Wrapping parser for Rust (#325) * deriving JSON Encoder for Scala AST types * websocket-based Parser Service * wrapper for Parser in Rust that includes client for Parser Service --- .gitignore | 6 + Cargo.toml | 5 + build.sbt | 22 +- common/rust/.gitkeep | 0 common/rust/parser/Cargo.toml | 20 + common/rust/parser/src/api.rs | 40 ++ common/rust/parser/src/jsclient.rs | 30 ++ common/rust/parser/src/lib.rs | 45 +++ common/rust/parser/src/main.rs | 16 + common/rust/parser/src/wsclient.rs | 208 +++++++++++ .../main/scala/org/enso/ParserService.scala | 68 ++++ .../org/enso/parserservice/Protocol.scala | 45 +++ .../scala/org/enso/parserservice/Server.scala | 109 ++++++ .../src/main/scala/org/enso/data/Tree.scala | 28 +- .../main/scala/org/enso/syntax/text/AST.scala | 353 ++++++++++-------- .../enso/syntax/text/ast/meta/Pattern.scala | 11 +- .../enso/syntax/text/ast/text/Escape.scala | 328 ++++++++++++---- .../scala/org/enso/syntax/text/Parser.scala | 5 +- .../org/enso/syntax/text/ParserTest.scala | 6 +- 19 files changed, 1107 insertions(+), 238 deletions(-) create mode 100644 Cargo.toml delete mode 100644 common/rust/.gitkeep create mode 100644 common/rust/parser/Cargo.toml create mode 100644 common/rust/parser/src/api.rs create mode 100644 common/rust/parser/src/jsclient.rs create mode 100644 common/rust/parser/src/lib.rs create mode 100644 common/rust/parser/src/main.rs create mode 100644 common/rust/parser/src/wsclient.rs create mode 100644 common/scala/parser-service/src/main/scala/org/enso/ParserService.scala create mode 100644 common/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala create mode 100644 common/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala diff --git a/.gitignore b/.gitignore index 09bfbd5e267..ded8317eeef 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,12 @@ target/ *.class *.log +########## +## Rust ## +########## + +Cargo.lock + ############# ## Haskell ## ############# diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000000..e56c8faf5a7 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] + +members = [ + "common/rust/parser" +] \ No newline at end of file diff --git a/build.sbt b/build.sbt index de5110f2105..d83f8e1bafd 100644 --- a/build.sbt +++ b/build.sbt @@ -10,6 +10,7 @@ import sbtassembly.AssemblyPlugin.defaultUniversalScript val scalacVersion = "2.12.10" val graalVersion = "19.2.0.1" +val circeVersion = "0.11.1" organization in ThisBuild := "org.enso" scalaVersion in ThisBuild := scalacVersion @@ -98,6 +99,7 @@ lazy val enso = (project in file(".")) pkg, project_manager, runtime, + parser_service, syntax, syntax_definition, unused @@ -131,8 +133,8 @@ val scala_compiler = Seq( "org.scala-lang" % "scala-compiler" % scalacVersion ) -val circe = Seq("circe-core", "circe-generic", "circe-yaml") - .map("io.circe" %% _ % "0.10.0") +val circe = Seq("circe-core", "circe-generic", "circe-parser") + .map("io.circe" %% _ % circeVersion) def akkaPkg(name: String) = akkaURL %% s"akka-$name" % akkaVersion def akkaHTTPPkg(name: String) = akkaURL %% s"akka-$name" % akkaHTTPVersion @@ -185,7 +187,7 @@ lazy val unused = (project in file("common/scala/unused")) lazy val syntax_definition = (project in file("common/scala/syntax/definition")) .dependsOn(logger, flexer) .settings( - libraryDependencies ++= monocle ++ cats ++ scala_compiler ++ Seq( + libraryDependencies ++= monocle ++ cats ++ circe ++ scala_compiler ++ Seq( "com.lihaoyi" %% "scalatags" % "0.7.0" ) ) @@ -230,7 +232,7 @@ lazy val syntax = (project in file("common/scala/syntax/specialization")) inConfig(Benchmark)(Defaults.testSettings), bench := (test in Benchmark).tag(Exclusive).value, parallelExecution in Benchmark := false, - libraryDependencies ++= Seq( + libraryDependencies ++= circe ++ Seq( "com.storm-enroute" %% "scalameter" % "0.17" % "bench", "org.scalatest" %% "scalatest" % "3.0.5" % Test, "com.lihaoyi" %% "pprint" % "0.5.3" @@ -249,11 +251,21 @@ lazy val syntax = (project in file("common/scala/syntax/specialization")) .value ) +lazy val parser_service = (project in file("common/scala/parser-service")) + .dependsOn(syntax) + .settings( + libraryDependencies ++= akka, + mainClass := Some("org.enso.ParserServiceMain") + ) + lazy val pkg = (project in file("common/scala/pkg")) .settings( mainClass in (Compile, run) := Some("org.enso.pkg.Main"), version := "0.1", - libraryDependencies ++= circe ++ Seq("commons-io" % "commons-io" % "2.6") + libraryDependencies ++= circe ++ Seq( + "io.circe" %% "circe-yaml" % "0.10.0", // separate from other circe deps because its independent project with its own versioning + "commons-io" % "commons-io" % "2.6" + ) ) lazy val file_manager = (project in file("common/scala/file-manager")) diff --git a/common/rust/.gitkeep b/common/rust/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/common/rust/parser/Cargo.toml b/common/rust/parser/Cargo.toml new file mode 100644 index 00000000000..19f35049cf4 --- /dev/null +++ b/common/rust/parser/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "parser" +version = "0.1.0" +authors = ["Michal Wawrzyniec Urbanczyk "] +edition = "2018" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +failure = "0.1" +matches = "0.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +shrinkwraprs = "0.2.1" +wasm-bindgen = "0.2" +wasm-bindgen-test = "0.2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +websocket = "0.23.0" diff --git a/common/rust/parser/src/api.rs b/common/rust/parser/src/api.rs new file mode 100644 index 00000000000..c0954cf5b01 --- /dev/null +++ b/common/rust/parser/src/api.rs @@ -0,0 +1,40 @@ +use failure::Fail; + +// ============ +// == Parser == +// ============ + +/// Entity being able to parse Luna programs into Luna's AST. +pub trait IsParser { + fn parse(&mut self, program: String) -> Result; +} + +// ========= +// == AST == +// ========= + +// TODO: placeholder until we have real AST, see: +// https://github.com/luna/enso/issues/296 +pub type AST = String; + +// =========== +// == Error == +// =========== + +pub type Result = std::result::Result; + +#[derive(Debug, Fail)] +pub enum Error { + /// Error due to inner workings of the parser. + #[fail(display = "Internal parser error: {:?}", _0)] + ParsingError(String), + /// Error related to wrapping = communication with the parser service. + #[fail(display = "Interop error: {}", _0)] + InteropError(#[cause] Box), +} + +/// Wraps an arbitrary `std::error::Error` as an `InteropError.` +pub fn interop_error(error: T) -> Error + where T: Fail { + Error::InteropError(Box::new(error)) +} diff --git a/common/rust/parser/src/jsclient.rs b/common/rust/parser/src/jsclient.rs new file mode 100644 index 00000000000..6fad0852a7f --- /dev/null +++ b/common/rust/parser/src/jsclient.rs @@ -0,0 +1,30 @@ +use crate::{api, api::IsParser}; +use failure::Fail; + +pub type Result = std::result::Result; + +#[derive(Debug, Fail)] +pub enum Error { + #[fail(display = "JS parser client has not been yet implemented!")] + NotImplemented, +} + +/// Wrapper over the JS-compiled parser. +/// +/// Can only be used when targeting WebAssembly. Not yet implemented. +pub struct Client {} + +impl Client { + // avoid warnings when compiling natively and having this usage cfg-ed out + #[cfg(not(target_arch = "wasm32"))] + #[allow(dead_code)] + pub fn new() -> Result { + Err(Error::NotImplemented) + } +} + +impl IsParser for Client { + fn parse(&mut self, _program: String) -> api::Result { + Err(api::interop_error(Error::NotImplemented)) + } +} diff --git a/common/rust/parser/src/lib.rs b/common/rust/parser/src/lib.rs new file mode 100644 index 00000000000..bf11c354d3b --- /dev/null +++ b/common/rust/parser/src/lib.rs @@ -0,0 +1,45 @@ +pub mod api; + +mod jsclient; +mod wsclient; + +use std::ops::DerefMut; + +/// Handle to a parser implementation. +/// +/// Currently this component is implemented as a wrapper over parser written +/// in Scala. Depending on compilation target (native or wasm) it uses either +/// implementation provided by `wsclient` or `jsclient`. +#[derive(shrinkwraprs::Shrinkwrap)] +#[shrinkwrap(mutable)] +pub struct Parser(pub Box); + +impl Parser { + /// Obtains a default parser implementation. + #[cfg(not(target_arch = "wasm32"))] + pub fn new() -> api::Result { + let client = wsclient::Client::new()?; + let parser = Box::new(client); + Ok(Parser(parser)) + } + + /// Obtains a default parser implementation. + #[cfg(target_arch = "wasm32")] + pub fn new() -> api::Result { + let client = jsclient::Client::new()?; + let parser = Box::new(client); + Ok(parser) + } + + /// Obtains a default parser implementation, panicking in case of failure. + pub fn new_or_panic() -> Parser { + Parser::new() + .unwrap_or_else(|e| panic!("Failed to create a parser: {:?}", e)) + } +} + +impl api::IsParser for Parser { + fn parse(&mut self, program: String) -> api::Result { + self.deref_mut().parse(program) + } +} diff --git a/common/rust/parser/src/main.rs b/common/rust/parser/src/main.rs new file mode 100644 index 00000000000..48eeaf85f32 --- /dev/null +++ b/common/rust/parser/src/main.rs @@ -0,0 +1,16 @@ +use parser::api::IsParser; + +/// Simple interactive tester - calls parser with its argument (or a +/// hardcoded default) and prints the result +fn main() { + let default_input = String::from("import Foo.Bar\nfoo = a + 2"); + let program = std::env::args().nth(1).unwrap_or(default_input); + println!("Will parse: {}", program); + + let mut parser = parser::Parser::new_or_panic(); + let output = parser.parse(program); + match output { + Ok(result) => println!("Parser responded with: {:?}", result), + Err(e) => println!("Failed to obtain a response: {:?}", e), + } +} diff --git a/common/rust/parser/src/wsclient.rs b/common/rust/parser/src/wsclient.rs new file mode 100644 index 00000000000..7e8c51862ac --- /dev/null +++ b/common/rust/parser/src/wsclient.rs @@ -0,0 +1,208 @@ +#![cfg(not(target_arch = "wasm32"))] + +use failure::Fail; +use std::default::Default; +use websocket::{ + stream::sync::TcpStream, ClientBuilder, Message, OwnedMessage, +}; + +use crate::api; +use api::Error::*; +use Error::*; + +type WsTcpClient = websocket::sync::Client; + +// ========================== +// == Constants & literals == +// ========================== + +pub const LOCALHOST: &str = "localhost"; +pub const DEFAULT_PORT: i32 = 30615; +pub const DEFAULT_HOSTNAME: &str = LOCALHOST; + +pub const HOSTNAME_VAR: &str = "ENSO_PARSER_HOSTNAME"; +pub const PORT_VAR: &str = "ENSO_PARSER_PORT"; + +// =========== +// == Error == +// =========== + +pub type Result = std::result::Result; + +#[derive(Debug, Fail)] +pub enum Error { + #[fail(display = "Failed to parse given address url: {}", _0)] + WrongUrl(#[cause] websocket::client::ParseError), + + #[fail(display = "Connection error: {}", _0)] + ConnectivityError(#[cause] websocket::WebSocketError), + + #[fail(display = "Peer has closed the connection")] + PeerClosedConnection, + + #[fail(display = "Received non-text response: {:?}", _0)] + NonTextResponse(websocket::OwnedMessage), + + #[fail(display = "JSON (de)serialization failed: {:?}", _0)] + JsonSerializationError(#[cause] serde_json::error::Error), +} + +impl From for api::Error { + fn from(e: Error) -> Self { + api::interop_error(e) + } +} +impl From for Error { + fn from(error: websocket::client::ParseError) -> Self { + WrongUrl(error) + } +} +impl From for Error { + fn from(error: websocket::WebSocketError) -> Self { + ConnectivityError(error) + } +} +impl From for Error { + fn from(error: serde_json::error::Error) -> Self { + JsonSerializationError(error) + } +} + +// ============== +// == Protocol == +// ============== + +/// All request supported by the Parser Service. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub enum Request { + ParseRequest { program: String }, +} + +/// All responses that Parser Service might reply with. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub enum Response { + Success { ast: String }, + Error { message: String }, +} + +// ============ +// == Config == +// ============ + +/// Describes a WS endpoint. +pub struct Config { + pub host: String, + pub port: i32, +} + +impl Config { + /// Formats URL String describing a WS endpoint. + pub fn address_string(&self) -> String { + format!("ws://{}:{}", self.host, self.port) + } + + /// Obtains a default WS endpoint to use to connect to parser service + /// using environment variables or, if they are not set, hardcoded + /// defaults. + pub fn from_env() -> Config { + let host = env_var_or(HOSTNAME_VAR, DEFAULT_HOSTNAME); + let port = env_var_or(PORT_VAR, Default::default()) + .parse() + .unwrap_or(DEFAULT_PORT); + Config { host, port } + } +} + +pub fn env_var_or(varname: &str, default_value: &str) -> String { + std::env::var(varname).unwrap_or_else(|_| default_value.into()) +} + +// ============ +// == Client == +// ============ + +/// Client to the Parser Service written in Scala. +/// +/// Connects through WebSocket to the running service. +pub struct Client { + connection: WsTcpClient, +} + +mod internal { + use super::*; + impl Client { + /// Serializes `Request` to JSON and sends to peer as a text message. + pub fn send_request(&mut self, request: Request) -> Result<()> { + let request_txt = serde_json::to_string(&request)?; + let message = Message::text(request_txt); + self.connection.send_message(&message)?; + Ok(()) + } + + /// Obtains a text message from peer and deserializes it using JSON + /// into a `Response`. + /// + /// Should be called exactly once after each `send_request` invocation. + pub fn recv_response(&mut self) -> Result { + let response = self.connection.recv_message()?; + match response { + OwnedMessage::Text(text) => Ok(serde_json::from_str(&text)?), + _ => Err(NonTextResponse(response)), + } + } + + /// Sends given `Request` to peer and receives a `Response`. + /// + /// Both request and response are exchanged in JSON using text messages + /// over WebSocket. + pub fn rpc_call(&mut self, request: Request) -> Result { + self.send_request(request)?; + self.recv_response() + } + } +} + +impl Client { + /// Creates a new `Client` connected to the already running parser service. + pub fn from_conf(config: &Config) -> Result { + let address = config.address_string(); + let mut builder = ClientBuilder::new(&address)?; + let connection = builder.connect_insecure()?; + Ok(Client { connection }) + } + + /// Creates a `Client` using configuration defined by environment or + /// defaults if environment is not set. + pub fn new() -> Result { + let config = Config::from_env(); + println!("Connecting to {}", config.address_string()); + let client = Client::from_conf(&config)?; + println!("Established connection with {}", config.address_string()); + Ok(client) + } +} + +impl api::IsParser for Client { + fn parse(&mut self, program: String) -> api::Result { + let request = Request::ParseRequest { program }; + let response = self.rpc_call(request)?; + match response { + Response::Success { ast } => Ok(ast), + Response::Error { message } => Err(ParsingError(message)), + } + } +} + +// =========== +// == tests == +// =========== + + +#[test] +fn wrong_url_reported() { + let invalid_hostname = String::from("bgjhkb 7"); + let wrong_config = Config { host: invalid_hostname, port: 8080 }; + let client = Client::from_conf(&wrong_config); + let got_wrong_url_error = matches::matches!(client, Err(WrongUrl(_))); + assert!(got_wrong_url_error, "expected WrongUrl error"); +} diff --git a/common/scala/parser-service/src/main/scala/org/enso/ParserService.scala b/common/scala/parser-service/src/main/scala/org/enso/ParserService.scala new file mode 100644 index 00000000000..538a59c2236 --- /dev/null +++ b/common/scala/parser-service/src/main/scala/org/enso/ParserService.scala @@ -0,0 +1,68 @@ +package org.enso + +import io.circe.syntax._ +import io.circe.generic.auto._ +import org.enso.flexer.Reader +import org.enso.parserservice.Protocol +import org.enso.parserservice.Server +import org.enso.syntax.text.AST +import org.enso.syntax.text.Parser + +import scala.util.Try + +object ParserService { + val HOSTNAME_VAR = "ENSO_PARSER_HOSTNAME" + val PORT_VAR = "ENSO_PARSER_PORT" + + val DEFAULT_PORT = 30615 + val DEFAULT_HOSTNAME = "localhost" + + /** Obtains configuration from environment, filling missing values with + * defaults. + */ + def configFromEnv(): Server.Config = { + val hostname = sys.env.getOrElse(HOSTNAME_VAR, DEFAULT_HOSTNAME) + val port = sys.env + .get(PORT_VAR) + .flatMap(str => Try { str.toInt }.toOption) + .getOrElse(DEFAULT_PORT) + Server.Config(hostname, port) + } +} + +/** Class that allows setting up parser service with given configuration. */ +case class ParserService() extends Server with Protocol { + import parserservice._ + import Protocol._ + + def serializeAst(ast: AST.Module): String = + ast.asJson.noSpaces + + def runParser(program: String): AST = + new Parser().run(new Reader(program)) + + def handleRequest(request: Request): Response = { + request match { + case ParseRequest(program) => + val ast = runParser(program) + Protocol.Success(ast.asJson.toString()) + case _ => + throw new Exception(f"unimplemented request: $request") + } + } +} + +/** Runs a simple WebSocket server that wraps Parser into a service. */ +object ParserServiceMain extends App { + import ParserService._ + println("Getting configuration from environment...") + val config = configFromEnv() + + println(s"Will serve ${config.addressString()}") + println( + s"To change configuration, restart with $HOSTNAME_VAR or " + + s"$PORT_VAR variables set to desired values" + ) + val service = ParserService() + service.start(config) +} diff --git a/common/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala b/common/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala new file mode 100644 index 00000000000..cfb1d825fc1 --- /dev/null +++ b/common/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala @@ -0,0 +1,45 @@ +package org.enso.parserservice + +import io.circe.parser._ +import io.circe.generic.auto._ +import io.circe.syntax._ + +/** Types implementing parser server protocol. + * + * The protocol always is single request -> single response. + */ +object Protocol { + sealed trait Request + final case class ParseRequest(program: String) extends Request + + sealed trait Response + final case class Success(ast: String) extends Response + final case class Error(message: String) extends Response +} + +/** Helper for implementing protocol over text-based transport. + * + * Requests and responses are marshaled as text using JSON + * (and default circe serialzation schema). + */ +trait Protocol { + import Protocol._ + + /** Generate [[Response]] for a given [[Request]]. + * + * Any [[Throwable]] thrown out of implementation will be translated into + * [[Protocol.Error]] message. + */ + def handleRequest(request: Request): Response + + def handleMessage(input: String): String = { + try { + decode[Request](input) match { + case Left(err) => throw err + case Right(request) => handleRequest(request).asJson.toString() + } + } catch { + case e: Throwable => (Error(e.toString): Response).asJson.noSpaces + } + } +} diff --git a/common/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala b/common/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala new file mode 100644 index 00000000000..8889071c798 --- /dev/null +++ b/common/scala/parser-service/src/main/scala/org/enso/parserservice/Server.scala @@ -0,0 +1,109 @@ +package org.enso.parserservice + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.HttpMethods.GET +import akka.http.scaladsl.model.HttpRequest +import akka.http.scaladsl.model.HttpResponse +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.ws.BinaryMessage +import akka.http.scaladsl.model.ws.Message +import akka.http.scaladsl.model.ws.TextMessage +import akka.http.scaladsl.model.ws.UpgradeToWebSocket +import akka.stream.ActorMaterializer +import akka.stream.scaladsl.Flow +import akka.stream.scaladsl.Sink +import akka.stream.scaladsl.Source + +import scala.concurrent.ExecutionContext +import scala.util.Failure +import scala.util.Success + +object Server { + + /** Describes endpoint to which [[Server]] can bind. */ + final case class Config(interface: String, port: Int) { + def addressString(): String = s"ws://$interface:$port" + } +} + +/** WebSocket server supporting synchronous request-response protocol. + * + * Server when run binds to endpoint and accepts establishing web socket + * connection for any number of peers. + * + * Server replies to each incoming text message with a single text message. + * Server accepts a single Text Message from a peer and responds with + * another Text Message. + */ +trait Server { + implicit val system: ActorSystem = ActorSystem() + implicit val materializer: ActorMaterializer = ActorMaterializer() + + /** Generate text reply for given request text message. */ + def handleMessage(input: String): String + + /** Akka stream defining server behavior. + * + * Incoming [[TextMessage]]s are replied to (see [[handleMessage]]). + * Incoming binary messages are ignored. + */ + val handlerFlow: Flow[Message, TextMessage.Strict, NotUsed] = + Flow[Message] + .flatMapConcat { + case tm: TextMessage => + val strict = tm.textStream.fold("")(_ + _) + strict.map(input => TextMessage(handleMessage(input))) + case bm: BinaryMessage => + bm.dataStream.runWith(Sink.ignore) + Source.empty + } + + /** Server behavior upon receiving HTTP request. + * + * As server implements websocket-based protocol, this implementation accepts + * only GET requests to set up WebSocket connection. + * + * The request's URI is not checked. + */ + val handleRequest: HttpRequest => HttpResponse = { + case req @ HttpRequest(GET, _, _, _, _) => + req.header[UpgradeToWebSocket] match { + case Some(upgrade) => + println("Establishing a new connection") + upgrade.handleMessages(handlerFlow) + case None => + HttpResponse( + StatusCodes.BadRequest, + entity = "Not a valid websocket request!" + ) + } + case r: HttpRequest => + r.discardEntityBytes() + HttpResponse(StatusCodes.MethodNotAllowed) + } + + /** Starts a HTTP server listening at the given endpoint. + * + * Function is asynchronous, will return immediately. If the server fails to + * start, function will exit the process with a non-zero code. + */ + def start(config: Server.Config): Unit = { + val bindingFuture = + Http().bindAndHandleSync( + handleRequest, + interface = config.interface, + port = config.port + ) + + bindingFuture.onComplete({ + case Success(_) => + println(s"Server online at ${config.addressString()}") + case Failure(exception) => + println(s"Failed to start server: $exception") + system.terminate() + System.exit(1) + })(ExecutionContext.global) + } +} diff --git a/common/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala b/common/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala index 2555be8ea3c..167d7afe665 100644 --- a/common/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala +++ b/common/scala/syntax/definition/src/main/scala/org/enso/data/Tree.scala @@ -1,6 +1,9 @@ package org.enso.data -case class Tree[K, V](value: Option[V], branches: Map[K, Tree[K, V]]) { +import io.circe.Decoder +import io.circe.Encoder + +final case class Tree[K, V](value: Option[V], branches: Map[K, Tree[K, V]]) { def +(item: (List[K], V)): Tree[K, V] = item._1 match { case Nil => this.copy(value = Some(item._2)) case p :: ps => { @@ -34,4 +37,27 @@ object Tree { def apply[K, V](): Tree[K, V] = new Tree(None, Map()) def apply[K, V](deps: (List[K], V)*): Tree[K, V] = deps.foldLeft(Tree[K, V]())(_ + _) + + ////////////////// + // JSON support // + ////////////////// + + /* Note [Tree Serialization] */ + implicit def jsonEncode[K: Encoder, V: Encoder]: Encoder[Tree[K, V]] = + Encoder.forProduct2("value", "branches")( + tree => tree.value -> tree.branches.toSeq + ) + + /* Note [Tree Serialization] + * We can't directly serialize Map[K,V], as circe tries to use whole K as a + * key string in the generated JSON. Thus, we serialize Map[K, V] by + * converting it to Seq[(K,V)] first. + */ + + /* Note [Tree Serialization] */ + implicit def jsonDecode[K: Decoder, V: Decoder]: Decoder[Tree[K, V]] = + Decoder.forProduct2("value", "branches")( + (value: Option[V], branches: Seq[(K, Tree[K, V])]) => + Tree(value, branches.toMap) + ) } diff --git a/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala b/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala index 523b14976d0..19721af962a 100644 --- a/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala +++ b/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala @@ -2,11 +2,14 @@ package org.enso.syntax.text import java.util.UUID -import shapeless.Id import cats.Foldable import cats.Functor import cats.derived._ import cats.implicits._ +import io.circe.Encoder +import io.circe.Json +import io.circe.generic.AutoDerivation +import io.circe.generic.auto._ import org.enso.data.List1._ import org.enso.data.Index import org.enso.data.List1 @@ -266,53 +269,27 @@ object AST { case a: ASTOf[_] => shape == a.shape case _ => false } - val repr: Repr.Builder = cls.repr(shape) - val span: Int = cls.repr(shape).span - def show(): String = repr.build() - def setID(newID: ID): ASTOf[T] = copy(id = Some(newID)) - def withNewID(): ASTOf[T] = copy(id = Some(UUID.randomUUID())) + val repr: Repr.Builder = cls.repr(shape) + val span: Int = cls.repr(shape).span + def show(): String = repr.build() + def setID(newID: ID): ASTOf[T] = copy(id = Some(newID)) + def withNewID(): ASTOf[T] = copy(id = Some(UUID.randomUUID())) def map(f: AST => AST): ASTOf[T] = copy(shape = cls.map(shape)(f)) def mapWithOff(f: (Index, AST) => AST): ASTOf[T] = copy(shape = cls.mapWithOff(shape)(f)) def zipWithOffset(): T[(Index, AST)] = cls.zipWithOffset(shape) + def encodeShape(): Json = cls.encode(shape) } - object ASTOf { - implicit def repr[T[_]]: Repr[ASTOf[T]] = _.repr - implicit def unwrap[T[_]](t: ASTOf[T]): T[AST] = t.shape + object ASTOf extends AutoDerivation { + implicit def repr[T[_]]: Repr[ASTOf[T]] = _.repr + implicit def unwrap[T[_]](t: ASTOf[T]): T[AST] = t.shape implicit def wrap[T[_]](t: T[AST])( implicit ev: ASTClass[T] ): ASTOf[T] = ASTOf(t) - } - //// ASTClass //// - - /** [[ASTClass]] implements set of AST operations based on a precise AST - * shape. Because the [[T]] parameter in [[ASTOf]] is covariant, we may lose - * information about the shape after we construct the AST, thus this instance - * is used to cache all necessary operations during AST construction. - */ - trait ASTClass[T[_]] { - def repr(t: T[AST]): Repr.Builder - def map(t: T[AST])(f: AST => AST): T[AST] - def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST] - def zipWithOffset(t: T[AST]): T[(Index, AST)] - } - object ASTClass { - def apply[T[_]](implicit cls: ASTClass[T]): ASTClass[T] = cls - implicit def instance[T[_]]( - implicit - evRepr: Repr[T[AST]], - evFtor: Functor[T], - evOzip: OffsetZip[T, AST] - ): ASTClass[T] = - new ASTClass[T] { - def repr(t: T[AST]): Repr.Builder = evRepr.repr(t) - def map(t: T[AST])(f: AST => AST): T[AST] = Functor[T].map(t)(f) - def zipWithOffset(t: T[AST]): T[(Index, AST)] = OffsetZip(t) - def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST] = - Functor[T].map(zipWithOffset(t))(f.tupled) - } + implicit def jsonEncoder[T[_]]: Encoder[ASTOf[T]] = + Encoder.forProduct2("shape", "id")(ast => ast.encodeShape() -> ast.id) } //// ASTOps //// @@ -352,7 +329,7 @@ object AST { * a type is phantom, then its last type argument is not used and we can * safely coerce it to something else. */ - trait Phantom + sealed trait Phantom implicit class PhantomOps[T[_] <: Phantom](ident: T[_]) { def coerce[S]: T[S] = ident.asInstanceOf[T[S]] } @@ -380,8 +357,8 @@ object AST { val any = UnapplyByType[Invalid] object Unrecognized { - val any = UnapplyByType[Unrecognized] - def unapply(t: AST) = Unapply[Unrecognized].run(_.str)(t) + val any = UnapplyByType[Unrecognized] + def unapply(t: AST) = Unapply[Unrecognized].run(_.str)(t) def apply(str: String): Unrecognized = UnrecognizedOf[AST](str) } object Unexpected { @@ -394,15 +371,15 @@ object AST { //// Instances //// object UnrecognizedOf { - implicit def ftor: Functor[UnrecognizedOf] = semi.functor - implicit def fold: Foldable[UnrecognizedOf] = semi.foldable + implicit def ftor: Functor[UnrecognizedOf] = semi.functor + implicit def fold: Foldable[UnrecognizedOf] = semi.foldable implicit def repr[T]: Repr[UnrecognizedOf[T]] = _.str implicit def ozip[T]: OffsetZip[UnrecognizedOf, T] = t => t.coerce } object UnexpectedOf { - implicit def ftor: Functor[UnexpectedOf] = semi.functor - implicit def fold: Foldable[UnexpectedOf] = semi.foldable - implicit def repr[T: Repr]: Repr[UnexpectedOf[T]] = t => Repr(t.stream) + implicit def ftor: Functor[UnexpectedOf] = semi.functor + implicit def fold: Foldable[UnexpectedOf] = semi.foldable + implicit def repr[T: Repr]: Repr[UnexpectedOf[T]] = t => Repr(t.stream) implicit def ozip[T: Repr]: OffsetZip[UnexpectedOf, T] = t => t.copy(stream = OffsetZip(t.stream)) } @@ -452,32 +429,32 @@ object AST { //// Instances //// object BlankOf { - implicit def ftor: Functor[BlankOf] = semi.functor - implicit def fold: Foldable[BlankOf] = semi.foldable + implicit def ftor: Functor[BlankOf] = semi.functor + implicit def fold: Foldable[BlankOf] = semi.foldable implicit def repr[T]: Repr[BlankOf[T]] = _.name implicit def ozip[T]: OffsetZip[BlankOf, T] = t => t.coerce } object VarOf { - implicit def ftor: Functor[VarOf] = semi.functor - implicit def fold: Foldable[VarOf] = semi.foldable + implicit def ftor: Functor[VarOf] = semi.functor + implicit def fold: Foldable[VarOf] = semi.foldable implicit def repr[T]: Repr[VarOf[T]] = _.name implicit def ozip[T]: OffsetZip[VarOf, T] = t => t.coerce } object ConsOf { - implicit def ftor: Functor[ConsOf] = semi.functor - implicit def fold: Foldable[ConsOf] = semi.foldable + implicit def ftor: Functor[ConsOf] = semi.functor + implicit def fold: Foldable[ConsOf] = semi.foldable implicit def repr[T]: Repr[ConsOf[T]] = _.name implicit def ozip[T]: OffsetZip[ConsOf, T] = t => t.coerce } object OprOf { - implicit def ftor: Functor[OprOf] = semi.functor - implicit def fold: Foldable[OprOf] = semi.foldable + implicit def ftor: Functor[OprOf] = semi.functor + implicit def fold: Foldable[OprOf] = semi.foldable implicit def repr[T]: Repr[OprOf[T]] = _.name implicit def ozip[T]: OffsetZip[OprOf, T] = t => t.coerce } object ModOf { - implicit def ftor: Functor[ModOf] = semi.functor - implicit def fold: Foldable[ModOf] = semi.foldable + implicit def ftor: Functor[ModOf] = semi.functor + implicit def fold: Foldable[ModOf] = semi.foldable implicit def repr[T]: Repr[ModOf[T]] = R + _.name + "=" implicit def ozip[T]: OffsetZip[ModOf, T] = t => t.coerce } @@ -485,10 +462,10 @@ object AST { //// Conversions //// trait Conversions1 { - implicit def strToVar(str: String): Var = Var(str) + implicit def strToVar(str: String): Var = Var(str) implicit def strToCons(str: String): Cons = Cons(str) - implicit def strToOpr(str: String): Opr = Opr(str) - implicit def strToMod(str: String): Mod = Mod(str) + implicit def strToOpr(str: String): Opr = Opr(str) + implicit def strToMod(str: String): Mod = Mod(str) } trait conversions extends Conversions1 { @@ -509,31 +486,31 @@ object AST { private val blank = BlankOf[AST]() val any = UnapplyByType[Blank] def unapply(t: AST) = Unapply[Blank].run(_ => true)(t) - def apply(): Blank = blank + def apply(): Blank = blank } object Var { - private val pool = new Pool[VarOf[AST]]() - val any = UnapplyByType[Var] - def unapply(t: AST) = Unapply[Var].run(_.name)(t) + private val pool = new Pool[VarOf[AST]]() + val any = UnapplyByType[Var] + def unapply(t: AST) = Unapply[Var].run(_.name)(t) def apply(name: String): Var = pool.get(VarOf[AST](name)) } object Cons { - private val pool = new Pool[ConsOf[AST]]() - val any = UnapplyByType[Cons] - def unapply(t: AST) = Unapply[Cons].run(_.name)(t) + private val pool = new Pool[ConsOf[AST]]() + val any = UnapplyByType[Cons] + def unapply(t: AST) = Unapply[Cons].run(_.name)(t) def apply(name: String): Cons = pool.get(ConsOf[AST](name)) } object Mod { - private val pool = new Pool[ModOf[AST]]() - val any = UnapplyByType[Mod] - def unapply(t: AST) = Unapply[Mod].run(_.name)(t) + private val pool = new Pool[ModOf[AST]]() + val any = UnapplyByType[Mod] + def unapply(t: AST) = Unapply[Mod].run(_.name)(t) def apply(name: String): Mod = pool.get(ModOf[AST](name)) } object Opr { - private val pool = new Pool[OprOf[AST]]() - val app = Opr(" ") - val any = UnapplyByType[Opr] - def unapply(t: AST) = Unapply[Opr].run(_.name)(t) + private val pool = new Pool[OprOf[AST]]() + val app = Opr(" ") + val any = UnapplyByType[Opr] + def unapply(t: AST) = Unapply[Opr].run(_.name)(t) def apply(name: String): Opr = pool.get(OprOf[AST](name)) } @@ -546,8 +523,8 @@ object AST { extends InvalidOf[T] with Phantom object InvalidSuffixOf { - implicit def ftor: Functor[InvalidSuffixOf] = semi.functor - implicit def fold: Foldable[InvalidSuffixOf] = semi.foldable + implicit def ftor: Functor[InvalidSuffixOf] = semi.functor + implicit def fold: Foldable[InvalidSuffixOf] = semi.foldable implicit def ozip[T]: OffsetZip[InvalidSuffixOf, T] = t => t.coerce implicit def repr[T]: Repr[InvalidSuffixOf[T]] = t => R + t.elem + t.suffix @@ -598,12 +575,12 @@ object AST { //// Smart Constructors //// - def apply(i: String): Number = Number(None, i) + def apply(i: String): Number = Number(None, i) def apply(b: String, i: String): Number = Number(Some(b), i) - def apply(i: Int): Number = Number(i.toString) - def apply(b: Int, i: String): Number = Number(b.toString, i) - def apply(b: String, i: Int): Number = Number(b, i.toString) - def apply(b: Int, i: Int): Number = Number(b.toString, i.toString) + def apply(i: Int): Number = Number(i.toString) + def apply(b: Int, i: String): Number = Number(b.toString, i) + def apply(b: String, i: Int): Number = Number(b, i.toString) + def apply(b: Int, i: Int): Number = Number(b.toString, i.toString) def apply(b: Option[String], i: String): Number = NumberOf[AST](b, i) def unapply(t: AST) = Unapply[Number].run(t => (t.base, t.int))(t) @@ -616,14 +593,14 @@ object AST { extends InvalidOf[T] with Phantom object DanglingBase { - val any = UnapplyByType[DanglingBase] + val any = UnapplyByType[DanglingBase] def apply(base: String): DanglingBase = DanglingBaseOf[AST](base) def unapply(t: AST) = Unapply[DanglingBase].run(_.base)(t) } object DanglingBaseOf { - implicit def ftor: Functor[DanglingBaseOf] = semi.functor - implicit def fold: Foldable[DanglingBaseOf] = semi.foldable + implicit def ftor: Functor[DanglingBaseOf] = semi.functor + implicit def fold: Foldable[DanglingBaseOf] = semi.foldable implicit def ozip[T]: OffsetZip[DanglingBaseOf, T] = t => t.coerce implicit def repr[T]: Repr[DanglingBaseOf[T]] = R + _.base + '_' } @@ -632,10 +609,10 @@ object AST { //// Instances //// object NumberOf { - implicit def fromInt[T](int: Int): Number = Number(int) - implicit def ftor: Functor[NumberOf] = semi.functor - implicit def fold: Foldable[NumberOf] = semi.foldable - implicit def ozip[T]: OffsetZip[NumberOf, T] = t => t.coerce + implicit def fromInt[T](int: Int): Number = Number(int) + implicit def ftor: Functor[NumberOf] = semi.functor + implicit def fold: Foldable[NumberOf] = semi.foldable + implicit def ozip[T]: OffsetZip[NumberOf, T] = t => t.coerce implicit def repr[T]: Repr[NumberOf[T]] = t => t.base.map(_ + "_").getOrElse("") + t.int } @@ -671,7 +648,7 @@ object AST { //// Definition //// - sealed trait Line[T] extends TextOf[T] + sealed trait Line[T] extends TextOf[T] sealed trait Block[T] extends TextOf[T] final case class UnclosedOf[T](line: Line[T]) @@ -689,15 +666,36 @@ object AST { with Phantom object Line { - final case class Raw[T](text: List[Segment._Raw[T]]) + /* Note [Circe and naming] */ + val Raw = LineRaw + /* Note [Circe and naming] */ + type Raw[T] = LineRaw[T] + + /* Note [Circe and naming] */ + val Fmt = LineFmt + /* Note [Circe and naming] */ + type Fmt[T] = LineFmt[T] + + /* Note [Circe and naming] */ + final case class LineRaw[T](text: List[Segment._Raw[T]]) extends Line[T] with Phantom { val quote = '"' } - final case class Fmt[T](text: List[Segment._Fmt[T]]) extends Line[T] { + /* Note [Circe and naming] */ + final case class LineFmt[T](text: List[Segment._Fmt[T]]) + extends Line[T] { val quote = '\'' } + /* Note [Circe and naming] + * ~~~~~~~~~~~~~~~~~~~~~~~~ + * To be able to use Circe automatic derivation for traits, all case + * classes in its subtree must bear unique names. So `Line.Fmt` and + * `Line.Raw` cannot be used as they would collide with [[Block.Raw]] + * and [[Block.Fmt]]. + */ + ////// INSTANCES ///// import Segment.implicits._ @@ -707,9 +705,9 @@ object AST { case t: Raw[T] => t.quote + t.text + t.quote case t: Fmt[T] => t.quote + t.text + t.quote } - implicit def ozip[T: Repr]: OffsetZip[Line, T] = { - case t: Raw[T] => t.coerce - case t: Fmt[T] => + implicit def ozip[T: Repr]: OffsetZip[Line, T] = { + case t: Raw[T] => t.coerce + case t: Fmt[T] => var offset = Index(t.quote.span) val text2 = for (elem <- t.text) yield { val offElem = elem.map(offset -> _) @@ -730,7 +728,7 @@ object AST { with Phantom { val quote = "\"\"\"" } - case class Fmt[T]( + final case class Fmt[T]( text: List[Line[Segment._Fmt[T]]], spaces: Int, offset: Int @@ -755,7 +753,7 @@ object AST { case Fmt(text, s, off) => q + s + text.map(line(off, _)) } } - implicit def ozip[T: Repr]: OffsetZip[Block, T] = { + implicit def ozip[T: Repr]: OffsetZip[Block, T] = { case body: Raw[T] => body.coerce case body: Fmt[T] => var offset = Index(body.quote.span) @@ -802,13 +800,14 @@ object AST { def apply(quote: String): InlineBlock = InlineBlockOf[AST](quote) } - def apply(text: TextOf[AST]): Text = text - def apply(segment: Segment.Fmt*): Text = Text(Line.Fmt(segment.to[List])) + def apply(text: TextOf[AST]): Text = text + def apply(segment: Segment.Fmt*): Text = Text(Line.Fmt(segment.to[List])) def apply(spaces: Int, off: Int, line: Block.Line[Segment.Fmt]*): Text = Text(Block.Fmt(line.to[List], spaces, off)) object Raw { - def apply(segment: Segment.Raw*): Text = Text(Line.Raw(segment.to[List])) + def apply(segment: Segment.Raw*): Text = + Text(Line.Raw(segment.to[List])) def apply(spaces: Int, off: Int, line: Block.Line[Segment.Raw]*): Text = Text(Block.Raw(line.to[List], spaces, off)) } @@ -828,15 +827,15 @@ object AST { t => t.copy(line = OffsetZip(t.line)) } object InvalidQuoteOf { - implicit def ftor: Functor[InvalidQuoteOf] = semi.functor - implicit def fold: Foldable[InvalidQuoteOf] = semi.foldable - implicit def repr[T: Repr]: Repr[InvalidQuoteOf[T]] = _.quote + implicit def ftor: Functor[InvalidQuoteOf] = semi.functor + implicit def fold: Foldable[InvalidQuoteOf] = semi.foldable + implicit def repr[T: Repr]: Repr[InvalidQuoteOf[T]] = _.quote implicit def ozip[T: Repr]: OffsetZip[InvalidQuoteOf, T] = t => t.coerce } object InlineBlockOf { - implicit def ftor: Functor[InlineBlockOf] = semi.functor - implicit def fold: Foldable[InlineBlockOf] = semi.foldable - implicit def repr[T: Repr]: Repr[InlineBlockOf[T]] = _.quote + implicit def ftor: Functor[InlineBlockOf] = semi.functor + implicit def fold: Foldable[InlineBlockOf] = semi.foldable + implicit def repr[T: Repr]: Repr[InlineBlockOf[T]] = _.quote implicit def ozip[T: Repr]: OffsetZip[InlineBlockOf, T] = t => t.coerce } @@ -862,7 +861,7 @@ object AST { final case class _Escape[T](code: Escape) extends _Fmt[T] with Phantom object Expr { def apply(t: Option[AST]): Fmt = _Expr(t) } - object Plain { def apply(s: String): Raw = _Plain(s) } + object Plain { def apply(s: String): Raw = _Plain(s) } //// Instances //// @@ -875,20 +874,20 @@ object AST { t => R + ("\\" + t.code.repr) implicit def ozipEscape[T]: OffsetZip[_Escape, T] = t => t.coerce - implicit def foldPlain: Foldable[_Plain] = semi.foldable + implicit def foldPlain: Foldable[_Plain] = semi.foldable implicit def ftorPlain[T]: Functor[_Plain] = semi.functor implicit def reprPlain[T]: Repr[_Plain[T]] = _.value implicit def ozipPlain[T]: OffsetZip[_Plain, T] = t => t.coerce - implicit def ftorExpr[T]: Functor[_Expr] = semi.functor - implicit def foldExpr: Foldable[_Expr] = semi.foldable + implicit def ftorExpr[T]: Functor[_Expr] = semi.functor + implicit def foldExpr: Foldable[_Expr] = semi.foldable implicit def reprExpr[T: Repr]: Repr[_Expr[T]] = R + '`' + _.value + '`' implicit def ozipExpr[T]: OffsetZip[_Expr, T] = _.map(Index.Start -> _) - implicit def ftorRaw[T]: Functor[_Raw] = semi.functor - implicit def foldRaw: Foldable[_Raw] = semi.foldable + implicit def ftorRaw[T]: Functor[_Raw] = semi.functor + implicit def foldRaw: Foldable[_Raw] = semi.foldable implicit def reprRaw[T]: Repr[_Raw[T]] = { case t: _Plain[T] => Repr(t) } @@ -896,8 +895,8 @@ object AST { case t: _Plain[T] => OffsetZip(t) } - implicit def ftorFmt[T]: Functor[_Fmt] = semi.functor - implicit def foldFmt: Foldable[_Fmt] = semi.foldable + implicit def ftorFmt[T]: Functor[_Fmt] = semi.functor + implicit def foldFmt: Foldable[_Fmt] = semi.foldable implicit def reprFmt[T: Repr]: Repr[_Fmt[T]] = { case t: _Plain[T] => Repr(t) case t: _Expr[T] => Repr(t) @@ -955,10 +954,10 @@ object AST { //// Smart Constructors //// object Prefix { - val any = UnapplyByType[Prefix] - def unapply(t: AST) = Unapply[Prefix].run(t => (t.fn, t.arg))(t) + val any = UnapplyByType[Prefix] + def unapply(t: AST) = Unapply[Prefix].run(t => (t.fn, t.arg))(t) def apply(fn: AST, off: Int, arg: AST): Prefix = PrefixOf(fn, off, arg) - def apply(fn: AST, arg: AST): Prefix = Prefix(fn, 1, arg) + def apply(fn: AST, arg: AST): Prefix = Prefix(fn, 1, arg) } object Infix { @@ -1040,18 +1039,18 @@ object AST { def unapply(t: AST) = Unapply[Left].run(t => (t.arg, t.opr))(t) def apply(arg: AST, off: Int, opr: Opr): Left = LeftOf(arg, off, opr) - def apply(arg: AST, opr: Opr): Left = Left(arg, 1, opr) + def apply(arg: AST, opr: Opr): Left = Left(arg, 1, opr) } object Right { val any = UnapplyByType[Right] def unapply(t: AST) = Unapply[Right].run(t => (t.opr, t.arg))(t) def apply(opr: Opr, off: Int, arg: AST): Right = RightOf(opr, off, arg) - def apply(opr: Opr, arg: AST): Right = Right(opr, 1, arg) + def apply(opr: Opr, arg: AST): Right = Right(opr, 1, arg) } object Sides { - val any = UnapplyByType[Sides] - def unapply(t: AST) = Unapply[Sides].run(_.opr)(t) + val any = UnapplyByType[Sides] + def unapply(t: AST) = Unapply[Sides].run(_.opr)(t) def apply(opr: Opr): Sides = SidesOf[AST](opr) } @@ -1074,10 +1073,10 @@ object AST { t => t.copy(arg = (Index(t.opr.span + t.off), t.arg)) } object SidesOf { - implicit def ftor: Functor[SidesOf] = semi.functor - implicit def fold: Foldable[SidesOf] = semi.foldable - implicit def repr[T: Repr]: Repr[SidesOf[T]] = t => R + t.opr - implicit def ozip[T]: OffsetZip[SidesOf, T] = t => t.coerce + implicit def ftor: Functor[SidesOf] = semi.functor + implicit def fold: Foldable[SidesOf] = semi.foldable + implicit def repr[T: Repr]: Repr[SidesOf[T]] = t => R + t.opr + implicit def ozip[T]: OffsetZip[SidesOf, T] = t => t.coerce } } } @@ -1155,26 +1154,26 @@ object AST { def toOptional: LineOf[Option[T]] = copy(elem = Some(elem)) } object LineOf { - implicit def ftorLine: Functor[LineOf] = semi.functor - implicit def fold: Foldable[LineOf] = semi.foldable - implicit def reprLine[T: Repr]: Repr[LineOf[T]] = t => R + t.elem + t.off + implicit def ftorLine: Functor[LineOf] = semi.functor + implicit def fold: Foldable[LineOf] = semi.foldable + implicit def reprLine[T: Repr]: Repr[LineOf[T]] = t => R + t.elem + t.off } object Line { // FIXME: Compatibility mode type NonEmpty = Line - val Required = Line - def apply[T](elem: T, off: Int) = LineOf(elem, off) + val Required = Line + def apply[T](elem: T, off: Int) = LineOf(elem, off) def apply[T](elem: T): LineOf[T] = LineOf(elem, 0) } object OptLine { - def apply(): OptLine = Line(None, 0) + def apply(): OptLine = Line(None, 0) def apply(elem: AST): OptLine = Line(Some(elem)) - def apply(off: Int): OptLine = Line(None, off) + def apply(off: Int): OptLine = Line(None, off) } } object BlockOf { - implicit def ftorBlock: Functor[BlockOf] = semi.functor - implicit def fold: Foldable[BlockOf] = semi.foldable + implicit def ftorBlock: Functor[BlockOf] = semi.functor + implicit def fold: Foldable[BlockOf] = semi.foldable implicit def reprBlock[T: Repr]: Repr[BlockOf[T]] = t => { val headRepr = if (t.isOrphan) R else newline val emptyLinesRepr = t.emptyLines.map(R + _ + newline) @@ -1207,11 +1206,11 @@ object AST { object Module { import Block._ type M = Module - val any = UnapplyByType[M] - def unapply(t: AST) = Unapply[M].run(_.lines)(t) - def apply(ls: List1[OptLine]): M = ModuleOf(ls) - def apply(l: OptLine): M = Module(List1(l)) - def apply(l: OptLine, ls: OptLine*): M = Module(List1(l, ls.to[List])) + val any = UnapplyByType[M] + def unapply(t: AST) = Unapply[M].run(_.lines)(t) + def apply(ls: List1[OptLine]): M = ModuleOf(ls) + def apply(l: OptLine): M = Module(List1(l)) + def apply(l: OptLine, ls: OptLine*): M = Module(List1(l, ls.to[List])) def apply(l: OptLine, ls: List[OptLine]): M = Module(List1(l, ls)) def traverseWithOff(m: M)(f: (Index, AST) => AST): M = { val lines2 = m.lines.map { line: OptLine => @@ -1222,8 +1221,8 @@ object AST { } } object ModuleOf { - implicit def ftor: Functor[ModuleOf] = semi.functor - implicit def fold: Foldable[ModuleOf] = semi.foldable + implicit def ftor: Functor[ModuleOf] = semi.functor + implicit def fold: Foldable[ModuleOf] = semi.foldable implicit def ozip[T]: OffsetZip[ModuleOf, T] = _.map(Index.Start -> _) implicit def repr[T: Repr]: Repr[ModuleOf[T]] = t => R + t.lines.head + t.lines.tail.map(newline + _) @@ -1323,14 +1322,14 @@ object AST { final case class Segment(head: AST, body: Option[SAST]) object Segment { - def apply(head: AST): Segment = Segment(head, None) - implicit def repr: Repr[Segment] = t => R + t.head + t.body + def apply(head: AST): Segment = Segment(head, None) + implicit def repr: Repr[Segment] = t => R + t.head + t.body } } object AmbiguousOf { - implicit def ftor: Functor[AmbiguousOf] = semi.functor - implicit def fold: Foldable[AmbiguousOf] = semi.foldable + implicit def ftor: Functor[AmbiguousOf] = semi.functor + implicit def fold: Foldable[AmbiguousOf] = semi.foldable implicit def repr[T]: Repr[AmbiguousOf[T]] = t => R + t.segs.map(Repr(_)) implicit def ozip[T]: OffsetZip[AmbiguousOf, T] = _.map(Index.Start -> _) } @@ -1514,8 +1513,8 @@ object AST { extends SpacelessASTOf[T] with Phantom object Comment { - val any = UnapplyByType[Comment] - val symbol = "#" + val any = UnapplyByType[Comment] + val symbol = "#" def apply(lines: List[String]): Comment = ASTOf(CommentOf(lines)) def unapply(t: AST): Option[List[String]] = Unapply[Comment].run(t => t.lines)(t) @@ -1538,7 +1537,7 @@ object AST { ////////////////////////////////////////////////////////////////////////////// type Documented = ASTOf[DocumentedOf] - case class DocumentedOf[T](doc: Doc, emptyLinesBetween: Int, ast: T) + final case class DocumentedOf[T](doc: Doc, emptyLinesBetween: Int, ast: T) extends ShapeOf[T] object Documented { val any = UnapplyByType[Documented] @@ -1560,6 +1559,9 @@ object AST { } implicit def offsetZip[T]: OffsetZip[DocumentedOf, T] = _.map(Index.Start -> _) + + implicit def toJson[T]: Encoder[DocumentedOf[T]] = + _ => throw new NotImplementedError() } ////////////////////////////////////////////////////////////////////////////// @@ -1569,10 +1571,10 @@ object AST { type Import = ASTOf[ImportOf] final case class ImportOf[T](path: List1[Cons]) extends SpacelessASTOf[T] object Import { - def apply(path: List1[Cons]): Import = ImportOf[AST](path) - def apply(head: Cons): Import = Import(head, List()) + def apply(path: List1[Cons]): Import = ImportOf[AST](path) + def apply(head: Cons): Import = Import(head, List()) def apply(head: Cons, tail: List[Cons]): Import = Import(List1(head, tail)) - def apply(head: Cons, tail: Cons*): Import = Import(head, tail.toList) + def apply(head: Cons, tail: Cons*): Import = Import(head, tail.toList) def unapply(t: AST): Option[List1[Cons]] = Unapply[Import].run(t => t.path)(t) val any = UnapplyByType[Import] @@ -1621,12 +1623,12 @@ object AST { type Group = ASTOf[GroupOf] final case class GroupOf[T](body: Option[T]) extends SpacelessASTOf[T] object Group { - val any = UnapplyByType[Group] - def unapply(t: AST) = Unapply[Group].run(_.body)(t) + val any = UnapplyByType[Group] + def unapply(t: AST) = Unapply[Group].run(_.body)(t) def apply(body: Option[AST]): Group = GroupOf(body) - def apply(body: AST): Group = Group(Some(body)) - def apply(body: SAST): Group = Group(body.el) - def apply(): Group = Group(None) + def apply(body: AST): Group = Group(Some(body)) + def apply(body: SAST): Group = Group(body.el) + def apply(): Group = Group(None) } object GroupOf { implicit def ftor: Functor[GroupOf] = semi.functor @@ -1645,9 +1647,9 @@ object AST { final case class DefOf[T](name: Cons, args: List[T], body: Option[T]) extends SpacelessASTOf[T] object Def { - val any = UnapplyByType[Def] - val symbol = "def" - def apply(name: Cons): Def = Def(name, List()) + val any = UnapplyByType[Def] + val symbol = "def" + def apply(name: Cons): Def = Def(name, List()) def apply(name: Cons, args: List[AST]): Def = Def(name, args, None) def apply(name: Cons, args: List[AST], body: Option[AST]): Def = DefOf(name, args, body) @@ -1688,6 +1690,41 @@ object AST { implicit def ozip[T]: OffsetZip[ForeignOf, T] = _.map(Index.Start -> _) } + //// ASTClass //// + + /** [[ASTClass]] implements set of AST operations based on a precise AST + * shape. Because the [[T]] parameter in [[ASTOf]] is covariant, we may lose + * information about the shape after we construct the AST, thus this instance + * is used to cache all necessary operations during AST construction. + */ + sealed trait ASTClass[T[_]] { + def repr(t: T[AST]): Repr.Builder + def map(t: T[AST])(f: AST => AST): T[AST] + def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST] + def zipWithOffset(t: T[AST]): T[(Index, AST)] + def encode(t: T[AST]): Json + } + object ASTClass { + def apply[T[_]](implicit cls: ASTClass[T]): ASTClass[T] = cls + implicit def instance[T[S] <: ShapeOf[S]]( + implicit + evRepr: Repr[T[AST]], + evFtor: Functor[T], + evOzip: OffsetZip[T, AST] + ): ASTClass[T] = + new ASTClass[T] { + def repr(t: T[AST]): Repr.Builder = evRepr.repr(t) + def map(t: T[AST])(f: AST => AST): T[AST] = Functor[T].map(t)(f) + def zipWithOffset(t: T[AST]): T[(Index, AST)] = OffsetZip(t) + def mapWithOff(t: T[AST])(f: (Index, AST) => AST): T[AST] = + Functor[T].map(zipWithOffset(t))(f.tupled) + def encode(t: T[AST]): Json = { + val shapeEncoder = implicitly[Encoder[ShapeOf[AST]]] + shapeEncoder(t) + } + } + } + ///////////////////////////////////////////////// ///////////////////////////////////////////////// ///////////////////////////////////////////////// @@ -1738,4 +1775,4 @@ object AST { val v1_x = vx.as[Var] println(v1_x) } -} \ No newline at end of file +} diff --git a/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala b/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala index 2da567c3539..345364151f6 100644 --- a/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala +++ b/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/meta/Pattern.scala @@ -5,7 +5,8 @@ import org.enso.syntax.text.AST.SAST import org.enso.syntax.text.prec.Operator import scala.annotation.tailrec -import org.enso.data.{Index, Shifted} +import org.enso.data.Index +import org.enso.data.Shifted import org.enso.syntax.text.ast.Repr //////////////////////////////////////////////////////////////////////////////// @@ -13,7 +14,9 @@ import org.enso.syntax.text.ast.Repr //////////////////////////////////////////////////////////////////////////////// object Pattern { - import cats.{Foldable, Functor, Traverse} + import cats.Foldable + import cats.Functor + import cats.Traverse import cats.derived._ type P = Pattern @@ -34,7 +37,7 @@ object Pattern { (nStream.reverse, nOff) } - trait Class + sealed trait Class object Class { final case object Normal extends Class final case object Pattern extends Class @@ -195,7 +198,7 @@ object Pattern { //// Result //// - case class Result(elem: Match, stream: AST.Stream) { + final case class Result(elem: Match, stream: AST.Stream) { def map(fn: Match => Match): Result = copy(elem = fn(elem)) } diff --git a/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala b/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala index 4fefbdba4f0..d7493a1b84d 100644 --- a/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala +++ b/common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala @@ -1,6 +1,7 @@ package org.enso.syntax.text.ast.text import org.enso.flexer.ADT +import org.enso.syntax.text.ast.text.Escape.Slash.toString sealed trait Escape { val repr: String @@ -8,7 +9,6 @@ sealed trait Escape { object Escape { - final case class Invalid(str: String) extends Escape { val repr = str } @@ -21,18 +21,46 @@ object Escape { sealed trait Unicode extends Escape object Unicode { - final case class Invalid(unicode: Unicode) extends Unicode { + /* Note [Circe and Naming] */ + type Invalid = InvalidUnicode + + /* Note [Circe and Naming] */ + val Invalid = InvalidUnicode + + /* Note [Circe and Naming] */ + final case class InvalidUnicode(unicode: Unicode) extends Unicode { val repr = unicode.repr } - abstract class U(pfx: String, sfx: String = "") extends Unicode { - val digits: String + /* NOTE [Circe and Naming] + * Name of the class above cannot be Invalid, as we already have + * Escape.Invalid. And to be able to derive JSON serialization with circe + * case class names within a trait subtree need to be unique. + * + * To keep this unpleasant detail hidden from library users, we introduce + * aliases for type and object named `Invalid`. + */ + + type U16 = _U16 + final case class _U16(digits: String) extends Unicode { + val pfx = "u" + val sfx = "" val repr = pfx + digits + sfx } - final case class U16 private (digits: String) extends U("u") - final case class U32 private (digits: String) extends U("U") - final case class U21 private (digits: String) extends U("u{", "}") + type U32 = _U32 + final case class _U32(digits: String) extends Unicode { + val pfx = "U" + val sfx = "" + val repr = pfx + digits + sfx + } + + type U21 = _U21 + final case class _U21(digits: String) extends Unicode { + val pfx = "u{" + val sfx = "}" + val repr = pfx + digits + sfx + } object Validator { val hexChars = @@ -43,8 +71,8 @@ object Escape { object U16 { def apply(digits: String): Unicode = - if (validate(digits)) U16(digits) - else Invalid(U16(digits)) + if (validate(digits)) _U16(digits) + else Invalid(_U16(digits)) def validate(digits: String) = { import Validator._ val validLength = digits.length == 4 @@ -54,8 +82,8 @@ object Escape { } object U32 { def apply(digits: String): Unicode = - if (validate(digits)) U32(digits) - else Invalid(U32(digits)) + if (validate(digits)) _U32(digits) + else Invalid(_U32(digits)) def validate(digits: String) = { import Validator._ val validLength = digits.length == 8 @@ -66,8 +94,8 @@ object Escape { } object U21 { def apply(digits: String): Unicode = - if (validate(digits)) U21(digits) - else Invalid(U21(digits)) + if (validate(digits)) _U21(digits) + else Invalid(_U21(digits)) def validate(digits: String) = { import Validator._ val validLength = digits.length >= 1 && digits.length <= 6 @@ -77,66 +105,236 @@ object Escape { } } - - abstract class Simple(val code: Int) extends Escape{ - def name = toString - val repr = name + case object Slash extends Escape { + val code: Int = '\\' + def name: String = toString + override val repr = "\\" + } + case object Quote extends Escape { + val code: Int = '\'' + def name: String = toString + override val repr = "\'" + } + case object RawQuote extends Escape { + val code: Int = '"' + def name: String = toString + override val repr = "\"" } - case object Slash extends Simple('\\') { override val repr = "\\" } - case object Quote extends Simple('\'') { override val repr = "\'" } - case object RawQuote extends Simple('"') { override val repr = "\"" } - // Reference: https://en.wikipedia.org/wiki/String_literal - sealed trait Character extends Simple + sealed trait Character extends Escape object Character { - case object a extends Simple('\u0007') with Character - case object b extends Simple('\u0008') with Character - case object f extends Simple('\u000C') with Character - case object n extends Simple('\n') with Character - case object r extends Simple('\r') with Character - case object t extends Simple('\u0009') with Character - case object v extends Simple('\u000B') with Character - case object e extends Simple('\u001B') with Character + case object a extends Character { + val code: Int = '\u0007' + def name: String = toString + override val repr = name + } + case object b extends Character { + val code: Int = '\u0008' + def name: String = toString + override val repr = name + } + case object f extends Character { + val code: Int = '\u000C' + def name: String = toString + override val repr = name + } + case object n extends Character { + val code: Int = '\n' + def name: String = toString + override val repr = name + } + case object r extends Character { + val code: Int = '\r' + def name: String = toString + override val repr = name + } + case object t extends Character { + val code: Int = '\u0009' + def name: String = toString + override val repr = name + } + case object v extends Character { + val code: Int = '\u000B' + def name: String = toString + override val repr = name + } + case object e extends Character { + val code: Int = '\u001B' + def name: String = toString + override val repr = name + } val codes = ADT.constructors[Character] } // Reference: https://en.wikipedia.org/wiki/Control_character - sealed trait Control extends Simple + sealed trait Control extends Escape object Control { - case object NUL extends Simple(0x00) with Control - case object SOH extends Simple(0x01) with Control - case object STX extends Simple(0x02) with Control - case object ETX extends Simple(0x03) with Control - case object EOT extends Simple(0x04) with Control - case object ENQ extends Simple(0x05) with Control - case object ACK extends Simple(0x06) with Control - case object BEL extends Simple(0x07) with Control - case object BS extends Simple(0x08) with Control - case object TAB extends Simple(0x09) with Control - case object LF extends Simple(0x0A) with Control - case object VT extends Simple(0x0B) with Control - case object FF extends Simple(0x0C) with Control - case object CR extends Simple(0x0D) with Control - case object SO extends Simple(0x0E) with Control - case object SI extends Simple(0x0F) with Control - case object DLE extends Simple(0x10) with Control - case object DC1 extends Simple(0x11) with Control - case object DC2 extends Simple(0x12) with Control - case object DC3 extends Simple(0x13) with Control - case object DC4 extends Simple(0x14) with Control - case object NAK extends Simple(0x15) with Control - case object SYN extends Simple(0x16) with Control - case object ETB extends Simple(0x17) with Control - case object CAN extends Simple(0x18) with Control - case object EM extends Simple(0x19) with Control - case object SUB extends Simple(0x1A) with Control - case object ESC extends Simple(0x1B) with Control - case object FS extends Simple(0x1C) with Control - case object GS extends Simple(0x1D) with Control - case object RS extends Simple(0x1E) with Control - case object US extends Simple(0x1F) with Control - case object DEL extends Simple(0x7F) with Control + case object NUL extends Control { + val code: Int = 0x00 + def name: String = toString + override val repr = name + } + case object SOH extends Control { + val code: Int = 0x01 + def name: String = toString + override val repr = name + } + case object STX extends Control { + val code: Int = 0x02 + def name: String = toString + override val repr = name + } + case object ETX extends Control { + val code: Int = 0x03 + def name: String = toString + override val repr = name + } + case object EOT extends Control { + val code: Int = 0x04 + def name: String = toString + override val repr = name + } + case object ENQ extends Control { + val code: Int = 0x05 + def name: String = toString + override val repr = name + } + case object ACK extends Control { + val code: Int = 0x06 + def name: String = toString + override val repr = name + } + case object BEL extends Control { + val code: Int = 0x07 + def name: String = toString + override val repr = name + } + case object BS extends Control { + val code: Int = 0x08 + def name: String = toString + override val repr = name + } + case object TAB extends Control { + val code: Int = 0x09 + def name: String = toString + override val repr = name + } + case object LF extends Control { + val code: Int = 0x0A + def name: String = toString + override val repr = name + } + case object VT extends Control { + val code: Int = 0x0B + def name: String = toString + override val repr = name + } + case object FF extends Control { + val code: Int = 0x0C + def name: String = toString + override val repr = name + } + case object CR extends Control { + val code: Int = 0x0D + def name: String = toString + override val repr = name + } + case object SO extends Control { + val code: Int = 0x0E + def name: String = toString + override val repr = name + } + case object SI extends Control { + val code: Int = 0x0F + def name: String = toString + override val repr = name + } + case object DLE extends Control { + val code: Int = 0x10 + def name: String = toString + override val repr = name + } + case object DC1 extends Control { + val code: Int = 0x11 + def name: String = toString + override val repr = name + } + case object DC2 extends Control { + val code: Int = 0x12 + def name: String = toString + override val repr = name + } + case object DC3 extends Control { + val code: Int = 0x13 + def name: String = toString + override val repr = name + } + case object DC4 extends Control { + val code: Int = 0x14 + def name: String = toString + override val repr = name + } + case object NAK extends Control { + val code: Int = 0x15 + def name: String = toString + override val repr = name + } + case object SYN extends Control { + val code: Int = 0x16 + def name: String = toString + override val repr = name + } + case object ETB extends Control { + val code: Int = 0x17 + def name: String = toString + override val repr = name + } + case object CAN extends Control { + val code: Int = 0x18 + def name: String = toString + override val repr = name + } + case object EM extends Control { + val code: Int = 0x19 + def name: String = toString + override val repr = name + } + case object SUB extends Control { + val code: Int = 0x1A + def name: String = toString + override val repr = name + } + case object ESC extends Control { + val code: Int = 0x1B + def name: String = toString + override val repr = name + } + case object FS extends Control { + val code: Int = 0x1C + def name: String = toString + override val repr = name + } + case object GS extends Control { + val code: Int = 0x1D + def name: String = toString + override val repr = name + } + case object RS extends Control { + val code: Int = 0x1E + def name: String = toString + override val repr = name + } + case object US extends Control { + val code: Int = 0x1F + def name: String = toString + override val repr = name + } + case object DEL extends Control { + val code: Int = 0x7F + def name: String = toString + override val repr = name + } val codes = ADT.constructors[Control] } -} \ No newline at end of file +} diff --git a/common/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala b/common/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala index 57571d6d51b..6f6ef7daddd 100644 --- a/common/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala +++ b/common/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala @@ -1,6 +1,7 @@ package org.enso.syntax.text -import org.enso.data.{Index, Span} +import org.enso.data.Index +import org.enso.data.Span import org.enso.flexer import org.enso.flexer.Reader import org.enso.syntax.text.ast.meta.Builtin @@ -218,7 +219,7 @@ class Parser { object Parser { type IDMap = Seq[(Span, AST.ID)] - def apply(): Parser = new Parser() + def apply(): Parser = new Parser() private val newEngine = flexer.Parser.compile(ParserDef()) //// Exceptions //// diff --git a/common/scala/syntax/specialization/src/test/scala/org/enso/syntax/text/ParserTest.scala b/common/scala/syntax/specialization/src/test/scala/org/enso/syntax/text/ParserTest.scala index e707e4040d2..57ab9627869 100644 --- a/common/scala/syntax/specialization/src/test/scala/org/enso/syntax/text/ParserTest.scala +++ b/common/scala/syntax/specialization/src/test/scala/org/enso/syntax/text/ParserTest.scala @@ -303,9 +303,9 @@ class ParserTest extends FlatSpec with Matchers { def _amb_group_(i: Int)(t: AST): Macro.Ambiguous = amb("(", List(List(")")), Shifted(i, t)) - val amb_group = _amb_group_(0)(_) - val amb_group_ = _amb_group_(1)(_) - val amb_group__ = _amb_group_(2)(_) + val amb_group = _amb_group_(0)(_) + val amb_group_ = _amb_group_(1)(_) + val amb_group__ = _amb_group_(2)(_) def group_(): Macro.Ambiguous = amb("(", List(List(")"))) def _amb_if(i: Int)(t: AST) =