enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-24 00:27:16 +03:00
Code Issues Projects Releases Wiki Activity

Scala/Rust JSON serialization protocol unification (#370)

Browse Source 
This PR updates JSON serialization in Scala in Rust, so they are compatible, implementing #297. The parser wrapper now uses the real AST in API. Still most of non-trivial use-cases will fail. Once #336 is done, it should finally work.
This commit is contained in:
Michał Wawrzyniec Urbańczyk 2019-12-04 16:34:54 +01:00 committed by GitHub
parent c7b3d31415
commit 07baa9212b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 358 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
common/rust/ast/core/Cargo.toml
View File

@ -13,6 +13,7 @@ failure = { version = "0.1.5" }
serde = { version = "1.0", features = ["derive", "rc"] }
serde_json = { version = "1.0" }
shrinkwraprs = { version = "0.2.1" }
uuid = { version = "0.8.1", features = ["serde", "v4"] }
ast-macros = { version = "0.1.0", path = "../macros" }
prelude = { version = "0.1.0", path = "../../prelude" }

178
common/rust/ast/core/src/lib.rs
View File

@ -4,12 +4,23 @@
use prelude::*;
use serde::{Serialize, Deserialize};
use serde::ser::{Serializer, SerializeStruct};
use serde::de::{Deserializer, Visitor};
use uuid::Uuid;
use ast_macros::*;
use shapely::*;
pub type Stream<T> = Vec<T>;
// ==============
// === Errors ===
// ==============
/// Exception raised by macro-generated TryFrom methods that try to "downcast"
/// enum type to its variant subtype if different constructor was used.
#[derive(Display, Debug, Fail)]
pub struct WrongEnum { pub expected_con: String }
// ============
// === Tree ===
// ============
@ -84,13 +95,17 @@ impl<T> Layer<T> for Layered<T> {
/// to either of the implementation need to be applied to the other one as well.
///
/// Each AST node is annotated with span and an optional ID.
#[derive(Eq, PartialEq, Debug, Shrinkwrap, Serialize, Deserialize)]
#[derive(Eq, PartialEq, Debug, Shrinkwrap)]
#[shrinkwrap(mutable)]
pub struct Ast {
#[serde(flatten)]
pub wrapped: Rc<WithID<WithSpan<Shape<Ast>>>>
}
impl Clone for Ast {
fn clone(&self) -> Self {
Ast { wrapped: self.wrapped.clone() }
}
}
impl Ast {
pub fn iter(&self) -> Rc<dyn Iterator<Item = &'_ Shape<Ast>> + '_> {
@ -99,12 +114,20 @@ impl Ast {
}
pub fn shape(&self) -> &Shape<Ast> {
&self.wrapped.wrapped.wrapped
self
}
/// Wraps given shape with an optional ID into Ast. Span will ba
/// automatically calculated based on Shape.
pub fn new<S: Into<Shape<Ast>>>(shape: S, id: Option<ID>) -> Ast {
let shape = shape.into();
let shape: Shape<Ast> = shape.into();
let span = shape.span();
Ast::new_with_span(shape, id, span)
}
pub fn new_with_span<S: Into<Shape<Ast>>>
(shape: S, id: Option<ID>, span: usize) -> Ast {
let shape = shape.into();
let with_span = WithSpan { wrapped: shape, span };
let with_id = WithID { wrapped: with_span, id };
Ast { wrapped: Rc::new(with_id) }
@ -126,6 +149,78 @@ From<T> for Ast {
}
}
// Serialization & Deserialization //
/// Literals used in `Ast` serialization and deserialization.
pub mod ast_schema {
pub const STRUCT_NAME: &str = "Ast";
pub const SHAPE: &str = "shape";
pub const ID: &str = "id";
pub const SPAN: &str = "span";
pub const FIELDS: [&str; 3] = [SHAPE, ID, SPAN];
pub const COUNT: usize = FIELDS.len();
}
impl Serialize for Ast {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer {
use ast_schema::*;
let mut state = serializer.serialize_struct(STRUCT_NAME, COUNT)?;
state.serialize_field(SHAPE, &self.shape())?;
if self.id.is_some() {
state.serialize_field(ID, &self.id)?;
}
state.serialize_field(SPAN, &self.span)?;
state.end()
}
}
/// Type to provide serde::de::Visitor to deserialize data into `Ast`.
struct AstDeserializationVisitor;
impl<'de> Visitor<'de> for AstDeserializationVisitor {
type Value = Ast;
fn expecting
(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
use ast_schema::*;
write!(formatter, "an object with `{}` and `{}` fields", SHAPE, SPAN)
}
fn visit_map<A>
(self, mut map: A) -> Result<Self::Value, A::Error>
where A: serde::de::MapAccess<'de>, {
use ast_schema::*;
let mut shape: Option<Shape<Ast>> = None;
let mut id: Option<Option<ID>> = None;
let mut span: Option<usize> = None;
while let Some(key) = map.next_key()? {
match key {
SHAPE => shape = Some(map.next_value()?),
ID => id = Some(map.next_value()?),
SPAN => span = Some(map.next_value()?),
_ => {},
}
}
let shape = shape.ok_or(serde::de::Error::missing_field(SHAPE))?;
let id = id.unwrap_or(None); // allow missing `id` field
let span = span.ok_or(serde::de::Error::missing_field(SPAN))?;
Ok(Ast::new_with_span(shape, id, span))
}
}
impl<'de> Deserialize<'de> for Ast {
fn deserialize<D>(deserializer: D) -> Result<Ast, D::Error>
where D: Deserializer<'de> {
use ast_schema::FIELDS;
let visitor = AstDeserializationVisitor;
deserializer.deserialize_struct("AstOf", &FIELDS, visitor)
}
}
// =============
// === Shape ===
// =============
@ -147,7 +242,7 @@ From<T> for Ast {
Text (Text<T>),
// === Expressions ===
App { func : T , off : usize , arg: T },
Prefix { func : T , off : usize , arg: T },
Infix { larg : T , loff : usize , opr: T , roff: usize , rarg: T },
SectLeft { arg : T , off : usize , opr: T },
SectRight { opr : T , off : usize , arg: T },
@ -211,22 +306,22 @@ pub type TextBlock<T> = Vec<TextLine<T>>;
// =============
#[ast] pub struct Block<T> {
ty : BlockType,
ident : usize,
empty_lines : usize,
first_line : BlockLine<T>,
lines : Vec<BlockLine<Option<T>>>,
is_orphan : bool,
pub ty : BlockType,
pub ident : usize,
pub empty_lines : usize,
pub first_line : BlockLine<T>,
pub lines : Vec<BlockLine<Option<T>>>,
pub is_orphan : bool,
}
#[ast] pub enum BlockType { Continuous, Discontinuous }
#[ast] pub struct BlockLine <T> { elem: T, off: usize }
#[ast] pub struct BlockLine <T> { pub elem: T, pub off: usize }
// ==============
// === Module ===
// ==============
#[ast] pub struct Module<T> { lines: Vec<BlockLine<Option<T>>> }
#[ast] pub struct Module<T> { pub lines: Vec<BlockLine<Option<T>>> }
// =============
// === Macro ===
@ -397,12 +492,12 @@ impl HasSpan for &str {
// === WithID ===
pub type ID = Uuid;
pub trait HasID {
fn id(&self) -> Option<ID>;
}
pub type ID = i32;
#[derive(Eq, PartialEq, Debug, Shrinkwrap, Serialize, Deserialize)]
#[shrinkwrap(mutable)]
pub struct WithID<T> {
@ -522,6 +617,15 @@ impl HasSpan for Var {
#[cfg(test)]
mod tests {
use super::*;
use serde::de::DeserializeOwned;
/// Assert that given value round trips JSON serialization.
fn round_trips<T>(input_val: &T)
where T: Serialize + DeserializeOwned + PartialEq + Debug {
let json_str = serde_json::to_string(&input_val).unwrap();
let deserialized_val: T = serde_json::from_str(&json_str).unwrap();
assert_eq!(*input_val, deserialized_val);
}
#[test]
fn var_smart_constructor() {
@ -547,16 +651,38 @@ mod tests {
#[test]
fn serialization_round_trip() {
let var_name = "foo";
let v1 = Var { name: var_name.to_string() };
let v1_str = serde_json::to_string(&v1).unwrap();
let v2: Var = serde_json::from_str(&v1_str).unwrap();
assert_eq!(v1, v2);
let make_var = || Var { name: "foo".into() };
round_trips(&make_var());
let id = Some(15);
let ast1 = Ast::new(v1, id);
let ast_str = serde_json::to_string(&ast1).unwrap();
let ast2: Ast = serde_json::from_str(&ast_str).unwrap();
assert_eq!(ast1, ast2);
let ast_without_id = Ast::new(make_var(), None);
round_trips(&ast_without_id);
let id = Uuid::parse_str("15").ok();
let ast_with_id = Ast::new(make_var(), id);
round_trips(&ast_with_id);
}
#[test]
fn deserialize_var() {
let var_name = "foo";
let uuid_str = "51e74fb9-75a4-499d-9ea3-a90a2663b4a1";
let sample_json = serde_json::json!({
"shape": { "Var":{"name": var_name}},
"id": uuid_str,
"span": var_name.len()
});
let sample_json_text = sample_json.to_string();
let ast: Ast = serde_json::from_str(&sample_json_text).unwrap();
let expected_uuid = Uuid::parse_str(uuid_str).ok();
assert_eq!(ast.id, expected_uuid);
let expected_span = 3;
assert_eq!(ast.span, expected_span);
let expected_var = Var { name: var_name.into() };
let expected_shape = Shape::from(expected_var);
assert_eq!(*ast.shape(), expected_shape);
}
}

54
common/rust/ast/macros/src/lib.rs
View File

@ -143,6 +143,8 @@ fn gen_variant_decl
/// Generate `From` trait implementations converting from each of extracted
/// types back into primary enumeration.
/// Generate `TryFrom` implementation from primary enumeration into each
/// extracted type.
fn gen_from_impls
( ident : &syn::Ident
, decl : &syn::DeriveInput
@ -150,6 +152,7 @@ fn gen_from_impls
) -> TokenStream {
let sum_label = &decl.ident;
let variant_label = &variant.ident;
let variant_name = variant_label.to_string();
let sum_params = &decl.generics.params
.iter().cloned().collect::<Vec<_>>();
@ -168,6 +171,57 @@ fn gen_from_impls
#sum_label::#ident(t)
}
}
// impl<'t, T> TryFrom<&'t Shape<T>> for &'t Infix<T> {
// type Error = WrongEnum;
// fn try_from(value: &'t Shape<T>) -> Result<Self, Self::Error> {
// match value {
// Shape::Infix(elem) => Ok (elem),
// _ => {
// let error = WrongEnum {
// expected_con : "Infix" };
// Err(error)
// },
// }
// }
// }
impl<'t, #(#sum_params),*> TryFrom<&'t #sum_label<#(#sum_params),*>>
for &'t #variant_label<#(#variant_params),*> {
type Error = WrongEnum;
fn try_from
(value: &'t #sum_label<#(#sum_params),*>)
-> Result<Self, Self::Error> {
match value {
#sum_label::#ident(elem) => Ok(elem),
_ => {
let error = WrongEnum {
expected_con: #variant_name.to_string() };
Err(error)
},
}
}
}
// same as above but for values
impl<#(#sum_params),*> TryFrom<#sum_label<#(#sum_params),*>>
for #variant_label<#(#variant_params),*> {
type Error = WrongEnum;
fn try_from
(value: #sum_label<#(#sum_params),*>)
-> Result<Self, Self::Error> {
match value {
#sum_label::#ident(elem) => Ok(elem),
_ => {
let error = WrongEnum {
expected_con: #variant_name.to_string() };
Err(error)
},
}
}
}
}
}

1
common/rust/parser/Cargo.toml
View File

@ -8,6 +8,7 @@ edition = "2018"
crate-type = ["cdylib", "rlib"]
[dependencies]
ast = { version = "0.1.0", path = "../ast/core" }
failure = "0.1"
matches = "0.1"
prelude = { version = "0.1.0", path = "../prelude" }

12
common/rust/parser/src/api.rs
View File

@ -1,22 +1,16 @@
use prelude::*;
pub type Ast = ast::Ast;
// ============
// == Parser ==
// ============
/// Entity being able to parse Luna programs into Luna's AST.
pub trait IsParser {
fn parse(&mut self, program: String) -> Result<AST>;
fn parse(&mut self, program: String) -> Result<Ast>;
}
// =========
// == AST ==
// =========
// TODO: placeholder until we have real AST, see:
// https://github.com/luna/enso/issues/296
pub type AST = String;
// ===========
// == Error ==
// ===========

2
common/rust/parser/src/jsclient.rs
View File

@ -24,7 +24,7 @@ impl Client {
}
impl IsParser for Client {
fn parse(&mut self, _program: String) -> api::Result<api::AST> {
fn parse(&mut self, _program: String) -> api::Result<api::Ast> {
Err(api::interop_error(Error::NotImplemented))
}
}

2
common/rust/parser/src/lib.rs
View File

@ -40,7 +40,7 @@ impl Parser {
}
impl api::IsParser for Parser {
fn parse(&mut self, program: String) -> api::Result<api::AST> {
fn parse(&mut self, program: String) -> api::Result<api::Ast> {
self.deref_mut().parse(program)
}
}

12
common/rust/parser/src/wsclient.rs
View File

@ -81,7 +81,7 @@ pub enum Request {
/// All responses that Parser Service might reply with.
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub enum Response {
Success { ast: String },
Success { ast_json: String },
Error { message: String },
}
@ -160,6 +160,12 @@ mod internal {
self.recv_response()
}
}
/// Deserialize AST from JSON text received from WS Parser Service.
pub fn from_json(json_text: &str) -> api::Result<api::Ast> {
let ast = serde_json::from_str::<api::Ast>(json_text);
Ok(ast.map_err(|e| JsonSerializationError(e))?)
}
}
impl Client {
@ -183,11 +189,11 @@ impl Client {
}
impl api::IsParser for Client {
fn parse(&mut self, program: String) -> api::Result<api::AST> {
fn parse(&mut self, program: String) -> api::Result<api::Ast> {
let request = Request::ParseRequest { program };
let response = self.rpc_call(request)?;
match response {
Response::Success { ast } => Ok(ast),
Response::Success { ast_json } => internal::from_json(&ast_json),
Response::Error { message } => Err(ParsingError(message)),
}
}

93
common/rust/parser/tests/parsing.rs Normal file
View File

@ -0,0 +1,93 @@
use prelude::*;
use parser::api::IsParser;
use ast::{Ast, Shape};
/// Takes Ast being a module with a single line and returns that line's AST.
fn expect_single_line(ast: &Ast) -> &Ast {
let shape = ast.shape();
let module: &ast::Module<Ast> = shape.try_into().unwrap();
assert_eq!(module.lines.len(), 1, "module expected to have a single line");
let line = module.lines.iter().nth(0).unwrap();
line.elem.as_ref().unwrap()
}
/// "Downcasts" given AST's Shape to `T`.
fn expect_shape<'t, T>(ast: &'t Ast) -> &'t T
where &'t Shape<Ast>: TryInto<&'t T> {
match ast.shape().try_into() {
Ok(shape) => shape,
_ => panic!("failed converting shape"),
}
}
/// Persists parser (which is expensive to construct, so we want to reuse it
/// between tests. Additionally, hosts a number of helper methods.
struct TestHelper(parser::Parser);
impl TestHelper {
fn new() -> TestHelper {
TestHelper(parser::Parser::new_or_panic())
}
fn parse_line(&mut self, program: &str) -> Ast {
let ast = self.0.parse(program.into()).unwrap();
let line = expect_single_line(&ast);
line.clone()
}
// TODO: make generic, should work for all shape subtypes.
fn parse_shape_var<F>(&mut self, program: &str, tester: F)
where F: FnOnce(&ast::Var) -> () {
let ast = self.parse_line(program);
let shape = expect_shape(&ast);
tester(shape);
}
fn deserialize_blank(&mut self) {
let _ast = self.parse_line("_");
}
fn deserialize_cons(&mut self) {
let _ast = self.parse_line("FooBar");
}
fn deserialize_mod(&mut self) {
let _ast = self.parse_line("+=");
}
fn deserialize_prefix(&mut self) {
let _ast = self.parse_line("foo bar");
}
fn deserialize_infix(&mut self) {
let _ast = self.parse_line("foo + bar");
}
fn deserialize_var(&mut self) {
self.parse_shape_var("foo", |var| {
let expected_var = ast::Var { name: "foo".into() };
assert_eq!(var, &expected_var);
});
}
fn run(&mut self) {
self.deserialize_blank();
self.deserialize_cons();
self.deserialize_mod();
self.deserialize_prefix();
self.deserialize_infix();
self.deserialize_var();
}
}
/// A single entry point for all the tests here using external parser.
///
/// Setting up the parser is costly, so we run all tests as a single batch.
/// Until proper CI solution for calling external parser is devised, this
/// test is marked with `#[ignore]`.
#[test]
#[ignore]
fn parser_tests() {
TestHelper::new().run()
}

2
common/scala/parser-service/src/main/scala/org/enso/parserservice/Protocol.scala
View File

@ -13,7 +13,7 @@ object Protocol {
final case class ParseRequest(program: String) extends Request
sealed trait Response
final case class Success(ast: String) extends Response
final case class Success(ast_json: String) extends Response
final case class Error(message: String) extends Response
}

64
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala
View File

@ -272,7 +272,7 @@ object Shape extends ShapeImplicit {
/// App ///
///////////
sealed trait App[T] extends Shape[T]
final case class Prefix[T](fn: T, off: Int, arg: T) extends App[T]
final case class Prefix[T](func: T, off: Int, arg: T) extends App[T]
final case class Infix[T](
larg: T,
loff: Int,
@ -674,15 +674,15 @@ object Shape extends ShapeImplicit {
implicit def ftor: Functor[Prefix] = semi.functor
implicit def fold: Foldable[Prefix] = semi.foldable
implicit def repr[T: Repr]: Repr[Prefix[T]] =
t => R + t.fn + t.off + t.arg
t => R + t.func + t.off + t.arg
implicit def ozip[T: HasSpan]: OffsetZip[Prefix, T] =
t =>
t.copy(
fn = (Index.Start, t.fn),
arg = (Index(t.fn.span + t.off), t.arg)
func = (Index.Start, t.func),
arg = (Index(t.func.span + t.off), t.arg)
)
implicit def span[T: HasSpan]: HasSpan[Prefix[T]] =
t => t.fn.span + t.off + t.arg.span
t => t.func.span + t.off + t.arg.span
}
object Infix {
@ -1135,7 +1135,7 @@ sealed trait ShapeImplicit {
* and are defined within the [[Shape]] object. Shapes contain information
* about names of children and spacing between them, for example, the
* [[Shape.Prefix]] shape contains reference to function being its first child
* ([[Shape.Prefix.fn]]), spacing between the function and its argument
* ([[Shape.Prefix.func]]), spacing between the function and its argument
* ([[Shape.Prefix.off]]), and the argument itself ([[Shape.Prefix.arg]]).
*
* ==[[ASTOf]] as Catamorphism==
@ -1236,7 +1236,7 @@ object AST {
def tokenize(ast: AST): Shifted.List1[AST] = {
@tailrec
def go(ast: AST, out: AST.Stream): Shifted.List1[AST] = ast match {
case App.Prefix.any(t) => go(t.fn, Shifted(t.off, t.arg) :: out)
case App.Prefix.any(t) => go(t.func, Shifted(t.off, t.arg) :: out)
case _ => Shifted.List1(ast, out)
}
go(ast, List())
@ -1354,7 +1354,9 @@ object AST {
}
}
object ASTOf extends AstImplicits {
object ASTOf extends AstImplicits
trait AstImplicits extends AstImplicits2 {
implicit def unwrap[T[_]](t: ASTOf[T]): T[AST] = t.shape
implicit def repr[T[S] <: Shape[S]]: Repr[ASTOf[T]] =
t => implicitly[Repr[Shape[AST]]].repr(t.shape)
@ -1363,39 +1365,35 @@ object AST {
t: T[AST]
)(implicit ev: HasSpan[T[AST]]): ASTOf[T] = ASTOf(t, ev.span(t))
}
trait AstImplicits extends AstImplicits2 {
implicit def encoder_spec(
implicit ev: Encoder[Shape[AST]]
): Encoder[AST] = encoder
}
trait AstImplicits2 {
implicit def encoder[T[_]](
implicit ev: Encoder[T[AST]]
): Encoder[ASTOf[T]] = (ast) => {
val obj1 = ev(ast.shape).asObject.get
val obj2 = obj1.mapValues(s => {
val s2 =
addField(s, "id", implicitly[Encoder[Option[ID]]].apply(ast.id))
addField(s2, "span", implicitly[Encoder[Int]].apply(ast.span))
})
Json.fromJsonObject(obj2)
// Note: [JSON Schema]
implicit def encoder[T[S] <: Shape[S]](
implicit ev: Encoder[Shape[AST]]
): Encoder[ASTOf[T]] = ast => {
import io.circe.syntax._
val shape = "shape" -> ev(ast.shape)
val id = ast.id.map("id" -> _.asJson)
val span = "span" -> ast.span.asJson
val fields = Seq(shape) ++ id.toSeq :+ span
Json.fromFields(fields)
}
}
// FIXME: refactor https://github.com/luna/enso/issues/297
def addField(base: Json, name: String, value: Json): Json = {
final case class NotAnObject() extends Exception {
override def getMessage: String =
s"Cannot add field {name} to a non-object JSON!"
}
val obj = base.asObject.getOrElse(throw NotAnObject())
val obj2 = (name, value) +: obj
Json.fromJsonObject(obj2)
}
/* Note: [JSON Schema]
* ~~~~~~~~~~~~~~~~~~~
* Each AST node is serialized to a map with `shape`, `span` and,
* optionally, `id` keys. `shape` is always serialized as if base trait
* were encoded, even if the final case class type is known. This is
* required for consistency with Rust AST implementation, which uses a
* monomorphic AST type and has no means of expressing types like
* `AstOf[Shape.Var]`.
*/
//// ASTOps ////
@ -1730,7 +1728,7 @@ object AST {
object Prefix {
val any = UnapplyByType[Prefix]
def unapply(t: AST) = Unapply[Prefix].run(t => (t.fn, t.arg))(t)
def unapply(t: AST) = Unapply[Prefix].run(t => (t.func, t.arg))(t)
def apply(fn: AST, off: Int, arg: AST): Prefix =
Shape.Prefix(fn, off, arg)
def apply(fn: AST, arg: AST): Prefix = Prefix(fn, 1, arg)

2
common/scala/syntax/specialization/src/main/scala/org/enso/syntax/text/Parser.scala
View File

@ -237,7 +237,7 @@ class Parser {
*/
def attachLocations(ast: AST, startOffset: Int): AST = ast match {
case App.Prefix.any(app) =>
val locatedFn = attachLocations(app.fn, startOffset)
val locatedFn = attachLocations(app.func, startOffset)
val locatedArg =
attachLocations(app.arg, startOffset + locatedFn.span + app.off)
App.Prefix(locatedFn, app.off, locatedArg)