From 00c10870e241ab115aac78e53b1f59c8f00b4fd7 Mon Sep 17 00:00:00 2001 From: howardwu Date: Sat, 1 Aug 2020 21:50:47 -0700 Subject: [PATCH] Implements working typed ast serialization --- ast/Cargo.toml | 2 + ast/tests/mod.rs | 1 + ast/tests/serialization/expected_ast.json | 91 +++++++++++++++++++ ast/tests/serialization/json.rs | 22 +++++ ast/tests/serialization/main.leo | 3 + ast/tests/serialization/mod.rs | 1 + typed/src/common/identifier.rs | 74 ++++++++++++++- typed/src/imports/import.rs | 2 +- typed/src/imports/import_symbol.rs | 4 +- typed/src/imports/package.rs | 2 +- typed/src/imports/package_access.rs | 2 +- typed/src/lib.rs | 9 +- typed/src/program.rs | 2 +- typed/tests/mod.rs | 1 + .../serialization/expected_typed_ast.json | 66 ++++++++++++++ typed/tests/serialization/json.rs | 79 ++++++++++++++++ typed/tests/serialization/main.leo | 3 + typed/tests/serialization/mod.rs | 1 + 18 files changed, 355 insertions(+), 10 deletions(-) create mode 100644 ast/tests/mod.rs create mode 100644 ast/tests/serialization/expected_ast.json create mode 100644 ast/tests/serialization/json.rs create mode 100644 ast/tests/serialization/main.leo create mode 100644 ast/tests/serialization/mod.rs create mode 100644 typed/tests/mod.rs create mode 100644 typed/tests/serialization/expected_typed_ast.json create mode 100644 typed/tests/serialization/json.rs create mode 100644 typed/tests/serialization/main.leo create mode 100644 typed/tests/serialization/mod.rs diff --git a/ast/Cargo.toml b/ast/Cargo.toml index 7050904de1..0df56b73a9 100644 --- a/ast/Cargo.toml +++ b/ast/Cargo.toml @@ -18,3 +18,5 @@ pest_derive = { version = "2.0" } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } thiserror = { version = "1.0" } + +[dev-dependencies] diff --git a/ast/tests/mod.rs b/ast/tests/mod.rs new file mode 100644 index 0000000000..1118e0b570 --- /dev/null +++ b/ast/tests/mod.rs @@ -0,0 +1 @@ +mod serialization; diff --git a/ast/tests/serialization/expected_ast.json b/ast/tests/serialization/expected_ast.json new file mode 100644 index 0000000000..f4286789e0 --- /dev/null +++ b/ast/tests/serialization/expected_ast.json @@ -0,0 +1,91 @@ +{ + "imports": [], + "circuits": [], + "functions": [ + { + "function_name": { + "value": "main", + "span": { + "input": "main", + "start": 9, + "end": 13 + } + }, + "parameters": [], + "returns": [], + "statements": [ + { + "Return": { + "return_": { + "Single": { + "Binary": { + "operation": "Add", + "left": { + "Value": { + "Implicit": { + "number": { + "value": "1", + "span": { + "input": "1", + "start": 29, + "end": 30 + } + }, + "span": { + "input": "1", + "start": 29, + "end": 30 + } + } + } + }, + "right": { + "Value": { + "Implicit": { + "number": { + "value": "1", + "span": { + "input": "1", + "start": 33, + "end": 34 + } + }, + "span": { + "input": "1", + "start": 33, + "end": 34 + } + } + } + }, + "span": { + "input": "1 + 1", + "start": 29, + "end": 34 + } + } + } + }, + "span": { + "input": "return 1 + 1", + "start": 22, + "end": 34 + } + } + } + ], + "span": { + "input": "function main() {\n return 1 + 1\n}\n", + "start": 0, + "end": 37 + } + } + ], + "tests": [], + "eoi": null, + "span": { + "input": "function main() {\n return 1 + 1\n}\n", + "start": 0, + "end": 37 + } +} \ No newline at end of file diff --git a/ast/tests/serialization/json.rs b/ast/tests/serialization/json.rs new file mode 100644 index 0000000000..1e5a2386e1 --- /dev/null +++ b/ast/tests/serialization/json.rs @@ -0,0 +1,22 @@ +use leo_ast::LeoAst; + +use std::path::PathBuf; + +#[test] +fn test_serialization() { + let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + program_filepath.push("tests/serialization/main.leo"); + + let expected = include_str!("./expected_ast.json"); + + // Loads the Leo code as a string from the given file path. + let program_string = LeoAst::load_file(&program_filepath).unwrap(); + + // Parses the Leo file and constructs an abstract syntax tree. + let ast = LeoAst::new(&program_filepath, &program_string).unwrap(); + + // Serializes the abstract syntax tree into JSON format. + let serialized_ast = LeoAst::to_json_string(&ast).unwrap(); + + assert_eq!(expected, serialized_ast); +} diff --git a/ast/tests/serialization/main.leo b/ast/tests/serialization/main.leo new file mode 100644 index 0000000000..ef22115243 --- /dev/null +++ b/ast/tests/serialization/main.leo @@ -0,0 +1,3 @@ +function main() { + return 1 + 1 +} diff --git a/ast/tests/serialization/mod.rs b/ast/tests/serialization/mod.rs new file mode 100644 index 0000000000..cff0e9089e --- /dev/null +++ b/ast/tests/serialization/mod.rs @@ -0,0 +1 @@ +mod json; diff --git a/typed/src/common/identifier.rs b/typed/src/common/identifier.rs index cd152b5ccd..cb9d5ae08f 100644 --- a/typed/src/common/identifier.rs +++ b/typed/src/common/identifier.rs @@ -1,11 +1,21 @@ use crate::Span; use leo_ast::common::Identifier as AstIdentifier; -use serde::{Deserialize, Serialize}; -use std::fmt; +use serde::{ + de::{self, Visitor}, + Deserialize, + Deserializer, + Serialize, + Serializer, +}; +use std::{collections::BTreeMap, fmt}; /// An identifier in the constrained program. -#[derive(Clone, Hash, Serialize, Deserialize)] +/// +/// Attention - When adding or removing fields from this struct, +/// please remember to update it's Serialize and Deserialize implementation +/// to reflect the new struct instantiation. +#[derive(Clone, Hash)] pub struct Identifier { pub name: String, pub span: Span, @@ -44,3 +54,61 @@ impl PartialEq for Identifier { } impl Eq for Identifier {} + +impl Serialize for Identifier { + fn serialize(&self, serializer: S) -> Result { + // Converts an element that implements Serialize into a string. + fn to_json_string(element: &E) -> Result { + serde_json::to_string(&element).map_err(|e| Error::custom(e.to_string())) + } + + // Load the struct elements into a BTreeMap (to preserve serialized ordering of keys). + let mut key: BTreeMap = BTreeMap::new(); + key.insert("name".to_string(), self.name.clone()); + key.insert("span".to_string(), to_json_string(&self.span)?); + + // Convert the serialized object into a string for use as a key. + serializer.serialize_str(&to_json_string(&key)?) + } +} + +impl<'de> Deserialize<'de> for Identifier { + fn deserialize>(deserializer: D) -> Result { + struct IdentifierVisitor; + + impl<'de> Visitor<'de> for IdentifierVisitor { + type Value = Identifier; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string encoding the typed Identifier struct") + } + + /// Implementation for recovering a string that serializes Identifier. + fn visit_str(self, value: &str) -> Result { + // Converts a serialized string into an element that implements Deserialize. + fn to_json_string<'a, D: Deserialize<'a>, Error: serde::de::Error>( + serialized: &'a str, + ) -> Result { + serde_json::from_str::<'a>(serialized).map_err(|e| Error::custom(e.to_string())) + } + + // Convert the serialized string into a BTreeMap to recover Identifier. + let key: BTreeMap = to_json_string(value)?; + + let name = match key.get("name") { + Some(name) => name.clone(), + None => return Err(E::custom("missing 'name' in serialized Identifier struct")), + }; + + let span: Span = match key.get("span") { + Some(span) => to_json_string(span)?, + None => return Err(E::custom("missing 'span' in serialized Identifier struct")), + }; + + Ok(Identifier { name, span }) + } + } + + deserializer.deserialize_str(IdentifierVisitor) + } +} diff --git a/typed/src/imports/import.rs b/typed/src/imports/import.rs index 5b7ce821c6..c049b60928 100644 --- a/typed/src/imports/import.rs +++ b/typed/src/imports/import.rs @@ -6,7 +6,7 @@ use leo_ast::imports::Import as AstImport; use serde::{Deserialize, Serialize}; use std::fmt; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct Import { pub package: Package, pub span: Span, diff --git a/typed/src/imports/import_symbol.rs b/typed/src/imports/import_symbol.rs index e8b57b3999..2a5668b244 100644 --- a/typed/src/imports/import_symbol.rs +++ b/typed/src/imports/import_symbol.rs @@ -4,7 +4,7 @@ use leo_ast::imports::ImportSymbol as AstImportSymbol; use serde::{Deserialize, Serialize}; use std::fmt; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct ImportSymbol { pub symbol: Identifier, pub alias: Option, @@ -31,7 +31,7 @@ impl fmt::Display for ImportSymbol { } } -// todo: remove this +// TODO (collin): remove this impl fmt::Debug for ImportSymbol { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if self.alias.is_some() { diff --git a/typed/src/imports/package.rs b/typed/src/imports/package.rs index 666f92e5b9..16776a2b97 100644 --- a/typed/src/imports/package.rs +++ b/typed/src/imports/package.rs @@ -4,7 +4,7 @@ use leo_ast::imports::Package as AstPackage; use serde::{Deserialize, Serialize}; use std::fmt; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct Package { pub name: Identifier, pub access: PackageAccess, diff --git a/typed/src/imports/package_access.rs b/typed/src/imports/package_access.rs index bf968f9920..d019b12534 100644 --- a/typed/src/imports/package_access.rs +++ b/typed/src/imports/package_access.rs @@ -4,7 +4,7 @@ use leo_ast::imports::PackageAccess as AstPackageAccess; use serde::{Deserialize, Serialize}; use std::fmt; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum PackageAccess { Star(Span), SubPackage(Box), diff --git a/typed/src/lib.rs b/typed/src/lib.rs index 6b0f9915d0..5f967f356b 100644 --- a/typed/src/lib.rs +++ b/typed/src/lib.rs @@ -38,6 +38,7 @@ use leo_ast::LeoAst; use serde_json; +#[derive(Debug, Eq, PartialEq)] pub struct LeoTypedAst { typed_ast: Program, } @@ -55,8 +56,14 @@ impl LeoTypedAst { self.typed_ast } - /// Serializes the abstract syntax tree into a JSON string. + /// Serializes the typed syntax tree into a JSON string. pub fn to_json_string(&self) -> Result { Ok(serde_json::to_string_pretty(&self.typed_ast)?) } + + /// Deserializes the JSON string into a typed syntax tree. + pub fn from_json_string(json: &str) -> Result { + let typed_ast: Program = serde_json::from_str(json)?; + Ok(Self { typed_ast }) + } } diff --git a/typed/src/program.rs b/typed/src/program.rs index 5b9d914845..f99fefa878 100644 --- a/typed/src/program.rs +++ b/typed/src/program.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; /// A simple program with statement expressions, program arguments and program returns. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct Program { pub name: String, pub expected_inputs: Vec, diff --git a/typed/tests/mod.rs b/typed/tests/mod.rs new file mode 100644 index 0000000000..1118e0b570 --- /dev/null +++ b/typed/tests/mod.rs @@ -0,0 +1 @@ +mod serialization; diff --git a/typed/tests/serialization/expected_typed_ast.json b/typed/tests/serialization/expected_typed_ast.json new file mode 100644 index 0000000000..1ee5aea9c6 --- /dev/null +++ b/typed/tests/serialization/expected_typed_ast.json @@ -0,0 +1,66 @@ +{ + "name": "leo_typed_tree", + "expected_inputs": [], + "imports": [], + "circuits": {}, + "functions": { + "{\"name\":\"main\",\"span\":\"{\\\"text\\\":\\\" function main() {\\\",\\\"line\\\":1,\\\"start\\\":10,\\\"end\\\":14}\"}": { + "function_name": "{\"name\":\"main\",\"span\":\"{\\\"text\\\":\\\" function main() {\\\",\\\"line\\\":1,\\\"start\\\":10,\\\"end\\\":14}\"}", + "inputs": [], + "returns": [], + "statements": [ + { + "Return": [ + [ + { + "Add": [ + { + "Implicit": [ + "1", + { + "text": " return 1 + 1", + "line": 2, + "start": 12, + "end": 13 + } + ] + }, + { + "Implicit": [ + "1", + { + "text": " return 1 + 1", + "line": 2, + "start": 16, + "end": 17 + } + ] + }, + { + "text": " return 1 + 1", + "line": 2, + "start": 12, + "end": 17 + } + ] + } + ], + { + "text": " return 1 + 1", + "line": 2, + "start": 5, + "end": 17 + } + ] + } + ], + "span": { + "text": " function main() {", + "line": 1, + "start": 1, + "end": 1 + } + } + }, + "tests": {} +} \ No newline at end of file diff --git a/typed/tests/serialization/json.rs b/typed/tests/serialization/json.rs new file mode 100644 index 0000000000..da3a78a9c2 --- /dev/null +++ b/typed/tests/serialization/json.rs @@ -0,0 +1,79 @@ +use leo_ast::LeoAst; +use leo_typed::LeoTypedAst; + +use std::path::PathBuf; + +fn to_typed_ast(program_filepath: &PathBuf) -> LeoTypedAst { + // Loads the Leo code as a string from the given file path. + let program_string = LeoAst::load_file(program_filepath).unwrap(); + + // Parses the Leo file and constructs an abstract syntax tree. + let ast = LeoAst::new(&program_filepath, &program_string).unwrap(); + + // Parse the abstract syntax tree and constructs a typed syntax tree. + let typed_ast = LeoTypedAst::new("leo_typed_tree", &ast); + + typed_ast +} + +#[test] +fn test_serialize() { + // Construct a typed syntax tree from the given test file. + let typed_ast = { + let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + program_filepath.push("tests/serialization/main.leo"); + + to_typed_ast(&program_filepath) + }; + + // Serializes the typed syntax tree into JSON format. + let serialized_typed_ast = typed_ast.to_json_string().unwrap(); + + // Load the expected typed syntax tree. + let expected = include_str!("expected_typed_ast.json"); + + println!("{}", serialized_typed_ast); + assert_eq!(expected, serialized_typed_ast); +} + +#[test] +fn test_deserialize() { + // Load the expected typed syntax tree. + let expected_typed_ast = { + let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + program_filepath.push("tests/serialization/main.leo"); + + to_typed_ast(&program_filepath) + }; + + // Construct a typed syntax tree by deserializing a typed syntax tree JSON file. + let serialized_typed_ast = include_str!("expected_typed_ast.json"); + let typed_ast = LeoTypedAst::from_json_string(serialized_typed_ast).unwrap(); + + assert_eq!(expected_typed_ast, typed_ast); +} + +#[test] +fn test_serialize_deserialize_serialize() { + // Construct a typed syntax tree from the given test file. + let typed_ast = { + let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + program_filepath.push("tests/serialization/main.leo"); + + to_typed_ast(&program_filepath) + }; + + // Serializes the typed syntax tree into JSON format. + let serialized_typed_ast = typed_ast.to_json_string().unwrap(); + + // Deserializes the typed syntax tree into a LeoTypedAst. + let typed_ast = LeoTypedAst::from_json_string(&serialized_typed_ast).unwrap(); + + // Reserializes the typed syntax tree into JSON format. + let reserialized_typed_ast = typed_ast.to_json_string().unwrap(); + + // Load the expected typed syntax tree. + let expected = include_str!("expected_typed_ast.json"); + + assert_eq!(serialized_typed_ast, reserialized_typed_ast); +} diff --git a/typed/tests/serialization/main.leo b/typed/tests/serialization/main.leo new file mode 100644 index 0000000000..ef22115243 --- /dev/null +++ b/typed/tests/serialization/main.leo @@ -0,0 +1,3 @@ +function main() { + return 1 + 1 +} diff --git a/typed/tests/serialization/mod.rs b/typed/tests/serialization/mod.rs new file mode 100644 index 0000000000..cff0e9089e --- /dev/null +++ b/typed/tests/serialization/mod.rs @@ -0,0 +1 @@ +mod json;