Merge pull request #808 from AleoHQ/feature/abnf-grammar-converter

Adds 'leo-abnf' binary package for markdown conversion
This commit is contained in:
Collin Chin 2021-03-30 15:36:06 -07:00 committed by GitHub
commit e1837e4f7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 333 additions and 1 deletions

91
Cargo.lock generated
View File

@ -2,6 +2,25 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "abnf"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "020babd9b431b239cffff9eb1e49d8cad5f3581674d3938540801e633a3b5f69"
dependencies = [
"abnf-core",
"nom 6.1.2",
]
[[package]]
name = "abnf-core"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b514944cb7199c4201f54406bc58676a3e4f37d40bf8e3dbe30652ca82e3ddb4"
dependencies = [
"nom 6.1.2",
]
[[package]]
name = "addr2line"
version = "0.14.1"
@ -157,6 +176,18 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bitvec"
version = "0.19.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "blake2"
version = "0.9.1"
@ -290,7 +321,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
dependencies = [
"nom",
"nom 5.1.2",
]
[[package]]
@ -1202,6 +1233,14 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "leo-abnf"
version = "1.2.3"
dependencies = [
"abnf",
"anyhow",
]
[[package]]
name = "leo-asg"
version = "1.2.3"
@ -1399,6 +1438,19 @@ dependencies = [
"snarkvm-r1cs",
]
[[package]]
name = "lexical-core"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374"
dependencies = [
"arrayvec",
"bitflags",
"cfg-if 1.0.0",
"ryu",
"static_assertions",
]
[[package]]
name = "libc"
version = "0.2.89"
@ -1651,6 +1703,19 @@ dependencies = [
"version_check",
]
[[package]]
name = "nom"
version = "6.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2"
dependencies = [
"bitvec",
"funty",
"lexical-core",
"memchr",
"version_check",
]
[[package]]
name = "notify"
version = "4.0.15"
@ -2036,6 +2101,12 @@ dependencies = [
"proc-macro2 1.0.24",
]
[[package]]
name = "radium"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
[[package]]
name = "rand"
version = "0.8.3"
@ -2700,6 +2771,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strsim"
version = "0.8.0"
@ -2770,6 +2847,12 @@ dependencies = [
"unicode-xid 0.2.1",
]
[[package]]
name = "tap"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tempfile"
version = "3.2.0"
@ -3325,6 +3408,12 @@ dependencies = [
"winapi-build",
]
[[package]]
name = "wyz"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
[[package]]
name = "zip"
version = "0.5.10"

View File

@ -29,6 +29,7 @@ members = [
"asg",
"ast",
"compiler",
"grammar",
"imports",
"input",
"linter",

18
grammar/Cargo.toml Normal file
View File

@ -0,0 +1,18 @@
[package]
name = "leo-abnf"
version = "1.2.3"
authors = [ "The Aleo Team <hello@aleo.org>" ]
description = "ABNF to Markdown converter"
edition = "2018"
keywords = [
"aleo",
"cryptography",
"leo",
"programming-language",
"zero-knowledge",
"leo-abnf"
]
[dependencies]
abnf = "0.10.0"
anyhow = "1.0"

224
grammar/src/main.rs Normal file
View File

@ -0,0 +1,224 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
// ABNF PARSING RULES
//
// Header:
// ```abnf
// ; Introduction
// ; -------------
// ```
//
// Code block in docs (note double whitespace after colon):
// ```abnf
// ; code
// ; code
//```
//
// Rule:
// ```abnf
// address = "address"
// ```
//
// Line:
// ``` abnf
// ;;;;;;;;;
// ```
//
use abnf::types::{Node, Rule};
use anyhow::Result;
use std::collections::{HashMap, HashSet};
/// Processor's scope. Used when code block or definition starts or ends.
#[derive(Debug, Clone)]
enum Scope {
Free,
Code,
Definition(Rule),
}
/// Transforms abnf file into Markdown.
#[derive(Debug, Clone)]
struct Processor<'a> {
rules: HashMap<String, Rule>,
grammar: &'a str,
scope: Scope,
line: u32,
out: String,
}
impl<'a> Processor<'a> {
fn new(grammar: &'a str, abnf: Vec<Rule>) -> Processor<'a> {
// we need a hashmap to pull rules easily
let rules: HashMap<String, Rule> = abnf.into_iter().map(|rule| (rule.name().to_string(), rule)).collect();
Processor {
grammar,
line: 0,
out: String::new(),
rules,
scope: Scope::Free,
}
}
/// Main function for this struct.
/// Goes through each line and transforms it into proper markdown.
fn process(&mut self) -> Result<()> {
let mut lines = self.grammar.lines();
let mut prev = "";
while let Some(line) = lines.next() {
self.line += 1;
// code block in comment (not highlighted as abnf)
if line.starts_with("; ") {
self.enter_scope(Scope::Code)?;
self.append_str(&line[3..]);
// just comment. end of code block
} else if line.starts_with("; ") {
self.enter_scope(Scope::Free)?;
self.append_str(&line[2..]);
// horizontal rule - section separator
} else if line.starts_with(";;;;;;;;;;") {
self.enter_scope(Scope::Free)?;
self.append_str("\n--------\n");
// empty line in comment. end of code block
} else if line.starts_with(";") {
self.enter_scope(Scope::Free)?;
self.append_str("\n\n");
// just empty line. end of doc, start of definition
} else if line.len() == 0 {
self.enter_scope(Scope::Free)?;
self.append_str("");
// definition (may be multiline)
} else {
// if there's an equality sign and previous line was empty
if line.contains("=") && prev.len() == 0 {
let (def, _) = line.split_at(line.find("=").unwrap());
let def = def.trim();
// try to find rule matching definition or fail
let rule = self.rules.get(&def.to_string()).map(|rule| rule.clone()).unwrap();
self.enter_scope(Scope::Definition(rule))?;
}
self.append_str(line);
}
prev = line;
}
Ok(())
}
/// Append new line into output, add newline character.
fn append_str(&mut self, line: &str) {
self.out.push_str(line);
self.out.push_str("\n");
}
/// Enter new scope (definition or code block). Allows customizing
/// pre and post lines for each scope entered or exited.
fn enter_scope(&mut self, new_scope: Scope) -> Result<()> {
match (&self.scope, &new_scope) {
// exchange scopes between Free and Code
(Scope::Free, Scope::Code) => self.append_str("```"),
(Scope::Code, Scope::Free) => self.append_str("```"),
// exchange scopes between Free and Definition
(Scope::Free, Scope::Definition(rule)) => {
self.append_str(&format!("<a name=\"{}\"></a>", rule.name()));
self.append_str("```abnf");
}
(Scope::Definition(rule), Scope::Free) => {
let mut rules: Vec<String> = Vec::new();
parse_abnf_node(rule.node(), &mut rules);
// 1. leave only unique keys
// 2. map each rule into a link
// 3. join results as a list
// Note: GitHub only allows custom tags with 'user-content-' prefix
let keys = rules
.into_iter()
.collect::<HashSet<_>>()
.into_iter()
.map(|tag| format!("[{}](#user-content-{})", &tag, tag))
.collect::<Vec<String>>()
.join(", ");
self.append_str("```");
if keys.len() > 0 {
self.append_str(&format!("\nGo to: _{}_;\n", keys));
}
}
(_, _) => (),
};
self.scope = new_scope;
Ok(())
}
}
/// Recursively parse ABNF Node and fill sum vec with found rule names.
fn parse_abnf_node(node: &Node, sum: &mut Vec<String>) {
match node {
// these two are just vectors of rules
Node::Alternation(vec) | Node::Concatenation(vec) => {
for node in vec {
parse_abnf_node(node, sum);
}
}
Node::Group(node) | Node::Optional(node) => parse_abnf_node(node.as_ref(), sum),
// push rulename if it is known
Node::Rulename(name) => sum.push(name.clone()),
// do nothing for other nodes
_ => (),
}
}
fn main() -> Result<()> {
// Take Leo ABNF grammar file.
let grammar = include_str!("../abnf-grammar.txt");
// A. Coglio's proposal for %s syntax for case-sensitive statements has not been implemented
// in this library, so we need to remove all occurrences of %s in the grammar file.
// Link to this proposal: https://www.kestrel.edu/people/coglio/vstte18.pdf
let grammar = &str::replace(grammar, "%s", "");
// Parse ABNF to get list of all definitions.
let parsed = abnf::rulelist(grammar).map_err(|e| {
eprintln!("{}", &e);
anyhow::anyhow!(e)
})?;
// Init parser and run it. That's it.
let mut parser = Processor::new(grammar, parsed);
parser.process()?;
// Print result of conversion to STDOUT.
println!("{}", parser.out);
Ok(())
}