mirror of
https://github.com/enso-org/enso.git
synced 2024-09-11 21:27:15 +03:00
Prepare the repo for working on rust code (#841)
This commit is contained in:
parent
2f404b7f08
commit
f7d4ef546a
4
.cargo/config
Normal file
4
.cargo/config
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
|
||||||
|
[build]
|
||||||
|
target-dir = "target/rust/"
|
||||||
|
rustflags = []
|
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -14,5 +14,5 @@
|
|||||||
Please include the following checklist in your PR:
|
Please include the following checklist in your PR:
|
||||||
|
|
||||||
- [ ] The documentation has been updated if necessary.
|
- [ ] The documentation has been updated if necessary.
|
||||||
- [ ] All code conforms to the [Scala](https://github.com/luna/enso/blob/main/docs/style-guide/scala.md) and [Java](https://github.com/luna/enso/blob/main/docs/style-guide/java.md) style guides.
|
- [ ] All code conforms to the [Scala](https://github.com/luna/enso/blob/main/docs/style-guide/scala.md), [Java](https://github.com/luna/enso/blob/main/docs/style-guide/java.md), and [Rust](https://github.com/luna/enso/blob/main/docs/style-guide/rust.md) style guides.
|
||||||
- [ ] All code has been tested where possible.
|
- [ ] All code has been tested where possible.
|
||||||
|
2
.github/settings.yml
vendored
2
.github/settings.yml
vendored
@ -204,7 +204,7 @@ branches:
|
|||||||
required_status_checks:
|
required_status_checks:
|
||||||
# Require branches to be up to date before merging.
|
# Require branches to be up to date before merging.
|
||||||
strict: true
|
strict: true
|
||||||
contexts: ["Test (macOS-latest)", "Test (ubuntu-latest)", "license/cla"]
|
contexts: ["Test Engine (macOS-latest)", "Test Engine (ubuntu-latest)", "Test Parser (macOS-latest)", "Test Parser (ubuntu-latest)", " Test Parser (windows-latest) ", "license/cla"]
|
||||||
enforce_admins: null
|
enforce_admins: null
|
||||||
restrictions: null
|
restrictions: null
|
||||||
|
|
||||||
|
55
.github/workflows/rust.yml
vendored
Normal file
55
.github/workflows/rust.yml
vendored
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
name: Parser CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ main ]
|
||||||
|
pull_request:
|
||||||
|
branches: [ "*" ]
|
||||||
|
|
||||||
|
env:
|
||||||
|
wasmpackVersion: 0.8.1
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
name: Test Parser
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
timeout-minutes: 10
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
os: [macOS-latest, ubuntu-latest, windows-latest]
|
||||||
|
fail-fast: false
|
||||||
|
steps:
|
||||||
|
- name: Checkout Parser Sources
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
# Install Tooling
|
||||||
|
- name: Install Rust
|
||||||
|
uses: actions-rs/toolchain@v1
|
||||||
|
with:
|
||||||
|
toolchain: nightly-2020-06-09
|
||||||
|
override: true
|
||||||
|
- name: Install wasm-pack
|
||||||
|
uses: actions-rs/cargo@v1
|
||||||
|
with:
|
||||||
|
command: install
|
||||||
|
args: wasm-pack --version ${{ env.wasmpackVersion }}
|
||||||
|
|
||||||
|
# Caches
|
||||||
|
- name: Cache Cargo Registry
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ~/.cargo/registry
|
||||||
|
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }}
|
||||||
|
restore-keys: ${{ runner.os }}-cargo-registry
|
||||||
|
- name: Cache Cargo Test
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ./target/rust
|
||||||
|
key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }}
|
||||||
|
restore-keys: ${{ runner.os }}-cargo-build
|
||||||
|
|
||||||
|
# Tests
|
||||||
|
- name: Test Parser
|
||||||
|
uses: actions-rs/cargo@v1
|
||||||
|
with:
|
||||||
|
command: test
|
6
.github/workflows/scala.yml
vendored
6
.github/workflows/scala.yml
vendored
@ -1,4 +1,4 @@
|
|||||||
name: Enso CI
|
name: Engine CI
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
@ -27,7 +27,7 @@ env:
|
|||||||
jobs:
|
jobs:
|
||||||
# This job is responsible for testing the codebase
|
# This job is responsible for testing the codebase
|
||||||
test:
|
test:
|
||||||
name: Test
|
name: Test Engine
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
strategy:
|
strategy:
|
||||||
@ -93,7 +93,7 @@ jobs:
|
|||||||
|
|
||||||
# This job is responsible for building the artifacts
|
# This job is responsible for building the artifacts
|
||||||
build:
|
build:
|
||||||
name: Build
|
name: Build Engine
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
steps:
|
steps:
|
||||||
|
40
.gitignore
vendored
40
.gitignore
vendored
@ -5,15 +5,9 @@
|
|||||||
|
|
||||||
graal_dumps
|
graal_dumps
|
||||||
|
|
||||||
##########
|
#########
|
||||||
## Java ##
|
## JVM ##
|
||||||
##########
|
#########
|
||||||
|
|
||||||
*.class
|
|
||||||
|
|
||||||
###########
|
|
||||||
## Scala ##
|
|
||||||
###########
|
|
||||||
|
|
||||||
graal_dumps/
|
graal_dumps/
|
||||||
target/
|
target/
|
||||||
@ -25,6 +19,8 @@ target/
|
|||||||
##########
|
##########
|
||||||
|
|
||||||
Cargo.lock
|
Cargo.lock
|
||||||
|
**/*.rs.bk
|
||||||
|
wasm-pack.log
|
||||||
|
|
||||||
#############
|
#############
|
||||||
## Haskell ##
|
## Haskell ##
|
||||||
@ -32,28 +28,12 @@ Cargo.lock
|
|||||||
|
|
||||||
dist
|
dist
|
||||||
cabal-dev
|
cabal-dev
|
||||||
*.o
|
|
||||||
*.hi
|
|
||||||
*.chi
|
|
||||||
*.chs.h
|
|
||||||
*.dyn_o
|
|
||||||
*.dyn_hi
|
|
||||||
.hpc
|
|
||||||
.hsenv
|
|
||||||
.cabal-sandbox/
|
|
||||||
cabal.sandbox.config
|
|
||||||
*.cabal
|
|
||||||
*.prof
|
|
||||||
*.aux
|
|
||||||
*.hp
|
|
||||||
*.DS_Store
|
|
||||||
.stack-work/
|
.stack-work/
|
||||||
|
|
||||||
############
|
############
|
||||||
## System ##
|
## System ##
|
||||||
############
|
############
|
||||||
|
|
||||||
# OSX
|
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
############
|
############
|
||||||
@ -70,6 +50,7 @@ cabal.sandbox.config
|
|||||||
######################
|
######################
|
||||||
|
|
||||||
.idea/
|
.idea/
|
||||||
|
.vscode/
|
||||||
*.swp
|
*.swp
|
||||||
.projections.json
|
.projections.json
|
||||||
|
|
||||||
@ -83,6 +64,7 @@ scaladoc/
|
|||||||
#######################
|
#######################
|
||||||
## Benchmark Reports ##
|
## Benchmark Reports ##
|
||||||
#######################
|
#######################
|
||||||
|
|
||||||
bench-report.xml
|
bench-report.xml
|
||||||
|
|
||||||
##############
|
##############
|
||||||
@ -97,10 +79,4 @@ bench-report.xml
|
|||||||
#########
|
#########
|
||||||
|
|
||||||
.editorconfig
|
.editorconfig
|
||||||
.bloop
|
.bloop/
|
||||||
|
|
||||||
|
|
||||||
#########
|
|
||||||
## NPM ##
|
|
||||||
#########
|
|
||||||
node_modules/
|
|
||||||
|
33
.rustfmt.toml
Normal file
33
.rustfmt.toml
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
|
||||||
|
# General Configuration
|
||||||
|
unstable_features = true
|
||||||
|
max_width = 80
|
||||||
|
error_on_line_overflow = true
|
||||||
|
newline_style = "Unix"
|
||||||
|
|
||||||
|
# Operators
|
||||||
|
binop_separator = "Front"
|
||||||
|
|
||||||
|
# Whitespace
|
||||||
|
blank_lines_upper_bound = 1
|
||||||
|
|
||||||
|
# Code Layout
|
||||||
|
brace_style = "SameLineWhere"
|
||||||
|
combine_control_expr = true
|
||||||
|
empty_item_single_line = true
|
||||||
|
fn_single_line = true
|
||||||
|
format_strings = true
|
||||||
|
inline_attribute_width = 80
|
||||||
|
space_before_colon = false
|
||||||
|
space_after_colon = false
|
||||||
|
type_punctuation_density = "Wide"
|
||||||
|
|
||||||
|
# Comments
|
||||||
|
comment_width = 80
|
||||||
|
wrap_comments = true
|
||||||
|
format_code_in_doc_comments = true
|
||||||
|
normalize_comments = true
|
||||||
|
|
||||||
|
# Macros
|
||||||
|
format_macro_matchers = true
|
||||||
|
format_macro_bodies = true
|
25
Cargo.toml
Normal file
25
Cargo.toml
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
[workspace]
|
||||||
|
|
||||||
|
members = [
|
||||||
|
"parser/flexer"
|
||||||
|
]
|
||||||
|
|
||||||
|
[profile.dev]
|
||||||
|
opt-level = 0
|
||||||
|
lto = false
|
||||||
|
debug = true
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
opt-level = 3
|
||||||
|
lto = true
|
||||||
|
debug = false
|
||||||
|
|
||||||
|
[profile.bench]
|
||||||
|
opt-level = 3
|
||||||
|
lto = true
|
||||||
|
debug = false
|
||||||
|
|
||||||
|
[profile.test]
|
||||||
|
opt-level = 0
|
||||||
|
lto = false
|
||||||
|
debug = true
|
@ -43,4 +43,4 @@ It is broken up into categories as follows:
|
|||||||
- [**Syntax:**](./syntax) A specification of Enso's syntax.
|
- [**Syntax:**](./syntax) A specification of Enso's syntax.
|
||||||
- [**Types:**](./types) A specification of Enso's type system and type theory.
|
- [**Types:**](./types) A specification of Enso's type system and type theory.
|
||||||
- [**Debugger:**](./debugger) A specification of Enso's debugger.
|
- [**Debugger:**](./debugger) A specification of Enso's debugger.
|
||||||
|
- [**Parser:**](./parser) Design and specification of the Enso parser.
|
||||||
|
25
docs/parser/README.md
Normal file
25
docs/parser/README.md
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
---
|
||||||
|
layout: docs-index
|
||||||
|
title: Enso's Parser
|
||||||
|
category: summary
|
||||||
|
tags: [parser, readme]
|
||||||
|
order: 0
|
||||||
|
---
|
||||||
|
|
||||||
|
# Enso's Parser
|
||||||
|
The parser is one of the most crucial components of the Enso runtime in that
|
||||||
|
_all_ code that a user writes must be parsed. This means that a good parser is
|
||||||
|
fast, responsive, and lightweight; it shouldn't baulk at having thousands of
|
||||||
|
lines of code thrown at it.
|
||||||
|
|
||||||
|
Enso's parser, however, is very special. In order to support interactive use it
|
||||||
|
has to narrow down the scope of a syntax error as much as possible, while still
|
||||||
|
providing useful output for the compiler around the rest of the parse errors.
|
||||||
|
This feature makes it more complex than many common parsers, so making this work
|
||||||
|
while still preserving performance is of paramount importance.
|
||||||
|
|
||||||
|
The various components of the parser's design and architecture are described
|
||||||
|
below:
|
||||||
|
|
||||||
|
- [**Tech Analysis:**](./tech-analysis.md) A brief overview of the reasons for
|
||||||
|
the implementation technologies for the parser.
|
68
docs/parser/tech-analysis.md
Normal file
68
docs/parser/tech-analysis.md
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
---
|
||||||
|
layout: developer-doc
|
||||||
|
title: Technology Analysis
|
||||||
|
category: syntax
|
||||||
|
tags: [parser, tech-analysis]
|
||||||
|
order: 1
|
||||||
|
---
|
||||||
|
|
||||||
|
# Parser Technology Analysis
|
||||||
|
As the Enso parser has some fairly unique requirements placed upon it, the
|
||||||
|
choice of implementation technology is of paramount importance. Choosing the
|
||||||
|
correct technology ensures that we can meet all of the requirements placed upon
|
||||||
|
the parser.
|
||||||
|
|
||||||
|
<!-- MarkdownTOC levels="2,3" autolink="true" -->
|
||||||
|
|
||||||
|
- [Technology Requirements for the Parser](#technology-requirements-for-the-parser)
|
||||||
|
- [Issues With the Previous Implementation](#issues-with-the-previous-implementation)
|
||||||
|
- [Choosing Rust](#choosing-rust)
|
||||||
|
- [Downsides of Rust](#downsides-of-rust)
|
||||||
|
|
||||||
|
<!-- /MarkdownTOC -->
|
||||||
|
|
||||||
|
## Technology Requirements for the Parser
|
||||||
|
As the parser has to work both for the Engine and for the IDE, it has a strange
|
||||||
|
set of requirements:
|
||||||
|
|
||||||
|
- The implementation language must be able to run on native platforms, as well
|
||||||
|
as in the browser via WASM (not JavaScript due to the marshalling overhead).
|
||||||
|
- The implementation language should permit _excellent_ native performance on
|
||||||
|
both native and web platforms, by giving implementers fine-grained control
|
||||||
|
over memory usage.
|
||||||
|
- The implementation language must be able to target all primary platforms:
|
||||||
|
macOS, Linux and Windows.
|
||||||
|
|
||||||
|
## Issues With the Previous Implementation
|
||||||
|
The previous implementation of the parser was implemented in Scala, and had some
|
||||||
|
serious issues that have necessitated this rewrite:
|
||||||
|
|
||||||
|
- **Performance:** The structures used to implement the parser proved inherently
|
||||||
|
difficult for a JIT to optimise, making performance far worse than expected on
|
||||||
|
the JVM.
|
||||||
|
- **ScalaJS Sub-Optimal Code Generation:** The JavaScript generated by ScalaJS
|
||||||
|
was very suboptimal for these structures, making the parser _even_ slower when
|
||||||
|
run in the browser.
|
||||||
|
- **JS as a Browser Target:** To transfer textual data between WASM and JS
|
||||||
|
incurs a significant marshalling overhead. As the IDE primarily works with
|
||||||
|
textual operations under the hood, this proved to be a significant slowdown.
|
||||||
|
|
||||||
|
## Choosing Rust
|
||||||
|
Rust, then, is an obvious choice for the following reasons:
|
||||||
|
|
||||||
|
- It can be compiled _natively_ into the IDE binary, providing them with
|
||||||
|
excellent performance.
|
||||||
|
- As a native language it can use JNI to directly create JVM objects on the JVM
|
||||||
|
heap, for use by the compiler.
|
||||||
|
- As a native language it can be called directly via JNI.
|
||||||
|
- There is potential in the future for employing Graal's LLVM bitcode
|
||||||
|
interpreter to execute the parser safely in a non-native context.
|
||||||
|
|
||||||
|
### Downsides of Rust
|
||||||
|
This is not to say that choosing rust doesn't come with some compromises:
|
||||||
|
|
||||||
|
- It significantly complicates the CI pipeline for the engine, as we will have
|
||||||
|
to build native artefacts for use by the runtime itself.
|
||||||
|
- As a non-JVM language, the complexity of working with it from Scala and Java
|
||||||
|
is increased. We will need to maintain a full definition of the AST in Scala
|
||||||
|
to permit the compiler to work properly with it.
|
27
parser/flexer/Cargo.toml
Normal file
27
parser/flexer/Cargo.toml
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
[package]
|
||||||
|
name = "flexer"
|
||||||
|
version = "0.0.1"
|
||||||
|
authors = [
|
||||||
|
"Enso Team <enso-dev@enso.org>",
|
||||||
|
"Ara Adkins <ara.adkins@enso.org"
|
||||||
|
]
|
||||||
|
edition = "2018"
|
||||||
|
|
||||||
|
description = "A finite-automata-based lexing engine."
|
||||||
|
readme = "README.md"
|
||||||
|
homepage = "https://github.com/luna/enso"
|
||||||
|
repository = "https://github.com/luna/enso"
|
||||||
|
license-file = "../../LICENSE"
|
||||||
|
|
||||||
|
keywords = ["lexer", "finite-automata"]
|
||||||
|
|
||||||
|
publish = false
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
name = "flexer"
|
||||||
|
crate-type = ["dylib", "rlib"]
|
||||||
|
test = true
|
||||||
|
bench = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
itertools = "0.8"
|
4
parser/flexer/README.md
Normal file
4
parser/flexer/README.md
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
# Flexer
|
||||||
|
This library provides a finite-automata-based lexing engine that can flexibly
|
||||||
|
tokenize an input stream.
|
||||||
|
|
1
parser/flexer/build.rs
Normal file
1
parser/flexer/build.rs
Normal file
@ -0,0 +1 @@
|
|||||||
|
fn main() {}
|
8
parser/flexer/src/automata.rs
Normal file
8
parser/flexer/src/automata.rs
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
//! Exports API for construction of Nondeterminist and Deterministic Finite
|
||||||
|
//! State Automata.
|
||||||
|
|
||||||
|
pub mod alphabet;
|
||||||
|
pub mod dfa;
|
||||||
|
pub mod nfa;
|
||||||
|
pub mod pattern;
|
||||||
|
pub mod state;
|
61
parser/flexer/src/automata/alphabet.rs
Normal file
61
parser/flexer/src/automata/alphabet.rs
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
//! Exports an alphabet (set of all valid input symbols) for Finite State
|
||||||
|
//! Automata (NFA and DFA).
|
||||||
|
|
||||||
|
use crate::automata::state::Symbol;
|
||||||
|
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::ops::RangeInclusive;
|
||||||
|
|
||||||
|
// ================
|
||||||
|
// === Alphabet ===
|
||||||
|
// ================
|
||||||
|
|
||||||
|
/// An alphabet describes a set of all the valid input symbols that a given
|
||||||
|
/// finite state automata (NFA or DFA) can operate over.
|
||||||
|
/// More information at: https://en.wikipedia.org/wiki/Deterministic_finite_automaton
|
||||||
|
/// The alphabet is meant to be represented as an interval. That is, if `a` and
|
||||||
|
/// `b` are in alphabet, then any symbol from `a..=b` is in alphabet too.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Alphabet {
|
||||||
|
/// The interval of all valid input symbols. The interval is further
|
||||||
|
/// divided into subintervals (i.e. `[a,z,A,Z]` should be understood as
|
||||||
|
/// `[a..=z,z..=A,A..=Z]`), in order to efficiently encode state
|
||||||
|
/// transitions that trigger not just on one but a whole range of symbols
|
||||||
|
/// (i.e. `a..=z`)
|
||||||
|
pub symbols:BTreeSet<Symbol>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for Alphabet {
|
||||||
|
fn default() -> Self {
|
||||||
|
Alphabet {
|
||||||
|
symbols:[Symbol { val:0 }].iter().cloned().collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Alphabet {
|
||||||
|
/// Inserts a range of symbols into the alphabet.
|
||||||
|
pub fn insert(&mut self, range:RangeInclusive<Symbol>) {
|
||||||
|
// The symbol range is associated with transition in automata. Therefore
|
||||||
|
// we: Mark the symbol with the new transition.
|
||||||
|
self.symbols.insert(Symbol {
|
||||||
|
val:range.start().val,
|
||||||
|
});
|
||||||
|
// Mark the symbol without the new transition.
|
||||||
|
self.symbols.insert(Symbol {
|
||||||
|
val:range.end().val + 1,
|
||||||
|
});
|
||||||
|
// This way each symbol in alphabet corresponds to a unique set of
|
||||||
|
// transitions.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<u32>> for Alphabet {
|
||||||
|
fn from(vec:Vec<u32>) -> Self {
|
||||||
|
let mut dict = Self::default();
|
||||||
|
for val in vec {
|
||||||
|
dict.symbols.insert(Symbol { val });
|
||||||
|
}
|
||||||
|
dict
|
||||||
|
}
|
||||||
|
}
|
156
parser/flexer/src/automata/dfa.rs
Normal file
156
parser/flexer/src/automata/dfa.rs
Normal file
@ -0,0 +1,156 @@
|
|||||||
|
//! Exports the structure for Deterministic Finite Automata.
|
||||||
|
|
||||||
|
use crate::automata::alphabet::Alphabet;
|
||||||
|
use crate::automata::state;
|
||||||
|
use crate::data::matrix::Matrix;
|
||||||
|
|
||||||
|
// =====================================
|
||||||
|
// === Deterministic Finite Automata ===
|
||||||
|
// =====================================
|
||||||
|
|
||||||
|
/// Function callback for an arbitrary state of finite automata.
|
||||||
|
/// It contains name of Rust procedure that is meant to be executed after
|
||||||
|
/// encountering a pattern (declared in `group::Rule.pattern`).
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Callback {
|
||||||
|
/// TODO[jv] Write better explanation after implementing rust code
|
||||||
|
/// generation. Priority is used during rust code generation.
|
||||||
|
pub priority:usize,
|
||||||
|
/// Name of Rust method that will be called when executing this callback.
|
||||||
|
pub name:String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DFA automata with a set of symbols, states and transitions.
|
||||||
|
/// Deterministic Finite Automata is a finite-state machine that accepts or
|
||||||
|
/// rejects a given sequence of symbols, by running through a state sequence
|
||||||
|
/// uniquely determined by the input symbol sequence. ___ ___
|
||||||
|
/// ___ ___ | 0 | -- 'D' --> | 1 | -- 'F' --> | 2 | -- 'A' --> | 3
|
||||||
|
/// | ‾‾‾ ‾‾‾ ‾‾‾ ‾‾‾
|
||||||
|
/// More information at: https://en.wikipedia.org/wiki/Deterministic_finite_automaton
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
pub struct DFA {
|
||||||
|
/// Finite set of all valid input symbols.
|
||||||
|
pub alphabet:Alphabet,
|
||||||
|
/// Transition matrix of deterministic finite state automata.
|
||||||
|
/// It contains next state for each pair of state and input symbol -
|
||||||
|
/// (state,symbol) => new state. For example, a transition matrix for
|
||||||
|
/// automata that accepts string "ABABAB...." would look like this:
|
||||||
|
/// states
|
||||||
|
/// | | A | B | <- symbols
|
||||||
|
/// | 0 | 1 | - |
|
||||||
|
/// | 1 | - | 0 |
|
||||||
|
/// Where `-` denotes `state::INVALID`.
|
||||||
|
pub links:Matrix<state::Id>,
|
||||||
|
/// Stores callback for each state (if it has one).
|
||||||
|
pub callbacks:Vec<Option<Callback>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<Vec<usize>>> for Matrix<state::Id> {
|
||||||
|
fn from(input:Vec<Vec<usize>>) -> Self {
|
||||||
|
let rows = input.len();
|
||||||
|
let columns = if rows == 0 { 0 } else { input[0].len() };
|
||||||
|
let mut matrix = Self::new(rows, columns);
|
||||||
|
for row in 0..rows {
|
||||||
|
for column in 0..columns {
|
||||||
|
matrix[(row, column)] = state::Id {
|
||||||
|
id:input[row][column],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
matrix
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===========
|
||||||
|
// == Tests ==
|
||||||
|
// ===========
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::automata::state;
|
||||||
|
|
||||||
|
const I:usize = state::INVALID.id;
|
||||||
|
|
||||||
|
/// DFA automata that accepts newline '\n'.
|
||||||
|
pub fn newline() -> DFA {
|
||||||
|
DFA {
|
||||||
|
alphabet:Alphabet::from(vec![10, 11]),
|
||||||
|
links:Matrix::from(vec![vec![I, 1, I], vec![I, I, I]]),
|
||||||
|
callbacks:vec![
|
||||||
|
None,
|
||||||
|
Some(Callback {
|
||||||
|
priority:2,
|
||||||
|
name:"group0_rule0".into(),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DFA automata that accepts any letter a..=z.
|
||||||
|
pub fn letter() -> DFA {
|
||||||
|
DFA {
|
||||||
|
alphabet:Alphabet::from(vec![97, 123]),
|
||||||
|
links:Matrix::from(vec![vec![I, 1, I], vec![I, I, I]]),
|
||||||
|
callbacks:vec![
|
||||||
|
None,
|
||||||
|
Some(Callback {
|
||||||
|
priority:2,
|
||||||
|
name:"group0_rule0".into(),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DFA automata that accepts any number of spaces ' '.
|
||||||
|
pub fn spaces() -> DFA {
|
||||||
|
DFA {
|
||||||
|
alphabet:Alphabet::from(vec![0, 32, 33]),
|
||||||
|
links:Matrix::from(vec![
|
||||||
|
vec![I, 1, I],
|
||||||
|
vec![I, 2, I],
|
||||||
|
vec![I, 2, I],
|
||||||
|
]),
|
||||||
|
callbacks:vec![
|
||||||
|
None,
|
||||||
|
Some(Callback {
|
||||||
|
priority:3,
|
||||||
|
name:"group0_rule0".into(),
|
||||||
|
}),
|
||||||
|
Some(Callback {
|
||||||
|
priority:3,
|
||||||
|
name:"group0_rule0".into(),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DFA automata that accepts one letter a..=z or any many spaces.
|
||||||
|
pub fn letter_and_spaces() -> DFA {
|
||||||
|
DFA {
|
||||||
|
alphabet:Alphabet::from(vec![32, 33, 97, 123]),
|
||||||
|
links:Matrix::from(vec![
|
||||||
|
vec![I, 1, I, 2, I],
|
||||||
|
vec![I, 3, I, I, I],
|
||||||
|
vec![I, I, I, I, I],
|
||||||
|
vec![I, 3, I, I, I],
|
||||||
|
]),
|
||||||
|
callbacks:vec![
|
||||||
|
None,
|
||||||
|
Some(Callback {
|
||||||
|
priority:4,
|
||||||
|
name:"group0_rule1".into(),
|
||||||
|
}),
|
||||||
|
Some(Callback {
|
||||||
|
priority:4,
|
||||||
|
name:"group0_rule0".into(),
|
||||||
|
}),
|
||||||
|
Some(Callback {
|
||||||
|
priority:4,
|
||||||
|
name:"group0_rule1".into(),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
365
parser/flexer/src/automata/nfa.rs
Normal file
365
parser/flexer/src/automata/nfa.rs
Normal file
@ -0,0 +1,365 @@
|
|||||||
|
//! Implementation of Nondeterministic Finite Automata and it's conversion to
|
||||||
|
//! DFA.
|
||||||
|
|
||||||
|
use crate::automata::alphabet::Alphabet;
|
||||||
|
use crate::automata::dfa::Callback;
|
||||||
|
use crate::automata::dfa::DFA;
|
||||||
|
use crate::automata::state;
|
||||||
|
use crate::automata::state::Link;
|
||||||
|
use crate::automata::state::State;
|
||||||
|
use crate::automata::state::Symbol;
|
||||||
|
use crate::data::matrix::Matrix;
|
||||||
|
|
||||||
|
use crate::automata::pattern::Pattern;
|
||||||
|
use itertools::Itertools;
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::ops::RangeInclusive;
|
||||||
|
|
||||||
|
// ========================================
|
||||||
|
// === Nondeterministic Finite Automata ===
|
||||||
|
// ========================================
|
||||||
|
|
||||||
|
/// Type alias for a state Id based on set of states.
|
||||||
|
/// It is used during NFA -> DFA transformation where multiple states can merge
|
||||||
|
/// together, thanks to epsilon links.
|
||||||
|
type StateSetId = BTreeSet<state::Id>;
|
||||||
|
|
||||||
|
/// NFA automata with a set of symbols, states and transitions.
|
||||||
|
/// Nondeterministic Finite Automata is a finite-state machine that accepts or
|
||||||
|
/// rejects a given sequence of symbols.
|
||||||
|
/// Compared to `DFA`, NFA can transition into multiple new states without
|
||||||
|
/// reading any symbol (so called epsilon link / transition),
|
||||||
|
/// ___ ___ ___ ___ ___
|
||||||
|
/// | 0 | -- 'N' --> | 1 | ----> | 2 | -- 'F' --> | 3 | -- 'A' --> | 4 |
|
||||||
|
/// ‾‾‾ ‾‾‾ ‾‾‾ ‾‾‾ ‾‾‾
|
||||||
|
/// More information at: https://en.wikipedia.org/wiki/Deterministic_finite_automaton
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
pub struct NFA {
|
||||||
|
/// Finite set of all valid input symbols.
|
||||||
|
pub alphabet:Alphabet,
|
||||||
|
/// Set of named NFA states with (epsilon) transitions.
|
||||||
|
pub states:Vec<State>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NFA {
|
||||||
|
/// Adds a new state to NFA and returns it's Id.
|
||||||
|
pub fn new_state(&mut self) -> state::Id {
|
||||||
|
let id = self.states.len();
|
||||||
|
self.states.push(State::default());
|
||||||
|
state::Id { id }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates an epsilon transition between two states.
|
||||||
|
/// Whenever the automata happens to be in `source` state it can
|
||||||
|
/// immediatelly move to `target` state (but does not have to).
|
||||||
|
pub fn connect(&mut self, source:state::Id, target:state::Id) {
|
||||||
|
self.states[source.id].epsilon_links.push(target);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates an ordinary transition (for a range of symbols) between two
|
||||||
|
/// states. If any symbol from such range happens to be on input when
|
||||||
|
/// the automata is in `source` state, it will immediatelly move to
|
||||||
|
/// `target` state.
|
||||||
|
pub fn connect_by(
|
||||||
|
&mut self,
|
||||||
|
source:state::Id,
|
||||||
|
target:state::Id,
|
||||||
|
symbols:&RangeInclusive<Symbol>,
|
||||||
|
) {
|
||||||
|
self.alphabet.insert(symbols.clone());
|
||||||
|
self.states[source.id].links.push(Link {
|
||||||
|
symbols:symbols.clone(),
|
||||||
|
target,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Transforms pattern to NFA.
|
||||||
|
/// The algorithm is based on: https://www.youtube.com/watch?v=RYNN-tb9WxI
|
||||||
|
pub fn new_pattern(
|
||||||
|
&mut self,
|
||||||
|
source:state::Id,
|
||||||
|
pattern:&Pattern,
|
||||||
|
) -> state::Id {
|
||||||
|
let current = self.new_state();
|
||||||
|
self.connect(source, current);
|
||||||
|
match pattern {
|
||||||
|
Pattern::Range(range) => {
|
||||||
|
let state = self.new_state();
|
||||||
|
self.connect_by(current, state, range);
|
||||||
|
state
|
||||||
|
}
|
||||||
|
Pattern::Many(body) => {
|
||||||
|
let s1 = self.new_state();
|
||||||
|
let s2 = self.new_pattern(s1, body);
|
||||||
|
let s3 = self.new_state();
|
||||||
|
self.connect(current, s1);
|
||||||
|
self.connect(current, s3);
|
||||||
|
self.connect(s2, s3);
|
||||||
|
self.connect(s3, s1);
|
||||||
|
s3
|
||||||
|
}
|
||||||
|
Pattern::And(patterns) => patterns
|
||||||
|
.iter()
|
||||||
|
.fold(current, |s, pat| self.new_pattern(s, pat)),
|
||||||
|
Pattern::Or(patterns) => {
|
||||||
|
let states = patterns
|
||||||
|
.iter()
|
||||||
|
.map(|pat| self.new_pattern(current, pat))
|
||||||
|
.collect_vec();
|
||||||
|
let end = self.new_state();
|
||||||
|
for state in states {
|
||||||
|
self.connect(state, end);
|
||||||
|
}
|
||||||
|
end
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// === NFA -> DFA ===
|
||||||
|
|
||||||
|
/// Merges states that are connected by epsilon links.
|
||||||
|
/// The algorithm is based on: https://www.youtube.com/watch?v=taClnxU-nao
|
||||||
|
fn eps_matrix(&self) -> Vec<StateSetId> {
|
||||||
|
fn fill_eps_matrix(
|
||||||
|
nfa:&NFA,
|
||||||
|
states:&mut Vec<StateSetId>,
|
||||||
|
computed:&mut Vec<bool>,
|
||||||
|
visited:&mut Vec<bool>,
|
||||||
|
state:state::Id,
|
||||||
|
) {
|
||||||
|
let mut state_set = StateSetId::new();
|
||||||
|
let mut circular = false;
|
||||||
|
visited[state.id] = true;
|
||||||
|
state_set.insert(state);
|
||||||
|
for &target in &nfa.states[state.id].epsilon_links {
|
||||||
|
if !visited[target.id] {
|
||||||
|
fill_eps_matrix(nfa, states, computed, visited, target);
|
||||||
|
}
|
||||||
|
state_set.insert(target);
|
||||||
|
state_set.extend(states[target.id].iter());
|
||||||
|
if !computed[target.id] {
|
||||||
|
circular = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !circular {
|
||||||
|
computed[state.id] = true
|
||||||
|
}
|
||||||
|
states[state.id] = state_set;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut states = vec![StateSetId::new(); self.states.len()];
|
||||||
|
let mut computed = vec![false; self.states.len()];
|
||||||
|
for id in 0..self.states.len() {
|
||||||
|
let mut visited = vec![false; states.len()];
|
||||||
|
fill_eps_matrix(
|
||||||
|
self,
|
||||||
|
&mut states,
|
||||||
|
&mut computed,
|
||||||
|
&mut visited,
|
||||||
|
state::Id { id },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
states
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Computes a transition matrix (state X symbol => state) for NFA.
|
||||||
|
/// Ignores epsilon links.
|
||||||
|
fn nfa_matrix(&self) -> Matrix<state::Id> {
|
||||||
|
let mut matrix =
|
||||||
|
Matrix::new(self.states.len(), self.alphabet.symbols.len());
|
||||||
|
|
||||||
|
for (state_ix, source) in self.states.iter().enumerate() {
|
||||||
|
let targets = source.targets(&self.alphabet);
|
||||||
|
for (voc_ix, &target) in targets.iter().enumerate() {
|
||||||
|
matrix[(state_ix, voc_ix)] = target;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
matrix
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&NFA> for DFA {
|
||||||
|
/// Transforms NFA into DFA.
|
||||||
|
/// The algorithm is based on: https://www.youtube.com/watch?v=taClnxU-nao
|
||||||
|
fn from(nfa:&NFA) -> Self {
|
||||||
|
let nfa_mat = nfa.nfa_matrix();
|
||||||
|
let eps_mat = nfa.eps_matrix();
|
||||||
|
let mut dfa_mat = Matrix::new(0, nfa.alphabet.symbols.len());
|
||||||
|
let mut dfa_eps_ixs = Vec::<StateSetId>::new();
|
||||||
|
let mut dfa_eps_map = HashMap::<StateSetId, state::Id>::new();
|
||||||
|
|
||||||
|
dfa_eps_ixs.push(eps_mat[0].clone());
|
||||||
|
dfa_eps_map.insert(eps_mat[0].clone(), state::Id { id:0 });
|
||||||
|
|
||||||
|
let mut i = 0;
|
||||||
|
while i < dfa_eps_ixs.len() {
|
||||||
|
dfa_mat.new_row();
|
||||||
|
for voc_ix in 0..nfa.alphabet.symbols.len() {
|
||||||
|
let mut eps_set = StateSetId::new();
|
||||||
|
for &eps_ix in &dfa_eps_ixs[i] {
|
||||||
|
let tgt = nfa_mat[(eps_ix.id, voc_ix)];
|
||||||
|
if tgt != state::INVALID {
|
||||||
|
eps_set.extend(eps_mat[tgt.id].iter());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !eps_set.is_empty() {
|
||||||
|
dfa_mat[(i, voc_ix)] = match dfa_eps_map.get(&eps_set) {
|
||||||
|
Some(&id) => id,
|
||||||
|
None => {
|
||||||
|
let id = state::Id {
|
||||||
|
id:dfa_eps_ixs.len(),
|
||||||
|
};
|
||||||
|
dfa_eps_ixs.push(eps_set.clone());
|
||||||
|
dfa_eps_map.insert(eps_set, id);
|
||||||
|
id
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut callbacks = vec![None; dfa_eps_ixs.len()];
|
||||||
|
let priority = dfa_eps_ixs.len();
|
||||||
|
for (dfa_ix, epss) in dfa_eps_ixs.into_iter().enumerate() {
|
||||||
|
let has_name = |&key:&state::Id| nfa.states[key.id].name.is_some();
|
||||||
|
if let Some(eps) = epss.into_iter().find(has_name) {
|
||||||
|
let rule = nfa.states[eps.id].name.as_ref().cloned().unwrap();
|
||||||
|
callbacks[dfa_ix] = Some(Callback {
|
||||||
|
name:rule,
|
||||||
|
priority,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DFA {
|
||||||
|
alphabet:nfa.alphabet.clone(),
|
||||||
|
links:dfa_mat,
|
||||||
|
callbacks,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===========
|
||||||
|
// == Tests ==
|
||||||
|
// ===========
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tests {
|
||||||
|
extern crate test;
|
||||||
|
|
||||||
|
use crate::automata::dfa;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use test::Bencher;
|
||||||
|
|
||||||
|
/// NFA automata that accepts newline '\n'.
|
||||||
|
pub fn newline() -> NFA {
|
||||||
|
NFA {
|
||||||
|
states:vec![
|
||||||
|
State::from(vec![1]),
|
||||||
|
State::from(vec![(10..=10, 2)]),
|
||||||
|
State::from(vec![3]).named("group0_rule0"),
|
||||||
|
State::default(),
|
||||||
|
],
|
||||||
|
alphabet:Alphabet::from(vec![10, 11]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// NFA automata that accepts any letter a..=z.
|
||||||
|
pub fn letter() -> NFA {
|
||||||
|
NFA {
|
||||||
|
states:vec![
|
||||||
|
State::from(vec![1]),
|
||||||
|
State::from(vec![(97..=122, 2)]),
|
||||||
|
State::from(vec![3]).named("group0_rule0"),
|
||||||
|
State::default(),
|
||||||
|
],
|
||||||
|
alphabet:Alphabet::from(vec![97, 123]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// NFA automata that accepts any number of spaces ' '.
|
||||||
|
pub fn spaces() -> NFA {
|
||||||
|
NFA {
|
||||||
|
states:vec![
|
||||||
|
State::from(vec![1]),
|
||||||
|
State::from(vec![2]),
|
||||||
|
State::from(vec![(32..=32, 3)]),
|
||||||
|
State::from(vec![4]),
|
||||||
|
State::from(vec![5, 8]),
|
||||||
|
State::from(vec![6]),
|
||||||
|
State::from(vec![(32..=32, 7)]),
|
||||||
|
State::from(vec![8]),
|
||||||
|
State::from(vec![5, 9]).named("group0_rule0"),
|
||||||
|
State::default(),
|
||||||
|
],
|
||||||
|
alphabet:Alphabet::from(vec![0, 32, 33]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// NFA automata that accepts one letter a..=z or many spaces ' '.
|
||||||
|
pub fn letter_and_spaces() -> NFA {
|
||||||
|
NFA {
|
||||||
|
states:vec![
|
||||||
|
State::from(vec![1, 3]),
|
||||||
|
State::from(vec![(97..=122, 2)]),
|
||||||
|
State::from(vec![11]).named("group0_rule0"),
|
||||||
|
State::from(vec![4]),
|
||||||
|
State::from(vec![(32..=32, 5)]),
|
||||||
|
State::from(vec![6]),
|
||||||
|
State::from(vec![7, 10]),
|
||||||
|
State::from(vec![8]),
|
||||||
|
State::from(vec![(32..=32, 9)]),
|
||||||
|
State::from(vec![10]),
|
||||||
|
State::from(vec![7, 11]).named("group0_rule1"),
|
||||||
|
State::default(),
|
||||||
|
],
|
||||||
|
alphabet:Alphabet::from(vec![32, 33, 97, 123]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_dfa_newline() {
|
||||||
|
assert_eq!(DFA::from(&newline()), dfa::tests::newline());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_dfa_letter() {
|
||||||
|
assert_eq!(DFA::from(&letter()), dfa::tests::letter());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_dfa_spaces() {
|
||||||
|
assert_eq!(DFA::from(&spaces()), dfa::tests::spaces());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_dfa_letter_and_spaces() {
|
||||||
|
assert_eq!(
|
||||||
|
DFA::from(&letter_and_spaces()),
|
||||||
|
dfa::tests::letter_and_spaces()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_dfa_newline(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| DFA::from(&newline()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_dfa_letter(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| DFA::from(&letter()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_dfa_spaces(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| DFA::from(&spaces()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_dfa_letter_and_spaces(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| DFA::from(&letter_and_spaces()))
|
||||||
|
}
|
||||||
|
}
|
164
parser/flexer/src/automata/pattern.rs
Normal file
164
parser/flexer/src/automata/pattern.rs
Normal file
@ -0,0 +1,164 @@
|
|||||||
|
//! Simple API for constructing regex patterns that are used in parser
|
||||||
|
//! implementation.
|
||||||
|
|
||||||
|
use crate::automata::state::Symbol;
|
||||||
|
use crate::parser;
|
||||||
|
|
||||||
|
use core::iter;
|
||||||
|
use itertools::Itertools;
|
||||||
|
use std::ops::BitAnd;
|
||||||
|
use std::ops::BitOr;
|
||||||
|
use std::ops::RangeInclusive;
|
||||||
|
|
||||||
|
// =============
|
||||||
|
// == Pattern ==
|
||||||
|
// =============
|
||||||
|
|
||||||
|
/// Simple regex pattern.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub enum Pattern {
|
||||||
|
/// Pattern that triggers on any symbol from given range.
|
||||||
|
Range(RangeInclusive<Symbol>),
|
||||||
|
/// Pattern that triggers on any given pattern from sequence.
|
||||||
|
Or(Vec<Pattern>),
|
||||||
|
/// Pattern that triggers when a sequence of patterns is encountered.
|
||||||
|
And(Vec<Pattern>),
|
||||||
|
/// Pattern that triggers on 0..N repetitions of given pattern.
|
||||||
|
Many(Box<Pattern>),
|
||||||
|
}
|
||||||
|
|
||||||
|
use Pattern::*;
|
||||||
|
|
||||||
|
impl BitOr<Pattern> for Pattern {
|
||||||
|
type Output = Pattern;
|
||||||
|
fn bitor(self, rhs:Pattern) -> Self::Output {
|
||||||
|
match (self, rhs) {
|
||||||
|
(Or(mut lhs), Or(rhs)) => {
|
||||||
|
lhs.extend(rhs);
|
||||||
|
Or(lhs)
|
||||||
|
}
|
||||||
|
(Or(mut lhs), rhs) => {
|
||||||
|
lhs.push(rhs);
|
||||||
|
Or(lhs)
|
||||||
|
}
|
||||||
|
(lhs, Or(mut rhs)) => {
|
||||||
|
rhs.push(lhs);
|
||||||
|
Or(rhs)
|
||||||
|
}
|
||||||
|
(lhs, rhs) => Or(vec![lhs, rhs]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BitAnd<Pattern> for Pattern {
|
||||||
|
type Output = Pattern;
|
||||||
|
fn bitand(self, rhs:Pattern) -> Self::Output {
|
||||||
|
match (self, rhs) {
|
||||||
|
(And(mut lhs), And(rhs)) => {
|
||||||
|
lhs.extend(rhs);
|
||||||
|
And(lhs)
|
||||||
|
}
|
||||||
|
(And(mut lhs), rhs) => {
|
||||||
|
lhs.push(rhs);
|
||||||
|
And(lhs)
|
||||||
|
}
|
||||||
|
(lhs, And(mut rhs)) => {
|
||||||
|
rhs.push(lhs);
|
||||||
|
And(rhs)
|
||||||
|
}
|
||||||
|
(lhs, rhs) => And(vec![lhs, rhs]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Pattern {
|
||||||
|
/// Pattern that never triggers.
|
||||||
|
pub fn never() -> Self { Pattern::symbols(1..=0) }
|
||||||
|
|
||||||
|
/// Pattern that always triggers.
|
||||||
|
pub fn always() -> Self {
|
||||||
|
Pattern::symbols(u32::min_value()..=u32::max_value())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that triggers on any char.
|
||||||
|
pub fn any_char() -> Self { Pattern::symbols(0..=u32::max_value()) }
|
||||||
|
|
||||||
|
/// Pattern that triggers on 0..N repetitions of given pattern.
|
||||||
|
pub fn many(self) -> Self { Many(Box::new(self)) }
|
||||||
|
|
||||||
|
/// Pattern that triggers on 1..N repetitions of given pattern.
|
||||||
|
pub fn many1(self) -> Self { self.clone() & self.many() }
|
||||||
|
|
||||||
|
/// Pattern that triggers on 0..=1 repetitions of given pattern.
|
||||||
|
pub fn opt(self) -> Self { self | Self::always() }
|
||||||
|
|
||||||
|
/// Pattern that triggers on given symbol
|
||||||
|
pub fn symbol(symbol:u32) -> Self { Pattern::symbols(symbol..=symbol) }
|
||||||
|
|
||||||
|
/// Pattern that triggers on any of the given symbols.
|
||||||
|
pub fn symbols(symbols:RangeInclusive<u32>) -> Self {
|
||||||
|
Pattern::Range(
|
||||||
|
Symbol {
|
||||||
|
val:*symbols.start(),
|
||||||
|
}..=Symbol { val:*symbols.end() },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that triggers on end of file.
|
||||||
|
pub fn eof() -> Self { Self::symbol(parser::EOF_CODE.val) }
|
||||||
|
|
||||||
|
/// Pattern that triggers on given character.
|
||||||
|
pub fn char(char:char) -> Self { Self::symbol(char as u32) }
|
||||||
|
|
||||||
|
/// Pattern that triggers on any of the given characters.
|
||||||
|
pub fn range(chars:RangeInclusive<char>) -> Self {
|
||||||
|
Pattern::symbols((*chars.start() as u32)..=(*chars.end() as u32))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that triggers when sequence of characters is encountered.
|
||||||
|
pub fn all(chars:&str) -> Self {
|
||||||
|
chars
|
||||||
|
.chars()
|
||||||
|
.fold(Self::never(), |pat, char| pat & Self::char(char))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that triggers on any characters from given sequence.
|
||||||
|
pub fn any(chars:&str) -> Self {
|
||||||
|
chars
|
||||||
|
.chars()
|
||||||
|
.fold(Self::never(), |pat, char| pat | Self::char(char))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that doesn't trigger on any given character from given sequence.
|
||||||
|
pub fn none(chars:&str) -> Self {
|
||||||
|
let max = u32::max_value();
|
||||||
|
let char_iter = chars.chars().map(|char| char as u32);
|
||||||
|
let char_iter2 = iter::once(0).chain(char_iter).chain(iter::once(max));
|
||||||
|
let mut codes = char_iter2.collect_vec();
|
||||||
|
|
||||||
|
codes.sort();
|
||||||
|
codes
|
||||||
|
.iter()
|
||||||
|
.tuple_windows()
|
||||||
|
.fold(Self::never(), |pat, (start, end)| {
|
||||||
|
if end < start {
|
||||||
|
pat
|
||||||
|
} else {
|
||||||
|
pat | Pattern::symbols(*start..=*end)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that triggers on any character but the one given.
|
||||||
|
pub fn not(char:char) -> Self { Self::none(&char.to_string()) }
|
||||||
|
|
||||||
|
/// Pattern that triggers on N repetitions of given pattern.
|
||||||
|
pub fn repeat(pat:Pattern, num:usize) -> Self {
|
||||||
|
(0..num).fold(Self::always(), |p, _| p & pat.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pattern that triggers on MIN..MAX repetitions of given pattern.
|
||||||
|
pub fn repeat_between(pat:Pattern, min:usize, max:usize) -> Self {
|
||||||
|
(min..max).fold(Self::never(), |p, n| p | Self::repeat(pat.clone(), n))
|
||||||
|
}
|
||||||
|
}
|
121
parser/flexer/src/automata/state.rs
Normal file
121
parser/flexer/src/automata/state.rs
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
//! This module exports State implementation for Nondeterministic Finite
|
||||||
|
//! Automata.
|
||||||
|
|
||||||
|
use crate::automata::alphabet::Alphabet;
|
||||||
|
use crate::automata::state;
|
||||||
|
|
||||||
|
use std::ops::RangeInclusive;
|
||||||
|
|
||||||
|
// =======================
|
||||||
|
// == State Of Automata ==
|
||||||
|
// =======================
|
||||||
|
|
||||||
|
/// Flag for invalid state.
|
||||||
|
/// When finite automata gets into invalid state the input sequence of symbols
|
||||||
|
/// is rejected.
|
||||||
|
pub const INVALID:Id = Id {
|
||||||
|
id:usize::max_value(),
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Newtype wrapper for finite automata input symbol.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
|
pub struct Symbol {
|
||||||
|
#[allow(missing_docs)]
|
||||||
|
pub val:u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Newtype wrapper for finite automata state ID.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
|
pub struct Id {
|
||||||
|
#[allow(missing_docs)]
|
||||||
|
pub id:usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for Id {
|
||||||
|
/// Returns state::INVALID. This is because every finite automata has an
|
||||||
|
/// invalid state and because all transitions in automata transition
|
||||||
|
/// matrix lead to invalid state by default.
|
||||||
|
fn default() -> Self { state::INVALID }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Named NFA state with a set of transitions (links).
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
pub struct State {
|
||||||
|
/// Set of transitions that don't require any symbol to trigger.
|
||||||
|
/// I.E. If there is an epsilon link from state A to state B, then whenever
|
||||||
|
/// we are in state A, we can freely move to state B.
|
||||||
|
pub epsilon_links:Vec<Id>,
|
||||||
|
/// Set of transitions that trigger with specific symbol on input.
|
||||||
|
/// When triggered, the automata will transition to the `link.target`.
|
||||||
|
pub links:Vec<Link>,
|
||||||
|
/// Name of the state.
|
||||||
|
/// We use it to autogenerate a call to Rust method with same name.
|
||||||
|
pub name:Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A transition to new automata state
|
||||||
|
/// that requires specific symbol on automata input to trigger.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Link {
|
||||||
|
/// Any symbol from the range will trigger this link.
|
||||||
|
pub symbols:RangeInclusive<Symbol>,
|
||||||
|
/// A state that is visited, after the link is triggered.
|
||||||
|
pub target:Id,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl State {
|
||||||
|
/// Updater for field `name`. Returns updated state.
|
||||||
|
pub fn named(mut self, name:&str) -> Self {
|
||||||
|
self.name = Some(name.to_owned());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns transition (next state) for each symbol in alphabet.
|
||||||
|
pub fn targets(&self, alphabet:&Alphabet) -> Vec<Id> {
|
||||||
|
let mut targets = vec![];
|
||||||
|
let mut index = 0;
|
||||||
|
let mut links = self.links.clone();
|
||||||
|
links.sort_by_key(|link| *link.symbols.start());
|
||||||
|
for &symbol in &alphabet.symbols {
|
||||||
|
while links.len() > index && *links[index].symbols.end() < symbol {
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
if links.len() <= index || *links[index].symbols.start() > symbol {
|
||||||
|
targets.push(state::INVALID);
|
||||||
|
} else {
|
||||||
|
targets.push(links[index].target);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
targets
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<usize>> for State {
|
||||||
|
/// Creates a state with epsilon links.
|
||||||
|
fn from(vec:Vec<usize>) -> Self {
|
||||||
|
let epsilon_links = vec.iter().cloned().map(|id| Id { id }).collect();
|
||||||
|
State {
|
||||||
|
epsilon_links,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<(RangeInclusive<u32>, usize)>> for State {
|
||||||
|
/// Creates a state with ordinary links.
|
||||||
|
fn from(vec:Vec<(RangeInclusive<u32>, usize)>) -> Self {
|
||||||
|
let link = |(range, id):(RangeInclusive<u32>, usize)| {
|
||||||
|
let start = Symbol { val:*range.start() };
|
||||||
|
let end = Symbol { val:*range.end() };
|
||||||
|
Link {
|
||||||
|
symbols:start..=end,
|
||||||
|
target:Id { id },
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let links = vec.iter().cloned().map(link).collect();
|
||||||
|
State {
|
||||||
|
links,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
3
parser/flexer/src/data.rs
Normal file
3
parser/flexer/src/data.rs
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
//! Generic datastructures, with multiple usecases.
|
||||||
|
|
||||||
|
pub mod matrix;
|
55
parser/flexer/src/data/matrix.rs
Normal file
55
parser/flexer/src/data/matrix.rs
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
//! Efficient representation of 2D matrix.
|
||||||
|
|
||||||
|
use std::ops::Index;
|
||||||
|
use std::ops::IndexMut;
|
||||||
|
|
||||||
|
// ============
|
||||||
|
// == Matrix ==
|
||||||
|
// ============
|
||||||
|
|
||||||
|
/// Efficient 2D matrix implemented on top of vector.
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
pub struct Matrix<T> {
|
||||||
|
/// The number of rows in matrix.
|
||||||
|
rows:usize,
|
||||||
|
/// The number of columns in matrix.
|
||||||
|
columns:usize,
|
||||||
|
/// Matrix implemented with vector.
|
||||||
|
matrix:Vec<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Index<(usize, usize)> for Matrix<T> {
|
||||||
|
type Output = T;
|
||||||
|
fn index(&self, index:(usize, usize)) -> &T {
|
||||||
|
&self.matrix[index.0 * self.columns + index.1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> IndexMut<(usize, usize)> for Matrix<T> {
|
||||||
|
fn index_mut(&mut self, index:(usize, usize)) -> &mut T {
|
||||||
|
&mut self.matrix[index.0 * self.columns + index.1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T:Default> Matrix<T> {
|
||||||
|
/// Constructs a new matrix for given number of rows and columns.
|
||||||
|
pub fn new(rows:usize, columns:usize) -> Self {
|
||||||
|
let mut matrix = Vec::with_capacity(rows * columns);
|
||||||
|
for _ in 0..matrix.capacity() {
|
||||||
|
matrix.push(Default::default())
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
rows,
|
||||||
|
columns,
|
||||||
|
matrix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Adds a new row to matrix, filled with default values.
|
||||||
|
pub fn new_row(&mut self) {
|
||||||
|
for _ in 0..self.columns {
|
||||||
|
self.matrix.push(Default::default());
|
||||||
|
}
|
||||||
|
self.rows += 1;
|
||||||
|
}
|
||||||
|
}
|
222
parser/flexer/src/group.rs
Normal file
222
parser/flexer/src/group.rs
Normal file
@ -0,0 +1,222 @@
|
|||||||
|
//! This module exports API for grouping multiple rules (Rust callbacks with
|
||||||
|
//! regex pattern) together.
|
||||||
|
|
||||||
|
use crate::automata::nfa::NFA;
|
||||||
|
use crate::automata::pattern::Pattern;
|
||||||
|
use crate::group::rule::Rule;
|
||||||
|
|
||||||
|
use itertools::Itertools;
|
||||||
|
|
||||||
|
pub mod rule;
|
||||||
|
|
||||||
|
// ===========
|
||||||
|
// == Group ==
|
||||||
|
// ===========
|
||||||
|
|
||||||
|
/// Struct that group rules together. It also inherits rules from parent group
|
||||||
|
/// (if it has one). Groups are the basic building block of flexer:
|
||||||
|
/// Flexer internally keeps a stack of groups, only one of them active at a
|
||||||
|
/// time. Each group contains set of regex patterns and callbacks (together
|
||||||
|
/// called `Rule`). Whenever a rule.pattern from active group is matched with
|
||||||
|
/// part of input the associated rule.callback is executed, which in turn may
|
||||||
|
/// exit the current groupor enter a new one. This allows us to nicely model a
|
||||||
|
/// situation, where certain part of program (like a string literal) should have
|
||||||
|
/// very different parsing rules than other (for example body of function). Note
|
||||||
|
/// that the input is first matched with first added rule, then with the second
|
||||||
|
/// etc. Therefore, if two rules overlap, only the callback of the first added
|
||||||
|
/// rule will be executed.
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub struct Group {
|
||||||
|
/// Unique ID.
|
||||||
|
pub id:usize,
|
||||||
|
/// Custom name which is used for debugging.
|
||||||
|
pub name:String,
|
||||||
|
/// Parent which we inherit rules from.
|
||||||
|
pub parent:Option<Box<Group>>,
|
||||||
|
/// Set of regex patterns with associated callbacks.
|
||||||
|
pub rules:Vec<Rule>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Group {
|
||||||
|
/// Adds new rule (regex pattern with associated callback) to group.
|
||||||
|
pub fn add_rule(&mut self, rule:Rule) { self.rules.push(rule) }
|
||||||
|
|
||||||
|
/// Returns rule builder for given pattern.
|
||||||
|
/// TODO[jv] better describe it's purpose once we agree on correct API.
|
||||||
|
pub fn rule(
|
||||||
|
&mut self,
|
||||||
|
pattern:Pattern,
|
||||||
|
) -> rule::Builder<impl FnMut(Rule) + '_> {
|
||||||
|
rule::Builder {
|
||||||
|
pattern,
|
||||||
|
callback:move |rule| self.add_rule(rule),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// All rules including parent rules.
|
||||||
|
pub fn rules(&self) -> Vec<&Rule> {
|
||||||
|
let mut parent = &self.parent;
|
||||||
|
let mut rules = (&self.rules).iter().collect_vec();
|
||||||
|
while let Some(state) = parent {
|
||||||
|
rules.extend((&state.rules).iter());
|
||||||
|
parent = &state.parent;
|
||||||
|
}
|
||||||
|
rules
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Canonical name of given rule.
|
||||||
|
fn callback_name(&self, rule_ix:usize) -> String {
|
||||||
|
format!("group{}_rule{}", self.id, rule_ix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&Group> for NFA {
|
||||||
|
/// Transforms Group to NFA.
|
||||||
|
/// Algorithm is based on: https://www.youtube.com/watch?v=RYNN-tb9WxI
|
||||||
|
fn from(group:&Group) -> Self {
|
||||||
|
let mut nfa = NFA::default();
|
||||||
|
let start = nfa.new_state();
|
||||||
|
let build = |rule:&Rule| nfa.new_pattern(start, &rule.pattern);
|
||||||
|
let states = group.rules().into_iter().map(build).collect_vec();
|
||||||
|
let end = nfa.new_state();
|
||||||
|
for (ix, state) in states.into_iter().enumerate() {
|
||||||
|
nfa.states[state.id].name = Some(group.callback_name(ix));
|
||||||
|
nfa.connect(state, end);
|
||||||
|
}
|
||||||
|
nfa
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// =============
|
||||||
|
// === Tests ===
|
||||||
|
// =============
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tests {
|
||||||
|
extern crate test;
|
||||||
|
|
||||||
|
use crate::automata::dfa::DFA;
|
||||||
|
use crate::automata::nfa;
|
||||||
|
use crate::automata::nfa::NFA;
|
||||||
|
use crate::automata::pattern::Pattern;
|
||||||
|
use crate::group::rule::Rule;
|
||||||
|
use crate::group::Group;
|
||||||
|
|
||||||
|
use std::default::Default;
|
||||||
|
use test::Bencher;
|
||||||
|
|
||||||
|
fn newline() -> Group {
|
||||||
|
let pattern = Pattern::char('\n');
|
||||||
|
let mut group = Group::default();
|
||||||
|
|
||||||
|
group.add_rule(Rule {
|
||||||
|
pattern,
|
||||||
|
callback:"".into(),
|
||||||
|
});
|
||||||
|
|
||||||
|
group
|
||||||
|
}
|
||||||
|
|
||||||
|
fn letter() -> Group {
|
||||||
|
let pattern = Pattern::range('a'..='z');
|
||||||
|
let mut group = Group::default();
|
||||||
|
|
||||||
|
group.add_rule(Rule {
|
||||||
|
pattern,
|
||||||
|
callback:"".into(),
|
||||||
|
});
|
||||||
|
|
||||||
|
group
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spaces() -> Group {
|
||||||
|
let pattern = Pattern::char(' ').many1();
|
||||||
|
let mut group = Group::default();
|
||||||
|
|
||||||
|
group.add_rule(Rule {
|
||||||
|
pattern,
|
||||||
|
callback:"".into(),
|
||||||
|
});
|
||||||
|
|
||||||
|
group
|
||||||
|
}
|
||||||
|
|
||||||
|
fn letter_and_spaces() -> Group {
|
||||||
|
let letter = Pattern::range('a'..='z');
|
||||||
|
let spaces = Pattern::char(' ').many1();
|
||||||
|
let mut group = Group::default();
|
||||||
|
|
||||||
|
group.add_rule(Rule {
|
||||||
|
pattern:letter,
|
||||||
|
callback:"".into(),
|
||||||
|
});
|
||||||
|
group.add_rule(Rule {
|
||||||
|
pattern:spaces,
|
||||||
|
callback:"".into(),
|
||||||
|
});
|
||||||
|
|
||||||
|
group
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hundred_rules() -> Group {
|
||||||
|
let pattern =
|
||||||
|
Pattern::all("The quick brown fox jumps over the lazy dog!!");
|
||||||
|
let mut group = Group::default();
|
||||||
|
|
||||||
|
for _ in 0..100 {
|
||||||
|
group.add_rule(Rule {
|
||||||
|
pattern:pattern.clone(),
|
||||||
|
callback:"".into(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_nfa_newline() {
|
||||||
|
assert_eq!(NFA::from(&newline()), nfa::tests::newline());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_nfa_letter() {
|
||||||
|
assert_eq!(NFA::from(&letter()), nfa::tests::letter());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_nfa_spaces() {
|
||||||
|
assert_eq!(NFA::from(&spaces()), nfa::tests::spaces());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_nfa_letter_and_spaces() {
|
||||||
|
assert_eq!(
|
||||||
|
NFA::from(&letter_and_spaces()),
|
||||||
|
nfa::tests::letter_and_spaces()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_nfa_newline(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| NFA::from(&newline()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_nfa_letter(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| NFA::from(&letter()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_nfa_spaces(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| NFA::from(&spaces()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_to_nfa_letter_and_spaces(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| NFA::from(&letter_and_spaces()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_hundred_rules(bencher:&mut Bencher) {
|
||||||
|
bencher.iter(|| DFA::from(&NFA::from(&hundred_rules())));
|
||||||
|
}
|
||||||
|
}
|
39
parser/flexer/src/group/rule.rs
Normal file
39
parser/flexer/src/group/rule.rs
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
//! An API for declaring Rust callbacks for encountered regex patterns.
|
||||||
|
//!
|
||||||
|
use crate::automata::pattern::Pattern;
|
||||||
|
|
||||||
|
// ==========
|
||||||
|
// == Rule ==
|
||||||
|
// ==========
|
||||||
|
|
||||||
|
/// A rule is a pair of regex pattern and callback.
|
||||||
|
/// The intention is to run the callback after encountering given pattern.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct Rule {
|
||||||
|
/// Pattern that triggers the callback.
|
||||||
|
pub pattern:Pattern,
|
||||||
|
/// Callback containing stringified Rust code.
|
||||||
|
pub callback:String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builder that allows us to add `Rule` to `Group` in a nice way.
|
||||||
|
/// It is possible this structure won't be useful in rust, since borrow checker
|
||||||
|
/// will likely influence the final API of rule construction.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct Builder<Callback> {
|
||||||
|
/// Pattern that triggers the callback.
|
||||||
|
pub pattern:Pattern,
|
||||||
|
/// Callback containing a closure.
|
||||||
|
pub callback:Callback,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<F:FnMut(Rule)> Builder<F> {
|
||||||
|
/// Feeds the input that triggered regex pattern to callback.
|
||||||
|
pub fn run(&mut self, program:String) {
|
||||||
|
let rule = Rule {
|
||||||
|
pattern:self.pattern.clone(),
|
||||||
|
callback:program,
|
||||||
|
};
|
||||||
|
(self.callback)(rule);
|
||||||
|
}
|
||||||
|
}
|
17
parser/flexer/src/lib.rs
Normal file
17
parser/flexer/src/lib.rs
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
#![feature(test)]
|
||||||
|
#![deny(unconditional_recursion)]
|
||||||
|
#![warn(missing_copy_implementations)]
|
||||||
|
#![warn(missing_debug_implementations)]
|
||||||
|
#![warn(missing_docs)]
|
||||||
|
#![warn(trivial_casts)]
|
||||||
|
#![warn(trivial_numeric_casts)]
|
||||||
|
#![warn(unsafe_code)]
|
||||||
|
#![warn(unused_import_braces)]
|
||||||
|
|
||||||
|
//! This module exports simple parser based on Deterministic Finite State
|
||||||
|
//! Automata for regular grammars (anything parsable with regex patterns).
|
||||||
|
|
||||||
|
pub mod automata;
|
||||||
|
pub mod data;
|
||||||
|
pub mod group;
|
||||||
|
pub mod parser;
|
15
parser/flexer/src/parser.rs
Normal file
15
parser/flexer/src/parser.rs
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
//! The entry point of flexer. It (is going to) contain API for parsing an input
|
||||||
|
//! string based on group of regex patterns.
|
||||||
|
|
||||||
|
use crate::automata::state::Symbol;
|
||||||
|
|
||||||
|
// ============
|
||||||
|
// == Parser ==
|
||||||
|
// ============
|
||||||
|
|
||||||
|
/// End Of File - This symbol is inserted at the end of each parser input.
|
||||||
|
/// We can use the maximum value of u32, because no `char` (unicode scalar) can
|
||||||
|
/// hold this value.
|
||||||
|
pub const EOF_CODE:Symbol = Symbol {
|
||||||
|
val:u32::max_value(),
|
||||||
|
};
|
Loading…
Reference in New Issue
Block a user