mirror of
https://github.com/enso-org/enso.git
synced 2024-12-23 10:05:06 +03:00
Stub Out the Parser Interface (#1065)
This commit is contained in:
parent
da3b2bbe31
commit
5345bdcb3e
@ -14,6 +14,7 @@ members = [
|
||||
"lib/rust/flexer-testing/definition",
|
||||
"lib/rust/flexer-testing/generation",
|
||||
"lib/rust/lazy-reader",
|
||||
"lib/rust/parser",
|
||||
]
|
||||
|
||||
[profile.dev]
|
||||
|
24
build.sbt
24
build.sbt
@ -798,10 +798,32 @@ lazy val `language-server` = (project in file("engine/language-server"))
|
||||
lazy val ast = (project in file("lib/scala/ast"))
|
||||
.settings(
|
||||
version := ensoVersion,
|
||||
GenerateAST.rustVersion := rustVersion,
|
||||
Cargo.rustVersion := rustVersion,
|
||||
Compile / sourceGenerators += GenerateAST.task
|
||||
)
|
||||
|
||||
lazy val parser = (project in file("lib/scala/parser"))
|
||||
.settings(
|
||||
fork := true,
|
||||
Cargo.rustVersion := rustVersion,
|
||||
Compile / compile / compileInputs := (Compile / compile / compileInputs)
|
||||
.dependsOn(Cargo("build --project parser"))
|
||||
.value,
|
||||
javaOptions += {
|
||||
val root = baseDirectory.value.getParentFile.getParentFile.getParentFile
|
||||
s"-Djava.library.path=$root/target/rust/debug"
|
||||
},
|
||||
libraryDependencies ++= Seq(
|
||||
"com.storm-enroute" %% "scalameter" % scalameterVersion % "bench",
|
||||
"org.scalatest" %%% "scalatest" % scalatestVersion % Test,
|
||||
),
|
||||
testFrameworks := List(
|
||||
new TestFramework("org.scalatest.tools.Framework"),
|
||||
new TestFramework("org.scalameter.ScalaMeterFramework")
|
||||
),
|
||||
)
|
||||
.dependsOn(ast)
|
||||
|
||||
lazy val runtime = (project in file("engine/runtime"))
|
||||
.configs(Benchmark)
|
||||
.settings(
|
||||
|
@ -47,8 +47,8 @@ directories, as explained below.
|
||||
|
||||
### Portable Enso Distribution Layout
|
||||
|
||||
All files in the directory structure, except for configuration, can be safely
|
||||
removed and the launcher will re-download them if needed.
|
||||
All files in the directory structure, except for the configuration, can be
|
||||
safely removed, and the launcher will re-download them if needed.
|
||||
|
||||
The directory structure is as follows:
|
||||
|
||||
@ -162,6 +162,9 @@ enso-1.0.0
|
||||
├── component # Contains all the executable tools and their dependencies.
|
||||
│ ├── runner.jar # The main executable of the distribution. CLI entry point.
|
||||
│ └── runtime.jar # The language runtime. It is loaded by other JVM components, like the runner.
|
||||
├── native-libraries # Contains all shared libraries that are used by JVM components.
|
||||
│ └── parser.so # The language parser. It is loaded by the runtime component.
|
||||
│ # Alternative extensions are .dll Windows and .dylib on Mac.
|
||||
└── std-lib # Contains all the libraries that are pre-installed within that compiler version.
|
||||
├── Http # Every version sub-directory is just an Enso package containing the library.
|
||||
│ ├── package.yaml
|
||||
|
42
docs/infrastructure/rust.md
Normal file
42
docs/infrastructure/rust.md
Normal file
@ -0,0 +1,42 @@
|
||||
---
|
||||
layout: developer-doc
|
||||
title: Rust
|
||||
category: infrastructure
|
||||
tags: [infrastructure, build]
|
||||
order: 1
|
||||
---
|
||||
|
||||
# Rust
|
||||
|
||||
The Rust project is built using Cargo which manages dependencies between the
|
||||
projects as well as external dependencies and allows for incremental
|
||||
compilation. The build configuration is defined in
|
||||
[`Cargo.toml`](../../Cargo.toml).
|
||||
|
||||
<!-- MarkdownTOC levels="2,3" autolink="true" -->
|
||||
|
||||
- [Shared Libraries](#shared-libraries)
|
||||
|
||||
<!-- /MarkdownTOC -->
|
||||
|
||||
# Java Native Interface
|
||||
|
||||
Although the entry point of the Enso project is a Java archive, it can still
|
||||
make use of native libraries built in Rust trough the JVM foreign function
|
||||
interface (FFI) named
|
||||
[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface)
|
||||
(JNI).
|
||||
|
||||
In order to generate a shared library, the `Cargo.toml` needs to enable
|
||||
compilation into a dynamic system library:
|
||||
|
||||
```
|
||||
crate-type = ["cdylib"]
|
||||
```
|
||||
|
||||
Invoking `cargo build` will then output the library into `target/rust/debug`
|
||||
with the extension `.so` on Linux, `.dll` on Windows and `.dylib` on macOS.
|
||||
|
||||
Then, to be able to load the library by `System.loadLibrary("lib_name")`,
|
||||
the Java application should be started with the option
|
||||
`-Djava.library.path=path/to/lib_folder`.
|
@ -14,9 +14,18 @@ the compiler and runtime to work with the AST.
|
||||
|
||||
<!-- MarkdownTOC levels="2,3" autolink="true" -->
|
||||
|
||||
- [Overall Architecture](#overall-architecture)
|
||||
|
||||
<!-- /MarkdownTOC -->
|
||||
|
||||
> The actionables for this section are:
|
||||
>
|
||||
> - Work out how on earth this is going to work.
|
||||
> - Produce a detailed design for this functionality.
|
||||
# Overall Architecture
|
||||
|
||||
The JVM foreign function interface (FFI), named
|
||||
[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface)
|
||||
(JNI), enables the Rust parser library to call and be called by the parser
|
||||
library implemented in Scala.
|
||||
|
||||
Specifically, in our architecture, the JNI is used for:
|
||||
|
||||
- Invoking Rust parser methods from Scala.
|
||||
- Invoking Scala AST constructors from Rust.
|
||||
|
26
lib/rust/parser/Cargo.toml
Normal file
26
lib/rust/parser/Cargo.toml
Normal file
@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "parser"
|
||||
version = "0.1.0"
|
||||
authors = ["Enso Team <enso-dev@enso.org>"]
|
||||
edition = "2018"
|
||||
|
||||
description = "A parser for the Enso language"
|
||||
readme = "README.md"
|
||||
homepage = "https://github.com/enso-org/enso/lib/rust/parser"
|
||||
repository = "https://github.com/enso-org/enso"
|
||||
license-file = "../../../LICENSE"
|
||||
|
||||
keywords = ["parser"]
|
||||
categories = ["parsing"]
|
||||
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "parser"
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
test = true
|
||||
bench = true
|
||||
|
||||
[dependencies]
|
||||
jni = { version = "0.17.0" }
|
||||
ast = { version = "0.1.0", path = "../ast" }
|
116
lib/rust/parser/src/jni.rs
Normal file
116
lib/rust/parser/src/jni.rs
Normal file
@ -0,0 +1,116 @@
|
||||
//! This module exports JNI interface for parser methods implemented in Rust.
|
||||
//!
|
||||
//! The basics steps to add a new method are following:
|
||||
//! 1. Add the new method in Scala (in `org.enso.parser.Parser`).
|
||||
//! 2. (Optional) Run `scalac Parser.scala; javah Parser` to generate the C API in `Parser.h`.
|
||||
//! Note that you can skip this step. It is merely a guidance for you, as it generates
|
||||
//! the correct function names and type signatures of all `Parser` native methods.
|
||||
//! Generally, the method interface is going to have the following shape:
|
||||
//! ```c
|
||||
//! JNIEXPORT $returnType JNICALL Java_$package_$className_$methodName
|
||||
//! (JNIEnv* env, jobject this, $argType1 $arg1, $argType2 $arg2)
|
||||
//! ```
|
||||
//! For example if the definition is:
|
||||
//! ```scala
|
||||
//! package org.enso.parser
|
||||
//!
|
||||
//! class Parser {
|
||||
//! @native def newMethod(string: String, array: Array[Int])
|
||||
//! }
|
||||
//! ```
|
||||
//! Then the JNI API is going to be:
|
||||
//! ```c
|
||||
//! JNIEXPORT jobject JNICALL Java_org_enso_parser_Parser_newMethod
|
||||
//! (JNIEnv* env, jobject this, jstring string, jintArray array)
|
||||
//! ```
|
||||
//! The list of all available types can be found in
|
||||
//! [oracle documentation](https://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/types.html).
|
||||
//! 3. Implement the new parser method in this file.
|
||||
//! For the above definition the implementation is going to be:
|
||||
//! ```rust
|
||||
//! use jni::JNIEnv;
|
||||
//! use jni::objects::*;
|
||||
//! use jni::sys::*;
|
||||
//!
|
||||
//! #[no_mangle]
|
||||
//! pub extern "system" fn Java_org_enso_parser_Parser_newMethod(
|
||||
//! env : JNIEnv, // the JVM enviroment, used for calling methods and constructors
|
||||
//! this : JClass, // the instance of `Parser`
|
||||
//! string : JString,
|
||||
//! array : jintArray,
|
||||
//! ) -> jweak { unimplemented!() }
|
||||
//! ```
|
||||
//! 4. (Optional) Generate a shared library from the Rust definition by `cargo build`.
|
||||
//! It will be generated into `target/rust/debug/`.
|
||||
//! This step is done automatically by `sbt`.
|
||||
|
||||
use jni::JNIEnv;
|
||||
use jni::objects::*;
|
||||
use jni::sys::*;
|
||||
|
||||
|
||||
|
||||
// ======================
|
||||
// === Parser JNI API ===
|
||||
// ======================
|
||||
|
||||
/// Parses a content a of single source file.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_org_enso_parser_Parser_parseStr(
|
||||
env : JNIEnv,
|
||||
_this : JClass,
|
||||
input : JString,
|
||||
) -> jweak {
|
||||
let txt = env.new_object(
|
||||
env.find_class("org/enso/ast/Ast$Txt$Text").unwrap(),
|
||||
"(Ljava/lang/String;)V",
|
||||
&[input.into()],
|
||||
).unwrap();
|
||||
|
||||
let non = env.get_static_field(
|
||||
env.find_class("scala/None$").unwrap(),
|
||||
"MODULE$",
|
||||
"Lscala/None$;",
|
||||
).unwrap().l().unwrap();
|
||||
|
||||
let ast = env.new_object(
|
||||
env.find_class("org/enso/ast/Ast$Ast").unwrap(),
|
||||
"(Lscala/Option;JJLjava/lang/Object;)V",
|
||||
&[non.into(), 0i64.into(), 0i64.into(), txt.into()],
|
||||
).unwrap();
|
||||
|
||||
ast.into_inner()
|
||||
}
|
||||
|
||||
/// Parses a single source file.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_org_enso_parser_Parser_parseFile(
|
||||
env : JNIEnv,
|
||||
this : JClass,
|
||||
filename : JString,
|
||||
) -> jweak {
|
||||
Java_org_enso_parser_Parser_parseStr(env, this, filename)
|
||||
}
|
||||
|
||||
|
||||
// === Tokens ===
|
||||
|
||||
/// Parses a content of a single source file into a stream of tokens.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_org_enso_parser_Parser_lexStr(
|
||||
env : JNIEnv,
|
||||
this : JClass,
|
||||
input : JString,
|
||||
) -> jweak {
|
||||
Java_org_enso_parser_Parser_parseStr(env, this, input)
|
||||
}
|
||||
|
||||
/// Parses a single source file into a stream of tokens.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_org_enso_parser_Parser_lexFile(
|
||||
env : JNIEnv,
|
||||
this : JClass,
|
||||
filename : JString,
|
||||
) -> jweak {
|
||||
Java_org_enso_parser_Parser_parseStr(env, this, filename)
|
||||
}
|
47
lib/rust/parser/src/lib.rs
Normal file
47
lib/rust/parser/src/lib.rs
Normal file
@ -0,0 +1,47 @@
|
||||
#![feature(test)]
|
||||
#![deny(unconditional_recursion)]
|
||||
#![warn(missing_copy_implementations)]
|
||||
#![warn(missing_debug_implementations)]
|
||||
#![warn(missing_docs)]
|
||||
#![warn(trivial_casts)]
|
||||
#![warn(trivial_numeric_casts)]
|
||||
#![warn(unsafe_code)]
|
||||
#![warn(unused_import_braces)]
|
||||
|
||||
//! This module exports the implementation of parser for the Enso language.
|
||||
|
||||
mod jni;
|
||||
|
||||
pub use crate::jni::*;
|
||||
|
||||
use ast::AnyAst;
|
||||
use ast::Ast;
|
||||
|
||||
|
||||
|
||||
// =======================
|
||||
// === Parser Rust API ===
|
||||
// =======================
|
||||
|
||||
/// Parse a content of a single source file.
|
||||
pub fn parse_str(input:String) -> AnyAst {
|
||||
Ast::new(ast::txt::Text{text:input})
|
||||
}
|
||||
|
||||
/// Parse a single source file.
|
||||
pub fn parse_file(filename:String) -> AnyAst {
|
||||
parse_str(filename)
|
||||
}
|
||||
|
||||
|
||||
// === Tokens ===
|
||||
|
||||
/// Parse a content of single source file.
|
||||
pub fn lexe_str(input:String) -> AnyAst {
|
||||
parse_str(input)
|
||||
}
|
||||
|
||||
/// Parse a single source file.
|
||||
pub fn lexe_file(filename:String) -> AnyAst {
|
||||
parse_str(filename)
|
||||
}
|
41
lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala
Normal file
41
lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala
Normal file
@ -0,0 +1,41 @@
|
||||
package org.enso.parser
|
||||
|
||||
import org.enso.ast.Ast
|
||||
|
||||
import scala.annotation.unused
|
||||
|
||||
|
||||
/** This is the Enso language parser.
|
||||
*
|
||||
* It is a wrapper of parser written in Rust that uses JNI to efficiently
|
||||
* construct scala AST directly without any serialization overhead.
|
||||
*
|
||||
* The methods are loaded from a native shared library `parser` that is located
|
||||
* in a directory specified by `-Djava.library.path` and has one of the extensions
|
||||
* `.dll`, `.so` or `dylib` depending on the platform (windows, linux or mac).
|
||||
*
|
||||
* The shared library itself is generated into `target/rust/debug` by executing
|
||||
* `cargo build -p parser`. Each method marked by `@native` must have a
|
||||
* corresponding counterpart in rust, otherwise the loading of the shared library
|
||||
* is going to fail at runtime with `UnsatisfiedLinkingError`.
|
||||
*/
|
||||
class Parser private () {
|
||||
/** Parses a content of a single source file. */
|
||||
@native def parseStr(@unused input: String): Ast.AnyAst
|
||||
|
||||
/** Parses a single source file. */
|
||||
@native def parseFile(@unused filename: String): Ast.AnyAst
|
||||
|
||||
/** Parses a content of a single source file into a stream of tokens. */
|
||||
@native def lexStr(@unused input: String): Ast.AnyAst
|
||||
|
||||
/** Parses a single source file into a stream of tokens. */
|
||||
@native def lexFile(@unused filename: String): Ast.AnyAst
|
||||
}
|
||||
|
||||
object Parser {
|
||||
System.loadLibrary("parser")
|
||||
|
||||
/** Constructs a new parser */
|
||||
def apply(): Parser = new Parser()
|
||||
}
|
@ -0,0 +1,20 @@
|
||||
package org.enso.parser
|
||||
|
||||
import org.enso.ast.Ast
|
||||
|
||||
import org.scalatest.flatspec.AnyFlatSpec
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
|
||||
|
||||
|
||||
class ParserTest extends AnyFlatSpec with Matchers {
|
||||
val parser: Parser = Parser()
|
||||
|
||||
it should "parse file" in {
|
||||
val expected = Ast.Ast(uid=None, len=0, off=0, ast=Ast.Txt.Text("Hello!"))
|
||||
assert(expected == parser.parseStr("Hello!"))
|
||||
assert(expected == parser.parseFile("Hello!"))
|
||||
assert(expected == parser.lexStr("Hello!"))
|
||||
assert(expected == parser.lexFile("Hello!"))
|
||||
}
|
||||
}
|
49
project/Cargo.scala
Normal file
49
project/Cargo.scala
Normal file
@ -0,0 +1,49 @@
|
||||
import sbt.Keys._
|
||||
import sbt._
|
||||
import sbt.internal.util.ManagedLogger
|
||||
|
||||
import scala.sys.process._
|
||||
|
||||
|
||||
|
||||
/** A wrapper for executing the command `cargo`. */
|
||||
object Cargo {
|
||||
|
||||
/** The version of rust that needs to be installed. */
|
||||
val rustVersion = settingKey[String]("rustc version used in the project")
|
||||
|
||||
private val cargoCmd = "cargo"
|
||||
|
||||
/** Checks rust version and executes the command `cargo $args`. */
|
||||
def apply(args: String): Def.Initialize[Task[Unit]] = Def.task {
|
||||
run(args, rustVersion.value, state.value.log)
|
||||
}
|
||||
|
||||
/** Checks rust version and executes the command `cargo $args`. */
|
||||
def run(args: String, rustVersion: String, log: ManagedLogger): Unit = {
|
||||
val cmd = s"$cargoCmd $args"
|
||||
|
||||
if (!cargoOk(log))
|
||||
throw new RuntimeException("Cargo isn't installed!")
|
||||
|
||||
if (!EnvironmentCheck.rustVersionOk(rustVersion, log))
|
||||
throw new RuntimeException("Rust version mismatch!")
|
||||
|
||||
log.info(cmd)
|
||||
|
||||
try cmd.!! catch {
|
||||
case _: RuntimeException =>
|
||||
throw new RuntimeException("Cargo command failed.")
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks that cargo is installed. Logs an error and returns false if not. */
|
||||
def cargoOk(log: ManagedLogger): Boolean = {
|
||||
try s"$cargoCmd version".!! catch {
|
||||
case _: RuntimeException =>
|
||||
log.error(s"The command `cargo` isn't on path. Did you install cargo?")
|
||||
return false
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
import java.io.IOException
|
||||
|
||||
import GenerateAST.cargoCmd
|
||||
import sbt._
|
||||
import sbt.internal.util.ManagedLogger
|
||||
|
||||
|
@ -1,31 +1,24 @@
|
||||
import java.io.IOException
|
||||
|
||||
import sbt.Keys._
|
||||
import sbt._
|
||||
import sbt.internal.util.ManagedLogger
|
||||
|
||||
import scala.sys.process._
|
||||
|
||||
|
||||
object GenerateAST {
|
||||
|
||||
val rustVersion = settingKey[String]("rustc version used in the project")
|
||||
private val cargoCmd = "cargo"
|
||||
|
||||
lazy val task = Def.task {
|
||||
val log = state.value.log
|
||||
val lib = baseDirectory.value.getParentFile.getParentFile / "lib"
|
||||
val lib = baseDirectory.value.getParentFile.getParentFile
|
||||
val source = lib / "rust/ast/src/ast.rs"
|
||||
val output = sourceManaged.value / "main/org/enso/ast/Ast.scala"
|
||||
val cache = streams.value.cacheStoreFactory.make("ast_source")
|
||||
|
||||
if (!EnvironmentCheck.rustVersionOk(rustVersion.value, log))
|
||||
throw new RuntimeException("Rust version mismatch!")
|
||||
|
||||
Tracked.diffInputs(cache, FileInfo.lastModified)(Set(source))
|
||||
{ source: ChangeReport[File] =>
|
||||
val rustVersion = Cargo.rustVersion.value
|
||||
if (source.modified.nonEmpty) {
|
||||
output.getParentFile.mkdirs
|
||||
generateAST(output, log)
|
||||
generateAST(rustVersion, output, log)
|
||||
}
|
||||
}
|
||||
|
||||
@ -33,17 +26,18 @@ object GenerateAST {
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the Scala AST in the specified file. All errors are reported in
|
||||
* Generates the Scala AST in the specified file. All errors are reported in
|
||||
* stderr and raise a runtime exception.
|
||||
*
|
||||
* @param out the file where the generated AST is going to be placed
|
||||
*/
|
||||
private def generateAST(out: File, log: ManagedLogger): Unit = {
|
||||
val command = s"$cargoCmd run -p ast -- --generate-scala-ast $out"
|
||||
def generateAST
|
||||
(rustVersion: String, out: File, log: ManagedLogger): Unit = {
|
||||
val args = s"run -p ast -- --generate-scala-ast $out"
|
||||
|
||||
log.info(s"Generating AST with the command: $command:")
|
||||
log.info(s"Generating Scala AST from Rust definitions.")
|
||||
|
||||
try {command.!!} catch {
|
||||
try Cargo.run(args, rustVersion, log) catch {
|
||||
case ex: RuntimeException =>
|
||||
log.error(s"Generation of the Scala AST failed.")
|
||||
throw ex
|
||||
|
Loading…
Reference in New Issue
Block a user