Stub Out the Parser Interface (#1065)

Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "lib/rust/flexer-testing/definition",
     "lib/rust/flexer-testing/generation",
     "lib/rust/lazy-reader",
+    "lib/rust/parser",
 ]
 
 [profile.dev]

build.sbt

@@ -798,10 +798,32 @@ lazy val `language-server` = (project in file("engine/language-server"))
 lazy val ast = (project in file("lib/scala/ast"))
   .settings(
     version := ensoVersion,
-    GenerateAST.rustVersion := rustVersion,
+    Cargo.rustVersion := rustVersion,
     Compile / sourceGenerators += GenerateAST.task
   )
 
+lazy val parser = (project in file("lib/scala/parser"))
+  .settings(
+    fork := true,
+    Cargo.rustVersion := rustVersion,
+    Compile / compile / compileInputs := (Compile / compile / compileInputs)
+      .dependsOn(Cargo("build --project parser"))
+      .value,
+    javaOptions += {
+      val root = baseDirectory.value.getParentFile.getParentFile.getParentFile
+      s"-Djava.library.path=$root/target/rust/debug"
+    },
+    libraryDependencies ++= Seq(
+      "com.storm-enroute" %% "scalameter" % scalameterVersion % "bench",
+      "org.scalatest" %%% "scalatest"     % scalatestVersion % Test,
+    ),
+    testFrameworks := List(
+      new TestFramework("org.scalatest.tools.Framework"),
+      new TestFramework("org.scalameter.ScalaMeterFramework")
+    ),
+  )
+  .dependsOn(ast)
+
 lazy val runtime = (project in file("engine/runtime"))
   .configs(Benchmark)
   .settings(

docs/distribution/distribution.md

@@ -47,8 +47,8 @@ directories, as explained below.
 
 ### Portable Enso Distribution Layout
 
-All files in the directory structure, except for configuration, can be safely
-removed and the launcher will re-download them if needed.
+All files in the directory structure, except for the configuration, can be
+safely removed, and the launcher will re-download them if needed.
 
 The directory structure is as follows:
 
@@ -162,6 +162,9 @@ enso-1.0.0
 ├── component        # Contains all the executable tools and their dependencies.
 │   ├── runner.jar   # The main executable of the distribution. CLI entry point.
 │   └── runtime.jar  # The language runtime. It is loaded by other JVM components, like the runner.
+├── native-libraries # Contains all shared libraries that are used by JVM components.
+│   └── parser.so    # The language parser. It is loaded by the runtime component.
+│                    # Alternative extensions are .dll Windows and .dylib on Mac.
 └── std-lib          # Contains all the libraries that are pre-installed within that compiler version.
     ├── Http         # Every version sub-directory is just an Enso package containing the library.
     │     ├── package.yaml

docs/infrastructure/rust.md

@@ -0,0 +1,42 @@
+---
+layout: developer-doc
+title: Rust
+category: infrastructure
+tags: [infrastructure, build]
+order: 1
+---
+
+# Rust
+
+The Rust project is built using Cargo which manages dependencies between the
+projects as well as external dependencies and allows for incremental
+compilation. The build configuration is defined in
+[`Cargo.toml`](../../Cargo.toml).
+
+<!-- MarkdownTOC levels="2,3" autolink="true" -->
+
+- [Shared Libraries](#shared-libraries)
+
+<!-- /MarkdownTOC -->
+
+# Java Native Interface
+
+Although the entry point of the Enso project is a Java archive, it can still
+make use of native libraries built in Rust trough the JVM foreign function
+interface (FFI) named
+[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface)
+(JNI).
+
+In order to generate a shared library, the `Cargo.toml` needs to enable
+compilation into a dynamic system library:
+
+```
+crate-type = ["cdylib"]
+```
+
+Invoking `cargo build` will then output the library into `target/rust/debug`
+with the extension `.so` on Linux, `.dll` on Windows and `.dylib` on macOS.
+
+Then, to be able to load the library by `System.loadLibrary("lib_name")`,
+the Java application should be started with the option
+`-Djava.library.path=path/to/lib_folder`.

docs/parser/jvm-object-generation.md

@@ -14,9 +14,18 @@ the compiler and runtime to work with the AST.
 
 <!-- MarkdownTOC levels="2,3" autolink="true" -->
 
+- [Overall Architecture](#overall-architecture)
+
 <!-- /MarkdownTOC -->
 
-> The actionables for this section are:
->
-> - Work out how on earth this is going to work.
-> - Produce a detailed design for this functionality.
+# Overall Architecture
+
+The JVM foreign function interface (FFI), named
+[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface)
+(JNI), enables the Rust parser library to call and be called by the parser
+library implemented in Scala.
+
+Specifically, in our architecture, the JNI is used for:
+
+- Invoking Rust parser methods from Scala.
+- Invoking Scala AST constructors from Rust.

lib/rust/parser/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name    = "parser"
+version = "0.1.0"
+authors = ["Enso Team <enso-dev@enso.org>"]
+edition = "2018"
+
+description  = "A parser for the Enso language"
+readme       = "README.md"
+homepage     = "https://github.com/enso-org/enso/lib/rust/parser"
+repository   = "https://github.com/enso-org/enso"
+license-file = "../../../LICENSE"
+
+keywords   = ["parser"]
+categories = ["parsing"]
+
+publish = false
+
+[lib]
+name       = "parser"
+crate-type = ["cdylib", "rlib"]
+test       = true
+bench      = true
+
+[dependencies]
+jni = { version = "0.17.0"                  }
+ast = { version = "0.1.0",  path = "../ast" }

lib/rust/parser/src/jni.rs

@@ -0,0 +1,116 @@
+//! This module exports JNI interface for parser methods implemented in Rust.
+//!
+//! The basics steps to add a new method are following:
+//! 1. Add the new method in Scala (in `org.enso.parser.Parser`).
+//! 2. (Optional) Run `scalac Parser.scala; javah Parser` to generate the C API in `Parser.h`.
+//!    Note that you can skip this step. It is merely a guidance for you, as it generates
+//!    the correct function names and type signatures of all `Parser` native methods.
+//!    Generally, the method interface is going to have the following shape:
+//!    ```c
+//!    JNIEXPORT $returnType JNICALL Java_$package_$className_$methodName
+//!      (JNIEnv* env, jobject this, $argType1 $arg1, $argType2 $arg2)
+//!    ```
+//!    For example if the definition is:
+//!    ```scala
+//!    package org.enso.parser
+//!
+//!    class Parser {
+//!      @native def newMethod(string: String, array: Array[Int])
+//!    }
+//!    ```
+//!    Then the JNI API is going to be:
+//!    ```c
+//!    JNIEXPORT jobject JNICALL Java_org_enso_parser_Parser_newMethod
+//!      (JNIEnv* env, jobject this, jstring string, jintArray array)
+//!    ```
+//!    The list of all available types can be found in
+//!    [oracle documentation](https://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/types.html).
+//! 3. Implement the new parser method in this file.
+//!    For the above definition the implementation is going to be:
+//!    ```rust
+//!    use jni::JNIEnv;
+//!    use jni::objects::*;
+//!    use jni::sys::*;
+//!
+//!    #[no_mangle]
+//!    pub extern "system" fn Java_org_enso_parser_Parser_newMethod(
+//!        env    : JNIEnv,    // the JVM enviroment, used for calling methods and constructors
+//!        this   : JClass,    // the instance of `Parser`
+//!        string : JString,
+//!        array  : jintArray,
+//!    ) -> jweak { unimplemented!() }
+//!    ```
+//! 4. (Optional) Generate a shared library from the Rust definition by `cargo build`.
+//!    It will be generated into `target/rust/debug/`.
+//!    This step is done automatically by `sbt`.
+
+use jni::JNIEnv;
+use jni::objects::*;
+use jni::sys::*;
+
+
+
+// ======================
+// === Parser JNI API ===
+// ======================
+
+/// Parses a content a of single source file.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_parseStr(
+    env   : JNIEnv,
+    _this : JClass,
+    input : JString,
+) -> jweak {
+    let txt = env.new_object(
+        env.find_class("org/enso/ast/Ast$Txt$Text").unwrap(),
+        "(Ljava/lang/String;)V",
+        &[input.into()],
+    ).unwrap();
+
+    let non = env.get_static_field(
+        env.find_class("scala/None$").unwrap(),
+        "MODULE$",
+        "Lscala/None$;",
+    ).unwrap().l().unwrap();
+
+    let ast = env.new_object(
+        env.find_class("org/enso/ast/Ast$Ast").unwrap(),
+        "(Lscala/Option;JJLjava/lang/Object;)V",
+        &[non.into(), 0i64.into(), 0i64.into(), txt.into()],
+    ).unwrap();
+
+    ast.into_inner()
+}
+
+/// Parses a single source file.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_parseFile(
+    env      : JNIEnv,
+    this     : JClass,
+    filename : JString,
+) -> jweak {
+    Java_org_enso_parser_Parser_parseStr(env, this, filename)
+}
+
+
+// === Tokens ===
+
+/// Parses a content of a single source file into a stream of tokens.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_lexStr(
+    env   : JNIEnv,
+    this  : JClass,
+    input : JString,
+) -> jweak {
+    Java_org_enso_parser_Parser_parseStr(env, this, input)
+}
+
+/// Parses a single source file into a stream of tokens.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_lexFile(
+    env      : JNIEnv,
+    this     : JClass,
+    filename : JString,
+) -> jweak {
+    Java_org_enso_parser_Parser_parseStr(env, this, filename)
+}

lib/rust/parser/src/lib.rs

@@ -0,0 +1,47 @@
+#![feature(test)]
+#![deny(unconditional_recursion)]
+#![warn(missing_copy_implementations)]
+#![warn(missing_debug_implementations)]
+#![warn(missing_docs)]
+#![warn(trivial_casts)]
+#![warn(trivial_numeric_casts)]
+#![warn(unsafe_code)]
+#![warn(unused_import_braces)]
+
+//! This module exports the implementation of parser for the Enso language.
+
+mod jni;
+
+pub use crate::jni::*;
+
+use ast::AnyAst;
+use ast::Ast;
+
+
+
+// =======================
+// === Parser Rust API ===
+// =======================
+
+/// Parse a content of a single source file.
+pub fn parse_str(input:String) -> AnyAst {
+    Ast::new(ast::txt::Text{text:input})
+}
+
+/// Parse a single source file.
+pub fn parse_file(filename:String) -> AnyAst {
+    parse_str(filename)
+}
+
+
+// === Tokens ===
+
+/// Parse a content of single source file.
+pub fn lexe_str(input:String) -> AnyAst {
+    parse_str(input)
+}
+
+/// Parse a single source file.
+pub fn lexe_file(filename:String) -> AnyAst {
+    parse_str(filename)
+}

lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala

@@ -0,0 +1,41 @@
+package org.enso.parser
+
+import org.enso.ast.Ast
+
+import scala.annotation.unused
+
+
+/** This is the Enso language parser.
+  *
+  * It is a wrapper of parser written in Rust that uses JNI to efficiently
+  * construct scala AST directly without any serialization overhead.
+  *
+  * The methods are loaded from a native shared library `parser` that is located
+  * in a directory specified by `-Djava.library.path` and has one of the extensions
+  * `.dll`, `.so` or `dylib` depending on the platform (windows, linux or mac).
+  *
+  * The shared library itself is generated into `target/rust/debug` by executing
+  * `cargo build -p parser`. Each method marked by `@native` must have a
+  * corresponding counterpart in rust, otherwise the loading of the shared library
+  * is going to fail at runtime with `UnsatisfiedLinkingError`.
+  */
+class Parser private () {
+  /**  Parses a content of a single source file. */
+  @native def parseStr(@unused input: String): Ast.AnyAst
+
+  /**  Parses a single source file. */
+  @native def parseFile(@unused filename: String): Ast.AnyAst
+
+  /**  Parses a content of a single source file into a stream of tokens. */
+  @native def lexStr(@unused input: String): Ast.AnyAst
+
+  /**  Parses a single source file into a stream of tokens. */
+  @native def lexFile(@unused filename: String): Ast.AnyAst
+}
+
+object Parser {
+  System.loadLibrary("parser")
+
+  /** Constructs a new parser */
+  def apply(): Parser = new Parser()
+}

lib/scala/parser/src/test/scala/org/enso/parser/ParserTest.scala

@@ -0,0 +1,20 @@
+package org.enso.parser
+
+import org.enso.ast.Ast
+
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+
+
+class ParserTest extends AnyFlatSpec with Matchers {
+  val parser: Parser = Parser()
+
+  it should "parse file" in {
+    val expected = Ast.Ast(uid=None, len=0, off=0, ast=Ast.Txt.Text("Hello!"))
+    assert(expected == parser.parseStr("Hello!"))
+    assert(expected == parser.parseFile("Hello!"))
+    assert(expected == parser.lexStr("Hello!"))
+    assert(expected == parser.lexFile("Hello!"))
+  }
+}

project/Cargo.scala

@@ -0,0 +1,49 @@
+import sbt.Keys._
+import sbt._
+import sbt.internal.util.ManagedLogger
+
+import scala.sys.process._
+
+
+
+/** A wrapper for executing the command `cargo`. */
+object Cargo {
+
+  /** The version of rust that needs to be installed. */
+  val rustVersion = settingKey[String]("rustc version used in the project")
+
+  private val cargoCmd = "cargo"
+
+  /** Checks rust version and executes the command `cargo $args`. */
+  def apply(args: String): Def.Initialize[Task[Unit]] = Def.task {
+    run(args, rustVersion.value, state.value.log)
+  }
+
+  /** Checks rust version and executes the command `cargo $args`. */
+  def run(args: String, rustVersion: String, log: ManagedLogger): Unit = {
+    val cmd = s"$cargoCmd $args"
+
+    if (!cargoOk(log))
+      throw new RuntimeException("Cargo isn't installed!")
+
+    if (!EnvironmentCheck.rustVersionOk(rustVersion, log))
+      throw new RuntimeException("Rust version mismatch!")
+
+    log.info(cmd)
+
+    try cmd.!! catch {
+      case _: RuntimeException =>
+        throw new RuntimeException("Cargo command failed.")
+    }
+  }
+
+  /** Checks that cargo is installed. Logs an error and returns false if not. */
+  def cargoOk(log: ManagedLogger): Boolean = {
+    try s"$cargoCmd version".!! catch {
+      case _: RuntimeException =>
+        log.error(s"The command `cargo` isn't on path. Did you install cargo?")
+        return false
+    }
+    true
+  }
+}

project/EnvironmentCheck.scala

@@ -1,6 +1,5 @@
 import java.io.IOException
 
-import GenerateAST.cargoCmd
 import sbt._
 import sbt.internal.util.ManagedLogger
 

project/GenerateAST.scala

@@ -1,31 +1,24 @@
-import java.io.IOException
-
 import sbt.Keys._
 import sbt._
 import sbt.internal.util.ManagedLogger
 
-import scala.sys.process._
+
 
 object GenerateAST {
 
-  val rustVersion = settingKey[String]("rustc version used in the project")
-  private val cargoCmd = "cargo"
-
   lazy val task = Def.task {
     val log    = state.value.log
-    val lib    = baseDirectory.value.getParentFile.getParentFile / "lib"
+    val lib    = baseDirectory.value.getParentFile.getParentFile
     val source = lib / "rust/ast/src/ast.rs"
     val output = sourceManaged.value / "main/org/enso/ast/Ast.scala"
     val cache  = streams.value.cacheStoreFactory.make("ast_source")
 
-    if (!EnvironmentCheck.rustVersionOk(rustVersion.value, log))
-      throw new RuntimeException("Rust version mismatch!")
-
     Tracked.diffInputs(cache, FileInfo.lastModified)(Set(source))
     { source: ChangeReport[File] =>
+      val rustVersion = Cargo.rustVersion.value
       if (source.modified.nonEmpty) {
           output.getParentFile.mkdirs
-          generateAST(output, log)
+          generateAST(rustVersion, output, log)
       }
     }
 
@@ -33,17 +26,18 @@ object GenerateAST {
   }
 
   /**
-    * Generates the Scala AST in the specified file. All errors are reported in 
+    * Generates the Scala AST in the specified file. All errors are reported in
     * stderr and raise a runtime exception.
     *
     * @param out the file where the generated AST is going to be placed
     */
-  private def generateAST(out: File, log: ManagedLogger): Unit = {
-    val command = s"$cargoCmd run -p ast -- --generate-scala-ast $out"
+  def generateAST
+  (rustVersion: String, out: File, log: ManagedLogger): Unit = {
+    val args = s"run -p ast -- --generate-scala-ast $out"
 
-    log.info(s"Generating AST with the command: $command:")
+    log.info(s"Generating Scala AST from Rust definitions.")
 
-    try {command.!!} catch {
+    try Cargo.run(args, rustVersion, log) catch {
       case ex: RuntimeException =>
         log.error(s"Generation of the Scala AST failed.")
         throw ex