Generate rust code from FlatBuffers specification. (https://github.com/enso-org/ide/pull/410)

Original commit: 4b0dc4390e

gui/docs/contributing.md

@@ -37,6 +37,7 @@ due to non-trival icon generation on these platforms. In order to develop the so
 need the following setup:
 
 - **The Rust Toolchain (nightly-2019-11-04)**
+
   This project uses several features available only in the nightly Rust toolchain.  Please use the
   [the Rust toolchain installer](https://rustup.rs) to install it:
 
@@ -49,6 +50,7 @@ need the following setup:
   ```
 
 - **Node and Node Package Manager LTS**
+
   In order to build the web and desktop applications you will need
   [the latest LTS version of node and npm](https://nodejs.org/en/download). Even minor release
   changes are known to cause serious issues, thus **we provide support for the latest LTS version
@@ -57,6 +59,17 @@ need the following setup:
   [Node Version Manager](https://github.com/nvm-sh/nvm) and running
   `nvm install --lts && nvm use --lts`.
 
+- **(Optional) FlatBuffer compiler `flatc`**
+
+  This dependency is needed only if you need to update files generated by the FlatBuffer from the
+  Engine Services binary protocol description. Otherwise, relying on the generated files that are
+  being stored in this repository is fine.
+
+  `flatc` must be in version *newer than 1.12* due to [this bug](). As of writing this text there
+  are no official releases with this issue fixed, however current binaries can be obtained from the
+  project's CI [build
+  artifacts](https://github.com/google/flatbuffers/actions?query=branch%3Amaster).
+
 <br/>
 
 ## Working with sources

gui/src/rust/Cargo.lock

@@ -535,6 +535,21 @@ dependencies = [
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "engine-api"
+version = "0.1.0"
+dependencies = [
+ "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ensogl-build-utilities 0.1.0",
+ "flatbuffers 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "reqwest 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "zip 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "zip-extensions 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "enso-callback"
 version = "0.1.0"
@@ -732,6 +747,22 @@ dependencies = [
  "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "flatbuffers"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "flatc-rust"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.14"
@@ -3107,6 +3138,14 @@ dependencies = [
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "zip-extensions"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "zip 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [metadata]
 "checksum Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
 "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
@@ -3170,6 +3209,8 @@ dependencies = [
 "checksum error-chain 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd"
 "checksum failure 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
 "checksum failure_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231"
+"checksum flatbuffers 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea0c34f669be9911826facafe996adfda978aeee67285a13556869e2d8b8331f"
+"checksum flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b37a2ed85bee7b6aa0d5305b4765bf4cc0f0cfbc25b86d524126a1ab755f6aed"
 "checksum flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42"
 "checksum flo_stream 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b02e0d3667b27514149c1ac9b372d700f3e6df4bbaf6b7c5df12915de2996049"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@@ -3406,3 +3447,4 @@ dependencies = [
 "checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum zip 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6df134e83b8f0f8153a094c7b0fd79dfebe437f1d76e7715afa18ed95ebe2fd7"
+"checksum zip-extensions 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9adcf027b355870f62cabaed021f9028231d2d84cad6e30a7abdaa4dc0390edd"

gui/src/rust/Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "ide",
     "ide/ast/impl",
     "ide/ast/macros",
+    "ide/engine-api",
     "ide/enso-protocol",
     "ide/json-rpc",
     "ide/parser",

gui/src/rust/ide/engine-api/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name    = "engine-api"
+version = "0.1.0"
+authors = ["Enso Team <contact@luna-lang.org>"]
+edition = "2018"
+build   = "build.rs"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+flatbuffers = "0.5"
+
+[build-dependencies]
+ensogl-build-utilities = { version = "0.1.0"  , path = "../../build"  }
+bytes                  = { version = "0.5.4"                          }
+flatc-rust             = { version = "0.1.2"                          }
+futures                = { version = "0.3.1"                          }
+reqwest                = { version = "0.10.1"                         }
+tokio                  = { version = "0.2.10" , features = ["macros"] }
+zip                    = { version = "0.5.0"                          }
+zip-extensions         = { version = "0.4.0"                          }

gui/src/rust/ide/engine-api/build.rs

@@ -0,0 +1,131 @@
+use flatc_rust;
+use std::env;
+use std::fs;
+use std::fs::File;
+use std::io::prelude::*;
+use std::path::PathBuf;
+use zip_extensions::read::ZipArchiveExtensions;
+
+
+
+// =========================
+// == Hardcoded constants ==
+// =========================
+
+/// The name of zip containing engine interface files.
+const ZIP_NAME:&str = "fbs-schema.zip";
+
+/// The directory structure inside downloaded engine interface folder.
+const ZIP_CONTENT:&str = "fbs-upload/fbs-schema/";
+
+/// Commit from `enso` repository that will be used to obtain artifacts from.
+/// If you change this commit manually, you must have `flatc` installed to regenerate interface
+/// files. Run `cargo build` to do so, before creating a commit.
+///
+/// Follow to `contribution.md` for more guidance about setting up the development environment.
+const COMMIT:&str = "7d82b1abee0f20b87b578c9ddd1a7f11330b9738";
+
+/// An URL pointing to engine interface files.
+pub fn interface_description_url() -> reqwest::Url {
+    let url = format!("https://packages.luna-lang.org/fbs-schema/nightly/{}/fbs-schema.zip",COMMIT);
+    let err = format!("{} is an invalid URL.",url);
+    reqwest::Url::parse(&url).expect(&err)
+}
+
+
+
+// ===================================
+// == Download Engine Api Artifacts ==
+// ===================================
+
+/// Struct for downloading engine artifacts.
+struct ApiProvider {
+    /// The path where downloaded artifacts will be stored.
+    out_dir:PathBuf,
+}
+
+impl ApiProvider {
+    /// Creates a provider that can download engine artifacts.
+    pub fn new() -> ApiProvider {
+        let out_dir = env::var("OUT_DIR").expect("OUT_DIR isn't environment variable").into();
+        ApiProvider {out_dir}
+    }
+
+    /// Downloads api artifacts into memory.
+    pub async fn download(&self) -> bytes::Bytes {
+        let url            = interface_description_url();
+        let get_error      = format!("Failed to get response from {}",    &url);
+        let download_error = format!("Failed to download contents of {}", &url);
+        let response       = reqwest::get(url).await.expect(&get_error);
+        response.bytes().await.expect(&download_error)
+    }
+
+    /// Saves unzipped artifacts into file.
+    pub fn unzip(&self, artifacts:bytes::Bytes) {
+        let zip_path     = self.out_dir.join(ZIP_NAME);
+        let display_path = zip_path.display();
+        let open_error   = format!("Failed to open {}", display_path);
+        let write_error  = format!("Failed to write {}",display_path);
+        let flush_error  = format!("Failed to flush {}",display_path);
+        let unzip_error  = format!("Failed to unzip {}",display_path);
+
+        let mut file = File::create(&zip_path).expect(&open_error);
+        file.write_all(&artifacts).expect(&write_error);
+        file.flush().expect(&flush_error);
+
+        let file        = File::open(&zip_path).expect(&open_error);
+        let mut archive = zip::ZipArchive::new(&file).expect(&open_error);
+        archive.extract(&self.out_dir).expect(&unzip_error);
+    }
+
+
+    /// Generates rust files from FlatBuffers schemas.
+    pub fn generate_files(&self) {
+        let fbs_dir = self.out_dir.join(ZIP_CONTENT);
+        for entry in fs::read_dir(&fbs_dir).expect("Could not read content of dir") {
+            let path   = entry.expect("Invalid content of dir").path();
+            let result = flatc_rust::run(flatc_rust::Args {
+                inputs  : &[&path],
+                out_dir : &PathBuf::from("./src/generated"),
+                ..Default::default()
+            });
+            if result.is_err() {
+                println!("cargo:info=Engine API files were not regenerated because `flatc` isn't \
+                         installed.");
+                break;
+            }
+        }
+    }
+
+    /// Places required artifacts in the target location.
+    pub async fn run(&self) {
+        let fingerprint = self.out_dir.join("engine.api.fingerprint");
+        let unchanged   = match fs::read_to_string(&fingerprint) {
+            Ok(commit) => commit == COMMIT,
+            Err(_)     => false,
+        };
+        if unchanged {
+            return
+        }
+
+        println!("cargo:info=Engine API artifacts version changed. Rebuilding.");
+        let artifacts = self.download().await;
+        self.unzip(artifacts);
+        self.generate_files();
+        fs::write(&fingerprint,COMMIT).expect("Unable to write artifacts fingerprint.");
+    }
+}
+
+
+
+// ==========
+// == main ==
+// ==========
+
+#[tokio::main]
+async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
+    let provider = ApiProvider::new();
+    provider.run().await;
+    println!("cargo:rerun-if-changed=build.rs");
+    Ok(())
+}

gui/src/rust/ide/engine-api/src/generated.rs

@@ -0,0 +1,8 @@
+//! Despite the name, this file is NOT generated. It just re-exports all generated submodules.
+
+// Generated code is not pretty warning-wise.
+#![allow(clippy::all)]
+#![allow(non_snake_case)]
+#![allow(unused_imports)]
+
+pub mod binary_protocol_generated;

gui/src/rust/ide/engine-api/src/lib.rs

@@ -0,0 +1,8 @@
+//! Crate containing the Engine Services binary protocol interface.
+
+#[allow(dead_code, unused_imports)]
+use flatbuffers;
+
+pub mod generated;
+
+pub use generated::binary_protocol_generated as binary_protocol;