Merge pull request #104 from uqbar-dao/hf/bundle-procs-into-binary

bundle bootstrap processes into binary

.gitignore

@@ -13,4 +13,4 @@ modules/**/wit
 target.wasm
 world
 .env
-modules/sqlite/sqlite_worker/.cargo/config.toml
+src/bootstrapped_processes.rs

Cargo.lock

@@ -2456,6 +2456,29 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "include_dir"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24b56e147e6187d61e9d0f039f10e070d0c0a887e24fe0bb9ca3f29bfde62cab"
+dependencies = [
+ "include_dir_impl",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "include_dir_impl"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a0c890c85da4bab7bce4204c707396bbd3c6c8a681716a51c8814cfc2b682df"
+dependencies = [
+ "anyhow",
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "indenter"
 version = "0.3.3"
@@ -3525,6 +3548,12 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "proc-macro-hack"
+version = "0.5.20+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.66"
@@ -4400,6 +4429,23 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
+[[package]]
+name = "static_dir"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8286dc044d09abcb8bf85440b94f2c41aee322733a58cd195cf830ee660cadf5"
+dependencies = [
+ "headers",
+ "http",
+ "hyper",
+ "include_dir",
+ "log",
+ "mime_guess",
+ "once_cell",
+ "urlencoding",
+ "warp",
+]
+
 [[package]]
 name = "string_cache"
 version = "0.8.7"
@@ -5058,6 +5104,7 @@ dependencies = [
  "serde_urlencoded",
  "sha2",
  "snow",
+ "static_dir",
  "thiserror",
  "tokio",
  "tokio-tungstenite 0.20.1",
@@ -5081,6 +5128,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "urlencoding"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb"
+
 [[package]]
 name = "utf-8"
 version = "0.7.6"

Cargo.toml

@@ -61,6 +61,7 @@ serde_json = "1.0"
 serde_urlencoded = "0.7"
 sha2 = "0.10"
 snow = { version = "0.9.3", features = ["ring-resolver"] }
+static_dir = "0.2.0"
 thiserror = "1.0"
 tokio = { version = "1.28", features = ["fs", "macros", "rt-multi-thread", "sync"] }
 tokio-tungstenite = "0.20.1"

build.rs

@@ -158,7 +158,14 @@ fn main() {
     .unwrap();
     run_command(Command::new("touch").args([&format!("{}/world", pwd.display())])).unwrap();
 
-    // Build wasm32-wasi apps.
+    // Build wasm32-wasi apps, zip, and add to bootstrapped_processes.rs
+    let mut bootstrapped_processes =
+        fs::File::create(format!("{}/src/bootstrapped_processes.rs", pwd.display(),)).unwrap();
+    writeln!(
+        bootstrapped_processes,
+        "pub static BOOTSTRAPPED_PROCESSES: &[(&str, &'static [u8])] = &[",
+    )
+    .unwrap();
     let modules_dir = format!("{}/modules", pwd.display());
     for entry in std::fs::read_dir(modules_dir).unwrap() {
         let entry_path = entry.unwrap().path();
@@ -185,12 +192,9 @@ fn main() {
         }
 
         // After processing all sub-apps, zip the parent's pkg/ directory
-        let writer = std::fs::File::create(format!(
-            "{}/target/{}.zip",
-            pwd.display(),
-            entry_path.file_name().unwrap().to_str().unwrap()
-        ))
-        .unwrap();
+        let zip_filename = format!("{}.zip", entry_path.file_name().unwrap().to_str().unwrap(),);
+        let zip_path = format!("{}/target/{}", pwd.display(), zip_filename,);
+        let writer = std::fs::File::create(&zip_path).unwrap();
         let options = zip::write::FileOptions::default()
             .compression_method(zip::CompressionMethod::Stored)
             .unix_permissions(0o755);
@@ -216,5 +220,14 @@ fn main() {
             }
         }
         zip.finish().unwrap();
+
+        // Add zip bytes to bootstrapped_processes.rs
+        writeln!(
+            bootstrapped_processes,
+            "    (\"{}\", include_bytes!(\"{}\")),",
+            zip_filename, zip_path,
+        )
+        .unwrap();
     }
+    writeln!(bootstrapped_processes, "];").unwrap();
 }

src/register.rs

@@ -7,6 +7,7 @@ use ring::rand::SystemRandom;
 use ring::signature;
 use ring::signature::KeyPair;
 use sha2::Sha256;
+use static_dir::static_dir;
 use std::sync::Arc;
 use tokio::sync::{mpsc, oneshot};
 use warp::{
@@ -128,11 +129,11 @@ pub async fn register(
     let ip = warp::any().map(move || ip.clone());
     let rpc_url = warp::any().map(move || rpc_url.clone());
 
-    let static_files = warp::path("static").and(warp::fs::dir("./src/register-ui/build/static/"));
+    let static_files = warp::path("static").and(static_dir!("src/register-ui/build/static/"));
 
     let react_app = warp::path::end()
         .and(warp::get())
-        .and(warp::fs::file("./src/register-ui/build/index.html"));
+        .map(move || warp::reply::html(include_str!("register-ui/build/index.html")));
 
     let api = warp::path("info")
         .and(

src/state.rs

@@ -10,6 +10,8 @@ use tokio::sync::Mutex;
 
 use crate::types::*;
 
+include!("bootstrapped_processes.rs");
+
 pub async fn load_state(
     our_name: String,
     home_directory_path: String,
@@ -344,8 +346,7 @@ async fn bootstrap(
             });
     }
 
-    let packages: Vec<(String, zip::ZipArchive<std::io::Cursor<Vec<u8>>>)> =
-        get_zipped_packages().await;
+    let packages = get_zipped_packages().await;
 
     for (package_name, mut package) in packages {
         // special case tester: only load it in if in simulation mode
@@ -591,29 +592,17 @@ async fn bootstrap(
     Ok(())
 }
 
-/// go into /target folder and get all .zip package files
-async fn get_zipped_packages() -> Vec<(String, zip::ZipArchive<std::io::Cursor<Vec<u8>>>)> {
+/// read in `include!()`ed .zip package files
+async fn get_zipped_packages() -> Vec<(String, zip::ZipArchive<std::io::Cursor<&'static [u8]>>)> {
     println!("fs: reading distro packages...\r");
-    let target_path = std::path::Path::new("target");
 
     let mut packages = Vec::new();
 
-    if let Ok(mut entries) = fs::read_dir(target_path).await {
-        while let Ok(Some(entry)) = entries.next_entry().await {
-            if entry.file_name().to_string_lossy().ends_with(".zip") {
-                let package_name = entry
-                    .file_name()
-                    .to_string_lossy()
-                    .trim_end_matches(".zip")
-                    .to_string();
-                if let Ok(bytes) = fs::read(entry.path()).await {
-                    if let Ok(zip) = zip::ZipArchive::new(std::io::Cursor::new(bytes)) {
-                        // add to list of packages
-                        println!("fs: found package: {}\r", package_name);
-                        packages.push((package_name, zip));
-                    }
-                }
-            }
+    for (package_name, bytes) in BOOTSTRAPPED_PROCESSES.iter() {
+        if let Ok(zip) = zip::ZipArchive::new(std::io::Cursor::new(*bytes)) {
+            // add to list of packages
+            println!("fs: found package: {}\r", package_name);
+            packages.push((package_name.to_string(), zip));
         }
     }