diff --git a/Cargo.toml b/Cargo.toml index 90557040..733d3121 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ aes-gcm = "0.10.2" anyhow = "1.0.71" async-recursion = "1.0.4" async-trait = "0.1.71" -base64 = "0.13.0" +base64 = "0.13" bincode = "1.3.3" blake3 = "1.4.1" bytes = "1.4.0" @@ -65,4 +65,4 @@ uuid = { version = "1.1.2", features = ["serde", "v4"] } warp = "0.3.5" wasmtime = "12.0.1" wasmtime-wasi = "12.0.1" -zip = "0.6" +zip = "0.6" \ No newline at end of file diff --git a/README.md b/README.md index bc4c0801..bbff744c 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,12 @@ cargo install --git https://github.com/bytecodealliance/cargo-component --locked # Initialize submodules, in particular the register app git submodule update --init --recursive +# Build the register app +cd src/register && ./build_all.sh && cd ../.. # Build the runtime, along with a number of booted-at-startup WASM modules including terminal and key_value # OPTIONAL: --release flag cargo +nightly build --release - -# Create the home directory for your node -# If you boot multiple nodes, make sure each has their own home directory. -mkdir home ``` ### Boot diff --git a/build-app.sh b/build-app.sh deleted file mode 100755 index c71d4239..00000000 --- a/build-app.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -debug_flag="--release" - -# Grab the full path to the target -target_path="$1" -name=$(basename "$target_path") - -if [[ "$2" == "--debug" ]]; then - debug_flag="" -fi - -pwd=$(pwd) - -rm -rf "$target_path/wit" || { echo "Command failed"; exit 1; } -cp -r wit "$target_path" || { echo "Command failed"; exit 1; } -mkdir -p "$target_path/target/bindings/$name" || { echo "Command failed"; exit 1; } - -cp target.wasm "$target_path/target/bindings/$name/" || { echo "Command failed"; exit 1; } -cp world "$target_path/target/bindings/$name/" || { echo "Command failed"; exit 1; } - -mkdir -p "$target_path/target/wasm32-unknown-unknown/release" || { echo "Command failed"; exit 1; } - -# Build the module using Cargo -cargo +nightly build \ - $debug_flag \ - --no-default-features \ - --manifest-path="$target_path/Cargo.toml" \ - --target "wasm32-wasi" || { - echo "Command failed"; exit 1; - } - -# Adapt the module using wasm-tools -wasm-tools component new "$target_path/target/wasm32-wasi/release/$name.wasm" -o "$target_path/target/wasm32-wasi/release/${name}_adapted.wasm" --adapt "$pwd/wasi_snapshot_preview1.wasm" || { echo "Command failed"; exit 1; } - -# Embed "wit" into the component and place it in the expected location -wasm-tools component embed wit --world uq-process "$target_path/target/wasm32-wasi/release/${name}_adapted.wasm" -o "$target_path/target/wasm32-unknown-unknown/release/$name.wasm" || { echo "Command failed"; exit 1; } diff --git a/build.rs b/build.rs index 109c3e22..6b47e39f 100644 --- a/build.rs +++ b/build.rs @@ -36,16 +36,18 @@ where fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { let pwd = std::env::current_dir().unwrap(); - println!("cargo:warning=building {}", target_path); + let start = std::time::Instant::now(); - // Copy in newly-made wit IF old one is outdated + // if and only if module's wit is outdated, re-set-up build environment if file_outdated( - format!("{}/wit/", pwd.display()), - format!("{}/modules/{}/wit/", target_path, name), + format!("{}/wit/uqbar.wit", pwd.display()), + format!("{}/wit/uqbar.wit", target_path), ) .unwrap_or(true) { - run_command(Command::new("cp").args(&["-r", "wit", target_path])).unwrap(); + println!("cargo:warning=wit outdated, rebuilding"); + run_command(Command::new("cp").args(&["wit/uqbar.wit", &format!("{}/wit", target_path)])) + .unwrap(); // create target/bindings directory fs::create_dir_all(&format!("{}/target/bindings/{}", target_path, name,)).unwrap(); // copy newly-made target.wasm into target/bindings @@ -63,6 +65,7 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { } // Build the module targeting wasm32-wasi run_command(Command::new("cargo").args(&[ + "+nightly", "build", "--release", "--no-default-features", @@ -71,6 +74,7 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { "wasm32-wasi", ])) .unwrap(); + // Adapt module to component with adapter based on wasi_snapshot_preview1.wasm run_command(Command::new("wasm-tools").args(&[ "component", @@ -110,6 +114,13 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { &wasm_dest_path, ])) .unwrap(); + + let end = std::time::Instant::now(); + println!( + "cargo:warning=building {} took {:?}", + target_path, + end.duration_since(start) + ); } fn main() { @@ -117,53 +128,9 @@ fn main() { println!("Skipping build script"); return; } - let build_enabled = std::env::var("BUILD_APPS") - .map(|v| v == "true") - .unwrap_or(true); // run by default - - if !build_enabled { - return; - } - // only execute if one of the modules has source code changes - const WASI_APPS: [&str; 7] = [ - "app_store", - "chess", - "homepage", - "http_proxy", - "orgs", - "qns_indexer", - "terminal", - ]; - // NOT YET building KV, waiting for deps to be ready - const NESTED_WASI_APPS: [(&str, &str); 2] = [ - ("key_value", "key_value"), - ("key_value", "key_value_worker"), - ]; - - if std::env::var("REBUILD_ALL").is_ok() { - } else { - for name in &WASI_APPS { - println!("cargo:rerun-if-changed=modules/{}/src", name); - println!("cargo:rerun-if-changed=modules/{}/Cargo.toml", name); - println!("cargo:rerun-if-changed=modules/{}/pkg/manifest.json", name); - println!("cargo:rerun-if-changed=modules/{}/pkg/metadata.json", name); - } - for (outer, inner) in &NESTED_WASI_APPS { - println!("cargo:rerun-if-changed=modules/{}/{}/src", outer, inner); - println!( - "cargo:rerun-if-changed=modules/{}/{}/Cargo.toml", - outer, inner - ); - println!("cargo:rerun-if-changed=modules/{}/pkg/manifest.json", outer); - println!("cargo:rerun-if-changed=modules/{}/pkg/metadata.json", outer); - } - } let pwd = std::env::current_dir().unwrap(); - // build the register app - run_command(Command::new("./build_all.sh").current_dir("src/register")).unwrap(); - // create target.wasm (compiled .wit) & world run_command(Command::new("wasm-tools").args(&[ "component", @@ -182,11 +149,6 @@ fn main() { let entry_path = entry.unwrap().path(); let package_name = entry_path.file_name().unwrap().to_str().unwrap(); - // NOT YET building KV, waiting for deps to be ready - if package_name == "key_value" { - continue; - } - // If Cargo.toml is present, build the app let parent_pkg_path = format!("{}/pkg", entry_path.display()); if entry_path.join("Cargo.toml").exists() { diff --git a/modules/app_store/Cargo-component.lock b/modules/app_store/app_store/Cargo-component.lock similarity index 100% rename from modules/app_store/Cargo-component.lock rename to modules/app_store/app_store/Cargo-component.lock diff --git a/modules/app_store/Cargo.lock b/modules/app_store/app_store/Cargo.lock similarity index 68% rename from modules/app_store/Cargo.lock rename to modules/app_store/app_store/Cargo.lock index 702995a4..70700207 100644 --- a/modules/app_store/Cargo.lock +++ b/modules/app_store/app_store/Cargo.lock @@ -18,7 +18,8 @@ dependencies = [ "rand", "serde", "serde_json", - "wit-bindgen", + "sha2", + "wit-bindgen 0.11.0", ] [[package]] @@ -32,29 +33,32 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] -name = "bitflags" -version = "2.4.0" +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] [[package]] name = "cargo-component-bindings" -version = "0.1.0" -source = "git+https://github.com/bytecodealliance/cargo-component#6a2996f280dd8671a2a2d3c83cbe09a39225b526" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" dependencies = [ "cargo-component-macro", - "wit-bindgen", + "wit-bindgen 0.13.0", ] [[package]] name = "cargo-component-macro" -version = "0.1.0" -source = "git+https://github.com/bytecodealliance/cargo-component#6a2996f280dd8671a2a2d3c83cbe09a39225b526" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" dependencies = [ "heck", "proc-macro2", @@ -62,7 +66,6 @@ dependencies = [ "syn", "wit-bindgen-core", "wit-bindgen-rust", - "wit-bindgen-rust-lib", "wit-component", ] @@ -72,6 +75,35 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cpufeatures" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -79,12 +111,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "form_urlencoded" -version = "1.2.0" +name = "generic-array" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ - "percent-encoding", + "typenum", + "version_check", ] [[package]] @@ -100,9 +133,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heck" @@ -119,21 +152,11 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "indexmap" -version = "2.0.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", "hashbrown", @@ -164,18 +187,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "memchr" -version = "2.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -184,24 +195,13 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] -[[package]] -name = "pulldown-cmark" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "quote" version = "1.0.33" @@ -249,24 +249,24 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -285,10 +285,21 @@ dependencies = [ ] [[package]] -name = "smallvec" -version = "1.11.0" +name = "sha2" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "spdx" @@ -301,9 +312,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -311,49 +322,16 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.6.0" +name = "typenum" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-segmentation" @@ -367,17 +345,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "url" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - [[package]] name = "version_check" version = "0.9.4" @@ -392,22 +359,23 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-encoder" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" dependencies = [ "leb128", ] [[package]] name = "wasm-metadata" -version = "0.10.3" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08dc59d1fa569150851542143ca79438ca56845ccb31696c70225c638e063471" +checksum = "14abc161bfda5b519aa229758b68f2a52b45a12b993808665c857d1a9a00223c" dependencies = [ "anyhow", "indexmap", "serde", + "serde_derive", "serde_json", "spdx", "wasm-encoder", @@ -416,9 +384,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.112.0" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e986b010f47fcce49cf8ea5d5f9e5d2737832f12b53ae8ae785bbe895d0877bf" +checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" dependencies = [ "indexmap", "semver", @@ -430,15 +398,24 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a3e8e965dc50e6eb4410d9a11720719fadc6a1713803ea5f3be390b81c8279" dependencies = [ - "bitflags 2.4.0", + "bitflags", +] + +[[package]] +name = "wit-bindgen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d92ce0ca6b6074059413a9581a637550c3a740581c854f9847ec293c8aed71" +dependencies = [ + "bitflags", "wit-bindgen-rust-macro", ] [[package]] name = "wit-bindgen-core" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77255512565dfbd0b61de466e854918041d1da53c7bc049d6188c6e02643dc1e" +checksum = "565b945ae074886071eccf9cdaf8ccd7b959c2b0d624095bea5fe62003e8b3e0" dependencies = [ "anyhow", "wit-component", @@ -447,54 +424,44 @@ dependencies = [ [[package]] name = "wit-bindgen-rust" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c60e6ea8598d1380e792f13d557007834f0fb799fea6503408cbc5debb4ae" +checksum = "5695ff4e41873ed9ce56d2787e6b5772bdad9e70e2c1d2d160621d1762257f4f" dependencies = [ "anyhow", "heck", "wasm-metadata", "wit-bindgen-core", - "wit-bindgen-rust-lib", "wit-component", ] -[[package]] -name = "wit-bindgen-rust-lib" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9fb7a43c7dc28b0b727d6ae01bf369981229b7539e768fba2b7a4df13feeeb" -dependencies = [ - "heck", - "wit-bindgen-core", -] - [[package]] name = "wit-bindgen-rust-macro" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cea5ed784da06da0e55836a6c160e7502dbe28771c2368a595e8606243bf22" +checksum = "a91835ea4231da1fe7971679d505ba14be7826e192b6357f08465866ef482e08" dependencies = [ "anyhow", "proc-macro2", + "quote", "syn", "wit-bindgen-core", "wit-bindgen-rust", - "wit-bindgen-rust-lib", "wit-component", ] [[package]] name = "wit-component" -version = "0.14.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d9f2d16dd55d1a372dcfd4b7a466ea876682a5a3cb97e71ec9eef04affa876" +checksum = "e87488b57a08e2cbbd076b325acbe7f8666965af174d69d5929cd373bd54547f" dependencies = [ "anyhow", - "bitflags 2.4.0", + "bitflags", "indexmap", "log", "serde", + "serde_derive", "serde_json", "wasm-encoder", "wasm-metadata", @@ -504,16 +471,17 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e8b849bea13cc2315426b16efe6eb6813466d78f5fde69b0bb150c9c40e0dc" +checksum = "f6ace9943d89bbf3dbbc71b966da0e7302057b311f36a4ac3d65ddfef17b52cf" dependencies = [ "anyhow", "id-arena", "indexmap", "log", - "pulldown-cmark", "semver", + "serde", + "serde_derive", + "serde_json", "unicode-xid", - "url", ] diff --git a/modules/app_store/Cargo.toml b/modules/app_store/app_store/Cargo.toml similarity index 97% rename from modules/app_store/Cargo.toml rename to modules/app_store/app_store/Cargo.toml index 25120858..4e095cb2 100644 --- a/modules/app_store/Cargo.toml +++ b/modules/app_store/app_store/Cargo.toml @@ -17,6 +17,7 @@ cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-co rand = "0.8.5" serde = {version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.10.8" wit-bindgen = { version = "0.11.0", default_features = false } [lib] diff --git a/modules/app_store/app_store/src/ft_worker_lib.rs b/modules/app_store/app_store/src/ft_worker_lib.rs new file mode 120000 index 00000000..1fe915a3 --- /dev/null +++ b/modules/app_store/app_store/src/ft_worker_lib.rs @@ -0,0 +1 @@ +../../ft_worker/src/ft_worker_lib.rs \ No newline at end of file diff --git a/modules/app_store/app_store/src/kernel_types.rs b/modules/app_store/app_store/src/kernel_types.rs new file mode 120000 index 00000000..047e48bc --- /dev/null +++ b/modules/app_store/app_store/src/kernel_types.rs @@ -0,0 +1 @@ +../../../../src/kernel_types.rs \ No newline at end of file diff --git a/modules/app_store/app_store/src/lib.rs b/modules/app_store/app_store/src/lib.rs new file mode 100644 index 00000000..46e5b727 --- /dev/null +++ b/modules/app_store/app_store/src/lib.rs @@ -0,0 +1,703 @@ +cargo_component_bindings::generate!(); +use bindings::{ + component::uq_process::types::*, get_capability, get_payload, print_to_terminal, receive, + send_request, send_response, Guest, +}; +use serde::{Deserialize, Serialize}; +use sha2::Digest; +use std::collections::{HashMap, HashSet}; + +#[allow(dead_code)] +mod kernel_types; +use kernel_types as kt; +use kernel_types::{PackageManifestEntry, PackageMetadata, PackageVersion}; + +#[allow(dead_code)] +mod process_lib; +use process_lib::PackageId; + +#[allow(dead_code)] +mod ft_worker_lib; +use ft_worker_lib::{ + spawn_receive_transfer, spawn_transfer, FTWorkerCommand, FTWorkerResult, FileTransferContext, +}; + +struct Component; + +/// Uqbar App Store: +/// acts as both a local package manager and a protocol to share packages across the network. +/// packages are apps; apps are packages. we use an onchain app listing contract to determine +/// what apps are available to download and what node(s) to download them from. +/// +/// once we know that list, we can request a package from a node and download it locally. +/// (we can also manually download an "untracked" package if we know its name and distributor node) +/// packages that are downloaded can then be installed! +/// +/// installed packages can be managed: +/// - given permissions (necessary to complete install) +/// - uninstalled / suspended +/// - deleted ("undownloaded") +/// - set to automatically update if a new version is available (on by default) + +// +// app store types +// + +/// this process's saved state +#[derive(Debug, Serialize, Deserialize)] +struct State { + pub packages: HashMap, + pub requested_packages: HashSet, +} + +/// state of an individual package we have downloaded +#[derive(Debug, Serialize, Deserialize)] +struct PackageState { + pub mirrored_from: NodeId, + pub listing_data: PackageListing, + pub mirroring: bool, // are we serving this package to others? + pub auto_update: bool, // if we get a listing data update, will we try to download it? +} + +/// just a sketch of what we might get from chain +#[derive(Debug, Serialize, Deserialize)] +struct PackageListing { + pub name: String, + pub publisher: NodeId, + pub description: Option, + pub website: Option, + pub version: PackageVersion, + pub version_hash: String, // sha256 hash of the package zip or whatever +} + +// +// app store API +// + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] // untagged as a meta-type for all requests +pub enum Req { + LocalRequest(LocalRequest), + RemoteRequest(RemoteRequest), + FTWorkerCommand(FTWorkerCommand), + FTWorkerResult(FTWorkerResult), +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] // untagged as a meta-type for all responses +pub enum Resp { + RemoteResponse(RemoteResponse), + FTWorkerResult(FTWorkerResult), + // note that we do not need to ourselves handle local responses, as + // those are given to others rather than received. + NewPackageResponse(NewPackageResponse), + DownloadResponse(DownloadResponse), + InstallResponse(InstallResponse), +} + +/// Local requests take this form. +#[derive(Debug, Serialize, Deserialize)] +pub enum LocalRequest { + /// expects a zipped package as payload: create a new package from it + /// if requested, will return a NewPackageResponse indicating success/failure + NewPackage { + package: PackageId, + mirror: bool, // sets whether we will mirror this package + }, + /// no payload; try to download a package from a specified node + /// if requested, will return a DownloadResponse indicating success/failure + Download { + package: PackageId, + install_from: NodeId, + }, + /// no payload; select a downloaded package and install it + /// if requested, will return an InstallResponse indicating success/failure + Install(PackageId), + /// no payload; select an installed package and uninstall it + /// no response will be given + Uninstall(PackageId), + /// no payload; select a downloaded package and delete it + /// no response will be given + Delete(PackageId), +} + +/// Remote requests, those sent between instantiations of this process +/// on different nodes, take this form. +#[derive(Debug, Serialize, Deserialize)] +pub enum RemoteRequest { + /// no payload; request a package from a node + /// remote node must return RemoteResponse::DownloadApproved, + /// at which point requester can expect a FTWorkerRequest::Receive + Download(PackageId), +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum RemoteResponse { + DownloadApproved, + DownloadDenied, // TODO expand on why +} + +// TODO for all: expand these to elucidate why something failed +// these are locally-given responses to local requests + +#[derive(Debug, Serialize, Deserialize)] +pub enum NewPackageResponse { + Success, + Failure, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum DownloadResponse { + Started, + Failure, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum InstallResponse { + Success, + Failure, +} + +// +// app store init() +// + +impl Guest for Component { + fn init(our: Address) { + assert_eq!(our.process, "main:app_store:uqbar"); + + // begin by granting messaging capabilities to http_server and terminal, + // so that they can send us requests. + process_lib::grant_messaging( + &our, + &Vec::from([ + ProcessId::from_str("http_server:sys:uqbar").unwrap(), + ProcessId::from_str("terminal:terminal:uqbar").unwrap(), + ]), + ); + print_to_terminal(0, &format!("app_store main proc: start")); + + // load in our saved state or initalize a new one if none exists + let mut state = process_lib::get_state::().unwrap_or(State { + packages: HashMap::new(), + requested_packages: HashSet::new(), + }); + + // active the main messaging loop: handle requests and responses + loop { + let (source, message) = match receive() { + Ok((source, message)) => (source, message), + Err((error, _context)) => { + // TODO handle net errors more usefully based on their context + print_to_terminal(0, &format!("net error: {:?}", error.kind)); + continue; + } + }; + match message { + Message::Request(req) => { + let Some(ref ipc) = req.ipc else { + continue; + }; + match &serde_json::from_str::(ipc) { + Ok(Req::LocalRequest(local_request)) => { + match handle_local_request(&our, &source, local_request, &mut state) { + Ok(None) => continue, + Ok(Some(resp)) => { + if req.expects_response.is_some() { + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&resp).unwrap()), + metadata: None, + }, + None, + ); + } + } + Err(err) => { + print_to_terminal( + 0, + &format!("app-store: local request error: {:?}", err), + ); + } + } + } + Ok(Req::RemoteRequest(remote_request)) => { + match handle_remote_request(&our, &source, remote_request, &mut state) { + Ok(None) => continue, + Ok(Some(resp)) => { + if req.expects_response.is_some() { + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&resp).unwrap()), + metadata: None, + }, + None, + ); + } + } + Err(err) => { + print_to_terminal( + 0, + &format!("app-store: remote request error: {:?}", err), + ); + } + } + } + Ok(Req::FTWorkerResult(FTWorkerResult::ReceiveSuccess(name))) => { + // do with file what you'd like here + print_to_terminal( + 0, + &format!("file_transfer: successfully received {:?}", name,), + ); + // remove leading / and .zip from file name to get package ID + let package_id = + match PackageId::from_str(name[1..].trim_end_matches(".zip")) { + Ok(package_id) => package_id, + Err(_) => { + print_to_terminal( + 0, + &format!("app store: bad package filename: {}", name), + ); + continue; + } + }; + if state.requested_packages.remove(&package_id) { + // auto-take zip from payload and request ourself with New + let _ = send_request( + &our, + &Request { + inherit: true, // will inherit payload! + expects_response: None, + ipc: Some( + serde_json::to_string(&Req::LocalRequest( + LocalRequest::NewPackage { + package: package_id, + mirror: true, + }, + )) + .unwrap(), + ), + metadata: None, + }, + None, + None, + ); + } + } + Ok(Req::FTWorkerCommand(_)) => { + spawn_receive_transfer(&our, ipc); + } + e => { + print_to_terminal( + 0, + &format!("app store bad request: {}, error {:?}", ipc, e), + ); + continue; + } + } + } + Message::Response((response, context)) => { + let Some(ref ipc) = response.ipc else { + continue; + }; + match &serde_json::from_str::(ipc) { + Ok(Resp::RemoteResponse(remote_response)) => match remote_response { + RemoteResponse::DownloadApproved => { + print_to_terminal( + 0, + "app store: download approved, should be starting", + ); + } + RemoteResponse::DownloadDenied => { + print_to_terminal( + 0, + "app store: could not download package from that node!", + ); + } + }, + Ok(Resp::FTWorkerResult(ft_worker_result)) => { + let Ok(context) = serde_json::from_str::(&context.unwrap_or_default()) else { + print_to_terminal(0, "file_transfer: got weird local request"); + continue; + }; + match ft_worker_result { + FTWorkerResult::SendSuccess => { + print_to_terminal( + 0, + &format!( + "file_transfer: successfully shared app {} in {:.4}s", + context.file_name, + std::time::SystemTime::now() + .duration_since(context.start_time) + .unwrap() + .as_secs_f64(), + ), + ); + } + e => { + print_to_terminal( + 0, + &format!("app store file transfer: error {:?}", e), + ); + } + } + } + e => { + print_to_terminal( + 0, + &format!("app store bad response: {}, error {:?}", ipc, e), + ); + continue; + } + } + } + } + } + } +} + +fn handle_local_request( + our: &Address, + source: &Address, + request: &LocalRequest, + state: &mut State, +) -> anyhow::Result> { + if our.node != source.node { + return Err(anyhow::anyhow!("local request from non-local node")); + } + match request { + LocalRequest::NewPackage { package, mirror } => { + let Some(mut payload) = get_payload() else { + return Err(anyhow::anyhow!("no payload")); + }; + let vfs_address = Address { + node: our.node.clone(), + process: ProcessId::from_str("vfs:sys:uqbar")?, + }; + + // produce the version hash for this new package + let mut hasher = sha2::Sha256::new(); + hasher.update(&payload.bytes); + let version_hash = format!("{:x}", hasher.finalize()); + + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::New, + })?), + None, + None, + 5, + )?; + + // add zip bytes + payload.mime = Some("application/zip".to_string()); + let _ = process_lib::send_and_await_response( + &vfs_address, + true, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::Add { + full_path: package.to_string(), + entry_type: kt::AddEntryType::ZipArchive, + }, + })?), + None, + Some(&payload), + 5, + )?; + + // save the zip file itself in VFS for sharing with other nodes + // call it .zip + let _ = process_lib::send_and_await_response( + &vfs_address, + true, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::Add { + full_path: format!("/{}.zip", package.to_string()), + entry_type: kt::AddEntryType::NewFile, + }, + })?), + None, + Some(&payload), + 5, + )?; + + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry("/metadata.json".into()), + })?), + None, + None, + 5, + )?; + let Some(payload) = get_payload() else { + return Err(anyhow::anyhow!("no metadata payload")); + }; + let metadata = String::from_utf8(payload.bytes)?; + let metadata = serde_json::from_str::(&metadata)?; + + let listing_data = PackageListing { + name: metadata.package, + publisher: our.node.clone(), + description: metadata.description, + website: metadata.website, + version: metadata.version, + version_hash, + }; + let package_state = PackageState { + mirrored_from: our.node.clone(), + listing_data, + mirroring: *mirror, + auto_update: true, + }; + state.packages.insert(package.clone(), package_state); + process_lib::set_state::(&state); + Ok(Some(Resp::NewPackageResponse(NewPackageResponse::Success))) + } + LocalRequest::Download { + package, + install_from, + } => Ok(Some(Resp::DownloadResponse( + match process_lib::send_and_await_response( + &Address { + node: install_from.clone(), + process: our.process.clone(), + }, + true, + Some(serde_json::to_string(&RemoteRequest::Download( + package.clone(), + ))?), + None, + None, + 5, + ) { + Ok((_source, Message::Response((resp, _context)))) => { + let Some(ipc) = resp.ipc else { + return Err(anyhow::anyhow!("no ipc in response")) + }; + let resp = serde_json::from_str::(&ipc)?; + match resp { + Resp::RemoteResponse(RemoteResponse::DownloadApproved) => { + state.requested_packages.insert(package.clone()); + process_lib::set_state::(&state); + DownloadResponse::Started + } + _ => DownloadResponse::Failure, + } + } + _ => DownloadResponse::Failure, + }, + ))), + LocalRequest::Install(package) => { + let vfs_address = Address { + node: our.node.clone(), + process: ProcessId::from_str("vfs:sys:uqbar")?, + }; + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry("/manifest.json".into()), + })?), + None, + None, + 5, + )?; + let Some(payload) = get_payload() else { + return Err(anyhow::anyhow!("no payload")); + }; + let manifest = String::from_utf8(payload.bytes)?; + let manifest = serde_json::from_str::>(&manifest)?; + for entry in manifest { + let path = if entry.process_wasm_path.starts_with("/") { + entry.process_wasm_path + } else { + format!("/{}", entry.process_wasm_path) + }; + + let (_, hash_response) = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetHash(path.clone()), + })?), + None, + None, + 5, + )?; + + let Message::Response((Response { ipc: Some(ipc), .. }, _)) = hash_response else { + return Err(anyhow::anyhow!("bad vfs response")); + }; + let kt::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc)? else { + return Err(anyhow::anyhow!("no hash in vfs")); + }; + + // build initial caps + let mut initial_capabilities: HashSet = HashSet::new(); + if entry.request_networking { + let Some(networking_cap) = get_capability( + &Address { + node: our.node.clone(), + process: ProcessId::from_str("kernel:sys:uqbar")?, + }, + &"\"network\"".to_string(), + ) else { + return Err(anyhow::anyhow!("app-store: no net cap")); + }; + initial_capabilities.insert(kt::de_wit_signed_capability(networking_cap)); + } + let Some(read_cap) = get_capability( + &vfs_address.clone(), + &serde_json::to_string(&serde_json::json!({ + "kind": "read", + "drive": package.to_string(), + }))?, + ) else { + return Err(anyhow::anyhow!("app-store: no read cap")); + }; + initial_capabilities.insert(kt::de_wit_signed_capability(read_cap)); + let Some(write_cap) = get_capability( + &vfs_address.clone(), + &serde_json::to_string(&serde_json::json!({ + "kind": "write", + "drive": package.to_string(), + }))?, + ) else { + return Err(anyhow::anyhow!("app-store: no write cap")); + }; + initial_capabilities.insert(kt::de_wit_signed_capability(write_cap)); + + for process_name in &entry.request_messaging { + let Ok(parsed_process_id) = ProcessId::from_str(&process_name) else { + // TODO handle arbitrary caps here + continue; + }; + let Some(messaging_cap) = get_capability( + &Address { + node: our.node.clone(), + process: parsed_process_id.clone(), + }, + &"\"messaging\"".into() + ) else { + print_to_terminal(0, &format!("app-store: no cap for {} to give away!", process_name)); + continue; + }; + initial_capabilities.insert(kt::de_wit_signed_capability(messaging_cap)); + } + + let process_id = format!("{}:{}", entry.process_name, package.to_string()); + let Ok(parsed_new_process_id) = ProcessId::from_str(&process_id) else { + return Err(anyhow::anyhow!("app-store: invalid process id!")); + }; + let _ = process_lib::send_request( + &Address { + node: our.node.clone(), + process: ProcessId::from_str("kernel:sys:uqbar")?, + }, + false, + Some(serde_json::to_string(&kt::KernelCommand::KillProcess( + kt::ProcessId::de_wit(parsed_new_process_id.clone()), + ))?), + None, + None, + None, + ); + + // kernel start process takes bytes as payload + wasm_bytes_handle... + // reconsider perhaps + let (_, _bytes_response) = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry(path), + })?), + None, + None, + 5, + )?; + + let Some(payload) = get_payload() else { + return Err(anyhow::anyhow!("no wasm bytes payload.")); + }; + + let _ = process_lib::send_and_await_response( + &Address { + node: our.node.clone(), + process: ProcessId::from_str("kernel:sys:uqbar")?, + }, + false, + Some(serde_json::to_string(&kt::KernelCommand::StartProcess { + id: kt::ProcessId::de_wit(parsed_new_process_id), + wasm_bytes_handle: hash, + on_panic: entry.on_panic, + initial_capabilities, + public: entry.public, + })?), + None, + Some(&payload), + 5, + )?; + } + Ok(Some(Resp::InstallResponse(InstallResponse::Success))) + } + LocalRequest::Uninstall(package) => { + // TODO + Ok(None) + } + LocalRequest::Delete(package) => { + // TODO + Ok(None) + } + } +} + +fn handle_remote_request( + our: &Address, + source: &Address, + request: &RemoteRequest, + state: &mut State, +) -> anyhow::Result> { + match request { + RemoteRequest::Download(package) => { + let Some(package_state) = state.packages.get(&package) else { + return Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadDenied))) + }; + if !package_state.mirroring { + return Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadDenied))); + } + // get the .zip from VFS and attach as payload to response + let vfs_address = Address { + node: our.node.clone(), + process: ProcessId::from_str("vfs:sys:uqbar")?, + }; + let file_name = format!("/{}.zip", package.to_string()); + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry(file_name.clone()), + })?), + None, + None, + 5, + )?; + // transfer will inherit the payload bytes we receive from VFS + spawn_transfer(&our, &file_name, None, &source); + Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadApproved))) + } + } +} diff --git a/modules/app_store/app_store/src/process_lib.rs b/modules/app_store/app_store/src/process_lib.rs new file mode 120000 index 00000000..9b9ec3f4 --- /dev/null +++ b/modules/app_store/app_store/src/process_lib.rs @@ -0,0 +1 @@ +../../../../src/process_lib.rs \ No newline at end of file diff --git a/modules/app_store/ft_worker/Cargo.lock b/modules/app_store/ft_worker/Cargo.lock new file mode 100644 index 00000000..7abc1fbd --- /dev/null +++ b/modules/app_store/ft_worker/Cargo.lock @@ -0,0 +1,415 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "cargo-component-bindings" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" +dependencies = [ + "cargo-component-macro", + "wit-bindgen 0.13.0", +] + +[[package]] +name = "cargo-component-macro" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "ft_worker" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "cargo-component-bindings", + "rand", + "serde", + "serde_json", + "wit-bindgen 0.9.0", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "indexmap" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +dependencies = [ + "equivalent", + "hashbrown", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + +[[package]] +name = "serde" +version = "1.0.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + +[[package]] +name = "spdx" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-encoder" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14abc161bfda5b519aa229758b68f2a52b45a12b993808665c857d1a9a00223c" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.115.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" +dependencies = [ + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5c3d15a04ce994fad2c5442a754b404ab1fee23c903a04a560f84f94fdf63c0" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wit-bindgen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d92ce0ca6b6074059413a9581a637550c3a740581c854f9847ec293c8aed71" +dependencies = [ + "bitflags", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "565b945ae074886071eccf9cdaf8ccd7b959c2b0d624095bea5fe62003e8b3e0" +dependencies = [ + "anyhow", + "wit-component", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5695ff4e41873ed9ce56d2787e6b5772bdad9e70e2c1d2d160621d1762257f4f" +dependencies = [ + "anyhow", + "heck", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91835ea4231da1fe7971679d505ba14be7826e192b6357f08465866ef482e08" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "wit-component" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87488b57a08e2cbbd076b325acbe7f8666965af174d69d5929cd373bd54547f" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ace9943d89bbf3dbbc71b966da0e7302057b311f36a4ac3d65ddfef17b52cf" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", +] diff --git a/modules/app_store/ft_worker/Cargo.toml b/modules/app_store/ft_worker/Cargo.toml new file mode 100644 index 00000000..9283a3e8 --- /dev/null +++ b/modules/app_store/ft_worker/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "ft_worker" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[profile.release] +panic = "abort" +opt-level = "s" +lto = true + +[dependencies] +anyhow = "1.0" +bincode = "1.3.3" +cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-component" } +rand = "0.8" +serde = {version = "1.0", features = ["derive"] } +serde_json = "1.0" +wit-bindgen = { version = "0.9.0", default_features = false } + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "component:microkernel-process" + +[package.metadata.component.target] +path = "wit" + +[package.metadata.component.dependencies] diff --git a/modules/app_store/ft_worker/src/ft_worker_lib.rs b/modules/app_store/ft_worker/src/ft_worker_lib.rs new file mode 100644 index 00000000..79085ed2 --- /dev/null +++ b/modules/app_store/ft_worker/src/ft_worker_lib.rs @@ -0,0 +1,134 @@ +use super::bindings::component::uq_process::types::*; +use super::bindings::{print_to_terminal, send_request, spawn, Address, Payload}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct FileTransferContext { + pub file_name: String, + pub file_size: Option, + pub start_time: std::time::SystemTime, +} + +/// sent as first Request to a newly spawned worker +/// the Receive command will be sent out to target +/// in order to prompt them to spawn a worker +#[derive(Debug, Serialize, Deserialize)] +pub enum FTWorkerCommand { + Send { + // make sure to attach file itself as payload + target: String, // annoying, but this is Address + file_name: String, + timeout: u64, + }, + Receive { + transfer_id: u64, + file_name: String, + file_size: u64, + total_chunks: u64, + timeout: u64, + }, +} + +/// sent as Response by worker to its parent +#[derive(Debug, Serialize, Deserialize)] +pub enum FTWorkerResult { + SendSuccess, + ReceiveSuccess(String), // name of file, bytes in payload + Err(TransferError), +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum TransferError { + TargetOffline, + TargetTimeout, + TargetRejected, + SourceFailed, +} + +pub fn spawn_transfer( + our: &Address, + file_name: &str, + file_bytes: Option>, // if None, expects to inherit payload! + to_addr: &Address, +) { + let transfer_id: u64 = rand::random(); + // spawn a worker and tell it to send the file + let Ok(worker_process_id) = spawn( + Some(&transfer_id.to_string()), + "/ft_worker.wasm".into(), + &OnPanic::None, // can set message-on-panic here + &Capabilities::All, + false, // not public + ) else { + print_to_terminal(0, "file_transfer: failed to spawn worker!"); + return; + }; + // tell the worker what to do + let payload_or_inherit = match file_bytes { + Some(bytes) => Some(Payload { mime: None, bytes }), + None => None, + }; + send_request( + &Address { + node: our.node.clone(), + process: worker_process_id, + }, + &Request { + inherit: !payload_or_inherit.is_some(), + expects_response: Some(61), + ipc: Some( + serde_json::to_string(&FTWorkerCommand::Send { + target: to_addr.to_string(), + file_name: file_name.into(), + timeout: 60, + }) + .unwrap(), + ), + metadata: None, + }, + Some( + &serde_json::to_string(&FileTransferContext { + file_name: file_name.into(), + file_size: match &payload_or_inherit { + Some(p) => Some(p.bytes.len() as u64), + None => None, // TODO + }, + start_time: std::time::SystemTime::now(), + }) + .unwrap(), + ), + payload_or_inherit.as_ref(), + ); +} + +pub fn spawn_receive_transfer(our: &Address, ipc: &str) { + let Ok(FTWorkerCommand::Receive { transfer_id, .. }) = serde_json::from_str(ipc) else { + print_to_terminal(0, "file_transfer: got weird request"); + return; + }; + let Ok(worker_process_id) = spawn( + Some(&transfer_id.to_string()), + "/ft_worker.wasm".into(), + &OnPanic::None, // can set message-on-panic here + &Capabilities::All, + false, // not public + ) else { + print_to_terminal(0, "file_transfer: failed to spawn worker!"); + return; + }; + // forward receive command to worker + send_request( + &Address { + node: our.node.clone(), + process: worker_process_id, + }, + &Request { + inherit: true, + expects_response: None, + ipc: Some(ipc.to_string()), + metadata: None, + }, + None, + None, + ); +} diff --git a/modules/app_store/ft_worker/src/lib.rs b/modules/app_store/ft_worker/src/lib.rs new file mode 100644 index 00000000..fd3a7ebc --- /dev/null +++ b/modules/app_store/ft_worker/src/lib.rs @@ -0,0 +1,227 @@ +cargo_component_bindings::generate!(); +use bindings::component::uq_process::types::*; +use bindings::{get_payload, print_to_terminal, receive, send_request, send_response, Guest}; +use serde::{Deserialize, Serialize}; + +struct Component; + +mod ft_worker_lib; +#[allow(dead_code)] +mod process_lib; +use ft_worker_lib::*; + +/// internal worker protocol +#[derive(Debug, Serialize, Deserialize)] +pub enum FTWorkerProtocol { + Ready, + Finished, +} + +impl Guest for Component { + fn init(our: Address) { + print_to_terminal(1, &format!("{}: start", our.process)); + + let Ok((parent_process, Message::Request(req))) = receive() else { + panic!("ft_worker: got bad init message"); + }; + + let command = serde_json::from_str::( + &req.ipc.expect("ft_worker: got empty init message"), + ) + .expect("ft_worker: got unparseable init message"); + + match command { + FTWorkerCommand::Send { + target, + file_name, + timeout, + } => { + let transfer_id: u64 = our.process.process().parse().unwrap(); + let Some(payload) = get_payload() else { + print_to_terminal(0, "FTWorker wasn't given payload, exiting"); + return + }; + let file_bytes = payload.bytes; + let mut file_size = file_bytes.len() as u64; + let mut offset: u64 = 0; + let mut chunk_size: u64 = 1048576; // 1MB + let total_chunks = (file_size as f64 / chunk_size as f64).ceil() as u64; + // send a file to another worker + // start by telling target to expect a file, + // then upon reciving affirmative response, + // send contents in chunks and wait for + // acknowledgement. + match bindings::send_and_await_response( + &Address::from_str(&target).unwrap(), + &Request { + inherit: false, + expects_response: Some(timeout), + ipc: Some( + serde_json::to_string(&FTWorkerCommand::Receive { + transfer_id, + file_name, + file_size, + total_chunks, + timeout, + }) + .unwrap(), + ), + metadata: None, + }, + None, + ) { + Err(send_error) => { + respond_to_parent(FTWorkerResult::Err(match send_error.kind { + SendErrorKind::Offline => TransferError::TargetOffline, + SendErrorKind::Timeout => TransferError::TargetTimeout, + })) + } + Ok((opp_worker, Message::Response((response, _)))) => { + let Ok(FTWorkerProtocol::Ready) = serde_json::from_str(&response.ipc.expect("ft_worker: got empty response")) else { + respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)); + return; + }; + // send file in chunks + loop { + if file_size < chunk_size { + // this is the last chunk, so we should expect a Finished response + chunk_size = file_size; + let payload = Payload { + mime: None, + bytes: file_bytes + [offset as usize..offset as usize + chunk_size as usize] + .to_vec(), + }; + send_request( + &opp_worker, + &Request { + inherit: false, + expects_response: Some(timeout), + ipc: None, + metadata: None, + }, + None, + Some(&payload), + ); + break; + } + let payload = Payload { + mime: None, + bytes: file_bytes + [offset as usize..offset as usize + chunk_size as usize] + .to_vec(), + }; + send_request( + &opp_worker, + &Request { + inherit: false, + expects_response: None, + ipc: None, + metadata: None, + }, + None, + Some(&payload), + ); + file_size -= chunk_size; + offset += chunk_size; + } + // now wait for Finished response + let Ok((receiving_worker, Message::Response((resp, _)))) = receive() else { + respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)); + return; + }; + let Ok(FTWorkerProtocol::Finished) = serde_json::from_str( + &resp.ipc.expect("ft_worker: got empty response"), + ) else { + respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)); + return; + }; + // return success to parent + respond_to_parent(FTWorkerResult::SendSuccess); + } + _ => respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)), + } + } + FTWorkerCommand::Receive { + transfer_id, + file_name, + file_size, + total_chunks, + timeout, + } => { + // send Ready response to counterparty + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&FTWorkerProtocol::Ready).unwrap()), + metadata: None, + }, + None, + ); + // receive a file from a worker, then send it to parent + // all messages will be chunks of file. when we receive the + // last chunk, send a Finished message to sender and Success to parent. + let mut file_bytes = Vec::new(); + let mut chunks_received = 0; + let start_time = std::time::Instant::now(); + loop { + let Ok((source, Message::Request(req))) = receive() else { + respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed)); + return; + }; + if start_time.elapsed().as_secs() > timeout { + respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed)); + return; + } + let Some(payload) = get_payload() else { + respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed)); + return; + }; + chunks_received += 1; + file_bytes.extend(payload.bytes); + if chunks_received == total_chunks { + break; + } + } + // send Finished message to sender + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&FTWorkerProtocol::Finished).unwrap()), + metadata: None, + }, + None, + ); + // send Success message to parent + send_request( + &parent_process, + &Request { + inherit: false, + expects_response: None, + ipc: Some( + serde_json::to_string(&FTWorkerResult::ReceiveSuccess(file_name)) + .unwrap(), + ), + metadata: None, + }, + None, + Some(&Payload { + mime: None, + bytes: file_bytes, + }), + ); + } + } + } +} + +fn respond_to_parent(result: FTWorkerResult) { + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&result).unwrap()), + metadata: None, + }, + None, + ); +} diff --git a/modules/app_store/ft_worker/src/process_lib.rs b/modules/app_store/ft_worker/src/process_lib.rs new file mode 120000 index 00000000..9b9ec3f4 --- /dev/null +++ b/modules/app_store/ft_worker/src/process_lib.rs @@ -0,0 +1 @@ +../../../../src/process_lib.rs \ No newline at end of file diff --git a/modules/app_store/pkg/manifest.json b/modules/app_store/pkg/manifest.json index 1c9ae034..a3d11081 100644 --- a/modules/app_store/pkg/manifest.json +++ b/modules/app_store/pkg/manifest.json @@ -5,7 +5,6 @@ "on_panic": "Restart", "request_networking": true, "request_messaging": [ - "http_bindings:http_bindings:uqbar", "terminal:terminal:uqbar", "filesystem:sys:uqbar", "http_server:sys:uqbar", diff --git a/modules/app_store/pkg/metadata.json b/modules/app_store/pkg/metadata.json index cbb01c6e..912510ce 100644 --- a/modules/app_store/pkg/metadata.json +++ b/modules/app_store/pkg/metadata.json @@ -1,5 +1,6 @@ { "package": "app_store", "publisher": "uqbar", + "version": [0, 1, 0], "description": "A package manager + app store. This JSON field is optional and you can add whatever you want in addition to this." } diff --git a/modules/app_store/src/kernel_types.rs b/modules/app_store/src/kernel_types.rs deleted file mode 120000 index 8311791c..00000000 --- a/modules/app_store/src/kernel_types.rs +++ /dev/null @@ -1 +0,0 @@ -../../../src/kernel_types.rs \ No newline at end of file diff --git a/modules/app_store/src/lib.rs b/modules/app_store/src/lib.rs deleted file mode 100644 index 90000a20..00000000 --- a/modules/app_store/src/lib.rs +++ /dev/null @@ -1,512 +0,0 @@ -cargo_component_bindings::generate!(); - -use bindings::{ - component::uq_process::types::*, get_capability, get_payload, print_to_terminal, receive, - send_request, send_response, Guest, -}; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; - -#[allow(dead_code)] -mod kernel_types; -use kernel_types as kt; - -#[allow(dead_code)] -mod process_lib; -use process_lib::PackageId; - -mod transfer_lib; - -struct Component; - -// #[derive(Serialize, Deserialize)] -// struct AppState { -// // TODO this should mirror onchain listing -// pub name: String, -// pub owner: NodeId, -// pub desc: String, -// pub website: Option, -// pub versions: Vec<(u32, String)>, // TODO -// } - -#[derive(Serialize, Deserialize)] -struct AppTrackerState { - pub mirrored_packages: HashSet, - pub requested_packages: HashMap, // who we're expecting it from -} - -#[derive(Debug, Serialize, Deserialize)] -pub enum AppTrackerRequest { - New { - package: PackageId, - mirror: bool, - }, - NewFromRemote { - package_id: PackageId, - install_from: NodeId, - }, - Install { - package: PackageId, - }, -} - -#[derive(Debug, Serialize, Deserialize)] -pub enum AppTrackerResponse { - New { package: String }, - NewFromRemote { package_id: PackageId }, - Install { package: String }, - Error { error: String }, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct PackageMetadata { - pub package: String, - pub publisher: String, - pub desc: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct PackageManifestEntry { - pub process_name: String, - pub process_wasm_path: String, - pub on_panic: kt::OnPanic, - pub request_networking: bool, - pub request_messaging: Vec, - pub public: bool, -} - -fn parse_command( - our: &Address, - source: &Address, - request_string: String, - state: &mut AppTrackerState, -) -> anyhow::Result> { - match serde_json::from_str(&request_string)? { - // create a new package based on local payload - AppTrackerRequest::New { package, mirror } => { - if our.node != source.node { - return Err(anyhow::anyhow!("new package request from non-local node")); - } - let Some(mut payload) = get_payload() else { - return Err(anyhow::anyhow!("no payload")); - }; - - let vfs_address = Address { - node: our.node.clone(), - process: ProcessId::from_str("vfs:sys:uqbar")?, - }; - - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::New, - })?), - None, - None, - 5, - )?; - - // add zip bytes - payload.mime = Some("application/zip".to_string()); - let _ = process_lib::send_and_await_response( - &vfs_address, - true, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::Add { - full_path: package.to_string(), - entry_type: kt::AddEntryType::ZipArchive, - }, - })?), - None, - Some(&payload), - 5, - )?; - - // save the zip file itself in VFS for sharing with other nodes - // call it .zip - let _ = process_lib::send_and_await_response( - &vfs_address, - true, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::Add { - full_path: format!("/{}.zip", package.to_string()), - entry_type: kt::AddEntryType::NewFile, - }, - })?), - None, - Some(&payload), - 5, - )?; - - // if mirror, save in our state - if mirror { - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetEntry("/metadata.json".into()), - })?), - None, - None, - 5, - )?; - let Some(payload) = get_payload() else { - return Err(anyhow::anyhow!("no metadata payload")); - }; - let metadata = String::from_utf8(payload.bytes)?; - let metadata = serde_json::from_str::(&metadata)?; - state - .mirrored_packages - .insert(PackageId::new(&metadata.package, &metadata.publisher)); - process_lib::set_state::(&state); - } - - Ok(Some(AppTrackerResponse::New { package: package.to_string() })) - } - // if we are the source, forward to install_from target. - // if we install_from, respond with package if we have it - AppTrackerRequest::NewFromRemote { - package_id, - install_from, - } => { - if our.node == source.node { - let _ = send_request( - &Address { - node: install_from.clone(), - process: our.process.clone(), - }, - &Request { - inherit: true, - expects_response: Some(5), // TODO - ipc: Some(serde_json::to_string(&AppTrackerRequest::NewFromRemote { - package_id: package_id.clone(), - install_from: install_from.clone(), - })?), - metadata: None, - }, - None, - None, - ); - state.requested_packages.insert(package_id, install_from); - process_lib::set_state::(&state); - Ok(None) - } else if our.node == install_from { - let Some(_mirror) = state.mirrored_packages.get(&package_id) else { - return Ok(Some(AppTrackerResponse::Error { error: "package not mirrored here!".into() })) - }; - // get the .zip from VFS and attach as payload to response - let vfs_address = Address { - node: our.node.clone(), - process: ProcessId::from_str("vfs:sys:uqbar")?, - }; - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package_id.to_string(), - action: kt::VfsAction::GetEntry(format!("/{}.zip", package_id.to_string())), - })?), - None, - None, - 5, - )?; - Ok(Some(AppTrackerResponse::NewFromRemote { package_id })) - } else { - // TODO what to do here? - Ok(None) - } - } - AppTrackerRequest::Install { package } => { - if our.node != source.node { - return Err(anyhow::anyhow!("install request from non-local node")); - } - let vfs_address = Address { - node: our.node.clone(), - process: ProcessId::from_str("vfs:sys:uqbar")?, - }; - - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetEntry("/manifest.json".into()), - })?), - None, - None, - 5, - )?; - let Some(payload) = get_payload() else { - return Err(anyhow::anyhow!("no payload")); - }; - let manifest = String::from_utf8(payload.bytes)?; - let manifest = serde_json::from_str::>(&manifest)?; - - for entry in manifest { - let path = if entry.process_wasm_path.starts_with("/") { - entry.process_wasm_path - } else { - format!("/{}", entry.process_wasm_path) - }; - - let (_, hash_response) = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetHash(path.clone()), - })?), - None, - None, - 5, - )?; - - let Message::Response((Response { ipc: Some(ipc), .. }, _)) = hash_response else { - return Err(anyhow::anyhow!("bad vfs response")); - }; - let kt::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc)? else { - return Err(anyhow::anyhow!("no hash in vfs")); - }; - - // build initial caps - let mut initial_capabilities: HashSet = HashSet::new(); - if entry.request_networking { - let Some(networking_cap) = get_capability( - &Address { - node: our.node.clone(), - process: ProcessId::from_str("kernel:sys:uqbar")?, - }, - &"\"network\"".to_string(), - ) else { - return Err(anyhow::anyhow!("app-store: no net cap")); - }; - initial_capabilities.insert(kt::de_wit_signed_capability(networking_cap)); - } - let Some(read_cap) = get_capability( - &vfs_address.clone(), - &serde_json::to_string(&serde_json::json!({ - "kind": "read", - "drive": package.to_string(), - }))?, - ) else { - return Err(anyhow::anyhow!("app-store: no read cap")); - }; - initial_capabilities.insert(kt::de_wit_signed_capability(read_cap)); - let Some(write_cap) = get_capability( - &vfs_address.clone(), - &serde_json::to_string(&serde_json::json!({ - "kind": "write", - "drive": package.to_string(), - }))?, - ) else { - return Err(anyhow::anyhow!("app-store: no write cap")); - }; - initial_capabilities.insert(kt::de_wit_signed_capability(write_cap)); - - for process_name in &entry.request_messaging { - let Ok(parsed_process_id) = ProcessId::from_str(&process_name) else { - // TODO handle arbitrary caps here - continue; - }; - let Some(messaging_cap) = get_capability( - &Address { - node: our.node.clone(), - process: parsed_process_id.clone(), - }, - &"\"messaging\"".into() - ) else { - print_to_terminal(0, &format!("app-store: no cap for {} to give away!", process_name)); - continue; - }; - initial_capabilities.insert(kt::de_wit_signed_capability(messaging_cap)); - } - - let process_id = format!("{}:{}", entry.process_name, package.to_string()); - let Ok(parsed_new_process_id) = ProcessId::from_str(&process_id) else { - return Err(anyhow::anyhow!("app-store: invalid process id!")); - }; - let _ = process_lib::send_request( - &Address { - node: our.node.clone(), - process: ProcessId::from_str("kernel:sys:uqbar")?, - }, - false, - Some(serde_json::to_string(&kt::KernelCommand::KillProcess( - kt::ProcessId::de_wit(parsed_new_process_id.clone()), - ))?), - None, - None, - None, - ); - - // kernel start process takes bytes as payload + wasm_bytes_handle... - // reconsider perhaps - let (_, _bytes_response) = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetEntry(path), - })?), - None, - None, - 5, - )?; - - let Some(payload) = get_payload() else { - return Err(anyhow::anyhow!("no wasm bytes payload.")); - }; - - let _ = process_lib::send_and_await_response( - &Address { - node: our.node.clone(), - process: ProcessId::from_str("kernel:sys:uqbar")?, - }, - false, - Some(serde_json::to_string(&kt::KernelCommand::StartProcess { - id: kt::ProcessId::de_wit(parsed_new_process_id), - wasm_bytes_handle: hash, - on_panic: entry.on_panic, - initial_capabilities, - public: entry.public, - })?), - None, - Some(&payload), - 5, - )?; - } - Ok(Some(AppTrackerResponse::Install { package: package.to_string() })) - } - } -} - -impl Guest for Component { - fn init(our: Address) { - assert_eq!(our.process.to_string(), "main:app_store:uqbar"); - - // grant messaging caps to http_server and terminal - let Some(our_messaging_cap) = bindings::get_capability( - &our, - &"\"messaging\"".into() - ) else { - panic!("missing self-messaging cap!") - }; - bindings::share_capability( - &ProcessId::from_str("http_server:sys:uqbar").unwrap(), - &our_messaging_cap, - ); - bindings::share_capability( - &ProcessId::from_str("terminal:terminal:uqbar").unwrap(), - &our_messaging_cap, - ); - - print_to_terminal(0, &format!("app_store main proc: start")); - - let mut state = process_lib::get_state::().unwrap_or(AppTrackerState { - mirrored_packages: HashSet::new(), - requested_packages: HashMap::new(), - }); - - loop { - let (source, message) = match receive() { - Ok((source, message)) => (source, message), - Err((error, _context)) => { - print_to_terminal(0, &format!("net error: {:?}", error.kind)); - continue; - } - }; - match message { - Message::Request(Request { - ipc, - expects_response, - metadata, - .. - }) => { - let Some(command) = ipc else { - continue; - }; - match parse_command(&our, &source, command, &mut state) { - Ok(response) => { - if let Some(_) = expects_response { - let _ = send_response( - &Response { - inherit: true, - ipc: Some(serde_json::to_string(&response).unwrap()), - metadata, - }, - None, // payload will be attached here if created in parse_command - ); - }; - } - Err(e) => { - print_to_terminal(0, &format!("app-store: got error {}", e)); - if let Some(_) = expects_response { - let error = AppTrackerResponse::Error { - error: format!("{}", e), - }; - let _ = send_response( - &Response { - inherit: false, - ipc: Some(serde_json::to_string(&error).unwrap()), - metadata, - }, - None, - ); - }; - } - } - } - Message::Response((response, _)) => { - print_to_terminal(0, &format!("app-store: got response {:?}", response)); - // only expecting NewFromRemote for apps we've requested - match serde_json::from_str(&response.ipc.unwrap_or_default()) { - Ok(AppTrackerResponse::NewFromRemote { package_id }) => { - if let Some(install_from) = state.requested_packages.remove(&package_id) - { - if install_from == source.node { - // auto-take zip from payload and request ourself with New - let _ = send_request( - &our, - &Request { - inherit: true, // will inherit payload! - expects_response: None, - ipc: Some( - serde_json::to_string(&AppTrackerRequest::New { - package: package_id, - mirror: true, - }) - .unwrap(), - ), - metadata: None, - }, - None, - None, - ); - } else { - print_to_terminal( - 0, - &format!( - "app-store: got install response from bad source: {}", - install_from - ), - ); - } - } - } - err => { - print_to_terminal( - 0, - &format!("app-store: got unexpected response {:?}", err), - ); - } - } - } - } - } - } -} diff --git a/modules/app_store/src/process_lib.rs b/modules/app_store/src/process_lib.rs deleted file mode 120000 index 77367fe0..00000000 --- a/modules/app_store/src/process_lib.rs +++ /dev/null @@ -1 +0,0 @@ -../../../src/process_lib.rs \ No newline at end of file diff --git a/modules/app_store/src/transfer_lib.rs b/modules/app_store/src/transfer_lib.rs deleted file mode 100644 index 2084e210..00000000 --- a/modules/app_store/src/transfer_lib.rs +++ /dev/null @@ -1,227 +0,0 @@ -use super::bindings::component::uq_process::types::*; -use crate::bindings::{get_payload, receive, send_request, send_response}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug)] -pub enum TransferError { - // in all errors, u64 is number of bytes successfully transferred - TargetOffline(u64), - TargetTimeout(u64), - TargetRejected(u64), - SourceFailed(u64), -} - -#[derive(Debug, Serialize, Deserialize)] -pub enum TransferMetadata { - Begin { - file_name: String, - file_size: u64, - total_chunks: u64, - }, -} - -pub fn transfer( - to_addr: Address, - bytes: Vec, - max_timeout: u64, -) -> ( - Result<(), TransferError>, - Vec)>>, -) { - let transfer_context_id: u64 = rand::random(); - let mut bytes_remaining: u64 = bytes.len() as u64; - let mut offset: u64 = 0; - let mut chunk_size: u64 = 1048576; // 1MB - let mut chunks_sent = 0; - let total_chunks = (bytes.len() as f64 / chunk_size as f64).ceil() as u64; - loop { - chunks_sent += 1; - if bytes_remaining < chunk_size { - chunk_size = bytes_remaining; - } - let payload = Payload { - mime: None, - bytes: bytes[offset as usize..offset as usize + chunk_size as usize].to_vec(), - }; - send_request( - &to_addr, - &Request { - inherit: false, - expects_response: Some(max_timeout), - ipc: None, - metadata: Some(if chunks_sent == 1 { - serde_json::to_string(&TransferMetadata::Begin { - file_name: "test".to_string(), - file_size: bytes.len() as u64, - total_chunks, - }) - .unwrap() - } else { - chunks_sent.to_string() - }), - }, - Some(&&transfer_context_id.to_string()), - Some(&payload), - ); - bytes_remaining -= chunk_size; - offset += chunk_size; - if bytes_remaining == 0 { - break; - } - } - let mut chunks_confirmed = 0; - let mut non_transfer_message_queue = Vec::new(); - loop { - let next = receive(); - if let Err((send_error, context)) = &next { - match context { - Some(_) => match send_error.kind { - SendErrorKind::Offline => { - return ( - Err(TransferError::TargetOffline(chunks_confirmed * chunk_size)), - non_transfer_message_queue, - ) - } - SendErrorKind::Timeout => { - return ( - Err(TransferError::TargetTimeout(chunks_confirmed * chunk_size)), - non_transfer_message_queue, - ) - } - }, - None => { - non_transfer_message_queue.push(next); - continue; - } - } - } - if let Ok((source, message)) = &next { - if source.process == to_addr.process { - match message { - Message::Request(_) => { - non_transfer_message_queue.push(next); - continue; - } - Message::Response((response, context)) => { - if transfer_context_id - == context - .as_ref() - .unwrap_or(&"".into()) - .parse::() - .unwrap_or(0) - { - chunks_confirmed += 1; - if response - .metadata - .as_ref() - .unwrap_or(&"".into()) - .parse::() - .unwrap_or(0) - != chunks_confirmed - { - return ( - Err(TransferError::TargetRejected( - chunks_confirmed * chunk_size, - )), - non_transfer_message_queue, - ); - } - if chunks_confirmed == chunks_sent { - return (Ok(()), non_transfer_message_queue); - } - } else { - non_transfer_message_queue.push(next); - } - } - } - } else { - non_transfer_message_queue.push(next); - continue; - } - } - } -} - -pub fn receive_transfer( - transfer_source: Address, - total_chunks: u64, - max_timeout: u64, -) -> ( - Result, TransferError>, - Vec)>>, -) { - let start_time = std::time::SystemTime::now(); - // get first payload then loop and receive rest - let mut file = match get_payload() { - Some(payload) => payload.bytes, - None => { - return (Err(TransferError::SourceFailed(0)), vec![]); - } - }; - // respond to first request - send_response( - &Response { - inherit: false, - ipc: None, - metadata: Some(1.to_string()), - }, - None, - ); - if total_chunks == 1 { - return (Ok(file), vec![]); - } - let mut chunk_num = 1; - let mut non_transfer_message_queue = Vec::new(); - loop { - let next = receive(); - if start_time.elapsed().expect("time error").as_secs() > max_timeout { - return ( - Err(TransferError::TargetTimeout(file.len() as u64)), - non_transfer_message_queue, - ); - } - if let Err(_) = &next { - non_transfer_message_queue.push(next); - } else if let Ok((source, message)) = &next { - // we know all messages from source process will be for this transfer, - // since they are sent sequentially and it's a single-file queue. - if source.process == transfer_source.process { - match message { - Message::Request(_) => { - let payload = match get_payload() { - Some(payload) => payload, - None => { - return ( - Err(TransferError::SourceFailed(file.len() as u64)), - non_transfer_message_queue, - ); - } - }; - chunk_num += 1; - file.extend(payload.bytes); - send_response( - &Response { - inherit: false, - ipc: None, - metadata: Some(chunk_num.to_string()), - }, - None, - ); - if chunk_num == total_chunks { - return (Ok(file), non_transfer_message_queue); - } - } - Message::Response(_) => { - return ( - Err(TransferError::SourceFailed(file.len() as u64)), - non_transfer_message_queue, - ); - } - } - } else { - non_transfer_message_queue.push(next); - continue; - } - } - } -} diff --git a/modules/chess/pkg/metadata.json b/modules/chess/pkg/metadata.json index be7718ff..f833a906 100644 --- a/modules/chess/pkg/metadata.json +++ b/modules/chess/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "chess", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/homepage/pkg/metadata.json b/modules/homepage/pkg/metadata.json index 24843095..5f440264 100644 --- a/modules/homepage/pkg/metadata.json +++ b/modules/homepage/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "homepage", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/http_proxy/pkg/metadata.json b/modules/http_proxy/pkg/metadata.json index 6acd4d15..8fc0ad14 100644 --- a/modules/http_proxy/pkg/metadata.json +++ b/modules/http_proxy/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "http_proxy", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/key_value/key_value/src/lib.rs b/modules/key_value/key_value/src/lib.rs index 4990355b..51946f80 100644 --- a/modules/key_value/key_value/src/lib.rs +++ b/modules/key_value/key_value/src/lib.rs @@ -191,6 +191,7 @@ impl Guest for Component { if let Some(e) = e.downcast_ref::() { send_response( &Response { + inherit: false, ipc: Some(serde_json::to_string(&e).unwrap()), metadata: None, }, diff --git a/modules/key_value/key_value_worker/src/lib.rs b/modules/key_value/key_value_worker/src/lib.rs index 2181a05b..7c892440 100644 --- a/modules/key_value/key_value_worker/src/lib.rs +++ b/modules/key_value/key_value_worker/src/lib.rs @@ -42,7 +42,7 @@ fn send_and_await_response_wrapped( }; let ( _, - Message::Response((Response { ipc, metadata }, _)), + Message::Response((Response { ipc, metadata, .. }, _)), ) = send_and_await_response( &Address { node: target_node, @@ -118,6 +118,7 @@ fn handle_message ( send_response( &Response { + inherit: false, ipc, metadata: None, }, @@ -137,6 +138,7 @@ fn handle_message ( None => { send_response( &Response { + inherit: false, ipc, metadata: None, }, @@ -155,6 +157,7 @@ fn handle_message ( ); send_response( &Response { + inherit: false, ipc, metadata: None, }, @@ -193,6 +196,7 @@ impl Guest for Component { if let Some(e) = e.downcast_ref::() { send_response( &Response { + inherit: false, ipc: Some(serde_json::to_string(&e).unwrap()), metadata: None, }, diff --git a/modules/key_value/pkg/metadata.json b/modules/key_value/pkg/metadata.json index 7db01e9b..549365f7 100644 --- a/modules/key_value/pkg/metadata.json +++ b/modules/key_value/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "key_value", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/orgs/pkg/metadata.json b/modules/orgs/pkg/metadata.json index b0d2380e..43e641c7 100644 --- a/modules/orgs/pkg/metadata.json +++ b/modules/orgs/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "orgs", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/qns_indexer/pkg/metadata.json b/modules/qns_indexer/pkg/metadata.json index 1d6a68e8..1a50d51f 100644 --- a/modules/qns_indexer/pkg/metadata.json +++ b/modules/qns_indexer/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "qns_indexer", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/terminal/pkg/metadata.json b/modules/terminal/pkg/metadata.json index b1a7dbff..5d8539c8 100644 --- a/modules/terminal/pkg/metadata.json +++ b/modules/terminal/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "terminal", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/src/filesystem/mod.rs b/src/filesystem/mod.rs index 49fd98da..3c9faf16 100644 --- a/src/filesystem/mod.rs +++ b/src/filesystem/mod.rs @@ -150,7 +150,34 @@ async fn bootstrap( for (package_name, mut package) in packages { println!("fs: handling package {package_name}...\r"); + // get and read metadata.json + let Ok(mut package_metadata_zip) = package.by_name("metadata.json") else { + println!( + "fs: missing metadata for package {}, skipping", + package_name + ); + continue; + }; + let mut metadata_content = Vec::new(); + package_metadata_zip + .read_to_end(&mut metadata_content) + .unwrap(); + drop(package_metadata_zip); + let package_metadata: serde_json::Value = + serde_json::from_slice(&metadata_content).expect("fs: metadata parse error"); + + println!("fs: found package metadata: {:?}\r", package_metadata); + + let package_name = package_metadata["package"] + .as_str() + .expect("fs: metadata parse error: bad package name"); + + let package_publisher = package_metadata["publisher"] + .as_str() + .expect("fs: metadata parse error: bad publisher name"); + // create a new package in VFS + let our_drive_name = [package_name, package_publisher].join(":"); vfs_messages.push(KernelMessage { id: rand::random(), source: Address { @@ -167,7 +194,7 @@ async fn bootstrap( expects_response: None, ipc: Some( serde_json::to_string::(&VfsRequest { - drive: package_name.clone(), + drive: our_drive_name.clone(), action: VfsAction::New, }) .unwrap(), @@ -208,7 +235,7 @@ async fn bootstrap( expects_response: None, ipc: Some( serde_json::to_string::(&VfsRequest { - drive: package_name.clone(), + drive: our_drive_name.clone(), action: VfsAction::Add { full_path: file_path, entry_type: AddEntryType::NewFile, @@ -227,32 +254,6 @@ async fn bootstrap( } } - // get and read metadata.json - let Ok(mut package_metadata_zip) = package.by_name("metadata.json") else { - println!( - "fs: missing metadata for package {}, skipping", - package_name - ); - continue; - }; - let mut metadata_content = Vec::new(); - package_metadata_zip - .read_to_end(&mut metadata_content) - .unwrap(); - drop(package_metadata_zip); - let package_metadata: serde_json::Value = - serde_json::from_slice(&metadata_content).expect("fs: metadata parse error"); - - println!("fs: found package metadata: {:?}\r", package_metadata); - - let package_name = package_metadata["package"] - .as_str() - .expect("fs: metadata parse error: bad package name"); - - let package_publisher = package_metadata["publisher"] - .as_str() - .expect("fs: metadata parse error: bad publisher name"); - // get and read manifest.json let Ok(mut package_manifest_zip) = package.by_name("manifest.json") else { println!( @@ -319,7 +320,7 @@ async fn bootstrap( }, params: serde_json::to_string(&serde_json::json!({ "kind": "read", - "drive": package_name, + "drive": our_drive_name, })) .unwrap(), }); @@ -330,7 +331,7 @@ async fn bootstrap( }, params: serde_json::to_string(&serde_json::json!({ "kind": "write", - "drive": package_name, + "drive": our_drive_name, })) .unwrap(), }); diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 1874963b..c220e32f 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -722,19 +722,19 @@ async fn handler( None => "".to_string(), }; // trim trailing "/" - let raw_path = normalize_path(path.as_str()); + let original_path = normalize_path(path.as_str()); let id: u64 = rand::random(); let real_headers = serialize_headers(&headers); let path_bindings = path_bindings.read().await; - let Ok(route) = path_bindings.recognize(&raw_path) else { + let Ok(route) = path_bindings.recognize(&original_path) else { return Ok(warp::reply::with_status(vec![], StatusCode::NOT_FOUND).into_response()); }; let bound_path = route.handler(); let app = bound_path.app.to_string(); let url_params: HashMap<&str, &str> = route.params().into_iter().collect(); - let raw_path = remove_process_id(&raw_path); + let raw_path = remove_process_id(&original_path); let path = remove_process_id(&bound_path.original_path); if bound_path.authenticated { @@ -899,12 +899,11 @@ async fn handler( signed_capabilities: None, } }; - let (response_sender, response_receiver) = oneshot::channel(); http_response_senders .lock() .await - .insert(id, (raw_path.clone(), response_sender)); + .insert(id, (original_path.clone(), response_sender)); send_to_loop.send(message).await.unwrap(); let timeout_duration = tokio::time::Duration::from_secs(15); // adjust as needed diff --git a/src/kernel/mod.rs b/src/kernel/mod.rs index fc3a9967..0711fa65 100644 --- a/src/kernel/mod.rs +++ b/src/kernel/mod.rs @@ -94,52 +94,6 @@ impl WasiView for ProcessWasi { } } -/// -/// intercept wasi random -/// - -// #[async_trait::async_trait] -// impl wasi::random::insecure::Host for ProcessWasi { -// async fn get_insecure_random_bytes(&mut self, len: u64) -> Result> { -// let mut bytes = Vec::with_capacity(len as usize); -// for _ in 0..len { -// bytes.push(rand::random()); -// } -// Ok(bytes) -// } - -// async fn get_insecure_random_u64(&mut self) -> Result { -// Ok(rand::random()) -// } -// } - -// #[async_trait::async_trait] -// impl wasi::random::insecure_seed::Host for ProcessWasi { -// async fn insecure_seed(&mut self) -> Result<(u64, u64)> { -// Ok((rand::random(), rand::random())) -// } -// } - -// #[async_trait::async_trait] -// impl wasi::random::random::Host for ProcessWasi { -// async fn get_random_bytes(&mut self, len: u64) -> Result> { -// let mut bytes = Vec::with_capacity(len as usize); -// getrandom::getrandom(&mut bytes[..])?; -// Ok(bytes) -// } - -// async fn get_random_u64(&mut self) -> Result { -// let mut bytes = Vec::with_capacity(8); -// getrandom::getrandom(&mut bytes[..])?; - -// let mut number = 0u64; -// for (i, &byte) in bytes.iter().enumerate() { -// number |= (byte as u64) << (i * 8); -// } -// Ok(number) -// } -// } - /// /// create the process API. this is where the functions that a process can use live. /// @@ -328,6 +282,8 @@ impl UqProcessImports for ProcessWasi { capabilities: wit::Capabilities, public: bool, ) -> Result> { + // save existing payload to restore later + let old_last_payload = self.process.last_payload.clone(); let vfs_address = wit::Address { node: self.process.metadata.our.node.clone(), process: VFS_PROCESS_ID.en_wit(), @@ -358,13 +314,19 @@ impl UqProcessImports for ProcessWasi { .await else { println!("spawn: GetHash fail"); + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let wit::Message::Response((wit::Response { ipc: Some(ipc), .. }, _)) = hash_response else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let t::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc).unwrap() else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let Ok(Ok(_)) = send_and_await_response( @@ -387,9 +349,13 @@ impl UqProcessImports for ProcessWasi { ) .await else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let Some(t::Payload { mime: _, ref bytes }) = self.process.last_payload else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; @@ -470,8 +436,12 @@ impl UqProcessImports for ProcessWasi { ) .await else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NameTaken)); }; + // reset payload to what it was + self.process.last_payload = old_last_payload; let wit::Message::Response((wit::Response { ipc: Some(ipc), .. }, _)) = response else { return Ok(Err(wit::SpawnError::NoFileAtPath)); }; @@ -1098,13 +1068,13 @@ impl Process { } /// persist process_map state for next bootup +/// and wait for filesystem to respond in the affirmative async fn persist_state( our_name: &String, send_to_loop: &t::MessageSender, process_map: &t::ProcessMap, ) -> Result<()> { let bytes = bincode::serialize(process_map)?; - send_to_loop .send(t::KernelMessage { id: rand::random(), @@ -1119,7 +1089,7 @@ async fn persist_state( rsvp: None, message: t::Message::Request(t::Request { inherit: true, - expects_response: Some(5), // TODO evaluate + expects_response: None, ipc: Some( serde_json::to_string(&t::FsAction::SetState(KERNEL_PROCESS_ID.clone())) .unwrap(), @@ -1818,7 +1788,6 @@ async fn start_process( ); process_map.insert(process_id, process_metadata.persisted); - if !process_metadata.reboot { // if new, persist let _ = persist_state(&our_name, &send_to_loop, &process_map).await; diff --git a/src/kernel_types.rs b/src/kernel_types.rs index 4e483a08..1e4af80e 100644 --- a/src/kernel_types.rs +++ b/src/kernel_types.rs @@ -311,6 +311,33 @@ impl VfsError { } } +// +// package types +// + +pub type PackageVersion = (u32, u32, u32); + +/// the type that gets deserialized from `metadata.json` in a package +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageMetadata { + pub package: String, + pub publisher: String, + pub version: PackageVersion, + pub description: Option, + pub website: Option, +} + +/// the type that gets deserialized from each entry in the array in `manifest.json` +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageManifestEntry { + pub process_name: String, + pub process_wasm_path: String, + pub on_panic: OnPanic, + pub request_networking: bool, + pub request_messaging: Vec, + pub public: bool, +} + // // conversions between wit types and kernel types (annoying!) // diff --git a/src/process_lib.rs b/src/process_lib.rs index bf8c61ca..c72472f1 100644 --- a/src/process_lib.rs +++ b/src/process_lib.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use super::bindings::component::uq_process::types::*; -use super::bindings::{Address, Payload, ProcessId, SendError}; +use super::bindings::{get_capability, share_capability, Address, Payload, ProcessId, SendError}; #[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct PackageId { @@ -276,6 +276,18 @@ where Ok(parsed) } +pub fn grant_messaging(our: &Address, grant_to: &Vec) { + let Some(our_messaging_cap) = get_capability( + our, + &"\"messaging\"".into() + ) else { + panic!("missing self-messaging cap!") + }; + for process in grant_to { + share_capability(&process, &our_messaging_cap); + } +} + // move these to better place! #[derive(Serialize, Deserialize, Debug)] pub enum FsAction { diff --git a/src/types.rs b/src/types.rs index bd1a5709..a2482b3d 100644 --- a/src/types.rs +++ b/src/types.rs @@ -435,6 +435,19 @@ pub struct ProcessContext { // filesystem.rs types // +pub type PackageVersion = (u32, u32, u32); + +/// the type that gets deserialized from `metadata.json` in a package +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageMetadata { + pub package: String, + pub publisher: String, + pub version: PackageVersion, + pub description: Option, + pub website: Option, +} + +/// the type that gets deserialized from each entry in the array in `manifest.json` #[derive(Debug, Serialize, Deserialize)] pub struct PackageManifestEntry { pub process_name: String, @@ -705,8 +718,16 @@ impl std::fmt::Display for KernelMessage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, - "{{\n id: {},\n source: {},\n target: {},\n rsvp: {:?},\n message: {},\n payload: {}\n}}", - self.id, self.source, self.target, self.rsvp, self.message, self.payload.is_some() + "{{\n id: {},\n source: {},\n target: {},\n rsvp: {},\n message: {},\n payload: {}\n}}", + self.id, + self.source, + self.target, + match &self.rsvp { + Some(rsvp) => rsvp.to_string(), + None => "None".to_string() + }, + self.message, + self.payload.is_some(), ) } } diff --git a/src/vfs.rs b/src/vfs.rs index aaefd262..9633b1d0 100644 --- a/src/vfs.rs +++ b/src/vfs.rs @@ -292,14 +292,12 @@ pub async fn vfs( let our_node = our_node.clone(); let send_to_loop = send_to_loop.clone(); let serialized_state = state_to_bytes(&drive_to_vfs).await; - tokio::spawn( - send_persist_state_message( - our_node.clone(), - send_to_loop, - respond_to_id, - serialized_state, - ) - ); + send_persist_state_message( + our_node.clone(), + send_to_loop, + respond_to_id, + serialized_state, + ).await; }, km = recv_from_loop.recv() => { let Some(km) = km else {