From d6f50e0e384e416973d1ae1eceb5841f61d8969f Mon Sep 17 00:00:00 2001 From: bitful-pannul <109035169+bitful-pannul@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:27:18 +0200 Subject: [PATCH 01/24] rpc: hotfix (#35) --- src/http_server/mod.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 1874963b..c220e32f 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -722,19 +722,19 @@ async fn handler( None => "".to_string(), }; // trim trailing "/" - let raw_path = normalize_path(path.as_str()); + let original_path = normalize_path(path.as_str()); let id: u64 = rand::random(); let real_headers = serialize_headers(&headers); let path_bindings = path_bindings.read().await; - let Ok(route) = path_bindings.recognize(&raw_path) else { + let Ok(route) = path_bindings.recognize(&original_path) else { return Ok(warp::reply::with_status(vec![], StatusCode::NOT_FOUND).into_response()); }; let bound_path = route.handler(); let app = bound_path.app.to_string(); let url_params: HashMap<&str, &str> = route.params().into_iter().collect(); - let raw_path = remove_process_id(&raw_path); + let raw_path = remove_process_id(&original_path); let path = remove_process_id(&bound_path.original_path); if bound_path.authenticated { @@ -899,12 +899,11 @@ async fn handler( signed_capabilities: None, } }; - let (response_sender, response_receiver) = oneshot::channel(); http_response_senders .lock() .await - .insert(id, (raw_path.clone(), response_sender)); + .insert(id, (original_path.clone(), response_sender)); send_to_loop.send(message).await.unwrap(); let timeout_duration = tokio::time::Duration::from_secs(15); // adjust as needed From 06385f3d3ab15730aa4d3dc2bce99f6b489a5e45 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 24 Oct 2023 09:57:25 -0700 Subject: [PATCH 02/24] hotfix: fix drives of initial apps and spawn() the culprit: https://github.com/uqbar-dao/uqbar/commit/c9010147d66285baecc688f5ab41e679fef784a2#diff-734a7b0476de23ada17258a7ec6ad91a9ebb29abea30d8554124d01b24fba409R335-R339 --- src/filesystem/mod.rs | 65 +++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/filesystem/mod.rs b/src/filesystem/mod.rs index 49fd98da..6c6cde52 100644 --- a/src/filesystem/mod.rs +++ b/src/filesystem/mod.rs @@ -150,7 +150,38 @@ async fn bootstrap( for (package_name, mut package) in packages { println!("fs: handling package {package_name}...\r"); + // get and read metadata.json + let Ok(mut package_metadata_zip) = package.by_name("metadata.json") else { + println!( + "fs: missing metadata for package {}, skipping", + package_name + ); + continue; + }; + let mut metadata_content = Vec::new(); + package_metadata_zip + .read_to_end(&mut metadata_content) + .unwrap(); + drop(package_metadata_zip); + let package_metadata: serde_json::Value = + serde_json::from_slice(&metadata_content).expect("fs: metadata parse error"); + + println!("fs: found package metadata: {:?}\r", package_metadata); + + let package_name = package_metadata["package"] + .as_str() + .expect("fs: metadata parse error: bad package name"); + + let package_publisher = package_metadata["publisher"] + .as_str() + .expect("fs: metadata parse error: bad publisher name"); + // create a new package in VFS + let our_drive_name = [ + package_name, + package_publisher, + ] + .join(":"); vfs_messages.push(KernelMessage { id: rand::random(), source: Address { @@ -167,7 +198,7 @@ async fn bootstrap( expects_response: None, ipc: Some( serde_json::to_string::(&VfsRequest { - drive: package_name.clone(), + drive: our_drive_name.clone(), action: VfsAction::New, }) .unwrap(), @@ -208,7 +239,7 @@ async fn bootstrap( expects_response: None, ipc: Some( serde_json::to_string::(&VfsRequest { - drive: package_name.clone(), + drive: our_drive_name.clone(), action: VfsAction::Add { full_path: file_path, entry_type: AddEntryType::NewFile, @@ -227,32 +258,6 @@ async fn bootstrap( } } - // get and read metadata.json - let Ok(mut package_metadata_zip) = package.by_name("metadata.json") else { - println!( - "fs: missing metadata for package {}, skipping", - package_name - ); - continue; - }; - let mut metadata_content = Vec::new(); - package_metadata_zip - .read_to_end(&mut metadata_content) - .unwrap(); - drop(package_metadata_zip); - let package_metadata: serde_json::Value = - serde_json::from_slice(&metadata_content).expect("fs: metadata parse error"); - - println!("fs: found package metadata: {:?}\r", package_metadata); - - let package_name = package_metadata["package"] - .as_str() - .expect("fs: metadata parse error: bad package name"); - - let package_publisher = package_metadata["publisher"] - .as_str() - .expect("fs: metadata parse error: bad publisher name"); - // get and read manifest.json let Ok(mut package_manifest_zip) = package.by_name("manifest.json") else { println!( @@ -319,7 +324,7 @@ async fn bootstrap( }, params: serde_json::to_string(&serde_json::json!({ "kind": "read", - "drive": package_name, + "drive": our_drive_name, })) .unwrap(), }); @@ -330,7 +335,7 @@ async fn bootstrap( }, params: serde_json::to_string(&serde_json::json!({ "kind": "write", - "drive": package_name, + "drive": our_drive_name, })) .unwrap(), }); From ead4b1f64acf3afce80b1ba501ebcb4adee4553d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:59:11 +0000 Subject: [PATCH 03/24] Format Rust code using rustfmt --- src/filesystem/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/filesystem/mod.rs b/src/filesystem/mod.rs index 6c6cde52..3c9faf16 100644 --- a/src/filesystem/mod.rs +++ b/src/filesystem/mod.rs @@ -177,11 +177,7 @@ async fn bootstrap( .expect("fs: metadata parse error: bad publisher name"); // create a new package in VFS - let our_drive_name = [ - package_name, - package_publisher, - ] - .join(":"); + let our_drive_name = [package_name, package_publisher].join(":"); vfs_messages.push(KernelMessage { id: rand::random(), source: Address { From ca724671e3240c7255abd0b732ccd2475d442cb2 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 19 Oct 2023 13:45:24 -0400 Subject: [PATCH 04/24] WIP --- src/kernel/mod.rs | 48 +---------------------------------------------- src/types.rs | 12 ++++++++++-- src/vfs.rs | 14 ++++++-------- 3 files changed, 17 insertions(+), 57 deletions(-) diff --git a/src/kernel/mod.rs b/src/kernel/mod.rs index fc3a9967..2eba13ac 100644 --- a/src/kernel/mod.rs +++ b/src/kernel/mod.rs @@ -94,52 +94,6 @@ impl WasiView for ProcessWasi { } } -/// -/// intercept wasi random -/// - -// #[async_trait::async_trait] -// impl wasi::random::insecure::Host for ProcessWasi { -// async fn get_insecure_random_bytes(&mut self, len: u64) -> Result> { -// let mut bytes = Vec::with_capacity(len as usize); -// for _ in 0..len { -// bytes.push(rand::random()); -// } -// Ok(bytes) -// } - -// async fn get_insecure_random_u64(&mut self) -> Result { -// Ok(rand::random()) -// } -// } - -// #[async_trait::async_trait] -// impl wasi::random::insecure_seed::Host for ProcessWasi { -// async fn insecure_seed(&mut self) -> Result<(u64, u64)> { -// Ok((rand::random(), rand::random())) -// } -// } - -// #[async_trait::async_trait] -// impl wasi::random::random::Host for ProcessWasi { -// async fn get_random_bytes(&mut self, len: u64) -> Result> { -// let mut bytes = Vec::with_capacity(len as usize); -// getrandom::getrandom(&mut bytes[..])?; -// Ok(bytes) -// } - -// async fn get_random_u64(&mut self) -> Result { -// let mut bytes = Vec::with_capacity(8); -// getrandom::getrandom(&mut bytes[..])?; - -// let mut number = 0u64; -// for (i, &byte) in bytes.iter().enumerate() { -// number |= (byte as u64) << (i * 8); -// } -// Ok(number) -// } -// } - /// /// create the process API. this is where the functions that a process can use live. /// @@ -1098,6 +1052,7 @@ impl Process { } /// persist process_map state for next bootup +/// and wait for filesystem to respond in the affirmative async fn persist_state( our_name: &String, send_to_loop: &t::MessageSender, @@ -1818,7 +1773,6 @@ async fn start_process( ); process_map.insert(process_id, process_metadata.persisted); - if !process_metadata.reboot { // if new, persist let _ = persist_state(&our_name, &send_to_loop, &process_map).await; diff --git a/src/types.rs b/src/types.rs index b037ce5b..a90fcf48 100644 --- a/src/types.rs +++ b/src/types.rs @@ -673,8 +673,16 @@ impl std::fmt::Display for KernelMessage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, - "{{\n id: {},\n source: {},\n target: {},\n rsvp: {:?},\n message: {},\n payload: {}\n}}", - self.id, self.source, self.target, self.rsvp, self.message, self.payload.is_some() + "{{\n id: {},\n source: {},\n target: {},\n rsvp: {},\n message: {},\n payload: {}\n}}", + self.id, + self.source, + self.target, + match &self.rsvp { + Some(rsvp) => rsvp.to_string(), + None => "None".to_string() + }, + self.message, + self.payload.is_some(), ) } } diff --git a/src/vfs.rs b/src/vfs.rs index aaefd262..9633b1d0 100644 --- a/src/vfs.rs +++ b/src/vfs.rs @@ -292,14 +292,12 @@ pub async fn vfs( let our_node = our_node.clone(); let send_to_loop = send_to_loop.clone(); let serialized_state = state_to_bytes(&drive_to_vfs).await; - tokio::spawn( - send_persist_state_message( - our_node.clone(), - send_to_loop, - respond_to_id, - serialized_state, - ) - ); + send_persist_state_message( + our_node.clone(), + send_to_loop, + respond_to_id, + serialized_state, + ).await; }, km = recv_from_loop.recv() => { let Some(km) = km else { From 4aa7cc9f9e6bacadd567f4141c15c455bdd09896 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 19 Oct 2023 16:24:05 -0400 Subject: [PATCH 05/24] WIP: use ft_worker in app_store --- .../{ => app_store}/Cargo-component.lock | 0 modules/app_store/{ => app_store}/Cargo.lock | 0 modules/app_store/{ => app_store}/Cargo.toml | 0 .../app_store/app_store/src/ft_worker_lib.rs | 1 + .../app_store/app_store/src/kernel_types.rs | 1 + modules/app_store/{ => app_store}/src/lib.rs | 4 +- .../app_store/app_store/src/process_lib.rs | 1 + modules/app_store/ft_worker/Cargo.lock | 514 ++++++++++++++++++ modules/app_store/ft_worker/Cargo.toml | 31 ++ .../app_store/ft_worker/src/ft_worker_lib.rs | 130 +++++ modules/app_store/ft_worker/src/lib.rs | 226 ++++++++ .../app_store/ft_worker/src/process_lib.rs | 1 + modules/app_store/src/kernel_types.rs | 1 - modules/app_store/src/process_lib.rs | 1 - modules/app_store/src/transfer_lib.rs | 227 -------- src/kernel/mod.rs | 3 +- 16 files changed, 909 insertions(+), 232 deletions(-) rename modules/app_store/{ => app_store}/Cargo-component.lock (100%) rename modules/app_store/{ => app_store}/Cargo.lock (100%) rename modules/app_store/{ => app_store}/Cargo.toml (100%) create mode 120000 modules/app_store/app_store/src/ft_worker_lib.rs create mode 120000 modules/app_store/app_store/src/kernel_types.rs rename modules/app_store/{ => app_store}/src/lib.rs (99%) create mode 120000 modules/app_store/app_store/src/process_lib.rs create mode 100644 modules/app_store/ft_worker/Cargo.lock create mode 100644 modules/app_store/ft_worker/Cargo.toml create mode 100644 modules/app_store/ft_worker/src/ft_worker_lib.rs create mode 100644 modules/app_store/ft_worker/src/lib.rs create mode 120000 modules/app_store/ft_worker/src/process_lib.rs delete mode 120000 modules/app_store/src/kernel_types.rs delete mode 120000 modules/app_store/src/process_lib.rs delete mode 100644 modules/app_store/src/transfer_lib.rs diff --git a/modules/app_store/Cargo-component.lock b/modules/app_store/app_store/Cargo-component.lock similarity index 100% rename from modules/app_store/Cargo-component.lock rename to modules/app_store/app_store/Cargo-component.lock diff --git a/modules/app_store/Cargo.lock b/modules/app_store/app_store/Cargo.lock similarity index 100% rename from modules/app_store/Cargo.lock rename to modules/app_store/app_store/Cargo.lock diff --git a/modules/app_store/Cargo.toml b/modules/app_store/app_store/Cargo.toml similarity index 100% rename from modules/app_store/Cargo.toml rename to modules/app_store/app_store/Cargo.toml diff --git a/modules/app_store/app_store/src/ft_worker_lib.rs b/modules/app_store/app_store/src/ft_worker_lib.rs new file mode 120000 index 00000000..1fe915a3 --- /dev/null +++ b/modules/app_store/app_store/src/ft_worker_lib.rs @@ -0,0 +1 @@ +../../ft_worker/src/ft_worker_lib.rs \ No newline at end of file diff --git a/modules/app_store/app_store/src/kernel_types.rs b/modules/app_store/app_store/src/kernel_types.rs new file mode 120000 index 00000000..047e48bc --- /dev/null +++ b/modules/app_store/app_store/src/kernel_types.rs @@ -0,0 +1 @@ +../../../../src/kernel_types.rs \ No newline at end of file diff --git a/modules/app_store/src/lib.rs b/modules/app_store/app_store/src/lib.rs similarity index 99% rename from modules/app_store/src/lib.rs rename to modules/app_store/app_store/src/lib.rs index 90000a20..36e62409 100644 --- a/modules/app_store/src/lib.rs +++ b/modules/app_store/app_store/src/lib.rs @@ -15,7 +15,9 @@ use kernel_types as kt; mod process_lib; use process_lib::PackageId; -mod transfer_lib; +#[allow(dead_code)] +mod ft_worker_lib; +use ft_worker_lib::*; struct Component; diff --git a/modules/app_store/app_store/src/process_lib.rs b/modules/app_store/app_store/src/process_lib.rs new file mode 120000 index 00000000..9b9ec3f4 --- /dev/null +++ b/modules/app_store/app_store/src/process_lib.rs @@ -0,0 +1 @@ +../../../../src/process_lib.rs \ No newline at end of file diff --git a/modules/app_store/ft_worker/Cargo.lock b/modules/app_store/ft_worker/Cargo.lock new file mode 100644 index 00000000..cc84648e --- /dev/null +++ b/modules/app_store/ft_worker/Cargo.lock @@ -0,0 +1,514 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "cargo-component-bindings" +version = "0.2.0" +source = "git+https://github.com/bytecodealliance/cargo-component#2c45278720196778704f5baf62066d355d48f702" +dependencies = [ + "cargo-component-macro", + "wit-bindgen 0.12.0", +] + +[[package]] +name = "cargo-component-macro" +version = "0.2.0" +source = "git+https://github.com/bytecodealliance/cargo-component#2c45278720196778704f5baf62066d355d48f702" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "ft_worker" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "cargo-component-bindings", + "rand", + "serde", + "serde_json", + "wit-bindgen 0.9.0", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +dependencies = [ + "equivalent", + "hashbrown", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + +[[package]] +name = "spdx" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-encoder" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34180c89672b3e4825c3a8db4b61a674f1447afd5fe2445b2d22c3d8b6ea086c" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c051ef041d348324b01ff0419f6f6593f094b4897d93c9cf52d5d1ac879ba" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.113.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "286049849b5a5bd09a8773171be96824afabffc7cc3df6caaf33a38db6cd07ae" +dependencies = [ + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5c3d15a04ce994fad2c5442a754b404ab1fee23c903a04a560f84f94fdf63c0" +dependencies = [ + "bitflags 2.4.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.12.0" +source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +dependencies = [ + "bitflags 2.4.0", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.12.0" +source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +dependencies = [ + "anyhow", + "wit-component", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.12.0" +source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +dependencies = [ + "anyhow", + "heck", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.12.0" +source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "wit-component" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e2bf941487fc5afa9e3fc94761f6b80ecef5a2bed6239b959d23d9de69e3448" +dependencies = [ + "anyhow", + "bitflags 2.4.0", + "indexmap", + "log", + "serde", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a39edca9abb16309def3843af73b58d47d243fe33a9ceee572446bcc57556b9a" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "pulldown-cmark", + "semver", + "serde", + "serde_json", + "unicode-xid", + "url", +] diff --git a/modules/app_store/ft_worker/Cargo.toml b/modules/app_store/ft_worker/Cargo.toml new file mode 100644 index 00000000..9283a3e8 --- /dev/null +++ b/modules/app_store/ft_worker/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "ft_worker" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[profile.release] +panic = "abort" +opt-level = "s" +lto = true + +[dependencies] +anyhow = "1.0" +bincode = "1.3.3" +cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-component" } +rand = "0.8" +serde = {version = "1.0", features = ["derive"] } +serde_json = "1.0" +wit-bindgen = { version = "0.9.0", default_features = false } + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "component:microkernel-process" + +[package.metadata.component.target] +path = "wit" + +[package.metadata.component.dependencies] diff --git a/modules/app_store/ft_worker/src/ft_worker_lib.rs b/modules/app_store/ft_worker/src/ft_worker_lib.rs new file mode 100644 index 00000000..514a5260 --- /dev/null +++ b/modules/app_store/ft_worker/src/ft_worker_lib.rs @@ -0,0 +1,130 @@ +use super::bindings::component::uq_process::types::*; +use super::bindings::{Address, Payload, print_to_terminal, send_request, spawn}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct FileTransferContext { + pub file_name: String, + pub file_size: u64, + pub start_time: std::time::SystemTime, +} + +/// sent as first Request to a newly spawned worker +/// the Receive command will be sent out to target +/// in order to prompt them to spawn a worker +#[derive(Debug, Serialize, Deserialize)] +pub enum FTWorkerCommand { + Send { + // make sure to attach file itself as payload + target: String, // annoying, but this is Address + file_name: String, + timeout: u64, + }, + Receive { + transfer_id: u64, + file_name: String, + file_size: u64, + total_chunks: u64, + timeout: u64, + }, +} + +/// sent as Response by worker to its parent +#[derive(Debug, Serialize, Deserialize)] +pub enum FTWorkerResult { + SendSuccess, + ReceiveSuccess(String), // name of file, bytes in payload + Err(TransferError), +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum TransferError { + TargetOffline, + TargetTimeout, + TargetRejected, + SourceFailed, +} + +pub fn spawn_transfer( + our: &Address, + file_name: &str, + file_bytes: Vec, + to_addr: &Address, +) { + let transfer_id: u64 = rand::random(); + // spawn a worker and tell it to send the file + let Ok(worker_process_id) = spawn( + Some(&transfer_id.to_string()), + "/ft_worker.wasm".into(), + &OnPanic::None, // can set message-on-panic here + &Capabilities::All, + false, // not public + ) else { + print_to_terminal(0, "file_transfer: failed to spawn worker!"); + return; + }; + // tell the worker what to do + send_request( + &Address { + node: our.node.clone(), + process: worker_process_id, + }, + &Request { + inherit: false, + expects_response: Some(61), + ipc: Some( + serde_json::to_string(&FTWorkerCommand::Send { + target: to_addr.to_string(), + file_name: file_name.into(), + timeout: 60, + }) + .unwrap(), + ), + metadata: None, + }, + Some( + &serde_json::to_string(&FileTransferContext { + file_name: file_name.into(), + file_size: file_bytes.len() as u64, + start_time: std::time::SystemTime::now(), + }) + .unwrap(), + ), + Some(&Payload { mime: None, bytes: file_bytes }), + ); +} + +pub fn spawn_receive_transfer( + our: &Address, + ipc: &str, +) { + let Ok(FTWorkerCommand::Receive { transfer_id, .. }) = serde_json::from_str(ipc) else { + print_to_terminal(0, "file_transfer: got weird request"); + return; + }; + let Ok(worker_process_id) = spawn( + Some(&transfer_id.to_string()), + "/ft_worker.wasm".into(), + &OnPanic::None, // can set message-on-panic here + &Capabilities::All, + false, // not public + ) else { + print_to_terminal(0, "file_transfer: failed to spawn worker!"); + return; + }; + // forward receive command to worker + send_request( + &Address { + node: our.node.clone(), + process: worker_process_id, + }, + &Request { + inherit: true, + expects_response: None, + ipc: Some(ipc.to_string()), + metadata: None, + }, + None, + None, + ); +} diff --git a/modules/app_store/ft_worker/src/lib.rs b/modules/app_store/ft_worker/src/lib.rs new file mode 100644 index 00000000..45f75cd9 --- /dev/null +++ b/modules/app_store/ft_worker/src/lib.rs @@ -0,0 +1,226 @@ +cargo_component_bindings::generate!(); +use bindings::component::uq_process::types::*; +use bindings::{get_payload, print_to_terminal, receive, send_request, send_response, Guest}; +use serde::{Deserialize, Serialize}; + +struct Component; + +mod ft_worker_lib; +#[allow(dead_code)] +mod process_lib; +use ft_worker_lib::*; + +/// internal worker protocol +#[derive(Debug, Serialize, Deserialize)] +pub enum FTWorkerProtocol { + Ready, + Finished, +} + +impl Guest for Component { + fn init(our: Address) { + print_to_terminal(1, &format!("{}: start", our.process)); + + let Ok((parent_process, Message::Request(req))) = receive() else { + panic!("ft_worker: got bad init message"); + }; + + let command = serde_json::from_str::( + &req.ipc.expect("ft_worker: got empty init message"), + ) + .expect("ft_worker: got unparseable init message"); + + match command { + FTWorkerCommand::Send { + target, + file_name, + timeout, + } => { + let transfer_id: u64 = our.process.process().parse().unwrap(); + let Some(payload) = get_payload() else { + panic!("ft_worker: got empty payload"); + }; + let file_bytes = payload.bytes; + let mut file_size = file_bytes.len() as u64; + let mut offset: u64 = 0; + let mut chunk_size: u64 = 1048576; // 1MB + let total_chunks = (file_size as f64 / chunk_size as f64).ceil() as u64; + // send a file to another worker + // start by telling target to expect a file, + // then upon reciving affirmative response, + // send contents in chunks and wait for + // acknowledgement. + match bindings::send_and_await_response( + &Address::from_str(&target).unwrap(), + &Request { + inherit: false, + expects_response: Some(timeout), + ipc: Some( + serde_json::to_string(&FTWorkerCommand::Receive { + transfer_id, + file_name, + file_size, + total_chunks, + timeout, + }) + .unwrap(), + ), + metadata: None, + }, + None, + ) { + Err(send_error) => { + respond_to_parent(FTWorkerResult::Err(match send_error.kind { + SendErrorKind::Offline => TransferError::TargetOffline, + SendErrorKind::Timeout => TransferError::TargetTimeout, + })) + } + Ok((opp_worker, Message::Response((response, _)))) => { + let Ok(FTWorkerProtocol::Ready) = serde_json::from_str(&response.ipc.expect("ft_worker: got empty response")) else { + respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)); + return; + }; + // send file in chunks + loop { + if file_size < chunk_size { + // this is the last chunk, so we should expect a Finished response + chunk_size = file_size; + let payload = Payload { + mime: None, + bytes: file_bytes + [offset as usize..offset as usize + chunk_size as usize] + .to_vec(), + }; + send_request( + &opp_worker, + &Request { + inherit: false, + expects_response: Some(timeout), + ipc: None, + metadata: None, + }, + None, + Some(&payload), + ); + break; + } + let payload = Payload { + mime: None, + bytes: file_bytes + [offset as usize..offset as usize + chunk_size as usize] + .to_vec(), + }; + send_request( + &opp_worker, + &Request { + inherit: false, + expects_response: None, + ipc: None, + metadata: None, + }, + None, + Some(&payload), + ); + file_size -= chunk_size; + offset += chunk_size; + } + // now wait for Finished response + let Ok((receiving_worker, Message::Response((resp, _)))) = receive() else { + respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)); + return; + }; + let Ok(FTWorkerProtocol::Finished) = serde_json::from_str( + &resp.ipc.expect("ft_worker: got empty response"), + ) else { + respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)); + return; + }; + // return success to parent + respond_to_parent(FTWorkerResult::SendSuccess); + } + _ => respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)), + } + } + FTWorkerCommand::Receive { + transfer_id, + file_name, + file_size, + total_chunks, + timeout, + } => { + // send Ready response to counterparty + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&FTWorkerProtocol::Ready).unwrap()), + metadata: None, + }, + None, + ); + // receive a file from a worker, then send it to parent + // all messages will be chunks of file. when we receive the + // last chunk, send a Finished message to sender and Success to parent. + let mut file_bytes = Vec::new(); + let mut chunks_received = 0; + let start_time = std::time::Instant::now(); + loop { + let Ok((source, Message::Request(req))) = receive() else { + respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed)); + return; + }; + if start_time.elapsed().as_secs() > timeout { + respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed)); + return; + } + let Some(payload) = get_payload() else { + respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed)); + return; + }; + chunks_received += 1; + file_bytes.extend(payload.bytes); + if chunks_received == total_chunks { + break; + } + } + // send Finished message to sender + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&FTWorkerProtocol::Finished).unwrap()), + metadata: None, + }, + None, + ); + // send Success message to parent + send_request( + &parent_process, + &Request { + inherit: false, + expects_response: None, + ipc: Some( + serde_json::to_string(&FTWorkerResult::ReceiveSuccess(file_name)) + .unwrap(), + ), + metadata: None, + }, + None, + Some(&Payload { + mime: None, + bytes: file_bytes, + }), + ); + } + } + } +} + +fn respond_to_parent(result: FTWorkerResult) { + send_response( + &Response { + inherit: false, + ipc: Some(serde_json::to_string(&result).unwrap()), + metadata: None, + }, + None, + ); +} diff --git a/modules/app_store/ft_worker/src/process_lib.rs b/modules/app_store/ft_worker/src/process_lib.rs new file mode 120000 index 00000000..9b9ec3f4 --- /dev/null +++ b/modules/app_store/ft_worker/src/process_lib.rs @@ -0,0 +1 @@ +../../../../src/process_lib.rs \ No newline at end of file diff --git a/modules/app_store/src/kernel_types.rs b/modules/app_store/src/kernel_types.rs deleted file mode 120000 index 8311791c..00000000 --- a/modules/app_store/src/kernel_types.rs +++ /dev/null @@ -1 +0,0 @@ -../../../src/kernel_types.rs \ No newline at end of file diff --git a/modules/app_store/src/process_lib.rs b/modules/app_store/src/process_lib.rs deleted file mode 120000 index 77367fe0..00000000 --- a/modules/app_store/src/process_lib.rs +++ /dev/null @@ -1 +0,0 @@ -../../../src/process_lib.rs \ No newline at end of file diff --git a/modules/app_store/src/transfer_lib.rs b/modules/app_store/src/transfer_lib.rs deleted file mode 100644 index 2084e210..00000000 --- a/modules/app_store/src/transfer_lib.rs +++ /dev/null @@ -1,227 +0,0 @@ -use super::bindings::component::uq_process::types::*; -use crate::bindings::{get_payload, receive, send_request, send_response}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug)] -pub enum TransferError { - // in all errors, u64 is number of bytes successfully transferred - TargetOffline(u64), - TargetTimeout(u64), - TargetRejected(u64), - SourceFailed(u64), -} - -#[derive(Debug, Serialize, Deserialize)] -pub enum TransferMetadata { - Begin { - file_name: String, - file_size: u64, - total_chunks: u64, - }, -} - -pub fn transfer( - to_addr: Address, - bytes: Vec, - max_timeout: u64, -) -> ( - Result<(), TransferError>, - Vec)>>, -) { - let transfer_context_id: u64 = rand::random(); - let mut bytes_remaining: u64 = bytes.len() as u64; - let mut offset: u64 = 0; - let mut chunk_size: u64 = 1048576; // 1MB - let mut chunks_sent = 0; - let total_chunks = (bytes.len() as f64 / chunk_size as f64).ceil() as u64; - loop { - chunks_sent += 1; - if bytes_remaining < chunk_size { - chunk_size = bytes_remaining; - } - let payload = Payload { - mime: None, - bytes: bytes[offset as usize..offset as usize + chunk_size as usize].to_vec(), - }; - send_request( - &to_addr, - &Request { - inherit: false, - expects_response: Some(max_timeout), - ipc: None, - metadata: Some(if chunks_sent == 1 { - serde_json::to_string(&TransferMetadata::Begin { - file_name: "test".to_string(), - file_size: bytes.len() as u64, - total_chunks, - }) - .unwrap() - } else { - chunks_sent.to_string() - }), - }, - Some(&&transfer_context_id.to_string()), - Some(&payload), - ); - bytes_remaining -= chunk_size; - offset += chunk_size; - if bytes_remaining == 0 { - break; - } - } - let mut chunks_confirmed = 0; - let mut non_transfer_message_queue = Vec::new(); - loop { - let next = receive(); - if let Err((send_error, context)) = &next { - match context { - Some(_) => match send_error.kind { - SendErrorKind::Offline => { - return ( - Err(TransferError::TargetOffline(chunks_confirmed * chunk_size)), - non_transfer_message_queue, - ) - } - SendErrorKind::Timeout => { - return ( - Err(TransferError::TargetTimeout(chunks_confirmed * chunk_size)), - non_transfer_message_queue, - ) - } - }, - None => { - non_transfer_message_queue.push(next); - continue; - } - } - } - if let Ok((source, message)) = &next { - if source.process == to_addr.process { - match message { - Message::Request(_) => { - non_transfer_message_queue.push(next); - continue; - } - Message::Response((response, context)) => { - if transfer_context_id - == context - .as_ref() - .unwrap_or(&"".into()) - .parse::() - .unwrap_or(0) - { - chunks_confirmed += 1; - if response - .metadata - .as_ref() - .unwrap_or(&"".into()) - .parse::() - .unwrap_or(0) - != chunks_confirmed - { - return ( - Err(TransferError::TargetRejected( - chunks_confirmed * chunk_size, - )), - non_transfer_message_queue, - ); - } - if chunks_confirmed == chunks_sent { - return (Ok(()), non_transfer_message_queue); - } - } else { - non_transfer_message_queue.push(next); - } - } - } - } else { - non_transfer_message_queue.push(next); - continue; - } - } - } -} - -pub fn receive_transfer( - transfer_source: Address, - total_chunks: u64, - max_timeout: u64, -) -> ( - Result, TransferError>, - Vec)>>, -) { - let start_time = std::time::SystemTime::now(); - // get first payload then loop and receive rest - let mut file = match get_payload() { - Some(payload) => payload.bytes, - None => { - return (Err(TransferError::SourceFailed(0)), vec![]); - } - }; - // respond to first request - send_response( - &Response { - inherit: false, - ipc: None, - metadata: Some(1.to_string()), - }, - None, - ); - if total_chunks == 1 { - return (Ok(file), vec![]); - } - let mut chunk_num = 1; - let mut non_transfer_message_queue = Vec::new(); - loop { - let next = receive(); - if start_time.elapsed().expect("time error").as_secs() > max_timeout { - return ( - Err(TransferError::TargetTimeout(file.len() as u64)), - non_transfer_message_queue, - ); - } - if let Err(_) = &next { - non_transfer_message_queue.push(next); - } else if let Ok((source, message)) = &next { - // we know all messages from source process will be for this transfer, - // since they are sent sequentially and it's a single-file queue. - if source.process == transfer_source.process { - match message { - Message::Request(_) => { - let payload = match get_payload() { - Some(payload) => payload, - None => { - return ( - Err(TransferError::SourceFailed(file.len() as u64)), - non_transfer_message_queue, - ); - } - }; - chunk_num += 1; - file.extend(payload.bytes); - send_response( - &Response { - inherit: false, - ipc: None, - metadata: Some(chunk_num.to_string()), - }, - None, - ); - if chunk_num == total_chunks { - return (Ok(file), non_transfer_message_queue); - } - } - Message::Response(_) => { - return ( - Err(TransferError::SourceFailed(file.len() as u64)), - non_transfer_message_queue, - ); - } - } - } else { - non_transfer_message_queue.push(next); - continue; - } - } - } -} diff --git a/src/kernel/mod.rs b/src/kernel/mod.rs index 2eba13ac..f099339c 100644 --- a/src/kernel/mod.rs +++ b/src/kernel/mod.rs @@ -1059,7 +1059,6 @@ async fn persist_state( process_map: &t::ProcessMap, ) -> Result<()> { let bytes = bincode::serialize(process_map)?; - send_to_loop .send(t::KernelMessage { id: rand::random(), @@ -1074,7 +1073,7 @@ async fn persist_state( rsvp: None, message: t::Message::Request(t::Request { inherit: true, - expects_response: Some(5), // TODO evaluate + expects_response: None, ipc: Some( serde_json::to_string(&t::FsAction::SetState(KERNEL_PROCESS_ID.clone())) .unwrap(), From 315bea98764c01d2712d7d5b41ec0413bcee5f28 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Fri, 20 Oct 2023 19:15:11 -0400 Subject: [PATCH 06/24] scaffold new app store api --- modules/app_store/app_store/src/lib.rs | 1003 +++++++++++++----------- src/kernel_types.rs | 27 + src/process_lib.rs | 14 +- src/types.rs | 13 +- 4 files changed, 614 insertions(+), 443 deletions(-) diff --git a/modules/app_store/app_store/src/lib.rs b/modules/app_store/app_store/src/lib.rs index 36e62409..24ad9fb2 100644 --- a/modules/app_store/app_store/src/lib.rs +++ b/modules/app_store/app_store/src/lib.rs @@ -1,5 +1,4 @@ cargo_component_bindings::generate!(); - use bindings::{ component::uq_process::types::*, get_capability, get_payload, print_to_terminal, receive, send_request, send_response, Guest, @@ -10,6 +9,7 @@ use std::collections::{HashMap, HashSet}; #[allow(dead_code)] mod kernel_types; use kernel_types as kt; +use kernel_types::{PackageManifestEntry, PackageMetadata, PackageVersion}; #[allow(dead_code)] mod process_lib; @@ -17,498 +17,619 @@ use process_lib::PackageId; #[allow(dead_code)] mod ft_worker_lib; -use ft_worker_lib::*; struct Component; -// #[derive(Serialize, Deserialize)] -// struct AppState { -// // TODO this should mirror onchain listing -// pub name: String, -// pub owner: NodeId, -// pub desc: String, -// pub website: Option, -// pub versions: Vec<(u32, String)>, // TODO -// } +/// Uqbar App Store: +/// acts as both a local package manager and a protocol to share packages across the network. +/// packages are apps; apps are packages. we use an onchain app listing contract to determine +/// what apps are available to download and what node(s) to download them from. +/// +/// once we know that list, we can request a package from a node and download it locally. +/// (we can also manually download an "untracked" package if we know its name and distributor node) +/// packages that are downloaded can then be installed! +/// +/// installed packages can be managed: +/// - given permissions (necessary to complete install) +/// - uninstalled / suspended +/// - deleted ("undownloaded") +/// - set to automatically update if a new version is available (on by default) -#[derive(Serialize, Deserialize)] -struct AppTrackerState { - pub mirrored_packages: HashSet, - pub requested_packages: HashMap, // who we're expecting it from +// +// app store types +// + +/// this process's saved state +#[derive(Debug, Serialize, Deserialize)] +struct State { + pub packages: HashMap, +} + +/// state of an individual package we have downloaded +#[derive(Debug, Serialize, Deserialize)] +struct PackageState { + pub mirrored_from: NodeId, + pub listing_data: Option, // None if package is unlisted + pub installed_version: Option, // None if downloaded but not installed + pub mirroring: bool, // are we serving this package to others? + pub auto_update: bool, // if we get a listing data update, will we try to download it? +} + +/// just a sketch of what we might get from chain +#[derive(Debug, Serialize, Deserialize)] +struct PackageListing { + pub name: String, + pub publisher: NodeId, + pub description: Option, + pub website: Option, + pub version: PackageVersion, + pub version_hash: String, // sha256 hash of the package zip or whatever +} + +// +// app store API +// + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] // untagged as a meta-type for all requests +pub enum Req { + LocalRequest(LocalRequest), + RemoteRequest(RemoteRequest), + FTWorkerCommand(ft_worker_lib::FTWorkerCommand), } #[derive(Debug, Serialize, Deserialize)] -pub enum AppTrackerRequest { - New { - package: PackageId, - mirror: bool, +#[serde(untagged)] // untagged as a meta-type for all responses +pub enum Resp { + // note that we do not need to ourselves handle local responses, as + // those are given to others rather than received. + RemoteResponse(RemoteResponse), + FTWorkerResult(ft_worker_lib::FTWorkerResult), +} + +/// Local requests take this form. +#[derive(Debug, Serialize, Deserialize)] +pub enum LocalRequest { + /// expects a zipped package as payload: create a new package from it + /// if requested, will return a NewPackageResponse indicating success/failure + NewPackage { + mirror: bool, // sets whether we will mirror this package }, - NewFromRemote { - package_id: PackageId, + /// no payload; try to download a package from a specified node + /// if requested, will return a DownloadResponse indicating success/failure + Download { + package: PackageId, install_from: NodeId, }, - Install { - package: PackageId, - }, + /// no payload; select a downloaded package and install it + /// if requested, will return an InstallResponse indicating success/failure + Install(PackageId), + /// no payload; select an installed package and uninstall it + /// no response will be given + Uninstall(PackageId), + /// no payload; select a downloaded package and delete it + /// no response will be given + Delete(PackageId), +} + +/// Remote requests, those sent between instantiations of this process +/// on different nodes, take this form. +#[derive(Debug, Serialize, Deserialize)] +pub enum RemoteRequest { + /// no payload; request a package from a node + /// remote node must return RemoteResponse::DownloadApproved, + /// at which point requester can expect a FTWorkerRequest::Receive + Download(PackageId), } #[derive(Debug, Serialize, Deserialize)] -pub enum AppTrackerResponse { - New { package: String }, - NewFromRemote { package_id: PackageId }, - Install { package: String }, - Error { error: String }, +pub enum RemoteResponse { + DownloadApproved, + DownloadDenied, // TODO expand on why +} + +// TODO for all: expand these to elucidate why something failed +// these are locally-given responses to local requests + +#[derive(Debug, Serialize, Deserialize)] +pub enum NewPackageResponse { + Success, + Failure, } #[derive(Debug, Serialize, Deserialize)] -pub struct PackageMetadata { - pub package: String, - pub publisher: String, - pub desc: Option, +pub enum DownloadResponse { + Success, + Failure, } #[derive(Debug, Serialize, Deserialize)] -pub struct PackageManifestEntry { - pub process_name: String, - pub process_wasm_path: String, - pub on_panic: kt::OnPanic, - pub request_networking: bool, - pub request_messaging: Vec, - pub public: bool, +pub enum InstallResponse { + Success, + Failure, } -fn parse_command( - our: &Address, - source: &Address, - request_string: String, - state: &mut AppTrackerState, -) -> anyhow::Result> { - match serde_json::from_str(&request_string)? { - // create a new package based on local payload - AppTrackerRequest::New { package, mirror } => { - if our.node != source.node { - return Err(anyhow::anyhow!("new package request from non-local node")); - } - let Some(mut payload) = get_payload() else { - return Err(anyhow::anyhow!("no payload")); - }; - - let vfs_address = Address { - node: our.node.clone(), - process: ProcessId::from_str("vfs:sys:uqbar")?, - }; - - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::New, - })?), - None, - None, - 5, - )?; - - // add zip bytes - payload.mime = Some("application/zip".to_string()); - let _ = process_lib::send_and_await_response( - &vfs_address, - true, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::Add { - full_path: package.to_string(), - entry_type: kt::AddEntryType::ZipArchive, - }, - })?), - None, - Some(&payload), - 5, - )?; - - // save the zip file itself in VFS for sharing with other nodes - // call it .zip - let _ = process_lib::send_and_await_response( - &vfs_address, - true, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::Add { - full_path: format!("/{}.zip", package.to_string()), - entry_type: kt::AddEntryType::NewFile, - }, - })?), - None, - Some(&payload), - 5, - )?; - - // if mirror, save in our state - if mirror { - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetEntry("/metadata.json".into()), - })?), - None, - None, - 5, - )?; - let Some(payload) = get_payload() else { - return Err(anyhow::anyhow!("no metadata payload")); - }; - let metadata = String::from_utf8(payload.bytes)?; - let metadata = serde_json::from_str::(&metadata)?; - state - .mirrored_packages - .insert(PackageId::new(&metadata.package, &metadata.publisher)); - process_lib::set_state::(&state); - } - - Ok(Some(AppTrackerResponse::New { package: package.to_string() })) - } - // if we are the source, forward to install_from target. - // if we install_from, respond with package if we have it - AppTrackerRequest::NewFromRemote { - package_id, - install_from, - } => { - if our.node == source.node { - let _ = send_request( - &Address { - node: install_from.clone(), - process: our.process.clone(), - }, - &Request { - inherit: true, - expects_response: Some(5), // TODO - ipc: Some(serde_json::to_string(&AppTrackerRequest::NewFromRemote { - package_id: package_id.clone(), - install_from: install_from.clone(), - })?), - metadata: None, - }, - None, - None, - ); - state.requested_packages.insert(package_id, install_from); - process_lib::set_state::(&state); - Ok(None) - } else if our.node == install_from { - let Some(_mirror) = state.mirrored_packages.get(&package_id) else { - return Ok(Some(AppTrackerResponse::Error { error: "package not mirrored here!".into() })) - }; - // get the .zip from VFS and attach as payload to response - let vfs_address = Address { - node: our.node.clone(), - process: ProcessId::from_str("vfs:sys:uqbar")?, - }; - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package_id.to_string(), - action: kt::VfsAction::GetEntry(format!("/{}.zip", package_id.to_string())), - })?), - None, - None, - 5, - )?; - Ok(Some(AppTrackerResponse::NewFromRemote { package_id })) - } else { - // TODO what to do here? - Ok(None) - } - } - AppTrackerRequest::Install { package } => { - if our.node != source.node { - return Err(anyhow::anyhow!("install request from non-local node")); - } - let vfs_address = Address { - node: our.node.clone(), - process: ProcessId::from_str("vfs:sys:uqbar")?, - }; - - let _ = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetEntry("/manifest.json".into()), - })?), - None, - None, - 5, - )?; - let Some(payload) = get_payload() else { - return Err(anyhow::anyhow!("no payload")); - }; - let manifest = String::from_utf8(payload.bytes)?; - let manifest = serde_json::from_str::>(&manifest)?; - - for entry in manifest { - let path = if entry.process_wasm_path.starts_with("/") { - entry.process_wasm_path - } else { - format!("/{}", entry.process_wasm_path) - }; - - let (_, hash_response) = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetHash(path.clone()), - })?), - None, - None, - 5, - )?; - - let Message::Response((Response { ipc: Some(ipc), .. }, _)) = hash_response else { - return Err(anyhow::anyhow!("bad vfs response")); - }; - let kt::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc)? else { - return Err(anyhow::anyhow!("no hash in vfs")); - }; - - // build initial caps - let mut initial_capabilities: HashSet = HashSet::new(); - if entry.request_networking { - let Some(networking_cap) = get_capability( - &Address { - node: our.node.clone(), - process: ProcessId::from_str("kernel:sys:uqbar")?, - }, - &"\"network\"".to_string(), - ) else { - return Err(anyhow::anyhow!("app-store: no net cap")); - }; - initial_capabilities.insert(kt::de_wit_signed_capability(networking_cap)); - } - let Some(read_cap) = get_capability( - &vfs_address.clone(), - &serde_json::to_string(&serde_json::json!({ - "kind": "read", - "drive": package.to_string(), - }))?, - ) else { - return Err(anyhow::anyhow!("app-store: no read cap")); - }; - initial_capabilities.insert(kt::de_wit_signed_capability(read_cap)); - let Some(write_cap) = get_capability( - &vfs_address.clone(), - &serde_json::to_string(&serde_json::json!({ - "kind": "write", - "drive": package.to_string(), - }))?, - ) else { - return Err(anyhow::anyhow!("app-store: no write cap")); - }; - initial_capabilities.insert(kt::de_wit_signed_capability(write_cap)); - - for process_name in &entry.request_messaging { - let Ok(parsed_process_id) = ProcessId::from_str(&process_name) else { - // TODO handle arbitrary caps here - continue; - }; - let Some(messaging_cap) = get_capability( - &Address { - node: our.node.clone(), - process: parsed_process_id.clone(), - }, - &"\"messaging\"".into() - ) else { - print_to_terminal(0, &format!("app-store: no cap for {} to give away!", process_name)); - continue; - }; - initial_capabilities.insert(kt::de_wit_signed_capability(messaging_cap)); - } - - let process_id = format!("{}:{}", entry.process_name, package.to_string()); - let Ok(parsed_new_process_id) = ProcessId::from_str(&process_id) else { - return Err(anyhow::anyhow!("app-store: invalid process id!")); - }; - let _ = process_lib::send_request( - &Address { - node: our.node.clone(), - process: ProcessId::from_str("kernel:sys:uqbar")?, - }, - false, - Some(serde_json::to_string(&kt::KernelCommand::KillProcess( - kt::ProcessId::de_wit(parsed_new_process_id.clone()), - ))?), - None, - None, - None, - ); - - // kernel start process takes bytes as payload + wasm_bytes_handle... - // reconsider perhaps - let (_, _bytes_response) = process_lib::send_and_await_response( - &vfs_address, - false, - Some(serde_json::to_string(&kt::VfsRequest { - drive: package.to_string(), - action: kt::VfsAction::GetEntry(path), - })?), - None, - None, - 5, - )?; - - let Some(payload) = get_payload() else { - return Err(anyhow::anyhow!("no wasm bytes payload.")); - }; - - let _ = process_lib::send_and_await_response( - &Address { - node: our.node.clone(), - process: ProcessId::from_str("kernel:sys:uqbar")?, - }, - false, - Some(serde_json::to_string(&kt::KernelCommand::StartProcess { - id: kt::ProcessId::de_wit(parsed_new_process_id), - wasm_bytes_handle: hash, - on_panic: entry.on_panic, - initial_capabilities, - public: entry.public, - })?), - None, - Some(&payload), - 5, - )?; - } - Ok(Some(AppTrackerResponse::Install { package: package.to_string() })) - } - } -} +// +// app store init() +// impl Guest for Component { fn init(our: Address) { - assert_eq!(our.process.to_string(), "main:app_store:uqbar"); + assert_eq!(our.process, "main:app_store:uqbar"); - // grant messaging caps to http_server and terminal - let Some(our_messaging_cap) = bindings::get_capability( + // begin by granting messaging capabilities to http_server and terminal, + // so that they can send us requests. + process_lib::grant_messaging( &our, - &"\"messaging\"".into() - ) else { - panic!("missing self-messaging cap!") - }; - bindings::share_capability( - &ProcessId::from_str("http_server:sys:uqbar").unwrap(), - &our_messaging_cap, + &Vec::from([ + ProcessId::from_str("http_server:sys:uqbar").unwrap(), + ProcessId::from_str("terminal:terminal:uqbar").unwrap(), + ]), ); - bindings::share_capability( - &ProcessId::from_str("terminal:terminal:uqbar").unwrap(), - &our_messaging_cap, - ); - print_to_terminal(0, &format!("app_store main proc: start")); - let mut state = process_lib::get_state::().unwrap_or(AppTrackerState { - mirrored_packages: HashSet::new(), - requested_packages: HashMap::new(), + // load in our saved state or initalize a new one if none exists + let mut state = process_lib::get_state::().unwrap_or(State { + packages: HashMap::new(), }); + // active the main messaging loop: handle requests and responses loop { let (source, message) = match receive() { Ok((source, message)) => (source, message), Err((error, _context)) => { + // TODO handle net errors more usefully based on their context print_to_terminal(0, &format!("net error: {:?}", error.kind)); continue; } }; match message { - Message::Request(Request { - ipc, - expects_response, - metadata, - .. - }) => { - let Some(command) = ipc else { + Message::Request(req) => { + let Some(ref ipc) = req.ipc else { continue; }; - match parse_command(&our, &source, command, &mut state) { - Ok(response) => { - if let Some(_) = expects_response { - let _ = send_response( - &Response { - inherit: true, - ipc: Some(serde_json::to_string(&response).unwrap()), - metadata, - }, - None, // payload will be attached here if created in parse_command - ); - }; - } - Err(e) => { - print_to_terminal(0, &format!("app-store: got error {}", e)); - if let Some(_) = expects_response { - let error = AppTrackerResponse::Error { - error: format!("{}", e), - }; - let _ = send_response( - &Response { - inherit: false, - ipc: Some(serde_json::to_string(&error).unwrap()), - metadata, - }, - None, - ); - }; - } - } - } - Message::Response((response, _)) => { - print_to_terminal(0, &format!("app-store: got response {:?}", response)); - // only expecting NewFromRemote for apps we've requested - match serde_json::from_str(&response.ipc.unwrap_or_default()) { - Ok(AppTrackerResponse::NewFromRemote { package_id }) => { - if let Some(install_from) = state.requested_packages.remove(&package_id) - { - if install_from == source.node { - // auto-take zip from payload and request ourself with New - let _ = send_request( - &our, - &Request { - inherit: true, // will inherit payload! - expects_response: None, - ipc: Some( - serde_json::to_string(&AppTrackerRequest::New { - package: package_id, - mirror: true, - }) - .unwrap(), - ), - metadata: None, - }, - None, - None, - ); - } else { + match &serde_json::from_str::(ipc) { + Ok(Req::LocalRequest(local_request)) => { + match handle_local_request(local_request) { + Ok(None) => continue, + Ok(Some(resp)) => { + if req.expects_response.is_some() { + send_response( + &Response { + inherit: false, + ipc: Some( + serde_json::to_string(&resp).unwrap(), + ), + metadata: None, + }, + None, + ); + } + } + Err(err) => { print_to_terminal( 0, - &format!( - "app-store: got install response from bad source: {}", - install_from - ), + &format!("app-store: local request error: {:?}", err), ); } } } - err => { - print_to_terminal( - 0, - &format!("app-store: got unexpected response {:?}", err), - ); + Ok(Req::RemoteRequest(remote_request)) => { + match handle_remote_request(remote_request) { + Ok(None) => continue, + Ok(Some(resp)) => { + if req.expects_response.is_some() { + send_response( + &Response { + inherit: false, + ipc: Some( + serde_json::to_string(&resp).unwrap(), + ), + metadata: None, + }, + None, + ); + } + } + Err(err) => { + print_to_terminal( + 0, + &format!("app-store: local request error: {:?}", err), + ); + } + } + } + Ok(Req::FTWorkerCommand(ft_worker_command)) => { + // TODO handle ft_worker commands + } + Err(_) => { + continue; } } } + Message::Response((response, context)) => { + let Some(ref ipc) = response.ipc else { + continue; + }; + match &serde_json::from_str::(ipc) { + Ok(Resp::RemoteResponse(remote_response)) => { + // TODO handle remote response + } + Ok(Resp::FTWorkerResult(ft_worker_result)) => { + // TODO handle ft_worker result + } + Err(_) => { + continue; + } + } + // // only expecting NewFromRemote for apps we've requested + // match serde_json::from_str(&response.ipc.unwrap_or_default()) { + // Ok(AppTrackerResponse::NewFromRemote { package_id }) => { + // if let Some(install_from) = state.requested_packages.remove(&package_id) + // { + // if install_from == source.node { + // // auto-take zip from payload and request ourself with New + // let _ = send_request( + // &our, + // &Request { + // inherit: true, // will inherit payload! + // expects_response: None, + // ipc: Some( + // serde_json::to_string(&AppTrackerRequest::New { + // package: package_id, + // mirror: true, + // }) + // .unwrap(), + // ), + // metadata: None, + // }, + // None, + // None, + // ); + // } else { + // print_to_terminal( + // 0, + // &format!( + // "app-store: got install response from bad source: {}", + // install_from + // ), + // ); + // } + // } + // } + // err => { + // print_to_terminal( + // 0, + // &format!("app-store: got unexpected response {:?}", err), + // ); + // } + // } + } } } } } + +fn handle_local_request(request: &LocalRequest) -> anyhow::Result> { + // TODO + Ok(None) +} + +fn handle_remote_request(request: &RemoteRequest) -> anyhow::Result> { + // TODO + Ok(None) +} + +// fn parse_command( +// our: &Address, +// source: &Address, +// request_string: String, +// state: &mut AppTrackerState, +// ) -> anyhow::Result> { +// match serde_json::from_str(&request_string)? { +// // create a new package based on local payload +// AppTrackerRequest::New { package, mirror } => { +// if our.node != source.node { +// return Err(anyhow::anyhow!("new package request from non-local node")); +// } +// let Some(mut payload) = get_payload() else { +// return Err(anyhow::anyhow!("no payload")); +// }; + +// let vfs_address = Address { +// node: our.node.clone(), +// process: ProcessId::from_str("vfs:sys:uqbar")?, +// }; + +// let _ = process_lib::send_and_await_response( +// &vfs_address, +// false, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::New, +// })?), +// None, +// None, +// 5, +// )?; + +// // add zip bytes +// payload.mime = Some("application/zip".to_string()); +// let _ = process_lib::send_and_await_response( +// &vfs_address, +// true, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::Add { +// full_path: package.to_string(), +// entry_type: kt::AddEntryType::ZipArchive, +// }, +// })?), +// None, +// Some(&payload), +// 5, +// )?; + +// // save the zip file itself in VFS for sharing with other nodes +// // call it .zip +// let _ = process_lib::send_and_await_response( +// &vfs_address, +// true, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::Add { +// full_path: format!("/{}.zip", package.to_string()), +// entry_type: kt::AddEntryType::NewFile, +// }, +// })?), +// None, +// Some(&payload), +// 5, +// )?; + +// // if mirror, save in our state +// if mirror { +// let _ = process_lib::send_and_await_response( +// &vfs_address, +// false, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::GetEntry("/metadata.json".into()), +// })?), +// None, +// None, +// 5, +// )?; +// let Some(payload) = get_payload() else { +// return Err(anyhow::anyhow!("no metadata payload")); +// }; +// let metadata = String::from_utf8(payload.bytes)?; +// let metadata = serde_json::from_str::(&metadata)?; +// state +// .mirrored_packages +// .insert(PackageId::new(&metadata.package, &metadata.publisher)); +// process_lib::set_state::(&state); +// } + +// Ok(Some(AppTrackerResponse::New { +// package: package.to_string(), +// })) +// } +// // if we are the source, forward to install_from target. +// // if we install_from, respond with package if we have it +// AppTrackerRequest::NewFromRemote { +// package_id, +// install_from, +// } => { +// if our.node == source.node { +// let _ = send_request( +// &Address { +// node: install_from.clone(), +// process: our.process.clone(), +// }, +// &Request { +// inherit: true, +// expects_response: Some(5), // TODO +// ipc: Some(serde_json::to_string(&AppTrackerRequest::NewFromRemote { +// package_id: package_id.clone(), +// install_from: install_from.clone(), +// })?), +// metadata: None, +// }, +// None, +// None, +// ); +// state.requested_packages.insert(package_id, install_from); +// process_lib::set_state::(&state); +// Ok(None) +// } else if our.node == install_from { +// let Some(_mirror) = state.mirrored_packages.get(&package_id) else { +// return Ok(Some(AppTrackerResponse::Error { error: "package not mirrored here!".into() })) +// }; +// // get the .zip from VFS and attach as payload to response +// let vfs_address = Address { +// node: our.node.clone(), +// process: ProcessId::from_str("vfs:sys:uqbar")?, +// }; +// let _ = process_lib::send_and_await_response( +// &vfs_address, +// false, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package_id.to_string(), +// action: kt::VfsAction::GetEntry(format!("/{}.zip", package_id.to_string())), +// })?), +// None, +// None, +// 5, +// )?; +// Ok(Some(AppTrackerResponse::NewFromRemote { package_id })) +// } else { +// // TODO what to do here? +// Ok(None) +// } +// } +// AppTrackerRequest::Install { package } => { +// if our.node != source.node { +// return Err(anyhow::anyhow!("install request from non-local node")); +// } +// let vfs_address = Address { +// node: our.node.clone(), +// process: ProcessId::from_str("vfs:sys:uqbar")?, +// }; + +// let _ = process_lib::send_and_await_response( +// &vfs_address, +// false, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::GetEntry("/manifest.json".into()), +// })?), +// None, +// None, +// 5, +// )?; +// let Some(payload) = get_payload() else { +// return Err(anyhow::anyhow!("no payload")); +// }; +// let manifest = String::from_utf8(payload.bytes)?; +// let manifest = serde_json::from_str::>(&manifest)?; + +// for entry in manifest { +// let path = if entry.process_wasm_path.starts_with("/") { +// entry.process_wasm_path +// } else { +// format!("/{}", entry.process_wasm_path) +// }; + +// let (_, hash_response) = process_lib::send_and_await_response( +// &vfs_address, +// false, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::GetHash(path.clone()), +// })?), +// None, +// None, +// 5, +// )?; + +// let Message::Response((Response { ipc: Some(ipc), .. }, _)) = hash_response else { +// return Err(anyhow::anyhow!("bad vfs response")); +// }; +// let kt::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc)? else { +// return Err(anyhow::anyhow!("no hash in vfs")); +// }; + +// // build initial caps +// let mut initial_capabilities: HashSet = HashSet::new(); +// if entry.request_networking { +// let Some(networking_cap) = get_capability( +// &Address { +// node: our.node.clone(), +// process: ProcessId::from_str("kernel:sys:uqbar")?, +// }, +// &"\"network\"".to_string(), +// ) else { +// return Err(anyhow::anyhow!("app-store: no net cap")); +// }; +// initial_capabilities.insert(kt::de_wit_signed_capability(networking_cap)); +// } +// let Some(read_cap) = get_capability( +// &vfs_address.clone(), +// &serde_json::to_string(&serde_json::json!({ +// "kind": "read", +// "drive": package.to_string(), +// }))?, +// ) else { +// return Err(anyhow::anyhow!("app-store: no read cap")); +// }; +// initial_capabilities.insert(kt::de_wit_signed_capability(read_cap)); +// let Some(write_cap) = get_capability( +// &vfs_address.clone(), +// &serde_json::to_string(&serde_json::json!({ +// "kind": "write", +// "drive": package.to_string(), +// }))?, +// ) else { +// return Err(anyhow::anyhow!("app-store: no write cap")); +// }; +// initial_capabilities.insert(kt::de_wit_signed_capability(write_cap)); + +// for process_name in &entry.request_messaging { +// let Ok(parsed_process_id) = ProcessId::from_str(&process_name) else { +// // TODO handle arbitrary caps here +// continue; +// }; +// let Some(messaging_cap) = get_capability( +// &Address { +// node: our.node.clone(), +// process: parsed_process_id.clone(), +// }, +// &"\"messaging\"".into() +// ) else { +// print_to_terminal(0, &format!("app-store: no cap for {} to give away!", process_name)); +// continue; +// }; +// initial_capabilities.insert(kt::de_wit_signed_capability(messaging_cap)); +// } + +// let process_id = format!("{}:{}", entry.process_name, package.to_string()); +// let Ok(parsed_new_process_id) = ProcessId::from_str(&process_id) else { +// return Err(anyhow::anyhow!("app-store: invalid process id!")); +// }; +// let _ = process_lib::send_request( +// &Address { +// node: our.node.clone(), +// process: ProcessId::from_str("kernel:sys:uqbar")?, +// }, +// false, +// Some(serde_json::to_string(&kt::KernelCommand::KillProcess( +// kt::ProcessId::de_wit(parsed_new_process_id.clone()), +// ))?), +// None, +// None, +// None, +// ); + +// // kernel start process takes bytes as payload + wasm_bytes_handle... +// // reconsider perhaps +// let (_, _bytes_response) = process_lib::send_and_await_response( +// &vfs_address, +// false, +// Some(serde_json::to_string(&kt::VfsRequest { +// drive: package.to_string(), +// action: kt::VfsAction::GetEntry(path), +// })?), +// None, +// None, +// 5, +// )?; + +// let Some(payload) = get_payload() else { +// return Err(anyhow::anyhow!("no wasm bytes payload.")); +// }; + +// let _ = process_lib::send_and_await_response( +// &Address { +// node: our.node.clone(), +// process: ProcessId::from_str("kernel:sys:uqbar")?, +// }, +// false, +// Some(serde_json::to_string(&kt::KernelCommand::StartProcess { +// id: kt::ProcessId::de_wit(parsed_new_process_id), +// wasm_bytes_handle: hash, +// on_panic: entry.on_panic, +// initial_capabilities, +// public: entry.public, +// })?), +// None, +// Some(&payload), +// 5, +// )?; +// } +// Ok(Some(AppTrackerResponse::Install { +// package: package.to_string(), +// })) +// } +// } +// } diff --git a/src/kernel_types.rs b/src/kernel_types.rs index 4e483a08..1e4af80e 100644 --- a/src/kernel_types.rs +++ b/src/kernel_types.rs @@ -311,6 +311,33 @@ impl VfsError { } } +// +// package types +// + +pub type PackageVersion = (u32, u32, u32); + +/// the type that gets deserialized from `metadata.json` in a package +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageMetadata { + pub package: String, + pub publisher: String, + pub version: PackageVersion, + pub description: Option, + pub website: Option, +} + +/// the type that gets deserialized from each entry in the array in `manifest.json` +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageManifestEntry { + pub process_name: String, + pub process_wasm_path: String, + pub on_panic: OnPanic, + pub request_networking: bool, + pub request_messaging: Vec, + pub public: bool, +} + // // conversions between wit types and kernel types (annoying!) // diff --git a/src/process_lib.rs b/src/process_lib.rs index bf8c61ca..c72472f1 100644 --- a/src/process_lib.rs +++ b/src/process_lib.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use super::bindings::component::uq_process::types::*; -use super::bindings::{Address, Payload, ProcessId, SendError}; +use super::bindings::{get_capability, share_capability, Address, Payload, ProcessId, SendError}; #[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct PackageId { @@ -276,6 +276,18 @@ where Ok(parsed) } +pub fn grant_messaging(our: &Address, grant_to: &Vec) { + let Some(our_messaging_cap) = get_capability( + our, + &"\"messaging\"".into() + ) else { + panic!("missing self-messaging cap!") + }; + for process in grant_to { + share_capability(&process, &our_messaging_cap); + } +} + // move these to better place! #[derive(Serialize, Deserialize, Debug)] pub enum FsAction { diff --git a/src/types.rs b/src/types.rs index a90fcf48..56d5c866 100644 --- a/src/types.rs +++ b/src/types.rs @@ -403,11 +403,22 @@ pub struct ProcessContext { // filesystem.rs types // +/// the type that gets deserialized from `metadata.json` in a package +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageMetadata { + pub package: String, + pub publisher: String, + pub version: Version, + pub description: Option, + pub website: Option, +} + +/// the type that gets deserialized from each entry in the array in `manifest.json` #[derive(Debug, Serialize, Deserialize)] pub struct PackageManifestEntry { pub process_name: String, pub process_wasm_path: String, - pub on_panic: OnPanic, + pub on_panic: kt::OnPanic, pub request_networking: bool, pub request_messaging: Vec, pub public: bool, From ab14386893ad83326a6372b6f14a92020bd6c2ba Mon Sep 17 00:00:00 2001 From: bitful-pannul Date: Thu, 19 Oct 2023 01:51:00 +0200 Subject: [PATCH 07/24] init move to http_server --- Cargo.toml | 4 ++-- src/http_server/mod.rs | 2 ++ src/http_server/server_fns.rs | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5f6ffcdc..dedbc8b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ rand = "0.8.4" reqwest = "0.11.18" ring = "0.16.20" rsa = "0.9" +route-recognizer = "0.3.1" rusoto_core = "0.48.0" rusoto_s3 = "0.48.0" rusoto_credential = "0.48.0" @@ -64,5 +65,4 @@ warp = "0.3.5" wasmtime = "12.0.1" wasmtime-wasi = "12.0.1" zip = "0.6" -base64 = "0.13" -route-recognizer = "0.3.1" +base64 = "0.13" \ No newline at end of file diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index c220e32f..81626866 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -7,6 +7,7 @@ use base64; use futures::SinkExt; use futures::StreamExt; use serde_urlencoded; +use base64; use route_recognizer::Router; use std::collections::HashMap; @@ -905,6 +906,7 @@ async fn handler( .await .insert(id, (original_path.clone(), response_sender)); + let message = km.unwrap(); // DOUBLECHECK send_to_loop.send(message).await.unwrap(); let timeout_duration = tokio::time::Duration::from_secs(15); // adjust as needed let result = tokio::time::timeout(timeout_duration, response_receiver).await; diff --git a/src/http_server/server_fns.rs b/src/http_server/server_fns.rs index 6183bff8..6008764d 100644 --- a/src/http_server/server_fns.rs +++ b/src/http_server/server_fns.rs @@ -10,6 +10,7 @@ use tokio::net::TcpListener; use tokio::sync::Mutex; use warp::http::{header::HeaderName, header::HeaderValue, HeaderMap}; use warp::ws::WebSocket; +use serde::{Deserialize, Serialize}; pub type SharedWriteStream = Arc>>; pub type WebSockets = Arc>>>>; From 1abf599c21750e7f96b203f3d423b21ba4bd29e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 00:29:39 +0000 Subject: [PATCH 08/24] Format Rust code using rustfmt --- src/http_server/mod.rs | 1 - src/http_server/server_fns.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 81626866..3167c247 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -7,7 +7,6 @@ use base64; use futures::SinkExt; use futures::StreamExt; use serde_urlencoded; -use base64; use route_recognizer::Router; use std::collections::HashMap; diff --git a/src/http_server/server_fns.rs b/src/http_server/server_fns.rs index 6008764d..6183bff8 100644 --- a/src/http_server/server_fns.rs +++ b/src/http_server/server_fns.rs @@ -10,7 +10,6 @@ use tokio::net::TcpListener; use tokio::sync::Mutex; use warp::http::{header::HeaderName, header::HeaderValue, HeaderMap}; use warp::ws::WebSocket; -use serde::{Deserialize, Serialize}; pub type SharedWriteStream = Arc>>; pub type WebSockets = Arc>>>>; From 086297f2241b14857d1661e2c1c9961431b162aa Mon Sep 17 00:00:00 2001 From: bitful-pannul Date: Thu, 19 Oct 2023 22:59:17 +0200 Subject: [PATCH 09/24] refactor params matching into url --- Cargo.toml | 2 +- src/http_server/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dedbc8b3..f3b5d2cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,4 +65,4 @@ warp = "0.3.5" wasmtime = "12.0.1" wasmtime-wasi = "12.0.1" zip = "0.6" -base64 = "0.13" \ No newline at end of file +base64 = "0.13" diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 3167c247..309f79ea 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -239,7 +239,7 @@ async fn http_handle_messages( let mut senders = http_response_senders.lock().await; match senders.remove(&id) { // if no corresponding entry, nowhere to send response - None => {} + None => { } Some((path, channel)) => { // if path is /rpc/message, return accordingly with base64 encoded payload if path == "/rpc:sys:uqbar/message".to_string() { From 4c62d01ac975be2d3c3f671828560bb77966fae8 Mon Sep 17 00:00:00 2001 From: bitful-pannul Date: Fri, 20 Oct 2023 00:10:58 +0200 Subject: [PATCH 10/24] paths now need to contain processId before the actual path --- src/http_server/server_fns.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http_server/server_fns.rs b/src/http_server/server_fns.rs index 6183bff8..9a2f5643 100644 --- a/src/http_server/server_fns.rs +++ b/src/http_server/server_fns.rs @@ -3,6 +3,7 @@ use futures::stream::SplitSink; use hmac::{Hmac, Mac}; use jwt::{Error, VerifyWithKey}; use serde::{Deserialize, Serialize}; +use regex::Regex; use sha2::Sha256; use std::collections::{HashMap, HashSet}; use std::sync::Arc; From 72194fca1f7efa23295281340274e8a2bee178cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 22:12:00 +0000 Subject: [PATCH 11/24] Format Rust code using rustfmt --- src/http_server/mod.rs | 2 +- src/http_server/server_fns.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 309f79ea..3167c247 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -239,7 +239,7 @@ async fn http_handle_messages( let mut senders = http_response_senders.lock().await; match senders.remove(&id) { // if no corresponding entry, nowhere to send response - None => { } + None => {} Some((path, channel)) => { // if path is /rpc/message, return accordingly with base64 encoded payload if path == "/rpc:sys:uqbar/message".to_string() { diff --git a/src/http_server/server_fns.rs b/src/http_server/server_fns.rs index 9a2f5643..1ec9e35b 100644 --- a/src/http_server/server_fns.rs +++ b/src/http_server/server_fns.rs @@ -2,8 +2,8 @@ use crate::types::*; use futures::stream::SplitSink; use hmac::{Hmac, Mac}; use jwt::{Error, VerifyWithKey}; -use serde::{Deserialize, Serialize}; use regex::Regex; +use serde::{Deserialize, Serialize}; use sha2::Sha256; use std::collections::{HashMap, HashSet}; use std::sync::Arc; From 0ac988802546139a55535f0ff6cd2e8783fd2eef Mon Sep 17 00:00:00 2001 From: bitful-pannul Date: Fri, 20 Oct 2023 16:12:11 +0200 Subject: [PATCH 12/24] add /encryptor route, remove regex --- src/http_server/server_fns.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/http_server/server_fns.rs b/src/http_server/server_fns.rs index 1ec9e35b..6183bff8 100644 --- a/src/http_server/server_fns.rs +++ b/src/http_server/server_fns.rs @@ -2,7 +2,6 @@ use crate::types::*; use futures::stream::SplitSink; use hmac::{Hmac, Mac}; use jwt::{Error, VerifyWithKey}; -use regex::Regex; use serde::{Deserialize, Serialize}; use sha2::Sha256; use std::collections::{HashMap, HashSet}; From bf1b3f0581bcdf30d3429a1162424d112c7cf6bb Mon Sep 17 00:00:00 2001 From: bitful-pannul Date: Fri, 20 Oct 2023 19:26:19 +0200 Subject: [PATCH 13/24] standardize names, path vs raw_path distinction --- src/http_server/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 3167c247..c220e32f 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -905,7 +905,6 @@ async fn handler( .await .insert(id, (original_path.clone(), response_sender)); - let message = km.unwrap(); // DOUBLECHECK send_to_loop.send(message).await.unwrap(); let timeout_duration = tokio::time::Duration::from_secs(15); // adjust as needed let result = tokio::time::timeout(timeout_duration, response_receiver).await; From 78468348c5a4f6858e9950ce26b2ea1e1c74ae6b Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Mon, 23 Oct 2023 21:05:16 -0400 Subject: [PATCH 14/24] just a bit of cleanup --- src/http_server/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index c220e32f..e5b2d612 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -899,6 +899,7 @@ async fn handler( signed_capabilities: None, } }; + let (response_sender, response_receiver) = oneshot::channel(); http_response_senders .lock() From 767a7b04f9bec0df7d2d47f61fa03d441e7d84c9 Mon Sep 17 00:00:00 2001 From: bitful-pannul <109035169+bitful-pannul@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:27:18 +0200 Subject: [PATCH 15/24] rpc: hotfix (#35) --- src/http_server/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index e5b2d612..c220e32f 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -899,7 +899,6 @@ async fn handler( signed_capabilities: None, } }; - let (response_sender, response_receiver) = oneshot::channel(); http_response_senders .lock() From 2735c0f04e14332a1f29bf3e3f0880085a439a1d Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 15:18:38 -0400 Subject: [PATCH 16/24] add in transfer logic --- modules/app_store/app_store/Cargo.lock | 66 ++ modules/app_store/app_store/Cargo.toml | 1 + modules/app_store/app_store/src/lib.rs | 862 ++++++++++-------- .../app_store/ft_worker/src/ft_worker_lib.rs | 24 +- modules/app_store/pkg/manifest.json | 1 - src/types.rs | 6 +- 6 files changed, 561 insertions(+), 399 deletions(-) diff --git a/modules/app_store/app_store/Cargo.lock b/modules/app_store/app_store/Cargo.lock index 702995a4..c601a168 100644 --- a/modules/app_store/app_store/Cargo.lock +++ b/modules/app_store/app_store/Cargo.lock @@ -18,6 +18,7 @@ dependencies = [ "rand", "serde", "serde_json", + "sha2", "wit-bindgen", ] @@ -42,6 +43,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "cargo-component-bindings" version = "0.1.0" @@ -72,6 +82,35 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cpufeatures" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -87,6 +126,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -284,6 +333,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "smallvec" version = "1.11.0" @@ -325,6 +385,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicase" version = "2.7.0" diff --git a/modules/app_store/app_store/Cargo.toml b/modules/app_store/app_store/Cargo.toml index 25120858..4e095cb2 100644 --- a/modules/app_store/app_store/Cargo.toml +++ b/modules/app_store/app_store/Cargo.toml @@ -17,6 +17,7 @@ cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-co rand = "0.8.5" serde = {version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.10.8" wit-bindgen = { version = "0.11.0", default_features = false } [lib] diff --git a/modules/app_store/app_store/src/lib.rs b/modules/app_store/app_store/src/lib.rs index 24ad9fb2..ffeee069 100644 --- a/modules/app_store/app_store/src/lib.rs +++ b/modules/app_store/app_store/src/lib.rs @@ -4,6 +4,7 @@ use bindings::{ send_request, send_response, Guest, }; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256, Sha512}; use std::collections::{HashMap, HashSet}; #[allow(dead_code)] @@ -17,6 +18,9 @@ use process_lib::PackageId; #[allow(dead_code)] mod ft_worker_lib; +use ft_worker_lib::{ + spawn_receive_transfer, spawn_transfer, FTWorkerCommand, FTWorkerResult, FileTransferContext, +}; struct Component; @@ -43,15 +47,15 @@ struct Component; #[derive(Debug, Serialize, Deserialize)] struct State { pub packages: HashMap, + pub requested_packages: HashMap, } /// state of an individual package we have downloaded #[derive(Debug, Serialize, Deserialize)] struct PackageState { pub mirrored_from: NodeId, - pub listing_data: Option, // None if package is unlisted - pub installed_version: Option, // None if downloaded but not installed - pub mirroring: bool, // are we serving this package to others? + pub listing_data: PackageListing, + pub mirroring: bool, // are we serving this package to others? pub auto_update: bool, // if we get a listing data update, will we try to download it? } @@ -75,16 +79,19 @@ struct PackageListing { pub enum Req { LocalRequest(LocalRequest), RemoteRequest(RemoteRequest), - FTWorkerCommand(ft_worker_lib::FTWorkerCommand), + FTWorkerCommand(FTWorkerCommand), } #[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] // untagged as a meta-type for all responses pub enum Resp { + RemoteResponse(RemoteResponse), + FTWorkerResult(FTWorkerResult), // note that we do not need to ourselves handle local responses, as // those are given to others rather than received. - RemoteResponse(RemoteResponse), - FTWorkerResult(ft_worker_lib::FTWorkerResult), + NewPackageResponse(NewPackageResponse), + DownloadResponse(DownloadResponse), + InstallResponse(InstallResponse), } /// Local requests take this form. @@ -93,6 +100,7 @@ pub enum LocalRequest { /// expects a zipped package as payload: create a new package from it /// if requested, will return a NewPackageResponse indicating success/failure NewPackage { + package: PackageId, mirror: bool, // sets whether we will mirror this package }, /// no payload; try to download a package from a specified node @@ -139,7 +147,7 @@ pub enum NewPackageResponse { #[derive(Debug, Serialize, Deserialize)] pub enum DownloadResponse { - Success, + Started, Failure, } @@ -171,6 +179,7 @@ impl Guest for Component { // load in our saved state or initalize a new one if none exists let mut state = process_lib::get_state::().unwrap_or(State { packages: HashMap::new(), + requested_packages: HashMap::new(), }); // active the main messaging loop: handle requests and responses @@ -183,6 +192,10 @@ impl Guest for Component { continue; } }; + print_to_terminal( + 0, + &format!("app-store: got message from {}: {:?}", source.to_string(), message), + ); match message { Message::Request(req) => { let Some(ref ipc) = req.ipc else { @@ -190,16 +203,14 @@ impl Guest for Component { }; match &serde_json::from_str::(ipc) { Ok(Req::LocalRequest(local_request)) => { - match handle_local_request(local_request) { + match handle_local_request(&our, &source, local_request, &mut state) { Ok(None) => continue, Ok(Some(resp)) => { if req.expects_response.is_some() { send_response( &Response { inherit: false, - ipc: Some( - serde_json::to_string(&resp).unwrap(), - ), + ipc: Some(serde_json::to_string(&resp).unwrap()), metadata: None, }, None, @@ -215,16 +226,14 @@ impl Guest for Component { } } Ok(Req::RemoteRequest(remote_request)) => { - match handle_remote_request(remote_request) { + match handle_remote_request(&our, &source, remote_request, &mut state) { Ok(None) => continue, Ok(Some(resp)) => { if req.expects_response.is_some() { send_response( &Response { inherit: false, - ipc: Some( - serde_json::to_string(&resp).unwrap(), - ), + ipc: Some(serde_json::to_string(&resp).unwrap()), metadata: None, }, None, @@ -234,15 +243,19 @@ impl Guest for Component { Err(err) => { print_to_terminal( 0, - &format!("app-store: local request error: {:?}", err), + &format!("app-store: remote request error: {:?}", err), ); } } } - Ok(Req::FTWorkerCommand(ft_worker_command)) => { - // TODO handle ft_worker commands + Ok(Req::FTWorkerCommand(_)) => { + spawn_receive_transfer(&our, ipc); } - Err(_) => { + e => { + print_to_terminal( + 0, + &format!("app store bad request: {}, error {:?}", ipc, e), + ); continue; } } @@ -252,384 +265,461 @@ impl Guest for Component { continue; }; match &serde_json::from_str::(ipc) { - Ok(Resp::RemoteResponse(remote_response)) => { - // TODO handle remote response - } + Ok(Resp::RemoteResponse(remote_response)) => match remote_response { + RemoteResponse::DownloadApproved => { + print_to_terminal( + 0, + "app store: download approved, should be starting", + ); + } + RemoteResponse::DownloadDenied => { + print_to_terminal( + 0, + "app store: could not download package from that node!", + ); + } + }, Ok(Resp::FTWorkerResult(ft_worker_result)) => { - // TODO handle ft_worker result + let Ok(context) = serde_json::from_str::(&context.unwrap_or_default()) else { + print_to_terminal(0, "file_transfer: got weird local request"); + continue; + }; + match ft_worker_result { + FTWorkerResult::SendSuccess => { + print_to_terminal( + 0, + &format!( + "file_transfer: successfully shared app {} in {:.4}s", + context.file_name, + std::time::SystemTime::now() + .duration_since(context.start_time) + .unwrap() + .as_secs_f64(), + ), + ); + } + FTWorkerResult::ReceiveSuccess(name) => { + // do with file what you'd like here + print_to_terminal( + 0, + &format!("file_transfer: successfully received {:?}", name,), + ); + // remove .zip from name + let package_id = + match PackageId::from_str(name.trim_end_matches(".zip")) { + Ok(package_id) => package_id, + Err(_) => { + print_to_terminal( + 0, + &format!( + "app store: bad package filename: {}", + name + ), + ); + continue; + } + }; + if let Some(install_from) = + state.requested_packages.remove(&package_id) + { + if install_from == source.node { + // auto-take zip from payload and request ourself with New + let _ = send_request( + &our, + &Request { + inherit: true, // will inherit payload! + expects_response: None, + ipc: Some( + serde_json::to_string(&Req::LocalRequest( + LocalRequest::NewPackage { + package: package_id, + mirror: true, + }, + )) + .unwrap(), + ), + metadata: None, + }, + None, + None, + ); + } else { + print_to_terminal( + 0, + &format!( + "app-store: got install response from bad source: {}", + install_from + ), + ); + } + } + } + FTWorkerResult::Err(e) => { + print_to_terminal( + 0, + &format!("app store file transfer: error {:?}", e), + ); + } + } } - Err(_) => { + e => { + print_to_terminal( + 0, + &format!("app store bad response: {}, error {:?}", ipc, e), + ); continue; } } - // // only expecting NewFromRemote for apps we've requested - // match serde_json::from_str(&response.ipc.unwrap_or_default()) { - // Ok(AppTrackerResponse::NewFromRemote { package_id }) => { - // if let Some(install_from) = state.requested_packages.remove(&package_id) - // { - // if install_from == source.node { - // // auto-take zip from payload and request ourself with New - // let _ = send_request( - // &our, - // &Request { - // inherit: true, // will inherit payload! - // expects_response: None, - // ipc: Some( - // serde_json::to_string(&AppTrackerRequest::New { - // package: package_id, - // mirror: true, - // }) - // .unwrap(), - // ), - // metadata: None, - // }, - // None, - // None, - // ); - // } else { - // print_to_terminal( - // 0, - // &format!( - // "app-store: got install response from bad source: {}", - // install_from - // ), - // ); - // } - // } - // } - // err => { - // print_to_terminal( - // 0, - // &format!("app-store: got unexpected response {:?}", err), - // ); - // } - // } } } } } } -fn handle_local_request(request: &LocalRequest) -> anyhow::Result> { - // TODO - Ok(None) +fn handle_local_request( + our: &Address, + source: &Address, + request: &LocalRequest, + state: &mut State, +) -> anyhow::Result> { + if our.node != source.node { + return Err(anyhow::anyhow!("local request from non-local node")); + } + match request { + LocalRequest::NewPackage { package, mirror } => { + let Some(mut payload) = get_payload() else { + return Err(anyhow::anyhow!("no payload")); + }; + let vfs_address = Address { + node: our.node.clone(), + process: ProcessId::from_str("vfs:sys:uqbar")?, + }; + + // produce the version hash for this new package + let mut hasher = sha2::Sha256::new(); + hasher.update(&payload.bytes); + let version_hash = format!("{:x}", hasher.finalize()); + + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::New, + })?), + None, + None, + 5, + )?; + + // add zip bytes + payload.mime = Some("application/zip".to_string()); + let _ = process_lib::send_and_await_response( + &vfs_address, + true, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::Add { + full_path: package.to_string(), + entry_type: kt::AddEntryType::ZipArchive, + }, + })?), + None, + Some(&payload), + 5, + )?; + + // save the zip file itself in VFS for sharing with other nodes + // call it .zip + let _ = process_lib::send_and_await_response( + &vfs_address, + true, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::Add { + full_path: format!("/{}.zip", package.to_string()), + entry_type: kt::AddEntryType::NewFile, + }, + })?), + None, + Some(&payload), + 5, + )?; + + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry("/metadata.json".into()), + })?), + None, + None, + 5, + )?; + let Some(payload) = get_payload() else { + return Err(anyhow::anyhow!("no metadata payload")); + }; + let metadata = String::from_utf8(payload.bytes)?; + let metadata = serde_json::from_str::(&metadata)?; + + let listing_data = PackageListing { + name: metadata.package, + publisher: our.node.clone(), + description: metadata.description, + website: metadata.website, + version: metadata.version, + version_hash, + }; + let package_state = PackageState { + mirrored_from: our.node.clone(), + listing_data, + mirroring: *mirror, + auto_update: true, + }; + state.packages.insert(package.clone(), package_state); + process_lib::set_state::(&state); + Ok(Some(Resp::NewPackageResponse(NewPackageResponse::Success))) + } + LocalRequest::Download { + package, + install_from, + } => Ok(Some(Resp::DownloadResponse( + match process_lib::send_and_await_response( + &Address { + node: install_from.clone(), + process: our.process.clone(), + }, + true, + Some(serde_json::to_string(&RemoteRequest::Download( + package.clone(), + ))?), + None, + None, + 5, + ) { + Ok((_source, Message::Response((resp, _context)))) => { + let Some(ipc) = resp.ipc else { + return Err(anyhow::anyhow!("no ipc in response")) + }; + let resp = serde_json::from_str::(&ipc)?; + match resp { + Resp::RemoteResponse(RemoteResponse::DownloadApproved) => { + state + .requested_packages + .insert(package.clone(), install_from.to_string()); + process_lib::set_state::(&state); + DownloadResponse::Started + } + _ => DownloadResponse::Failure, + } + } + _ => DownloadResponse::Failure, + }, + ))), + LocalRequest::Install(package) => { + let vfs_address = Address { + node: our.node.clone(), + process: ProcessId::from_str("vfs:sys:uqbar")?, + }; + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry("/manifest.json".into()), + })?), + None, + None, + 5, + )?; + let Some(payload) = get_payload() else { + return Err(anyhow::anyhow!("no payload")); + }; + let manifest = String::from_utf8(payload.bytes)?; + let manifest = serde_json::from_str::>(&manifest)?; + for entry in manifest { + let path = if entry.process_wasm_path.starts_with("/") { + entry.process_wasm_path + } else { + format!("/{}", entry.process_wasm_path) + }; + + let (_, hash_response) = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetHash(path.clone()), + })?), + None, + None, + 5, + )?; + + let Message::Response((Response { ipc: Some(ipc), .. }, _)) = hash_response else { + return Err(anyhow::anyhow!("bad vfs response")); + }; + let kt::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc)? else { + return Err(anyhow::anyhow!("no hash in vfs")); + }; + + // build initial caps + let mut initial_capabilities: HashSet = HashSet::new(); + if entry.request_networking { + let Some(networking_cap) = get_capability( + &Address { + node: our.node.clone(), + process: ProcessId::from_str("kernel:sys:uqbar")?, + }, + &"\"network\"".to_string(), + ) else { + return Err(anyhow::anyhow!("app-store: no net cap")); + }; + initial_capabilities.insert(kt::de_wit_signed_capability(networking_cap)); + } + let Some(read_cap) = get_capability( + &vfs_address.clone(), + &serde_json::to_string(&serde_json::json!({ + "kind": "read", + "drive": package.to_string(), + }))?, + ) else { + return Err(anyhow::anyhow!("app-store: no read cap")); + }; + initial_capabilities.insert(kt::de_wit_signed_capability(read_cap)); + let Some(write_cap) = get_capability( + &vfs_address.clone(), + &serde_json::to_string(&serde_json::json!({ + "kind": "write", + "drive": package.to_string(), + }))?, + ) else { + return Err(anyhow::anyhow!("app-store: no write cap")); + }; + initial_capabilities.insert(kt::de_wit_signed_capability(write_cap)); + + for process_name in &entry.request_messaging { + let Ok(parsed_process_id) = ProcessId::from_str(&process_name) else { + // TODO handle arbitrary caps here + continue; + }; + let Some(messaging_cap) = get_capability( + &Address { + node: our.node.clone(), + process: parsed_process_id.clone(), + }, + &"\"messaging\"".into() + ) else { + print_to_terminal(0, &format!("app-store: no cap for {} to give away!", process_name)); + continue; + }; + initial_capabilities.insert(kt::de_wit_signed_capability(messaging_cap)); + } + + let process_id = format!("{}:{}", entry.process_name, package.to_string()); + let Ok(parsed_new_process_id) = ProcessId::from_str(&process_id) else { + return Err(anyhow::anyhow!("app-store: invalid process id!")); + }; + let _ = process_lib::send_request( + &Address { + node: our.node.clone(), + process: ProcessId::from_str("kernel:sys:uqbar")?, + }, + false, + Some(serde_json::to_string(&kt::KernelCommand::KillProcess( + kt::ProcessId::de_wit(parsed_new_process_id.clone()), + ))?), + None, + None, + None, + ); + + // kernel start process takes bytes as payload + wasm_bytes_handle... + // reconsider perhaps + let (_, _bytes_response) = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry(path), + })?), + None, + None, + 5, + )?; + + let Some(payload) = get_payload() else { + return Err(anyhow::anyhow!("no wasm bytes payload.")); + }; + + let _ = process_lib::send_and_await_response( + &Address { + node: our.node.clone(), + process: ProcessId::from_str("kernel:sys:uqbar")?, + }, + false, + Some(serde_json::to_string(&kt::KernelCommand::StartProcess { + id: kt::ProcessId::de_wit(parsed_new_process_id), + wasm_bytes_handle: hash, + on_panic: entry.on_panic, + initial_capabilities, + public: entry.public, + })?), + None, + Some(&payload), + 5, + )?; + } + Ok(Some(Resp::InstallResponse(InstallResponse::Success))) + } + LocalRequest::Uninstall(package) => { + // TODO + Ok(None) + } + LocalRequest::Delete(package) => { + // TODO + Ok(None) + } + } } -fn handle_remote_request(request: &RemoteRequest) -> anyhow::Result> { - // TODO - Ok(None) +fn handle_remote_request( + our: &Address, + source: &Address, + request: &RemoteRequest, + state: &mut State, +) -> anyhow::Result> { + match request { + RemoteRequest::Download(package) => { + print_to_terminal(0, &format!("app store: got download request for {:?}", package)); + print_to_terminal(0, &format!("app store: state: {:?}", state)); + let Some(package_state) = state.packages.get(&package) else { + return Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadDenied))) + }; + if !package_state.mirroring { + return Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadDenied))); + } + // get the .zip from VFS and attach as payload to response + let vfs_address = Address { + node: our.node.clone(), + process: ProcessId::from_str("vfs:sys:uqbar")?, + }; + let file_name = format!("/{}.zip", package.to_string()); + let _ = process_lib::send_and_await_response( + &vfs_address, + false, + Some(serde_json::to_string(&kt::VfsRequest { + drive: package.to_string(), + action: kt::VfsAction::GetEntry(file_name.clone()), + })?), + None, + None, + 5, + )?; + // transfer will inherit the payload bytes we receive from VFS + spawn_transfer(&our, &file_name, None, &source); + Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadApproved))) + } + } } - -// fn parse_command( -// our: &Address, -// source: &Address, -// request_string: String, -// state: &mut AppTrackerState, -// ) -> anyhow::Result> { -// match serde_json::from_str(&request_string)? { -// // create a new package based on local payload -// AppTrackerRequest::New { package, mirror } => { -// if our.node != source.node { -// return Err(anyhow::anyhow!("new package request from non-local node")); -// } -// let Some(mut payload) = get_payload() else { -// return Err(anyhow::anyhow!("no payload")); -// }; - -// let vfs_address = Address { -// node: our.node.clone(), -// process: ProcessId::from_str("vfs:sys:uqbar")?, -// }; - -// let _ = process_lib::send_and_await_response( -// &vfs_address, -// false, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::New, -// })?), -// None, -// None, -// 5, -// )?; - -// // add zip bytes -// payload.mime = Some("application/zip".to_string()); -// let _ = process_lib::send_and_await_response( -// &vfs_address, -// true, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::Add { -// full_path: package.to_string(), -// entry_type: kt::AddEntryType::ZipArchive, -// }, -// })?), -// None, -// Some(&payload), -// 5, -// )?; - -// // save the zip file itself in VFS for sharing with other nodes -// // call it .zip -// let _ = process_lib::send_and_await_response( -// &vfs_address, -// true, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::Add { -// full_path: format!("/{}.zip", package.to_string()), -// entry_type: kt::AddEntryType::NewFile, -// }, -// })?), -// None, -// Some(&payload), -// 5, -// )?; - -// // if mirror, save in our state -// if mirror { -// let _ = process_lib::send_and_await_response( -// &vfs_address, -// false, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::GetEntry("/metadata.json".into()), -// })?), -// None, -// None, -// 5, -// )?; -// let Some(payload) = get_payload() else { -// return Err(anyhow::anyhow!("no metadata payload")); -// }; -// let metadata = String::from_utf8(payload.bytes)?; -// let metadata = serde_json::from_str::(&metadata)?; -// state -// .mirrored_packages -// .insert(PackageId::new(&metadata.package, &metadata.publisher)); -// process_lib::set_state::(&state); -// } - -// Ok(Some(AppTrackerResponse::New { -// package: package.to_string(), -// })) -// } -// // if we are the source, forward to install_from target. -// // if we install_from, respond with package if we have it -// AppTrackerRequest::NewFromRemote { -// package_id, -// install_from, -// } => { -// if our.node == source.node { -// let _ = send_request( -// &Address { -// node: install_from.clone(), -// process: our.process.clone(), -// }, -// &Request { -// inherit: true, -// expects_response: Some(5), // TODO -// ipc: Some(serde_json::to_string(&AppTrackerRequest::NewFromRemote { -// package_id: package_id.clone(), -// install_from: install_from.clone(), -// })?), -// metadata: None, -// }, -// None, -// None, -// ); -// state.requested_packages.insert(package_id, install_from); -// process_lib::set_state::(&state); -// Ok(None) -// } else if our.node == install_from { -// let Some(_mirror) = state.mirrored_packages.get(&package_id) else { -// return Ok(Some(AppTrackerResponse::Error { error: "package not mirrored here!".into() })) -// }; -// // get the .zip from VFS and attach as payload to response -// let vfs_address = Address { -// node: our.node.clone(), -// process: ProcessId::from_str("vfs:sys:uqbar")?, -// }; -// let _ = process_lib::send_and_await_response( -// &vfs_address, -// false, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package_id.to_string(), -// action: kt::VfsAction::GetEntry(format!("/{}.zip", package_id.to_string())), -// })?), -// None, -// None, -// 5, -// )?; -// Ok(Some(AppTrackerResponse::NewFromRemote { package_id })) -// } else { -// // TODO what to do here? -// Ok(None) -// } -// } -// AppTrackerRequest::Install { package } => { -// if our.node != source.node { -// return Err(anyhow::anyhow!("install request from non-local node")); -// } -// let vfs_address = Address { -// node: our.node.clone(), -// process: ProcessId::from_str("vfs:sys:uqbar")?, -// }; - -// let _ = process_lib::send_and_await_response( -// &vfs_address, -// false, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::GetEntry("/manifest.json".into()), -// })?), -// None, -// None, -// 5, -// )?; -// let Some(payload) = get_payload() else { -// return Err(anyhow::anyhow!("no payload")); -// }; -// let manifest = String::from_utf8(payload.bytes)?; -// let manifest = serde_json::from_str::>(&manifest)?; - -// for entry in manifest { -// let path = if entry.process_wasm_path.starts_with("/") { -// entry.process_wasm_path -// } else { -// format!("/{}", entry.process_wasm_path) -// }; - -// let (_, hash_response) = process_lib::send_and_await_response( -// &vfs_address, -// false, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::GetHash(path.clone()), -// })?), -// None, -// None, -// 5, -// )?; - -// let Message::Response((Response { ipc: Some(ipc), .. }, _)) = hash_response else { -// return Err(anyhow::anyhow!("bad vfs response")); -// }; -// let kt::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc)? else { -// return Err(anyhow::anyhow!("no hash in vfs")); -// }; - -// // build initial caps -// let mut initial_capabilities: HashSet = HashSet::new(); -// if entry.request_networking { -// let Some(networking_cap) = get_capability( -// &Address { -// node: our.node.clone(), -// process: ProcessId::from_str("kernel:sys:uqbar")?, -// }, -// &"\"network\"".to_string(), -// ) else { -// return Err(anyhow::anyhow!("app-store: no net cap")); -// }; -// initial_capabilities.insert(kt::de_wit_signed_capability(networking_cap)); -// } -// let Some(read_cap) = get_capability( -// &vfs_address.clone(), -// &serde_json::to_string(&serde_json::json!({ -// "kind": "read", -// "drive": package.to_string(), -// }))?, -// ) else { -// return Err(anyhow::anyhow!("app-store: no read cap")); -// }; -// initial_capabilities.insert(kt::de_wit_signed_capability(read_cap)); -// let Some(write_cap) = get_capability( -// &vfs_address.clone(), -// &serde_json::to_string(&serde_json::json!({ -// "kind": "write", -// "drive": package.to_string(), -// }))?, -// ) else { -// return Err(anyhow::anyhow!("app-store: no write cap")); -// }; -// initial_capabilities.insert(kt::de_wit_signed_capability(write_cap)); - -// for process_name in &entry.request_messaging { -// let Ok(parsed_process_id) = ProcessId::from_str(&process_name) else { -// // TODO handle arbitrary caps here -// continue; -// }; -// let Some(messaging_cap) = get_capability( -// &Address { -// node: our.node.clone(), -// process: parsed_process_id.clone(), -// }, -// &"\"messaging\"".into() -// ) else { -// print_to_terminal(0, &format!("app-store: no cap for {} to give away!", process_name)); -// continue; -// }; -// initial_capabilities.insert(kt::de_wit_signed_capability(messaging_cap)); -// } - -// let process_id = format!("{}:{}", entry.process_name, package.to_string()); -// let Ok(parsed_new_process_id) = ProcessId::from_str(&process_id) else { -// return Err(anyhow::anyhow!("app-store: invalid process id!")); -// }; -// let _ = process_lib::send_request( -// &Address { -// node: our.node.clone(), -// process: ProcessId::from_str("kernel:sys:uqbar")?, -// }, -// false, -// Some(serde_json::to_string(&kt::KernelCommand::KillProcess( -// kt::ProcessId::de_wit(parsed_new_process_id.clone()), -// ))?), -// None, -// None, -// None, -// ); - -// // kernel start process takes bytes as payload + wasm_bytes_handle... -// // reconsider perhaps -// let (_, _bytes_response) = process_lib::send_and_await_response( -// &vfs_address, -// false, -// Some(serde_json::to_string(&kt::VfsRequest { -// drive: package.to_string(), -// action: kt::VfsAction::GetEntry(path), -// })?), -// None, -// None, -// 5, -// )?; - -// let Some(payload) = get_payload() else { -// return Err(anyhow::anyhow!("no wasm bytes payload.")); -// }; - -// let _ = process_lib::send_and_await_response( -// &Address { -// node: our.node.clone(), -// process: ProcessId::from_str("kernel:sys:uqbar")?, -// }, -// false, -// Some(serde_json::to_string(&kt::KernelCommand::StartProcess { -// id: kt::ProcessId::de_wit(parsed_new_process_id), -// wasm_bytes_handle: hash, -// on_panic: entry.on_panic, -// initial_capabilities, -// public: entry.public, -// })?), -// None, -// Some(&payload), -// 5, -// )?; -// } -// Ok(Some(AppTrackerResponse::Install { -// package: package.to_string(), -// })) -// } -// } -// } diff --git a/modules/app_store/ft_worker/src/ft_worker_lib.rs b/modules/app_store/ft_worker/src/ft_worker_lib.rs index 514a5260..79085ed2 100644 --- a/modules/app_store/ft_worker/src/ft_worker_lib.rs +++ b/modules/app_store/ft_worker/src/ft_worker_lib.rs @@ -1,11 +1,11 @@ use super::bindings::component::uq_process::types::*; -use super::bindings::{Address, Payload, print_to_terminal, send_request, spawn}; +use super::bindings::{print_to_terminal, send_request, spawn, Address, Payload}; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] pub struct FileTransferContext { pub file_name: String, - pub file_size: u64, + pub file_size: Option, pub start_time: std::time::SystemTime, } @@ -48,7 +48,7 @@ pub enum TransferError { pub fn spawn_transfer( our: &Address, file_name: &str, - file_bytes: Vec, + file_bytes: Option>, // if None, expects to inherit payload! to_addr: &Address, ) { let transfer_id: u64 = rand::random(); @@ -64,13 +64,17 @@ pub fn spawn_transfer( return; }; // tell the worker what to do + let payload_or_inherit = match file_bytes { + Some(bytes) => Some(Payload { mime: None, bytes }), + None => None, + }; send_request( &Address { node: our.node.clone(), process: worker_process_id, }, &Request { - inherit: false, + inherit: !payload_or_inherit.is_some(), expects_response: Some(61), ipc: Some( serde_json::to_string(&FTWorkerCommand::Send { @@ -85,19 +89,19 @@ pub fn spawn_transfer( Some( &serde_json::to_string(&FileTransferContext { file_name: file_name.into(), - file_size: file_bytes.len() as u64, + file_size: match &payload_or_inherit { + Some(p) => Some(p.bytes.len() as u64), + None => None, // TODO + }, start_time: std::time::SystemTime::now(), }) .unwrap(), ), - Some(&Payload { mime: None, bytes: file_bytes }), + payload_or_inherit.as_ref(), ); } -pub fn spawn_receive_transfer( - our: &Address, - ipc: &str, -) { +pub fn spawn_receive_transfer(our: &Address, ipc: &str) { let Ok(FTWorkerCommand::Receive { transfer_id, .. }) = serde_json::from_str(ipc) else { print_to_terminal(0, "file_transfer: got weird request"); return; diff --git a/modules/app_store/pkg/manifest.json b/modules/app_store/pkg/manifest.json index 1c9ae034..a3d11081 100644 --- a/modules/app_store/pkg/manifest.json +++ b/modules/app_store/pkg/manifest.json @@ -5,7 +5,6 @@ "on_panic": "Restart", "request_networking": true, "request_messaging": [ - "http_bindings:http_bindings:uqbar", "terminal:terminal:uqbar", "filesystem:sys:uqbar", "http_server:sys:uqbar", diff --git a/src/types.rs b/src/types.rs index 56d5c866..7c4537d8 100644 --- a/src/types.rs +++ b/src/types.rs @@ -403,12 +403,14 @@ pub struct ProcessContext { // filesystem.rs types // +pub type PackageVersion = (u32, u32, u32); + /// the type that gets deserialized from `metadata.json` in a package #[derive(Debug, Serialize, Deserialize)] pub struct PackageMetadata { pub package: String, pub publisher: String, - pub version: Version, + pub version: PackageVersion, pub description: Option, pub website: Option, } @@ -418,7 +420,7 @@ pub struct PackageMetadata { pub struct PackageManifestEntry { pub process_name: String, pub process_wasm_path: String, - pub on_panic: kt::OnPanic, + pub on_panic: OnPanic, pub request_networking: bool, pub request_messaging: Vec, pub public: bool, From 7b5c70b6eabeca2117aee26263e01f71748d4e33 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 15:21:55 -0400 Subject: [PATCH 17/24] merge from main --- modules/app_store/app_store/Cargo.lock | 220 +++++++------------------ modules/app_store/app_store/Cargo.toml | 32 ---- modules/app_store/ft_worker/Cargo.lock | 181 +++++--------------- 3 files changed, 102 insertions(+), 331 deletions(-) delete mode 100644 modules/app_store/app_store/Cargo.toml diff --git a/modules/app_store/app_store/Cargo.lock b/modules/app_store/app_store/Cargo.lock index c601a168..70700207 100644 --- a/modules/app_store/app_store/Cargo.lock +++ b/modules/app_store/app_store/Cargo.lock @@ -19,7 +19,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "wit-bindgen", + "wit-bindgen 0.11.0", ] [[package]] @@ -33,15 +33,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -54,17 +48,17 @@ dependencies = [ [[package]] name = "cargo-component-bindings" -version = "0.1.0" -source = "git+https://github.com/bytecodealliance/cargo-component#6a2996f280dd8671a2a2d3c83cbe09a39225b526" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" dependencies = [ "cargo-component-macro", - "wit-bindgen", + "wit-bindgen 0.13.0", ] [[package]] name = "cargo-component-macro" -version = "0.1.0" -source = "git+https://github.com/bytecodealliance/cargo-component#6a2996f280dd8671a2a2d3c83cbe09a39225b526" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" dependencies = [ "heck", "proc-macro2", @@ -72,7 +66,6 @@ dependencies = [ "syn", "wit-bindgen-core", "wit-bindgen-rust", - "wit-bindgen-rust-lib", "wit-component", ] @@ -117,15 +110,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "form_urlencoded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" -dependencies = [ - "percent-encoding", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -149,9 +133,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heck" @@ -168,21 +152,11 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "indexmap" -version = "2.0.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", "hashbrown", @@ -213,18 +187,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "memchr" -version = "2.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -233,24 +195,13 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] -[[package]] -name = "pulldown-cmark" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "quote" version = "1.0.33" @@ -298,24 +249,24 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", @@ -324,9 +275,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -346,9 +297,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "spdx" @@ -361,65 +312,26 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "typenum" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-segmentation" @@ -433,17 +345,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "url" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - [[package]] name = "version_check" version = "0.9.4" @@ -458,22 +359,23 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-encoder" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" dependencies = [ "leb128", ] [[package]] name = "wasm-metadata" -version = "0.10.3" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08dc59d1fa569150851542143ca79438ca56845ccb31696c70225c638e063471" +checksum = "14abc161bfda5b519aa229758b68f2a52b45a12b993808665c857d1a9a00223c" dependencies = [ "anyhow", "indexmap", "serde", + "serde_derive", "serde_json", "spdx", "wasm-encoder", @@ -482,9 +384,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.112.0" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e986b010f47fcce49cf8ea5d5f9e5d2737832f12b53ae8ae785bbe895d0877bf" +checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" dependencies = [ "indexmap", "semver", @@ -496,15 +398,24 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a3e8e965dc50e6eb4410d9a11720719fadc6a1713803ea5f3be390b81c8279" dependencies = [ - "bitflags 2.4.0", + "bitflags", +] + +[[package]] +name = "wit-bindgen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d92ce0ca6b6074059413a9581a637550c3a740581c854f9847ec293c8aed71" +dependencies = [ + "bitflags", "wit-bindgen-rust-macro", ] [[package]] name = "wit-bindgen-core" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77255512565dfbd0b61de466e854918041d1da53c7bc049d6188c6e02643dc1e" +checksum = "565b945ae074886071eccf9cdaf8ccd7b959c2b0d624095bea5fe62003e8b3e0" dependencies = [ "anyhow", "wit-component", @@ -513,54 +424,44 @@ dependencies = [ [[package]] name = "wit-bindgen-rust" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c60e6ea8598d1380e792f13d557007834f0fb799fea6503408cbc5debb4ae" +checksum = "5695ff4e41873ed9ce56d2787e6b5772bdad9e70e2c1d2d160621d1762257f4f" dependencies = [ "anyhow", "heck", "wasm-metadata", "wit-bindgen-core", - "wit-bindgen-rust-lib", "wit-component", ] -[[package]] -name = "wit-bindgen-rust-lib" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9fb7a43c7dc28b0b727d6ae01bf369981229b7539e768fba2b7a4df13feeeb" -dependencies = [ - "heck", - "wit-bindgen-core", -] - [[package]] name = "wit-bindgen-rust-macro" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cea5ed784da06da0e55836a6c160e7502dbe28771c2368a595e8606243bf22" +checksum = "a91835ea4231da1fe7971679d505ba14be7826e192b6357f08465866ef482e08" dependencies = [ "anyhow", "proc-macro2", + "quote", "syn", "wit-bindgen-core", "wit-bindgen-rust", - "wit-bindgen-rust-lib", "wit-component", ] [[package]] name = "wit-component" -version = "0.14.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d9f2d16dd55d1a372dcfd4b7a466ea876682a5a3cb97e71ec9eef04affa876" +checksum = "e87488b57a08e2cbbd076b325acbe7f8666965af174d69d5929cd373bd54547f" dependencies = [ "anyhow", - "bitflags 2.4.0", + "bitflags", "indexmap", "log", "serde", + "serde_derive", "serde_json", "wasm-encoder", "wasm-metadata", @@ -570,16 +471,17 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e8b849bea13cc2315426b16efe6eb6813466d78f5fde69b0bb150c9c40e0dc" +checksum = "f6ace9943d89bbf3dbbc71b966da0e7302057b311f36a4ac3d65ddfef17b52cf" dependencies = [ "anyhow", "id-arena", "indexmap", "log", - "pulldown-cmark", "semver", + "serde", + "serde_derive", + "serde_json", "unicode-xid", - "url", ] diff --git a/modules/app_store/app_store/Cargo.toml b/modules/app_store/app_store/Cargo.toml deleted file mode 100644 index 4e095cb2..00000000 --- a/modules/app_store/app_store/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "app_store" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[profile.release] -panic = "abort" -opt-level = "s" -lto = true - -[dependencies] -anyhow = "1.0" -bincode = "1.3.3" -cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-component" } -rand = "0.8.5" -serde = {version = "1.0", features = ["derive"] } -serde_json = "1.0" -sha2 = "0.10.8" -wit-bindgen = { version = "0.11.0", default_features = false } - -[lib] -crate-type = ["cdylib"] - -[package.metadata.component] -package = "component:uq-process" - -[package.metadata.component.target] -path = "wit" - -[package.metadata.component.dependencies] diff --git a/modules/app_store/ft_worker/Cargo.lock b/modules/app_store/ft_worker/Cargo.lock index cc84648e..7abc1fbd 100644 --- a/modules/app_store/ft_worker/Cargo.lock +++ b/modules/app_store/ft_worker/Cargo.lock @@ -19,29 +19,23 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "cargo-component-bindings" -version = "0.2.0" -source = "git+https://github.com/bytecodealliance/cargo-component#2c45278720196778704f5baf62066d355d48f702" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" dependencies = [ "cargo-component-macro", - "wit-bindgen 0.12.0", + "wit-bindgen 0.13.0", ] [[package]] name = "cargo-component-macro" -version = "0.2.0" -source = "git+https://github.com/bytecodealliance/cargo-component#2c45278720196778704f5baf62066d355d48f702" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/cargo-component#5cddf6df7fd5ab5aa9bfd66f4cf2e2f6cc7d72f9" dependencies = [ "heck", "proc-macro2", @@ -64,15 +58,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "form_urlencoded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" -dependencies = [ - "percent-encoding", -] - [[package]] name = "ft_worker" version = "0.1.0" @@ -99,9 +84,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heck" @@ -118,16 +103,6 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "indexmap" version = "2.0.2" @@ -163,18 +138,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "memchr" -version = "2.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -190,17 +153,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pulldown-cmark" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "quote" version = "1.0.33" @@ -254,18 +206,18 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", @@ -309,51 +261,12 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.10.1" @@ -366,23 +279,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "url" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -391,22 +287,23 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-encoder" -version = "0.33.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34180c89672b3e4825c3a8db4b61a674f1447afd5fe2445b2d22c3d8b6ea086c" +checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" dependencies = [ "leb128", ] [[package]] name = "wasm-metadata" -version = "0.10.7" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2c051ef041d348324b01ff0419f6f6593f094b4897d93c9cf52d5d1ac879ba" +checksum = "14abc161bfda5b519aa229758b68f2a52b45a12b993808665c857d1a9a00223c" dependencies = [ "anyhow", "indexmap", "serde", + "serde_derive", "serde_json", "spdx", "wasm-encoder", @@ -415,9 +312,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.113.3" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "286049849b5a5bd09a8773171be96824afabffc7cc3df6caaf33a38db6cd07ae" +checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" dependencies = [ "indexmap", "semver", @@ -429,22 +326,24 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5c3d15a04ce994fad2c5442a754b404ab1fee23c903a04a560f84f94fdf63c0" dependencies = [ - "bitflags 2.4.0", + "bitflags", ] [[package]] name = "wit-bindgen" -version = "0.12.0" -source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d92ce0ca6b6074059413a9581a637550c3a740581c854f9847ec293c8aed71" dependencies = [ - "bitflags 2.4.0", + "bitflags", "wit-bindgen-rust-macro", ] [[package]] name = "wit-bindgen-core" -version = "0.12.0" -source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "565b945ae074886071eccf9cdaf8ccd7b959c2b0d624095bea5fe62003e8b3e0" dependencies = [ "anyhow", "wit-component", @@ -453,8 +352,9 @@ dependencies = [ [[package]] name = "wit-bindgen-rust" -version = "0.12.0" -source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5695ff4e41873ed9ce56d2787e6b5772bdad9e70e2c1d2d160621d1762257f4f" dependencies = [ "anyhow", "heck", @@ -465,8 +365,9 @@ dependencies = [ [[package]] name = "wit-bindgen-rust-macro" -version = "0.12.0" -source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91835ea4231da1fe7971679d505ba14be7826e192b6357f08465866ef482e08" dependencies = [ "anyhow", "proc-macro2", @@ -479,15 +380,16 @@ dependencies = [ [[package]] name = "wit-component" -version = "0.14.5" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e2bf941487fc5afa9e3fc94761f6b80ecef5a2bed6239b959d23d9de69e3448" +checksum = "e87488b57a08e2cbbd076b325acbe7f8666965af174d69d5929cd373bd54547f" dependencies = [ "anyhow", - "bitflags 2.4.0", + "bitflags", "indexmap", "log", "serde", + "serde_derive", "serde_json", "wasm-encoder", "wasm-metadata", @@ -497,18 +399,17 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.11.3" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a39edca9abb16309def3843af73b58d47d243fe33a9ceee572446bcc57556b9a" +checksum = "f6ace9943d89bbf3dbbc71b966da0e7302057b311f36a4ac3d65ddfef17b52cf" dependencies = [ "anyhow", "id-arena", "indexmap", "log", - "pulldown-cmark", "semver", "serde", + "serde_derive", "serde_json", "unicode-xid", - "url", ] From 9660c26a99bb4ab476570e56b6b2c312490e5925 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 15:23:21 -0400 Subject: [PATCH 18/24] restore cargo.toml that github decided to delete --- modules/app_store/app_store/Cargo.toml | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 modules/app_store/app_store/Cargo.toml diff --git a/modules/app_store/app_store/Cargo.toml b/modules/app_store/app_store/Cargo.toml new file mode 100644 index 00000000..4e095cb2 --- /dev/null +++ b/modules/app_store/app_store/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "app_store" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[profile.release] +panic = "abort" +opt-level = "s" +lto = true + +[dependencies] +anyhow = "1.0" +bincode = "1.3.3" +cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-component" } +rand = "0.8.5" +serde = {version = "1.0", features = ["derive"] } +serde_json = "1.0" +sha2 = "0.10.8" +wit-bindgen = { version = "0.11.0", default_features = false } + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "component:uq-process" + +[package.metadata.component.target] +path = "wit" + +[package.metadata.component.dependencies] From f4b0046df351b599ab03bba4b44f4890fd7938ce Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 16:18:37 -0400 Subject: [PATCH 19/24] nice and good again! kernel: save payload during spawn() --- modules/app_store/app_store/src/lib.rs | 116 ++++++++++--------------- modules/app_store/ft_worker/src/lib.rs | 3 +- src/kernel/mod.rs | 16 ++++ 3 files changed, 65 insertions(+), 70 deletions(-) diff --git a/modules/app_store/app_store/src/lib.rs b/modules/app_store/app_store/src/lib.rs index ffeee069..46e5b727 100644 --- a/modules/app_store/app_store/src/lib.rs +++ b/modules/app_store/app_store/src/lib.rs @@ -4,7 +4,7 @@ use bindings::{ send_request, send_response, Guest, }; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256, Sha512}; +use sha2::Digest; use std::collections::{HashMap, HashSet}; #[allow(dead_code)] @@ -47,7 +47,7 @@ struct Component; #[derive(Debug, Serialize, Deserialize)] struct State { pub packages: HashMap, - pub requested_packages: HashMap, + pub requested_packages: HashSet, } /// state of an individual package we have downloaded @@ -80,6 +80,7 @@ pub enum Req { LocalRequest(LocalRequest), RemoteRequest(RemoteRequest), FTWorkerCommand(FTWorkerCommand), + FTWorkerResult(FTWorkerResult), } #[derive(Debug, Serialize, Deserialize)] @@ -179,7 +180,7 @@ impl Guest for Component { // load in our saved state or initalize a new one if none exists let mut state = process_lib::get_state::().unwrap_or(State { packages: HashMap::new(), - requested_packages: HashMap::new(), + requested_packages: HashSet::new(), }); // active the main messaging loop: handle requests and responses @@ -192,10 +193,6 @@ impl Guest for Component { continue; } }; - print_to_terminal( - 0, - &format!("app-store: got message from {}: {:?}", source.to_string(), message), - ); match message { Message::Request(req) => { let Some(ref ipc) = req.ipc else { @@ -248,6 +245,47 @@ impl Guest for Component { } } } + Ok(Req::FTWorkerResult(FTWorkerResult::ReceiveSuccess(name))) => { + // do with file what you'd like here + print_to_terminal( + 0, + &format!("file_transfer: successfully received {:?}", name,), + ); + // remove leading / and .zip from file name to get package ID + let package_id = + match PackageId::from_str(name[1..].trim_end_matches(".zip")) { + Ok(package_id) => package_id, + Err(_) => { + print_to_terminal( + 0, + &format!("app store: bad package filename: {}", name), + ); + continue; + } + }; + if state.requested_packages.remove(&package_id) { + // auto-take zip from payload and request ourself with New + let _ = send_request( + &our, + &Request { + inherit: true, // will inherit payload! + expects_response: None, + ipc: Some( + serde_json::to_string(&Req::LocalRequest( + LocalRequest::NewPackage { + package: package_id, + mirror: true, + }, + )) + .unwrap(), + ), + metadata: None, + }, + None, + None, + ); + } + } Ok(Req::FTWorkerCommand(_)) => { spawn_receive_transfer(&our, ipc); } @@ -298,63 +336,7 @@ impl Guest for Component { ), ); } - FTWorkerResult::ReceiveSuccess(name) => { - // do with file what you'd like here - print_to_terminal( - 0, - &format!("file_transfer: successfully received {:?}", name,), - ); - // remove .zip from name - let package_id = - match PackageId::from_str(name.trim_end_matches(".zip")) { - Ok(package_id) => package_id, - Err(_) => { - print_to_terminal( - 0, - &format!( - "app store: bad package filename: {}", - name - ), - ); - continue; - } - }; - if let Some(install_from) = - state.requested_packages.remove(&package_id) - { - if install_from == source.node { - // auto-take zip from payload and request ourself with New - let _ = send_request( - &our, - &Request { - inherit: true, // will inherit payload! - expects_response: None, - ipc: Some( - serde_json::to_string(&Req::LocalRequest( - LocalRequest::NewPackage { - package: package_id, - mirror: true, - }, - )) - .unwrap(), - ), - metadata: None, - }, - None, - None, - ); - } else { - print_to_terminal( - 0, - &format!( - "app-store: got install response from bad source: {}", - install_from - ), - ); - } - } - } - FTWorkerResult::Err(e) => { + e => { print_to_terminal( 0, &format!("app store file transfer: error {:?}", e), @@ -505,9 +487,7 @@ fn handle_local_request( let resp = serde_json::from_str::(&ipc)?; match resp { Resp::RemoteResponse(RemoteResponse::DownloadApproved) => { - state - .requested_packages - .insert(package.clone(), install_from.to_string()); + state.requested_packages.insert(package.clone()); process_lib::set_state::(&state); DownloadResponse::Started } @@ -692,8 +672,6 @@ fn handle_remote_request( ) -> anyhow::Result> { match request { RemoteRequest::Download(package) => { - print_to_terminal(0, &format!("app store: got download request for {:?}", package)); - print_to_terminal(0, &format!("app store: state: {:?}", state)); let Some(package_state) = state.packages.get(&package) else { return Ok(Some(Resp::RemoteResponse(RemoteResponse::DownloadDenied))) }; diff --git a/modules/app_store/ft_worker/src/lib.rs b/modules/app_store/ft_worker/src/lib.rs index 45f75cd9..fd3a7ebc 100644 --- a/modules/app_store/ft_worker/src/lib.rs +++ b/modules/app_store/ft_worker/src/lib.rs @@ -38,7 +38,8 @@ impl Guest for Component { } => { let transfer_id: u64 = our.process.process().parse().unwrap(); let Some(payload) = get_payload() else { - panic!("ft_worker: got empty payload"); + print_to_terminal(0, "FTWorker wasn't given payload, exiting"); + return }; let file_bytes = payload.bytes; let mut file_size = file_bytes.len() as u64; diff --git a/src/kernel/mod.rs b/src/kernel/mod.rs index f099339c..0711fa65 100644 --- a/src/kernel/mod.rs +++ b/src/kernel/mod.rs @@ -282,6 +282,8 @@ impl UqProcessImports for ProcessWasi { capabilities: wit::Capabilities, public: bool, ) -> Result> { + // save existing payload to restore later + let old_last_payload = self.process.last_payload.clone(); let vfs_address = wit::Address { node: self.process.metadata.our.node.clone(), process: VFS_PROCESS_ID.en_wit(), @@ -312,13 +314,19 @@ impl UqProcessImports for ProcessWasi { .await else { println!("spawn: GetHash fail"); + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let wit::Message::Response((wit::Response { ipc: Some(ipc), .. }, _)) = hash_response else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let t::VfsResponse::GetHash(Some(hash)) = serde_json::from_str(&ipc).unwrap() else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let Ok(Ok(_)) = send_and_await_response( @@ -341,9 +349,13 @@ impl UqProcessImports for ProcessWasi { ) .await else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; let Some(t::Payload { mime: _, ref bytes }) = self.process.last_payload else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NoFileAtPath)); }; @@ -424,8 +436,12 @@ impl UqProcessImports for ProcessWasi { ) .await else { + // reset payload to what it was + self.process.last_payload = old_last_payload; return Ok(Err(wit::SpawnError::NameTaken)); }; + // reset payload to what it was + self.process.last_payload = old_last_payload; let wit::Message::Response((wit::Response { ipc: Some(ipc), .. }, _)) = response else { return Ok(Err(wit::SpawnError::NoFileAtPath)); }; From 41ba7ed7eaa90f9a58105431ae9c01b8dc9b01f9 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 16:26:02 -0400 Subject: [PATCH 20/24] update metadata for modules to contain a version --- modules/app_store/pkg/metadata.json | 1 + modules/chess/pkg/metadata.json | 3 ++- modules/homepage/pkg/metadata.json | 3 ++- modules/http_proxy/pkg/metadata.json | 3 ++- modules/key_value/pkg/metadata.json | 3 ++- modules/orgs/pkg/metadata.json | 3 ++- modules/qns_indexer/pkg/metadata.json | 3 ++- modules/terminal/pkg/metadata.json | 3 ++- 8 files changed, 15 insertions(+), 7 deletions(-) diff --git a/modules/app_store/pkg/metadata.json b/modules/app_store/pkg/metadata.json index cbb01c6e..912510ce 100644 --- a/modules/app_store/pkg/metadata.json +++ b/modules/app_store/pkg/metadata.json @@ -1,5 +1,6 @@ { "package": "app_store", "publisher": "uqbar", + "version": [0, 1, 0], "description": "A package manager + app store. This JSON field is optional and you can add whatever you want in addition to this." } diff --git a/modules/chess/pkg/metadata.json b/modules/chess/pkg/metadata.json index be7718ff..f833a906 100644 --- a/modules/chess/pkg/metadata.json +++ b/modules/chess/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "chess", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/homepage/pkg/metadata.json b/modules/homepage/pkg/metadata.json index 24843095..5f440264 100644 --- a/modules/homepage/pkg/metadata.json +++ b/modules/homepage/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "homepage", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/http_proxy/pkg/metadata.json b/modules/http_proxy/pkg/metadata.json index 6acd4d15..8fc0ad14 100644 --- a/modules/http_proxy/pkg/metadata.json +++ b/modules/http_proxy/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "http_proxy", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/key_value/pkg/metadata.json b/modules/key_value/pkg/metadata.json index 7db01e9b..549365f7 100644 --- a/modules/key_value/pkg/metadata.json +++ b/modules/key_value/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "key_value", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/orgs/pkg/metadata.json b/modules/orgs/pkg/metadata.json index b0d2380e..43e641c7 100644 --- a/modules/orgs/pkg/metadata.json +++ b/modules/orgs/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "orgs", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/qns_indexer/pkg/metadata.json b/modules/qns_indexer/pkg/metadata.json index 1d6a68e8..1a50d51f 100644 --- a/modules/qns_indexer/pkg/metadata.json +++ b/modules/qns_indexer/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "qns_indexer", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } diff --git a/modules/terminal/pkg/metadata.json b/modules/terminal/pkg/metadata.json index b1a7dbff..5d8539c8 100644 --- a/modules/terminal/pkg/metadata.json +++ b/modules/terminal/pkg/metadata.json @@ -1,4 +1,5 @@ { "package": "terminal", - "publisher": "uqbar" + "publisher": "uqbar", + "version": [0, 1, 0] } From 8e36d7776abdec448856b0f011fb4a061829f3f4 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 17:26:33 -0400 Subject: [PATCH 21/24] small fixes to kv; build.rs working much better now --- build-app.sh | 37 ----------- build.rs | 66 +++++-------------- modules/key_value/key_value/src/lib.rs | 1 + modules/key_value/key_value_worker/src/lib.rs | 6 +- 4 files changed, 21 insertions(+), 89 deletions(-) delete mode 100755 build-app.sh diff --git a/build-app.sh b/build-app.sh deleted file mode 100755 index c71d4239..00000000 --- a/build-app.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -debug_flag="--release" - -# Grab the full path to the target -target_path="$1" -name=$(basename "$target_path") - -if [[ "$2" == "--debug" ]]; then - debug_flag="" -fi - -pwd=$(pwd) - -rm -rf "$target_path/wit" || { echo "Command failed"; exit 1; } -cp -r wit "$target_path" || { echo "Command failed"; exit 1; } -mkdir -p "$target_path/target/bindings/$name" || { echo "Command failed"; exit 1; } - -cp target.wasm "$target_path/target/bindings/$name/" || { echo "Command failed"; exit 1; } -cp world "$target_path/target/bindings/$name/" || { echo "Command failed"; exit 1; } - -mkdir -p "$target_path/target/wasm32-unknown-unknown/release" || { echo "Command failed"; exit 1; } - -# Build the module using Cargo -cargo +nightly build \ - $debug_flag \ - --no-default-features \ - --manifest-path="$target_path/Cargo.toml" \ - --target "wasm32-wasi" || { - echo "Command failed"; exit 1; - } - -# Adapt the module using wasm-tools -wasm-tools component new "$target_path/target/wasm32-wasi/release/$name.wasm" -o "$target_path/target/wasm32-wasi/release/${name}_adapted.wasm" --adapt "$pwd/wasi_snapshot_preview1.wasm" || { echo "Command failed"; exit 1; } - -# Embed "wit" into the component and place it in the expected location -wasm-tools component embed wit --world uq-process "$target_path/target/wasm32-wasi/release/${name}_adapted.wasm" -o "$target_path/target/wasm32-unknown-unknown/release/$name.wasm" || { echo "Command failed"; exit 1; } diff --git a/build.rs b/build.rs index d2734d36..63145cb9 100644 --- a/build.rs +++ b/build.rs @@ -36,16 +36,17 @@ where fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { let pwd = std::env::current_dir().unwrap(); - println!("cargo:warning=building {}", target_path); + let start = std::time::Instant::now(); - // Copy in newly-made wit IF old one is outdated + // if and only if module's wit is outdated, re-set-up build environment if file_outdated( - format!("{}/wit/", pwd.display()), - format!("{}/modules/{}/wit/", target_path, name), + format!("{}/wit/uqbar.wit", pwd.display()), + format!("{}/wit/uqbar.wit", target_path), ) .unwrap_or(true) { - run_command(Command::new("cp").args(&["-r", "wit", target_path])).unwrap(); + println!("cargo:warning=wit outdated, rebuilding"); + run_command(Command::new("cp").args(&["wit/uqbar.wit", &format!("{}/wit", target_path)])).unwrap(); // create target/bindings directory fs::create_dir_all(&format!("{}/target/bindings/{}", target_path, name,)).unwrap(); // copy newly-made target.wasm into target/bindings @@ -63,6 +64,7 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { } // Build the module targeting wasm32-wasi run_command(Command::new("cargo").args(&[ + "+nightly", "build", "--release", "--no-default-features", @@ -71,6 +73,7 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { "wasm32-wasi", ])) .unwrap(); + // Adapt module to component with adapter based on wasi_snapshot_preview1.wasm run_command(Command::new("wasm-tools").args(&[ "component", @@ -110,6 +113,13 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { &wasm_dest_path, ])) .unwrap(); + + let end = std::time::Instant::now(); + println!( + "cargo:warning=building {} took {:?}", + target_path, + end.duration_since(start) + ); } fn main() { @@ -117,47 +127,6 @@ fn main() { println!("Skipping build script"); return; } - let build_enabled = std::env::var("BUILD_APPS") - .map(|v| v == "true") - .unwrap_or(true); // run by default - - if !build_enabled { - return; - } - // only execute if one of the modules has source code changes - const WASI_APPS: [&str; 7] = [ - "app_store", - "chess", - "homepage", - "http_proxy", - "orgs", - "qns_indexer", - "terminal", - ]; - // NOT YET building KV, waiting for deps to be ready - const NESTED_WASI_APPS: [(&str, &str); 2] = [ - ("key_value", "key_value"), - ("key_value", "key_value_worker"), - ]; - - if std::env::var("REBUILD_ALL").is_ok() { - } else { - for name in &WASI_APPS { - println!("cargo:rerun-if-changed=modules/{}/src", name); - println!("cargo:rerun-if-changed=modules/{}/Cargo.toml", name); - println!("cargo:rerun-if-changed=modules/{}/pkg/manifest.json", name); - println!("cargo:rerun-if-changed=modules/{}/pkg/metadata.json", name); - } - for (outer, inner) in &NESTED_WASI_APPS { - println!("cargo:rerun-if-changed=modules/{}/{}/src", outer, inner); - println!( - "cargo:rerun-if-changed=modules/{}/{}/Cargo.toml", - outer, inner - ); - println!("cargo:rerun-if-changed=modules/{}/pkg/manifest.json", outer); - println!("cargo:rerun-if-changed=modules/{}/pkg/metadata.json", outer); - } - } let pwd = std::env::current_dir().unwrap(); // Create target.wasm (compiled .wit) & world @@ -178,11 +147,6 @@ fn main() { let entry_path = entry.unwrap().path(); let package_name = entry_path.file_name().unwrap().to_str().unwrap(); - // NOT YET building KV, waiting for deps to be ready - if package_name == "key_value" { - continue; - } - // If Cargo.toml is present, build the app let parent_pkg_path = format!("{}/pkg", entry_path.display()); if entry_path.join("Cargo.toml").exists() { diff --git a/modules/key_value/key_value/src/lib.rs b/modules/key_value/key_value/src/lib.rs index 4990355b..51946f80 100644 --- a/modules/key_value/key_value/src/lib.rs +++ b/modules/key_value/key_value/src/lib.rs @@ -191,6 +191,7 @@ impl Guest for Component { if let Some(e) = e.downcast_ref::() { send_response( &Response { + inherit: false, ipc: Some(serde_json::to_string(&e).unwrap()), metadata: None, }, diff --git a/modules/key_value/key_value_worker/src/lib.rs b/modules/key_value/key_value_worker/src/lib.rs index 2181a05b..7c892440 100644 --- a/modules/key_value/key_value_worker/src/lib.rs +++ b/modules/key_value/key_value_worker/src/lib.rs @@ -42,7 +42,7 @@ fn send_and_await_response_wrapped( }; let ( _, - Message::Response((Response { ipc, metadata }, _)), + Message::Response((Response { ipc, metadata, .. }, _)), ) = send_and_await_response( &Address { node: target_node, @@ -118,6 +118,7 @@ fn handle_message ( send_response( &Response { + inherit: false, ipc, metadata: None, }, @@ -137,6 +138,7 @@ fn handle_message ( None => { send_response( &Response { + inherit: false, ipc, metadata: None, }, @@ -155,6 +157,7 @@ fn handle_message ( ); send_response( &Response { + inherit: false, ipc, metadata: None, }, @@ -193,6 +196,7 @@ impl Guest for Component { if let Some(e) = e.downcast_ref::() { send_response( &Response { + inherit: false, ipc: Some(serde_json::to_string(&e).unwrap()), metadata: None, }, From b5f57d3e710cf5cb32eb661053b9651b29a144f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 21:26:54 +0000 Subject: [PATCH 22/24] Format Rust code using rustfmt --- build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.rs b/build.rs index 63145cb9..260636ff 100644 --- a/build.rs +++ b/build.rs @@ -46,7 +46,8 @@ fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) { .unwrap_or(true) { println!("cargo:warning=wit outdated, rebuilding"); - run_command(Command::new("cp").args(&["wit/uqbar.wit", &format!("{}/wit", target_path)])).unwrap(); + run_command(Command::new("cp").args(&["wit/uqbar.wit", &format!("{}/wit", target_path)])) + .unwrap(); // create target/bindings directory fs::create_dir_all(&format!("{}/target/bindings/{}", target_path, name,)).unwrap(); // copy newly-made target.wasm into target/bindings From 3ef3fea473c0dbf0589c3ee5cf307dce32eff850 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 20:23:04 -0400 Subject: [PATCH 23/24] match cargo.toml in main --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 90557040..dedbc8b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ aes-gcm = "0.10.2" anyhow = "1.0.71" async-recursion = "1.0.4" async-trait = "0.1.71" -base64 = "0.13.0" bincode = "1.3.3" blake3 = "1.4.1" bytes = "1.4.0" @@ -48,8 +47,8 @@ public-ip = "0.2.2" rand = "0.8.4" reqwest = "0.11.18" ring = "0.16.20" -route-recognizer = "0.3.1" rsa = "0.9" +route-recognizer = "0.3.1" rusoto_core = "0.48.0" rusoto_s3 = "0.48.0" rusoto_credential = "0.48.0" @@ -66,3 +65,4 @@ warp = "0.3.5" wasmtime = "12.0.1" wasmtime-wasi = "12.0.1" zip = "0.6" +base64 = "0.13" \ No newline at end of file From 47f7f5eecbfabd23421375f3772f692298e82002 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Tue, 24 Oct 2023 20:39:44 -0400 Subject: [PATCH 24/24] remove register app build from build.rs (it's too slow!) and add it to readme instead --- README.md | 6 ++---- build.rs | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bc4c0801..bbff744c 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,12 @@ cargo install --git https://github.com/bytecodealliance/cargo-component --locked # Initialize submodules, in particular the register app git submodule update --init --recursive +# Build the register app +cd src/register && ./build_all.sh && cd ../.. # Build the runtime, along with a number of booted-at-startup WASM modules including terminal and key_value # OPTIONAL: --release flag cargo +nightly build --release - -# Create the home directory for your node -# If you boot multiple nodes, make sure each has their own home directory. -mkdir home ``` ### Boot diff --git a/build.rs b/build.rs index 4d663c9d..6b47e39f 100644 --- a/build.rs +++ b/build.rs @@ -131,9 +131,6 @@ fn main() { let pwd = std::env::current_dir().unwrap(); - // build the register app - run_command(Command::new("./build_all.sh").current_dir("src/register")).unwrap(); - // create target.wasm (compiled .wit) & world run_command(Command::new("wasm-tools").args(&[ "component",