WIP: use ft_worker in app_store

This commit is contained in:
dr-frmr 2023-10-19 16:24:05 -04:00
parent ca724671e3
commit 4aa7cc9f9e
No known key found for this signature in database
16 changed files with 909 additions and 232 deletions

View File

@ -0,0 +1 @@
../../ft_worker/src/ft_worker_lib.rs

View File

@ -0,0 +1 @@
../../../../src/kernel_types.rs

View File

@ -15,7 +15,9 @@ use kernel_types as kt;
mod process_lib;
use process_lib::PackageId;
mod transfer_lib;
#[allow(dead_code)]
mod ft_worker_lib;
use ft_worker_lib::*;
struct Component;

View File

@ -0,0 +1 @@
../../../../src/process_lib.rs

514
modules/app_store/ft_worker/Cargo.lock generated Normal file
View File

@ -0,0 +1,514 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "anyhow"
version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "cargo-component-bindings"
version = "0.2.0"
source = "git+https://github.com/bytecodealliance/cargo-component#2c45278720196778704f5baf62066d355d48f702"
dependencies = [
"cargo-component-macro",
"wit-bindgen 0.12.0",
]
[[package]]
name = "cargo-component-macro"
version = "0.2.0"
source = "git+https://github.com/bytecodealliance/cargo-component#2c45278720196778704f5baf62066d355d48f702"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
"wit-component",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "form_urlencoded"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
dependencies = [
"percent-encoding",
]
[[package]]
name = "ft_worker"
version = "0.1.0"
dependencies = [
"anyhow",
"bincode",
"cargo-component-bindings",
"rand",
"serde",
"serde_json",
"wit-bindgen 0.9.0",
]
[[package]]
name = "getrandom"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "hashbrown"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
[[package]]
name = "heck"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "id-arena"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005"
[[package]]
name = "idna"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"
dependencies = [
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "indexmap"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
dependencies = [
"equivalent",
"hashbrown",
"serde",
]
[[package]]
name = "itoa"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "leb128"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
[[package]]
name = "libc"
version = "0.2.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
[[package]]
name = "log"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "memchr"
version = "2.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
[[package]]
name = "percent-encoding"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
[[package]]
name = "ppv-lite86"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "proc-macro2"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
dependencies = [
"unicode-ident",
]
[[package]]
name = "pulldown-cmark"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998"
dependencies = [
"bitflags 1.3.2",
"memchr",
"unicase",
]
[[package]]
name = "quote"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "ryu"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "semver"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
[[package]]
name = "serde"
version = "1.0.188"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.188"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "smallvec"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
[[package]]
name = "spdx"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71"
dependencies = [
"smallvec",
]
[[package]]
name = "syn"
version = "2.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tinyvec"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "unicase"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-normalization"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-xid"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "url"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
]
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-encoder"
version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34180c89672b3e4825c3a8db4b61a674f1447afd5fe2445b2d22c3d8b6ea086c"
dependencies = [
"leb128",
]
[[package]]
name = "wasm-metadata"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f2c051ef041d348324b01ff0419f6f6593f094b4897d93c9cf52d5d1ac879ba"
dependencies = [
"anyhow",
"indexmap",
"serde",
"serde_json",
"spdx",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasmparser"
version = "0.113.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "286049849b5a5bd09a8773171be96824afabffc7cc3df6caaf33a38db6cd07ae"
dependencies = [
"indexmap",
"semver",
]
[[package]]
name = "wit-bindgen"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5c3d15a04ce994fad2c5442a754b404ab1fee23c903a04a560f84f94fdf63c0"
dependencies = [
"bitflags 2.4.0",
]
[[package]]
name = "wit-bindgen"
version = "0.12.0"
source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a"
dependencies = [
"bitflags 2.4.0",
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen-core"
version = "0.12.0"
source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a"
dependencies = [
"anyhow",
"wit-component",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.12.0"
source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a"
dependencies = [
"anyhow",
"heck",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.12.0"
source = "git+https://github.com/bytecodealliance/wit-bindgen.git#6de9afcee29170f721c5d9f34dce87d0852c053a"
dependencies = [
"anyhow",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
"wit-component",
]
[[package]]
name = "wit-component"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e2bf941487fc5afa9e3fc94761f6b80ecef5a2bed6239b959d23d9de69e3448"
dependencies = [
"anyhow",
"bitflags 2.4.0",
"indexmap",
"log",
"serde",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a39edca9abb16309def3843af73b58d47d243fe33a9ceee572446bcc57556b9a"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"pulldown-cmark",
"semver",
"serde",
"serde_json",
"unicode-xid",
"url",
]

View File

@ -0,0 +1,31 @@
[package]
name = "ft_worker"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[profile.release]
panic = "abort"
opt-level = "s"
lto = true
[dependencies]
anyhow = "1.0"
bincode = "1.3.3"
cargo-component-bindings = { git = "https://github.com/bytecodealliance/cargo-component" }
rand = "0.8"
serde = {version = "1.0", features = ["derive"] }
serde_json = "1.0"
wit-bindgen = { version = "0.9.0", default_features = false }
[lib]
crate-type = ["cdylib"]
[package.metadata.component]
package = "component:microkernel-process"
[package.metadata.component.target]
path = "wit"
[package.metadata.component.dependencies]

View File

@ -0,0 +1,130 @@
use super::bindings::component::uq_process::types::*;
use super::bindings::{Address, Payload, print_to_terminal, send_request, spawn};
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize)]
pub struct FileTransferContext {
pub file_name: String,
pub file_size: u64,
pub start_time: std::time::SystemTime,
}
/// sent as first Request to a newly spawned worker
/// the Receive command will be sent out to target
/// in order to prompt them to spawn a worker
#[derive(Debug, Serialize, Deserialize)]
pub enum FTWorkerCommand {
Send {
// make sure to attach file itself as payload
target: String, // annoying, but this is Address
file_name: String,
timeout: u64,
},
Receive {
transfer_id: u64,
file_name: String,
file_size: u64,
total_chunks: u64,
timeout: u64,
},
}
/// sent as Response by worker to its parent
#[derive(Debug, Serialize, Deserialize)]
pub enum FTWorkerResult {
SendSuccess,
ReceiveSuccess(String), // name of file, bytes in payload
Err(TransferError),
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TransferError {
TargetOffline,
TargetTimeout,
TargetRejected,
SourceFailed,
}
pub fn spawn_transfer(
our: &Address,
file_name: &str,
file_bytes: Vec<u8>,
to_addr: &Address,
) {
let transfer_id: u64 = rand::random();
// spawn a worker and tell it to send the file
let Ok(worker_process_id) = spawn(
Some(&transfer_id.to_string()),
"/ft_worker.wasm".into(),
&OnPanic::None, // can set message-on-panic here
&Capabilities::All,
false, // not public
) else {
print_to_terminal(0, "file_transfer: failed to spawn worker!");
return;
};
// tell the worker what to do
send_request(
&Address {
node: our.node.clone(),
process: worker_process_id,
},
&Request {
inherit: false,
expects_response: Some(61),
ipc: Some(
serde_json::to_string(&FTWorkerCommand::Send {
target: to_addr.to_string(),
file_name: file_name.into(),
timeout: 60,
})
.unwrap(),
),
metadata: None,
},
Some(
&serde_json::to_string(&FileTransferContext {
file_name: file_name.into(),
file_size: file_bytes.len() as u64,
start_time: std::time::SystemTime::now(),
})
.unwrap(),
),
Some(&Payload { mime: None, bytes: file_bytes }),
);
}
pub fn spawn_receive_transfer(
our: &Address,
ipc: &str,
) {
let Ok(FTWorkerCommand::Receive { transfer_id, .. }) = serde_json::from_str(ipc) else {
print_to_terminal(0, "file_transfer: got weird request");
return;
};
let Ok(worker_process_id) = spawn(
Some(&transfer_id.to_string()),
"/ft_worker.wasm".into(),
&OnPanic::None, // can set message-on-panic here
&Capabilities::All,
false, // not public
) else {
print_to_terminal(0, "file_transfer: failed to spawn worker!");
return;
};
// forward receive command to worker
send_request(
&Address {
node: our.node.clone(),
process: worker_process_id,
},
&Request {
inherit: true,
expects_response: None,
ipc: Some(ipc.to_string()),
metadata: None,
},
None,
None,
);
}

View File

@ -0,0 +1,226 @@
cargo_component_bindings::generate!();
use bindings::component::uq_process::types::*;
use bindings::{get_payload, print_to_terminal, receive, send_request, send_response, Guest};
use serde::{Deserialize, Serialize};
struct Component;
mod ft_worker_lib;
#[allow(dead_code)]
mod process_lib;
use ft_worker_lib::*;
/// internal worker protocol
#[derive(Debug, Serialize, Deserialize)]
pub enum FTWorkerProtocol {
Ready,
Finished,
}
impl Guest for Component {
fn init(our: Address) {
print_to_terminal(1, &format!("{}: start", our.process));
let Ok((parent_process, Message::Request(req))) = receive() else {
panic!("ft_worker: got bad init message");
};
let command = serde_json::from_str::<FTWorkerCommand>(
&req.ipc.expect("ft_worker: got empty init message"),
)
.expect("ft_worker: got unparseable init message");
match command {
FTWorkerCommand::Send {
target,
file_name,
timeout,
} => {
let transfer_id: u64 = our.process.process().parse().unwrap();
let Some(payload) = get_payload() else {
panic!("ft_worker: got empty payload");
};
let file_bytes = payload.bytes;
let mut file_size = file_bytes.len() as u64;
let mut offset: u64 = 0;
let mut chunk_size: u64 = 1048576; // 1MB
let total_chunks = (file_size as f64 / chunk_size as f64).ceil() as u64;
// send a file to another worker
// start by telling target to expect a file,
// then upon reciving affirmative response,
// send contents in chunks and wait for
// acknowledgement.
match bindings::send_and_await_response(
&Address::from_str(&target).unwrap(),
&Request {
inherit: false,
expects_response: Some(timeout),
ipc: Some(
serde_json::to_string(&FTWorkerCommand::Receive {
transfer_id,
file_name,
file_size,
total_chunks,
timeout,
})
.unwrap(),
),
metadata: None,
},
None,
) {
Err(send_error) => {
respond_to_parent(FTWorkerResult::Err(match send_error.kind {
SendErrorKind::Offline => TransferError::TargetOffline,
SendErrorKind::Timeout => TransferError::TargetTimeout,
}))
}
Ok((opp_worker, Message::Response((response, _)))) => {
let Ok(FTWorkerProtocol::Ready) = serde_json::from_str(&response.ipc.expect("ft_worker: got empty response")) else {
respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected));
return;
};
// send file in chunks
loop {
if file_size < chunk_size {
// this is the last chunk, so we should expect a Finished response
chunk_size = file_size;
let payload = Payload {
mime: None,
bytes: file_bytes
[offset as usize..offset as usize + chunk_size as usize]
.to_vec(),
};
send_request(
&opp_worker,
&Request {
inherit: false,
expects_response: Some(timeout),
ipc: None,
metadata: None,
},
None,
Some(&payload),
);
break;
}
let payload = Payload {
mime: None,
bytes: file_bytes
[offset as usize..offset as usize + chunk_size as usize]
.to_vec(),
};
send_request(
&opp_worker,
&Request {
inherit: false,
expects_response: None,
ipc: None,
metadata: None,
},
None,
Some(&payload),
);
file_size -= chunk_size;
offset += chunk_size;
}
// now wait for Finished response
let Ok((receiving_worker, Message::Response((resp, _)))) = receive() else {
respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected));
return;
};
let Ok(FTWorkerProtocol::Finished) = serde_json::from_str(
&resp.ipc.expect("ft_worker: got empty response"),
) else {
respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected));
return;
};
// return success to parent
respond_to_parent(FTWorkerResult::SendSuccess);
}
_ => respond_to_parent(FTWorkerResult::Err(TransferError::TargetRejected)),
}
}
FTWorkerCommand::Receive {
transfer_id,
file_name,
file_size,
total_chunks,
timeout,
} => {
// send Ready response to counterparty
send_response(
&Response {
inherit: false,
ipc: Some(serde_json::to_string(&FTWorkerProtocol::Ready).unwrap()),
metadata: None,
},
None,
);
// receive a file from a worker, then send it to parent
// all messages will be chunks of file. when we receive the
// last chunk, send a Finished message to sender and Success to parent.
let mut file_bytes = Vec::new();
let mut chunks_received = 0;
let start_time = std::time::Instant::now();
loop {
let Ok((source, Message::Request(req))) = receive() else {
respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed));
return;
};
if start_time.elapsed().as_secs() > timeout {
respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed));
return;
}
let Some(payload) = get_payload() else {
respond_to_parent(FTWorkerResult::Err(TransferError::SourceFailed));
return;
};
chunks_received += 1;
file_bytes.extend(payload.bytes);
if chunks_received == total_chunks {
break;
}
}
// send Finished message to sender
send_response(
&Response {
inherit: false,
ipc: Some(serde_json::to_string(&FTWorkerProtocol::Finished).unwrap()),
metadata: None,
},
None,
);
// send Success message to parent
send_request(
&parent_process,
&Request {
inherit: false,
expects_response: None,
ipc: Some(
serde_json::to_string(&FTWorkerResult::ReceiveSuccess(file_name))
.unwrap(),
),
metadata: None,
},
None,
Some(&Payload {
mime: None,
bytes: file_bytes,
}),
);
}
}
}
}
fn respond_to_parent(result: FTWorkerResult) {
send_response(
&Response {
inherit: false,
ipc: Some(serde_json::to_string(&result).unwrap()),
metadata: None,
},
None,
);
}

View File

@ -0,0 +1 @@
../../../../src/process_lib.rs

View File

@ -1 +0,0 @@
../../../src/kernel_types.rs

View File

@ -1 +0,0 @@
../../../src/process_lib.rs

View File

@ -1,227 +0,0 @@
use super::bindings::component::uq_process::types::*;
use crate::bindings::{get_payload, receive, send_request, send_response};
use serde::{Deserialize, Serialize};
#[derive(Debug)]
pub enum TransferError {
// in all errors, u64 is number of bytes successfully transferred
TargetOffline(u64),
TargetTimeout(u64),
TargetRejected(u64),
SourceFailed(u64),
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TransferMetadata {
Begin {
file_name: String,
file_size: u64,
total_chunks: u64,
},
}
pub fn transfer(
to_addr: Address,
bytes: Vec<u8>,
max_timeout: u64,
) -> (
Result<(), TransferError>,
Vec<Result<(Address, Message), (SendError, Option<Context>)>>,
) {
let transfer_context_id: u64 = rand::random();
let mut bytes_remaining: u64 = bytes.len() as u64;
let mut offset: u64 = 0;
let mut chunk_size: u64 = 1048576; // 1MB
let mut chunks_sent = 0;
let total_chunks = (bytes.len() as f64 / chunk_size as f64).ceil() as u64;
loop {
chunks_sent += 1;
if bytes_remaining < chunk_size {
chunk_size = bytes_remaining;
}
let payload = Payload {
mime: None,
bytes: bytes[offset as usize..offset as usize + chunk_size as usize].to_vec(),
};
send_request(
&to_addr,
&Request {
inherit: false,
expects_response: Some(max_timeout),
ipc: None,
metadata: Some(if chunks_sent == 1 {
serde_json::to_string(&TransferMetadata::Begin {
file_name: "test".to_string(),
file_size: bytes.len() as u64,
total_chunks,
})
.unwrap()
} else {
chunks_sent.to_string()
}),
},
Some(&&transfer_context_id.to_string()),
Some(&payload),
);
bytes_remaining -= chunk_size;
offset += chunk_size;
if bytes_remaining == 0 {
break;
}
}
let mut chunks_confirmed = 0;
let mut non_transfer_message_queue = Vec::new();
loop {
let next = receive();
if let Err((send_error, context)) = &next {
match context {
Some(_) => match send_error.kind {
SendErrorKind::Offline => {
return (
Err(TransferError::TargetOffline(chunks_confirmed * chunk_size)),
non_transfer_message_queue,
)
}
SendErrorKind::Timeout => {
return (
Err(TransferError::TargetTimeout(chunks_confirmed * chunk_size)),
non_transfer_message_queue,
)
}
},
None => {
non_transfer_message_queue.push(next);
continue;
}
}
}
if let Ok((source, message)) = &next {
if source.process == to_addr.process {
match message {
Message::Request(_) => {
non_transfer_message_queue.push(next);
continue;
}
Message::Response((response, context)) => {
if transfer_context_id
== context
.as_ref()
.unwrap_or(&"".into())
.parse::<u64>()
.unwrap_or(0)
{
chunks_confirmed += 1;
if response
.metadata
.as_ref()
.unwrap_or(&"".into())
.parse::<u64>()
.unwrap_or(0)
!= chunks_confirmed
{
return (
Err(TransferError::TargetRejected(
chunks_confirmed * chunk_size,
)),
non_transfer_message_queue,
);
}
if chunks_confirmed == chunks_sent {
return (Ok(()), non_transfer_message_queue);
}
} else {
non_transfer_message_queue.push(next);
}
}
}
} else {
non_transfer_message_queue.push(next);
continue;
}
}
}
}
pub fn receive_transfer(
transfer_source: Address,
total_chunks: u64,
max_timeout: u64,
) -> (
Result<Vec<u8>, TransferError>,
Vec<Result<(Address, Message), (SendError, Option<Context>)>>,
) {
let start_time = std::time::SystemTime::now();
// get first payload then loop and receive rest
let mut file = match get_payload() {
Some(payload) => payload.bytes,
None => {
return (Err(TransferError::SourceFailed(0)), vec![]);
}
};
// respond to first request
send_response(
&Response {
inherit: false,
ipc: None,
metadata: Some(1.to_string()),
},
None,
);
if total_chunks == 1 {
return (Ok(file), vec![]);
}
let mut chunk_num = 1;
let mut non_transfer_message_queue = Vec::new();
loop {
let next = receive();
if start_time.elapsed().expect("time error").as_secs() > max_timeout {
return (
Err(TransferError::TargetTimeout(file.len() as u64)),
non_transfer_message_queue,
);
}
if let Err(_) = &next {
non_transfer_message_queue.push(next);
} else if let Ok((source, message)) = &next {
// we know all messages from source process will be for this transfer,
// since they are sent sequentially and it's a single-file queue.
if source.process == transfer_source.process {
match message {
Message::Request(_) => {
let payload = match get_payload() {
Some(payload) => payload,
None => {
return (
Err(TransferError::SourceFailed(file.len() as u64)),
non_transfer_message_queue,
);
}
};
chunk_num += 1;
file.extend(payload.bytes);
send_response(
&Response {
inherit: false,
ipc: None,
metadata: Some(chunk_num.to_string()),
},
None,
);
if chunk_num == total_chunks {
return (Ok(file), non_transfer_message_queue);
}
}
Message::Response(_) => {
return (
Err(TransferError::SourceFailed(file.len() as u64)),
non_transfer_message_queue,
);
}
}
} else {
non_transfer_message_queue.push(next);
continue;
}
}
}
}

View File

@ -1059,7 +1059,6 @@ async fn persist_state(
process_map: &t::ProcessMap,
) -> Result<()> {
let bytes = bincode::serialize(process_map)?;
send_to_loop
.send(t::KernelMessage {
id: rand::random(),
@ -1074,7 +1073,7 @@ async fn persist_state(
rsvp: None,
message: t::Message::Request(t::Request {
inherit: true,
expects_response: Some(5), // TODO evaluate
expects_response: None,
ipc: Some(
serde_json::to_string(&t::FsAction::SetState(KERNEL_PROCESS_ID.clone()))
.unwrap(),