uqbar-dao/nectar
1
1
Fork 0
mirror of https://github.com/uqbar-dao/nectar.git synced 2024-11-23 03:44:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

add kernel fixes & update key_value

Browse Source
This commit is contained in:
hosted-fornet 2023-10-09 16:31:49 -07:00
parent e4cfdea148
commit 08b34ad194
16 changed files with 222 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

269
build.rs
View File

@ -33,6 +33,95 @@ where
}
}
fn build_app(target_path: &str, name: &str, parent_pkg_path: Option<&str>) {
let pwd = std::env::current_dir().unwrap();
// Copy in newly-made wit IF old one is outdated
if file_outdated(
format!("{}/wit/", pwd.display()),
format!("{}/modules/{}/wit/", target_path, name),
)
.unwrap_or(true)
{
run_command(Command::new("cp").args(&[
"-r",
"wit",
target_path,
]))
.unwrap();
// create target/bindings directory
fs::create_dir_all(&format!(
"{}/target/bindings/{}",
target_path,
name,
))
.unwrap();
// copy newly-made target.wasm into target/bindings
run_command(Command::new("cp").args(&[
"target.wasm",
&format!(
"{}/target/bindings/{}/",
target_path,
name,
),
]))
.unwrap();
// copy newly-made world into target/bindings
run_command(Command::new("cp").args(&[
"world",
&format!(
"{}/target/bindings/{}/",
target_path,
name,
),
]))
.unwrap();
}
// Build the module targeting wasm32-wasi
run_command(Command::new("cargo").args(&[
"build",
"--release",
"--no-default-features",
&format!("--manifest-path={}/Cargo.toml", target_path),
"--target",
"wasm32-wasi",
]))
.unwrap();
// Adapt module to component with adapter based on wasi_snapshot_preview1.wasm
run_command(Command::new("wasm-tools").args(&[
"component",
"new",
&format!("{}/target/wasm32-wasi/release/{}.wasm", target_path, name),
"-o",
&format!("{}/target/wasm32-wasi/release/{}_adapted.wasm", target_path, name),
"--adapt",
&format!("{}/wasi_snapshot_preview1.wasm", pwd.display()),
]))
.unwrap();
// Determine the destination for the .wasm file after embedding wit
let wasm_dest_path = if let Some(parent_pkg) = parent_pkg_path {
format!("{}/{}.wasm", parent_pkg, name)
} else {
let pkg_folder = format!("{}/pkg/", target_path);
let _ = run_command(Command::new("mkdir").args(&["-p", &pkg_folder]));
format!("{}/{}.wasm", pkg_folder, name)
};
// Embed "wit" into the component
run_command(Command::new("wasm-tools").args(&[
"component",
"embed",
"wit",
"--world",
"uq-process",
&format!("{}/target/wasm32-wasi/release/{}_adapted.wasm", target_path, name),
"-o",
&wasm_dest_path,
]))
.unwrap();
}
fn main() {
if std::env::var("SKIP_BUILD_SCRIPT").is_ok() {
println!("Skipping build script");
@ -57,15 +146,17 @@ fn main() {
"rpc",
"terminal",
];
for name in WASI_APPS {
println!("cargo:rerun-if-changed=modules/{}/src", name);
println!("cargo:rerun-if-changed=modules/{}/pkg/manifest.json", name);
println!("cargo:rerun-if-changed=modules/{}/pkg/metadata.json", name);
if std::env::var("REBUILD_ALL").is_ok() {} else {
for name in &WASI_APPS {
println!("cargo:rerun-if-changed=modules/{}/src", name);
println!("cargo:rerun-if-changed=modules/{}/pkg/manifest.json", name);
println!("cargo:rerun-if-changed=modules/{}/pkg/metadata.json", name);
}
}
let pwd = std::env::current_dir().unwrap();
// create target.wasm (compiled .wit) & world
// Create target.wasm (compiled .wit) & world
run_command(Command::new("wasm-tools").args(&[
"component",
"wit",
@ -78,133 +169,49 @@ fn main() {
run_command(Command::new("touch").args(&[&format!("{}/world", pwd.display())])).unwrap();
// Build wasm32-wasi apps.
for name in WASI_APPS {
// copy in newly-made wit IF old one is outdated
if file_outdated(
format!("{}/wit/", pwd.display()),
format!("{}/modules/{}/wit/", pwd.display(), name),
)
.unwrap_or(true)
{
run_command(Command::new("cp").args(&[
"-r",
"wit",
&format!("{}/modules/{}", pwd.display(), name),
]))
.unwrap();
// create target/bindings directory
fs::create_dir_all(&format!(
"{}/modules/{}/target/bindings/{}",
pwd.display(),
name,
name
))
.unwrap();
// copy newly-made target.wasm into target/bindings
run_command(Command::new("cp").args(&[
"target.wasm",
&format!(
"{}/modules/{}/target/bindings/{}/",
pwd.display(),
name,
name
),
]))
.unwrap();
// copy newly-made world into target/bindings
run_command(Command::new("cp").args(&[
"world",
&format!(
"{}/modules/{}/target/bindings/{}/",
pwd.display(),
name,
name
),
]))
.unwrap();
}
// build the module targeting wasm32-wasi
run_command(Command::new("cargo").args(&[
"build",
"--release",
"--no-default-features",
&format!(
"--manifest-path={}/modules/{}/Cargo.toml",
pwd.display(),
name
),
"--target",
"wasm32-wasi",
]))
.unwrap();
// adapt module to component with adapter based on wasi_snapshot_preview1.wasm
run_command(Command::new("wasm-tools").args(&[
"component",
"new",
&format!(
"{}/modules/{}/target/wasm32-wasi/release/{}.wasm",
pwd.display(),
name,
name
),
"-o",
&format!(
"{}/modules/{}/target/wasm32-wasi/release/{}_adapted.wasm",
pwd.display(),
name,
name
),
"--adapt",
&format!("{}/wasi_snapshot_preview1.wasm", pwd.display()),
]))
.unwrap();
// put wit into component & place final .wasm in /pkg
let pkg_folder = format!("{}/modules/{}/pkg/", pwd.display(), name);
let _ = run_command(Command::new("mkdir").args(&["-p", &pkg_folder]));
run_command(Command::new("wasm-tools").args(&[
"component",
"embed",
"wit",
"--world",
"uq-process",
&format!(
"{}/modules/{}/target/wasm32-wasi/release/{}_adapted.wasm",
pwd.display(),
name,
name
),
"-o",
&format!("{}/{}.wasm", pkg_folder, name),
]))
.unwrap();
// from the pkg folder, create a zip archive and save in target directory
let writer =
std::fs::File::create(format!("{}/target/{}.zip", pwd.display(), name)).unwrap();
let options = zip::write::FileOptions::default()
.compression_method(zip::CompressionMethod::Stored) // or CompressionMethod::Deflated
.unix_permissions(0o755);
let mut zip = zip::ZipWriter::new(writer);
let modules_dir = format!("{}/modules", pwd.display());
for entry in std::fs::read_dir(&modules_dir).unwrap() {
let entry_path = entry.unwrap().path();
// If Cargo.toml is present, build the app
if entry_path.join("Cargo.toml").exists() {
build_app(&entry_path.display().to_string(), &entry_path.file_name().unwrap().to_str().unwrap(), None);
} else if entry_path.is_dir() {
let parent_pkg_path = format!("{}/pkg", entry_path.display());
fs::create_dir_all(&parent_pkg_path).unwrap();
for entry in walkdir::WalkDir::new(&pkg_folder) {
let entry = entry.unwrap();
let path = entry.path();
let name = path
.strip_prefix(std::path::Path::new(&pkg_folder))
.unwrap();
// Write a directory or file to the ZIP archive
if path.is_file() {
zip.start_file(name.to_string_lossy().into_owned(), options)
.unwrap();
let mut file = std::fs::File::open(path).unwrap();
let mut buffer = Vec::new();
file.read_to_end(&mut buffer).unwrap();
zip.write_all(&buffer).unwrap();
} else if name.as_os_str().len() != 0 {
zip.add_directory(name.to_string_lossy().into_owned(), options)
.unwrap();
// Otherwise, consider it a directory containing subdirectories with potential apps
for sub_entry in std::fs::read_dir(&entry_path).unwrap() {
let sub_entry_path = sub_entry.unwrap().path();
if sub_entry_path.join("Cargo.toml").exists() {
build_app(&sub_entry_path.display().to_string(), &sub_entry_path.file_name().unwrap().to_str().unwrap(), Some(&parent_pkg_path));
}
}
// After processing all sub-apps, zip the parent's pkg/ directory
let writer = std::fs::File::create(format!("{}/target/{}.zip", pwd.display(), entry_path.file_name().unwrap().to_str().unwrap())).unwrap();
let options = zip::write::FileOptions::default()
.compression_method(zip::CompressionMethod::Stored)
.unix_permissions(0o755);
let mut zip = zip::ZipWriter::new(writer);
for sub_entry in walkdir::WalkDir::new(&parent_pkg_path) {
let sub_entry = sub_entry.unwrap();
let path = sub_entry.path();
let name = path.strip_prefix(std::path::Path::new(&parent_pkg_path)).unwrap();
// Write a directory or file to the ZIP archive
if path.is_file() {
zip.start_file(name.to_string_lossy().into_owned(), options)
.unwrap();
let mut file = std::fs::File::open(path).unwrap();
let mut buffer = Vec::new();
file.read_to_end(&mut buffer).unwrap();
zip.write_all(&buffer).unwrap();
} else if name.as_os_str().len() != 0 {
zip.add_directory(name.to_string_lossy().into_owned(), options)
.unwrap();
}
}
zip.finish().unwrap();
}
zip.finish().unwrap();
}
}

4
modules/app_tracker/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "app_tracker",
"process_wasm_path": "app_tracker.wasm",
"process_wasm_path": "/app_tracker.wasm",
"on_panic": "Restart",
"request_networking": true,
"request_messaging": [
@ -18,4 +18,4 @@
],
"grant_messaging": []
}
]
]

4
modules/chess/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "chess",
"process_wasm_path": "chess.wasm",
"process_wasm_path": "/chess.wasm",
"on_panic": "Restart",
"request_networking": true,
"request_messaging": [
@ -11,4 +11,4 @@
],
"grant_messaging": []
}
]
]

4
modules/homepage/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "homepage",
"process_wasm_path": "homepage.wasm",
"process_wasm_path": "/homepage.wasm",
"on_panic": "Restart",
"request_networking": false,
"request_messaging": [
@ -11,4 +11,4 @@
],
"grant_messaging": []
}
]
]

4
modules/http_bindings/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "http_bindings",
"process_wasm_path": "http_bindings.wasm",
"process_wasm_path": "/http_bindings.wasm",
"on_panic": "Restart",
"request_networking": false,
"request_messaging": [
@ -14,4 +14,4 @@
"http_server:sys:uqbar"
]
}
]
]

4
modules/http_proxy/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "http_proxy",
"process_wasm_path": "http_proxy.wasm",
"process_wasm_path": "/http_proxy.wasm",
"on_panic": "Restart",
"request_networking": false,
"request_messaging": [
@ -11,4 +11,4 @@
],
"grant_messaging": []
}
]
]

31
modules/key_value/key_value/src/lib.rs
View File

@ -54,23 +54,26 @@ fn handle_message (
))
}
print_to_terminal(0, "key_value: 0");
// (1)
let vfs_address = Address {
node: our.node.clone(),
process: ProcessId::Name("vfs".into()),
process: kt::ProcessId::new("vfs", "sys", "uqbar").en_wit(),
};
let vfs_drive = format!("{}{}", PREFIX, drive);
let _ = process_lib::send_and_await_response(
&vfs_address,
false,
Some(serde_json::to_string(&kt::VfsRequest::New {
Some(serde_json::to_string(&kt::VfsRequest {
drive: vfs_drive.clone(),
action: kt::VfsAction::New,
}).unwrap()),
None,
None,
15,
).unwrap();
print_to_terminal(0, "key_value: 1");
// (2)
let vfs_read = get_capability(
&vfs_address,
@ -80,9 +83,8 @@ fn handle_message (
&vfs_address,
&make_cap("write", &vfs_drive),
).ok_or(anyhow::anyhow!("New failed: no vfs 'write' capability found"))?;
let Some(spawned_process_id) = spawn(
&ProcessId::Id(0),
"key_value",
let Ok(spawned_process_id) = spawn(
None,
"/key_value_worker.wasm",
&OnPanic::None, // TODO: notify us
&Capabilities::Some(vec![vfs_read, vfs_write]),
@ -91,19 +93,21 @@ fn handle_message (
panic!("couldn't spawn"); // TODO
};
print_to_terminal(0, "key_value: 2");
// (3)
send_requests(&vec![
// grant caps to source
(
Address {
node: our.node.clone(),
process: ProcessId::Name("kernel".into()),
process: kt::ProcessId::new("kernel", "sys", "uqbar").en_wit(),
},
Request {
inherit: false,
expects_response: None,
ipc: Some(serde_json::to_string(&kt::KernelCommand::GrantCapability {
to_process: kt::de_wit_process_id(source.process.clone()),
to_process: kt::ProcessId::de_wit(source.process.clone()),
// to_process: kt::de_wit_process_id(source.process.clone()),
params: make_cap("read", drive),
}).unwrap()),
metadata: None,
@ -114,13 +118,13 @@ fn handle_message (
(
Address {
node: our.node.clone(),
process: ProcessId::Name("kernel".into()),
process: kt::ProcessId::new("kernel", "sys", "uqbar").en_wit(),
},
Request {
inherit: false,
expects_response: None,
ipc: Some(serde_json::to_string(&kt::KernelCommand::GrantCapability {
to_process: kt::de_wit_process_id(source.process.clone()),
to_process: kt::ProcessId::de_wit(source.process.clone()),
params: make_cap("write", drive),
}).unwrap()),
metadata: None,
@ -131,15 +135,15 @@ fn handle_message (
(
Address {
node: our.node.clone(),
process: ProcessId::Name("kernel".into()),
process: kt::ProcessId::new("kernel", "sys", "uqbar").en_wit(),
},
Request {
inherit: false,
expects_response: None,
ipc: Some(serde_json::to_string(&kt::KernelCommand::GrantCapability {
to_process: kt::de_wit_process_id(spawned_process_id.clone()),
to_process: kt::ProcessId::de_wit(spawned_process_id.clone()),
params: serde_json::to_string(&serde_json::json!({
"messaging": kt::de_wit_process_id(our.process.clone()),
"messaging": kt::ProcessId::de_wit(our.process.clone()),
})).unwrap(),
}).unwrap()),
metadata: None,
@ -164,6 +168,7 @@ fn handle_message (
),
]);
print_to_terminal(0, "key_value: 3");
// (4)
drive_to_process.insert(drive.into(), spawned_process_id);
// TODO
@ -243,7 +248,7 @@ fn handle_message (
impl Guest for Component {
fn init(our: Address) {
print_to_terminal(1, "key_value: begin");
print_to_terminal(0, "key_value: begin");
let mut drive_to_process: HashMap<String, ProcessId> = HashMap::new();

31
modules/key_value/key_value_worker/src/lib.rs
View File

@ -26,29 +26,29 @@ fn get_payload_wrapped() -> Option<(Option<String>, Vec<u8>)> {
fn send_and_await_response_wrapped(
target_node: String,
target_process: Result<u64, String>,
target_process: String,
target_package: String,
target_publisher: String,
request_ipc: Option<String>,
request_metadata: Option<String>,
payload: Option<(Option<String>, Vec<u8>)>,
timeout: u64,
) -> (
(String, Result<u64, String>),
(Option<String>, Option<String>),
) {
) -> (Option<String>, Option<String>) {
let payload = match payload {
None => None,
Some((mime, bytes)) => Some(Payload { mime, bytes }),
};
let (
Address { node, process },
_,
Message::Response((Response { ipc, metadata }, _)),
) = send_and_await_response(
&Address {
node: target_node,
process: match target_process {
Ok(id) => ProcessId::Id(id),
Err(name) => ProcessId::Name(name),
},
process: kt::ProcessId::new(
&target_process,
&target_package,
&target_publisher,
).en_wit(),
},
&Request {
inherit: false,
@ -63,16 +63,7 @@ fn send_and_await_response_wrapped(
).unwrap() else {
panic!("");
};
(
(
node,
match process {
ProcessId::Id(id) => Ok(id),
ProcessId::Name(name) => Err(name),
},
),
(ipc, metadata)
)
(ipc, metadata)
}
fn handle_message (

14
modules/key_value/pkg/manifest.json Normal file
View File

@ -0,0 +1,14 @@
[
{
"process_name": "key_value",
"process_wasm_path": "/key_value.wasm",
"on_panic": "Restart",
"request_networking": false,
"request_messaging": [
"vfs:sys:uqbar"
],
"grant_messaging": [
"all"
]
}
]

4
modules/key_value/pkg/metadata.json Normal file
View File

@ -0,0 +1,4 @@
{
"package": "sys",
"publisher": "uqbar"
}

4
modules/orgs/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "orgs",
"process_wasm_path": "orgs.wasm",
"process_wasm_path": "/orgs.wasm",
"on_panic": "Restart",
"request_networking": true,
"request_messaging": [
@ -9,4 +9,4 @@
],
"grant_messaging": []
}
]
]

4
modules/qns_indexer/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "qns_indexer",
"process_wasm_path": "qns_indexer.wasm",
"process_wasm_path": "/qns_indexer.wasm",
"on_panic": "Restart",
"request_networking": true,
"request_messaging": [
@ -14,4 +14,4 @@
"filesystem:sys:uqbar"
]
}
]
]

4
modules/rpc/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "rpc",
"process_wasm_path": "rpc.wasm",
"process_wasm_path": "/rpc.wasm",
"on_panic": "Restart",
"request_networking": false,
"request_messaging": [
@ -9,4 +9,4 @@
],
"grant_messaging": []
}
]
]

4
modules/terminal/pkg/manifest.json
View File

@ -1,7 +1,7 @@
[
{
"process_name": "terminal",
"process_wasm_path": "terminal.wasm",
"process_wasm_path": "/terminal.wasm",
"on_panic": "Restart",
"request_networking": true,
"request_messaging": [
@ -11,4 +11,4 @@
"all"
]
}
]
]

28
src/filesystem/mod.rs
View File

@ -191,7 +191,11 @@ async fn bootstrap(
.enclosed_name()
.expect("fs: name error reading package.zip")
.to_owned();
println!("fs: found file {}...\r", file_path.display());
let mut file_path = file_path.to_string_lossy().to_string();
if !file_path.starts_with("/") {
file_path = format!("/{}", file_path);
}
println!("fs: found file {}...\r", file_path);
let mut file_content = Vec::new();
file.read_to_end(&mut file_content).unwrap();
vfs_message_sender
@ -213,7 +217,7 @@ async fn bootstrap(
serde_json::to_string::<VfsRequest>(&VfsRequest {
drive: package_name.clone(),
action: VfsAction::Add {
full_path: file_path.to_string_lossy().to_string(),
full_path: file_path,
entry_type: AddEntryType::NewFile,
},
})
@ -278,8 +282,12 @@ async fn bootstrap(
// for each process-entry in manifest.json:
for mut entry in package_manifest {
let wasm_bytes = &mut Vec::new();
let mut file_path = format!("{}", entry.process_wasm_path);
if file_path.starts_with("/") {
file_path = format!("{}", &file_path[1..]);
}
package
.by_name(&format!("{}", entry.process_wasm_path))
.by_name(&file_path)
.expect("fs: no wasm found in package!")
.read_to_end(wasm_bytes)
.unwrap();
@ -287,10 +295,11 @@ async fn bootstrap(
// spawn the requested capabilities
// remember: out of thin air, because this is the root distro
let mut requested_caps = HashSet::new();
entry.request_messaging.push(format!(
let our_process_id = format!(
"{}:{}:{}",
entry.process_name, package_name, package_publisher
));
);
entry.request_messaging.push(our_process_id.clone());
for process_name in &entry.request_messaging {
requested_caps.insert(Capability {
issuer: Address {
@ -319,16 +328,13 @@ async fn bootstrap(
public_process = true;
continue;
}
let process_id = ProcessId::from_str(process_name).unwrap();
caps_to_grant.push((
ProcessId::from_str(process_name).unwrap(),
process_id.clone(),
Capability {
issuer: Address {
node: our_name.to_string(),
process: ProcessId::new(
Some(&entry.process_name),
package_name,
package_publisher,
),
process: ProcessId::from_str(&our_process_id).unwrap(),
},
params: "\"messaging\"".into(),
},

3
src/kernel/mod.rs
View File

@ -1896,7 +1896,8 @@ async fn make_event_loop(
match process_map.get(&kernel_message.source.process) {
None => {}, // this should only get hit by kernel?
Some(persisted) => {
if !persisted.capabilities.contains(&t::Capability {
if !persisted.public
&& !persisted.capabilities.contains(&t::Capability {
issuer: t::Address {
node: our_name.clone(),
process: kernel_message.target.process.clone(),