diff --git a/Cargo.lock b/Cargo.lock index 7913eca2..16b1c638 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1402,6 +1402,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "build_packages" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "fs-err", + "kit 0.7.7", + "serde_json", + "tokio", + "walkdir", + "zip 0.6.6", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -2638,18 +2652,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" -[[package]] -name = "filetime" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" -dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.59.0", -] - [[package]] name = "fixed-hash" version = "0.8.0" @@ -3649,7 +3651,7 @@ dependencies = [ [[package]] name = "kinode" -version = "0.9.5" +version = "0.9.7" dependencies = [ "aes-gcm", "alloy 0.2.1", @@ -3664,7 +3666,6 @@ dependencies = [ "clap", "crossterm", "dashmap 5.5.3", - "flate2", "futures", "generic-array", "hex", @@ -3672,7 +3673,6 @@ dependencies = [ "http 1.1.0", "indexmap", "jwt", - "kit", "lazy_static", "lib", "libc", @@ -3693,25 +3693,22 @@ dependencies = [ "snow", "socket2 0.5.7", "static_dir", - "tar", "thiserror", "tokio", "tokio-tungstenite 0.21.0", "unicode-segmentation", "unicode-width", "url", - "walkdir", "warp", "wasi-common", "wasmtime", "wasmtime-wasi", - "zip 0.6.6", "zip 1.1.4", ] [[package]] name = "kinode_lib" -version = "0.9.5" +version = "0.9.7" dependencies = [ "lib", ] @@ -3798,6 +3795,43 @@ dependencies = [ "zip 0.6.6", ] +[[package]] +name = "kit" +version = "0.7.7" +source = "git+https://github.com/kinode-dao/kit?rev=9c94b4b#9c94b4bd3f2a9dc2eabb2da9bc2ef5e6eb07af9d" +dependencies = [ + "alloy 0.1.4", + "alloy-sol-macro", + "alloy-sol-types", + "anyhow", + "base64 0.21.7", + "cargo_metadata", + "clap", + "color-eyre", + "dirs 5.0.1", + "fs-err", + "git2", + "hex", + "kinode_process_lib 0.9.2", + "nix 0.27.1", + "regex", + "reqwest", + "rpassword", + "semver 1.0.23", + "serde", + "serde_json", + "sha2", + "tokio", + "toml", + "tracing", + "tracing-appender", + "tracing-error", + "tracing-subscriber", + "walkdir", + "wit-bindgen", + "zip 0.6.6", +] + [[package]] name = "kns_indexer" version = "0.2.0" @@ -3834,10 +3868,10 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "lib" -version = "0.9.5" +version = "0.9.7" dependencies = [ "alloy 0.2.1", - "kit", + "kit 0.7.6", "lazy_static", "rand 0.8.5", "ring", @@ -3893,7 +3927,6 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall", ] [[package]] @@ -5951,17 +5984,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "tar" -version = "0.4.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "target-lexicon" version = "0.12.16" @@ -7725,17 +7747,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xattr" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" -dependencies = [ - "libc", - "linux-raw-sys", - "rustix", -] - [[package]] name = "zerocopy" version = "0.7.35" diff --git a/Cargo.toml b/Cargo.toml index c1413943..6a5dee6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kinode_lib" authors = ["KinodeDAO"] -version = "0.9.5" +version = "0.9.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" @@ -27,7 +27,7 @@ members = [ "kinode/packages/terminal/help", "kinode/packages/terminal/hi", "kinode/packages/terminal/kfetch", "kinode/packages/terminal/kill", "kinode/packages/terminal/m", "kinode/packages/terminal/top", "kinode/packages/terminal/net_diagnostics", "kinode/packages/terminal/peer", "kinode/packages/terminal/peers", - "kinode/packages/tester/tester", + "kinode/packages/tester/tester", "scripts/build_packages", ] default-members = ["lib"] resolver = "2" diff --git a/Dockerfile.buildruntime b/Dockerfile.buildruntime index 3b4bdd31..6cbf1f35 100644 --- a/Dockerfile.buildruntime +++ b/Dockerfile.buildruntime @@ -10,5 +10,4 @@ ENV NVM_DIR=/root/.nvm \ WORKDIR /input # Set the default command to run the build script -# TODO: once build is idempotent, remove the `rm -rf` line -CMD ["/bin/bash", "-c", ". ~/.bashrc && . ~/.cargo/env && . $NVM_DIR/nvm.sh && rm -rf target/ kinode/packages/*/pkg/*wasm kinode/packages/*/*/target/ kinode/packages/*/pkg/api.zip kinode/packages/*/*/wit kinode/packages/app_store/pkg/ui kinode/packages/homepage/pkg/ui kinode/src/register-ui/build && ./scripts/build-release.py && cp -r /tmp/kinode-release/* /output && chmod 664 /output/* && rm -rf target/ kinode/packages/*/pkg/*wasm kinode/packages/*/*/target/ kinode/packages/*/pkg/api.zip kinode/packages/*/*/wit kinode/packages/app_store/pkg/ui kinode/packages/homepage/pkg/ui kinode/src/register-ui/build"] +CMD ["/bin/bash", "-c", ". ~/.bashrc && . ~/.cargo/env && . $NVM_DIR/nvm.sh && ./scripts/build-release.py && cp -r /tmp/kinode-release/* /output && chmod 664 /output/* && find . -user root -print0 2>/dev/null | xargs -0 rm -rf"] diff --git a/README.md b/README.md index 4a7bf501..84a8bafd 100644 --- a/README.md +++ b/README.md @@ -28,26 +28,26 @@ On certain operating systems, you may need to install these dependencies if they git clone git@github.com:kinode-dao/kinode.git -# Get some stuff so we can build Wasm. +# Install Rust and some `cargo` tools so we can build the runtime and Wasm. -cd kinode +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh cargo install wasm-tools rustup install nightly -rustup target add wasm32-wasi -rustup target add wasm32-wasi --toolchain nightly -rustup target add wasm32-wasip1 rustup target add wasm32-wasip1 --toolchain nightly cargo install cargo-wasi # Install NPM so we can build frontends for "distro" packages. # https://docs.npmjs.com/downloading-and-installing-node-js-and-npm -# If you want to skip this step, run cargo build with the environment variable SKIP_BUILD_FRONTEND=true +# If you want to skip this step, build the packages with `cargo run -p build_packages -- --skip-build-frontend` to neglect building the frontends -# Build the runtime, along with a number of "distro" Wasm modules. -# The compiled binary will be at `kinode/target/debug/kinode` -# OPTIONAL: --release flag (slower build; faster runtime; binary at `kinode/target/release/kinode`) +# Build the "distro" Wasm modules, then, build the runtime. +# The compiled packages will be at `kinode/target/packages.zip`. +# The compiled binary will be at `kinode/target/debug/kinode`. +# OPTIONAL: --release flag (slower build; faster runtime; binary at `kinode/target/release/kinode`). -cargo +nightly build -p kinode +cd kinode +cargo run -p build_packages +cargo build -p kinode ``` ## Security Status diff --git a/kinode/Cargo.toml b/kinode/Cargo.toml index 1468ab02..24bbd0f1 100644 --- a/kinode/Cargo.toml +++ b/kinode/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kinode" authors = ["KinodeDAO"] -version = "0.9.5" +version = "0.9.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" @@ -14,12 +14,7 @@ path = "src/main.rs" [build-dependencies] anyhow = "1.0.71" -flate2 = "1.0" -kit = { git = "https://github.com/kinode-dao/kit", tag = "v0.7.6" } -tar = "0.4" -tokio = "1.28" -walkdir = "2.4" -zip = "0.6" +sha2 = "0.10.8" [features] simulation-mode = [] diff --git a/kinode/build.rs b/kinode/build.rs index 388bb658..a3ec15df 100644 --- a/kinode/build.rs +++ b/kinode/build.rs @@ -1,12 +1,9 @@ -use std::{ - fs::{self, File}, - io::{BufReader, Cursor, Read, Write}, - path::{Path, PathBuf}, -}; +use std::io::Read; +use std::path::{Path, PathBuf}; -use flate2::read::GzDecoder; -use tar::Archive; -use zip::write::FileOptions; +use sha2::Digest; + +const CANONICAL_PACKAGES_ZIP_PATH: &str = "../target/packages.zip"; macro_rules! p { ($($tokens: tt)*) => { @@ -14,214 +11,64 @@ macro_rules! p { } } -/// get cargo features to compile packages with -fn get_features() -> String { - let mut features = "".to_string(); - for (key, _) in std::env::vars() { - if key.starts_with("CARGO_FEATURE_") { - let feature = key - .trim_start_matches("CARGO_FEATURE_") - .to_lowercase() - .replace("_", "-"); - features.push_str(&feature); +fn compute_hash(file_path: &Path) -> anyhow::Result { + let input_file = std::fs::File::open(file_path)?; + let mut reader = std::io::BufReader::new(input_file); + let mut hasher = sha2::Sha256::new(); + let mut buffer = [0; 1024]; // buffer for chunks of the file + + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; } + hasher.update(&buffer[..count]); } - features -} -/// print `cargo:rerun-if-changed=PATH` for each path of interest -fn output_reruns(dir: &Path) { - // Check files individually - if let Ok(entries) = fs::read_dir(dir) { - for entry in entries.filter_map(|e| e.ok()) { - let path = entry.path(); - if path.is_dir() { - if let Some(dirname) = path.file_name().and_then(|n| n.to_str()) { - if dirname == "ui" || dirname == "target" { - // do not prompt a rerun if only UI/build files have changed - continue; - } - // If the entry is a directory not in rerun_files, recursively walk it - output_reruns(&path); - } - } else { - if let Some(filename) = path.file_name().and_then(|n| n.to_str()) { - if filename.ends_with(".zip") || filename.ends_with(".wasm") { - // do not prompt a rerun for compiled outputs - continue; - } - // any other changed file within a package subdir prompts a rerun - println!("cargo::rerun-if-changed={}", path.display()); - } - } - } - } -} - -fn _untar_gz_file(path: &Path, dest: &Path) -> std::io::Result<()> { - // Open the .tar.gz file - let tar_gz = File::open(path)?; - let tar_gz_reader = BufReader::new(tar_gz); - - // Decode the gzip layer - let tar = GzDecoder::new(tar_gz_reader); - - // Create a new archive from the tar file - let mut archive = Archive::new(tar); - - // Unpack the archive into the specified destination directory - archive.unpack(dest)?; - - Ok(()) -} - -fn build_and_zip_package( - entry_path: PathBuf, - parent_pkg_path: &str, - features: &str, -) -> anyhow::Result<(String, String, Vec)> { - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async { - kit::build::execute( - &entry_path, - true, - false, - true, - features, - None, - None, - None, - vec![], - vec![], - false, - false, - false, - ) - .await - .map_err(|e| anyhow::anyhow!("{:?}", e))?; - - let mut writer = Cursor::new(Vec::new()); - let options = FileOptions::default() - .compression_method(zip::CompressionMethod::Deflated) - .unix_permissions(0o755) - .last_modified_time(zip::DateTime::from_date_and_time(2023, 6, 19, 0, 0, 0).unwrap()); - { - let mut zip = zip::ZipWriter::new(&mut writer); - - for sub_entry in walkdir::WalkDir::new(parent_pkg_path) { - let sub_entry = sub_entry?; - let path = sub_entry.path(); - let name = path.strip_prefix(Path::new(parent_pkg_path))?; - - if path.is_file() { - zip.start_file(name.to_string_lossy(), options)?; - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer)?; - zip.write_all(&buffer)?; - } else if !name.as_os_str().is_empty() { - zip.add_directory(name.to_string_lossy(), options)?; - } - } - zip.finish()?; - } - - let zip_contents = writer.into_inner(); - let zip_filename = format!("{}.zip", entry_path.file_name().unwrap().to_str().unwrap()); - Ok((entry_path.display().to_string(), zip_filename, zip_contents)) - }) + Ok(format!("{:x}", hasher.finalize())) } fn main() -> anyhow::Result<()> { - if std::env::var("SKIP_BUILD_SCRIPT").is_ok() { - p!("skipping build script"); - return Ok(()); + let path_to_packages_zip = match std::env::var("PATH_TO_PACKAGES_ZIP") { + Ok(env_var) => env_var, + Err(_) => { + let canonical_path = PathBuf::from(CANONICAL_PACKAGES_ZIP_PATH); + if canonical_path.exists() { + p!("No path given via PATH_TO_PACKAGES_ZIP envvar. Defaulting to path of `kinode/target/packages.zip`."); + CANONICAL_PACKAGES_ZIP_PATH.to_string() + } else { + return Err(anyhow::anyhow!("You must build packages.zip with scripts/build_packages or set PATH_TO_PACKAGES_ZIP to point to your desired pacakges.zip (default path at kinode/target/packages.zip was not populated).")); + } + } + }; + let path = PathBuf::from(&path_to_packages_zip); + if !path.exists() { + return Err(anyhow::anyhow!( + "Path to packages {path_to_packages_zip} does not exist." + )); } + let path_to_packages_zip_path = PathBuf::from(&path_to_packages_zip).canonicalize()?; + let canonical_packages_zip_path = PathBuf::from(CANONICAL_PACKAGES_ZIP_PATH).canonicalize()?; + if path_to_packages_zip_path != canonical_packages_zip_path { + std::fs::copy(&path_to_packages_zip_path, &canonical_packages_zip_path)?; + } + + // build core frontends let pwd = std::env::current_dir()?; - let parent_dir = pwd.parent().unwrap(); - let packages_dir = pwd.join("packages"); + let core_frontends = vec!["src/register-ui"]; - if std::env::var("SKIP_BUILD_FRONTEND").is_ok() { - p!("skipping frontend builds"); - } else { - // build core frontends - let core_frontends = vec![ - "src/register-ui", - "packages/app_store/ui", - "packages/homepage/ui", - // chess when brought in - ]; - - // for each frontend, execute build.sh - for frontend in core_frontends { - let status = std::process::Command::new("sh") - .current_dir(pwd.join(frontend)) - .arg("./build.sh") - .status()?; - if !status.success() { - return Err(anyhow::anyhow!("Failed to build frontend: {}", frontend)); - } + // for each frontend, execute build.sh + for frontend in core_frontends { + let status = std::process::Command::new("sh") + .current_dir(pwd.join(frontend)) + .arg("./build.sh") + .status()?; + if !status.success() { + return Err(anyhow::anyhow!("Failed to build frontend: {}", frontend)); } } - output_reruns(&packages_dir); - - let features = get_features(); - - let results: Vec)>> = fs::read_dir(&packages_dir)? - .filter_map(|entry| { - let entry_path = match entry { - Ok(e) => e.path(), - Err(_) => return None, - }; - let child_pkg_path = entry_path.join("pkg"); - if !child_pkg_path.exists() { - // don't run on, e.g., `.DS_Store` - return None; - } - Some(build_and_zip_package( - entry_path.clone(), - child_pkg_path.to_str().unwrap(), - &features, - )) - }) - .collect(); - - // Process results, e.g., write to `bootstrapped_processes.rs` - // This part remains sequential - let mut bootstrapped_processes = vec![]; - writeln!( - bootstrapped_processes, - "pub static BOOTSTRAPPED_PROCESSES: &[(&str, &[u8], &[u8])] = &[" - )?; - - for result in results { - match result { - Ok((entry_path, zip_filename, zip_contents)) => { - // Further processing, like saving ZIP files and updating bootstrapped_processes - let metadata_path = format!("{}/metadata.json", entry_path); - let zip_path = format!("{}/target/{}", parent_dir.display(), zip_filename); - fs::write(&zip_path, &zip_contents)?; - - writeln!( - bootstrapped_processes, - " (\"{}\", include_bytes!(\"{}\"), include_bytes!(\"{}\")),", - zip_filename, metadata_path, zip_path, - )?; - } - Err(e) => return Err(e), - } - } - - writeln!(bootstrapped_processes, "];")?; - let target_dir = pwd.join("../target"); - if !target_dir.exists() { - fs::create_dir_all(&target_dir)?; - } - let bootstrapped_processes_path = target_dir.join("bootstrapped_processes.rs"); - fs::write(&bootstrapped_processes_path, bootstrapped_processes)?; - let version = if let Ok(version) = std::env::var("DOCKER_BUILD_IMAGE_VERSION") { // embed the DOCKER_BUILD_IMAGE_VERSION version @@ -230,5 +77,8 @@ fn main() -> anyhow::Result<()> { }; println!("cargo:rustc-env=DOCKER_BUILD_IMAGE_VERSION={version}"); + let packages_zip_hash = compute_hash(&canonical_packages_zip_path)?; + println!("cargo:rustc-env=PACKAGES_ZIP_HASH={packages_zip_hash}"); + Ok(()) } diff --git a/kinode/packages/app_store/api/app_store:sys-v1.wit b/kinode/packages/app_store/api/app_store:sys-v1.wit index 352af4ce..360d7cfe 100644 --- a/kinode/packages/app_store/api/app_store:sys-v1.wit +++ b/kinode/packages/app_store/api/app_store:sys-v1.wit @@ -184,6 +184,8 @@ interface downloads { auto-update(auto-update-request), /// Notify that a download is complete download-complete(download-complete-request), + /// Auto-update-download complete + auto-download-complete(auto-download-complete-request), /// Get files for a package get-files(option), /// Remove a file @@ -243,6 +245,12 @@ interface downloads { err: option, } + /// Request for an auto-download complete + record auto-download-complete-request { + download-info: download-complete-request, + manifest-hash: string, + } + /// Represents a hash mismatch error record hash-mismatch { desired: string, diff --git a/kinode/packages/app_store/app_store/src/lib.rs b/kinode/packages/app_store/app_store/src/lib.rs index f3b5731b..e7a9d433 100644 --- a/kinode/packages/app_store/app_store/src/lib.rs +++ b/kinode/packages/app_store/app_store/src/lib.rs @@ -30,7 +30,7 @@ //! It delegates these responsibilities to the downloads and chain processes respectively. //! use crate::kinode::process::downloads::{ - DownloadCompleteRequest, DownloadResponses, ProgressUpdate, + AutoDownloadCompleteRequest, DownloadCompleteRequest, DownloadResponses, ProgressUpdate, }; use crate::kinode::process::main::{ ApisResponse, GetApiResponse, InstallPackageRequest, InstallResponse, LocalRequest, @@ -65,6 +65,7 @@ pub enum Req { LocalRequest(LocalRequest), Progress(ProgressUpdate), DownloadComplete(DownloadCompleteRequest), + AutoDownloadComplete(AutoDownloadCompleteRequest), Http(http::server::HttpServerRequest), } @@ -161,6 +162,40 @@ fn handle_message( }, ); } + Req::AutoDownloadComplete(req) => { + if !message.is_local(&our) { + return Err(anyhow::anyhow!( + "auto download complete from non-local node" + )); + } + // auto_install case: + // the downloads process has given us the new package manifest's + // capability hashes, and the old package's capability hashes. + // we can use these to determine if the new package has the same + // capabilities as the old one, and if so, auto-install it. + + let manifest_hash = req.manifest_hash; + let package_id = req.download_info.package_id; + let version_hash = req.download_info.version_hash; + + if let Some(package) = state.packages.get(&package_id.clone().to_process_lib()) { + if package.manifest_hash == Some(manifest_hash) { + print_to_terminal(1, "auto_install:main, manifest_hash match"); + if let Err(e) = + utils::install(&package_id, None, &version_hash, state, &our.node) + { + print_to_terminal(1, &format!("error auto_installing package: {e}")); + } else { + println!( + "auto_installed update for package: {:?}", + &package_id.to_process_lib() + ); + } + } else { + print_to_terminal(1, "auto_install:main, manifest_hash do not match"); + } + } + } Req::DownloadComplete(req) => { if !message.is_local(&our) { return Err(anyhow::anyhow!("download complete from non-local node")); @@ -182,41 +217,6 @@ fn handle_message( .unwrap(), }, ); - - // auto_install case: - // the downloads process has given us the new package manifest's - // capability hashes, and the old package's capability hashes. - // we can use these to determine if the new package has the same - // capabilities as the old one, and if so, auto-install it. - if let Some(context) = message.context() { - let manifest_hash = String::from_utf8(context.to_vec())?; - if let Some(package) = - state.packages.get(&req.package_id.clone().to_process_lib()) - { - if package.manifest_hash == Some(manifest_hash) { - print_to_terminal(1, "auto_install:main, manifest_hash match"); - if let Err(e) = utils::install( - &req.package_id, - None, - &req.version_hash, - state, - &our.node, - ) { - print_to_terminal( - 1, - &format!("error auto_installing package: {e}"), - ); - } else { - println!( - "auto_installed update for package: {:?}", - &req.package_id.to_process_lib() - ); - } - } else { - print_to_terminal(1, "auto_install:main, manifest_hash do not match"); - } - } - } } } } else { diff --git a/kinode/packages/app_store/downloads/src/lib.rs b/kinode/packages/app_store/downloads/src/lib.rs index 8a8bfc31..31c91f13 100644 --- a/kinode/packages/app_store/downloads/src/lib.rs +++ b/kinode/packages/app_store/downloads/src/lib.rs @@ -42,9 +42,9 @@ //! mechanism is implemented in the FT worker for improved modularity and performance. //! use crate::kinode::process::downloads::{ - AutoUpdateRequest, DirEntry, DownloadCompleteRequest, DownloadError, DownloadRequests, - DownloadResponses, Entry, FileEntry, HashMismatch, LocalDownloadRequest, RemoteDownloadRequest, - RemoveFileRequest, + AutoDownloadCompleteRequest, AutoUpdateRequest, DirEntry, DownloadCompleteRequest, + DownloadError, DownloadRequests, DownloadResponses, Entry, FileEntry, HashMismatch, + LocalDownloadRequest, RemoteDownloadRequest, RemoveFileRequest, }; use std::{collections::HashSet, io::Read, str::FromStr}; @@ -245,7 +245,7 @@ fn handle_message( // if we have a pending auto_install, forward that context to the main process. // it will check if the caps_hashes match (no change in capabilities), and auto_install if it does. - let context = if auto_updates.remove(&( + let manifest_hash = if auto_updates.remove(&( req.package_id.clone().to_process_lib(), req.version_hash.clone(), )) { @@ -253,7 +253,7 @@ fn handle_message( req.package_id.clone().to_process_lib(), req.version_hash.clone(), ) { - Ok(manifest_hash) => Some(manifest_hash.as_bytes().to_vec()), + Ok(manifest_hash) => Some(manifest_hash), Err(e) => { print_to_terminal( 1, @@ -267,13 +267,26 @@ fn handle_message( }; // pushed to UI via websockets - let mut request = Request::to(("our", "main", "app_store", "sys")) - .body(serde_json::to_vec(&req)?); + Request::to(("our", "main", "app_store", "sys")) + .body(serde_json::to_vec(&req)?) + .send()?; - if let Some(ctx) = context { - request = request.context(ctx); + // trigger auto-update install trigger to main:app_store:sys + if let Some(manifest_hash) = manifest_hash { + let auto_download_complete_req = AutoDownloadCompleteRequest { + download_info: req.clone(), + manifest_hash, + }; + print_to_terminal( + 1, + &format!( + "auto_update download complete: triggering install on main:app_store:sys" + ), + ); + Request::to(("our", "main", "app_store", "sys")) + .body(serde_json::to_vec(&auto_download_complete_req)?) + .send()?; } - request.send()?; } DownloadRequests::GetFiles(maybe_id) => { // if not local, throw to the boonies. diff --git a/kinode/packages/app_store/ui/src/components/PackageSelector.tsx b/kinode/packages/app_store/ui/src/components/PackageSelector.tsx index 7fc66743..ae28fffd 100644 --- a/kinode/packages/app_store/ui/src/components/PackageSelector.tsx +++ b/kinode/packages/app_store/ui/src/components/PackageSelector.tsx @@ -6,7 +6,7 @@ interface PackageSelectorProps { } const PackageSelector: React.FC = ({ onPackageSelect }) => { - const { installed } = useAppsStore(); + const { installed, fetchInstalled } = useAppsStore(); const [selectedPackage, setSelectedPackage] = useState(""); const [customPackage, setCustomPackage] = useState(""); const [isCustomPackageSelected, setIsCustomPackageSelected] = useState(false); @@ -18,6 +18,10 @@ const PackageSelector: React.FC = ({ onPackageSelect }) => } }, [selectedPackage, onPackageSelect]); + useEffect(() => { + fetchInstalled(); + }, []); + const handlePackageChange = (e: React.ChangeEvent) => { const value = e.target.value; if (value === "custom") { diff --git a/kinode/packages/app_store/ui/src/pages/DownloadPage.tsx b/kinode/packages/app_store/ui/src/pages/DownloadPage.tsx index cd90c6be..c933d3a7 100644 --- a/kinode/packages/app_store/ui/src/pages/DownloadPage.tsx +++ b/kinode/packages/app_store/ui/src/pages/DownloadPage.tsx @@ -1,11 +1,10 @@ import React, { useState, useEffect, useCallback, useMemo } from "react"; -import { useParams, useNavigate } from "react-router-dom"; +import { useParams } from "react-router-dom"; import { FaDownload, FaSpinner, FaChevronDown, FaChevronUp, FaRocket, FaTrash, FaPlay } from "react-icons/fa"; import useAppsStore from "../store"; import { MirrorSelector } from '../components'; export default function DownloadPage() { - const navigate = useNavigate(); const { id } = useParams<{ id: string }>(); const { listings, @@ -28,6 +27,9 @@ export default function DownloadPage() { const [isMirrorOnline, setIsMirrorOnline] = useState(null); const [showCapApproval, setShowCapApproval] = useState(false); const [manifest, setManifest] = useState(null); + const [isInstalling, setIsInstalling] = useState(false); + const [isCheckingLaunch, setIsCheckingLaunch] = useState(false); + const [launchPath, setLaunchPath] = useState(null); const app = useMemo(() => listings[id || ""], [listings, id]); const appDownloads = useMemo(() => downloads[id || ""] || [], [downloads, id]); @@ -101,6 +103,36 @@ export default function DownloadPage() { return versionData ? installedApp.our_version_hash === versionData.hash : false; }, [app, selectedVersion, installedApp, sortedVersions]); + const checkLaunchPath = useCallback(() => { + if (!app) return; + setIsCheckingLaunch(true); + const appId = `${app.package_id.package_name}:${app.package_id.publisher_node}`; + fetchHomepageApps().then(() => { + const path = getLaunchUrl(appId); + setLaunchPath(path); + setIsCheckingLaunch(false); + if (path) { + setIsInstalling(false); + } + }); + }, [app, fetchHomepageApps, getLaunchUrl]); + + useEffect(() => { + if (isInstalling) { + const checkInterval = setInterval(checkLaunchPath, 500); + const timeout = setTimeout(() => { + clearInterval(checkInterval); + setIsInstalling(false); + setIsCheckingLaunch(false); + }, 5000); + + return () => { + clearInterval(checkInterval); + clearTimeout(timeout); + }; + } + }, [isInstalling, checkLaunchPath]); + const handleDownload = useCallback(() => { if (!id || !selectedMirror || !app || !selectedVersion) return; const versionData = sortedVersions.find(v => v.version === selectedVersion); @@ -130,36 +162,87 @@ export default function DownloadPage() { } }, [id, app, appDownloads]); - const canDownload = useMemo(() => { - return selectedMirror && (isMirrorOnline === true || selectedMirror.startsWith('http')) && !isDownloading && !isDownloaded; - }, [selectedMirror, isMirrorOnline, isDownloading, isDownloaded]); - const confirmInstall = useCallback(() => { if (!id || !selectedVersion) return; const versionData = sortedVersions.find(v => v.version === selectedVersion); if (versionData) { + setIsInstalling(true); + setLaunchPath(null); installApp(id, versionData.hash).then(() => { - fetchData(id); setShowCapApproval(false); setManifest(null); + fetchData(id); }); } }, [id, selectedVersion, sortedVersions, installApp, fetchData]); const handleLaunch = useCallback(() => { - if (app) { - const launchUrl = getLaunchUrl(`${app.package_id.package_name}:${app.package_id.publisher_node}`); - if (launchUrl) { - window.location.href = launchUrl; - } + if (launchPath) { + window.location.href = launchPath; } - }, [app, getLaunchUrl]); + }, [launchPath]); const canLaunch = useMemo(() => { if (!app) return false; return !!getLaunchUrl(`${app.package_id.package_name}:${app.package_id.publisher_node}`); }, [app, getLaunchUrl]); + const canDownload = useMemo(() => { + return selectedMirror && (isMirrorOnline === true || selectedMirror.startsWith('http')) && !isDownloading && !isDownloaded; + }, [selectedMirror, isMirrorOnline, isDownloading, isDownloaded]); + + const renderActionButton = () => { + if (isCurrentVersionInstalled || launchPath) { + return ( + + ); + } + + if (isInstalling || isCheckingLaunch) { + return ( + + ); + } + + if (isDownloaded) { + return ( + + ); + } + + return ( + + ); + }; + if (!app) { return

Loading app details...

; } @@ -176,15 +259,22 @@ export default function DownloadPage() {

{`${app.package_id.package_name}.${app.package_id.publisher_node}`}

- {installedApp && ( + {launchPath ? ( - )} + ) : isInstalling || isCheckingLaunch ? ( + + ) : installedApp ? ( + + ) : null}

{app.metadata?.description}

@@ -207,39 +297,7 @@ export default function DownloadPage() { onMirrorSelect={handleMirrorSelect} /> - {isCurrentVersionInstalled ? ( - - ) : isDownloaded ? ( - - ) : ( - - )} + {renderActionButton()}
diff --git a/kinode/packages/app_store/ui/src/pages/PublishPage.tsx b/kinode/packages/app_store/ui/src/pages/PublishPage.tsx index c72d0eec..e9c16c44 100644 --- a/kinode/packages/app_store/ui/src/pages/PublishPage.tsx +++ b/kinode/packages/app_store/ui/src/pages/PublishPage.tsx @@ -12,7 +12,7 @@ const NAME_INVALID = "Package name must contain only valid characters (a-z, 0-9, export default function PublishPage() { const { openConnectModal } = useConnectModal(); - const { ourApps, fetchOurApps, installed, downloads } = useAppsStore(); + const { ourApps, fetchOurApps, downloads } = useAppsStore(); const publicClient = usePublicClient(); const { address, isConnected, isConnecting } = useAccount(); diff --git a/kinode/packages/app_store/ui/src/store/index.ts b/kinode/packages/app_store/ui/src/store/index.ts index c87e23e8..c6c0deec 100644 --- a/kinode/packages/app_store/ui/src/store/index.ts +++ b/kinode/packages/app_store/ui/src/store/index.ts @@ -218,12 +218,6 @@ const useAppsStore = create()((set, get) => ({ }); if (res.status === HTTP_STATUS.CREATED) { await get().fetchInstalled(); - - // hacky: a small delay (500ms) before fetching homepage apps - // to give the app time to add itself to the homepage - // might make sense to add more state and do retry logic instead. - await new Promise(resolve => setTimeout(resolve, 500)); - await get().fetchHomepageApps(); } } catch (error) { diff --git a/kinode/src/main.rs b/kinode/src/main.rs index f5c23bc2..31140a5f 100644 --- a/kinode/src/main.rs +++ b/kinode/src/main.rs @@ -1,5 +1,3 @@ -#![feature(async_closure)] -#![feature(btree_extract_if)] use anyhow::Result; use clap::{arg, value_parser, Command}; use lib::types::core::{ @@ -66,10 +64,17 @@ pub const MULTICALL_ADDRESS: &str = "0xcA11bde05977b3631167028862bE2a173976CA11" #[tokio::main] async fn main() { + // embed values in binary for inspection without running & print on boot + // e.g., to inspect without running, use + // ```bash + // strings kinode | grep DOCKER_BUILD_IMAGE_VERSION + // ``` println!( - "\nDOCKER_BUILD_IMAGE_VERSION: {}\n", - env!("DOCKER_BUILD_IMAGE_VERSION") + "\nDOCKER_BUILD_IMAGE_VERSION: {}\nPACKAGES_ZIP_HASH: {}\n", + env!("DOCKER_BUILD_IMAGE_VERSION"), + env!("PACKAGES_ZIP_HASH"), ); + let app = build_command(); let matches = app.get_matches(); @@ -713,11 +718,11 @@ fn build_command() -> Command { ) .arg( arg!(--"max-peers" "Maximum number of peers to hold active connections with (default 32)") - .value_parser(value_parser!(u32)), + .value_parser(value_parser!(u64)), ) .arg( arg!(--"max-passthroughs" "Maximum number of passthroughs serve as a router (default 0)") - .value_parser(value_parser!(u32)), + .value_parser(value_parser!(u64)), ) .arg( arg!(--"soft-ulimit" "Enforce a static maximum number of file descriptors (default fetched from system)") diff --git a/kinode/src/net/connect.rs b/kinode/src/net/connect.rs index 12931a2a..b06ea7b5 100644 --- a/kinode/src/net/connect.rs +++ b/kinode/src/net/connect.rs @@ -6,25 +6,40 @@ use tokio::sync::mpsc; /// if target is a peer, queue to be routed /// otherwise, create peer and initiate routing -pub async fn send_to_peer(ext: &IdentityExt, data: &NetData, km: KernelMessage) { +pub async fn send_to_peer(ext: &IdentityExt, data: &NetData, mut km: KernelMessage) { if let Some(mut peer) = data.peers.get_mut(&km.target.node) { - peer.sender.send(km).expect("net: peer sender was dropped"); - peer.set_last_message(); - } else { - let Some(peer_id) = data.pki.get(&km.target.node) else { - return utils::error_offline(km, &ext.network_error_tx).await; - }; - let (mut peer, peer_rx) = Peer::new(peer_id.clone(), false); - // send message to be routed - peer.send(km); - data.peers.insert(peer_id.name.clone(), peer).await; - tokio::spawn(connect_to_peer( - ext.clone(), - data.clone(), - peer_id.clone(), - peer_rx, - )); + match peer.send(km) { + Ok(()) => { + peer.set_last_message(); + return; + } + Err(e_km) => { + // peer connection was closed, remove it and try to reconnect + data.peers.remove(&peer.identity.name).await; + km = e_km.0; + } + } } + let Some(peer_id) = data.pki.get(&km.target.node) else { + return utils::error_offline(km, &ext.network_error_tx).await; + }; + let (mut peer, peer_rx) = Peer::new(peer_id.clone(), false); + // send message to be routed + match peer.send(km) { + Ok(()) => { + peer.set_last_message(); + } + Err(e_km) => { + return utils::error_offline(e_km.0, &ext.network_error_tx).await; + } + }; + data.peers.insert(peer_id.name.clone(), peer).await; + tokio::spawn(connect_to_peer( + ext.clone(), + data.clone(), + peer_id.clone(), + peer_rx, + )); } /// based on peer's identity, either use one of their diff --git a/kinode/src/net/mod.rs b/kinode/src/net/mod.rs index 11226dc2..7dc71d03 100644 --- a/kinode/src/net/mod.rs +++ b/kinode/src/net/mod.rs @@ -69,7 +69,6 @@ pub async fn networking( peers, pending_passthroughs, active_passthroughs, - max_peers, max_passthroughs, fds_limit: 10, // small hardcoded limit that gets replaced by fd_manager soon after boot }; @@ -212,7 +211,7 @@ async fn handle_local_request( printout.push_str(&format!( "we have connections with {} peers ({} max):\r\n", data.peers.peers().len(), - data.max_peers, + data.peers.max_peers(), )); let now = std::time::SystemTime::now() @@ -342,16 +341,17 @@ async fn handle_fdman(km: &KernelMessage, request_body: &[u8], data: &mut NetDat match req { lib::core::FdManagerRequest::FdsLimit(fds_limit) => { data.fds_limit = fds_limit; - if data.max_peers > fds_limit { - data.max_peers = fds_limit; - } + data.peers.set_max_peers(fds_limit); // TODO combine with max_peers check + // only update passthrough limit if it's higher than the new fds limit + // most nodes have passthroughs disabled, meaning this will keep it at 0 if data.max_passthroughs > fds_limit { data.max_passthroughs = fds_limit; } // TODO cull passthroughs too if data.peers.peers().len() >= data.fds_limit as usize { let diff = data.peers.peers().len() - data.fds_limit as usize; + println!("net: culling {diff} peer(s)\r\n"); data.peers.cull(diff).await; } } diff --git a/kinode/src/net/types.rs b/kinode/src/net/types.rs index f5119283..7b37d918 100644 --- a/kinode/src/net/types.rs +++ b/kinode/src/net/types.rs @@ -6,6 +6,7 @@ use { dashmap::DashMap, ring::signature::Ed25519KeyPair, serde::{Deserialize, Serialize}, + std::sync::atomic::AtomicU64, std::sync::Arc, tokio::net::TcpStream, tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}, @@ -57,7 +58,7 @@ pub struct RoutingRequest { #[derive(Clone)] pub struct Peers { - max_peers: u64, + max_peers: Arc, send_to_loop: MessageSender, peers: Arc>, } @@ -65,7 +66,7 @@ pub struct Peers { impl Peers { pub fn new(max_peers: u64, send_to_loop: MessageSender) -> Self { Self { - max_peers, + max_peers: Arc::new(max_peers.into()), send_to_loop, peers: Arc::new(DashMap::new()), } @@ -75,6 +76,15 @@ impl Peers { &self.peers } + pub fn max_peers(&self) -> u64 { + self.max_peers.load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_max_peers(&self, max_peers: u64) { + self.max_peers + .store(max_peers, std::sync::atomic::Ordering::Relaxed); + } + pub fn get(&self, name: &str) -> Option> { self.peers.get(name) } @@ -94,7 +104,7 @@ impl Peers { /// remove the one with the oldest last_message. pub async fn insert(&self, name: String, peer: Peer) { self.peers.insert(name, peer); - if self.peers.len() > self.max_peers as usize { + if self.peers.len() as u64 > self.max_peers.load(std::sync::atomic::Ordering::Relaxed) { let oldest = self .peers .iter() @@ -102,7 +112,7 @@ impl Peers { .unwrap() .key() .clone(); - self.peers.remove(&oldest); + self.remove(&oldest).await; crate::fd_manager::send_fd_manager_hit_fds_limit( &Address::new("our", NET_PROCESS_ID.clone()), &self.send_to_loop, @@ -122,7 +132,7 @@ impl Peers { sorted_peers.sort_by_key(|p| p.last_message); to_remove.extend(sorted_peers.iter().take(n)); for peer in to_remove { - self.peers.remove(&peer.identity.name); + self.remove(&peer.identity.name).await; } crate::fd_manager::send_fd_manager_hit_fds_limit( &Address::new("our", NET_PROCESS_ID.clone()), @@ -189,9 +199,13 @@ impl Peer { } /// Send a message to the peer. - pub fn send(&mut self, km: KernelMessage) { - self.sender.send(km).expect("net: peer sender was dropped"); + pub fn send( + &mut self, + km: KernelMessage, + ) -> Result<(), tokio::sync::mpsc::error::SendError> { + self.sender.send(km)?; self.set_last_message(); + Ok(()) } /// Update the last message time to now. @@ -222,7 +236,6 @@ pub struct NetData { pub pending_passthroughs: PendingPassthroughs, /// only used by routers pub active_passthroughs: ActivePassthroughs, - pub max_peers: u64, pub max_passthroughs: u64, pub fds_limit: u64, } diff --git a/kinode/src/state.rs b/kinode/src/state.rs index a86c3e35..7b14ad0f 100644 --- a/kinode/src/state.rs +++ b/kinode/src/state.rs @@ -14,7 +14,8 @@ use std::{ }; use tokio::{fs, io::AsyncWriteExt, sync::Mutex}; -include!("../../target/bootstrapped_processes.rs"); +static PACKAGES_ZIP: &[u8] = include_bytes!("../../target/packages.zip"); +const FILE_TO_METADATA: &str = "file_to_metadata.json"; pub async fn load_state( our_name: String, @@ -381,7 +382,7 @@ async fn bootstrap( current.capabilities.extend(runtime_caps.clone()); } - let packages = get_zipped_packages().await; + let packages = get_zipped_packages(); for (package_metadata, mut package) in packages.clone() { let package_name = package_metadata.properties.package_name.as_str(); @@ -412,7 +413,7 @@ async fn bootstrap( let mut zip_file = fs::File::create(format!("{}/{}.zip", &pkg_path, &our_drive_name)).await?; let package_zip_bytes = package.clone().into_inner().into_inner(); - zip_file.write_all(package_zip_bytes).await?; + zip_file.write_all(&package_zip_bytes).await?; // for each file in package.zip, write to vfs folder for i in 0..package.len() { @@ -713,20 +714,28 @@ fn sign_cap(cap: Capability, keypair: Arc) -> Vec } /// read in `include!()`ed .zip package files -async fn get_zipped_packages() -> Vec<( - Erc721Metadata, - zip::ZipArchive>, -)> { +fn get_zipped_packages() -> Vec<(Erc721Metadata, zip::ZipArchive>>)> { let mut packages = Vec::new(); - for (package_name, metadata_bytes, bytes) in BOOTSTRAPPED_PROCESSES.iter() { - if let Ok(zip) = zip::ZipArchive::new(std::io::Cursor::new(*bytes)) { - if let Ok(metadata) = serde_json::from_slice::(metadata_bytes) { - packages.push((metadata, zip)); - } else { - println!("fs: metadata for package {package_name} is not valid Erc721Metadata!\r",); - } - } + let mut packages_zip = zip::ZipArchive::new(std::io::Cursor::new(PACKAGES_ZIP)).unwrap(); + let mut file_to_metadata = vec![]; + packages_zip + .by_name(FILE_TO_METADATA) + .unwrap() + .read_to_end(&mut file_to_metadata) + .unwrap(); + let file_to_metadata: HashMap = + serde_json::from_slice(&file_to_metadata).unwrap(); + + for (file_name, metadata) in file_to_metadata { + let mut zip_bytes = vec![]; + packages_zip + .by_name(&file_name) + .unwrap() + .read_to_end(&mut zip_bytes) + .unwrap(); + let zip_archive = zip::ZipArchive::new(std::io::Cursor::new(zip_bytes)).unwrap(); + packages.push((metadata, zip_archive)); } packages diff --git a/lib/Cargo.toml b/lib/Cargo.toml index a4914b97..abf98728 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lib" authors = ["KinodeDAO"] -version = "0.9.5" +version = "0.9.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" diff --git a/lib/src/core.rs b/lib/src/core.rs index f3fb16ce..fc74bcbd 100644 --- a/lib/src/core.rs +++ b/lib/src/core.rs @@ -1104,10 +1104,12 @@ impl Identity { match &self.routing { NodeRouting::Routers(_) => None, NodeRouting::Direct { ip, ports } | NodeRouting::Both { ip, ports, .. } => { - if let Some(port) = ports.get("ws") - && *port != 0 - { - Some((ip, port)) + if let Some(port) = ports.get("ws") { + if *port != 0 { + Some((ip, port)) + } else { + None + } } else { None } @@ -1118,10 +1120,12 @@ impl Identity { match &self.routing { NodeRouting::Routers(_) => None, NodeRouting::Direct { ip, ports } | NodeRouting::Both { ip, ports, .. } => { - if let Some(port) = ports.get("tcp") - && *port != 0 - { - Some((ip, port)) + if let Some(port) = ports.get("tcp") { + if *port != 0 { + Some((ip, port)) + } else { + None + } } else { None } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index e99aac02..d9306949 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(let_chains)] +//#![feature(let_chains)] pub mod core; pub mod eth; diff --git a/scripts/build-release.py b/scripts/build-release.py index 4fd9fa09..bd36ac1e 100755 --- a/scripts/build-release.py +++ b/scripts/build-release.py @@ -27,14 +27,32 @@ def build_and_move(feature, tmp_dir, architecture, os_name): zip_prefix = f"kinode-{architecture}-{os_name}" release_env = os.environ.copy() - release_env["CARGO_PROFILE_RELEASE_LTO"] = f"fat" - release_env["CARGO_PROFILE_RELEASE_CODEGEN_UNITS"] = f"1" - release_env["CARGO_PROFILE_RELEASE_STRIP"] = f"symbols" + release_env["CARGO_PROFILE_RELEASE_LTO"] = "fat" + release_env["CARGO_PROFILE_RELEASE_CODEGEN_UNITS"] = "1" + release_env["CARGO_PROFILE_RELEASE_STRIP"] = "symbols" if feature: - subprocess.run(["cargo", "+nightly", "build", "--release", "-p", "kinode", "--features", feature], check=True, env=release_env) + release_env["PATH_TO_PACKAGES_ZIP"] = f"../target/packages-{feature}.zip" + subprocess.run( + ["cargo", "run", "-p", "build_packages", "--", "--features", feature], + check=True, + #stdout=subprocess.PIPE, + #stderr=subprocess.PIPE, + ) + subprocess.run( + ["cargo", "build", "--release", "-p", "kinode", "--features", feature], + check=True, + env=release_env, + #stdout=subprocess.PIPE, + #stderr=subprocess.PIPE, + ) zip_name = f"{zip_prefix}-{feature}.zip" else: - subprocess.run(["cargo", "+nightly", "build", "--release", "-p", "kinode"], check=True, env=release_env) + subprocess.run(["cargo", "run", "-p", "build_packages"], check=True) + subprocess.run( + ["cargo", "build", "--release", "-p", "kinode"], + check=True, + env=release_env, + ) zip_name = f"{zip_prefix}.zip" # Move and rename the binary @@ -74,4 +92,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/scripts/build_packages/Cargo.toml b/scripts/build_packages/Cargo.toml new file mode 100644 index 00000000..ccb9b122 --- /dev/null +++ b/scripts/build_packages/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "build_packages" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.71" +clap = "4" +fs-err = "2.11" +kit = { git = "https://github.com/kinode-dao/kit", rev = "9c94b4b" } +serde_json = "1" +tokio = "1.28" +walkdir = "2.4" +zip = "0.6" diff --git a/scripts/build_packages/src/main.rs b/scripts/build_packages/src/main.rs new file mode 100644 index 00000000..02f15d77 --- /dev/null +++ b/scripts/build_packages/src/main.rs @@ -0,0 +1,185 @@ +use std::{ + io::{Cursor, Read, Write}, + path::{Path, PathBuf}, +}; + +use clap::{Arg, Command}; +use fs_err as fs; +use zip::write::FileOptions; + +fn zip_directory(dir_path: &Path) -> anyhow::Result> { + let mut writer = Cursor::new(Vec::new()); + let options = FileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .unix_permissions(0o755) + .last_modified_time(zip::DateTime::from_date_and_time(2023, 6, 19, 0, 0, 0).unwrap()); + { + let mut zip = zip::ZipWriter::new(&mut writer); + + for sub_entry in walkdir::WalkDir::new(dir_path) { + let sub_entry = sub_entry?; + let path = sub_entry.path(); + let name = path.strip_prefix(dir_path)?; + + if path.is_file() { + zip.start_file(name.to_string_lossy(), options)?; + let mut file = fs::File::open(path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + zip.write_all(&buffer)?; + } else if !name.as_os_str().is_empty() { + zip.add_directory(name.to_string_lossy(), options)?; + } + } + zip.finish()?; + } + + let zip_contents = writer.into_inner(); + Ok(zip_contents) +} + +fn build_and_zip_package( + entry_path: PathBuf, + parent_pkg_path: &str, + skip_frontend: bool, + features: &str, +) -> anyhow::Result<(PathBuf, String, Vec)> { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + kit::build::execute( + &entry_path, + skip_frontend, + false, + true, + features, + None, + None, + None, + vec![], + vec![], + false, + false, + false, + false, + ) + .await + .map_err(|e| anyhow::anyhow!("{:?}", e))?; + + let zip_contents = zip_directory(&Path::new(parent_pkg_path))?; + let zip_filename = format!("{}.zip", entry_path.file_name().unwrap().to_str().unwrap()); + Ok((entry_path, zip_filename, zip_contents)) + }) +} + +fn main() -> anyhow::Result<()> { + let matches = Command::new("build_packages") + .about("Build the core Kinode packages.") + .arg( + Arg::new("FEATURES") + .long("features") + .help("List of features to compile packages with") + .action(clap::ArgAction::Append), + ) + .arg( + Arg::new("SKIP_FRONTEND") + .long("skip-build-frontend") + .help("Skip building the frontend") + .action(clap::ArgAction::SetTrue), + ) + .arg( + Arg::new("OUTPUT_FILENAME") + .long("output-filename") + .help("Set output filename (default: packages-{features}.zip)") + .action(clap::ArgAction::Set), + ) + .get_matches(); + + // kinode/target/debug/build_package + let current_exe_dir = std::env::current_exe() // build_package + .unwrap(); + let top_level_dir = current_exe_dir + .parent() // debug/ + .unwrap() + .parent() // target/ + .unwrap() + .parent() // kinode/ + .unwrap(); + let kinode_dir = top_level_dir.join("kinode"); + let packages_dir = kinode_dir.join("packages"); + + let mut features = matches + .get_many::("FEATURES") + .unwrap_or_default() + .map(|s| s.to_owned()) + .collect::>(); + features.sort(); + let features = features.join(","); + + let skip_frontend = matches.get_flag("SKIP_FRONTEND"); + + let results: Vec)>> = fs::read_dir(&packages_dir)? + .filter_map(|entry| { + let entry_path = match entry { + Ok(e) => e.path(), + Err(_) => return None, + }; + let child_pkg_path = entry_path.join("pkg"); + if !child_pkg_path.exists() { + // don't run on, e.g., `.DS_Store` + return None; + } + Some(build_and_zip_package( + entry_path.clone(), + child_pkg_path.to_str().unwrap(), + skip_frontend, + &features, + )) + }) + .collect(); + + let mut file_to_metadata = std::collections::HashMap::new(); + + let target_dir = top_level_dir.join("target"); + let target_packages_dir = target_dir.join("packages"); + // fresh + if target_packages_dir.exists() { + fs::remove_dir_all(&target_packages_dir)?; + } + fs::create_dir_all(&target_packages_dir)?; + + for result in results { + match result { + Ok((entry_path, zip_filename, zip_contents)) => { + let metadata_path = entry_path.join("metadata.json"); + let metadata_contents = fs::read_to_string(&metadata_path)?; + let metadata_contents: serde_json::Value = + serde_json::from_str(&metadata_contents)?; + file_to_metadata.insert(zip_filename.clone(), metadata_contents); + let zip_path = target_packages_dir.join(&zip_filename); + fs::write(&zip_path, &zip_contents)?; + } + Err(e) => return Err(anyhow::anyhow!("{e:?}")), + } + } + + let file_to_metadata = serde_json::to_value(&file_to_metadata)?; + let file_to_metadata = serde_json::to_string_pretty(&file_to_metadata)?; + let file_to_metadata_path = target_packages_dir.join("file_to_metadata.json"); + fs::write(&file_to_metadata_path, file_to_metadata)?; + + let package_zip_file_name = match matches.get_one::("OUTPUT_FILENAME") { + Some(filename) => filename.to_string(), + None => { + if features.is_empty() { + "packages.zip".to_string() + } else { + format!("packages-{features}.zip") + } + } + }; + let package_zip_path = target_dir.join(package_zip_file_name); + let package_zip_contents = zip_directory(&target_packages_dir)?; + fs::write(package_zip_path, package_zip_contents)?; + + Ok(()) +}