From 71d81ddea74deeeae6d1666a55f0939cfe56472c Mon Sep 17 00:00:00 2001 From: Lyn <0xlynett@proton.me> Date: Fri, 19 Jul 2024 16:29:45 +0100 Subject: [PATCH 1/8] use build args in dockerfile --- Dockerfile | 6 +++--- README.md | 30 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd6d6b75..2f1f5822 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM debian:12-slim AS downloader +ARG VERSION WORKDIR /tmp/download RUN apt-get update -RUN apt-get install wget curl openssl jq unzip -y +RUN apt-get install unzip -y -ADD https://api.github.com/repos/kinode-dao/kinode/releases releases.json -RUN wget "https://github.com/kinode-dao/kinode/releases/download/$(cat releases.json | jq -r '.[0].tag_name')/kinode-x86_64-unknown-linux-gnu.zip" +ADD "https://github.com/kinode-dao/kinode/releases/download/${VERSION}/kinode-x86_64-unknown-linux-gnu.zip" kinode-x86_64-unknown-linux-gnu.zip RUN unzip kinode-x86_64-unknown-linux-gnu.zip FROM debian:12-slim diff --git a/README.md b/README.md index 2ffece5c..3e30146b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@

- Kinode is a general-purpose sovereign cloud computer, built for crypto. This repo contains the core runtime and processes. @@ -17,10 +16,10 @@ Then follow the instructions to [install it](https://book.kinode.org/install.htm If you have questions, join the [Kinode discord](https://discord.gg/TCgdca5Bjt) and drop us a line in `#dev-support`. - ## Setup On certain operating systems, you may need to install these dependencies if they are not already present: + - openssl-sys: https://docs.rs/crate/openssl-sys/0.9.19 - libclang 5.0: https://rust-lang.github.io/rust-bindgen/requirements.html @@ -54,6 +53,7 @@ No security audits of this crate have ever been performed. This software is unde Make sure not to use the same home directory for two nodes at once! You can use any name for the home directory: here we just use `home`. The `--` here separates cargo arguments from binary arguments. TODO: document feature flags in `--simulation-mode` + ```bash # OPTIONAL: --release flag cargo +nightly run -p kinode -- home @@ -66,6 +66,7 @@ On boot you will be prompted to navigate to `localhost:8080` (or whatever HTTP p By default, a node will use the [hardcoded providers](./kinode/src/eth/default_providers_mainnet.json) for the network it is booted on. A node can use a WebSockets RPC URL directly, or use another Kinode as a relay point. To adjust the providers a node uses, just create and modify the `.eth_providers` file in the node's home folder (set at boot). See the Kinode Book for more docs, and see the [default providers file here](./kinode/src/eth/default_providers_mainnet.json) for a template to create `.eth_providers`. You may also add a RPC provider or otherwise modify your configuration by sending messages from the terminal to the `eth:distro:sys` process. You can get one for free at `alchemy.com`. Use this message format to add a provider -- this will make your node's performance better when accessing a blockchain: + ``` m our@eth:distro:sys '{"AddProvider": {"chain_id": , "trusted": true, "provider": {"RpcUrl": ""}}}' ``` @@ -121,18 +122,18 @@ The `sys` publisher is not a real node ID, but it's also not a special case valu - CTRL+R to search history, CTRL+R again to toggle through search results, CTRL+G to cancel search - `m
''`: send an inter-process message.
is formatted as @. is formatted as ::. JSON containing spaces must be wrapped in single-quotes (`''`). - - Example: `m our@eth:distro:sys "SetPublic" -a 5` - - the '-a' flag is used to expect a response with a given timeout - - `our` will always be interpolated by the system as your node's name + - Example: `m our@eth:distro:sys "SetPublic" -a 5` + - the '-a' flag is used to expect a response with a given timeout + - `our` will always be interpolated by the system as your node's name - `hi `: send a text message to another node's command line. - - Example: `hi ben.os hello world` + - Example: `hi ben.os hello world` - `top `: display kernel debugging info about a process. Leave the process ID blank to display info about all processes and get the total number of running processes. - - Example: `top net:distro:sys` - - Example: `top` + - Example: `top net:distro:sys` + - Example: `top` - `cat `: print the contents of a file in the terminal - - Example: `cat /terminal:sys/pkg/scripts.json` + - Example: `cat /terminal:sys/pkg/scripts.json` - `echo `: print `text` to the terminal - - Example: `echo foo` + - Example: `echo foo` - `net_diagnostics`: print some useful networking diagnostic data - `peers`: print the peers the node currently hold connections with - `peer `: print the peer's PKI info, if it exists @@ -146,12 +147,15 @@ The image includes EXPOSE directives for TCP port `8080` and TCP port `9000`. Po If you are running a direct node, you must map port `9000` to the same port on the host and on your router. Otherwise, your Kinode will not be able to connect to the rest of the network as connection info is written to the chain, and this information is based on the view from inside the Docker container. To build a local Docker image, run the following command in this project root. -``` -docker build -t 0xlynett/kinode . + +```bash +# The `VERSION` may be replaced with the tag of a GitHub release +docker build -t 0xlynett/kinode . --build-arg VERSION=v0.8.6 ``` For example: -``` + +```bash docker volume create kinode-volume docker run -d -p 8080:8080 -it --name my-kinode \ From 6298afdd5808b479c787bfc73b2c48c915773a47 Mon Sep 17 00:00:00 2001 From: Lyn <0xlynett@proton.me> Date: Fri, 19 Jul 2024 16:32:37 +0100 Subject: [PATCH 2/8] return formatting to normal --- README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3e30146b..5d3350fb 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@

+ Kinode is a general-purpose sovereign cloud computer, built for crypto. This repo contains the core runtime and processes. @@ -16,10 +17,10 @@ Then follow the instructions to [install it](https://book.kinode.org/install.htm If you have questions, join the [Kinode discord](https://discord.gg/TCgdca5Bjt) and drop us a line in `#dev-support`. + ## Setup On certain operating systems, you may need to install these dependencies if they are not already present: - - openssl-sys: https://docs.rs/crate/openssl-sys/0.9.19 - libclang 5.0: https://rust-lang.github.io/rust-bindgen/requirements.html @@ -53,7 +54,6 @@ No security audits of this crate have ever been performed. This software is unde Make sure not to use the same home directory for two nodes at once! You can use any name for the home directory: here we just use `home`. The `--` here separates cargo arguments from binary arguments. TODO: document feature flags in `--simulation-mode` - ```bash # OPTIONAL: --release flag cargo +nightly run -p kinode -- home @@ -66,7 +66,6 @@ On boot you will be prompted to navigate to `localhost:8080` (or whatever HTTP p By default, a node will use the [hardcoded providers](./kinode/src/eth/default_providers_mainnet.json) for the network it is booted on. A node can use a WebSockets RPC URL directly, or use another Kinode as a relay point. To adjust the providers a node uses, just create and modify the `.eth_providers` file in the node's home folder (set at boot). See the Kinode Book for more docs, and see the [default providers file here](./kinode/src/eth/default_providers_mainnet.json) for a template to create `.eth_providers`. You may also add a RPC provider or otherwise modify your configuration by sending messages from the terminal to the `eth:distro:sys` process. You can get one for free at `alchemy.com`. Use this message format to add a provider -- this will make your node's performance better when accessing a blockchain: - ``` m our@eth:distro:sys '{"AddProvider": {"chain_id": , "trusted": true, "provider": {"RpcUrl": ""}}}' ``` @@ -122,18 +121,18 @@ The `sys` publisher is not a real node ID, but it's also not a special case valu - CTRL+R to search history, CTRL+R again to toggle through search results, CTRL+G to cancel search - `m
''`: send an inter-process message.
is formatted as @. is formatted as ::. JSON containing spaces must be wrapped in single-quotes (`''`). - - Example: `m our@eth:distro:sys "SetPublic" -a 5` - - the '-a' flag is used to expect a response with a given timeout - - `our` will always be interpolated by the system as your node's name + - Example: `m our@eth:distro:sys "SetPublic" -a 5` + - the '-a' flag is used to expect a response with a given timeout + - `our` will always be interpolated by the system as your node's name - `hi `: send a text message to another node's command line. - - Example: `hi ben.os hello world` + - Example: `hi ben.os hello world` - `top `: display kernel debugging info about a process. Leave the process ID blank to display info about all processes and get the total number of running processes. - - Example: `top net:distro:sys` - - Example: `top` + - Example: `top net:distro:sys` + - Example: `top` - `cat `: print the contents of a file in the terminal - - Example: `cat /terminal:sys/pkg/scripts.json` + - Example: `cat /terminal:sys/pkg/scripts.json` - `echo `: print `text` to the terminal - - Example: `echo foo` + - Example: `echo foo` - `net_diagnostics`: print some useful networking diagnostic data - `peers`: print the peers the node currently hold connections with - `peer `: print the peer's PKI info, if it exists @@ -147,7 +146,6 @@ The image includes EXPOSE directives for TCP port `8080` and TCP port `9000`. Po If you are running a direct node, you must map port `9000` to the same port on the host and on your router. Otherwise, your Kinode will not be able to connect to the rest of the network as connection info is written to the chain, and this information is based on the view from inside the Docker container. To build a local Docker image, run the following command in this project root. - ```bash # The `VERSION` may be replaced with the tag of a GitHub release docker build -t 0xlynett/kinode . --build-arg VERSION=v0.8.6 @@ -161,4 +159,4 @@ docker volume create kinode-volume docker run -d -p 8080:8080 -it --name my-kinode \ --mount type=volume,source=kinode-volume,destination=/kinode-home \ 0xlynett/kinode -``` +``` \ No newline at end of file From 5106f43c56d51024b9c8ed9fa3c786ac0be9bd66 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 09:44:59 -0700 Subject: [PATCH 3/8] bump versions to 0.8.7 --- Cargo.toml | 2 +- kinode/Cargo.toml | 2 +- lib/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e37cf88e..f4bb93ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kinode_lib" authors = ["KinodeDAO"] -version = "0.8.6" +version = "0.8.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" diff --git a/kinode/Cargo.toml b/kinode/Cargo.toml index c0f5ee85..5238cca4 100644 --- a/kinode/Cargo.toml +++ b/kinode/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kinode" authors = ["KinodeDAO"] -version = "0.8.6" +version = "0.8.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index b79fc8a7..909693c2 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lib" authors = ["KinodeDAO"] -version = "0.8.6" +version = "0.8.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" From 2259449321f0613c37db929886150d6dce0a4a3c Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 14:46:10 -0700 Subject: [PATCH 4/8] eth: avoid ws provider race condition by checking before replacing --- Cargo.lock | 6 +++--- kinode/src/eth/mod.rs | 7 +++++-- kinode/src/eth/subscription.rs | 8 ++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a695cf9c..da4ff72a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3203,7 +3203,7 @@ dependencies = [ [[package]] name = "kinode" -version = "0.8.6" +version = "0.8.7" dependencies = [ "aes-gcm", "alloy", @@ -3259,7 +3259,7 @@ dependencies = [ [[package]] name = "kinode_lib" -version = "0.8.6" +version = "0.8.7" dependencies = [ "lib", ] @@ -3376,7 +3376,7 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "lib" -version = "0.8.6" +version = "0.8.7" dependencies = [ "alloy", "kit", diff --git a/kinode/src/eth/mod.rs b/kinode/src/eth/mod.rs index 53d640aa..190e9739 100644 --- a/kinode/src/eth/mod.rs +++ b/kinode/src/eth/mod.rs @@ -626,8 +626,11 @@ async fn fulfill_request( is_replacement_successful = false; return (); }; - aps.urls.remove(index); - aps.urls.insert(0, url_provider.clone()); + let old_provider = aps.urls.remove(index); + match old_provider.pubsub { + None => aps.urls.insert(0, url_provider.clone()), + Some(_) => aps.urls.insert(0, old_provider), + } }); if !is_replacement_successful { verbose_print( diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 3c7e3797..ccf554d5 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -232,8 +232,12 @@ async fn build_subscription( is_replacement_successful = false; return (); }; - aps.urls.remove(index); - aps.urls.insert(0, url_provider.clone()); + let old_provider = aps.urls.remove(index); + match old_provider.pubsub { + None => aps.urls.insert(0, url_provider.clone()), + Some(_) => aps.urls.insert(0, old_provider), + } + }); if !is_replacement_successful { verbose_print( From e876891e1cf576d81efba9d34d3f5cd72f1bbc89 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 21:46:37 +0000 Subject: [PATCH 5/8] Format Rust code using rustfmt --- kinode/src/eth/subscription.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index ccf554d5..781f8441 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -237,7 +237,6 @@ async fn build_subscription( None => aps.urls.insert(0, url_provider.clone()), Some(_) => aps.urls.insert(0, old_provider), } - }); if !is_replacement_successful { verbose_print( From 81fa748d0a5f515ff41f0a7d780dc47af51ee0de Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 8 Aug 2024 16:05:13 +0300 Subject: [PATCH 6/8] eth provider: try and get more diagnostics to provider term --- kinode/src/eth/mod.rs | 55 ++++++++++++++++++++++++---------- kinode/src/eth/subscription.rs | 1 - 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/kinode/src/eth/mod.rs b/kinode/src/eth/mod.rs index 190e9739..b23742d2 100644 --- a/kinode/src/eth/mod.rs +++ b/kinode/src/eth/mod.rs @@ -354,19 +354,25 @@ async fn handle_message( Ok(EthSub { id, .. }) => id, Err(EthSubError { id, .. }) => id, }; - if let Some(sub_map) = state.active_subscriptions.get(&rsvp) { - if let Some(ActiveSub::Remote { - provider_node, - sender, - .. - }) = sub_map.get(&sub_id) - { - if provider_node == &km.source.node { - if let Ok(()) = sender.send(eth_sub_result).await { - // successfully sent a subscription update from a - // remote provider to one of our processes - return Ok(()); + if let Some(mut sub_map) = state.active_subscriptions.get_mut(&rsvp) { + if let Some(sub) = sub_map.get(&sub_id) { + if let ActiveSub::Remote { + provider_node, + sender, + .. + } = sub + { + if provider_node == &km.source.node { + if let Ok(()) = sender.send(eth_sub_result).await { + // successfully sent a subscription update from a + // remote provider to one of our processes + return Ok(()); + } } + // failed to send subscription update to process, + // unsubscribe from provider and close + sub.close(sub_id, state).await; + sub_map.remove(&sub_id); } } } @@ -479,12 +485,29 @@ async fn handle_eth_action( .await; } EthAction::UnsubscribeLogs(sub_id) => { - let mut sub_map = state - .active_subscriptions - .entry(km.source.clone()) - .or_insert(HashMap::new()); + let Some(mut sub_map) = state.active_subscriptions.get_mut(&km.source) else { + verbose_print( + &state.print_tx, + "eth: got unsubscribe but no matching subscription found", + ) + .await; + error_message( + &state.our, + km.id, + km.source, + EthError::MalformedRequest, + &state.send_to_loop, + ) + .await; + return Ok(()); + }; if let Some(sub) = sub_map.remove(&sub_id) { sub.close(sub_id, state).await; + verbose_print( + &state.print_tx, + &format!("eth: closed subscription {} for {}", sub_id, km.source.node), + ) + .await; kernel_message( &state.our, km.id, diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 781f8441..92a3216a 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -113,7 +113,6 @@ pub async fn create_new_subscription( let (keepalive_err_sender, keepalive_err_receiver) = tokio::sync::mpsc::channel(1); response_channels.insert(keepalive_km_id, keepalive_err_sender); - let response_channels = response_channels.clone(); subs.insert( remote_sub_id, ActiveSub::Remote { From 19a750b5e5c86f3fd6fe16d4e0213c76dcd61644 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 8 Aug 2024 18:15:24 +0300 Subject: [PATCH 7/8] eth: manually unsubscribe if we break remote_subscription due to error --- kinode/src/eth/subscription.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 92a3216a..875de503 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -470,7 +470,7 @@ async fn maintain_remote_subscription( true, Some(30), IncomingReq::SubKeepalive(remote_sub_id), - &send_to_loop, + send_to_loop, ).await; } _incoming = net_error_rx.recv() => { @@ -487,6 +487,23 @@ async fn maintain_remote_subscription( } } }; + // tell provider node we don't need their services anymore + // (in case they did not close the subscription on their side, + // such as in the 2-hour timeout case) + kernel_message( + our, + rand::random(), + Address { + node: provider_node.to_string(), + process: ETH_PROCESS_ID.clone(), + }, + None, + true, + None, + EthAction::UnsubscribeLogs(sub_id), + send_to_loop, + ) + .await; active_subscriptions .entry(target.clone()) .and_modify(|sub_map| { From a288fc1fd4f508e0d4ca16d40afc55c7d0281295 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 8 Aug 2024 19:12:13 +0300 Subject: [PATCH 8/8] more diagnostics, fixes --- kinode/src/eth/mod.rs | 37 +++++++++++++++++++++++++++------- kinode/src/eth/subscription.rs | 2 +- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/kinode/src/eth/mod.rs b/kinode/src/eth/mod.rs index b23742d2..cec64a51 100644 --- a/kinode/src/eth/mod.rs +++ b/kinode/src/eth/mod.rs @@ -322,6 +322,7 @@ async fn handle_message( ) .await; } + Ok(()) } Message::Request(req) => { let timeout = req.expects_response.unwrap_or(60); @@ -330,7 +331,7 @@ async fn handle_message( }; match req { IncomingReq::EthAction(eth_action) => { - return handle_eth_action(state, km, timeout, eth_action).await; + handle_eth_action(state, km, timeout, eth_action).await } IncomingReq::EthConfigAction(eth_config_action) => { kernel_message( @@ -344,10 +345,16 @@ async fn handle_message( &state.send_to_loop, ) .await; + Ok(()) } IncomingReq::EthSubResult(eth_sub_result) => { // forward this to rsvp, if we have the sub id in our active subs let Some(rsvp) = km.rsvp else { + verbose_print( + &state.print_tx, + "eth: got eth_sub_result with no rsvp, ignoring", + ) + .await; return Ok(()); // no rsvp, no need to forward }; let sub_id = match eth_sub_result { @@ -371,8 +378,14 @@ async fn handle_message( } // failed to send subscription update to process, // unsubscribe from provider and close + verbose_print( + &state.print_tx, + "eth: got eth_sub_result but provider node did not match or local sub was already closed", + ) + .await; sub.close(sub_id, state).await; sub_map.remove(&sub_id); + return Ok(()); } } } @@ -380,13 +393,16 @@ async fn handle_message( // so they can stop sending us updates verbose_print( &state.print_tx, - "eth: got eth_sub_result but no matching sub found, unsubscribing", + &format!( + "eth: got eth_sub_result but no matching sub {} found, unsubscribing", + sub_id + ), ) .await; kernel_message( - &state.our.clone(), + &state.our, km.id, - km.source.clone(), + km.source, None, true, None, @@ -394,6 +410,7 @@ async fn handle_message( &state.send_to_loop, ) .await; + Ok(()) } IncomingReq::SubKeepalive(sub_id) => { // source expects that we have a local sub for them with this id @@ -426,11 +443,11 @@ async fn handle_message( &state.send_to_loop, ) .await; + Ok(()) } } } } - Ok(()) } async fn handle_eth_action( @@ -488,7 +505,10 @@ async fn handle_eth_action( let Some(mut sub_map) = state.active_subscriptions.get_mut(&km.source) else { verbose_print( &state.print_tx, - "eth: got unsubscribe but no matching subscription found", + &format!( + "eth: got unsubscribe from {} but no subscription found", + km.source + ), ) .await; error_message( @@ -522,7 +542,10 @@ async fn handle_eth_action( } else { verbose_print( &state.print_tx, - "eth: got unsubscribe but no matching subscription found", + &format!( + "eth: got unsubscribe from {} but no subscription {} found", + km.source, sub_id + ), ) .await; error_message( diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 875de503..21e78021 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -500,7 +500,7 @@ async fn maintain_remote_subscription( None, true, None, - EthAction::UnsubscribeLogs(sub_id), + EthAction::UnsubscribeLogs(remote_sub_id), send_to_loop, ) .await;