kns_indexer -> kimap_indexer

This commit is contained in:
bitful-pannul 2024-06-13 01:09:12 -07:00
parent 2aac98ed14
commit 37e7dde388
12 changed files with 275 additions and 270 deletions

View File

@ -19,7 +19,7 @@ members = [
"kinode/packages/chess/chess",
"kinode/packages/homepage/homepage",
"kinode/packages/kino_updates/widget",
"kinode/packages/kns_indexer/kns_indexer", "kinode/packages/kns_indexer/get_block", "kinode/packages/kns_indexer/state",
"kinode/packages/kimap_indexer/kimap_indexer", "kinode/packages/kimap_indexer/get_block", "kinode/packages/kimap_indexer/state",
"kinode/packages/settings/settings",
"kinode/packages/terminal/terminal",
"kinode/packages/terminal/alias", "kinode/packages/terminal/cat", "kinode/packages/terminal/echo", "kinode/packages/terminal/hi", "kinode/packages/terminal/kfetch", "kinode/packages/terminal/kill", "kinode/packages/terminal/m", "kinode/packages/terminal/top",

View File

@ -53,7 +53,7 @@ pub const APP_SHARE_TIMEOUT: u64 = 120; // 120s
#[cfg(not(feature = "simulation-mode"))]
const CONTRACT_ADDRESS: &str = "0x52185B6a6017E6f079B994452F234f7C2533787B"; // optimism
#[cfg(feature = "simulation-mode")]
const CONTRACT_ADDRESS: &str = "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318"; // local
const CONTRACT_ADDRESS: &str = "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707"; // note temp kimap address!
#[cfg(not(feature = "simulation-mode"))]
const CONTRACT_FIRST_BLOCK: u64 = 118_590_088;

View File

@ -1,4 +1,4 @@
interface kns-indexer {
interface kimap-indexer {
/// IndexerRequests are used to query discrete information from the indexer
/// for example, if you want to know the human readable name for a namehash,
/// you would send a NamehashToName request.
@ -34,7 +34,7 @@ interface kns-indexer {
}
}
world kns-indexer-sys-v0 {
import kns-indexer;
world kimap-indexer-sys-v0 {
import kimap-indexer;
include process-v0;
}

View File

@ -1,6 +1,6 @@
[package]
name = "kns_indexer"
version = "0.3.0"
name = "kimap_indexer"
version = "0.1.0"
edition = "2021"
[features]

View File

@ -1,19 +1,21 @@
use crate::kinode::process::kns_indexer::{
use crate::kinode::process::kimap_indexer::{
GetStateRequest, IndexerRequests, NamehashToNameRequest, NodeInfoRequest,
};
use alloy_sol_types::{sol, SolEvent};
use alloy_primitives::FixedBytes;
use alloy_sol_types::{sol, SolCall, SolEvent};
use kinode_process_lib::{
await_message, call_init, eth, net::KnsUpdate, println, Address, Message, Request, Response,
await_message, call_init,
eth::{self, Provider, TransactionInput, TransactionRequest},
net, println, Address, Message, Request, Response,
};
use serde::{Deserialize, Serialize};
use std::{
collections::{hash_map::HashMap, BTreeMap},
net::{IpAddr, Ipv4Addr, Ipv6Addr},
str::FromStr,
};
wit_bindgen::generate!({
path: "target/wit",
world: "kns-indexer-sys-v0",
world: "kimap-indexer-sys-v0",
generate_unused_types: true,
additional_derives: [serde::Deserialize, serde::Serialize],
});
@ -29,9 +31,9 @@ const CHAIN_ID: u64 = 10; // optimism
const CHAIN_ID: u64 = 31337; // local
#[cfg(not(feature = "simulation-mode"))]
const KNS_FIRST_BLOCK: u64 = 114_923_786; // optimism
const KIMAP_FIRST_BLOCK: u64 = 114_923_786; // optimism, adjust
#[cfg(feature = "simulation-mode")]
const KNS_FIRST_BLOCK: u64 = 1; // local
const KIMAP_FIRST_BLOCK: u64 = 1; // local
#[derive(Clone, Debug, Serialize, Deserialize)]
struct State {
@ -40,80 +42,27 @@ struct State {
contract_address: String,
// namehash to human readable name
names: HashMap<String, String>,
// temporary hash->name mapping
hashes: HashMap<String, String>,
// notehash->note mapping
// note, do not need this here, adding relevant notes directly to KNS rn.
// notes: HashMap<String, Note>,
// NOTE: wip knsUpdates not 1-1 rn
nodes: HashMap<String, Node>,
// human readable name to most recent on-chain routing information as json
// TODO: optional params knsUpdate? also include tba.
nodes: HashMap<String, net::KnsUpdate>,
// last block we have an update from
block: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct Node {
pub name: String, // actual username / domain name
pub hash: String, // hex namehash of node
// pub tba: String, can query for this as events come in too.
pub parent_hash: String, // hex namehash of parent node, top level = 0x0?
pub public_key: Option<String>,
pub ips: Vec<String>,
pub ports: BTreeMap<String, u16>,
pub routers: Vec<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct IndexedNote {
pub name: String, // note full name
pub hash: String, // hex namehash of note (in key already?)
pub node_hash: String, // hex namehash of node
pub value: String, // note value, hex/bytes instead?
}
sol! {
// Kimap events
event Mint(bytes32 indexed parenthash, bytes32 indexed childhash, bytes name);
event Fact(bytes32 indexed nodehash, bytes32 indexed facthash, bytes note, bytes data);
event Note(bytes32 indexed nodehash, bytes32 indexed notehash, bytes note, bytes data);
event Edit(bytes32 indexed note, bytes data);
// event Edit(bytes32 indexed note, bytes data);
event Zero(address indexed zerotba);
}
fn subscribe_to_logs(eth_provider: &eth::Provider, from_block: u64, filter: eth::Filter) {
loop {
match eth_provider.subscribe(1, filter.clone().from_block(from_block)) {
Ok(()) => break,
Err(_) => {
println!("failed to subscribe to chain! trying again in 5s...");
std::thread::sleep(std::time::Duration::from_secs(5));
continue;
}
}
}
println!("subscribed to logs successfully");
}
// TEMP. Either remove when event reimitting working with anvil,
// or refactor into better structure(!)
fn add_temp_hardcoded_tlzs(state: &mut State) {
// add some hardcoded top level zones
state.names.insert(
"0xdeeac81ae11b64e7cab86d089c306e5d223552a630f02633ce170d2786ff1bbd".to_string(),
"os".to_string(),
);
state.hashes.insert(
"os".to_string(),
"0xdeeac81ae11b64e7cab86d089c306e5d223552a630f02633ce170d2786ff1bbd".to_string(),
);
state.names.insert(
"0x137d9e4cc0479164d40577620cb3b41b083c6e8dbf58f8523be76d207d6fd8ea".to_string(),
"dev".to_string(),
);
state.hashes.insert(
"dev".to_string(),
"0x137d9e4cc0479164d40577620cb3b41b083c6e8dbf58f8523be76d207d6fd8ea".to_string(),
function get (
bytes32 node
) external view returns (
address tokenBoundAccount,
address tokenOwner,
bytes memory note
);
}
@ -126,18 +75,14 @@ fn init(our: Address) {
// us to quickly verify we have the updated mapping with root hash, but right
// now it's tricky to recover from missed events.
let mut state = State {
let state = State {
chain_id: CHAIN_ID,
contract_address: KIMAP_ADDRESS.to_string(),
names: HashMap::new(),
hashes: HashMap::new(),
nodes: HashMap::new(),
// notes: HashMap::new(),
block: KNS_FIRST_BLOCK,
names: HashMap::new(),
block: KIMAP_FIRST_BLOCK,
};
add_temp_hardcoded_tlzs(&mut state);
match main(our, state) {
Ok(_) => {}
Err(e) => {
@ -147,6 +92,9 @@ fn init(our: Address) {
}
fn main(our: Address, mut state: State) -> anyhow::Result<()> {
#[cfg(feature = "simulation-mode")]
add_temp_hardcoded_tlzs(&mut state);
let filter = eth::Filter::new()
.address(state.contract_address.parse::<eth::Address>().unwrap())
.from_block(state.block - 1)
@ -155,13 +103,13 @@ fn main(our: Address, mut state: State) -> anyhow::Result<()> {
"Mint(bytes32,bytes32,bytes)",
"Fact(bytes32,bytes32,bytes,bytes)",
"Note(bytes32,bytes32,bytes,bytes)",
"Edit(bytes32,bytes)",
// "Edit(bytes32,bytes)",
"Zero(address)",
]);
// 60s timeout -- these calls can take a long time
// if they do time out, we try them again
let eth_provider = eth::Provider::new(state.chain_id, 60);
let eth_provider: eth::Provider = eth::Provider::new(state.chain_id, 60);
println!(
"subscribing, state.block: {}, chain_id: {}",
@ -176,7 +124,7 @@ fn main(our: Address, mut state: State) -> anyhow::Result<()> {
match eth_provider.get_logs(&filter) {
Ok(logs) => {
for log in logs {
match handle_log(&our, &mut state, &log) {
match handle_log(&our, &mut state, &log, &eth_provider) {
Ok(_) => {}
Err(e) => {
println!("log-handling error! {e:?}");
@ -219,23 +167,24 @@ fn main(our: Address, mut state: State) -> anyhow::Result<()> {
&filter,
)?;
} else {
let Ok(request) = serde_json::from_slice::<IndexerRequests>(&body) else {
let Ok(request) = serde_json::from_slice(&body) else {
println!("got invalid message");
continue;
};
match request {
// IndexerRequests, especially NamehashToName, relevant anymore? if they're mostly queried from the net runtime?
IndexerRequests::NamehashToName(NamehashToNameRequest { ref hash, block }) => {
if block <= state.block {
Response::new()
.body(serde_json::to_vec(&state.names.get(hash))?)
.send()?;
} else {
pending_requests
.entry(block)
.or_insert(vec![])
.push(request);
}
// if block <= state.block {
// Response::new()
// .body(serde_json::to_vec(&state.names.get(hash))?)
// .send()?;
// } else {
// pending_requests
// .entry(block)
// .or_insert(vec![])
// .push(request);
// }
}
IndexerRequests::NodeInfo(NodeInfoRequest { ref name, block }) => {
if block <= state.block {
@ -279,7 +228,7 @@ fn handle_eth_message(
match eth_result {
Ok(eth::EthSub { result, .. }) => {
if let eth::SubscriptionResult::Log(log) = result {
match handle_log(our, state, &log) {
match handle_log(our, state, &log, eth_provider) {
Ok(_) => {}
Err(e) => {
println!("log-handling error! {e:?}");
@ -333,6 +282,179 @@ fn handle_eth_message(
Ok(())
}
fn handle_log(
our: &Address,
state: &mut State,
log: &eth::Log,
eth_provider: &Provider,
) -> anyhow::Result<()> {
let mut node: Option<String> = None;
match log.topics()[0] {
Mint::SIGNATURE_HASH => {
let decoded = Mint::decode_log_data(log.data(), true).unwrap();
let parent_hash = decoded.parenthash.to_string();
let child_hash = decoded.childhash.to_string();
let label = String::from_utf8(decoded.name.to_vec())?;
let name = get_full_name(state, &label, &parent_hash);
let get_call = getCall {
node: FixedBytes::<32>::from_str(&child_hash).unwrap(),
}
.abi_encode();
let get_tx = TransactionRequest::default()
.to(state.contract_address.parse::<eth::Address>().unwrap())
.input(TransactionInput::new(get_call.into()));
let res = eth_provider
.call(get_tx, None)
.map_err(|e| anyhow::anyhow!("tba get_call error: {:?}", e))?;
let get_return = getCall::abi_decode_returns(&res, false)?;
let tba = get_return.tokenBoundAccount.to_string();
state.names.insert(child_hash.clone(), name.clone());
println!(
"got mint, name: {}, child_hash: {}, tba: {}",
name, child_hash, tba
);
state
.nodes
.entry(name.clone())
.or_insert_with(|| net::KnsUpdate {
name: name.clone(),
// tbh owner should be a separate one from tba. (although we won't index transfers so won't be up to date)
owner: tba,
node: child_hash.clone(),
public_key: String::new(),
ips: Vec::new(),
ports: BTreeMap::new(),
routers: Vec::new(),
});
node = Some(name);
}
Note::SIGNATURE_HASH => {
let decoded = Note::decode_log_data(log.data(), true).unwrap();
let note = String::from_utf8(decoded.note.to_vec())?;
let _note_hash: String = decoded.notehash.to_string();
let node_hash = decoded.nodehash.to_string();
let name = get_node_name(state, &node_hash);
println!(
"got note, from name: {}, note: {}, note_hash: {}",
name, note, node_hash
);
match note.as_str() {
"~ws-port" => {
let port_bytes = decoded.data.to_vec();
if port_bytes.len() != 2 {
return Err(anyhow::anyhow!("Invalid data length for port number"));
} else {
let port = u16::from_be_bytes([port_bytes[0], port_bytes[1]]);
state.nodes.entry(name.clone()).and_modify(|node| {
node.ports.insert("ws".to_string(), port);
// port defined, -> direct
node.routers = vec![];
});
node = Some(name.clone());
}
}
"~tcp-port" => {
let port_str: String = String::from_utf8(decoded.data.to_vec())?;
// from be_bytes soon, for debugging now.
let port = port_str.parse::<u16>()?;
state.nodes.entry(name.clone()).and_modify(|node| {
node.ports.insert("tcp".to_string(), port);
// port defined, -> direct
node.routers = vec![];
});
node = Some(name.clone());
}
"~net-key" => {
state.nodes.entry(name.clone()).and_modify(|node| {
let pubkey = hex::encode(&decoded.data);
node.public_key = pubkey;
});
node = Some(name.clone());
}
"~routers" => {
state.nodes.entry(name.clone()).and_modify(|node| {
if let Ok(routers) = decode_routers(&decoded.data) {
node.routers = routers;
// -> indirect
node.ports = BTreeMap::new();
node.ips = vec![];
}
});
node = Some(name.clone());
}
"~ip" => {
// todo big-endian ipv6 encoding, current for debugging.
let ip = String::from_utf8(decoded.data.to_vec())?;
state.nodes.entry(name.clone()).and_modify(|node| {
node.ips.push(ip);
// -> direct
node.routers = vec![];
});
node = Some(name.clone());
}
_ => {}
}
}
_ => {}
}
if let Some(node) = node {
if let Some(node_info) = state.nodes.get(&node) {
if node_info.public_key != ""
&& ((!node_info.ips.is_empty() && !node_info.ports.is_empty())
|| node_info.routers.len() > 0)
{
println!("sending kns update for node: {}", node_info.node);
Request::new()
.target((&our.node, "net", "distro", "sys"))
.body(rmp_serde::to_vec(&net::NetAction::KnsUpdate(
node_info.clone(),
))?)
.send()?;
}
}
}
Ok(())
}
// helpers
fn get_node_name(state: &mut State, parent_hash: &str) -> String {
let mut current_hash = parent_hash;
let mut components = Vec::new(); // Collect components in a vector
let mut visited_hashes = std::collections::HashSet::new();
while let Some(parent_name) = state.names.get(current_hash) {
if !visited_hashes.insert(current_hash) {
break;
}
components.push(parent_name.clone());
// Update current_hash to the parent's hash for the next iteration
if let Some(new_parent_hash) = state.names.get(parent_name) {
current_hash = new_parent_hash;
} else {
break;
}
}
components.reverse();
components.join(".")
}
/// note, unlike get_node_name, includes the label.
/// e.g label "testing" with parenthash_resolved = "parent.os" would return "testing.parent.os"
fn get_full_name(state: &mut State, label: &str, parent_hash: &str) -> String {
let mut current_hash = parent_hash;
let mut full_name = label.to_string();
@ -344,7 +466,8 @@ fn get_full_name(state: &mut State, label: &str, parent_hash: &str) -> String {
}
full_name = format!("{}.{}", full_name, parent_name);
if let Some(new_parent_hash) = state.hashes.get(parent_name) {
// Update current_hash to the parent's hash for the next iteration
if let Some(new_parent_hash) = state.names.get(parent_name) {
current_hash = new_parent_hash;
} else {
break;
@ -354,182 +477,64 @@ fn get_full_name(state: &mut State, label: &str, parent_hash: &str) -> String {
full_name
}
/// Decodes bytes into an IP address, expecting either 4 bytes (IPv4) or 16 bytes (IPv6).
fn decode_bytes_to_ip(bytes: &[u8]) -> anyhow::Result<IpAddr> {
match bytes.len() {
4 => Ok(IpAddr::V4(Ipv4Addr::new(
bytes[0], bytes[1], bytes[2], bytes[3],
))),
16 => {
let addr = Ipv6Addr::from(
<[u8; 16]>::try_from(bytes)
.map_err(|_| anyhow::anyhow!("Invalid length for IPv6"))?,
);
Ok(IpAddr::V6(addr))
}
_ => Err(anyhow::anyhow!("Invalid byte length for IP address")),
}
}
/// Decodes bytes into a u16 port number, expecting exactly 2 bytes.
fn decode_bytes_to_port(bytes: &[u8]) -> anyhow::Result<u16> {
if bytes.len() == 2 {
Ok(u16::from_be_bytes([bytes[0], bytes[1]]))
} else {
Err(anyhow::anyhow!("Invalid byte length for port number"))
}
// TEMP. Either remove when event reimitting working with anvil,
// or refactor into better structure(!)
#[cfg(feature = "simulation-mode")]
fn add_temp_hardcoded_tlzs(state: &mut State) {
// add some hardcoded top level zones
state.names.insert(
"0xdeeac81ae11b64e7cab86d089c306e5d223552a630f02633ce170d2786ff1bbd".to_string(),
"os".to_string(),
);
state.names.insert(
"0x137d9e4cc0479164d40577620cb3b41b083c6e8dbf58f8523be76d207d6fd8ea".to_string(),
"dev".to_string(),
);
}
/// Decodes bytes into an array of node identities, expecting UTF-8 encoded strings separated by newlines.
fn decode_routers(bytes: &[u8]) -> anyhow::Result<Vec<String>> {
let data = std::str::from_utf8(bytes).map_err(|_| anyhow::anyhow!("Invalid UTF-8 data"))?;
let routers = data
.lines() // Assuming each router is separated by a newline
.map(str::to_owned)
.collect();
fn decode_routers(data: &[u8]) -> anyhow::Result<Vec<String>> {
let data_str = std::str::from_utf8(data)?;
let routers = data_str.split(',').map(str::to_owned).collect();
Ok(routers)
}
fn handle_log(our: &Address, state: &mut State, log: &eth::Log) -> anyhow::Result<()> {
let mut node: Option<String> = None;
// /// Decodes bytes into an IP address, expecting either 4 bytes (IPv4) or 16 bytes (IPv6).
// fn decode_bytes_to_ip(bytes: &[u8]) -> anyhow::Result<IpAddr> {
// match bytes.len() {
// 4 => Ok(IpAddr::V4(Ipv4Addr::new(
// bytes[0], bytes[1], bytes[2], bytes[3],
// ))),
// 16 => {
// let addr = Ipv6Addr::from(
// <[u8; 16]>::try_from(bytes)
// .map_err(|_| anyhow::anyhow!("Invalid length for IPv6"))?,
// );
// Ok(IpAddr::V6(addr))
// }
// _ => Err(anyhow::anyhow!("Invalid byte length for IP address")),
// }
// }
match log.topics()[0] {
Mint::SIGNATURE_HASH => {
let decoded = Mint::decode_log_data(log.data(), true).unwrap();
// /// Decodes bytes into a u16 port number, expecting exactly 2 bytes.
// fn decode_bytes_to_port(bytes: &[u8]) -> anyhow::Result<u16> {
// if bytes.len() == 2 {
// Ok(u16::from_be_bytes([bytes[0], bytes[1]]))
// } else {
// Err(anyhow::anyhow!("Invalid byte length for port number"))
// }
// }
let name = String::from_utf8(decoded.name.to_vec())?;
let parent_hash = decoded.parenthash.to_string();
let node_hash = decoded.childhash.to_string();
println!(
"got name, node_hash, parent_node: {:?}, {:?}, {:?}",
name, node_hash, parent_hash
);
let full_name = get_full_name(state, &name, &parent_hash);
println!("got full hierarchical name: {:?}", full_name);
state.names.insert(node_hash.clone(), full_name);
node = Some(node_hash.clone());
state.hashes.insert(node_hash, name);
}
Note::SIGNATURE_HASH => {
let decoded = Note::decode_log_data(log.data(), true).unwrap();
let note = String::from_utf8(decoded.note.to_vec())?;
let _notehash: String = decoded.notehash.to_string();
let node_hash = decoded.nodehash.to_string();
let full_note_name = get_full_name(state, &note, &node_hash);
println!("got full note name: {:?}", full_note_name);
// println!("note hash: {:?}", _notehash);
// println!("node_hash: {:?}", node_hash);
// let note_value = String::from_utf8(decoded.data.to_vec())?;
// println!("got note value: {:?}", note_value);
// generalize, cleaner system
match note.as_str() {
"~ws-port" => {
let port = decode_bytes_to_port(&decoded.data)?;
state.nodes.entry(node_hash.clone()).and_modify(|node| {
node.ports.insert("ws".to_string(), port);
// port defined, -> direct
node.routers = vec![];
});
}
"~tcp-port" => {
let port = decode_bytes_to_port(&decoded.data)?;
state.nodes.entry(node_hash.clone()).and_modify(|node| {
node.ports.insert("tcp".to_string(), port);
// port defined, -> direct
node.routers = vec![];
});
}
"~net-key" => {
state.nodes.entry(node_hash.clone()).and_modify(|node| {
let pubkey = hex::encode(&decoded.data);
node.public_key = Some(pubkey);
});
}
"~routers" => {
state.nodes.entry(node_hash.clone()).and_modify(|node| {
if let Ok(routers) = decode_routers(&decoded.data) {
node.routers = routers;
// -> indirect
node.ports = BTreeMap::new();
node.ips = vec![];
}
});
}
"~ip" => {
state.nodes.entry(node_hash.clone()).and_modify(|node| {
if let Ok(ip) = decode_bytes_to_ip(&decoded.data) {
node.ips.push(ip.to_string());
// -> direct
node.routers = vec![];
}
});
}
_ => {}
}
}
Edit::SIGNATURE_HASH => {
let _decoded = Edit::decode_log_data(log.data(), true).unwrap();
println!("got updated note!");
// todo get saved nodename etc and update if needed
// recursion?
}
Zero::SIGNATURE_HASH => {
// println!("got zeroth log: {:?}", log);
}
_ => {
println!("got other log: {:?}", log);
}
}
if let Some(node) = node {
if let Some(info) = state.nodes.get(&node) {
if let Some(pubkey) = &info.public_key {
// indirect case:
if !info.routers.is_empty() {
// send to KNS
let update = KnsUpdate {
name: info.name.clone(),
owner: "".to_string(),
node: info.hash.clone(),
routers: info.routers.clone(),
public_key: pubkey.clone(),
ips: info.ips.clone(),
ports: info.ports.clone(),
};
Request::new()
.target((&our.node, "net", "distro", "sys"))
.body(rmp_serde::to_vec(&update)?)
.send()?;
} else if info.ips.len() > 0 && info.ports.len() > 0 {
// send to KNS
let update = KnsUpdate {
name: info.name.clone(),
owner: "".to_string(),
node: info.hash.clone(),
routers: info.routers.clone(),
public_key: pubkey.clone(),
ips: info.ips.clone(),
ports: info.ports.clone(),
};
Request::new()
.target((&our.node, "net", "distro", "sys"))
.body(rmp_serde::to_vec(&update)?)
.send()?;
}
fn subscribe_to_logs(eth_provider: &eth::Provider, from_block: u64, filter: eth::Filter) {
loop {
match eth_provider.subscribe(1, filter.clone().from_block(from_block)) {
Ok(()) => break,
Err(_) => {
println!("failed to subscribe to chain! trying again in 5s...");
std::thread::sleep(std::time::Duration::from_secs(5));
continue;
}
}
}
Ok(())
println!("subscribed to logs successfully");
}

View File

@ -1,9 +1,9 @@
{
"name": "KNS Indexer",
"name": "KiMap Indexer",
"description": "Kinode OS PKI indexer",
"image": "",
"properties": {
"package_name": "kns_indexer",
"package_name": "kimap_indexer",
"current_version": "0.2.1",
"publisher": "sys",
"mirrors": [],