From 330e2c425f05e418c0bff5a1ca75accba8cebb38 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente Date: Mon, 27 Jan 2020 22:09:28 +0900 Subject: [PATCH] More work on fixing bugs --- scripts/run_full_node.sh | 4 ++-- src/full_node.py | 40 +++++++++++++++++++++++++++++------ src/rpc/rpc_server.py | 28 ++++++++++++++++++++++-- src/server/start_full_node.py | 4 ++++ src/store.py | 35 +++++++++++++++++++++++++----- tests/rpc/test_rpc.py | 7 +++++- tests/test_store.py | 17 ++++++++++++--- 7 files changed, 115 insertions(+), 20 deletions(-) diff --git a/scripts/run_full_node.sh b/scripts/run_full_node.sh index c8867ffdeb17..8a69197d727d 100755 --- a/scripts/run_full_node.sh +++ b/scripts/run_full_node.sh @@ -2,7 +2,7 @@ . scripts/common.sh # Starts a full node -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -r 8555 -_run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 +_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 3 +# _run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 wait diff --git a/src/full_node.py b/src/full_node.py index 5c8126a01c25..3173ae015b55 100644 --- a/src/full_node.py +++ b/src/full_node.py @@ -206,6 +206,7 @@ class FullNode: log.info("Starting to perform sync with peers.") log.info("Waiting to receive tips from peers.") # TODO: better way to tell that we have finished receiving tips + log.error("STARTING SYNC") await asyncio.sleep(5) highest_weight: uint64 = uint64(0) tip_block: FullBlock @@ -492,22 +493,33 @@ class FullNode: block: Optional[FullBlock] = await self.store.get_potential_block( uint32(height) ) + log.warning(f"starting {height}") assert block is not None prev_block: Optional[FullBlock] = await self.store.get_potential_block( uint32(height - 1) ) - if prev_block is None: - prev_block = await self.store.get_block(block.prev_header_hash) - assert prev_block is not None - start = time.time() + try: + if prev_block is None: + print("IF") + prev_block = await self.store.get_block(block.prev_header_hash) + print(f"prev {prev_block.header_hash}") + else: + print("ELSE") + assert prev_block is not None + print("Assert passed") + start = time.time() + except Exception as e: + print(f"Excepted {e} {type(e)}") + log.warning(f"Locking {height}") async with self.store.lock: # The block gets permanantly added to the blockchain validated, pos = prevalidate_results[index] index += 1 - + log.warning(f"Receivinng block {height}") result = await self.blockchain.receive_block( block, prev_block.header_block, validated, pos ) + log.warning(f"Received {height}") if ( result == ReceiveBlockResult.INVALID_BLOCK or result == ReceiveBlockResult.DISCONNECTED_BLOCK @@ -516,15 +528,23 @@ class FullNode: log.info( f"Took {time.time() - start} seconds to validate and add block {block.height}." ) - # Always immediately add the block to the database, after updating blockchain state - await self.store.add_block(block) + log.warning(f"ADding{height}") + try: + # Always immediately add the block to the database, after updating blockchain state + await self.store.add_block(block) + except Exception as e: + print(f"Excepted 2 {e} {type(e)}") + log.warning(f"ADdded{height}") assert ( max([h.height for h in self.blockchain.get_current_tips()]) >= height ) + log.warning(f"Setting pot") await self.store.set_proof_of_time_estimate_ips( self.blockchain.get_next_ips(block.header_block) ) + log.warning(f"Exiting lock") + log.warning(f"Exited lock") assert max([h.height for h in self.blockchain.get_current_tips()]) == tip_height log.info(f"Finished sync up to height {tip_height}") @@ -1032,11 +1052,17 @@ class FullNode: if self.blockchain.cointains_block(header_hash): return + log.error("Locking block") async with self.store.lock: + log.error("getting sync mode") if await self.store.get_sync_mode(): + log.error("IN SYNC MODE") # Add the block to our potential tips list await self.store.add_potential_tip(block.block) + log.error("ADDED TIP, returning") return + else: + log.error("NOT IN SYNC MODE") prevalidate_block = await self.blockchain.pre_validate_blocks([block.block]) val, pos = prevalidate_block[0] diff --git a/src/rpc/rpc_server.py b/src/rpc/rpc_server.py index c8635293f053..3658acae1691 100644 --- a/src/rpc/rpc_server.py +++ b/src/rpc/rpc_server.py @@ -1,16 +1,18 @@ import dataclasses import json -from typing import Any, Callable, List, Optional +from typing import Any, Callable, List, Optional, Dict, Tuple from aiohttp import web +from blspy import PublicKey from src.full_node import FullNode from src.types.header_block import SmallHeaderBlock from src.types.full_block import FullBlock from src.types.peer_info import PeerInfo from src.types.challenge import Challenge -from src.util.ints import uint16, uint64 +from src.util.ints import uint16, uint32, uint64 +from src.consensus.block_rewards import calculate_block_reward from src.util.byte_types import hexstr_to_bytes @@ -169,6 +171,27 @@ class RpcApiHandler: self.stop_cb() return obj_to_response("") + async def get_pool_balances(self, request) -> web.Response: + """ + Retrieves the coinbase balances earned by all pools. + TODO: remove after transactions and coins are added. + """ + + async with self.full_node.store.lock: + ppks: List[ + Tuple[uint32, PublicKey] + ] = await self.full_node.store.get_pool_pks_hack() + + coin_balances: Dict[str, uint64] = {} + for height, pk in ppks: + pool_pk = f"0x{bytes(pk).hex()}" + if pool_pk not in coin_balances: + coin_balances[pool_pk] = uint64(0) + coin_balances[pool_pk] = uint64( + coin_balances[pool_pk] + calculate_block_reward(height) + ) + return obj_to_response(coin_balances) + async def get_heaviest_block_seen(self, request) -> web.Response: """ Returns the heaviest block ever seen, whether it's been added to the blockchain or not @@ -206,6 +229,7 @@ async def start_rpc_server(full_node: FullNode, stop_node_cb: Callable, rpc_port web.post("/open_connection", handler.open_connection), web.post("/close_connection", handler.close_connection), web.post("/stop_node", handler.stop_node), + web.post("/get_pool_balances", handler.get_pool_balances), web.post("/get_heaviest_block_seen", handler.get_heaviest_block_seen), ] ) diff --git a/src/server/start_full_node.py b/src/server/start_full_node.py index 913d932ee97c..c1d882e54818 100644 --- a/src/server/start_full_node.py +++ b/src/server/start_full_node.py @@ -134,14 +134,18 @@ async def main(): ) _ = await server.start_client(peer_info, None) + log.info(" 0 Closing ser") # Awaits for server and all connections to close await server.await_closed() + log.info(" 1 Closing ser") # Waits for the rpc server to close if rpc_cleanup is not None: await rpc_cleanup() + log.info(" 2 Closing ser") await store.close() + log.info(" 3 Closing ser") await asyncio.get_running_loop().shutdown_asyncgens() log.info("Node fully closed.") diff --git a/src/store.py b/src/store.py index ad3c2f20de0c..efcd2eecec3b 100644 --- a/src/store.py +++ b/src/store.py @@ -3,6 +3,7 @@ import logging import aiosqlite from typing import Dict, List, Optional, Tuple +from blspy import PublicKey from src.types.body import Body from src.types.full_block import FullBlock from src.types.header import HeaderData @@ -54,7 +55,9 @@ class FullNodeStore: self.db_name = db_name # All full blocks which have been added to the blockchain. Header_hash -> block + log.warn("CREATING CON") self.db = await aiosqlite.connect(self.db_name) + log.warn("CREATED CON") await self.db.execute( "CREATE TABLE IF NOT EXISTS blocks(height bigint, header_hash text PRIMARY KEY, block blob)" ) @@ -67,7 +70,7 @@ class FullNodeStore: # Headers await self.db.execute( "CREATE TABLE IF NOT EXISTS small_header_blocks(height bigint, header_hash " - "text PRIMARY KEY, small_header_block blob)" + "text PRIMARY KEY, pool_pk text, small_header_block blob)" ) # Height index so we can look up in order of height for sync purposes @@ -110,18 +113,28 @@ class FullNodeStore: await self.db.commit() async def add_block(self, block: FullBlock) -> None: + log.warning("1") await self.db.execute( "INSERT OR REPLACE INTO blocks VALUES(?, ?, ?)", (block.height, block.header_hash.hex(), bytes(block)), ) + log.warning("2") assert block.header_block.challenge is not None + log.warning("3") small_header_block: SmallHeaderBlock = SmallHeaderBlock( block.header_block.header, block.header_block.challenge ) - await self.db.execute( - ("INSERT OR REPLACE INTO small_header_blocks VALUES(?, ?, ?)"), - (block.height, block.header_hash.hex(), bytes(small_header_block),), + log.warning("Starting to add shb") + res = await self.db.execute( + ("INSERT OR REPLACE INTO small_header_blocks VALUES(?, ?, ?, ?)"), + ( + block.height, + block.header_hash.hex(), + bytes(block.header_block.proof_of_space.pool_pubkey).hex(), + bytes(small_header_block), + ), ) + log.warning(f"ADDED {res}") await self.db.commit() async def get_block(self, header_hash: bytes32) -> Optional[FullBlock]: @@ -150,7 +163,19 @@ class FullNodeStore: async def get_small_header_blocks(self) -> List[SmallHeaderBlock]: cursor = await self.db.execute("SELECT * from small_header_blocks") rows = await cursor.fetchall() - return [SmallHeaderBlock.from_bytes(row[2]) for row in rows] + return [SmallHeaderBlock.from_bytes(row[3]) for row in rows] + + async def get_pool_pks_hack(self) -> List[Tuple[uint32, PublicKey]]: + # TODO: this API call is a hack to allow us to see block winners. Replace with coin/UTXU set. + cursor = await self.db.execute("SELECT * from small_header_blocks") + rows = await cursor.fetchall() + return [ + ( + SmallHeaderBlock.from_bytes(row[3]).height, + PublicKey.from_bytes(bytes.fromhex(row[2])), + ) + for row in rows + ] async def add_potential_block(self, block: FullBlock) -> None: await self.db.execute( diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 32abd9272707..60211cb7776b 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1,5 +1,6 @@ import asyncio from typing import Any, Dict +import os import pytest @@ -42,8 +43,11 @@ class TestRpc: test_node_1_port = 21234 test_node_2_port = 21235 test_rpc_port = 21236 + db_filename = "blockchain_test" - store = await FullNodeStore.create("blockchain_test") + if os.path.isfile(db_filename): + os.remove(db_filename) + store = await FullNodeStore.create(db_filename) await store._clear_database() blocks = bt.get_consecutive_blocks(test_constants, 10, [], 10) b: Blockchain = await Blockchain.create({}, test_constants) @@ -81,6 +85,7 @@ class TestRpc: small_header_block = await client.get_header(state["lca"].header_hash) assert small_header_block.header == blocks[6].header_block.header + assert len(await client.get_pool_balances()) > 0 assert len(await client.get_connections()) == 0 full_node_2 = FullNode(store, b) diff --git a/tests/test_store.py b/tests/test_store.py index b3753db575c4..9b0e7793efff 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,6 +1,7 @@ import asyncio from secrets import token_bytes from typing import Any, Dict +import os import pytest from src.consensus.constants import constants @@ -37,9 +38,19 @@ class TestStore: @pytest.mark.asyncio async def test_basic_store(self): blocks = bt.get_consecutive_blocks(test_constants, 9, [], 9, b"0") + db_filename = "blockchain_test" + db_filename_2 = "blockchain_test_2" + db_filename_3 = "blockchain_test_3" - db = await FullNodeStore.create("blockchain_test") - db_2 = await FullNodeStore.create("blockchain_test_2") + if os.path.isfile(db_filename): + os.remove(db_filename) + if os.path.isfile(db_filename_2): + os.remove(db_filename_2) + if os.path.isfile(db_filename_3): + os.remove(db_filename_3) + + db = await FullNodeStore.create(db_filename) + db_2 = await FullNodeStore.create(db_filename_2) try: await db._clear_database() @@ -135,7 +146,7 @@ class TestStore: raise # Different database should have different data - db_3 = await FullNodeStore.create("blockchain_test_3") + db_3 = await FullNodeStore.create(db_filename_3) assert db_3.get_unfinished_block_leader() == (0, (1 << 64) - 1) await db.close()