diff --git a/README.md b/README.md index fbd158892094..a77a37c08ad6 100644 --- a/README.md +++ b/README.md @@ -229,8 +229,6 @@ Due to the nature of proof of space lookups by the harvester in the current alph the number of plots on a physical drive to 50 or less. This limit should significantly increase before beta. You can also run the simulation, which runs all servers and multiple full nodes, locally, at once. -If you want to run the simulation, change the introducer ip in ./config/config.yaml so that the -full node points to the local introducer (127.0.0.1:8445). Note the the simulation is local only and requires installation of timelords and VDFs. diff --git a/config/config.yaml b/config/config.yaml index 43a15dfb4d5d..d2284522348c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -54,6 +54,14 @@ full_node: # The full node server (if run) will run on this host and port host: 127.0.0.1 port: 8444 + + # Run multiple nodes with different databases by changing the database_id + database_id: 1 + + # If True, starts an RPC server at the following port + start_rpc_server: True + rpc_port: 8555 + enable_upnp: True # Don't send any more than these number of headers and blocks, in one message max_headers_to_send: 25 @@ -63,8 +71,6 @@ full_node: # If node is more than these blocks behind, will do a sync sync_blocks_behind_threshold: 20 - # This SSH key is for the ui SSH server - ssh_filename: config/ssh_host_key # How often to connect to introducer if we need to learn more peers introducer_connect_interval: 500 # Continue trying to connect to more peers until this number of connections @@ -72,6 +78,9 @@ full_node: # Only connect to peers who we have heard about in the last recent_peer_threshold seconds recent_peer_threshold: 6000 + connect_to_farmer: False + connect_to_timelord: False + farmer_peer: host: 127.0.0.1 port: 8447 @@ -79,12 +88,20 @@ full_node: host: 127.0.0.1 port: 8446 introducer_peer: - # To run the simulation, set host to 127.0.0.1 and port to 8445 - # host: 127.0.0.1 - # port: 8445 host: introducer.chia.net # Chia AWS introducer IPv4/IPv6 port: 8444 +ui: + # The ui node server (if run) will run on this host and port + host: 127.0.0.1 + port: 8222 + + # Which port to use to communicate with the full node + rpc_port: 8555 + + # This SSH key is for the ui SSH server + ssh_filename: config/ssh_host_key + introducer: host: 127.0.0.1 port: 8445 diff --git a/scripts/run_all.sh b/scripts/run_all.sh index 9ce03aad3834..3224111b5099 100755 --- a/scripts/run_all.sh +++ b/scripts/run_all.sh @@ -6,7 +6,7 @@ _run_bg_cmd python -m src.server.start_harvester _run_bg_cmd python -m src.server.start_timelord _run_bg_cmd python -m src.server.start_farmer -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -f -t -r 8555 -_run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 +_run_bg_cmd python -m src.server.start_full_node --port=8444 --database_id=1 --connect_to_farmer=True --connect_to_timelord=True --rpc_port=8555 +_run_bg_cmd python -m src.ui.start_ui --port=8222 --rpc_port=8555 wait diff --git a/scripts/run_all_simulation.sh b/scripts/run_all_simulation.sh index 5d049cc54232..0c45614d65fd 100755 --- a/scripts/run_all_simulation.sh +++ b/scripts/run_all_simulation.sh @@ -1,7 +1,7 @@ . .venv/bin/activate . scripts/common.sh -echo "Starting local blockchain simulation. Make sure full node is configured to point to the local introducer (127.0.0.1:8445) in config/config.py." +echo "Starting local blockchain simulation. Runs a local introducer and chia system." echo "Note that this simulation will not work if connected to external nodes." # Starts a harvester, farmer, timelord, introducer, and 3 full nodes, locally. @@ -12,9 +12,9 @@ _run_bg_cmd python -m src.server.start_harvester _run_bg_cmd python -m src.server.start_timelord _run_bg_cmd python -m src.server.start_farmer _run_bg_cmd python -m src.server.start_introducer -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -f -t -r 8555 -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8002 -id 2 -r 8556 -_run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 -_run_bg_cmd python -m src.ui.start_ui 8223 -r 8556 +_run_bg_cmd python -m src.server.start_full_node --port=8444 --database_id=1 --connect_to_farmer=True --connect_to_timelord=True --rpc_port=8555 --introducer_peer.host="127.0.0.1" --introducer_peer.port=8445 +_run_bg_cmd python -m src.server.start_full_node --port=8002 --database_id=2 --rpc_port=8556 --introducer_peer.host="127.0.0.1" --introducer_peer.port=8445 +_run_bg_cmd python -m src.ui.start_ui --port=8222 --rpc_port=8555 +_run_bg_cmd python -m src.ui.start_ui --port=8223 --rpc_port=8556 wait \ No newline at end of file diff --git a/scripts/run_farming.sh b/scripts/run_farming.sh index 7e55ba10892a..119747b18d30 100755 --- a/scripts/run_farming.sh +++ b/scripts/run_farming.sh @@ -5,7 +5,7 @@ _run_bg_cmd python -m src.server.start_harvester _run_bg_cmd python -m src.server.start_farmer -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -f -r 8555 -_run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 +_run_bg_cmd python -m src.server.start_full_node --port=8444 --database_id=1 --connect_to_farmer=True --rpc_port=8555 +_run_bg_cmd python -m src.ui.start_ui --port=8222 --rpc_port=8555 wait diff --git a/scripts/run_full_node.sh b/scripts/run_full_node.sh index 5c9ef88358ac..de58c7d4ef99 100755 --- a/scripts/run_full_node.sh +++ b/scripts/run_full_node.sh @@ -2,7 +2,7 @@ . scripts/common.sh # Starts a full node -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -f -r 8555 +_run_bg_cmd python -m src.server.start_full_node --port=8444 --database_id=1 --connect_to_farmer=True --connect_to_timelord=True --rpc_port=8555 _run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 wait diff --git a/scripts/run_timelord.sh b/scripts/run_timelord.sh index 20e8c1f1284c..68c418ab3fab 100755 --- a/scripts/run_timelord.sh +++ b/scripts/run_timelord.sh @@ -4,7 +4,7 @@ # Starts a timelord, and a full node _run_bg_cmd python -m src.server.start_timelord -_run_bg_cmd python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -t -r 8555 -_run_bg_cmd python -m src.ui.start_ui 8222 -r 8555 +_run_bg_cmd python -m src.server.start_full_node --port=8444 --database_id=1 --connect_to_timelord=True --rpc_port=8555 +_run_bg_cmd python -m src.ui.start_ui --port=8222 --rpc_port=8555 wait diff --git a/src/farmer.py b/src/farmer.py index 3a2e232c7af8..3b12757396f8 100644 --- a/src/farmer.py +++ b/src/farmer.py @@ -1,12 +1,9 @@ import logging -import os from hashlib import sha256 from typing import Any, Dict, List, Set from blspy import PrependSignature, PrivateKey, Util -from yaml import safe_load -from definitions import ROOT_DIR from src.consensus.block_rewards import calculate_block_reward from src.consensus.constants import constants from src.consensus.pot_iterations import calculate_iterations_quality @@ -27,15 +24,9 @@ HARVESTER PROTOCOL (FARMER <-> HARVESTER) class Farmer: - def __init__(self): - config_filename = os.path.join(ROOT_DIR, "config", "config.yaml") - key_config_filename = os.path.join(ROOT_DIR, "config", "keys.yaml") - if not os.path.isfile(key_config_filename): - raise RuntimeError( - "Keys not generated. Run python3.7 ./scripts/regenerate_keys.py." - ) - self.config = safe_load(open(config_filename, "r"))["farmer"] - self.key_config = safe_load(open(key_config_filename, "r")) + def __init__(self, farmer_config: Dict, key_config: Dict): + self.config = farmer_config + self.key_config = key_config self.harvester_responses_header_hash: Dict[bytes32, bytes32] = {} self.harvester_responses_challenge: Dict[bytes32, bytes32] = {} self.harvester_responses_proofs: Dict[bytes32, ProofOfSpace] = {} diff --git a/src/full_node.py b/src/full_node.py index cb168c1b7a17..b10d7ce41241 100644 --- a/src/full_node.py +++ b/src/full_node.py @@ -1,18 +1,15 @@ import asyncio import concurrent import logging -import os import time from asyncio import Event from hashlib import sha256 from secrets import token_bytes -from typing import AsyncGenerator, List, Optional, Tuple +from typing import AsyncGenerator, List, Optional, Tuple, Dict -import yaml from blspy import PrivateKey, Signature from chiapos import Verifier -from definitions import ROOT_DIR from src.blockchain import Blockchain, ReceiveBlockResult from src.consensus.constants import constants from src.consensus.pot_iterations import calculate_iterations @@ -40,16 +37,13 @@ OutboundMessageGenerator = AsyncGenerator[OutboundMessage, None] class FullNode: - store: FullNodeStore - blockchain: Blockchain - - def __init__(self, store: FullNodeStore, blockchain: Blockchain): - config_filename = os.path.join(ROOT_DIR, "config", "config.yaml") - self.config = yaml.safe_load(open(config_filename, "r"))["full_node"] - self.store = store - self.blockchain = blockchain - self._shut_down = False # Set to true to close all infinite loops + def __init__(self, store: FullNodeStore, blockchain: Blockchain, config: Dict): + self.store: FullNodeStore = store + self.blockchain: Blockchain = blockchain + self.config: Dict = config + self._shut_down: bool = False # Set to true to close all infinite loops self.server: Optional[ChiaServer] = None + log.warning(f"{self.config}") def _set_server(self, server: ChiaServer): self.server = server diff --git a/src/harvester.py b/src/harvester.py index 0ecfc26c44a0..ebb421f76159 100644 --- a/src/harvester.py +++ b/src/harvester.py @@ -5,7 +5,6 @@ import asyncio from typing import Dict, Optional, Tuple from blspy import PrependSignature, PrivateKey, PublicKey, Util -from yaml import safe_load from chiapos import DiskProver from definitions import ROOT_DIR @@ -20,23 +19,10 @@ log = logging.getLogger(__name__) class Harvester: - def __init__(self): - config_filename = os.path.join(ROOT_DIR, "config", "config.yaml") - plot_config_filename = os.path.join(ROOT_DIR, "config", "plots.yaml") - key_config_filename = os.path.join(ROOT_DIR, "config", "keys.yaml") - - if not os.path.isfile(key_config_filename): - raise RuntimeError( - "Keys not generated. Run python3.7 ./scripts/regenerate_keys.py." - ) - if not os.path.isfile(plot_config_filename): - raise RuntimeError( - "Plots not generated. Run python3.7 ./scripts/create_plots.py." - ) - - self.config = safe_load(open(config_filename, "r"))["harvester"] - self.key_config = safe_load(open(key_config_filename, "r")) - self.plot_config = safe_load(open(plot_config_filename, "r")) + def __init__(self, config: Dict, key_config: Dict, plot_config: Dict): + self.config: Dict = config + self.key_config: Dict = key_config + self.plot_config: Dict = plot_config # From filename to prover self.provers: Dict[str, DiskProver] = {} @@ -44,7 +30,7 @@ class Harvester: # From quality to (challenge_hash, filename, index) self.challenge_hashes: Dict[bytes32, Tuple[bytes32, str, uint8]] = {} self._plot_notification_task = asyncio.create_task(self._plot_notification()) - self._is_shutdown = False + self._is_shutdown: bool = False async def _plot_notification(self): """ diff --git a/src/introducer.py b/src/introducer.py index ea975ce3fcbf..ede83a72d93f 100644 --- a/src/introducer.py +++ b/src/introducer.py @@ -1,11 +1,7 @@ import asyncio import logging -import os from typing import AsyncGenerator, Dict -import yaml - -from definitions import ROOT_DIR from src.protocols.peer_protocol import Peers, RequestPeers from src.server.outbound_message import Delivery, Message, NodeType, OutboundMessage from src.server.server import ChiaServer @@ -16,9 +12,8 @@ log = logging.getLogger(__name__) class Introducer: - def __init__(self): - config_filename = os.path.join(ROOT_DIR, "config", "config.yaml") - self.config = yaml.safe_load(open(config_filename, "r"))["introducer"] + def __init__(self, config: Dict): + self.config: Dict = config self.vetted: Dict[bytes32, bool] = {} def set_server(self, server: ChiaServer): diff --git a/src/server/start_farmer.py b/src/server/start_farmer.py index 46bec25b7aba..ec4ce7340ab1 100644 --- a/src/server/start_farmer.py +++ b/src/server/start_farmer.py @@ -1,6 +1,7 @@ import asyncio import signal from typing import List +import logging try: import uvloop @@ -14,24 +15,33 @@ from src.protocols.harvester_protocol import HarvesterHandshake from src.server.outbound_message import Delivery, Message, NodeType, OutboundMessage from src.server.server import ChiaServer from src.types.peer_info import PeerInfo -from src.util.network import parse_host_port from src.util.logging import initialize_logging +from src.util.config import load_config, load_config_cli from setproctitle import setproctitle initialize_logging("Farmer %(name)-25s") +log = logging.getLogger(__name__) setproctitle("chia_farmer") async def main(): - farmer = Farmer() + config = load_config_cli("config.yaml", "farmer") + try: + key_config = load_config("keys.yaml") + except FileNotFoundError: + raise RuntimeError( + "Keys not generated. Run python3 ./scripts/regenerate_keys.py." + ) + + farmer = Farmer(config, key_config) + harvester_peer = PeerInfo( - farmer.config["harvester_peer"]["host"], farmer.config["harvester_peer"]["port"] + config["harvester_peer"]["host"], config["harvester_peer"]["port"] ) full_node_peer = PeerInfo( - farmer.config["full_node_peer"]["host"], farmer.config["full_node_peer"]["port"] + config["full_node_peer"]["host"], config["full_node_peer"]["port"] ) - host, port = parse_host_port(farmer) - server = ChiaServer(port, farmer, NodeType.FARMER) + server = ChiaServer(config["port"], farmer, NodeType.FARMER) asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all) @@ -39,20 +49,20 @@ async def main(): async def on_connect(): # Sends a handshake to the harvester pool_sks: List[PrivateKey] = [ - PrivateKey.from_bytes(bytes.fromhex(ce)) - for ce in farmer.key_config["pool_sks"] + PrivateKey.from_bytes(bytes.fromhex(ce)) for ce in key_config["pool_sks"] ] msg = HarvesterHandshake([sk.get_public_key() for sk in pool_sks]) yield OutboundMessage( NodeType.HARVESTER, Message("harvester_handshake", msg), Delivery.BROADCAST ) - _ = await server.start_server(host, on_connect) + _ = await server.start_server(config["host"], on_connect) await asyncio.sleep(1) # Prevents TCP simultaneous connect with harvester _ = await server.start_client(harvester_peer, None) _ = await server.start_client(full_node_peer, None) await server.await_closed() + log.info("Farmer fully closed.") if uvloop is not None: diff --git a/src/server/start_full_node.py b/src/server/start_full_node.py index d6ab0517d80d..04014d9624e7 100644 --- a/src/server/start_full_node.py +++ b/src/server/start_full_node.py @@ -2,8 +2,7 @@ import asyncio import logging import logging.config import signal -import sys -from typing import Dict, List +from typing import List, Dict import miniupnpc @@ -22,8 +21,8 @@ from src.server.server import ChiaServer from src.types.full_block import FullBlock from src.types.header_block import SmallHeaderBlock from src.types.peer_info import PeerInfo -from src.util.network import parse_host_port from src.util.logging import initialize_logging +from src.util.config import load_config_cli from setproctitle import setproctitle setproctitle("chia_full_node") @@ -57,11 +56,10 @@ async def load_header_blocks_from_store( async def main(): + config = load_config_cli("config.yaml", "full_node") + # Create the store (DB) and full node instance - db_id = 0 - if "-id" in sys.argv: - db_id = int(sys.argv[sys.argv.index("-id") + 1]) - store = await FullNodeStore.create(f"blockchain_{db_id}.db") + store = await FullNodeStore.create(f"blockchain_{config['database_id']}.db") genesis: FullBlock = FullBlock.from_bytes(constants["GENESIS_BLOCK"]) await store.add_block(genesis) @@ -72,25 +70,26 @@ async def main(): ] = await load_header_blocks_from_store(store) blockchain = await Blockchain.create(small_header_blocks) - full_node = FullNode(store, blockchain) - # Starts the full node server (which full nodes can connect to) - host, port = parse_host_port(full_node) + full_node = FullNode(store, blockchain, config) - if full_node.config["enable_upnp"]: - log.info(f"Attempting to enable UPnP (open up port {port})") + if config["enable_upnp"]: + log.info(f"Attempting to enable UPnP (open up port {config['port']})") try: upnp = miniupnpc.UPnP() upnp.discoverdelay = 5 upnp.discover() upnp.selectigd() - upnp.addportmapping(port, "TCP", upnp.lanaddr, port, "chia", "") - log.info(f"Port {port} opened with UPnP.") + upnp.addportmapping( + config["port"], "TCP", upnp.lanaddr, config["port"], "chia", "" + ) + log.info(f"Port {config['port']} opened with UPnP.") except Exception as e: log.warning(f"UPnP failed: {e}") - server = ChiaServer(port, full_node, NodeType.FULL_NODE) + # Starts the full node server (which full nodes can connect to) + server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE) full_node._set_server(server) - _ = await server.start_server(host, full_node._on_connect) + _ = await server.start_server(config["host"], full_node._on_connect) rpc_cleanup = None def master_close_cb(): @@ -102,32 +101,29 @@ async def main(): server.close_all() server_closed = True - if "-r" in sys.argv: + if config["start_rpc_server"]: # Starts the RPC server if -r is provided - index = sys.argv.index("-r") - rpc_port = int(sys.argv[index + 1]) - rpc_cleanup = await start_rpc_server(full_node, master_close_cb, rpc_port) + rpc_cleanup = await start_rpc_server( + full_node, master_close_cb, config["rpc_port"] + ) asyncio.get_running_loop().add_signal_handler(signal.SIGINT, master_close_cb) asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, master_close_cb) - connect_to_farmer = "-f" in sys.argv - connect_to_timelord = "-t" in sys.argv - full_node._start_bg_tasks() log.info("Waiting to connect to some peers...") await asyncio.sleep(3) log.info(f"Connected to {len(server.global_connections.get_connections())} peers.") - if connect_to_farmer and not server_closed: + if config["connect_to_farmer"] and not server_closed: peer_info = PeerInfo( full_node.config["farmer_peer"]["host"], full_node.config["farmer_peer"]["port"], ) _ = await server.start_client(peer_info, None) - if connect_to_timelord and not server_closed: + if config["connect_to_timelord"] and not server_closed: peer_info = PeerInfo( full_node.config["timelord_peer"]["host"], full_node.config["timelord_peer"]["port"], diff --git a/src/server/start_harvester.py b/src/server/start_harvester.py index 0ef1ded6752e..585fd24af5c4 100644 --- a/src/server/start_harvester.py +++ b/src/server/start_harvester.py @@ -1,5 +1,6 @@ import asyncio import signal +import logging try: import uvloop @@ -10,19 +11,32 @@ from src.harvester import Harvester from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.types.peer_info import PeerInfo -from src.util.network import parse_host_port from src.util.logging import initialize_logging +from src.util.config import load_config, load_config_cli from setproctitle import setproctitle initialize_logging("Harvester %(name)-22s") +log = logging.getLogger(__name__) setproctitle("chia_harvester") async def main(): - harvester = Harvester() - host, port = parse_host_port(harvester) - server = ChiaServer(port, harvester, NodeType.HARVESTER) - _ = await server.start_server(host, None) + config = load_config_cli("config.yaml", "harvester") + try: + key_config = load_config("keys.yaml") + except FileNotFoundError: + raise RuntimeError( + "Keys not generated. Run python3 ./scripts/regenerate_keys.py." + ) + try: + plot_config = load_config("plots.yaml") + except FileNotFoundError: + raise RuntimeError( + "Plots not generated. Run python3.7 ./scripts/create_plots.py." + ) + harvester = Harvester(config, key_config, plot_config) + server = ChiaServer(config["port"], harvester, NodeType.HARVESTER) + _ = await server.start_server(config["port"], None) asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all) @@ -35,6 +49,7 @@ async def main(): await server.await_closed() harvester._shutdown() await harvester._await_shutdown() + log.info("Harvester fully closed.") if uvloop is not None: diff --git a/src/server/start_introducer.py b/src/server/start_introducer.py index 92bd7f264108..b79ea8227aaf 100644 --- a/src/server/start_introducer.py +++ b/src/server/start_introducer.py @@ -1,5 +1,6 @@ import asyncio import signal +import logging try: import uvloop @@ -9,25 +10,27 @@ except ImportError: from src.introducer import Introducer from src.server.outbound_message import NodeType from src.server.server import ChiaServer -from src.util.network import parse_host_port from src.util.logging import initialize_logging +from src.util.config import load_config_cli from setproctitle import setproctitle initialize_logging("Introducer %(name)-21s") +log = logging.getLogger(__name__) setproctitle("chia_introducer") async def main(): - introducer = Introducer() - host, port = parse_host_port(introducer) - server = ChiaServer(port, introducer, NodeType.INTRODUCER) + config = load_config_cli("config.yaml", "introducer") + introducer = Introducer(config) + server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER) introducer.set_server(server) - _ = await server.start_server(host, None) + _ = await server.start_server(config["host"], None) asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all) await server.await_closed() + log.info("Introducer fully closed.") if uvloop is not None: diff --git a/src/server/start_timelord.py b/src/server/start_timelord.py index 3afe46651e9e..18c11111190c 100644 --- a/src/server/start_timelord.py +++ b/src/server/start_timelord.py @@ -1,5 +1,6 @@ import asyncio import signal +import logging try: import uvloop @@ -10,19 +11,20 @@ from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.timelord import Timelord from src.types.peer_info import PeerInfo -from src.util.network import parse_host_port from src.util.logging import initialize_logging +from src.util.config import load_config_cli from setproctitle import setproctitle initialize_logging("Timelord %(name)-23s") +log = logging.getLogger(__name__) setproctitle("chia_timelord") async def main(): - timelord = Timelord() - host, port = parse_host_port(timelord) - server = ChiaServer(port, timelord, NodeType.TIMELORD) - _ = await server.start_server(host, None) + config = load_config_cli("config.yaml", "timelord") + timelord = Timelord(config) + server = ChiaServer(config["port"], timelord, NodeType.TIMELORD) + _ = await server.start_server(config["host"], None) def signal_received(): server.close_all() @@ -43,6 +45,7 @@ async def main(): server.push_message(msg) await server.await_closed() + log.info("Timelord fully closed.") if uvloop is not None: diff --git a/src/timelord.py b/src/timelord.py index a91a39f99d0b..334399556d75 100644 --- a/src/timelord.py +++ b/src/timelord.py @@ -1,14 +1,11 @@ import asyncio import io import logging -import os import time from asyncio import Lock, StreamReader, StreamWriter from typing import Dict, List, Optional, Tuple -from yaml import safe_load -from definitions import ROOT_DIR from lib.chiavdf.inkfish.classgroup import ClassGroup from lib.chiavdf.inkfish.create_discriminant import create_discriminant from lib.chiavdf.inkfish.proof_of_time import check_proof_of_time_nwesolowski @@ -25,9 +22,8 @@ log = logging.getLogger(__name__) class Timelord: - def __init__(self): - config_filename = os.path.join(ROOT_DIR, "config", "config.yaml") - self.config = safe_load(open(config_filename, "r"))["timelord"] + def __init__(self, config: Dict): + self.config: Dict = config self.free_servers: List[Tuple[str, str]] = list( zip(self.config["vdf_server_ips"], self.config["vdf_server_ports"]) ) diff --git a/src/ui/start_ui.py b/src/ui/start_ui.py index 5b180f7112a1..52dc0cd5c8e9 100644 --- a/src/ui/start_ui.py +++ b/src/ui/start_ui.py @@ -1,12 +1,9 @@ import asyncio import signal -import sys -import yaml -import os from src.ui.prompt_ui import start_ssh_server -from definitions import ROOT_DIR from src.util.logging import initialize_logging +from src.util.config import load_config_cli from setproctitle import setproctitle initialize_logging("UI %(name)-29s") @@ -14,15 +11,10 @@ setproctitle("chia_full_node_ui") async def main(): - config_filename = os.path.join(ROOT_DIR, "config", "config.yaml") - config = yaml.safe_load(open(config_filename, "r"))["full_node"] + config = load_config_cli("config.yaml", "ui") - rpc_index = sys.argv.index("-r") - rpc_port = int(sys.argv[rpc_index + 1]) - - port = int(sys.argv[1]) await_all_closed, ui_close_cb = await start_ssh_server( - port, config["ssh_filename"], rpc_port + config["port"], config["ssh_filename"], config["rpc_port"] ) asyncio.get_running_loop().add_signal_handler( diff --git a/src/util/config.py b/src/util/config.py new file mode 100644 index 000000000000..da39e033ce71 --- /dev/null +++ b/src/util/config.py @@ -0,0 +1,82 @@ +import os + +import yaml +import argparse +from typing import Dict, Any, Callable, Optional +from definitions import ROOT_DIR + + +def load_config(filename: str, sub_config: Optional[str] = None) -> Dict: + config_filename = os.path.join(ROOT_DIR, "config", filename) + if sub_config is not None: + return yaml.safe_load(open(config_filename, "r"))[sub_config] + else: + return yaml.safe_load(open(config_filename, "r")) + + +def load_config_cli(filename: str, sub_config: Optional[str] = None) -> Dict: + """ + Loads configuration from the specified filename, in the config directory, + and then overrides any properties using the passed in command line arguments. + Nested properties in the config file can be used in the command line with ".", + for example --farmer_peer.host. Does not support lists. + """ + config = load_config(filename, sub_config) + + flattened_props = flatten_properties(config) + parser = argparse.ArgumentParser() + + for prop_name, value in flattened_props.items(): + if type(value) is list: + continue + prop_type: Callable = str2bool if type(value) is bool else type(value) # type: ignore + parser.add_argument(f"--{prop_name}", type=prop_type, dest=prop_name) + + for key, value in vars(parser.parse_args()).items(): + if value is not None: + flattened_props[key] = value + + return unflatten_properties(flattened_props) + + +def flatten_properties(config: Dict): + properties = {} + for key, value in config.items(): + if type(value) is dict: + for key_2, value_2 in flatten_properties(value).items(): + properties[key + "." + key_2] = value_2 + else: + properties[key] = value + return properties + + +def unflatten_properties(config: Dict): + properties: Dict = {} + for key, value in config.items(): + if "." in key: + add_property(properties, key, value) + else: + properties[key] = value + return properties + + +def add_property(d: Dict, partial_key: str, value: Any): + key_1, key_2 = partial_key.split(".") + if key_1 not in d: + d[key_1] = {} + if "." in key_2: + add_property(d, key_2, value) + else: + d[key_1][key_2] = value + + +def str2bool(v: Any) -> bool: + # Source from https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("Boolean value expected.") diff --git a/src/util/network.py b/src/util/network.py index 1228498bbafd..55ae27168664 100644 --- a/src/util/network.py +++ b/src/util/network.py @@ -1,16 +1,8 @@ import secrets -import sys -from typing import Tuple from src.types.sized_bytes import bytes32 -def parse_host_port(api) -> Tuple[str, int]: - host: str = sys.argv[1] if len(sys.argv) >= 3 else api.config["host"] - port: int = int(sys.argv[2]) if len(sys.argv) >= 3 else api.config["port"] - return (host, port) - - def create_node_id() -> bytes32: """Generates a transient random node_id.""" return bytes32(secrets.token_bytes(32)) diff --git a/tests/setup_nodes.py b/tests/setup_nodes.py index e95cf62545b2..637a013c5928 100644 --- a/tests/setup_nodes.py +++ b/tests/setup_nodes.py @@ -8,6 +8,7 @@ from src.server.connection import NodeType from src.server.server import ChiaServer from src.types.full_block import FullBlock from tests.block_tools import BlockTools +from src.util.config import load_config bt = BlockTools() @@ -42,12 +43,13 @@ async def setup_two_nodes(): await store_1.add_block(FullBlock.from_bytes(test_constants["GENESIS_BLOCK"])) await store_2.add_block(FullBlock.from_bytes(test_constants["GENESIS_BLOCK"])) - full_node_1 = FullNode(store_1, b_1) + config = load_config("config.yaml") + full_node_1 = FullNode(store_1, b_1, config) server_1 = ChiaServer(21234, full_node_1, NodeType.FULL_NODE) _ = await server_1.start_server("127.0.0.1", full_node_1._on_connect) full_node_1._set_server(server_1) - full_node_2 = FullNode(store_2, b_2) + full_node_2 = FullNode(store_2, b_2, config) server_2 = ChiaServer(21235, full_node_2, NodeType.FULL_NODE) full_node_2._set_server(server_2)