Added total_effective_plot_size for get_harvesters RPC API (#15770)

* Added `total_effective_plot_size` for `get_harvesters` RPC API

* Fixed lint error

* Change the type of `total_effective_plot_size` in Receiver to `int`

* Fixed lint error

* Fixed lint error 2

* Changed text

* Added test

* Fixed lint error

* Fixed lint error 2

* Fixed a test issue

* Added config.py to tests/cmds

chia/cmds/farm_funcs.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from chia.cmds.cmds_util import get_any_service_client
@@ -8,24 +9,29 @@ from chia.consensus.block_record import BlockRecord
 from chia.rpc.farmer_rpc_client import FarmerRpcClient
 from chia.rpc.full_node_rpc_client import FullNodeRpcClient
 from chia.rpc.wallet_rpc_client import WalletRpcClient
+from chia.util.default_root import DEFAULT_ROOT_PATH
 from chia.util.misc import format_bytes, format_minutes
 from chia.util.network import is_localhost
 
 SECONDS_PER_BLOCK = (24 * 3600) / 4608
 
 
-async def get_harvesters_summary(farmer_rpc_port: Optional[int]) -> Optional[Dict[str, Any]]:
-    async with get_any_service_client(FarmerRpcClient, farmer_rpc_port) as (farmer_client, _):
+async def get_harvesters_summary(
+    farmer_rpc_port: Optional[int], root_path: Path = DEFAULT_ROOT_PATH
+) -> Optional[Dict[str, Any]]:
+    async with get_any_service_client(FarmerRpcClient, farmer_rpc_port, root_path) as (farmer_client, _):
         return await farmer_client.get_harvesters_summary()
 
 
-async def get_blockchain_state(rpc_port: Optional[int]) -> Optional[Dict[str, Any]]:
-    async with get_any_service_client(FullNodeRpcClient, rpc_port) as (client, _):
+async def get_blockchain_state(
+    rpc_port: Optional[int], root_path: Path = DEFAULT_ROOT_PATH
+) -> Optional[Dict[str, Any]]:
+    async with get_any_service_client(FullNodeRpcClient, rpc_port, root_path) as (client, _):
         return await client.get_blockchain_state()
 
 
-async def get_average_block_time(rpc_port: Optional[int]) -> float:
-    async with get_any_service_client(FullNodeRpcClient, rpc_port) as (client, _):
+async def get_average_block_time(rpc_port: Optional[int], root_path: Path = DEFAULT_ROOT_PATH) -> float:
+    async with get_any_service_client(FullNodeRpcClient, rpc_port, root_path) as (client, _):
         blocks_to_compare = 500
         blockchain_state = await client.get_blockchain_state()
         curr: Optional[BlockRecord] = blockchain_state["peak"]
@@ -45,8 +51,10 @@ async def get_average_block_time(rpc_port: Optional[int]) -> float:
         return (curr.timestamp - past_curr.timestamp) / (curr.height - past_curr.height)
 
 
-async def get_wallets_stats(wallet_rpc_port: Optional[int]) -> Optional[Dict[str, Any]]:
-    async with get_any_service_client(WalletRpcClient, wallet_rpc_port) as (wallet_client, _):
+async def get_wallets_stats(
+    wallet_rpc_port: Optional[int], root_path: Path = DEFAULT_ROOT_PATH
+) -> Optional[Dict[str, Any]]:
+    async with get_any_service_client(WalletRpcClient, wallet_rpc_port, root_path) as (wallet_client, _):
         return await wallet_client.get_farmed_amount()
 
 
@@ -78,15 +86,16 @@ async def summary(
     wallet_rpc_port: Optional[int],
     harvester_rpc_port: Optional[int],
     farmer_rpc_port: Optional[int],
+    root_path: Path = DEFAULT_ROOT_PATH,
 ) -> None:
-    harvesters_summary = await get_harvesters_summary(farmer_rpc_port)
-    blockchain_state = await get_blockchain_state(rpc_port)
+    harvesters_summary = await get_harvesters_summary(farmer_rpc_port, root_path)
+    blockchain_state = await get_blockchain_state(rpc_port, root_path)
     farmer_running = False if harvesters_summary is None else True  # harvesters uses farmer rpc too
 
     wallet_not_ready: bool = False
     amounts = None
     try:
-        amounts = await get_wallets_stats(wallet_rpc_port)
+        amounts = await get_wallets_stats(wallet_rpc_port, root_path)
     except Exception:
         wallet_not_ready = True
     wallet_not_running: bool = True if amounts is None else False
@@ -111,6 +120,7 @@ async def summary(
 
     class PlotStats:
         total_plot_size = 0
+        total_effective_plot_size = 0
         total_plots = 0
 
     if harvesters_summary is not None:
@@ -132,10 +142,15 @@ async def summary(
                     print(f"   Loading plots: {syncing['plot_files_processed']} / {syncing['plot_files_total']}")
                 else:
                     total_plot_size_harvester = harvester_dict["total_plot_size"]
+                    total_effective_plot_size_harvester = harvester_dict["total_effective_plot_size"]
                     plot_count_harvester = harvester_dict["plots"]
                     PlotStats.total_plot_size += total_plot_size_harvester
+                    PlotStats.total_effective_plot_size += total_effective_plot_size_harvester
                     PlotStats.total_plots += plot_count_harvester
-                    print(f"   {plot_count_harvester} plots of size: {format_bytes(total_plot_size_harvester)}")
+                    print(
+                        f"   {plot_count_harvester} plots of size: {format_bytes(total_plot_size_harvester)} on-disk, "
+                        f"{format_bytes(total_effective_plot_size_harvester)}e (effective)"
+                    )
 
         if len(harvesters_local) > 0:
             print(f"Local Harvester{'s' if len(harvesters_local) > 1 else ''}")
@@ -146,8 +161,10 @@ async def summary(
 
         print(f"Plot count for all harvesters: {PlotStats.total_plots}")
 
-        print("Total size of plots: ", end="")
-        print(format_bytes(PlotStats.total_plot_size))
+        print(
+            f"Total size of plots: {format_bytes(PlotStats.total_plot_size)}, "
+            f"{format_bytes(PlotStats.total_effective_plot_size)}e (effective)"
+        )
     else:
         print("Plot count: Unknown")
         print("Total size of plots: Unknown")
@@ -160,8 +177,10 @@ async def summary(
 
     minutes = -1
     if blockchain_state is not None and harvesters_summary is not None:
-        proportion = PlotStats.total_plot_size / blockchain_state["space"] if blockchain_state["space"] else -1
-        minutes = int((await get_average_block_time(rpc_port) / 60) / proportion) if proportion else -1
+        proportion = (
+            PlotStats.total_effective_plot_size / blockchain_state["space"] if blockchain_state["space"] else -1
+        )
+        minutes = int((await get_average_block_time(rpc_port, root_path) / 60) / proportion) if proportion else -1
 
     if harvesters_summary is not None and PlotStats.total_plots == 0:
         print("Expected time to win: Never (no plots)")

chia/plot_sync/receiver.py

@@ -7,6 +7,7 @@ from typing import Any, Awaitable, Callable, Collection, Dict, List, Optional
 
 from typing_extensions import Protocol
 
+from chia.consensus.pos_quality import UI_ACTUAL_SPACE_CONSTANT_FACTOR, _expected_plot_size
 from chia.plot_sync.delta import Delta, PathListDelta, PlotListDelta
 from chia.plot_sync.exceptions import (
     InvalidIdentifierError,
@@ -82,6 +83,7 @@ class Receiver:
     _keys_missing: List[str]
     _duplicates: List[str]
     _total_plot_size: int
+    _total_effective_plot_size: int
     _update_callback: ReceiverUpdateCallback
 
     def __init__(
@@ -97,6 +99,7 @@ class Receiver:
         self._keys_missing = []
         self._duplicates = []
         self._total_plot_size = 0
+        self._total_effective_plot_size = 0
         self._update_callback = update_callback
 
     async def trigger_callback(self, update: Optional[Delta] = None) -> None:
@@ -114,6 +117,7 @@ class Receiver:
         self._keys_missing.clear()
         self._duplicates.clear()
         self._total_plot_size = 0
+        self._total_effective_plot_size = 0
 
     def connection(self) -> WSChiaConnection:
         return self._connection
@@ -142,6 +146,9 @@ class Receiver:
     def total_plot_size(self) -> int:
         return self._total_plot_size
 
+    def total_effective_plot_size(self) -> int:
+        return self._total_effective_plot_size
+
     async def _process(
         self, method: Callable[[T_PlotSyncMessage], Any], message_type: ProtocolMessageTypes, message: T_PlotSyncMessage
     ) -> None:
@@ -329,6 +336,9 @@ class Receiver:
         self._keys_missing = self._current_sync.delta.keys_missing.additions.copy()
         self._duplicates = self._current_sync.delta.duplicates.additions.copy()
         self._total_plot_size = sum(plot.file_size for plot in self._plots.values())
+        self._total_effective_plot_size = int(
+            sum(UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(plot.size)) for plot in self._plots.values())
+        )
         # Save current sync as last sync and create a new current sync
         self._last_sync = self._current_sync
         self._current_sync = Sync()
@@ -357,6 +367,7 @@ class Receiver:
             "no_key_filenames": get_list_or_len(self._keys_missing, counts_only),
             "duplicates": get_list_or_len(self._duplicates, counts_only),
             "total_plot_size": self._total_plot_size,
+            "total_effective_plot_size": self._total_effective_plot_size,
             "syncing": syncing,
             "last_sync_time": self._last_sync.time_done,
         }

tests/cmds/config.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+parallel = True
+install_timelord = False
+checkout_blocks_and_plots = True

tests/cmds/test_farm_cmd.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import re
+from typing import Tuple
+
+import pytest
+from _pytest.capture import CaptureFixture
+
+from chia.cmds.farm_funcs import summary
+from chia.farmer.farmer import Farmer
+from chia.farmer.farmer_api import FarmerAPI
+from chia.full_node.full_node import FullNode
+from chia.harvester.harvester import Harvester
+from chia.harvester.harvester_api import HarvesterAPI
+from chia.server.start_service import Service
+from chia.simulator.block_tools import BlockTools
+from chia.simulator.full_node_simulator import FullNodeSimulator
+from chia.simulator.time_out_assert import time_out_assert
+from chia.wallet.wallet_node import WalletNode
+from chia.wallet.wallet_node_api import WalletNodeAPI
+
+
+@pytest.mark.asyncio
+async def test_farm_summary_command(
+    capsys: CaptureFixture[str],
+    farmer_one_harvester_simulator_wallet: Tuple[
+        Service[Harvester, HarvesterAPI],
+        Service[Farmer, FarmerAPI],
+        Service[FullNode, FullNodeSimulator],
+        Service[WalletNode, WalletNodeAPI],
+        BlockTools,
+    ],
+) -> None:
+    harvester_service, farmer_service, full_node_service, wallet_service, bt = farmer_one_harvester_simulator_wallet
+    harvester: Harvester = harvester_service._node
+    farmer: Farmer = farmer_service._node
+
+    async def receiver_available() -> bool:
+        return harvester.server.node_id in farmer.plot_sync_receivers
+
+    # Wait for the receiver to show up
+    await time_out_assert(20, receiver_available)
+    receiver = farmer.plot_sync_receivers[harvester.server.node_id]
+    # And wait until the first sync from the harvester to the farmer is done
+    await time_out_assert(20, receiver.initial_sync, False)
+
+    assert full_node_service.rpc_server and full_node_service.rpc_server.webserver
+    assert wallet_service.rpc_server and wallet_service.rpc_server.webserver
+    assert farmer_service.rpc_server and farmer_service.rpc_server.webserver
+
+    full_node_rpc_port = full_node_service.rpc_server.webserver.listen_port
+    wallet_rpc_port = wallet_service.rpc_server.webserver.listen_port
+    farmer_rpc_port = farmer_service.rpc_server.webserver.listen_port
+
+    await summary(full_node_rpc_port, wallet_rpc_port, None, farmer_rpc_port, bt.root_path)
+
+    captured = capsys.readouterr()
+    match = re.search(r"^.+(Farming status:.+)$", captured.out, re.DOTALL)
+    assert match is not None
+    lines = match.group(1).split("\n")
+
+    assert lines[0] == "Farming status: Not synced or not connected to peers"
+    assert "Total chia farmed:" in lines[1]
+    assert "User transaction fees:" in lines[2]
+    assert "Block rewards:" in lines[3]
+    assert "Last height farmed:" in lines[4]
+    assert lines[5] == "Local Harvester"
+    assert "e (effective)" in lines[6]
+    assert "Plot count for all harvesters:" in lines[7]
+    assert "e (effective)" in lines[8]
+    assert "Estimated network space:" in lines[9]
+    assert "Expected time to win:" in lines[10]

tests/conftest.py

@@ -598,6 +598,25 @@ async def two_nodes_one_block():
         yield _
 
 
+@pytest_asyncio.fixture(scope="function")
+async def farmer_one_harvester_simulator_wallet(
+    tmp_path: Path,
+) -> AsyncIterator[
+    Tuple[
+        Service[Harvester, HarvesterAPI],
+        Service[Farmer, FarmerAPI],
+        Service[FullNode, FullNodeSimulator],
+        Service[WalletNode, WalletNodeAPI],
+        BlockTools,
+    ]
+]:
+    async for sim_and_wallet in setup_simulators_and_wallets_service(1, 1, {}):
+        nodes, wallets, bt = sim_and_wallet
+        async for farmer_harvester in setup_farmer_multi_harvester(bt, 1, tmp_path, bt.constants, start_services=True):
+            harvester_services, farmer_service, _ = farmer_harvester
+            yield harvester_services[0], farmer_service, nodes[0], wallets[0], bt
+
+
 @pytest_asyncio.fixture(scope="function")
 async def farmer_one_harvester(tmp_path: Path, get_b_tools: BlockTools) -> AsyncIterator[Tuple[List[Service], Service]]:
     async for _ in setup_farmer_multi_harvester(get_b_tools, 1, tmp_path, get_b_tools.constants, start_services=True):

tests/plot_sync/test_receiver.py

@@ -10,6 +10,7 @@ from typing import Any, Callable, List, Tuple, Type, Union
 import pytest
 from blspy import G1Element
 
+from chia.consensus.pos_quality import UI_ACTUAL_SPACE_CONSTANT_FACTOR, _expected_plot_size
 from chia.plot_sync.delta import Delta
 from chia.plot_sync.receiver import Receiver, Sync
 from chia.plot_sync.util import ErrorCodes, State
@@ -41,6 +42,8 @@ def assert_default_values(receiver: Receiver) -> None:
     assert receiver.invalid() == []
     assert receiver.keys_missing() == []
     assert receiver.duplicates() == []
+    assert receiver.total_plot_size() == 0
+    assert receiver.total_effective_plot_size() == 0
 
 
 async def dummy_callback(_: bytes32, __: Delta) -> None:
@@ -180,8 +183,10 @@ def plot_sync_setup() -> Tuple[Receiver, List[SyncStepData]]:
 
     # Manually add the plots we want to remove in tests
     receiver._plots = {plot_info.filename: plot_info for plot_info in plot_info_list[0:10]}
-    receiver._total_plot_size = sum(plot.file_size for plot in receiver._plots.values())
-
+    receiver._total_plot_size = sum(plot.file_size for plot in receiver.plots().values())
+    receiver._total_effective_plot_size = int(
+        sum(UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(plot.size)) for plot in receiver.plots().values())
+    )
     sync_steps: List[SyncStepData] = [
         SyncStepData(State.idle, receiver.sync_started, PlotSyncStart, False, uint64(0), uint32(len(plot_info_list))),
         SyncStepData(State.loaded, receiver.process_loaded, PlotSyncPlotList, plot_info_list[10:20], True),
@@ -243,6 +248,9 @@ async def test_to_dict(counts_only: bool) -> None:
     assert get_list_or_len(plot_sync_dict_1["no_key_filenames"], not counts_only) == 0
     assert get_list_or_len(plot_sync_dict_1["duplicates"], not counts_only) == 0
     assert plot_sync_dict_1["total_plot_size"] == sum(plot.file_size for plot in receiver.plots().values())
+    assert plot_sync_dict_1["total_effective_plot_size"] == int(
+        sum(UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(plot.size)) for plot in receiver.plots().values())
+    )
     assert plot_sync_dict_1["syncing"] is None
     assert plot_sync_dict_1["last_sync_time"] is None
     assert plot_sync_dict_1["connection"] == {
@@ -286,6 +294,9 @@ async def test_to_dict(counts_only: bool) -> None:
     assert get_list_or_len(sync_steps[State.duplicates].args[0], counts_only) == plot_sync_dict_3["duplicates"]
 
     assert plot_sync_dict_3["total_plot_size"] == sum(plot.file_size for plot in receiver.plots().values())
+    assert plot_sync_dict_3["total_effective_plot_size"] == int(
+        sum(UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(plot.size)) for plot in receiver.plots().values())
+    )
     assert plot_sync_dict_3["last_sync_time"] > 0
     assert plot_sync_dict_3["syncing"] is None
 

tests/plot_sync/test_sync_simulated.py

@@ -277,7 +277,7 @@ def create_example_plots(count: int) -> List[PlotInfo]:
 
     return [
         PlotInfo(
-            prover=DiskProver(f"{x}", bytes32(token_bytes(32)), x % 255),
+            prover=DiskProver(f"{x}", bytes32(token_bytes(32)), 25 + x % 26),
             pool_public_key=None,
             pool_contract_puzzle_hash=None,
             plot_public_key=G1Element(),