farmer|gui: Enable paginated plot loading and improved state reporting (#11367)

* farmer: Adjust notifications from the farmer to the UI

- Only send the data for the harvester which actually sent an update
- Notify for each loaded batch during initial loading

* Enable improved farmer/harvester GUI

chia-blockchain-gui

@@ -1 +1 @@
-Subproject commit 4fc7fe11cc668277aa0d4c68e1f8a48765db9241
+Subproject commit a6724910a82e1e719784a7a0885da13b30533c66

chia/farmer/farmer.py

@@ -256,11 +256,14 @@ class Farmer:
         self.state_changed("close_connection", {})
         if connection.connection_type is NodeType.HARVESTER:
             del self.plot_sync_receivers[connection.peer_node_id]
+            self.state_changed("harvester_removed", {"node_id": connection.peer_node_id})
 
-    async def plot_sync_callback(self, peer_id: bytes32, delta: Delta) -> None:
-        log.info(f"plot_sync_callback: peer_id {peer_id}, delta {delta}")
-        if not delta.empty():
-            self.state_changed("new_plots", await self.get_harvesters())
+    async def plot_sync_callback(self, peer_id: bytes32, delta: Optional[Delta]) -> None:
+        log.debug(f"plot_sync_callback: peer_id {peer_id}, delta {delta}")
+        receiver: Receiver = self.plot_sync_receivers[peer_id]
+        harvester_updated: bool = delta is not None and not delta.empty()
+        if receiver.initial_sync() or harvester_updated:
+            self.state_changed("harvester_update", receiver.to_dict(True))
 
     async def _pool_get_pool_info(self, pool_config: PoolWalletConfig) -> Optional[Dict]:
         try:

chia/plot_sync/receiver.py

@@ -61,10 +61,12 @@ class Receiver:
     _keys_missing: List[str]
     _duplicates: List[str]
     _total_plot_size: int
-    _update_callback: Callable[[bytes32, Delta], Coroutine[Any, Any, None]]
+    _update_callback: Callable[[bytes32, Optional[Delta]], Coroutine[Any, Any, None]]
 
     def __init__(
-        self, connection: WSChiaConnection, update_callback: Callable[[bytes32, Delta], Coroutine[Any, Any, None]]
+        self,
+        connection: WSChiaConnection,
+        update_callback: Callable[[bytes32, Optional[Delta]], Coroutine[Any, Any, None]],
     ) -> None:
         self._connection = connection
         self._current_sync = Sync()
@@ -76,6 +78,12 @@ class Receiver:
         self._total_plot_size = 0
         self._update_callback = update_callback  # type: ignore[assignment, misc]
 
+    async def trigger_callback(self, update: Optional[Delta] = None) -> None:
+        try:
+            await self._update_callback(self._connection.peer_node_id, update)  # type: ignore[misc,call-arg]
+        except Exception as e:
+            log.error(f"_update_callback raised: {e}")
+
     def reset(self) -> None:
         self._current_sync = Sync()
         self._last_sync = Sync()
@@ -176,6 +184,9 @@ class Receiver:
             self._current_sync.delta.valid.additions[plot_info.filename] = plot_info
             self._current_sync.bump_plots_processed()
 
+        # Let the callback receiver know about the sync progress updates
+        await self.trigger_callback()
+
         if plot_infos.final:
             self._current_sync.state = State.removed
 
@@ -205,6 +216,9 @@ class Receiver:
             if not is_removal:
                 self._current_sync.bump_plots_processed()
 
+        # Let the callback receiver know about the sync progress updates
+        await self.trigger_callback()
+
         if paths.final:
             self._current_sync.state = next_state
 
@@ -293,10 +307,7 @@ class Receiver:
         self._last_sync = self._current_sync
         self._current_sync = Sync()
         # Let the callback receiver know if this sync cycle caused any update
-        try:
-            await self._update_callback(self._connection.peer_node_id, update)  # type: ignore[misc,call-arg]
-        except Exception as e:
-            log.error(f"_update_callback raised: {e}")
+        await self.trigger_callback(update)
 
     async def sync_done(self, data: PlotSyncDone) -> None:
         await self._process(self._sync_done, ProtocolMessageTypes.plot_sync_done, data)

chia/rpc/farmer_rpc_api.py

@@ -109,10 +109,19 @@ class FarmerRpcApi:
                     "wallet_ui",
                 )
             )
-        elif change == "new_plots":
+        elif change == "harvester_update":
             payloads.append(
                 create_payload_dict(
-                    "get_harvesters",
+                    "harvester_update",
+                    change_data,
+                    self.service_name,
+                    "wallet_ui",
+                )
+            )
+        elif change == "harvester_removed":
+            payloads.append(
+                create_payload_dict(
+                    "harvester_removed",
                     change_data,
                     self.service_name,
                     "wallet_ui",