simplify fee estimator by not leaking MempoolItem into its interface (#14685)

chia/full_node/bitcoin_fee_estimator.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
 from chia.full_node.fee_estimate_store import FeeStore
-from chia.full_node.fee_estimation import EmptyFeeMempoolInfo, FeeBlockInfo, FeeMempoolInfo
+from chia.full_node.fee_estimation import EmptyFeeMempoolInfo, FeeBlockInfo, FeeMempoolInfo, MempoolItemInfo
 from chia.full_node.fee_estimator import SmartFeeEstimator
 from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
 from chia.full_node.fee_tracker import FeeTracker
 from chia.types.clvm_cost import CLVMCost
 from chia.types.fee_rate import FeeRateV2
-from chia.types.mempool_item import MempoolItem
 from chia.util.ints import uint32, uint64
 
 
@@ -35,11 +34,11 @@ class BitcoinFeeEstimator(FeeEstimatorInterface):
         self.block_height = block_info.block_height
         self.tracker.process_block(block_info.block_height, block_info.included_items)
 
-    def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         self.last_mempool_info = mempool_info
         self.tracker.add_tx(mempool_item)
 
-    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         self.last_mempool_info = mempool_info
         self.tracker.remove_tx(mempool_item)
 

chia/full_node/fee_estimation.py

@@ -6,11 +6,26 @@ from typing import List
 
 from chia.types.clvm_cost import CLVMCost
 from chia.types.fee_rate import FeeRate
-from chia.types.mempool_item import MempoolItem
 from chia.types.mojos import Mojos
 from chia.util.ints import uint32, uint64
 
 
+@dataclass(frozen=True)
+class MempoolItemInfo:
+    """
+    The information the fee estimator is passed for each mempool item that's
+    added, removed from the mempool and included in blocks
+    """
+
+    cost: int
+    fee: int
+    height_added_to_mempool: uint32
+
+    @property
+    def fee_per_cost(self) -> float:
+        return self.fee / self.cost
+
+
 @dataclass(frozen=True)
 class MempoolInfo:
     """
@@ -75,4 +90,4 @@ class FeeBlockInfo:  # See BlockRecord
     """
 
     block_height: uint32
-    included_items: List[MempoolItem]
+    included_items: List[MempoolItemInfo]

chia/full_node/fee_estimator_example.py

@@ -3,11 +3,10 @@ from __future__ import annotations
 from typing import Any, Dict, List
 
 from chia.full_node.fee_estimate import FeeEstimateV2
-from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo
+from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo, MempoolItemInfo
 from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
 from chia.types.clvm_cost import CLVMCost
 from chia.types.fee_rate import FeeRateV2
-from chia.types.mempool_item import MempoolItem
 from chia.util.ints import uint64
 
 MIN_MOJO_PER_COST = 5
@@ -31,10 +30,10 @@ class FeeEstimatorExample(FeeEstimatorInterface):
     def new_block(self, block_info: FeeBlockInfo) -> None:
         pass
 
-    def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         pass
 
-    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         pass
 
     def estimate_fee_rate(self, *, time_offset_seconds: int) -> FeeRateV2:

chia/full_node/fee_estimator_interface.py

@@ -2,10 +2,9 @@ from __future__ import annotations
 
 from typing_extensions import Protocol
 
-from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo
+from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo, MempoolItemInfo
 from chia.types.clvm_cost import CLVMCost
 from chia.types.fee_rate import FeeRateV2
-from chia.types.mempool_item import MempoolItem
 from chia.util.ints import uint32
 
 
@@ -18,11 +17,11 @@ class FeeEstimatorInterface(Protocol):
         """A new transaction block has been added to the blockchain"""
         pass
 
-    def add_mempool_item(self, mempool_item_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def add_mempool_item(self, mempool_item_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         """A MempoolItem (transaction and associated info) has been added to the mempool"""
         pass
 
-    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         """A MempoolItem (transaction and associated info) has been removed from the mempool"""
         pass
 

chia/full_node/fee_tracker.py

@@ -6,6 +6,7 @@ from dataclasses import dataclass
 from typing import List, Optional, Tuple
 
 from chia.full_node.fee_estimate_store import FeeStore
+from chia.full_node.fee_estimation import MempoolItemInfo
 from chia.full_node.fee_estimator_constants import (
     FEE_ESTIMATOR_VERSION,
     INFINITE_FEE_RATE,
@@ -26,7 +27,6 @@ from chia.full_node.fee_estimator_constants import (
     SUFFICIENT_FEE_TXS,
 )
 from chia.full_node.fee_history import FeeStatBackup, FeeTrackerBackup
-from chia.types.mempool_item import MempoolItem
 from chia.util.ints import uint8, uint32, uint64
 
 
@@ -129,7 +129,7 @@ class FeeStat:  # TxConfirmStats
 
         self.old_unconfirmed_txs = [0 for _ in range(0, len(buckets))]
 
-    def tx_confirmed(self, blocks_to_confirm: int, item: MempoolItem) -> None:
+    def tx_confirmed(self, blocks_to_confirm: int, item: MempoolItemInfo) -> None:
         if blocks_to_confirm < 1:
             raise ValueError("tx_confirmed called with < 1 block to confirm")
 
@@ -164,7 +164,7 @@ class FeeStat:  # TxConfirmStats
         self.unconfirmed_txs[block_index][bucket_index] += 1
         return bucket_index
 
-    def remove_tx(self, latest_seen_height: uint32, item: MempoolItem, bucket_index: int) -> None:
+    def remove_tx(self, latest_seen_height: uint32, item: MempoolItemInfo, bucket_index: int) -> None:
         if item.height_added_to_mempool is None:
             return
         block_ago = latest_seen_height - item.height_added_to_mempool
@@ -475,7 +475,7 @@ class FeeTracker:
         )
         self.fee_store.store_fee_data(backup)
 
-    def process_block(self, block_height: uint32, items: List[MempoolItem]) -> None:
+    def process_block(self, block_height: uint32, items: List[MempoolItemInfo]) -> None:
         """A new block has been farmed and these transactions have been included in that block"""
         if block_height <= self.latest_seen_height:
             # Ignore reorgs
@@ -498,7 +498,7 @@ class FeeTracker:
             self.first_recorded_height = block_height
             self.log.info(f"Fee Estimator first recorded height: {self.first_recorded_height}")
 
-    def process_block_tx(self, current_height: uint32, item: MempoolItem) -> None:
+    def process_block_tx(self, current_height: uint32, item: MempoolItemInfo) -> None:
         if item.height_added_to_mempool is None:
             raise ValueError("process_block_tx called with item.height_added_to_mempool=None")
 
@@ -510,7 +510,7 @@ class FeeTracker:
         self.med_horizon.tx_confirmed(blocks_to_confirm, item)
         self.long_horizon.tx_confirmed(blocks_to_confirm, item)
 
-    def add_tx(self, item: MempoolItem) -> None:
+    def add_tx(self, item: MempoolItemInfo) -> None:
         if item.height_added_to_mempool < self.latest_seen_height:
             self.log.info(f"Processing Item from pending pool: cost={item.cost} fee={item.fee}")
 
@@ -521,7 +521,7 @@ class FeeTracker:
         self.med_horizon.new_mempool_tx(self.latest_seen_height, bucket_index)
         self.long_horizon.new_mempool_tx(self.latest_seen_height, bucket_index)
 
-    def remove_tx(self, item: MempoolItem) -> None:
+    def remove_tx(self, item: MempoolItemInfo) -> None:
         bucket_index = get_bucket_index(self.buckets, item.fee_per_cost * 1000)
         self.short_horizon.remove_tx(self.latest_seen_height, item, bucket_index)
         self.med_horizon.remove_tx(self.latest_seen_height, item, bucket_index)

chia/full_node/mempool.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional
 
 from sortedcontainers import SortedDict
 
-from chia.full_node.fee_estimation import FeeMempoolInfo, MempoolInfo
+from chia.full_node.fee_estimation import FeeMempoolInfo, MempoolInfo, MempoolItemInfo
 from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.sized_bytes import bytes32
@@ -111,7 +111,9 @@ class Mempool:
             assert self._total_mempool_cost >= 0
             info = FeeMempoolInfo(self.mempool_info, self._total_mempool_cost, self._total_mempool_fees, datetime.now())
             if reason != MempoolRemoveReason.BLOCK_INCLUSION:
-                self.fee_estimator.remove_mempool_item(info, item)
+                self.fee_estimator.remove_mempool_item(
+                    info, MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool)
+                )
 
     def add_to_pool(self, item: MempoolItem) -> None:
         """
@@ -143,7 +145,7 @@ class Mempool:
         self._total_mempool_cost = CLVMCost(uint64(self._total_mempool_cost + item.cost))
         self._total_mempool_fees = self._total_mempool_fees + item.fee
         info = FeeMempoolInfo(self.mempool_info, self._total_mempool_cost, self._total_mempool_fees, datetime.now())
-        self.fee_estimator.add_mempool_item(info, item)
+        self.fee_estimator.add_mempool_item(info, MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
 
     def at_full_capacity(self, cost: int) -> bool:
         """

chia/full_node/mempool_manager.py

@@ -16,7 +16,7 @@ from chia.consensus.constants import ConsensusConstants
 from chia.consensus.cost_calculator import NPCResult
 from chia.full_node.bitcoin_fee_estimator import create_bitcoin_fee_estimator
 from chia.full_node.bundle_tools import simple_solution_generator
-from chia.full_node.fee_estimation import FeeBlockInfo, MempoolInfo
+from chia.full_node.fee_estimation import FeeBlockInfo, MempoolInfo, MempoolItemInfo
 from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
 from chia.full_node.mempool import Mempool, MempoolRemoveReason
 from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions, mempool_check_time_locks
@@ -600,7 +600,7 @@ class MempoolManager:
             return []
         assert new_peak.timestamp is not None
         self.fee_estimator.new_block_height(new_peak.height)
-        included_items = []
+        included_items: List[MempoolItemInfo] = []
 
         use_optimization: bool = self.peak is not None and new_peak.prev_transaction_block_hash == self.peak.header_hash
         self.peak = new_peak
@@ -612,7 +612,7 @@ class MempoolManager:
                 for spend in last_npc_result.conds.spends:
                     items: List[MempoolItem] = self.mempool.get_spends_by_coin_id(bytes32(spend.coin_id))
                     for item in items:
-                        included_items.append(item)
+                        included_items.append(MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
                         self.remove_seen(item.name)
                         spendbundle_ids_to_remove.append(item.name)
                 self.mempool.remove_from_pool(spendbundle_ids_to_remove, MempoolRemoveReason.BLOCK_INCLUSION)
@@ -632,7 +632,7 @@ class MempoolManager:
                 if result == MempoolInclusionStatus.FAILED and err == Err.DOUBLE_SPEND:
                     # Item was in mempool, but after the new block it's a double spend.
                     # Item is most likely included in the block.
-                    included_items.append(item)
+                    included_items.append(MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
 
         potential_txs = self._pending_cache.drain(new_peak.height)
         potential_txs.update(self._conflict_cache.drain())

tests/core/mempool/test_mempool_fee_estimator.py

@@ -5,21 +5,18 @@ from typing import Optional
 
 import pytest
 
-from chia.consensus.cost_calculator import NPCResult
 from chia.full_node.bitcoin_fee_estimator import BitcoinFeeEstimator
 from chia.full_node.coin_store import CoinStore
 from chia.full_node.fee_estimate_store import FeeStore
+from chia.full_node.fee_estimation import MempoolItemInfo
 from chia.full_node.fee_estimator import SmartFeeEstimator
 from chia.full_node.fee_tracker import FeeTracker
 from chia.full_node.mempool_manager import MempoolManager
-from chia.simulator.wallet_tools import WalletTool
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.coin_record import CoinRecord
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.types.mempool_inclusion_status import MempoolInclusionStatus
-from chia.types.mempool_item import MempoolItem
-from chia.types.spend_bundle_conditions import SpendBundleConditions
 from chia.util.ints import uint32, uint64
 from tests.core.consensus.test_pot_iterations import test_constants
 from tests.core.mempool.test_mempool_manager import (
@@ -35,41 +32,31 @@ async def test_basics() -> None:
     fee_store = FeeStore()
     fee_tracker = FeeTracker(fee_store)
 
-    wallet_tool = WalletTool(test_constants)
-    ph = wallet_tool.get_new_puzzlehash()
-    coin = Coin(ph, ph, uint64(10000))
-    spend_bundle = wallet_tool.generate_signed_transaction(uint64(10000), ph, coin)
     cost = uint64(5000000)
     for i in range(300, 700):
         i = uint32(i)
         items = []
         for _ in range(2, 100):
             fee = uint64(10000000)
-            mempool_item = MempoolItem(
+            mempool_item = MempoolItemInfo(
-                spend_bundle,
+                cost,
                 fee,
-                NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
-                spend_bundle.name(),
                 uint32(i - 1),
             )
             items.append(mempool_item)
 
             fee1 = uint64(200000)
-            mempool_item1 = MempoolItem(
+            mempool_item1 = MempoolItemInfo(
-                spend_bundle,
+                cost,
                 fee1,
-                NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
-                spend_bundle.name(),
                 uint32(i - 40),
             )
             items.append(mempool_item1)
 
             fee2 = uint64(0)
-            mempool_item2 = MempoolItem(
+            mempool_item2 = MempoolItemInfo(
-                spend_bundle,
+                cost,
                 fee2,
-                NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
-                spend_bundle.name(),
                 uint32(i - 270),
             )
             items.append(mempool_item2)
@@ -92,10 +79,6 @@ async def test_fee_increase() -> None:
         btc_fee_estimator: BitcoinFeeEstimator = mempool_manager.mempool.fee_estimator  # type: ignore
         fee_tracker = btc_fee_estimator.get_tracker()
         estimator = SmartFeeEstimator(fee_tracker, uint64(test_constants.MAX_BLOCK_COST_CLVM))
-        wallet_tool = WalletTool(test_constants)
-        ph = wallet_tool.get_new_puzzlehash()
-        coin = Coin(ph, ph, uint64(10000))
-        spend_bundle = wallet_tool.generate_signed_transaction(uint64(10000), ph, coin)
         random = Random(x=1)
         for i in range(300, 700):
             i = uint32(i)
@@ -104,11 +87,9 @@ async def test_fee_increase() -> None:
                 fee = uint64(0)
                 included_height = uint32(random.randint(i - 60, i - 1))
                 cost = uint64(5000000)
-                mempool_item = MempoolItem(
+                mempool_item = MempoolItemInfo(
-                    spend_bundle,
+                    cost,
                     fee,
-                    NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
-                    spend_bundle.name(),
                     included_height,
                 )
                 items.append(mempool_item)

tests/fee_estimation/test_fee_estimation_integration.py

@@ -14,6 +14,7 @@ from chia.full_node.fee_estimation import (
     FeeBlockInfo,
     FeeMempoolInfo,
     MempoolInfo,
+    MempoolItemInfo,
 )
 from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
 from chia.full_node.fee_tracker import FeeTracker
@@ -67,11 +68,11 @@ class FeeEstimatorInterfaceIntegrationVerificationObject(FeeEstimatorInterface):
         self.current_block_height = block_info.block_height
         self.new_block_called_count += 1
 
-    def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         """A MempoolItem (transaction and associated info) has been added to the mempool"""
         self.add_mempool_item_called_count += 1
 
-    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
+    def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
         """A MempoolItem (transaction and associated info) has been removed from the mempool"""
         self.remove_mempool_item_called_count += 1
 
@@ -143,7 +144,7 @@ def test_mempool_manager_fee_estimator_new_block() -> None:
     mempool = Mempool(test_mempool_info, fee_estimator)
     item = make_mempoolitem()
     height = uint32(4)
-    included_items = [item]
+    included_items = [MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool)]
     mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
     assert mempool.fee_estimator.new_block_called_count == 1  # type: ignore[attr-defined]
 
@@ -187,7 +188,7 @@ def test_current_block_height_new_block() -> None:
     fee_estimator = FeeEstimatorInterfaceIntegrationVerificationObject()
     mempool = Mempool(test_mempool_info, fee_estimator)
     height = uint32(10)
-    included_items: List[MempoolItem] = []
+    included_items: List[MempoolItemInfo] = []
     mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
     assert mempool.fee_estimator.current_block_height == height  # type: ignore[attr-defined]
 
@@ -196,7 +197,7 @@ def test_current_block_height_new_height_then_new_block() -> None:
     fee_estimator = FeeEstimatorInterfaceIntegrationVerificationObject()
     mempool = Mempool(test_mempool_info, fee_estimator)
     height = uint32(11)
-    included_items: List[MempoolItem] = []
+    included_items: List[MempoolItemInfo] = []
     fee_estimator.new_block_height(uint32(height - 1))
     mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
     assert mempool.fee_estimator.current_block_height == height  # type: ignore[attr-defined]
@@ -206,7 +207,7 @@ def test_current_block_height_new_block_then_new_height() -> None:
     fee_estimator = FeeEstimatorInterfaceIntegrationVerificationObject()
     mempool = Mempool(test_mempool_info, fee_estimator)
     height = uint32(12)
-    included_items: List[MempoolItem] = []
+    included_items: List[MempoolItemInfo] = []
     fee_estimator.new_block_height(uint32(height - 1))
     mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
     fee_estimator.new_block_height(uint32(height + 1))

tests/fee_estimation/test_fee_estimation_unit_tests.py

@@ -4,19 +4,13 @@ import logging
 from typing import List
 
 import pytest
-from chia_rs import Coin
 
-from chia.consensus.cost_calculator import NPCResult
 from chia.full_node.bitcoin_fee_estimator import create_bitcoin_fee_estimator
-from chia.full_node.fee_estimation import FeeBlockInfo
+from chia.full_node.fee_estimation import FeeBlockInfo, MempoolItemInfo
 from chia.full_node.fee_estimator_constants import INFINITE_FEE_RATE, INITIAL_STEP
 from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
 from chia.full_node.fee_tracker import get_bucket_index, init_buckets
-from chia.simulator.block_tools import test_constants
-from chia.simulator.wallet_tools import WalletTool
 from chia.types.fee_rate import FeeRateV2
-from chia.types.mempool_item import MempoolItem
-from chia.types.spend_bundle_conditions import SpendBundleConditions
 from chia.util.ints import uint32, uint64
 from chia.util.math import make_monotonically_decreasing
 
@@ -52,19 +46,10 @@ def test_single_estimate() -> None:
 
 
 def make_block(
-    wallet_tool: WalletTool, height: uint32, num_tx: int, cost: uint64, fee: uint64, num_blocks_wait_in_mempool: int
+    height: uint32, num_tx: int, cost: uint64, fee: uint64, num_blocks_wait_in_mempool: int
-) -> List[MempoolItem]:
+) -> List[MempoolItemInfo]:
-    items = []
+    block_included = uint32(height - num_blocks_wait_in_mempool)
-    ph = wallet_tool.get_new_puzzlehash()
+    return [MempoolItemInfo(cost, fee, block_included)] * num_tx
-    coin = Coin(ph, ph, uint64(10000))
-    spend_bundle = wallet_tool.generate_signed_transaction(uint64(10000), ph, coin)
-
-    for n in range(num_tx):
-        block_included = uint32(height - num_blocks_wait_in_mempool)
-        conds = SpendBundleConditions([], 0, 0, 0, None, None, [], cost)
-        mempool_item = MempoolItem(spend_bundle, fee, NPCResult(None, conds, cost), spend_bundle.name(), block_included)
-        items.append(mempool_item)
-    return items
 
 
 def test_steady_fee_pressure() -> None:
@@ -74,7 +59,6 @@ def test_steady_fee_pressure() -> None:
     """
     max_block_cost_clvm = uint64(1000 * 1000)
     estimator = create_bitcoin_fee_estimator(max_block_cost_clvm)
-    wallet_tool = WalletTool(test_constants)
     cost = uint64(5000000)
     fee = uint64(10000000)
     num_blocks_wait_in_mempool = 5
@@ -84,7 +68,7 @@ def test_steady_fee_pressure() -> None:
     estimates_during = []
     for height in range(start, end):
         height = uint32(height)
-        items = make_block(wallet_tool, height, 1, cost, fee, num_blocks_wait_in_mempool)
+        items = make_block(height, 1, cost, fee, num_blocks_wait_in_mempool)
         estimator.new_block(FeeBlockInfo(uint32(height), items))
         estimates_during.append(estimator.estimate_fee_rate(time_offset_seconds=40 * height))