Chia-Network/chia-blockchain
1
0
Fork 0
mirror of https://github.com/Chia-Network/chia-blockchain.git synced 2024-09-19 06:39:51 +03:00
Code Issues Packages Projects Releases Wiki Activity

simplify fee estimator by not leaking MempoolItem into its interface (#14685)

Browse Source
This commit is contained in:
Arvid Norberg 2023-02-27 23:34:11 +01:00 committed by GitHub
parent 6b680cf204
commit 5a5cb2e58e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 64 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
chia/full_node/bitcoin_fee_estimator.py
View File

@ -1,13 +1,12 @@
from __future__ import annotations
from chia.full_node.fee_estimate_store import FeeStore
from chia.full_node.fee_estimation import EmptyFeeMempoolInfo, FeeBlockInfo, FeeMempoolInfo
from chia.full_node.fee_estimation import EmptyFeeMempoolInfo, FeeBlockInfo, FeeMempoolInfo, MempoolItemInfo
from chia.full_node.fee_estimator import SmartFeeEstimator
from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
from chia.full_node.fee_tracker import FeeTracker
from chia.types.clvm_cost import CLVMCost
from chia.types.fee_rate import FeeRateV2
from chia.types.mempool_item import MempoolItem
from chia.util.ints import uint32, uint64
@ -35,11 +34,11 @@ class BitcoinFeeEstimator(FeeEstimatorInterface):
self.block_height = block_info.block_height
self.tracker.process_block(block_info.block_height, block_info.included_items)
def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
self.last_mempool_info = mempool_info
self.tracker.add_tx(mempool_item)
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
self.last_mempool_info = mempool_info
self.tracker.remove_tx(mempool_item)

19
chia/full_node/fee_estimation.py
View File

@ -6,11 +6,26 @@ from typing import List
from chia.types.clvm_cost import CLVMCost
from chia.types.fee_rate import FeeRate
from chia.types.mempool_item import MempoolItem
from chia.types.mojos import Mojos
from chia.util.ints import uint32, uint64
@dataclass(frozen=True)
class MempoolItemInfo:
"""
The information the fee estimator is passed for each mempool item that's
added, removed from the mempool and included in blocks
"""
cost: int
fee: int
height_added_to_mempool: uint32
@property
def fee_per_cost(self) -> float:
return self.fee / self.cost
@dataclass(frozen=True)
class MempoolInfo:
"""
@ -75,4 +90,4 @@ class FeeBlockInfo: # See BlockRecord
"""
block_height: uint32
included_items: List[MempoolItem]
included_items: List[MempoolItemInfo]

7
chia/full_node/fee_estimator_example.py
View File

@ -3,11 +3,10 @@ from __future__ import annotations
from typing import Any, Dict, List
from chia.full_node.fee_estimate import FeeEstimateV2
from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo
from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo, MempoolItemInfo
from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
from chia.types.clvm_cost import CLVMCost
from chia.types.fee_rate import FeeRateV2
from chia.types.mempool_item import MempoolItem
from chia.util.ints import uint64
MIN_MOJO_PER_COST = 5
@ -31,10 +30,10 @@ class FeeEstimatorExample(FeeEstimatorInterface):
def new_block(self, block_info: FeeBlockInfo) -> None:
pass
def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
pass
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
pass
def estimate_fee_rate(self, *, time_offset_seconds: int) -> FeeRateV2:

7
chia/full_node/fee_estimator_interface.py
View File

@ -2,10 +2,9 @@ from __future__ import annotations
from typing_extensions import Protocol
from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo
from chia.full_node.fee_estimation import FeeBlockInfo, FeeMempoolInfo, MempoolItemInfo
from chia.types.clvm_cost import CLVMCost
from chia.types.fee_rate import FeeRateV2
from chia.types.mempool_item import MempoolItem
from chia.util.ints import uint32
@ -18,11 +17,11 @@ class FeeEstimatorInterface(Protocol):
"""A new transaction block has been added to the blockchain"""
pass
def add_mempool_item(self, mempool_item_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def add_mempool_item(self, mempool_item_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
"""A MempoolItem (transaction and associated info) has been added to the mempool"""
pass
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
"""A MempoolItem (transaction and associated info) has been removed from the mempool"""
pass

14
chia/full_node/fee_tracker.py
View File

@ -6,6 +6,7 @@ from dataclasses import dataclass
from typing import List, Optional, Tuple
from chia.full_node.fee_estimate_store import FeeStore
from chia.full_node.fee_estimation import MempoolItemInfo
from chia.full_node.fee_estimator_constants import (
FEE_ESTIMATOR_VERSION,
INFINITE_FEE_RATE,
@ -26,7 +27,6 @@ from chia.full_node.fee_estimator_constants import (
SUFFICIENT_FEE_TXS,
)
from chia.full_node.fee_history import FeeStatBackup, FeeTrackerBackup
from chia.types.mempool_item import MempoolItem
from chia.util.ints import uint8, uint32, uint64
@ -129,7 +129,7 @@ class FeeStat: # TxConfirmStats
self.old_unconfirmed_txs = [0 for _ in range(0, len(buckets))]
def tx_confirmed(self, blocks_to_confirm: int, item: MempoolItem) -> None:
def tx_confirmed(self, blocks_to_confirm: int, item: MempoolItemInfo) -> None:
if blocks_to_confirm < 1:
raise ValueError("tx_confirmed called with < 1 block to confirm")
@ -164,7 +164,7 @@ class FeeStat: # TxConfirmStats
self.unconfirmed_txs[block_index][bucket_index] += 1
return bucket_index
def remove_tx(self, latest_seen_height: uint32, item: MempoolItem, bucket_index: int) -> None:
def remove_tx(self, latest_seen_height: uint32, item: MempoolItemInfo, bucket_index: int) -> None:
if item.height_added_to_mempool is None:
return
block_ago = latest_seen_height - item.height_added_to_mempool
@ -475,7 +475,7 @@ class FeeTracker:
)
self.fee_store.store_fee_data(backup)
def process_block(self, block_height: uint32, items: List[MempoolItem]) -> None:
def process_block(self, block_height: uint32, items: List[MempoolItemInfo]) -> None:
"""A new block has been farmed and these transactions have been included in that block"""
if block_height <= self.latest_seen_height:
# Ignore reorgs
@ -498,7 +498,7 @@ class FeeTracker:
self.first_recorded_height = block_height
self.log.info(f"Fee Estimator first recorded height: {self.first_recorded_height}")
def process_block_tx(self, current_height: uint32, item: MempoolItem) -> None:
def process_block_tx(self, current_height: uint32, item: MempoolItemInfo) -> None:
if item.height_added_to_mempool is None:
raise ValueError("process_block_tx called with item.height_added_to_mempool=None")
@ -510,7 +510,7 @@ class FeeTracker:
self.med_horizon.tx_confirmed(blocks_to_confirm, item)
self.long_horizon.tx_confirmed(blocks_to_confirm, item)
def add_tx(self, item: MempoolItem) -> None:
def add_tx(self, item: MempoolItemInfo) -> None:
if item.height_added_to_mempool < self.latest_seen_height:
self.log.info(f"Processing Item from pending pool: cost={item.cost} fee={item.fee}")
@ -521,7 +521,7 @@ class FeeTracker:
self.med_horizon.new_mempool_tx(self.latest_seen_height, bucket_index)
self.long_horizon.new_mempool_tx(self.latest_seen_height, bucket_index)
def remove_tx(self, item: MempoolItem) -> None:
def remove_tx(self, item: MempoolItemInfo) -> None:
bucket_index = get_bucket_index(self.buckets, item.fee_per_cost * 1000)
self.short_horizon.remove_tx(self.latest_seen_height, item, bucket_index)
self.med_horizon.remove_tx(self.latest_seen_height, item, bucket_index)

8
chia/full_node/mempool.py
View File

@ -6,7 +6,7 @@ from typing import Dict, List, Optional
from sortedcontainers import SortedDict
from chia.full_node.fee_estimation import FeeMempoolInfo, MempoolInfo
from chia.full_node.fee_estimation import FeeMempoolInfo, MempoolInfo, MempoolItemInfo
from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
from chia.types.blockchain_format.coin import Coin
from chia.types.blockchain_format.sized_bytes import bytes32
@ -111,7 +111,9 @@ class Mempool:
assert self._total_mempool_cost >= 0
info = FeeMempoolInfo(self.mempool_info, self._total_mempool_cost, self._total_mempool_fees, datetime.now())
if reason != MempoolRemoveReason.BLOCK_INCLUSION:
self.fee_estimator.remove_mempool_item(info, item)
self.fee_estimator.remove_mempool_item(
info, MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool)
)
def add_to_pool(self, item: MempoolItem) -> None:
"""
@ -143,7 +145,7 @@ class Mempool:
self._total_mempool_cost = CLVMCost(uint64(self._total_mempool_cost + item.cost))
self._total_mempool_fees = self._total_mempool_fees + item.fee
info = FeeMempoolInfo(self.mempool_info, self._total_mempool_cost, self._total_mempool_fees, datetime.now())
self.fee_estimator.add_mempool_item(info, item)
self.fee_estimator.add_mempool_item(info, MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
def at_full_capacity(self, cost: int) -> bool:
"""

8
chia/full_node/mempool_manager.py
View File

@ -16,7 +16,7 @@ from chia.consensus.constants import ConsensusConstants
from chia.consensus.cost_calculator import NPCResult
from chia.full_node.bitcoin_fee_estimator import create_bitcoin_fee_estimator
from chia.full_node.bundle_tools import simple_solution_generator
from chia.full_node.fee_estimation import FeeBlockInfo, MempoolInfo
from chia.full_node.fee_estimation import FeeBlockInfo, MempoolInfo, MempoolItemInfo
from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
from chia.full_node.mempool import Mempool, MempoolRemoveReason
from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions, mempool_check_time_locks
@ -600,7 +600,7 @@ class MempoolManager:
return []
assert new_peak.timestamp is not None
self.fee_estimator.new_block_height(new_peak.height)
included_items = []
included_items: List[MempoolItemInfo] = []
use_optimization: bool = self.peak is not None and new_peak.prev_transaction_block_hash == self.peak.header_hash
self.peak = new_peak
@ -612,7 +612,7 @@ class MempoolManager:
for spend in last_npc_result.conds.spends:
items: List[MempoolItem] = self.mempool.get_spends_by_coin_id(bytes32(spend.coin_id))
for item in items:
included_items.append(item)
included_items.append(MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
self.remove_seen(item.name)
spendbundle_ids_to_remove.append(item.name)
self.mempool.remove_from_pool(spendbundle_ids_to_remove, MempoolRemoveReason.BLOCK_INCLUSION)
@ -632,7 +632,7 @@ class MempoolManager:
if result == MempoolInclusionStatus.FAILED and err == Err.DOUBLE_SPEND:
# Item was in mempool, but after the new block it's a double spend.
# Item is most likely included in the block.
included_items.append(item)
included_items.append(MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
potential_txs = self._pending_cache.drain(new_peak.height)
potential_txs.update(self._conflict_cache.drain())

37
tests/core/mempool/test_mempool_fee_estimator.py
View File

@ -5,21 +5,18 @@ from typing import Optional
import pytest
from chia.consensus.cost_calculator import NPCResult
from chia.full_node.bitcoin_fee_estimator import BitcoinFeeEstimator
from chia.full_node.coin_store import CoinStore
from chia.full_node.fee_estimate_store import FeeStore
from chia.full_node.fee_estimation import MempoolItemInfo
from chia.full_node.fee_estimator import SmartFeeEstimator
from chia.full_node.fee_tracker import FeeTracker
from chia.full_node.mempool_manager import MempoolManager
from chia.simulator.wallet_tools import WalletTool
from chia.types.blockchain_format.coin import Coin
from chia.types.blockchain_format.sized_bytes import bytes32
from chia.types.coin_record import CoinRecord
from chia.types.condition_opcodes import ConditionOpcode
from chia.types.mempool_inclusion_status import MempoolInclusionStatus
from chia.types.mempool_item import MempoolItem
from chia.types.spend_bundle_conditions import SpendBundleConditions
from chia.util.ints import uint32, uint64
from tests.core.consensus.test_pot_iterations import test_constants
from tests.core.mempool.test_mempool_manager import (
@ -35,41 +32,31 @@ async def test_basics() -> None:
fee_store = FeeStore()
fee_tracker = FeeTracker(fee_store)
wallet_tool = WalletTool(test_constants)
ph = wallet_tool.get_new_puzzlehash()
coin = Coin(ph, ph, uint64(10000))
spend_bundle = wallet_tool.generate_signed_transaction(uint64(10000), ph, coin)
cost = uint64(5000000)
for i in range(300, 700):
i = uint32(i)
items = []
for _ in range(2, 100):
fee = uint64(10000000)
mempool_item = MempoolItem(
spend_bundle,
mempool_item = MempoolItemInfo(
cost,
fee,
NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
spend_bundle.name(),
uint32(i - 1),
)
items.append(mempool_item)
fee1 = uint64(200000)
mempool_item1 = MempoolItem(
spend_bundle,
mempool_item1 = MempoolItemInfo(
cost,
fee1,
NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
spend_bundle.name(),
uint32(i - 40),
)
items.append(mempool_item1)
fee2 = uint64(0)
mempool_item2 = MempoolItem(
spend_bundle,
mempool_item2 = MempoolItemInfo(
cost,
fee2,
NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
spend_bundle.name(),
uint32(i - 270),
)
items.append(mempool_item2)
@ -92,10 +79,6 @@ async def test_fee_increase() -> None:
btc_fee_estimator: BitcoinFeeEstimator = mempool_manager.mempool.fee_estimator # type: ignore
fee_tracker = btc_fee_estimator.get_tracker()
estimator = SmartFeeEstimator(fee_tracker, uint64(test_constants.MAX_BLOCK_COST_CLVM))
wallet_tool = WalletTool(test_constants)
ph = wallet_tool.get_new_puzzlehash()
coin = Coin(ph, ph, uint64(10000))
spend_bundle = wallet_tool.generate_signed_transaction(uint64(10000), ph, coin)
random = Random(x=1)
for i in range(300, 700):
i = uint32(i)
@ -104,11 +87,9 @@ async def test_fee_increase() -> None:
fee = uint64(0)
included_height = uint32(random.randint(i - 60, i - 1))
cost = uint64(5000000)
mempool_item = MempoolItem(
spend_bundle,
mempool_item = MempoolItemInfo(
cost,
fee,
NPCResult(None, SpendBundleConditions([], 0, 0, 0, None, None, [], cost), cost),
spend_bundle.name(),
included_height,
)
items.append(mempool_item)

13
tests/fee_estimation/test_fee_estimation_integration.py
View File

@ -14,6 +14,7 @@ from chia.full_node.fee_estimation import (
FeeBlockInfo,
FeeMempoolInfo,
MempoolInfo,
MempoolItemInfo,
)
from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
from chia.full_node.fee_tracker import FeeTracker
@ -67,11 +68,11 @@ class FeeEstimatorInterfaceIntegrationVerificationObject(FeeEstimatorInterface):
self.current_block_height = block_info.block_height
self.new_block_called_count += 1
def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def add_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
"""A MempoolItem (transaction and associated info) has been added to the mempool"""
self.add_mempool_item_called_count += 1
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItem) -> None:
def remove_mempool_item(self, mempool_info: FeeMempoolInfo, mempool_item: MempoolItemInfo) -> None:
"""A MempoolItem (transaction and associated info) has been removed from the mempool"""
self.remove_mempool_item_called_count += 1
@ -143,7 +144,7 @@ def test_mempool_manager_fee_estimator_new_block() -> None:
mempool = Mempool(test_mempool_info, fee_estimator)
item = make_mempoolitem()
height = uint32(4)
included_items = [item]
included_items = [MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool)]
mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
assert mempool.fee_estimator.new_block_called_count == 1 # type: ignore[attr-defined]
@ -187,7 +188,7 @@ def test_current_block_height_new_block() -> None:
fee_estimator = FeeEstimatorInterfaceIntegrationVerificationObject()
mempool = Mempool(test_mempool_info, fee_estimator)
height = uint32(10)
included_items: List[MempoolItem] = []
included_items: List[MempoolItemInfo] = []
mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
assert mempool.fee_estimator.current_block_height == height # type: ignore[attr-defined]
@ -196,7 +197,7 @@ def test_current_block_height_new_height_then_new_block() -> None:
fee_estimator = FeeEstimatorInterfaceIntegrationVerificationObject()
mempool = Mempool(test_mempool_info, fee_estimator)
height = uint32(11)
included_items: List[MempoolItem] = []
included_items: List[MempoolItemInfo] = []
fee_estimator.new_block_height(uint32(height - 1))
mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
assert mempool.fee_estimator.current_block_height == height # type: ignore[attr-defined]
@ -206,7 +207,7 @@ def test_current_block_height_new_block_then_new_height() -> None:
fee_estimator = FeeEstimatorInterfaceIntegrationVerificationObject()
mempool = Mempool(test_mempool_info, fee_estimator)
height = uint32(12)
included_items: List[MempoolItem] = []
included_items: List[MempoolItemInfo] = []
fee_estimator.new_block_height(uint32(height - 1))
mempool.fee_estimator.new_block(FeeBlockInfo(height, included_items))
fee_estimator.new_block_height(uint32(height + 1))

28
tests/fee_estimation/test_fee_estimation_unit_tests.py
View File

@ -4,19 +4,13 @@ import logging
from typing import List
import pytest
from chia_rs import Coin
from chia.consensus.cost_calculator import NPCResult
from chia.full_node.bitcoin_fee_estimator import create_bitcoin_fee_estimator
from chia.full_node.fee_estimation import FeeBlockInfo
from chia.full_node.fee_estimation import FeeBlockInfo, MempoolItemInfo
from chia.full_node.fee_estimator_constants import INFINITE_FEE_RATE, INITIAL_STEP
from chia.full_node.fee_estimator_interface import FeeEstimatorInterface
from chia.full_node.fee_tracker import get_bucket_index, init_buckets
from chia.simulator.block_tools import test_constants
from chia.simulator.wallet_tools import WalletTool
from chia.types.fee_rate import FeeRateV2
from chia.types.mempool_item import MempoolItem
from chia.types.spend_bundle_conditions import SpendBundleConditions
from chia.util.ints import uint32, uint64
from chia.util.math import make_monotonically_decreasing
@ -52,19 +46,10 @@ def test_single_estimate() -> None:
def make_block(
wallet_tool: WalletTool, height: uint32, num_tx: int, cost: uint64, fee: uint64, num_blocks_wait_in_mempool: int
) -> List[MempoolItem]:
items = []
ph = wallet_tool.get_new_puzzlehash()
coin = Coin(ph, ph, uint64(10000))
spend_bundle = wallet_tool.generate_signed_transaction(uint64(10000), ph, coin)
for n in range(num_tx):
block_included = uint32(height - num_blocks_wait_in_mempool)
conds = SpendBundleConditions([], 0, 0, 0, None, None, [], cost)
mempool_item = MempoolItem(spend_bundle, fee, NPCResult(None, conds, cost), spend_bundle.name(), block_included)
items.append(mempool_item)
return items
height: uint32, num_tx: int, cost: uint64, fee: uint64, num_blocks_wait_in_mempool: int
) -> List[MempoolItemInfo]:
block_included = uint32(height - num_blocks_wait_in_mempool)
return [MempoolItemInfo(cost, fee, block_included)] * num_tx
def test_steady_fee_pressure() -> None:
@ -74,7 +59,6 @@ def test_steady_fee_pressure() -> None:
"""
max_block_cost_clvm = uint64(1000 * 1000)
estimator = create_bitcoin_fee_estimator(max_block_cost_clvm)
wallet_tool = WalletTool(test_constants)
cost = uint64(5000000)
fee = uint64(10000000)
num_blocks_wait_in_mempool = 5
@ -84,7 +68,7 @@ def test_steady_fee_pressure() -> None:
estimates_during = []
for height in range(start, end):
height = uint32(height)
items = make_block(wallet_tool, height, 1, cost, fee, num_blocks_wait_in_mempool)
items = make_block(height, 1, cost, fee, num_blocks_wait_in_mempool)
estimator.new_block(FeeBlockInfo(uint32(height), items))
estimates_during.append(estimator.estimate_fee_rate(time_offset_seconds=40 * height))