Chia-Network/chia-blockchain
1
0
Fork 0
mirror of https://github.com/Chia-Network/chia-blockchain.git synced 2024-09-11 10:56:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint: into main from release/2.1.4 @ 55c064a292 (#17141)

Browse Source 
Source hash: 55c064a292
Remaining commits: 0
This commit is contained in:
Chris Marslender 2023-12-22 08:32:34 -06:00 committed by GitHub
commit 739b0d84b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 84 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
chia/consensus/default_constants.py
View File

@ -43,8 +43,8 @@ DEFAULT_CONSTANTS = ConsensusConstants(
"3d8765d3a597ec1d99663f6c9816d915b9f68613ac94009884c4addaefcce6af"
),
MAX_VDF_WITNESS_SIZE=64,
# Size of mempool = 50x the size of block
MEMPOOL_BLOCK_BUFFER=50,
# Size of mempool = 10x the size of block
MEMPOOL_BLOCK_BUFFER=10,
# Max coin amount, fits into 64 bits
MAX_COIN_AMOUNT=uint64((1 << 64) - 1),
# Max block cost in clvm cost units

40
chia/full_node/mempool.py
View File

@ -48,11 +48,16 @@ class Mempool:
_block_height: uint32
_timestamp: uint64
_total_fee: int
_total_cost: int
def __init__(self, mempool_info: MempoolInfo, fee_estimator: FeeEstimatorInterface):
self._db_conn = sqlite3.connect(":memory:")
self._items = {}
self._block_height = uint32(0)
self._timestamp = uint64(0)
self._total_fee = 0
self._total_cost = 0
with self._db_conn:
# name means SpendBundle hash
@ -75,8 +80,6 @@ class Mempool:
"""
)
self._db_conn.execute("CREATE INDEX name_idx ON tx(name)")
self._db_conn.execute("CREATE INDEX fee_sum ON tx(fee)")
self._db_conn.execute("CREATE INDEX cost_sum ON tx(cost)")
self._db_conn.execute("CREATE INDEX feerate ON tx(fee_per_cost)")
self._db_conn.execute(
"CREATE INDEX assert_before ON tx(assert_before_height, assert_before_seconds) "
@ -121,16 +124,10 @@ class Mempool:
)
def total_mempool_fees(self) -> int:
with self._db_conn:
cursor = self._db_conn.execute("SELECT SUM(fee) FROM tx")
val = cursor.fetchone()[0]
return uint64(0) if val is None else uint64(val)
return self._total_fee
def total_mempool_cost(self) -> CLVMCost:
with self._db_conn:
cursor = self._db_conn.execute("SELECT SUM(cost) FROM tx")
val = cursor.fetchone()[0]
return CLVMCost(uint64(0) if val is None else uint64(val))
return CLVMCost(uint64(self._total_cost))
def all_items(self) -> Iterator[MempoolItem]:
with self._db_conn:
@ -193,7 +190,7 @@ class Mempool:
return 0
# TODO: make MempoolItem.cost be CLVMCost
current_cost = int(self.total_mempool_cost())
current_cost = self._total_cost
# Iterates through all spends in increasing fee per cost
with self._db_conn:
@ -256,9 +253,19 @@ class Mempool:
for batch in to_batches(items, SQLITE_MAX_VARIABLE_NUMBER):
args = ",".join(["?"] * len(batch.entries))
with self._db_conn:
cursor = self._db_conn.execute(
f"SELECT SUM(cost), SUM(fee) FROM tx WHERE name in ({args})", batch.entries
)
cost_to_remove, fee_to_remove = cursor.fetchone()
self._db_conn.execute(f"DELETE FROM tx WHERE name in ({args})", batch.entries)
self._db_conn.execute(f"DELETE FROM spends WHERE tx in ({args})", batch.entries)
self._total_cost -= cost_to_remove
self._total_fee -= fee_to_remove
assert self._total_cost >= 0
assert self._total_fee >= 0
if reason != MempoolRemoveReason.BLOCK_INCLUSION:
info = FeeMempoolInfo(
self.mempool_info, self.total_mempool_cost(), self.total_mempool_fees(), datetime.now()
@ -311,11 +318,12 @@ class Mempool:
if fee_per_cost > item.fee_per_cost:
return Err.INVALID_FEE_LOW_FEE
to_remove.append(name)
self.remove_from_pool(to_remove, MempoolRemoveReason.EXPIRED)
# if we don't find any entries, it's OK to add this entry
total_cost = int(self.total_mempool_cost())
if total_cost + item.cost > self.mempool_info.max_size_in_cost:
if self._total_cost + item.cost > self.mempool_info.max_size_in_cost:
# pick the items with the lowest fee per cost to remove
cursor = self._db_conn.execute(
"""SELECT name FROM tx
@ -329,6 +337,7 @@ class Mempool:
(self.mempool_info.max_size_in_cost - item.cost,),
)
to_remove = [bytes32(row[0]) for row in cursor]
self.remove_from_pool(to_remove, MempoolRemoveReason.POOL_FULL)
# TODO: In the future, for the "fee_per_cost" field, opt for
@ -355,6 +364,9 @@ class Mempool:
item.spend_bundle, item.npc_result, item.height_added_to_mempool, item.bundle_coin_spends
)
self._total_cost += item.cost
self._total_fee += item.fee
info = FeeMempoolInfo(self.mempool_info, self.total_mempool_cost(), self.total_mempool_fees(), datetime.now())
self.fee_estimator.add_mempool_item(info, MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
return None
@ -364,7 +376,7 @@ class Mempool:
Checks whether the mempool is at full capacity and cannot accept a transaction with size cost.
"""
return self.total_mempool_cost() + cost > self.mempool_info.max_size_in_cost
return self._total_cost + cost > self.mempool_info.max_size_in_cost
def create_bundle_from_mempool_items(
self, item_inclusion_filter: Callable[[bytes32], bool]

6
chia/full_node/mempool_manager.py
View File

@ -632,6 +632,12 @@ class MempoolManager:
spendbundle_ids_to_remove.add(item.name)
self.mempool.remove_from_pool(list(spendbundle_ids_to_remove), MempoolRemoveReason.BLOCK_INCLUSION)
else:
log.warning(
"updating the mempool using the slow-path. "
f"peak: {self.peak.header_hash} "
f"new-peak-prev: {new_peak.prev_transaction_block_hash} "
f"coins: {'not set' if spent_coins is None else 'set'}"
)
old_pool = self.mempool
self.mempool = Mempool(old_pool.mempool_info, old_pool.fee_estimator)
self.seen_bundle_hashes = {}

28
tests/core/mempool/test_mempool.py
View File

@ -58,7 +58,7 @@ from tests.core.mempool.test_mempool_manager import (
spend_bundle_from_conditions,
)
from tests.core.node_height import node_height_at_least
from tests.util.misc import BenchmarkRunner
from tests.util.misc import BenchmarkRunner, invariant_check_mempool
from tests.util.time_out_assert import time_out_assert
BURN_PUZZLE_HASH = bytes32(b"0" * 32)
@ -335,7 +335,9 @@ async def respond_transaction(
self.full_node.full_node_store.pending_tx_request.pop(spend_name)
if spend_name in self.full_node.full_node_store.peers_with_tx:
self.full_node.full_node_store.peers_with_tx.pop(spend_name)
return await self.full_node.add_transaction(tx.transaction, spend_name, peer, test)
ret = await self.full_node.add_transaction(tx.transaction, spend_name, peer, test)
invariant_check_mempool(self.full_node.mempool_manager.mempool)
return ret
async def next_block(full_node_1, wallet_a, bt) -> Coin:
@ -579,6 +581,7 @@ class TestMempoolManager:
)
peer = await connect_and_get_peer(server_1, server_2, self_hostname)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
for block in blocks:
await full_node_1.full_node.add_block(block)
await time_out_assert(60, node_height_at_least, True, full_node_1, start_height + 3)
@ -594,12 +597,14 @@ class TestMempoolManager:
# Fee increase is insufficient, the old spendbundle must stay
self.assert_sb_in_pool(full_node_1, sb1_1)
self.assert_sb_not_in_pool(full_node_1, sb1_2)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
sb1_3 = await self.gen_and_send_sb(full_node_1, peer, wallet_a, coin1, fee=MEMPOOL_MIN_FEE_INCREASE)
# Fee increase is sufficiently high, sb1_1 gets replaced with sb1_3
self.assert_sb_not_in_pool(full_node_1, sb1_1)
self.assert_sb_in_pool(full_node_1, sb1_3)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
sb2 = generate_test_spend_bundle(wallet_a, coin2, fee=MEMPOOL_MIN_FEE_INCREASE)
sb12 = SpendBundle.aggregate((sb2, sb1_3))
@ -609,6 +614,7 @@ class TestMempoolManager:
# of coins spent in sb1_3
self.assert_sb_in_pool(full_node_1, sb12)
self.assert_sb_not_in_pool(full_node_1, sb1_3)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
sb3 = generate_test_spend_bundle(wallet_a, coin3, fee=uint64(MEMPOOL_MIN_FEE_INCREASE * 2))
sb23 = SpendBundle.aggregate((sb2, sb3))
@ -618,16 +624,19 @@ class TestMempoolManager:
# coins that are spent in the latter (specifically, coin1)
self.assert_sb_in_pool(full_node_1, sb12)
self.assert_sb_not_in_pool(full_node_1, sb23)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
await self.send_sb(full_node_1, sb3)
# Adding non-conflicting sb3 should succeed
self.assert_sb_in_pool(full_node_1, sb3)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
sb4_1 = generate_test_spend_bundle(wallet_a, coin4, fee=MEMPOOL_MIN_FEE_INCREASE)
sb1234_1 = SpendBundle.aggregate((sb12, sb3, sb4_1))
await self.send_sb(full_node_1, sb1234_1)
# sb1234_1 should not be in pool as it decreases total fees per cost
self.assert_sb_not_in_pool(full_node_1, sb1234_1)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
sb4_2 = generate_test_spend_bundle(wallet_a, coin4, fee=uint64(MEMPOOL_MIN_FEE_INCREASE * 2))
sb1234_2 = SpendBundle.aggregate((sb12, sb3, sb4_2))
@ -637,6 +646,7 @@ class TestMempoolManager:
self.assert_sb_in_pool(full_node_1, sb1234_2)
self.assert_sb_not_in_pool(full_node_1, sb12)
self.assert_sb_not_in_pool(full_node_1, sb3)
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
@pytest.mark.anyio
async def test_invalid_signature(self, one_node_one_block, wallet_a):
@ -668,6 +678,7 @@ class TestMempoolManager:
ack: TransactionAck = TransactionAck.from_bytes(res.data)
assert ack.status == MempoolInclusionStatus.FAILED.value
assert ack.error == Err.BAD_AGGREGATE_SIGNATURE.name
invariant_check_mempool(full_node_1.full_node.mempool_manager.mempool)
async def condition_tester(
self,
@ -2763,13 +2774,16 @@ def test_full_mempool(items: List[int], add: int, expected: List[int]) -> None:
CLVMCost(uint64(100)),
)
mempool = Mempool(mempool_info, fee_estimator)
invariant_check_mempool(mempool)
fee_rate: float = 3.0
for i in items:
mempool.add_to_pool(item_cost(i, fee_rate))
fee_rate -= 0.1
invariant_check_mempool(mempool)
# now, add the item we're testing
mempool.add_to_pool(item_cost(add, 3.1))
invariant_check_mempool(mempool)
ordered_items = list(mempool.items_by_feerate())
@ -2808,12 +2822,14 @@ def test_limit_expiring_transactions(height: bool, items: List[int], expected: L
)
mempool = Mempool(mempool_info, fee_estimator)
mempool.new_tx_block(uint32(10), uint64(100000))
invariant_check_mempool(mempool)
# fill the mempool with regular transactions (without expiration)
fee_rate: float = 3.0
for i in range(1, 20):
mempool.add_to_pool(item_cost(i, fee_rate))
fee_rate -= 0.1
invariant_check_mempool(mempool)
# now add the expiring transactions from the test case
fee_rate = 2.7
@ -2825,6 +2841,7 @@ def test_limit_expiring_transactions(height: bool, items: List[int], expected: L
ret = mempool.add_to_pool(mk_item([coin], cost=cost, fee=int(cost * fee_rate), assert_before_height=15))
else:
ret = mempool.add_to_pool(mk_item([coin], cost=cost, fee=int(cost * fee_rate), assert_before_seconds=10400))
invariant_check_mempool(mempool)
if increase_fee:
fee_rate += 0.1
assert ret is None
@ -2848,6 +2865,7 @@ def test_limit_expiring_transactions(height: bool, items: List[int], expected: L
print(f"- cost: {item.cost} TTL: {ttl}")
assert mempool.total_mempool_cost() > 90
invariant_check_mempool(mempool)
@pytest.mark.parametrize(
@ -2883,6 +2901,7 @@ def test_get_items_by_coin_ids(items: List[MempoolItem], coin_ids: List[bytes32]
mempool = Mempool(mempool_info, fee_estimator)
for i in items:
mempool.add_to_pool(i)
invariant_check_mempool(mempool)
result = mempool.get_items_by_coin_ids(coin_ids)
assert set(result) == set(expected)
@ -2905,6 +2924,7 @@ def test_aggregating_on_a_solution_then_a_more_cost_saving_one_appears() -> None
sb = SpendBundle.aggregate(spend_bundles)
mi = mempool_item_from_spendbundle(sb)
mempool.add_to_pool(mi)
invariant_check_mempool(mempool)
saved_cost = run_for_cost(
sb.coin_spends[0].puzzle_reveal, sb.coin_spends[0].solution, len(mi.additions), mi.cost
)
@ -2925,9 +2945,11 @@ def test_aggregating_on_a_solution_then_a_more_cost_saving_one_appears() -> None
highest_fee = 58282830
sb_high_rate = make_test_spendbundle(coins[1], fee=highest_fee)
agg_and_add_sb_returning_cost_info(mempool, [sb_A, sb_high_rate])
invariant_check_mempool(mempool)
# Create a ~2 FPC item that spends the eligible coin using the same solution A
sb_low_rate = make_test_spendbundle(coins[2], fee=highest_fee // 5)
saved_cost_on_solution_A = agg_and_add_sb_returning_cost_info(mempool, [sb_A, sb_low_rate])
invariant_check_mempool(mempool)
result = mempool.create_bundle_from_mempool_items(always)
assert result is not None
agg, _ = result
@ -2941,6 +2963,7 @@ def test_aggregating_on_a_solution_then_a_more_cost_saving_one_appears() -> None
# (which has ~10 FPC) but before sb_A2 (which has ~2 FPC)
sb_mid_rate = make_test_spendbundle(coins[i], fee=38004852 - i)
saved_cost_on_solution_B = agg_and_add_sb_returning_cost_info(mempool, [sb_B, sb_mid_rate])
invariant_check_mempool(mempool)
# We'd save more cost if we went with solution B instead of A
assert saved_cost_on_solution_B > saved_cost_on_solution_A
# If we process everything now, the 3 x ~3 FPC items get skipped because
@ -2953,6 +2976,7 @@ def test_aggregating_on_a_solution_then_a_more_cost_saving_one_appears() -> None
# We ran with solution A and missed bigger savings on solution B
assert mempool.size() == 5
assert [c.coin for c in agg.coin_spends] == [coins[0], coins[1], coins[2]]
invariant_check_mempool(mempool)
def test_get_puzzle_and_solution_for_coin_failure():

8
tests/core/mempool/test_mempool_manager.py
View File

@ -46,6 +46,7 @@ from chia.wallet.util.tx_config import DEFAULT_TX_CONFIG
from chia.wallet.wallet import Wallet
from chia.wallet.wallet_coin_record import WalletCoinRecord
from chia.wallet.wallet_node import WalletNode
from tests.util.misc import invariant_check_mempool
from tests.util.setup_nodes import OldSimulatorsAndWallets
IDENTITY_PUZZLE = SerializedProgram.to(1)
@ -121,6 +122,7 @@ async def instantiate_mempool_manager(
mempool_manager = MempoolManager(get_coin_record, constants)
test_block_record = create_test_block_record(height=block_height, timestamp=block_timestamp)
await mempool_manager.new_peak(test_block_record, None)
invariant_check_mempool(mempool_manager.mempool)
return mempool_manager
@ -348,7 +350,9 @@ async def add_spendbundle(
mempool_manager: MempoolManager, sb: SpendBundle, sb_name: bytes32
) -> Tuple[Optional[uint64], MempoolInclusionStatus, Optional[Err]]:
npc_result = await mempool_manager.pre_validate_spendbundle(sb, None, sb_name)
return await mempool_manager.add_spend_bundle(sb, npc_result, sb_name, TEST_HEIGHT)
ret = await mempool_manager.add_spend_bundle(sb, npc_result, sb_name, TEST_HEIGHT)
invariant_check_mempool(mempool_manager.mempool)
return ret
async def generate_and_add_spendbundle(
@ -1033,6 +1037,7 @@ async def test_assert_before_expiration(
block_record = create_test_block_record(height=uint32(11), timestamp=uint64(10019))
await mempool_manager.new_peak(block_record, None)
invariant_check_mempool(mempool_manager.mempool)
still_in_pool = mempool_manager.get_spendbundle(bundle_name) == bundle
assert still_in_pool != expect_eviction
@ -1385,6 +1390,7 @@ async def test_coin_spending_different_ways_then_finding_it_spent_in_new_peak(ne
test_coin_records = {coin_id: CoinRecord(coin, uint32(0), TEST_HEIGHT, False, uint64(0))}
block_record = create_test_block_record(height=new_height)
await mempool_manager.new_peak(block_record, [coin_id])
invariant_check_mempool(mempool_manager.mempool)
# As the coin was a spend in all the mempool items we had, nothing should be left now
assert len(mempool_manager.mempool.get_items_by_coin_id(coin_id)) == 0
assert mempool_manager.mempool.size() == 0

17
tests/util/misc.py
View File

@ -23,6 +23,7 @@ from chia_rs import Coin
from typing_extensions import Protocol, final
import chia
from chia.full_node.mempool import Mempool
from chia.types.blockchain_format.sized_bytes import bytes32
from chia.types.condition_opcodes import ConditionOpcode
from chia.util.hash import std_hash
@ -407,3 +408,19 @@ def create_logger(file: TextIO = sys.stdout) -> logging.Logger:
logger.addHandler(hdlr=stream_handler)
return logger
def invariant_check_mempool(mempool: Mempool) -> None:
with mempool._db_conn:
cursor = mempool._db_conn.execute("SELECT SUM(cost) FROM tx")
val = cursor.fetchone()[0]
if val is None:
val = 0
assert mempool._total_cost == val
with mempool._db_conn:
cursor = mempool._db_conn.execute("SELECT SUM(fee) FROM tx")
val = cursor.fetchone()[0]
if val is None:
val = 0
assert mempool._total_fee == val