Chia-Network/chia-blockchain
1
0
Fork 0
mirror of https://github.com/Chia-Network/chia-blockchain.git synced 2024-09-11 10:56:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

streamable: Simplify and force correct usage (#10509)

Browse Source 
* streamable: Merge `strictdataclass` into `Streamable` class

* tests: Test not supported streamable types

* streamable: Reorder decorators

* streamable: Simplify streamable decorator and force correct usage/syntax

* streamable: Just move some stuff around in the file

* streamable: Improve syntax error messages

* mypy: Drop `type_checking.py` and `test_type_checking.py` from exclusion

* streamable: Use cached fields instead of `__annotations__`

This is now possible after merging `__post_init__` into `Streamable`

* Introduce `DefinitionError` as `StreamableError`

* `/t` -> `    `
This commit is contained in:
dustinface 2022-04-09 03:29:32 +02:00 committed by GitHub
parent 930a4b6825
commit a48fd43100
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
61 changed files with 549 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
benchmarks/streamable.py
View File

@ -17,14 +17,14 @@ from chia.util.streamable import Streamable, streamable
_version = 1
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class BenchmarkInner(Streamable):
a: str
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class BenchmarkMiddle(Streamable):
a: uint64
b: List[bytes32]
@ -33,8 +33,8 @@ class BenchmarkMiddle(Streamable):
e: BenchmarkInner
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class BenchmarkClass(Streamable):
a: Optional[BenchmarkMiddle]
b: Optional[BenchmarkMiddle]

6
chia/clvm/spend_sim.py
View File

@ -38,15 +38,15 @@ and is designed so that you could test with it and then swap in a real rpc clien
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SimFullBlock(Streamable):
transactions_generator: Optional[BlockGenerator]
height: uint32 # Note that height is not on a regular FullBlock
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SimBlockRecord(Streamable):
reward_claims_incorporated: List[Coin]
height: uint32
@ -69,8 +69,8 @@ class SimBlockRecord(Streamable):
)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SimStore(Streamable):
timestamp: uint64
block_height: uint32

2
chia/consensus/block_record.py
View File

@ -11,8 +11,8 @@ from chia.util.ints import uint8, uint32, uint64, uint128
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class BlockRecord(Streamable):
"""
This class is not included or hashed into the blockchain, but it is kept in memory as a more

2
chia/consensus/cost_calculator.py
View File

@ -6,8 +6,8 @@ from chia.util.ints import uint16, uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NPCResult(Streamable):
error: Optional[uint16]
conds: Optional[SpendBundleConditions]

2
chia/consensus/multiprocess_validation.py
View File

@ -35,8 +35,8 @@ from chia.util.streamable import Streamable, dataclass_from_dict, streamable
log = logging.getLogger(__name__)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PreValidationResult(Streamable):
error: Optional[uint16]
required_iters: Optional[uint64] # Iff error is None

2
chia/full_node/block_height_map.py
View File

@ -13,8 +13,8 @@ from chia.util.db_wrapper import DBWrapper2
log = logging.getLogger(__name__)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SesCache(Streamable):
content: List[Tuple[uint32, bytes]]

2
chia/full_node/full_node_store.py
View File

@ -29,8 +29,8 @@ from chia.util.streamable import Streamable, streamable
log = logging.getLogger(__name__)
@dataclasses.dataclass(frozen=True)
@streamable
@dataclasses.dataclass(frozen=True)
class FullNodeStorePeakResult(Streamable):
added_eos: Optional[EndOfSubSlotBundle]
new_signage_points: List[Tuple[uint8, SignagePoint]]

2
chia/full_node/signage_point.py
View File

@ -5,8 +5,8 @@ from chia.types.blockchain_format.vdf import VDFInfo, VDFProof
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SignagePoint(Streamable):
cc_vdf: Optional[VDFInfo]
cc_proof: Optional[VDFProof]

4
chia/plotting/manager.py
View File

@ -32,16 +32,16 @@ log = logging.getLogger(__name__)
CURRENT_VERSION: uint16 = uint16(0)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class CacheEntry(Streamable):
pool_public_key: Optional[G1Element]
pool_contract_puzzle_hash: Optional[bytes32]
plot_public_key: G1Element
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class DiskCache(Streamable):
version: uint16
data: List[Tuple[bytes32, CacheEntry]]

2
chia/pools/pool_config.py
View File

@ -25,8 +25,8 @@ pool_list:
log = logging.getLogger(__name__)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PoolWalletConfig(Streamable):
launcher_id: bytes32
pool_url: str

4
chia/pools/pool_wallet_info.py
View File

@ -38,8 +38,8 @@ LEAVING_POOL = PoolSingletonState.LEAVING_POOL
FARMING_TO_POOL = PoolSingletonState.FARMING_TO_POOL
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PoolState(Streamable):
"""
`PoolState` is a type that is serialized to the blockchain to track the state of the user's pool singleton
@ -97,8 +97,8 @@ def create_pool_state(
return ps
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PoolWalletInfo(Streamable):
"""
Internal Pool Wallet state, not destined for the blockchain. This can be completely derived with

10
chia/protocols/farmer_protocol.py
View File

@ -15,8 +15,8 @@ Note: When changing this file, also change protocol_message_types.py, and the pr
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewSignagePoint(Streamable):
challenge_hash: bytes32
challenge_chain_sp: bytes32
@ -26,8 +26,8 @@ class NewSignagePoint(Streamable):
signage_point_index: uint8
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class DeclareProofOfSpace(Streamable):
challenge_hash: bytes32
challenge_chain_sp: bytes32
@ -41,16 +41,16 @@ class DeclareProofOfSpace(Streamable):
pool_signature: Optional[G2Element]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestSignedValues(Streamable):
quality_string: bytes32
foliage_block_data_hash: bytes32
foliage_transaction_block_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class FarmingInfo(Streamable):
challenge_hash: bytes32
sp_hash: bytes32
@ -60,8 +60,8 @@ class FarmingInfo(Streamable):
total_plots: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SignedValues(Streamable):
quality_string: bytes32
foliage_block_data_signature: G2Element

50
chia/protocols/full_node_protocol.py
View File

@ -18,8 +18,8 @@ Note: When changing this file, also change protocol_message_types.py, and the pr
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewPeak(Streamable):
header_hash: bytes32
height: uint32
@ -28,102 +28,102 @@ class NewPeak(Streamable):
unfinished_reward_block_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewTransaction(Streamable):
transaction_id: bytes32
cost: uint64
fees: uint64
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestTransaction(Streamable):
transaction_id: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondTransaction(Streamable):
transaction: SpendBundle
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestProofOfWeight(Streamable):
total_number_of_blocks: uint32
tip: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondProofOfWeight(Streamable):
wp: WeightProof
tip: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestBlock(Streamable):
height: uint32
include_transaction_block: bool
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectBlock(Streamable):
height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestBlocks(Streamable):
start_height: uint32
end_height: uint32
include_transaction_block: bool
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondBlocks(Streamable):
start_height: uint32
end_height: uint32
blocks: List[FullBlock]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectBlocks(Streamable):
start_height: uint32
end_height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondBlock(Streamable):
block: FullBlock
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewUnfinishedBlock(Streamable):
unfinished_reward_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestUnfinishedBlock(Streamable):
unfinished_reward_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondUnfinishedBlock(Streamable):
unfinished_block: UnfinishedBlock
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewSignagePointOrEndOfSubSlot(Streamable):
prev_challenge_hash: Optional[bytes32]
challenge_hash: bytes32
@ -131,16 +131,16 @@ class NewSignagePointOrEndOfSubSlot(Streamable):
last_rc_infusion: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestSignagePointOrEndOfSubSlot(Streamable):
challenge_hash: bytes32
index_from_challenge: uint8
last_rc_infusion: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondSignagePoint(Streamable):
index_from_challenge: uint8
challenge_chain_vdf: VDFInfo
@ -149,20 +149,20 @@ class RespondSignagePoint(Streamable):
reward_chain_proof: VDFProof
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondEndOfSubSlot(Streamable):
end_of_slot_bundle: EndOfSubSlotBundle
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestMempoolTransactions(Streamable):
filter: bytes
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewCompactVDF(Streamable):
height: uint32
header_hash: bytes32
@ -170,8 +170,8 @@ class NewCompactVDF(Streamable):
vdf_info: VDFInfo
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestCompactVDF(Streamable):
height: uint32
header_hash: bytes32
@ -179,8 +179,8 @@ class RequestCompactVDF(Streamable):
vdf_info: VDFInfo
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondCompactVDF(Streamable):
height: uint32
header_hash: bytes32
@ -189,15 +189,15 @@ class RespondCompactVDF(Streamable):
vdf_proof: VDFProof
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestPeers(Streamable):
"""
Return full list of peers
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondPeers(Streamable):
peer_list: List[TimestampedPeerInfo]

32
chia/protocols/harvester_protocol.py
View File

@ -14,23 +14,23 @@ Note: When changing this file, also change protocol_message_types.py, and the pr
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PoolDifficulty(Streamable):
difficulty: uint64
sub_slot_iters: uint64
pool_contract_puzzle_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class HarvesterHandshake(Streamable):
farmer_public_keys: List[G1Element]
pool_public_keys: List[G1Element]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewSignagePointHarvester(Streamable):
challenge_hash: bytes32
difficulty: uint64
@ -40,8 +40,8 @@ class NewSignagePointHarvester(Streamable):
pool_difficulties: List[PoolDifficulty]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewProofOfSpace(Streamable):
challenge_hash: bytes32
sp_hash: bytes32
@ -50,8 +50,8 @@ class NewProofOfSpace(Streamable):
signage_point_index: uint8
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestSignatures(Streamable):
plot_identifier: str
challenge_hash: bytes32
@ -59,8 +59,8 @@ class RequestSignatures(Streamable):
messages: List[bytes32]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondSignatures(Streamable):
plot_identifier: str
challenge_hash: bytes32
@ -70,8 +70,8 @@ class RespondSignatures(Streamable):
message_signatures: List[Tuple[bytes32, G2Element]]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class Plot(Streamable):
filename: str
size: uint8
@ -83,30 +83,30 @@ class Plot(Streamable):
time_modified: uint64
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestPlots(Streamable):
pass
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondPlots(Streamable):
plots: List[Plot]
failed_to_open_filenames: List[str]
no_key_filenames: List[str]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncIdentifier(Streamable):
timestamp: uint64
sync_id: uint64
message_id: uint64
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncStart(Streamable):
identifier: PlotSyncIdentifier
initial: bool
@ -120,8 +120,8 @@ class PlotSyncStart(Streamable):
)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncPathList(Streamable):
identifier: PlotSyncIdentifier
data: List[str]
@ -131,8 +131,8 @@ class PlotSyncPathList(Streamable):
return f"PlotSyncPathList: identifier {self.identifier}, count {len(self.data)}, final {self.final}"
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncPlotList(Streamable):
identifier: PlotSyncIdentifier
data: List[Plot]
@ -142,8 +142,8 @@ class PlotSyncPlotList(Streamable):
return f"PlotSyncPlotList: identifier {self.identifier}, count {len(self.data)}, final {self.final}"
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncDone(Streamable):
identifier: PlotSyncIdentifier
duration: uint64
@ -152,8 +152,8 @@ class PlotSyncDone(Streamable):
return f"PlotSyncDone: identifier {self.identifier}, duration {self.duration}"
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncError(Streamable):
code: int16
message: str
@ -163,8 +163,8 @@ class PlotSyncError(Streamable):
return f"PlotSyncError: code {self.code}, count {self.message}, expected_identifier {self.expected_identifier}"
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PlotSyncResponse(Streamable):
identifier: PlotSyncIdentifier
message_type: int16

4
chia/protocols/introducer_protocol.py
View File

@ -10,15 +10,15 @@ Note: When changing this file, also change protocol_message_types.py, and the pr
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestPeersIntroducer(Streamable):
"""
Return full list of peers
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondPeersIntroducer(Streamable):
peer_list: List[TimestampedPeerInfo]

26
chia/protocols/pool_protocol.py
View File

@ -33,8 +33,8 @@ class PoolErrorCode(Enum):
# Used to verify GET /farmer and GET /login
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class AuthenticationPayload(Streamable):
method_name: str
launcher_id: bytes32
@ -43,8 +43,8 @@ class AuthenticationPayload(Streamable):
# GET /pool_info
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class GetPoolInfoResponse(Streamable):
name: str
logo_url: str
@ -60,8 +60,8 @@ class GetPoolInfoResponse(Streamable):
# POST /partial
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PostPartialPayload(Streamable):
launcher_id: bytes32
authentication_token: uint64
@ -71,16 +71,16 @@ class PostPartialPayload(Streamable):
harvester_id: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PostPartialRequest(Streamable):
payload: PostPartialPayload
aggregate_signature: G2Element
# Response in success case
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PostPartialResponse(Streamable):
new_difficulty: uint64
@ -89,8 +89,8 @@ class PostPartialResponse(Streamable):
# Response in success case
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class GetFarmerResponse(Streamable):
authentication_public_key: G1Element
payout_instructions: str
@ -101,8 +101,8 @@ class GetFarmerResponse(Streamable):
# POST /farmer
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PostFarmerPayload(Streamable):
launcher_id: bytes32
authentication_token: uint64
@ -111,16 +111,16 @@ class PostFarmerPayload(Streamable):
suggested_difficulty: Optional[uint64]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PostFarmerRequest(Streamable):
payload: PostFarmerPayload
signature: G2Element
# Response in success case
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PostFarmerResponse(Streamable):
welcome_message: str
@ -128,8 +128,8 @@ class PostFarmerResponse(Streamable):
# PUT /farmer
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PutFarmerPayload(Streamable):
launcher_id: bytes32
authentication_token: uint64
@ -138,16 +138,16 @@ class PutFarmerPayload(Streamable):
suggested_difficulty: Optional[uint64]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PutFarmerRequest(Streamable):
payload: PutFarmerPayload
signature: G2Element
# Response in success case
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PutFarmerResponse(Streamable):
authentication_public_key: Optional[bool]
payout_instructions: Optional[bool]
@ -158,8 +158,8 @@ class PutFarmerResponse(Streamable):
# Response in error case for all endpoints of the pool protocol
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ErrorResponse(Streamable):
error_code: uint16
error_message: Optional[str]

2
chia/protocols/shared_protocol.py
View File

@ -19,8 +19,8 @@ class Capability(IntEnum):
BASE = 1 # Base capability just means it supports the chia protocol at mainnet
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class Handshake(Streamable):
network_id: str
protocol_version: str

14
chia/protocols/timelord_protocol.py
View File

@ -16,8 +16,8 @@ Note: When changing this file, also change protocol_message_types.py, and the pr
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewPeakTimelord(Streamable):
reward_chain_block: RewardChainBlock
difficulty: uint64
@ -31,8 +31,8 @@ class NewPeakTimelord(Streamable):
passes_ses_height_but_not_yet_included: bool
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewUnfinishedBlockTimelord(Streamable):
reward_chain_block: RewardChainBlockUnfinished # Reward chain trunk data
difficulty: uint64
@ -44,8 +44,8 @@ class NewUnfinishedBlockTimelord(Streamable):
rc_prev: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewInfusionPointVDF(Streamable):
unfinished_reward_hash: bytes32
challenge_chain_ip_vdf: VDFInfo
@ -56,8 +56,8 @@ class NewInfusionPointVDF(Streamable):
infused_challenge_chain_ip_proof: Optional[VDFProof]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewSignagePointVDF(Streamable):
index_from_challenge: uint8
challenge_chain_sp_vdf: VDFInfo
@ -66,14 +66,14 @@ class NewSignagePointVDF(Streamable):
reward_chain_sp_proof: VDFProof
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewEndOfSubSlotVDF(Streamable):
end_of_sub_slot_bundle: EndOfSubSlotBundle
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestCompactProofOfTime(Streamable):
new_proof_of_time: VDFInfo
header_hash: bytes32
@ -81,8 +81,8 @@ class RequestCompactProofOfTime(Streamable):
field_vdf: uint8
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondCompactProofOfTime(Streamable):
vdf_info: VDFInfo
vdf_proof: VDFProof

58
chia/protocols/wallet_protocol.py
View File

@ -15,15 +15,15 @@ Note: When changing this file, also change protocol_message_types.py, and the pr
"""
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestPuzzleSolution(Streamable):
coin_name: bytes32
height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PuzzleSolutionResponse(Streamable):
coin_name: bytes32
height: uint32
@ -31,35 +31,35 @@ class PuzzleSolutionResponse(Streamable):
solution: Program
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondPuzzleSolution(Streamable):
response: PuzzleSolutionResponse
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectPuzzleSolution(Streamable):
coin_name: bytes32
height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SendTransaction(Streamable):
transaction: SpendBundle
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TransactionAck(Streamable):
txid: bytes32
status: uint8 # MempoolInclusionStatus
error: Optional[str]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class NewPeakWallet(Streamable):
header_hash: bytes32
height: uint32
@ -67,34 +67,34 @@ class NewPeakWallet(Streamable):
fork_point_with_previous_peak: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestBlockHeader(Streamable):
height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondBlockHeader(Streamable):
header_block: HeaderBlock
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectHeaderRequest(Streamable):
height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestRemovals(Streamable):
height: uint32
header_hash: bytes32
coin_names: Optional[List[bytes32]]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondRemovals(Streamable):
height: uint32
header_hash: bytes32
@ -102,23 +102,23 @@ class RespondRemovals(Streamable):
proofs: Optional[List[Tuple[bytes32, bytes]]]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectRemovalsRequest(Streamable):
height: uint32
header_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestAdditions(Streamable):
height: uint32
header_hash: Optional[bytes32]
puzzle_hashes: Optional[List[bytes32]]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondAdditions(Streamable):
height: uint32
header_hash: bytes32
@ -126,75 +126,75 @@ class RespondAdditions(Streamable):
proofs: Optional[List[Tuple[bytes32, bytes, Optional[bytes]]]]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectAdditionsRequest(Streamable):
height: uint32
header_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestHeaderBlocks(Streamable):
start_height: uint32
end_height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RejectHeaderBlocks(Streamable):
start_height: uint32
end_height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondHeaderBlocks(Streamable):
start_height: uint32
end_height: uint32
header_blocks: List[HeaderBlock]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class CoinState(Streamable):
coin: Coin
spent_height: Optional[uint32]
created_height: Optional[uint32]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RegisterForPhUpdates(Streamable):
puzzle_hashes: List[bytes32]
min_height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondToPhUpdates(Streamable):
puzzle_hashes: List[bytes32]
min_height: uint32
coin_states: List[CoinState]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RegisterForCoinUpdates(Streamable):
coin_ids: List[bytes32]
min_height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondToCoinUpdates(Streamable):
coin_ids: List[bytes32]
min_height: uint32
coin_states: List[CoinState]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class CoinStateUpdate(Streamable):
height: uint32
fork_height: uint32
@ -202,27 +202,27 @@ class CoinStateUpdate(Streamable):
items: List[CoinState]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestChildren(Streamable):
coin_name: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondChildren(Streamable):
coin_states: List[CoinState]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RequestSESInfo(Streamable):
start_height: uint32
end_height: uint32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RespondSESInfo(Streamable):
reward_chain_hash: List[bytes32]
heights: List[List[uint32]]

2
chia/seeder/peer_record.py
View File

@ -6,8 +6,8 @@ from chia.util.ints import uint32, uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PeerRecord(Streamable):
peer_id: str
ip_address: str

2
chia/server/address_manager_store.py
View File

@ -21,8 +21,8 @@ from typing import Any, Dict, List, Optional, Tuple
log = logging.getLogger(__name__)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PeerDataSerialization(Streamable):
"""
Serializable property bag for the peer data that was previously stored in sqlite.

2
chia/server/outbound_message.py
View File

@ -31,8 +31,8 @@ class Delivery(IntEnum):
SPECIFIC = 6
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class Message(Streamable):
type: uint8 # one of ProtocolMessageTypes
# message id

4
chia/simulator/simulator_protocol.py
View File

@ -5,14 +5,14 @@ from chia.util.ints import uint32
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class FarmNewBlockProtocol(Streamable):
puzzle_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ReorgProtocol(Streamable):
old_index: uint32
new_index: uint32

2
chia/timelord/timelord.py
View File

@ -41,8 +41,8 @@ from chia.util.streamable import Streamable, streamable
log = logging.getLogger(__name__)
@dataclasses.dataclass(frozen=True)
@streamable
@dataclasses.dataclass(frozen=True)
class BlueboxProcessData(Streamable):
challenge: bytes32
size_bits: uint16

2
chia/types/blockchain_format/classgroup.py
View File

@ -5,8 +5,8 @@ from chia.types.blockchain_format.sized_bytes import bytes100
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ClassgroupElement(Streamable):
"""
Represents a classgroup element (a,b,c) where a, b, and c are 512 bit signed integers. However this is using

2
chia/types/blockchain_format/coin.py
View File

@ -9,8 +9,8 @@ from chia.util.ints import uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class Coin(Streamable):
"""
This structure is used in the body for the reward and fees genesis coins.

8
chia/types/blockchain_format/foliage.py
View File

@ -10,8 +10,8 @@ from chia.util.ints import uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TransactionsInfo(Streamable):
# Information that goes along with each transaction block
generator_root: bytes32 # sha256 of the block generator in this block
@ -22,8 +22,8 @@ class TransactionsInfo(Streamable):
reward_claims_incorporated: List[Coin] # These can be in any order
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class FoliageTransactionBlock(Streamable):
# Information that goes along with each transaction block that is relevant for light clients
prev_transaction_block_hash: bytes32
@ -34,8 +34,8 @@ class FoliageTransactionBlock(Streamable):
transactions_info_hash: bytes32
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class FoliageBlockData(Streamable):
# Part of the block that is signed by the plot key
unfinished_reward_block_hash: bytes32
@ -45,8 +45,8 @@ class FoliageBlockData(Streamable):
extension_data: bytes32 # Used for future updates. Can be any 32 byte value initially
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class Foliage(Streamable):
# The entire foliage block, containing signature and the unsigned back pointer
# The hash of this is the "header hash". Note that for unfinished blocks, the prev_block_hash

2
chia/types/blockchain_format/pool_target.py
View File

@ -5,8 +5,8 @@ from chia.util.ints import uint32
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PoolTarget(Streamable):
puzzle_hash: bytes32
max_height: uint32 # A max height of 0 means it is valid forever

2
chia/types/blockchain_format/proof_of_space.py
View File

@ -15,8 +15,8 @@ from chia.util.streamable import Streamable, streamable
log = logging.getLogger(__name__)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ProofOfSpace(Streamable):
challenge: bytes32
pool_public_key: Optional[G1Element] # Only one of these two should be present

4
chia/types/blockchain_format/reward_chain_block.py
View File

@ -10,8 +10,8 @@ from chia.util.ints import uint8, uint32, uint128
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RewardChainBlockUnfinished(Streamable):
total_iters: uint128
signage_point_index: uint8
@ -23,8 +23,8 @@ class RewardChainBlockUnfinished(Streamable):
reward_chain_sp_signature: G2Element
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RewardChainBlock(Streamable):
weight: uint128
height: uint32

10
chia/types/blockchain_format/slots.py
View File

@ -10,8 +10,8 @@ from chia.util.ints import uint8, uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ChallengeBlockInfo(Streamable): # The hash of this is used as the challenge_hash for the ICC VDF
proof_of_space: ProofOfSpace
challenge_chain_sp_vdf: Optional[VDFInfo] # Only present if not the first sp
@ -19,8 +19,8 @@ class ChallengeBlockInfo(Streamable): # The hash of this is used as the challen
challenge_chain_ip_vdf: VDFInfo
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ChallengeChainSubSlot(Streamable):
challenge_chain_end_of_slot_vdf: VDFInfo
infused_challenge_chain_sub_slot_hash: Optional[bytes32] # Only at the end of a slot
@ -29,14 +29,14 @@ class ChallengeChainSubSlot(Streamable):
new_difficulty: Optional[uint64] # Only at the end of epoch, sub-epoch, and slot
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class InfusedChallengeChainSubSlot(Streamable):
infused_challenge_chain_end_of_slot_vdf: VDFInfo
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RewardChainSubSlot(Streamable):
end_of_slot_vdf: VDFInfo
challenge_chain_sub_slot_hash: bytes32
@ -44,8 +44,8 @@ class RewardChainSubSlot(Streamable):
deficit: uint8 # 16 or less. usually zero
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SubSlotProofs(Streamable):
challenge_chain_slot_proof: VDFProof
infused_challenge_chain_slot_proof: Optional[VDFProof]

2
chia/types/blockchain_format/sub_epoch_summary.py
View File

@ -6,8 +6,8 @@ from chia.util.ints import uint8, uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SubEpochSummary(Streamable):
prev_subepoch_summary_hash: bytes32
reward_chain_hash: bytes32 # hash of reward chain at end of last segment

4
chia/types/blockchain_format/vdf.py
View File

@ -44,16 +44,16 @@ def verify_vdf(
)
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class VDFInfo(Streamable):
challenge: bytes32 # Used to generate the discriminant (VDF group)
number_of_iterations: uint64
output: ClassgroupElement
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class VDFProof(Streamable):
witness_type: uint8
witness: bytes

2
chia/types/coin_record.py
View File

@ -8,8 +8,8 @@ from chia.util.ints import uint32, uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class CoinRecord(Streamable):
"""
These are values that correspond to a CoinName that are used

2
chia/types/coin_spend.py
View File

@ -7,8 +7,8 @@ from chia.util.chain_utils import additions_for_solution, fee_for_solution
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class CoinSpend(Streamable):
"""
This is a rather disparate data structure that validates coin transfers. It's generally populated

2
chia/types/condition_with_args.py
View File

@ -5,8 +5,8 @@ from chia.types.condition_opcodes import ConditionOpcode
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ConditionWithArgs(Streamable):
"""
This structure is used to store parsed CLVM conditions

2
chia/types/end_of_slot_bundle.py
View File

@ -10,8 +10,8 @@ from chia.types.blockchain_format.slots import (
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class EndOfSubSlotBundle(Streamable):
challenge_chain: ChallengeChainSubSlot
infused_challenge_chain: Optional[InfusedChallengeChainSubSlot]

2
chia/types/full_block.py
View File

@ -11,8 +11,8 @@ from chia.util.ints import uint32
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class FullBlock(Streamable):
# All the information required to validate a block
finished_sub_slots: List[EndOfSubSlotBundle] # If first sb

2
chia/types/generator_types.py
View File

@ -21,8 +21,8 @@ class CompressorArg:
end: int
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class BlockGenerator(Streamable):
program: SerializedProgram
generator_refs: List[SerializedProgram]

2
chia/types/header_block.py
View File

@ -8,8 +8,8 @@ from chia.types.end_of_slot_bundle import EndOfSubSlotBundle
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class HeaderBlock(Streamable):
# Same as a FullBlock but without TransactionInfo and Generator (but with filter), used by light clients
finished_sub_slots: List[EndOfSubSlotBundle] # If first sb

2
chia/types/mempool_item.py
View File

@ -10,8 +10,8 @@ from chia.util.ints import uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class MempoolItem(Streamable):
spend_bundle: SpendBundle
fee: uint64

4
chia/types/peer_info.py
View File

@ -6,8 +6,8 @@ from chia.util.ints import uint16, uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class PeerInfo(Streamable):
host: str
port: uint16
@ -59,8 +59,8 @@ class PeerInfo(Streamable):
return group
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TimestampedPeerInfo(Streamable):
host: str
port: uint16

2
chia/types/spend_bundle.py
View File

@ -15,8 +15,8 @@ from chia.wallet.util.debug_spend_bundle import debug_spend_bundle
from .coin_spend import CoinSpend
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SpendBundle(Streamable):
"""
This is a list of coins being spent along with their solution programs, and a single

4
chia/types/spend_bundle_conditions.py
View File

@ -8,8 +8,8 @@ from chia.util.streamable import Streamable, streamable
# the Spend and SpendBundleConditions classes are mirrors of native types, returned by
# run_generator
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class Spend(Streamable):
coin_id: bytes32
puzzle_hash: bytes32
@ -19,8 +19,8 @@ class Spend(Streamable):
agg_sig_me: List[Tuple[bytes48, bytes]]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SpendBundleConditions(Streamable):
spends: List[Spend]
reserve_fee: uint64

2
chia/types/unfinished_block.py
View File

@ -10,8 +10,8 @@ from chia.util.ints import uint32
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class UnfinishedBlock(Streamable):
# Full block, without the final VDFs
finished_sub_slots: List[EndOfSubSlotBundle] # If first sb

2
chia/types/unfinished_header_block.py
View File

@ -8,8 +8,8 @@ from chia.types.end_of_slot_bundle import EndOfSubSlotBundle
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class UnfinishedHeaderBlock(Streamable):
# Same as a FullBlock but without TransactionInfo and Generator, used by light clients
finished_sub_slots: List[EndOfSubSlotBundle] # If first sb

14
chia/types/weight_proof.py
View File

@ -11,8 +11,8 @@ from chia.util.ints import uint8, uint32, uint64, uint128
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SubEpochData(Streamable):
reward_chain_hash: bytes32
num_blocks_overflow: uint8
@ -31,8 +31,8 @@ class SubEpochData(Streamable):
# total number of challenge blocks == total number of reward chain blocks
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SubSlotData(Streamable):
# if infused
proof_of_space: Optional[ProofOfSpace]
@ -65,37 +65,37 @@ class SubSlotData(Streamable):
return False
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class SubEpochChallengeSegment(Streamable):
sub_epoch_n: uint32
sub_slots: List[SubSlotData]
rc_slot_end_info: Optional[VDFInfo] # in first segment of each sub_epoch
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
# this is used only for serialization to database
class SubEpochSegments(Streamable):
challenge_segments: List[SubEpochChallengeSegment]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
# this is used only for serialization to database
class RecentChainData(Streamable):
recent_chain_data: List[HeaderBlock]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class ProofBlockHeader(Streamable):
finished_sub_slots: List[EndOfSubSlotBundle]
reward_chain_block: RewardChainBlock
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class WeightProof(Streamable):
sub_epochs: List[SubEpochData]
sub_epoch_segments: List[SubEpochChallengeSegment] # sampled sub epoch

264
chia/util/streamable.py
View File

@ -5,7 +5,7 @@ import io
import pprint
import sys
from enum import Enum
from typing import Any, BinaryIO, Dict, get_type_hints, List, Tuple, Type, TypeVar, Callable, Optional, Iterator
from typing import Any, BinaryIO, Dict, get_type_hints, List, Tuple, Type, TypeVar, Union, Callable, Optional, Iterator
from blspy import G1Element, G2Element, PrivateKey
from typing_extensions import Literal
@ -14,20 +14,31 @@ from chia.types.blockchain_format.sized_bytes import bytes32
from chia.util.byte_types import hexstr_to_bytes
from chia.util.hash import std_hash
from chia.util.ints import int64, int512, uint32, uint64, uint128
from chia.util.type_checking import is_type_List, is_type_SpecificOptional, is_type_Tuple, strictdataclass
if sys.version_info < (3, 8):
def get_args(t: Type[Any]) -> Tuple[Any, ...]:
return getattr(t, "__args__", ())
def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
return getattr(t, "__origin__", None)
else:
from typing import get_args
from typing import get_args, get_origin
pp = pprint.PrettyPrinter(indent=1, width=120, compact=True)
class StreamableError(Exception):
pass
class DefinitionError(StreamableError):
pass
# TODO: Remove hack, this allows streaming these objects from binary
size_hints = {
"PrivateKey": PrivateKey.PRIVATE_KEY_SIZE,
@ -48,6 +59,27 @@ big_ints = [uint64, int64, uint128, int512]
_T_Streamable = TypeVar("_T_Streamable", bound="Streamable")
# Caches to store the fields and (de)serialization methods for all available streamable classes.
FIELDS_FOR_STREAMABLE_CLASS = {}
STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS = {}
PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS = {}
def is_type_List(f_type: Type) -> bool:
return get_origin(f_type) == list or f_type == list
def is_type_SpecificOptional(f_type) -> bool:
"""
Returns true for types such as Optional[T], but not Optional, or T.
"""
return get_origin(f_type) == Union and get_args(f_type)[1]() is None
def is_type_Tuple(f_type: Type) -> bool:
return get_origin(f_type) == tuple or f_type == tuple
def dataclass_from_dict(klass, d):
"""
Converts a dictionary based on a dataclass, into an instance of that dataclass.
@ -124,81 +156,6 @@ def recurse_jsonify(d):
return d
STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS = {}
PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS = {}
FIELDS_FOR_STREAMABLE_CLASS = {}
def streamable(cls: Any):
"""
This is a decorator for class definitions. It applies the strictdataclass decorator,
which checks all types at construction. It also defines a simple serialization format,
and adds parse, from bytes, stream, and __bytes__ methods.
The primitives are:
* Sized ints serialized in big endian format, e.g. uint64
* Sized bytes serialized in big endian format, e.g. bytes32
* BLS public keys serialized in bls format (48 bytes)
* BLS signatures serialized in bls format (96 bytes)
* bool serialized into 1 byte (0x01 or 0x00)
* bytes serialized as a 4 byte size prefix and then the bytes.
* ConditionOpcode is serialized as a 1 byte value.
* str serialized as a 4 byte size prefix and then the utf-8 representation in bytes.
An item is one of:
* primitive
* Tuple[item1, .. itemx]
* List[item1, .. itemx]
* Optional[item]
* Custom item
A streamable must be a Tuple at the root level (although a dataclass is used here instead).
Iters are serialized in the following way:
1. A tuple of x items is serialized by appending the serialization of each item.
2. A List is serialized into a 4 byte size prefix (number of items) and the serialization of each item.
3. An Optional is serialized into a 1 byte prefix of 0x00 or 0x01, and if it's one, it's followed by the
serialization of the item.
4. A Custom item is serialized by calling the .parse method, passing in the stream of bytes into it. An example is
a CLVM program.
All of the constituents must have parse/from_bytes, and stream/__bytes__ and therefore
be of fixed size. For example, int cannot be a constituent since it is not a fixed size,
whereas uint32 can be.
Furthermore, a get_hash() member is added, which performs a serialization and a sha256.
This class is used for deterministic serialization and hashing, for consensus critical
objects such as the block header.
Make sure to use the Streamable class as a parent class when using the streamable decorator,
as it will allow linters to recognize the methods that are added by the decorator. Also,
use the @dataclass(frozen=True) decorator as well, for linters to recognize constructor
arguments.
"""
cls1 = strictdataclass(cls)
t = type(cls.__name__, (cls1, Streamable), {})
stream_functions = []
parse_functions = []
try:
hints = get_type_hints(t)
fields = {field.name: hints.get(field.name, field.type) for field in dataclasses.fields(t)}
except Exception:
fields = {}
FIELDS_FOR_STREAMABLE_CLASS[t] = fields
for _, f_type in fields.items():
stream_functions.append(cls.function_to_stream_one_item(f_type))
parse_functions.append(cls.function_to_parse_one_item(f_type))
STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS[t] = stream_functions
PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[t] = parse_functions
return t
def parse_bool(f: BinaryIO) -> bool:
bool_byte = f.read(1)
assert bool_byte is not None and len(bool_byte) == 1 # Checks for EOF
@ -298,7 +255,158 @@ def stream_str(item: Any, f: BinaryIO) -> None:
f.write(str_bytes)
def streamable(cls: Any):
"""
This decorator forces correct streamable protocol syntax/usage and populates the caches for types hints and
(de)serialization methods for all members of the class. The correct usage is:
@streamable
@dataclass(frozen=True)
class Example(Streamable):
...
The order how the decorator are applied and the inheritance from Streamable are forced. The explicit inheritance is
required because mypy doesn't analyse the type returned by decorators, so we can't just inherit from inside the
decorator. The dataclass decorator is required to fetch type hints, let mypy validate constructor calls and restrict
direct modification of objects by `frozen=True`.
"""
correct_usage_string: str = (
"Correct usage is:\n\n@streamable\n@dataclass(frozen=True)\nclass Example(Streamable):\n ..."
)
if not dataclasses.is_dataclass(cls):
raise DefinitionError(f"@dataclass(frozen=True) required first. {correct_usage_string}")
try:
object.__new__(cls)._streamable_test_if_dataclass_frozen_ = None
except dataclasses.FrozenInstanceError:
pass
else:
raise DefinitionError(f"dataclass needs to be frozen. {correct_usage_string}")
if not issubclass(cls, Streamable):
raise DefinitionError(f"Streamable inheritance required. {correct_usage_string}")
stream_functions = []
parse_functions = []
try:
hints = get_type_hints(cls)
fields = {field.name: hints.get(field.name, field.type) for field in dataclasses.fields(cls)}
except Exception:
fields = {}
FIELDS_FOR_STREAMABLE_CLASS[cls] = fields
for _, f_type in fields.items():
stream_functions.append(cls.function_to_stream_one_item(f_type))
parse_functions.append(cls.function_to_parse_one_item(f_type))
STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = stream_functions
PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = parse_functions
return cls
class Streamable:
"""
This class defines a simple serialization format, and adds methods to parse from/to bytes and json. It also
validates and parses all fields at construction in ´__post_init__` to make sure all fields have the correct type
and can be streamed/parsed properly.
The available primitives are:
* Sized ints serialized in big endian format, e.g. uint64
* Sized bytes serialized in big endian format, e.g. bytes32
* BLS public keys serialized in bls format (48 bytes)
* BLS signatures serialized in bls format (96 bytes)
* bool serialized into 1 byte (0x01 or 0x00)
* bytes serialized as a 4 byte size prefix and then the bytes.
* ConditionOpcode is serialized as a 1 byte value.
* str serialized as a 4 byte size prefix and then the utf-8 representation in bytes.
An item is one of:
* primitive
* Tuple[item1, .. itemx]
* List[item1, .. itemx]
* Optional[item]
* Custom item
A streamable must be a Tuple at the root level (although a dataclass is used here instead).
Iters are serialized in the following way:
1. A tuple of x items is serialized by appending the serialization of each item.
2. A List is serialized into a 4 byte size prefix (number of items) and the serialization of each item.
3. An Optional is serialized into a 1 byte prefix of 0x00 or 0x01, and if it's one, it's followed by the
serialization of the item.
4. A Custom item is serialized by calling the .parse method, passing in the stream of bytes into it. An example is
a CLVM program.
All of the constituents must have parse/from_bytes, and stream/__bytes__ and therefore
be of fixed size. For example, int cannot be a constituent since it is not a fixed size,
whereas uint32 can be.
Furthermore, a get_hash() member is added, which performs a serialization and a sha256.
This class is used for deterministic serialization and hashing, for consensus critical
objects such as the block header.
Make sure to use the streamable decorator when inheriting from the Streamable class to prepare the streaming caches.
"""
def post_init_parse(self, item: Any, f_name: str, f_type: Type) -> Any:
if is_type_List(f_type):
collected_list: List = []
inner_type: Type = get_args(f_type)[0]
# wjb assert inner_type != get_args(List)[0] # type: ignore
if not is_type_List(type(item)):
raise ValueError(f"Wrong type for {f_name}, need a list.")
for el in item:
collected_list.append(self.post_init_parse(el, f_name, inner_type))
return collected_list
if is_type_SpecificOptional(f_type):
if item is None:
return None
else:
inner_type: Type = get_args(f_type)[0] # type: ignore
return self.post_init_parse(item, f_name, inner_type)
if is_type_Tuple(f_type):
collected_list = []
if not is_type_Tuple(type(item)) and not is_type_List(type(item)):
raise ValueError(f"Wrong type for {f_name}, need a tuple.")
if len(item) != len(get_args(f_type)):
raise ValueError(f"Wrong number of elements in tuple {f_name}.")
for i in range(len(item)):
inner_type = get_args(f_type)[i]
tuple_item = item[i]
collected_list.append(self.post_init_parse(tuple_item, f_name, inner_type))
return tuple(collected_list)
if not isinstance(item, f_type):
try:
item = f_type(item)
except (TypeError, AttributeError, ValueError):
try:
item = f_type.from_bytes(item)
except Exception:
item = f_type.from_bytes(bytes(item))
if not isinstance(item, f_type):
raise ValueError(f"Wrong type for {f_name}")
return item
def __post_init__(self):
try:
fields = FIELDS_FOR_STREAMABLE_CLASS[type(self)]
except Exception:
fields = {}
data = self.__dict__
for (f_name, f_type) in fields.items():
if f_name not in data:
raise ValueError(f"Field {f_name} not present")
try:
if not isinstance(data[f_name], f_type):
object.__setattr__(self, f_name, self.post_init_parse(data[f_name], f_name, f_type))
except TypeError:
# Throws a TypeError because we cannot call isinstance for subscripted generics like Optional[int]
object.__setattr__(self, f_name, self.post_init_parse(data[f_name], f_name, f_type))
@classmethod
def function_to_parse_one_item(cls, f_type: Type) -> Callable[[BinaryIO], Any]:
"""

103
chia/util/type_checking.py
View File

@ -1,103 +0,0 @@
import dataclasses
import sys
from typing import Any, List, Optional, Tuple, Type, Union
if sys.version_info < (3, 8):
def get_args(t: Type[Any]) -> Tuple[Any, ...]:
return getattr(t, "__args__", ())
def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
return getattr(t, "__origin__", None)
else:
from typing import get_args, get_origin
def is_type_List(f_type: Type) -> bool:
return get_origin(f_type) == list or f_type == list
def is_type_SpecificOptional(f_type) -> bool:
"""
Returns true for types such as Optional[T], but not Optional, or T.
"""
return get_origin(f_type) == Union and get_args(f_type)[1]() is None
def is_type_Tuple(f_type: Type) -> bool:
return get_origin(f_type) == tuple or f_type == tuple
def strictdataclass(cls: Any):
class _Local:
"""
Dataclass where all fields must be type annotated, and type checking is performed
at initialization, even recursively through Lists. Non-annotated fields are ignored.
Also, for any fields which have a type with .from_bytes(bytes) or constructor(bytes),
bytes can be passed in and the type can be constructed.
"""
def parse_item(self, item: Any, f_name: str, f_type: Type) -> Any:
if is_type_List(f_type):
collected_list: List = []
inner_type: Type = get_args(f_type)[0]
# wjb assert inner_type != get_args(List)[0] # type: ignore
if not is_type_List(type(item)):
raise ValueError(f"Wrong type for {f_name}, need a list.")
for el in item:
collected_list.append(self.parse_item(el, f_name, inner_type))
return collected_list
if is_type_SpecificOptional(f_type):
if item is None:
return None
else:
inner_type: Type = get_args(f_type)[0] # type: ignore
return self.parse_item(item, f_name, inner_type)
if is_type_Tuple(f_type):
collected_list = []
if not is_type_Tuple(type(item)) and not is_type_List(type(item)):
raise ValueError(f"Wrong type for {f_name}, need a tuple.")
if len(item) != len(get_args(f_type)):
raise ValueError(f"Wrong number of elements in tuple {f_name}.")
for i in range(len(item)):
inner_type = get_args(f_type)[i]
tuple_item = item[i]
collected_list.append(self.parse_item(tuple_item, f_name, inner_type))
return tuple(collected_list)
if not isinstance(item, f_type):
try:
item = f_type(item)
except (TypeError, AttributeError, ValueError):
try:
item = f_type.from_bytes(item)
except Exception:
item = f_type.from_bytes(bytes(item))
if not isinstance(item, f_type):
raise ValueError(f"Wrong type for {f_name}")
return item
def __post_init__(self):
try:
fields = self.__annotations__ # pylint: disable=no-member
except Exception:
fields = {}
data = self.__dict__
for (f_name, f_type) in fields.items():
if f_name not in data:
raise ValueError(f"Field {f_name} not present")
try:
if not isinstance(data[f_name], f_type):
object.__setattr__(self, f_name, self.parse_item(data[f_name], f_name, f_type))
except TypeError:
# Throws a TypeError because we cannot call isinstance for subscripted generics like Optional[int]
object.__setattr__(self, f_name, self.parse_item(data[f_name], f_name, f_type))
class NoTypeChecking:
__no_type_check__ = True
cls1 = dataclasses.dataclass(cls, init=False, frozen=True) # type: ignore
if dataclasses.fields(cls1) == ():
return type(cls.__name__, (cls1, _Local, NoTypeChecking), {})
return type(cls.__name__, (cls1, _Local), {})

2
chia/wallet/block_record.py
View File

@ -6,8 +6,8 @@ from chia.types.header_block import HeaderBlock
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class HeaderBlockRecord(Streamable):
"""
These are values that are stored in the wallet database, corresponding to information

4
chia/wallet/cat_wallet/cat_info.py
View File

@ -7,8 +7,8 @@ from chia.wallet.lineage_proof import LineageProof
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class CATInfo(Streamable):
limitations_program_hash: bytes32
my_tail: Optional[Program] # this is the program
@ -16,8 +16,8 @@ class CATInfo(Streamable):
# We used to store all of the lineage proofs here but it was very slow to serialize for a lot of transactions
# so we moved it to CATLineageStore. We keep this around for migration purposes.
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class LegacyCATInfo(Streamable):
limitations_program_hash: bytes32
my_tail: Optional[Program] # this is the program

2
chia/wallet/did_wallet/did_info.py
View File

@ -9,8 +9,8 @@ from chia.types.blockchain_format.program import Program
from chia.types.blockchain_format.coin import Coin
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class DIDInfo(Streamable):
origin_coin: Optional[Coin] # Coin ID of this coin is our DID
backup_ids: List[bytes]

2
chia/wallet/lineage_proof.py
View File

@ -7,8 +7,8 @@ from chia.util.ints import uint64
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class LineageProof(Streamable):
parent_name: Optional[bytes32] = None
inner_puzzle_hash: Optional[bytes32] = None

2
chia/wallet/rl_wallet/rl_wallet.py
View File

@ -34,8 +34,8 @@ from chia.wallet.wallet_coin_record import WalletCoinRecord
from chia.wallet.wallet_info import WalletInfo
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class RLInfo(Streamable):
type: str
admin_pubkey: Optional[bytes]

2
chia/wallet/settings/settings_objects.py
View File

@ -3,8 +3,8 @@ from dataclasses import dataclass
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class BackupInitialized(Streamable):
"""
Stores user decision regarding import of backup info

2
chia/wallet/trade_record.py
View File

@ -9,8 +9,8 @@ from chia.wallet.trading.offer import Offer
from chia.wallet.trading.trade_status import TradeStatus
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TradeRecord(Streamable):
"""
Used for storing transaction data and status in wallets.

2
chia/wallet/transaction_record.py
View File

@ -12,8 +12,8 @@ from chia.util.streamable import Streamable, streamable
from chia.wallet.util.transaction_type import TransactionType
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TransactionRecord(Streamable):
"""
Used for storing transaction data and status in wallets.

4
chia/wallet/wallet_info.py
View File

@ -5,8 +5,8 @@ from chia.util.ints import uint8, uint32
from chia.util.streamable import Streamable, streamable
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class WalletInfo(Streamable):
"""
This object represents the wallet data as it is stored in DB.
@ -24,8 +24,8 @@ class WalletInfo(Streamable):
data: str
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class WalletInfoBackup(Streamable):
"""
Used for transforming list of WalletInfo objects into bytes.

2
mypy.ini
View File

File diff suppressed because one or more lines are too long

206
tests/core/util/test_streamable.py
View File

@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import List, Optional, Tuple
from typing import Dict, List, Optional, Tuple
import io
import pytest
@ -14,6 +14,7 @@ from chia.types.full_block import FullBlock
from chia.types.weight_proof import SubEpochChallengeSegment
from chia.util.ints import uint8, uint32, uint64
from chia.util.streamable import (
DefinitionError,
Streamable,
streamable,
parse_bool,
@ -25,13 +26,147 @@ from chia.util.streamable import (
parse_tuple,
parse_size_hints,
parse_str,
is_type_List,
is_type_SpecificOptional,
)
from tests.setup_nodes import test_constants
def test_basic():
def test_int_not_supported() -> None:
with raises(NotImplementedError):
@streamable
@dataclass(frozen=True)
class TestClassInt(Streamable):
a: int
def test_float_not_supported() -> None:
with raises(NotImplementedError):
@streamable
@dataclass(frozen=True)
class TestClassFloat(Streamable):
a: float
def test_dict_not_suppported() -> None:
with raises(NotImplementedError):
@streamable
@dataclass(frozen=True)
class TestClassDict(Streamable):
a: Dict[str, str]
def test_pure_dataclass_not_supported() -> None:
@dataclass(frozen=True)
class DataClassOnly:
a: uint8
with raises(NotImplementedError):
@streamable
@dataclass(frozen=True)
class TestClassDataclass(Streamable):
a: DataClassOnly
def test_plain_class_not_supported() -> None:
class PlainClass:
a: uint8
with raises(NotImplementedError):
@streamable
@dataclass(frozen=True)
class TestClassPlain(Streamable):
a: PlainClass
def test_basic_list():
a = [1, 2, 3]
assert is_type_List(type(a))
assert is_type_List(List)
assert is_type_List(List[int])
assert is_type_List(List[uint8])
assert is_type_List(list)
assert not is_type_List(Tuple)
assert not is_type_List(tuple)
assert not is_type_List(dict)
def test_not_lists():
assert not is_type_List(Dict)
def test_basic_optional():
assert is_type_SpecificOptional(Optional[int])
assert is_type_SpecificOptional(Optional[Optional[int]])
assert not is_type_SpecificOptional(List[int])
def test_StrictDataClass():
@streamable
@dataclass(frozen=True)
class TestClass1(Streamable):
a: uint8
b: str
good: TestClass1 = TestClass1(24, "!@12")
assert TestClass1.__name__ == "TestClass1"
assert good
assert good.a == 24
assert good.b == "!@12"
good2 = TestClass1(52, bytes([1, 2, 3]))
assert good2.b == str(bytes([1, 2, 3]))
def test_StrictDataClassBad():
@streamable
@dataclass(frozen=True)
class TestClass2(Streamable):
a: uint8
b = 0
assert TestClass2(25)
with raises(TypeError):
TestClass2(1, 2) # pylint: disable=too-many-function-args
def test_StrictDataClassLists():
@streamable
@dataclass(frozen=True)
class TestClass(Streamable):
a: List[uint8]
b: List[List[uint8]]
assert TestClass([1, 2, 3], [[uint8(200), uint8(25)], [uint8(25)]])
with raises(ValueError):
TestClass({"1": 1}, [[uint8(200), uint8(25)], [uint8(25)]])
with raises(ValueError):
TestClass([1, 2, 3], [uint8(200), uint8(25)])
def test_StrictDataClassOptional():
@streamable
@dataclass(frozen=True)
class TestClass(Streamable):
a: Optional[uint8]
b: Optional[uint8]
c: Optional[Optional[uint8]]
d: Optional[Optional[uint8]]
good = TestClass(12, None, 13, None)
assert good
def test_basic():
@streamable
@dataclass(frozen=True)
class TestClass(Streamable):
a: uint32
b: uint32
@ -48,8 +183,8 @@ def test_basic():
def test_variable_size():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClass2(Streamable):
a: uint32
b: uint32
@ -60,8 +195,8 @@ def test_variable_size():
with raises(NotImplementedError):
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClass3(Streamable):
a: int
@ -72,8 +207,8 @@ def test_json(bt):
assert FullBlock.from_json_dict(dict_block) == block
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class OptionalTestClass(Streamable):
a: Optional[str]
b: Optional[bool]
@ -99,13 +234,13 @@ def test_optional_json(a: Optional[str], b: Optional[bool], c: Optional[List[Opt
def test_recursive_json():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClass1(Streamable):
a: List[uint32]
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClass2(Streamable):
a: uint32
b: List[Optional[List[TestClass1]]]
@ -130,8 +265,8 @@ def test_ambiguous_deserialization_optionals():
with raises(AssertionError):
SubEpochChallengeSegment.from_bytes(b"\x00\x00\x00\x03\xff\xff\xff\xff")
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassOptional(Streamable):
a: Optional[uint8]
@ -144,8 +279,8 @@ def test_ambiguous_deserialization_optionals():
def test_ambiguous_deserialization_int():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassUint(Streamable):
a: uint32
@ -155,8 +290,8 @@ def test_ambiguous_deserialization_int():
def test_ambiguous_deserialization_list():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassList(Streamable):
a: List[uint8]
@ -166,8 +301,8 @@ def test_ambiguous_deserialization_list():
def test_ambiguous_deserialization_tuple():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassTuple(Streamable):
a: Tuple[uint8, str]
@ -177,8 +312,8 @@ def test_ambiguous_deserialization_tuple():
def test_ambiguous_deserialization_str():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassStr(Streamable):
a: str
@ -188,8 +323,8 @@ def test_ambiguous_deserialization_str():
def test_ambiguous_deserialization_bytes():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassBytes(Streamable):
a: bytes
@ -205,8 +340,8 @@ def test_ambiguous_deserialization_bytes():
def test_ambiguous_deserialization_bool():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassBool(Streamable):
a: bool
@ -219,8 +354,8 @@ def test_ambiguous_deserialization_bool():
def test_ambiguous_deserialization_program():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class TestClassProgram(Streamable):
a: Program
@ -233,8 +368,8 @@ def test_ambiguous_deserialization_program():
def test_streamable_empty():
@dataclass(frozen=True)
@streamable
@dataclass(frozen=True)
class A(Streamable):
pass
@ -414,3 +549,42 @@ def test_parse_str():
# EOF off by one
with raises(AssertionError):
parse_str(io.BytesIO(b"\x00\x00\x02\x01" + b"a" * 512))
def test_wrong_decorator_order():
with raises(DefinitionError):
@dataclass(frozen=True)
@streamable
class WrongDecoratorOrder(Streamable):
pass
def test_dataclass_not_frozen():
with raises(DefinitionError):
@streamable
@dataclass(frozen=False)
class DataclassNotFrozen(Streamable):
pass
def test_dataclass_missing():
with raises(DefinitionError):
@streamable
class DataclassMissing(Streamable):
pass
def test_streamable_inheritance_missing():
with raises(DefinitionError):
@streamable
@dataclass(frozen=True)
class StreamableInheritanceMissing:
pass

91
tests/core/util/test_type_checking.py
View File

@ -1,91 +0,0 @@
import unittest
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
from pytest import raises
from chia.util.ints import uint8
from chia.util.type_checking import is_type_List, is_type_SpecificOptional, strictdataclass
class TestIsTypeList(unittest.TestCase):
def test_basic_list(self):
a = [1, 2, 3]
assert is_type_List(type(a))
assert is_type_List(List)
assert is_type_List(List[int])
assert is_type_List(List[uint8])
assert is_type_List(list)
assert not is_type_List(Tuple)
assert not is_type_List(tuple)
assert not is_type_List(dict)
def test_not_lists(self):
assert not is_type_List(Dict)
class TestIsTypeSpecificOptional(unittest.TestCase):
def test_basic_optional(self):
assert is_type_SpecificOptional(Optional[int])
assert is_type_SpecificOptional(Optional[Optional[int]])
assert not is_type_SpecificOptional(List[int])
class TestStrictClass(unittest.TestCase):
def test_StrictDataClass(self):
@dataclass(frozen=True)
@strictdataclass
class TestClass1:
a: int
b: str
good: TestClass1 = TestClass1(24, "!@12")
assert TestClass1.__name__ == "TestClass1"
assert good
assert good.a == 24
assert good.b == "!@12"
good2 = TestClass1(52, bytes([1, 2, 3]))
assert good2.b == str(bytes([1, 2, 3]))
def test_StrictDataClassBad(self):
@dataclass(frozen=True)
@strictdataclass
class TestClass2:
a: int
b = 0
assert TestClass2(25)
with raises(TypeError):
TestClass2(1, 2) # pylint: disable=too-many-function-args
def test_StrictDataClassLists(self):
@dataclass(frozen=True)
@strictdataclass
class TestClass:
a: List[int]
b: List[List[uint8]]
assert TestClass([1, 2, 3], [[uint8(200), uint8(25)], [uint8(25)]])
with raises(ValueError):
TestClass({"1": 1}, [[uint8(200), uint8(25)], [uint8(25)]])
with raises(ValueError):
TestClass([1, 2, 3], [uint8(200), uint8(25)])
def test_StrictDataClassOptional(self):
@dataclass(frozen=True)
@strictdataclass
class TestClass:
a: Optional[int]
b: Optional[int]
c: Optional[Optional[int]]
d: Optional[Optional[int]]
good = TestClass(12, None, 13, None)
assert good
if __name__ == "__main__":
unittest.main()