optimize get_block_generator() (#10104)

* optimize get_block_generator()

* add a v1 compatible get_generator() to speed up get_block_generator() with v1 databases. Add test. Add error log in case generator_from_block() fails.

* speed up test_full_block_utils

benchmarks/block_store.py

@@ -8,9 +8,17 @@ import sys
 
 from chia.util.db_wrapper import DBWrapper
 from chia.util.ints import uint128, uint64, uint32, uint8
-from chia.types.blockchain_format.classgroup import ClassgroupElement
-from utils import rewards, rand_hash, setup_db, rand_g1, rand_g2, rand_bytes
-from chia.types.blockchain_format.vdf import VDFInfo, VDFProof
+from utils import (
+    rewards,
+    rand_hash,
+    setup_db,
+    rand_g1,
+    rand_g2,
+    rand_bytes,
+    rand_vdf,
+    rand_vdf_proof,
+    rand_class_group_element,
+)
 from chia.types.full_block import FullBlock
 from chia.consensus.block_record import BlockRecord
 from chia.types.blockchain_format.proof_of_space import ProofOfSpace
@@ -28,22 +36,6 @@ NUM_ITERS = 20000
 random.seed(123456789)
 
 
-def rand_class_group_element() -> ClassgroupElement:
-    return ClassgroupElement(rand_bytes(100))
-
-
-def rand_vdf() -> VDFInfo:
-    return VDFInfo(rand_hash(), uint64(random.randint(100000, 1000000000)), rand_class_group_element())
-
-
-def rand_vdf_proof() -> VDFProof:
-    return VDFProof(
-        uint8(1),  # witness_type
-        rand_hash(),  # witness
-        bool(random.randint(0, 1)),  # normalized_to_identity
-    )
-
-
 with open("clvm_generator.bin", "rb") as f:
     clvm_generator = f.read()
 

benchmarks/utils.py

@@ -1,8 +1,10 @@
 from chia.consensus.default_constants import DEFAULT_CONSTANTS
-from chia.util.ints import uint64, uint32
+from chia.util.ints import uint64, uint32, uint8
 from chia.consensus.coinbase import create_farmer_coin, create_pool_coin
+from chia.types.blockchain_format.classgroup import ClassgroupElement
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.types.blockchain_format.vdf import VDFInfo, VDFProof
 from chia.util.db_wrapper import DBWrapper
 from typing import Tuple
 from pathlib import Path
@@ -46,6 +48,24 @@ def rand_g2() -> G2Element:
     return AugSchemeMPL.sign(sk, b"foobar")
 
 
+def rand_class_group_element() -> ClassgroupElement:
+    # TODO: address hint errors and remove ignores
+    #       error: Argument 1 to "ClassgroupElement" has incompatible type "bytes"; expected "bytes100"  [arg-type]
+    return ClassgroupElement(rand_bytes(100))  # type: ignore[arg-type]
+
+
+def rand_vdf() -> VDFInfo:
+    return VDFInfo(rand_hash(), uint64(random.randint(100000, 1000000000)), rand_class_group_element())
+
+
+def rand_vdf_proof() -> VDFProof:
+    return VDFProof(
+        uint8(1),  # witness_type
+        rand_hash(),  # witness
+        bool(random.randint(0, 1)),  # normalized_to_identity
+    )
+
+
 async def setup_db(name: str, db_version: int) -> DBWrapper:
     db_filename = Path(name)
     try:

chia/consensus/blockchain.py

@@ -881,18 +881,22 @@ class Blockchain(BlockchainInterface):
         ):
             # We are not in a reorg, no need to look up alternate header hashes
             # (we can get them from height_to_hash)
-            for ref_height in block.transactions_generator_ref_list:
-                header_hash = self.height_to_hash(ref_height)
+            if self.block_store.db_wrapper.db_version == 2:
+                # in the v2 database, we can look up blocks by height directly
+                # (as long as we're in the main chain)
+                result = await self.block_store.get_generators_at(block.transactions_generator_ref_list)
+            else:
+                for ref_height in block.transactions_generator_ref_list:
+                    header_hash = self.height_to_hash(ref_height)
 
-                # if ref_height is invalid, this block should have failed with
-                # FUTURE_GENERATOR_REFS before getting here
-                assert header_hash is not None
+                    # if ref_height is invalid, this block should have failed with
+                    # FUTURE_GENERATOR_REFS before getting here
+                    assert header_hash is not None
 
-                ref_block = await self.block_store.get_full_block(header_hash)
-                assert ref_block is not None
-                if ref_block.transactions_generator is None:
-                    raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
-                result.append(ref_block.transactions_generator)
+                    ref_gen = await self.block_store.get_generator(header_hash)
+                    if ref_gen is None:
+                        raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
+                    result.append(ref_gen)
         else:
             # First tries to find the blocks in additional_blocks
             reorg_chain: Dict[uint32, FullBlock] = {}
@@ -933,15 +937,17 @@ class Blockchain(BlockchainInterface):
                 else:
                     if ref_height in additional_height_dict:
                         ref_block = additional_height_dict[ref_height]
+                        assert ref_block is not None
+                        if ref_block.transactions_generator is None:
+                            raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
+                        result.append(ref_block.transactions_generator)
                     else:
                         header_hash = self.height_to_hash(ref_height)
-                        # TODO: address hint error and remove ignore
-                        #       error: Argument 1 to "get_full_block" of "Blockchain" has incompatible type
-                        #       "Optional[bytes32]"; expected "bytes32"  [arg-type]
-                        ref_block = await self.get_full_block(header_hash)  # type: ignore[arg-type]
-                    assert ref_block is not None
-                    if ref_block.transactions_generator is None:
-                        raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
-                    result.append(ref_block.transactions_generator)
+                        if header_hash is None:
+                            raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
+                        gen = await self.block_store.get_generator(header_hash)
+                        if gen is None:
+                            raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
+                        result.append(gen)
         assert len(result) == len(ref_list)
         return BlockGenerator(block.transactions_generator, result, [])

chia/full_node/block_store.py

@@ -7,10 +7,13 @@ import zstd
 from chia.consensus.block_record import BlockRecord
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.full_block import FullBlock
+from chia.types.blockchain_format.program import SerializedProgram
 from chia.types.weight_proof import SubEpochChallengeSegment, SubEpochSegments
+from chia.util.errors import Err
 from chia.util.db_wrapper import DBWrapper
 from chia.util.ints import uint32
 from chia.util.lru_cache import LRUCache
+from chia.util.full_block_utils import generator_from_block
 
 log = logging.getLogger(__name__)
 
@@ -298,6 +301,64 @@ class BlockStore:
                 ret.append(self.maybe_decompress(row[0]))
             return ret
 
+    async def get_generator(self, header_hash: bytes32) -> Optional[SerializedProgram]:
+
+        cached = self.block_cache.get(header_hash)
+        if cached is not None:
+            log.debug(f"cache hit for block {header_hash.hex()}")
+            return cached.transactions_generator
+
+        formatted_str = "SELECT block, height from full_blocks WHERE header_hash=?"
+        async with self.db.execute(formatted_str, (self.maybe_to_hex(header_hash),)) as cursor:
+            row = await cursor.fetchone()
+            if row is None:
+                return None
+            if self.db_wrapper.db_version == 2:
+                block_bytes = zstd.decompress(row[0])
+            else:
+                block_bytes = row[0]
+
+            try:
+                return generator_from_block(block_bytes)
+            except Exception as e:
+                log.error(f"cheap parser failed for block at height {row[1]}: {e}")
+                # this is defensive, on the off-chance that
+                # generator_from_block() fails, fall back to the reliable
+                # definition of parsing a block
+                b = FullBlock.from_bytes(block_bytes)
+                return b.transactions_generator
+
+    async def get_generators_at(self, heights: List[uint32]) -> List[SerializedProgram]:
+        assert self.db_wrapper.db_version == 2
+
+        if len(heights) == 0:
+            return []
+
+        generators: Dict[uint32, SerializedProgram] = {}
+        heights_db = tuple(heights)
+        formatted_str = (
+            f"SELECT block, height from full_blocks "
+            f'WHERE in_main_chain=1 AND height in ({"?," * (len(heights_db) - 1)}?)'
+        )
+        async with self.db.execute(formatted_str, heights_db) as cursor:
+            async for row in cursor:
+                block_bytes = zstd.decompress(row[0])
+
+                try:
+                    gen = generator_from_block(block_bytes)
+                except Exception as e:
+                    log.error(f"cheap parser failed for block at height {row[1]}: {e}")
+                    # this is defensive, on the off-chance that
+                    # generator_from_block() fails, fall back to the reliable
+                    # definition of parsing a block
+                    b = FullBlock.from_bytes(block_bytes)
+                    gen = b.transactions_generator
+                if gen is None:
+                    raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR)
+                generators[uint32(row[1])] = gen
+
+        return [generators[h] for h in heights]
+
     async def get_block_records_by_hash(self, header_hashes: List[bytes32]):
         """
         Returns a list of Block Records, ordered by the same order in which header_hashes are passed in.

chia/util/full_block_utils.py

@@ -0,0 +1,208 @@
+from typing import Optional, Callable
+
+from chia.types.blockchain_format.program import SerializedProgram
+from clvm_rs import serialized_length
+from blspy import G1Element, G2Element
+
+
+def skip_list(buf: memoryview, skip_item: Callable[[memoryview], memoryview]) -> memoryview:
+    n = int.from_bytes(buf[:4], "big", signed=False)
+    buf = buf[4:]
+    for i in range(n):
+        buf = skip_item(buf)
+    return buf
+
+
+def skip_bytes(buf: memoryview) -> memoryview:
+    n = int.from_bytes(buf[:4], "big", signed=False)
+    buf = buf[4:]
+    assert n >= 0
+    return buf[n:]
+
+
+def skip_optional(buf: memoryview, skip_item: Callable[[memoryview], memoryview]) -> memoryview:
+
+    if buf[0] == 0:
+        return buf[1:]
+    assert buf[0] == 1
+    return skip_item(buf[1:])
+
+
+def skip_bytes32(buf: memoryview) -> memoryview:
+    return buf[32:]
+
+
+def skip_uint32(buf: memoryview) -> memoryview:
+    return buf[4:]
+
+
+def skip_uint64(buf: memoryview) -> memoryview:
+    return buf[8:]
+
+
+def skip_uint128(buf: memoryview) -> memoryview:
+    return buf[16:]
+
+
+def skip_uint8(buf: memoryview) -> memoryview:
+    return buf[1:]
+
+
+def skip_bool(buf: memoryview) -> memoryview:
+    assert buf[0] in [0, 1]
+    return buf[1:]
+
+
+# def skip_class_group_element(buf: memoryview) -> memoryview:
+#    return buf[100:]  # bytes100
+
+
+def skip_vdf_info(buf: memoryview) -> memoryview:
+    #    buf = skip_bytes32(buf)
+    #    buf = skip_uint64(buf)
+    #    return skip_class_group_element(buf)
+    return buf[32 + 8 + 100 :]
+
+
+def skip_vdf_proof(buf: memoryview) -> memoryview:
+    buf = skip_uint8(buf)  # witness_type
+    buf = skip_bytes(buf)  # witness
+    return skip_bool(buf)  # normalized_to_identity
+
+
+def skip_challenge_chain_sub_slot(buf: memoryview) -> memoryview:
+    buf = skip_vdf_info(buf)
+    buf = skip_optional(buf, skip_bytes32)  # infused challenge chain sub skit hash
+    buf = skip_optional(buf, skip_bytes32)  # subepoch_summary_hash
+    buf = skip_optional(buf, skip_uint64)  # new_sub_slot_iters
+    return skip_optional(buf, skip_uint64)  # new_difficulty
+
+
+def skip_infused_challenge_chain(buf: memoryview) -> memoryview:
+    return skip_vdf_info(buf)  # infused_challenge_chain_end_of_slot_vdf
+
+
+def skip_reward_chain_sub_slot(buf: memoryview) -> memoryview:
+    buf = skip_vdf_info(buf)  # end_of_slot_vdf
+    buf = skip_bytes32(buf)  # challenge_chain_sub_slot_hash
+    buf = skip_optional(buf, skip_bytes32)  # infused_challenge_chain_sub_slot_hash
+    return skip_uint8(buf)
+
+
+def skip_sub_slot_proofs(buf: memoryview) -> memoryview:
+    buf = skip_vdf_proof(buf)  # challenge_chain_slot_proof
+    buf = skip_optional(buf, skip_vdf_proof)  # infused_challenge_chain_slot_proof
+    return skip_vdf_proof(buf)  # reward_chain_slot_proof
+
+
+def skip_end_of_sub_slot_bundle(buf: memoryview) -> memoryview:
+    buf = skip_challenge_chain_sub_slot(buf)
+    buf = skip_optional(buf, skip_infused_challenge_chain)
+    buf = skip_reward_chain_sub_slot(buf)
+    return skip_sub_slot_proofs(buf)
+
+
+def skip_g1_element(buf: memoryview) -> memoryview:
+    return buf[G1Element.SIZE :]
+
+
+def skip_g2_element(buf: memoryview) -> memoryview:
+    return buf[G2Element.SIZE :]
+
+
+def skip_proof_of_space(buf: memoryview) -> memoryview:
+    buf = skip_bytes32(buf)  # challenge
+    buf = skip_optional(buf, skip_g1_element)  # pool_public_key
+    buf = skip_optional(buf, skip_bytes32)  # pool_contract_puzzle_hash
+    buf = skip_g1_element(buf)  # plot_public_key
+    buf = skip_uint8(buf)  # size
+    return skip_bytes(buf)  # proof
+
+
+def skip_reward_chain_block(buf: memoryview) -> memoryview:
+    buf = skip_uint128(buf)  # weight
+    buf = skip_uint32(buf)  # height
+    buf = skip_uint128(buf)  # total_iters
+    buf = skip_uint8(buf)  # signage_point_index
+    buf = skip_bytes32(buf)  # pos_ss_cc_challenge_hash
+
+    buf = skip_proof_of_space(buf)  # proof_of_space
+    buf = skip_optional(buf, skip_vdf_info)  # challenge_chain_sp_vdf
+    buf = skip_g2_element(buf)  # challenge_chain_sp_signature
+    buf = skip_vdf_info(buf)  # challenge_chain_ip_vdf
+    buf = skip_optional(buf, skip_vdf_info)  # reward_chain_sp_vdf
+    buf = skip_g2_element(buf)  # reward_chain_sp_signature
+    buf = skip_vdf_info(buf)  # reward_chain_ip_vdf
+    buf = skip_optional(buf, skip_vdf_info)  # infused_challenge_chain_ip_vdf
+    return skip_bool(buf)  # is_transaction_block
+
+
+def skip_pool_target(buf: memoryview) -> memoryview:
+    # buf = skip_bytes32(buf)  # puzzle_hash
+    # return skip_uint32(buf)  # max_height
+    return buf[32 + 4 :]
+
+
+def skip_foliage_block_data(buf: memoryview) -> memoryview:
+    buf = skip_bytes32(buf)  # unfinished_reward_block_hash
+    buf = skip_pool_target(buf)  # pool_target
+    buf = skip_optional(buf, skip_g2_element)  # pool_signature
+    buf = skip_bytes32(buf)  # farmer_reward_puzzle_hash
+    return skip_bytes32(buf)  # extension_data
+
+
+def skip_foliage(buf: memoryview) -> memoryview:
+    buf = skip_bytes32(buf)  # prev_block_hash
+    buf = skip_bytes32(buf)  # reward_block_hash
+    buf = skip_foliage_block_data(buf)  # foliage_block_data
+    buf = skip_g2_element(buf)  # foliage_block_data_signature
+    buf = skip_optional(buf, skip_bytes32)  # foliage_transaction_block_hash
+    return skip_optional(buf, skip_g2_element)  # foliage_transaction_block_signature
+
+
+def skip_foliage_transaction_block(buf: memoryview) -> memoryview:
+    # buf = skip_bytes32(buf)  # prev_transaction_block_hash
+    # buf = skip_uint64(buf)  # timestamp
+    # buf = skip_bytes32(buf)  # filter_hash
+    # buf = skip_bytes32(buf)  # additions_root
+    # buf = skip_bytes32(buf)  # removals_root
+    # return skip_bytes32(buf)  # transactions_info_hash
+    return buf[32 + 8 + 32 + 32 + 32 + 32 :]
+
+
+def skip_coin(buf: memoryview) -> memoryview:
+    # buf = skip_bytes32(buf)  # parent_coin_info
+    # buf = skip_bytes32(buf)  # puzzle_hash
+    # return skip_uint64(buf)  # amount
+    return buf[32 + 32 + 8 :]
+
+
+def skip_transactions_info(buf: memoryview) -> memoryview:
+    # buf = skip_bytes32(buf)  # generator_root
+    # buf = skip_bytes32(buf)  # generator_refs_root
+    # buf = skip_g2_element(buf)  # aggregated_signature
+    # buf = skip_uint64(buf)  # fees
+    # buf = skip_uint64(buf)  # cost
+    buf = buf[32 + 32 + G2Element.SIZE + 8 + 8 :]
+    return skip_list(buf, skip_coin)
+
+
+def generator_from_block(buf: memoryview) -> Optional[SerializedProgram]:
+    buf = skip_list(buf, skip_end_of_sub_slot_bundle)  # finished_sub_slots
+    buf = skip_reward_chain_block(buf)  # reward_chain_block
+    buf = skip_optional(buf, skip_vdf_proof)  # challenge_chain_sp_proof
+    buf = skip_vdf_proof(buf)  # challenge_chain_ip_proof
+    buf = skip_optional(buf, skip_vdf_proof)  # reward_chain_sp_proof
+    buf = skip_vdf_proof(buf)  # reward_chain_ip_proof
+    buf = skip_optional(buf, skip_vdf_proof)  # infused_challenge_chain_ip_proof
+    buf = skip_foliage(buf)  # foliage
+    buf = skip_optional(buf, skip_foliage_transaction_block)  # foliage_transaction_block
+    buf = skip_optional(buf, skip_transactions_info)  # transactions_info
+
+    # this is the transactions_generator optional
+    if buf[0] == 0:
+        return None
+
+    buf = buf[1:]
+    length = serialized_length(buf)
+    return SerializedProgram.from_bytes(bytes(buf[:length]))

tests/block_tools.py

@@ -2003,7 +2003,7 @@ def create_test_unfinished_block(
         foliage_transaction_block,
         transactions_info,
         block_generator.program if block_generator else None,
-        block_generator.block_height_list if block_generator else [],  # TODO: can block_generator ever be None?
+        block_generator.block_height_list if block_generator else [],
     )
 
 

tests/core/full_node/test_block_store.py

@@ -5,13 +5,17 @@ import sqlite3
 import dataclasses
 
 import pytest
+from clvm.casts import int_to_bytes
 
 from chia.consensus.blockchain import Blockchain
+from chia.consensus.full_block_to_block_record import header_block_to_sub_block_record
+from chia.consensus.default_constants import DEFAULT_CONSTANTS
 from chia.full_node.block_store import BlockStore
 from chia.full_node.coin_store import CoinStore
 from chia.full_node.hint_store import HintStore
 from chia.util.ints import uint8
 from chia.types.blockchain_format.vdf import VDFProof
+from chia.types.blockchain_format.program import SerializedProgram
 from tests.blockchain.blockchain_test_utils import _validate_and_add_block
 from tests.util.db_connection import DBConnection
 from tests.setup_nodes import bt, test_constants
@@ -221,3 +225,44 @@ class TestBlockStore:
                 block_store.rollback_cache_block(block.header_hash)
                 b = await block_store.get_full_block(block.header_hash)
                 assert b.challenge_chain_ip_proof == proof
+
+    @pytest.mark.asyncio
+    async def test_get_generator(self, db_version):
+        blocks = bt.get_consecutive_blocks(10)
+
+        def generator(i: int) -> SerializedProgram:
+            return SerializedProgram.from_bytes(int_to_bytes(i))
+
+        async with DBConnection(db_version) as db_wrapper:
+            store = await BlockStore.create(db_wrapper)
+
+            new_blocks = []
+            for i, block in enumerate(blocks):
+                block = dataclasses.replace(block, transactions_generator=generator(i))
+                block_record = header_block_to_sub_block_record(
+                    DEFAULT_CONSTANTS, 0, block, 0, False, 0, max(0, block.height - 1), None
+                )
+                await store.add_full_block(block.header_hash, block, block_record)
+                await store.set_in_chain([(block_record.header_hash,)])
+                await store.set_peak(block_record.header_hash)
+                new_blocks.append(block)
+
+            if db_version == 2:
+                expected_generators = list(map(lambda x: x.transactions_generator, new_blocks[1:10]))
+                generators = await store.get_generators_at(range(1, 10))
+                assert generators == expected_generators
+
+                # test out-of-order heights
+                expected_generators = list(
+                    map(lambda x: x.transactions_generator, [new_blocks[i] for i in [4, 8, 3, 9]])
+                )
+                generators = await store.get_generators_at([4, 8, 3, 9])
+                assert generators == expected_generators
+
+                with pytest.raises(KeyError):
+                    await store.get_generators_at([100])
+
+            assert await store.get_generator(blocks[2].header_hash) == new_blocks[2].transactions_generator
+            assert await store.get_generator(blocks[4].header_hash) == new_blocks[4].transactions_generator
+            assert await store.get_generator(blocks[6].header_hash) == new_blocks[6].transactions_generator
+            assert await store.get_generator(blocks[7].header_hash) == new_blocks[7].transactions_generator

tests/util/test_full_block_utils.py

@@ -0,0 +1,245 @@
+import random
+import pytest
+
+from chia.util.full_block_utils import generator_from_block
+from chia.types.full_block import FullBlock
+from chia.util.ints import uint128, uint64, uint32, uint8
+from chia.types.blockchain_format.pool_target import PoolTarget
+from chia.types.blockchain_format.foliage import Foliage, FoliageTransactionBlock, TransactionsInfo, FoliageBlockData
+from chia.types.blockchain_format.proof_of_space import ProofOfSpace
+from chia.types.blockchain_format.reward_chain_block import RewardChainBlock
+from chia.types.blockchain_format.program import SerializedProgram
+from chia.types.blockchain_format.slots import (
+    ChallengeChainSubSlot,
+    InfusedChallengeChainSubSlot,
+    RewardChainSubSlot,
+    SubSlotProofs,
+)
+from chia.types.end_of_slot_bundle import EndOfSubSlotBundle
+
+from benchmarks.utils import rand_hash, rand_bytes, rewards, rand_g1, rand_g2, rand_vdf, rand_vdf_proof
+
+test_g2s = [rand_g2() for _ in range(10)]
+test_g1s = [rand_g1() for _ in range(10)]
+test_hashes = [rand_hash() for _ in range(100)]
+test_vdfs = [rand_vdf() for _ in range(100)]
+test_vdf_proofs = [rand_vdf_proof() for _ in range(100)]
+
+
+def g2():
+    return random.sample(test_g2s, 1)[0]
+
+
+def g1():
+    return random.sample(test_g1s, 1)[0]
+
+
+def hsh():
+    return random.sample(test_hashes, 1)[0]
+
+
+def vdf():
+    return random.sample(test_vdfs, 1)[0]
+
+
+def vdf_proof():
+    return random.sample(test_vdf_proofs, 1)[0]
+
+
+def get_proof_of_space():
+    for pool_pk in [g1(), None]:
+        for plot_hash in [hsh(), None]:
+            yield ProofOfSpace(
+                hsh(),  # challenge
+                pool_pk,
+                plot_hash,
+                g1(),  # plot_public_key
+                uint8(32),
+                rand_bytes(8 * 32),
+            )
+
+
+def get_reward_chain_block(height):
+    for has_transactions in [True, False]:
+        for challenge_chain_sp_vdf in [vdf(), None]:
+            for reward_chain_sp_vdf in [vdf(), None]:
+                for infused_challenge_chain_ip_vdf in [vdf(), None]:
+                    for proof_of_space in get_proof_of_space():
+                        weight = uint128(random.randint(0, 1000000000))
+                        iters = uint128(123456)
+                        sp_index = uint8(0)
+                        yield RewardChainBlock(
+                            weight,
+                            uint32(height),
+                            iters,
+                            sp_index,
+                            hsh(),  # pos_ss_cc_challenge_hash
+                            proof_of_space,
+                            challenge_chain_sp_vdf,
+                            g2(),  # challenge_chain_sp_signature
+                            vdf(),  # challenge_chain_ip_vdf
+                            reward_chain_sp_vdf,
+                            g2(),  # reward_chain_sp_signature
+                            vdf(),  # reward_chain_ip_vdf
+                            infused_challenge_chain_ip_vdf,
+                            has_transactions,
+                        )
+
+
+def get_foliage_block_data():
+    for pool_signature in [g2(), None]:
+        pool_target = PoolTarget(
+            hsh(),  # puzzle_hash
+            uint32(0),  # max_height
+        )
+
+        yield FoliageBlockData(
+            hsh(),  # unfinished_reward_block_hash
+            pool_target,
+            pool_signature,  # pool_signature
+            hsh(),  # farmer_reward_puzzle_hash
+            hsh(),  # extension_data
+        )
+
+
+def get_foliage():
+    for foliage_block_data in get_foliage_block_data():
+        for foliage_transaction_block_hash in [hsh(), None]:
+            for foliage_transaction_block_signature in [g2(), None]:
+                yield Foliage(
+                    hsh(),  # prev_block_hash
+                    hsh(),  # reward_block_hash
+                    foliage_block_data,
+                    g2(),  # foliage_block_data_signature
+                    foliage_transaction_block_hash,
+                    foliage_transaction_block_signature,
+                )
+
+
+def get_foliage_transaction_block():
+    yield None
+    timestamp = uint64(1631794488)
+    yield FoliageTransactionBlock(
+        hsh(),  # prev_transaction_block
+        timestamp,
+        hsh(),  # filter_hash
+        hsh(),  # additions_root
+        hsh(),  # removals_root
+        hsh(),  # transactions_info_hash
+    )
+
+
+def get_transactions_info(height):
+    yield None
+    farmer_coin, pool_coin = rewards(uint32(height))
+    reward_claims_incorporated = [farmer_coin, pool_coin]
+    fees = uint64(random.randint(0, 150000))
+
+    yield TransactionsInfo(
+        hsh(),  # generator_root
+        hsh(),  # generator_refs_root
+        g2(),  # aggregated_signature
+        fees,
+        uint64(random.randint(0, 12000000000)),  # cost
+        reward_claims_incorporated,
+    )
+
+
+def get_challenge_chain_sub_slot():
+    for infused_chain_sub_slot_hash in [hsh(), None]:
+        for sub_epoch_summary_hash in [hsh(), None]:
+            for new_sub_slot_iters in [uint64(random.randint(0, 4000000000)), None]:
+                for new_difficulty in [uint64(random.randint(1, 30)), None]:
+                    yield ChallengeChainSubSlot(
+                        vdf(),  # challenge_chain_end_of_slot_vdf
+                        infused_chain_sub_slot_hash,
+                        sub_epoch_summary_hash,
+                        new_sub_slot_iters,
+                        new_difficulty,
+                    )
+
+
+def get_reward_chain_sub_slot():
+    for infused_challenge_chain_sub_slot_hash in [hsh(), None]:
+        yield RewardChainSubSlot(
+            vdf(),  # end_of_slot_vdf
+            hsh(),  # challenge_chain_sub_slot_hash
+            infused_challenge_chain_sub_slot_hash,
+            uint8(random.randint(0, 255)),  # deficit
+        )
+
+
+def get_sub_slot_proofs():
+    for infused_challenge_chain_slot_proof in [vdf_proof(), None]:
+        yield SubSlotProofs(
+            vdf_proof(),  # challenge_chain_slot_proof
+            infused_challenge_chain_slot_proof,
+            vdf_proof(),  # reward_chain_slot_proof
+        )
+
+
+def get_end_of_sub_slot():
+    for challenge_chain in get_challenge_chain_sub_slot():
+        for infused_challenge_chain in [InfusedChallengeChainSubSlot(vdf()), None]:
+            for reward_chain in get_reward_chain_sub_slot():
+                for proofs in get_sub_slot_proofs():
+                    yield EndOfSubSlotBundle(
+                        challenge_chain,
+                        infused_challenge_chain,
+                        reward_chain,
+                        proofs,
+                    )
+
+
+def get_finished_sub_slots():
+    yield []
+    yield [s for s in get_end_of_sub_slot()]
+
+
+def get_full_blocks():
+
+    random.seed(123456789)
+
+    generator = SerializedProgram.from_bytes(bytes.fromhex("ff01820539"))
+
+    for foliage in get_foliage():
+        for foliage_transaction_block in get_foliage_transaction_block():
+            height = random.randint(0, 1000000)
+            for reward_chain_block in get_reward_chain_block(height):
+                for transactions_info in get_transactions_info(height):
+                    for challenge_chain_sp_proof in [vdf_proof(), None]:
+                        for reward_chain_sp_proof in [vdf_proof(), None]:
+                            for infused_challenge_chain_ip_proof in [vdf_proof(), None]:
+                                for finished_sub_slots in get_finished_sub_slots():
+
+                                    yield FullBlock(
+                                        finished_sub_slots,
+                                        reward_chain_block,
+                                        challenge_chain_sp_proof,
+                                        vdf_proof(),  # challenge_chain_ip_proof
+                                        reward_chain_sp_proof,
+                                        vdf_proof(),  # reward_chain_ip_proof
+                                        infused_challenge_chain_ip_proof,
+                                        foliage,
+                                        foliage_transaction_block,
+                                        transactions_info,
+                                        generator,  # transactions_generator
+                                        [],  # transactions_generator_ref_list
+                                    )
+
+
+class TestFullBlockParser:
+    @pytest.mark.asyncio
+    async def test_parser(self):
+
+        # loop over every combination of Optionals being set and not set
+        # along with random values for the FullBlock fields. Ensure
+        # generator_from_block() successfully parses out the generator object
+        # correctly
+        for block in get_full_blocks():
+
+            block_bytes = bytes(block)
+            gen = generator_from_block(block_bytes)
+            assert gen == block.transactions_generator
+            # this doubles the run-time of this test, with questionable utility
+            # assert gen == FullBlock.from_bytes(block_bytes).transactions_generator