diff --git a/Cargo.lock b/Cargo.lock index c0b473f5788..bcd785df73a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6569,7 +6569,7 @@ dependencies = [ [[package]] name = "milhouse" version = "0.9.0" -source = "git+https://github.com/sigp/milhouse?rev=d67aabd2a7c405b3c16f4aaff159b137dc5b5e38#d67aabd2a7c405b3c16f4aaff159b137dc5b5e38" +source = "git+https://github.com/sigp/milhouse?rev=e7c611a7d768cf303b02797d9d2000f9b5e2bb41#e7c611a7d768cf303b02797d9d2000f9b5e2bb41" dependencies = [ "alloy-primitives", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index 0befd6819c0..87475ea375e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -296,7 +296,7 @@ debug = true [patch.crates-io] # FIXME(eip-7688): EIP-7688 development patches ssz_types = { git = "https://github.com/sigp/ssz_types", rev = "8955d22edc7633f26fe3b5ac933facb5121d717a" } -milhouse = { git = "https://github.com/sigp/milhouse", rev = "d67aabd2a7c405b3c16f4aaff159b137dc5b5e38" } +milhouse = { git = "https://github.com/sigp/milhouse", rev = "e7c611a7d768cf303b02797d9d2000f9b5e2bb41" } ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", rev = "974f78f421fbf70cba4e8f84e0fa28c23ad54f34" } ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", rev = "974f78f421fbf70cba4e8f84e0fa28c23ad54f34" } tree_hash = { git = "https://github.com/sigp/tree_hash", rev = "e5ea1875edddf12438506c3751cbb313f9c52d08" } diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index a6e7c68a152..db96de160aa 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -64,8 +64,6 @@ pub struct ChainConfig { pub paranoid_block_proposal: bool, /// Optionally set timeout for calls to checkpoint sync endpoint. pub checkpoint_sync_url_timeout: u64, - /// The offset before the start of a proposal slot at which payload attributes should be sent. - /// /// Low values are useful for execution engines which don't improve their payload after the /// first call, and high values are useful for ensuring the EL is given ample notice. pub prepare_payload_lookahead: Duration, @@ -163,7 +161,7 @@ impl Default for ChainConfig { always_reset_payload_statuses: false, paranoid_block_proposal: false, checkpoint_sync_url_timeout: 60, - prepare_payload_lookahead: Duration::from_secs(4), + prepare_payload_lookahead: Duration::from_secs(4), // This value isn't actually read except in tests. optimistic_finalized_sync: true, shuffling_cache_size: crate::shuffling_cache::DEFAULT_CACHE_SIZE, diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index fc57915ebd1..0a54dab1f75 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -461,6 +461,37 @@ where })? .ok_or("Finalized block missing from remote, it returned 404")?; let block_root = block.canonical_root(); + // If --wss-checkpoint was provided, verify it against the downloaded checkpoint + // before initializing the chain from it. --wss-checkpoint's epoch may be older + // than the downloaded state's own finalized epoch (the same tolerance that + // `verify_weak_subjectivity_checkpoint` already applies once the chain is + // running), so look up the root at that epoch instead of only ever comparing + // against the freshest downloaded block. + if let Some(wss_checkpoint) = config.chain.weak_subjectivity_checkpoint { + let finalized_checkpoint = state.finalized_checkpoint(); + let verified_root = if wss_checkpoint.epoch < finalized_checkpoint.epoch { + let target_slot = wss_checkpoint.epoch.start_slot(E::slots_per_epoch()); + *state.get_block_root(target_slot).map_err(|e| { + format!( + "Unable to find block root for --wss-checkpoint epoch {:?}: {:?}", + wss_checkpoint.epoch, e + ) + })? + } else { + block_root + }; + if verified_root != wss_checkpoint.root { + return Err(format!( + "Checkpoint sync block root mismatch. Expected (from --wss-checkpoint): {:?}, \ + Found: {:?}. The checkpoint provider may be serving a different chain.", + wss_checkpoint.root, verified_root + )); + } + info!( + block_root = ?verified_root, + "Verified checkpoint block root matches --wss-checkpoint" + ); + } debug!("Downloaded finalized block"); diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b7372f55593..b95b118177e 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -533,6 +533,7 @@ pub fn get_config( clap_utils::parse_required(cli_args, "genesis-state-url-timeout") .map(Duration::from_secs)?; + let genesis_state_url_opt = clap_utils::parse_optional::(cli_args, "genesis-state-url")?; let checkpoint_sync_url_opt = diff --git a/consensus/types/src/beacon_state_summary/beacon_state_part.rs b/consensus/types/src/beacon_state_summary/beacon_state_part.rs new file mode 100644 index 00000000000..11bab1cdae5 --- /dev/null +++ b/consensus/types/src/beacon_state_summary/beacon_state_part.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use ssz_types::ProgressiveVariableList; +use tree_hash_derive::TreeHash; + +use crate::core::Hash256; + +/// One fetchable slice of state data, returned by `beacon_state_parts_by_range`: the chunk's +/// index, its raw bytes, and a Merkle proof (`branch`) tying it to an already-verified +/// [`super::BeaconStateSnapshot`] summary root. Different chunks can be fetched from different, +/// mutually untrusting peers and verified independently. +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] +#[serde(deny_unknown_fields)] +pub struct BeaconStatePart { + #[serde(with = "serde_utils::quoted_u64")] + pub chunk_index: u64, + pub data: ProgressiveVariableList, + pub branch: ProgressiveVariableList, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(BeaconStatePart); +} diff --git a/consensus/types/src/beacon_state_summary/beacon_state_snapshot.rs b/consensus/types/src/beacon_state_summary/beacon_state_snapshot.rs new file mode 100644 index 00000000000..bca296e953c --- /dev/null +++ b/consensus/types/src/beacon_state_summary/beacon_state_snapshot.rs @@ -0,0 +1,25 @@ +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use ssz_types::ProgressiveVariableList; +use tree_hash_derive::TreeHash; + +use crate::{beacon_state_summary::BeaconStateSummary, core::EthSpec, core::Hash256}; + +/// Response to a `beacon_state_summary` request: a [`BeaconStateSummary`] plus a Merkle proof +/// (`state_branch`) tying its root to the already-trusted block root the requester started from. +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] +#[cfg_attr(feature = "arbitrary", arbitrary(bound = "E: EthSpec"))] +pub struct BeaconStateSnapshot { + pub summary: BeaconStateSummary, + pub state_branch: ProgressiveVariableList, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::MinimalEthSpec; + + ssz_and_tree_hash_tests!(BeaconStateSnapshot); +} diff --git a/consensus/types/src/beacon_state_summary/beacon_state_summary.rs b/consensus/types/src/beacon_state_summary/beacon_state_summary.rs new file mode 100644 index 00000000000..965f00ce833 --- /dev/null +++ b/consensus/types/src/beacon_state_summary/beacon_state_summary.rs @@ -0,0 +1,292 @@ +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use ssz_types::{BitVector, FixedVector}; +use tree_hash_derive::TreeHash; + +use crate::{ + ExecutionBlockHash, ExecutionPayloadBidGloas, Fork, + attestation::Checkpoint, + beacon_state_summary::ListSummary, + block::BeaconBlockHeader, + builder::{BuilderIndex, BuilderPendingPayment}, + core::{Epoch, EthSpec, Hash256, Slot}, + execution::Eth1Data, + sync_committee::SyncCommittee, +}; + +/// Mirror of `BeaconState` (Gloas) with every `List`/`ProgressiveList` field replaced by a +/// [`ListSummary`], and every other field left unchanged. Shares the same `hash_tree_root` as the +/// `BeaconStateGloas` it summarizes, since `progressive_container` merkleization (like regular +/// `Container` merkleization) depends only on the ordered sequence of field roots, and a +/// `ListSummary` built from a list's real `items_root`/`num_items` reproduces that list's root +/// exactly. +/// +/// Only targets Gloas: checkpoint sync only ever needs a recent finalized state, and Gloas is +/// already the live/rolling-out fork (`plataberget`, then `hoodi`/`sepolia`, then mainnet). +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] +#[cfg_attr(feature = "arbitrary", arbitrary(bound = "E: EthSpec"))] +#[tree_hash( + struct_behaviour = "progressive_container", + active_fields( + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + ) +)] +pub struct BeaconStateSummary { + // Versioning + pub genesis_time: u64, + pub genesis_validators_root: Hash256, + pub slot: Slot, + pub fork: Fork, + + // History + pub latest_block_header: BeaconBlockHeader, + pub block_roots: FixedVector, + pub state_roots: FixedVector, + pub historical_roots: ListSummary, + + // Ethereum 1.0 chain data + pub eth1_data: Eth1Data, + pub eth1_data_votes: ListSummary, + #[serde(with = "serde_utils::quoted_u64")] + pub eth1_deposit_index: u64, + + // Registry + pub validators: ListSummary, + pub balances: ListSummary, + + // Randomness + pub randao_mixes: FixedVector, + + // Slashings + #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] + pub slashings: FixedVector, + + // Participation (Gloas uses ProgressiveList, summarized like any other list) + pub previous_epoch_participation: ListSummary, + pub current_epoch_participation: ListSummary, + + // Finality + pub justification_bits: BitVector, + pub previous_justified_checkpoint: Checkpoint, + pub current_justified_checkpoint: Checkpoint, + pub finalized_checkpoint: Checkpoint, + + // Inactivity + pub inactivity_scores: ListSummary, + + // Light-client sync committees + pub current_sync_committee: Arc>, + pub next_sync_committee: Arc>, + + // Execution + pub latest_block_hash: ExecutionBlockHash, + #[serde(with = "serde_utils::quoted_u64")] + pub next_withdrawal_index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub next_withdrawal_validator_index: u64, + pub historical_summaries: ListSummary, + + // Electra + #[serde(with = "serde_utils::quoted_u64")] + pub deposit_requests_start_index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub deposit_balance_to_consume: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub exit_balance_to_consume: u64, + pub earliest_exit_epoch: Epoch, + #[serde(with = "serde_utils::quoted_u64")] + pub consolidation_balance_to_consume: u64, + pub earliest_consolidation_epoch: Epoch, + pub pending_deposits: ListSummary, + pub pending_partial_withdrawals: ListSummary, + pub pending_consolidations: ListSummary, + + // Fulu + #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] + pub proposer_lookahead: FixedVector, + + // Gloas + pub builders: ListSummary, + #[serde(with = "serde_utils::quoted_u64")] + pub next_withdrawal_builder_index: BuilderIndex, + pub execution_payload_availability: BitVector, + pub builder_pending_payments: + FixedVector, + pub builder_pending_withdrawals: ListSummary, + pub latest_execution_payload_bid: ExecutionPayloadBidGloas, + pub payload_expected_withdrawals: ListSummary, + pub ptc_window: FixedVector, E::PtcWindowLength>, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BeaconStateGloas, MinimalEthSpec, test_utils::test_arbitrary_instance}; + use tree_hash::{MerkleHasher, ProgressiveMerkleHasher, TreeHash, TreeHashType}; + use typenum::Unsigned; + + ssz_and_tree_hash_tests!(BeaconStateSummary); + + /// Merkleizes `items` to a regular (non-progressive) `List`'s items_root, matching the SSZ + /// spec's `merkleize(pack(items), limit=chunk_count(type))` exactly, using only public + /// `tree_hash` primitives (not milhouse's still-private tree access). + fn regular_items_root(items: &[T], capacity: usize) -> Hash256 { + let leaves = match T::tree_hash_type() { + TreeHashType::Basic => capacity.div_ceil(T::tree_hash_packing_factor()), + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => capacity, + }; + let mut hasher = MerkleHasher::with_leaves(leaves); + match T::tree_hash_type() { + TreeHashType::Basic => { + for item in items { + hasher + .write(&item.tree_hash_packed_encoding()) + .expect("capacity is large enough for all items"); + } + } + _ => { + for item in items { + hasher + .write(item.tree_hash_root().as_slice()) + .expect("capacity is large enough for all items"); + } + } + } + hasher.finish().expect("no partial buffer remains") + } + + /// Same as `regular_items_root`, but for `ProgressiveList` fields (EIP-7916 merkleization, + /// no fixed capacity). + fn progressive_items_root(items: &[T]) -> Hash256 { + let mut hasher = ProgressiveMerkleHasher::new(); + match T::tree_hash_type() { + TreeHashType::Basic => { + for item in items { + hasher + .write(&item.tree_hash_packed_encoding()) + .expect("progressive hasher accepts any number of chunks"); + } + } + _ => { + for item in items { + hasher + .write(item.tree_hash_root().as_slice()) + .expect("progressive hasher accepts any number of chunks"); + } + } + } + hasher.finish().expect("no partial buffer remains") + } + + fn list_summary_regular(items: &[T], capacity: usize) -> ListSummary { + ListSummary { + items_root: regular_items_root(items, capacity), + num_items: items.len() as u64, + } + } + + fn list_summary_progressive(items: &[T]) -> ListSummary { + ListSummary { + items_root: progressive_items_root(items), + num_items: items.len() as u64, + } + } + + /// Proves `BeaconStateSummary::tree_hash_root()` equals the real `BeaconStateGloas` it + /// summarizes, end to end, on a randomly generated (but structurally valid) state. This is + /// the concrete verification of the equivalence argument the whole design rests on, built + /// without needing milhouse's still-missing public items_root accessor: item values are read + /// out via milhouse's existing public `to_vec()`, and their roots are recomputed + /// independently using only public `tree_hash` primitives. + #[test] + fn summary_root_matches_real_gloas_state() { + let state: BeaconStateGloas = test_arbitrary_instance(); + + let historical_roots_limit = + ::HistoricalRootsLimit::to_usize(); + let eth1_voting_period = + ::SlotsPerEth1VotingPeriod::to_usize(); + + let summary = BeaconStateSummary:: { + genesis_time: state.genesis_time, + genesis_validators_root: state.genesis_validators_root, + slot: state.slot, + fork: state.fork, + latest_block_header: state.latest_block_header.clone(), + block_roots: FixedVector::new(state.block_roots.to_vec()).unwrap(), + state_roots: FixedVector::new(state.state_roots.to_vec()).unwrap(), + historical_roots: list_summary_regular( + &state.historical_roots.to_vec(), + historical_roots_limit, + ), + eth1_data: state.eth1_data.clone(), + eth1_data_votes: list_summary_regular( + &state.eth1_data_votes.to_vec(), + eth1_voting_period, + ), + eth1_deposit_index: state.eth1_deposit_index, + validators: list_summary_progressive(&state.validators.to_vec()), + balances: list_summary_progressive(&state.balances.to_vec()), + randao_mixes: FixedVector::new(state.randao_mixes.to_vec()).unwrap(), + slashings: FixedVector::new(state.slashings.to_vec()).unwrap(), + previous_epoch_participation: list_summary_progressive( + &state.previous_epoch_participation.to_vec(), + ), + current_epoch_participation: list_summary_progressive( + &state.current_epoch_participation.to_vec(), + ), + justification_bits: state.justification_bits.clone(), + previous_justified_checkpoint: state.previous_justified_checkpoint, + current_justified_checkpoint: state.current_justified_checkpoint, + finalized_checkpoint: state.finalized_checkpoint, + inactivity_scores: list_summary_progressive(&state.inactivity_scores.to_vec()), + current_sync_committee: state.current_sync_committee.clone(), + next_sync_committee: state.next_sync_committee.clone(), + latest_block_hash: state.latest_block_hash, + next_withdrawal_index: state.next_withdrawal_index, + next_withdrawal_validator_index: state.next_withdrawal_validator_index, + historical_summaries: list_summary_regular( + &state.historical_summaries.to_vec(), + historical_roots_limit, + ), + deposit_requests_start_index: state.deposit_requests_start_index, + deposit_balance_to_consume: state.deposit_balance_to_consume, + exit_balance_to_consume: state.exit_balance_to_consume, + earliest_exit_epoch: state.earliest_exit_epoch, + consolidation_balance_to_consume: state.consolidation_balance_to_consume, + earliest_consolidation_epoch: state.earliest_consolidation_epoch, + pending_deposits: list_summary_progressive(&state.pending_deposits.to_vec()), + pending_partial_withdrawals: list_summary_progressive( + &state.pending_partial_withdrawals.to_vec(), + ), + pending_consolidations: list_summary_progressive( + &state.pending_consolidations.to_vec(), + ), + proposer_lookahead: FixedVector::new(state.proposer_lookahead.to_vec()).unwrap(), + builders: list_summary_progressive(&state.builders.to_vec()), + next_withdrawal_builder_index: state.next_withdrawal_builder_index, + execution_payload_availability: state.execution_payload_availability.clone(), + builder_pending_payments: FixedVector::new(state.builder_pending_payments.to_vec()) + .unwrap(), + builder_pending_withdrawals: list_summary_progressive( + &state.builder_pending_withdrawals.to_vec(), + ), + latest_execution_payload_bid: state.latest_execution_payload_bid.clone(), + payload_expected_withdrawals: list_summary_progressive( + &state.payload_expected_withdrawals.to_vec(), + ), + ptc_window: FixedVector::new(state.ptc_window.to_vec()).unwrap(), + }; + + assert_eq!( + summary.tree_hash_root(), + state.tree_hash_root(), + "BeaconStateSummary must reproduce the real BeaconStateGloas's hash_tree_root" + ); + } +} diff --git a/consensus/types/src/beacon_state_summary/chunk_proof.rs b/consensus/types/src/beacon_state_summary/chunk_proof.rs new file mode 100644 index 00000000000..519736ea08e --- /dev/null +++ b/consensus/types/src/beacon_state_summary/chunk_proof.rs @@ -0,0 +1,587 @@ +//! Builds the `data`/`branch` halves of a `BeaconStatePart` for one chunk: the raw item bytes, +//! and a Merkle proof that the chunk's subtree is included in its field's full items root (what +//! `List`/`ProgressiveList::tree_hash_items_root()` returns). Not yet wired into an actual +//! `beacon_state_parts_by_range` handler (no caller exists in this crate yet), hence `#[allow(dead_code)]` +//! below; every function is exercised and verified correct by this module's own tests, which +//! reconstruct the root from a generated chunk + branch and check it against a real +//! `milhouse::List`/`ProgressiveList`'s `tree_hash_items_root()`. +#![allow(dead_code)] + +use std::ops::Range; + +use ethereum_hashing::hash32_concat; +use ssz::Encode; +use tree_hash::{Hash256, TreeHash, TreeHashType}; + +/// `hash(a, b)`, the one primitive every SSZ merkle structure (flat or progressive) is built +/// from. +fn hash_pair(a: &Hash256, b: &Hash256) -> Hash256 { + Hash256::from(hash32_concat(a.as_slice(), b.as_slice())) +} + +/// Precomputed zero-subtree roots: `zero_hashes[0]` is a zero leaf, `zero_hashes[d]` is the root +/// of an all-zero subtree of depth `d` (`2^d` zero leaves). Lets a proof reach an astronomically +/// large SSZ capacity (e.g. a list limit of 2^30) without ever materializing the padding. +fn zero_hashes(max_depth: usize) -> Vec { + let mut zh = vec![Hash256::ZERO]; + for _ in 0..max_depth { + let prev = *zh.last().expect("just pushed at least one entry"); + zh.push(hash_pair(&prev, &prev)); + } + zh +} + +/// The 32-byte merkle leaves for `items`, matching exactly what `MerkleHasher`/ +/// `ProgressiveMerkleHasher` would stream internally: for `Basic` (packable) types, each item's +/// packed encoding is concatenated and split into 32-byte chunks; for everything else, one leaf +/// per item (`tree_hash_root()`). +fn item_leaves(items: &[T]) -> Vec { + match T::tree_hash_type() { + TreeHashType::Basic => { + let mut bytes = Vec::new(); + for item in items { + bytes.extend_from_slice(&item.tree_hash_packed_encoding()); + } + bytes + .chunks(32) + .map(|chunk| { + let mut leaf = [0u8; 32]; + if let Some(dest) = leaf.get_mut(..chunk.len()) { + dest.copy_from_slice(chunk); + } + Hash256::from_slice(&leaf) + }) + .collect() + } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + items.iter().map(|item| item.tree_hash_root()).collect() + } + } +} + +/// Every level of a flat binary tree over `leaves`, zero-padded up to the next power of two. +/// `levels[0]` is the (padded) leaf level, `levels.last()` is the single-node root level. +fn flat_tree_levels(leaves: &[Hash256]) -> Vec> { + let padded_len = leaves.len().max(1).next_power_of_two(); + let mut level: Vec = (0..padded_len) + .map(|i| leaves.get(i).copied().unwrap_or(Hash256::ZERO)) + .collect(); + let mut levels = vec![level.clone()]; + while level.len() > 1 { + level = level.chunks(2).map(hash_pair_chunk).collect(); + levels.push(level.clone()); + } + levels +} + +/// `hash_pair` over a 2-element chunk from `.chunks(2)`; always exactly 2 elements here, since +/// every level built by this module has a power-of-two length. +fn hash_pair_chunk(pair: &[Hash256]) -> Hash256 { + match pair { + [a, b] => hash_pair(a, b), + [a] => hash_pair(a, a), + [] => Hash256::ZERO, + _ => unreachable!("chunks(2) never yields more than 2 elements"), + } +} + +/// The root and sibling path (bottom to top, standard merkle proof order) for the subtree +/// spanning leaves `[start, start + len)` of a tree built from `flat_tree_levels`. `start` must +/// be a multiple of `len`, and `len` must be a power of two. +fn flat_subtree_proof( + levels: &[Vec], + start: usize, + len: usize, +) -> (Hash256, Vec) { + let subtree_level = len.trailing_zeros() as usize; + let mut index = start.checked_div(len.max(1)).unwrap_or(0); + let subtree_root = levels + .get(subtree_level) + .and_then(|level| level.get(index)) + .copied() + .expect("subtree_level/index are in bounds by construction"); + + let mut siblings = Vec::new(); + let top = levels.len().saturating_sub(1); + for level in levels.get(subtree_level..top).unwrap_or(&[]) { + let sibling_index = index ^ 1; + siblings.push( + level + .get(sibling_index) + .copied() + .expect("sibling_index is in bounds by construction"), + ); + index = index.checked_div(2).unwrap_or(0); + } + (subtree_root, siblings) +} + +/// Extra siblings needed to extend a proof from a subtree root over `real_capacity` leaves (a +/// power of two) up to a declared SSZ `full_capacity` (also a power of two, `>= real_capacity`). +/// Real data always starts at leaf 0, so the real subtree sits as the left child at every one of +/// these extra depths, with an all-zero sibling on the right each time. +fn pad_to_capacity(real_capacity: usize, full_capacity: usize) -> Vec { + debug_assert!(full_capacity >= real_capacity); + debug_assert!(real_capacity.is_power_of_two() && full_capacity.is_power_of_two()); + // The real subtree's own depth (it spans `real_capacity` leaves); each extension step's + // sibling must span the SAME width as the node it's paired with at that step, which doubles + // every step, so the siblings start at this depth and increase from there, not from a bare + // zero leaf (depth 0) each time. + let real_depth = real_capacity.max(1).trailing_zeros() as usize; + let extra_depth = full_capacity + .checked_div(real_capacity.max(1)) + .unwrap_or(1) + .trailing_zeros() as usize; + let total_depth = real_depth.saturating_add(extra_depth); + let zh = zero_hashes(total_depth); + (real_depth..total_depth) + .map(|d| zh.get(d).copied().expect("d < total_depth == zh.len() - 1")) + .collect() +} + +/// `branch` for a chunk of a plain (non-progressive) `List` field: `[start, start + len)` of +/// `items`, proved against the field's full SSZ capacity (`capacity_items`, converted internally +/// to leaves via the item type's packing factor). +pub fn plain_list_chunk_branch( + items: &[T], + capacity_items: usize, + range: Range, +) -> Vec { + let packing_factor = match T::tree_hash_type() { + TreeHashType::Basic => T::tree_hash_packing_factor(), + _ => 1, + }; + let leaf_range_start = range.start.checked_div(packing_factor).unwrap_or(0); + let leaf_range_len = range + .len() + .div_ceil(packing_factor) + .max(1) + .next_power_of_two(); + let full_capacity_leaves = capacity_items + .div_ceil(packing_factor) + .max(1) + .next_power_of_two(); + + let leaves = item_leaves(items); + let levels = flat_tree_levels(&leaves); + let real_capacity = levels.first().map_or(1, Vec::len); + + let (_subtree_root, mut branch) = flat_subtree_proof(&levels, leaf_range_start, leaf_range_len); + branch.extend(pad_to_capacity(real_capacity, full_capacity_leaves)); + branch +} + +/// Start/size (in leaves) of each `ProgressiveList` level that has at least one leaf, given the +/// field's total leaf count. Level `i` holds `4^i` leaves; this walks levels until they cover +/// `total_leaves`, so the last entry may be only partially real (zero-padded internally). +fn progressive_level_bounds(total_leaves: usize) -> Vec<(usize, usize)> { + let mut bounds = Vec::new(); + let mut start = 0usize; + let mut size = 1usize; + while start < total_leaves { + bounds.push((start, size)); + start = start.saturating_add(size); + size = size.saturating_mul(4); + } + bounds +} + +/// Each level's own full root (`L_i`), computed from that level's leaf slice, zero-padded to the +/// level's declared size (`4^i`) if it's the last, partial level. +fn progressive_level_roots(leaves: &[Hash256]) -> Vec { + progressive_level_bounds(leaves.len()) + .into_iter() + .map(|(start, size)| { + let end = start.saturating_add(size).min(leaves.len()); + let level_leaves = leaves.get(start..end).unwrap_or(&[]); + // A full level matches `size` exactly; a partial final level is zero-padded inside + // `flat_tree_levels_fixed`, matching `size` since `size` is itself a power of two + // and `level_leaves.len() <= size`. + flat_tree_levels_fixed(level_leaves, size) + }) + .collect() +} + +/// Like `flat_tree_levels`, but padded to exactly `capacity` (not `leaves.len()`'s own next +/// power of two), returning just the root. Used for progressive levels, whose declared size +/// (`4^i`) may be larger than the real leaves present in a partial final level. +fn flat_tree_levels_fixed(leaves: &[Hash256], capacity: usize) -> Hash256 { + let mut level: Vec = (0..capacity) + .map(|i| leaves.get(i).copied().unwrap_or(Hash256::ZERO)) + .collect(); + while level.len() > 1 { + level = level.chunks(2).map(hash_pair_chunk).collect(); + } + level.first().copied().unwrap_or(Hash256::ZERO) +} + +/// Folds `level_roots[from..]` the same way `ProgressiveMerkleHasher` folds trailing levels: +/// starting from `ZERO` (nothing deeper), combining from the deepest level backward. Returns +/// `ZERO` if there's nothing at or after `from`. +fn fold_levels_from(level_roots: &[Hash256], from: usize) -> Hash256 { + let mut result = Hash256::ZERO; + if from >= level_roots.len() { + return result; + } + for root in level_roots.get(from..).unwrap_or(&[]).iter().rev() { + result = hash_pair(root, &result); + } + result +} + +/// `branch` for a chunk of a `ProgressiveList` field, local chunk index `local_chunk_index` +/// (`0` is the irregular folded first chunk; everything after is a round `items_per_chunk`-sized +/// slice). Proves the chunk against the field's full items root (what `List`/`ProgressiveList`'s +/// own `tree_hash_items_root()` getter returns, pre-`mix_in_length`), not the field's overall +/// `tree_hash_root()`, matching `ListSummary`'s own `{ items_root, num_items }` shape: a verifier +/// who already trusts the containing `BeaconStateSummary` already has `num_items`, so `branch` +/// only needs to cover the items_root half. +/// +/// For the first chunk (`local_chunk_index == 0`): the chunk's own data *is* every item in every +/// folded level, so the verifier can recompute each folded level's root directly, no sibling +/// needed for them. The branch is always exactly one hash, standing in for "everything deeper +/// than the folded prefix" (`Hash256::ZERO` if the first chunk is the *only* chunk), since the +/// real algorithm always combines the deepest active level with something on its right, even +/// when that something is the zero placeholder for "no more data". +/// +/// For a round chunk at level `m`: the verifier only has a slice of level `m`'s own items, not +/// the earlier levels' raw data, so it needs each earlier level's full root (`L_0..L_{m-1}`) as +/// explicit siblings, then the standard within-level binary-tree siblings for its own slice of +/// level `m`, then (if there's anything deeper than `m`) one more hash for that. +/// +/// `first_chunk_size`/`items_per_chunk` are the field's own values (e.g. +/// `ChunkedField::Validators.first_chunk_size()`/`.items_per_chunk()`), passed as plain numbers +/// so this function stays independent of `ChunkedField` itself. +pub fn progressive_chunk_branch( + items: &[T], + first_chunk_size: usize, + items_per_chunk: usize, + local_chunk_index: usize, +) -> Vec { + let leaves = item_leaves(items); + let level_roots = progressive_level_roots(&leaves); + let packing_factor = match T::tree_hash_type() { + TreeHashType::Basic => T::tree_hash_packing_factor(), + _ => 1, + }; + + let packing_factor = packing_factor.max(1); + if local_chunk_index == 0 { + let first_chunk_leaves = first_chunk_size.checked_div(packing_factor).unwrap_or(0); + let folded_levels = progressive_level_bounds(first_chunk_leaves).len(); + return vec![fold_levels_from(&level_roots, folded_levels)]; + } + + // Which level this round chunk lives in, and where within it. + let first_chunk_leaves = first_chunk_size.checked_div(packing_factor).unwrap_or(0); + let items_per_chunk_leaves = items_per_chunk.checked_div(packing_factor).unwrap_or(0); + let chunk_start_leaf = first_chunk_leaves.saturating_add( + local_chunk_index + .saturating_sub(1) + .saturating_mul(items_per_chunk_leaves), + ); + + let level_bounds = progressive_level_bounds(leaves.len()); + let (m, &(level_start, level_size)) = level_bounds + .iter() + .enumerate() + .find(|&(_, &(start, size))| { + chunk_start_leaf >= start && chunk_start_leaf < start.saturating_add(size) + }) + .expect("a round chunk always falls within some level, by construction"); + + // Standard binary-tree siblings within level m, for this chunk's slice of it. + let level_end = level_start.saturating_add(level_size).min(leaves.len()); + let level_leaves = leaves.get(level_start..level_end).unwrap_or(&[]); + let levels_within = flat_tree_levels(level_leaves_padded(level_leaves, level_size).as_slice()); + let local_start = chunk_start_leaf.saturating_sub(level_start); + let local_len = items_per_chunk_leaves.min(level_size); + let (_chunk_root, mut branch) = flat_subtree_proof(&levels_within, local_start, local_len); + + // Earlier levels' full roots, L_0..L_{m-1}, as explicit siblings (verifier has no raw data + // for them). + let mut chain_siblings: Vec = level_roots.get(..m).unwrap_or(&[]).to_vec(); + chain_siblings.reverse(); // closest-first: L_{m-1}, L_{m-2}, ..., L_0. + branch.extend(chain_siblings); + + // Everything deeper than level m, folded into a single hash (ZERO if m is the deepest level + // with any data at all), always pushed for a uniform verification format. + branch.push(fold_levels_from(&level_roots, m.saturating_add(1))); + + branch +} + +/// Pads `leaves` up to exactly `capacity` with zero leaves (capacity is always a power of two +/// here, a progressive level's declared `4^i` size). +fn level_leaves_padded(leaves: &[Hash256], capacity: usize) -> Vec { + let mut padded = leaves.to_vec(); + padded.resize(capacity, Hash256::ZERO); + padded +} + +/// The raw SSZ-encoded bytes for `items[range]`, concatenated: the `data` half of a +/// `BeaconStatePart`. +pub fn chunk_data(items: &[T], range: Range) -> Vec { + items + .get(range) + .unwrap_or(&[]) + .iter() + .flat_map(|item| item.as_ssz_bytes()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use bls::FixedBytesExtended; + use milhouse::{List, ProgressiveList}; + use typenum::U1024; + + /// Local re-implementation of `ChunkedField::num_chunks`/`chunk_range`'s math, parameterized + /// directly rather than tied to the enum, so tests can use small, fast synthetic sizes + /// instead of the real multi-thousand-item field constants. + fn chunk_range( + len: usize, + first_chunk_size: usize, + items_per_chunk: usize, + i: usize, + ) -> Option> { + if len == 0 { + return None; + } + if i == 0 { + return Some(0..first_chunk_size.min(len)); + } + let start = first_chunk_size + (i - 1) * items_per_chunk; + if start >= len { + return None; + } + Some(start..(start + items_per_chunk).min(len)) + } + + /// Reconstructs a root from a chunk's own root and a standard bottom-up sibling path, + /// using `index`'s parity at each step to know whether the sibling is on the left or right. + /// Once `index` reaches 0 it stays 0, which correctly treats every remaining step as "we are + /// the left child, sibling is zero-padding on the right", with no special-casing needed. + fn fold_branch(mut node: Hash256, mut index: usize, siblings: &[Hash256]) -> Hash256 { + for sibling in siblings { + node = if index % 2 == 1 { + hash_pair(sibling, &node) + } else { + hash_pair(&node, sibling) + }; + index /= 2; + } + node + } + + fn leaf_root(items: &[T]) -> Hash256 { + let leaves = item_leaves(items); + *flat_tree_levels(&leaves).last().unwrap().first().unwrap() + } + + #[test] + fn plain_list_branch_reconstructs_real_items_root() { + let capacity_items = 1024; // must match the `U1024` capacity used below + for len in [0usize, 1, 3, 4, 7, 16, 33, 64] { + let items: Vec = (0..len as u64).collect(); + let list = List::::new(items.clone()).unwrap(); + let expected = list.tree_hash_items_root(); + + let items_per_chunk = 4; + let mut i = 0; + loop { + let Some(range) = chunk_range(len, items_per_chunk, items_per_chunk, i) else { + break; + }; + let chunk_items = &items[range.clone()]; + let branch = plain_list_chunk_branch(&items, capacity_items, range.clone()); + + let packing_factor = u64::tree_hash_packing_factor(); + let leaf_start = range.start / packing_factor; + let leaf_len = range.len().div_ceil(packing_factor).next_power_of_two(); + let chunk_root = leaf_root(chunk_items); + let reconstructed = fold_branch(chunk_root, leaf_start / leaf_len, &branch); + + assert_eq!( + reconstructed, expected, + "len={len} chunk={i} range={range:?} mismatch" + ); + i += 1; + } + } + } + + #[test] + fn progressive_branch_reconstructs_real_items_root_unpacked() { + // Hash256 is TreeHashType::Vector (one leaf per item, no packing), exercising the + // Container/List/Vector branch of item_leaves, matching real fields like Validators. + let first_chunk_size = 5; // levels 0+1 folded (1 + 4 = 5), small and fast to test. + let items_per_chunk = 4; // level 2 (16 items) onward splits into 4-item round chunks. + + for len in [0usize, 1, 2, 5, 6, 9, 20, 21, 25, 40] { + let items: Vec = (0..len as u64) + .map(|i| Hash256::from_low_u64_be(i + 1)) + .collect(); + let plist = ProgressiveList::::try_from(items.clone()).unwrap(); + let expected = plist.tree_hash_items_root(); + + let mut i = 0; + loop { + let Some(range) = chunk_range(len, first_chunk_size, items_per_chunk, i) else { + break; + }; + let branch = progressive_chunk_branch(&items, first_chunk_size, items_per_chunk, i); + + let reconstructed = if i == 0 { + // First chunk: recompute each folded level directly from its own data, then + // combine with the single "deeper" sibling. + let folded_items = &items[range.clone()]; + let level_roots = progressive_level_roots(&item_leaves(folded_items)); + let deeper = branch[0]; + fold_levels_from_seed(&level_roots, deeper) + } else { + // Round chunk: branch = [within-level siblings] ++ [chain siblings, + // L_{m-1}..L_0] ++ [deeper, 1 entry]. Figure out which level (m) and where + // within it this chunk sits, the same way progressive_chunk_branch does, to + // know where to split branch back apart. + let chunk_start_leaf = first_chunk_size + (i - 1) * items_per_chunk; + let level_bounds = progressive_level_bounds(len); + let (_m, &(level_start, level_size)) = level_bounds + .iter() + .enumerate() + .find(|&(_, &(start, size))| { + chunk_start_leaf >= start && chunk_start_leaf < start + size + }) + .unwrap(); + let local_start = chunk_start_leaf - level_start; + let local_len = items_per_chunk.min(level_size); + let within_len = + (local_len.trailing_zeros()..level_size.trailing_zeros()).len(); + + // Pad the chunk's real items up to its full nominal leaf width before + // hashing: the chunk's own `range` may be clipped short (trailing partial + // chunk), but the subtree it proves against is always the full nominal + // width, zero-padded, same as the production code's own padding. + let chunk_items = &items[range.clone()]; + let mut chunk_leaves = item_leaves(chunk_items); + chunk_leaves.resize(local_len, Hash256::ZERO); + let chunk_root = *flat_tree_levels(&chunk_leaves) + .last() + .unwrap() + .first() + .unwrap(); + let l_m = + fold_branch(chunk_root, local_start / local_len, &branch[..within_len]); + + let deeper = *branch.last().unwrap(); + let chain_siblings = &branch[within_len..branch.len() - 1]; // L_{m-1}..L_0 + let mut result = hash_pair(&l_m, &deeper); // Chain_m = hash(L_m, deeper) + for sib in chain_siblings { + result = hash_pair(sib, &result); + } + result + }; + + assert_eq!( + reconstructed, expected, + "len={len} chunk={i} range={range:?} mismatch" + ); + i += 1; + } + } + } + + #[test] + fn progressive_branch_reconstructs_real_items_root_packed() { + // u64 is TreeHashType::Basic with packing_factor 4, exercising the packed path that + // chunking.rs's `packing_factor` fix (see project memory) was specifically about. + let packing_factor = 4usize; + let first_chunk_size = 5 * packing_factor; // 5 leaves' worth of items (levels 0+1). + let items_per_chunk = 4 * packing_factor; // 4 leaves' worth of items per round chunk. + + for len in [ + 0usize, 1, 4, 16, 19, 20, 21, 36, 40, 64, 80, 83, 84, 85, 100, + ] { + let items: Vec = (0..len as u64).collect(); + let plist = ProgressiveList::::try_from(items.clone()).unwrap(); + let expected = plist.tree_hash_items_root(); + + let first_chunk_leaves = first_chunk_size / packing_factor; + let items_per_chunk_leaves = items_per_chunk / packing_factor; + let total_leaves = len.div_ceil(packing_factor); + + let mut i = 0; + loop { + let Some(range) = chunk_range(len, first_chunk_size, items_per_chunk, i) else { + break; + }; + let branch = progressive_chunk_branch(&items, first_chunk_size, items_per_chunk, i); + + let reconstructed = if i == 0 { + let folded_items = &items[range.clone()]; + let level_roots = progressive_level_roots(&item_leaves(folded_items)); + let deeper = branch[0]; + fold_levels_from_seed(&level_roots, deeper) + } else { + let chunk_start_leaf = first_chunk_leaves + (i - 1) * items_per_chunk_leaves; + let level_bounds = progressive_level_bounds(total_leaves); + let (_m, &(level_start, level_size)) = level_bounds + .iter() + .enumerate() + .find(|&(_, &(start, size))| { + chunk_start_leaf >= start && chunk_start_leaf < start + size + }) + .unwrap(); + let local_start = chunk_start_leaf - level_start; + let local_len = items_per_chunk_leaves.min(level_size); + let within_len = + (local_len.trailing_zeros()..level_size.trailing_zeros()).len(); + + let chunk_items = &items[range.clone()]; + let mut chunk_leaves = item_leaves(chunk_items); + chunk_leaves.resize(local_len, Hash256::ZERO); + let chunk_root = *flat_tree_levels(&chunk_leaves) + .last() + .unwrap() + .first() + .unwrap(); + let l_m = + fold_branch(chunk_root, local_start / local_len, &branch[..within_len]); + + let deeper = *branch.last().unwrap(); + let chain_siblings = &branch[within_len..branch.len() - 1]; + let mut result = hash_pair(&l_m, &deeper); + for sib in chain_siblings { + result = hash_pair(sib, &result); + } + result + }; + + assert_eq!( + reconstructed, expected, + "len={len} chunk={i} range={range:?} mismatch" + ); + i += 1; + } + } + } + + /// Same as `fold_levels_from`, but starting from an explicit seed instead of `ZERO`. + fn fold_levels_from_seed(level_roots: &[Hash256], seed: Hash256) -> Hash256 { + let mut result = seed; + for root in level_roots.iter().rev() { + result = hash_pair(root, &result); + } + result + } + + #[test] + fn chunk_data_is_concatenated_ssz_bytes() { + let items: Vec = vec![1, 2, 3, 4, 5]; + let data = chunk_data(&items, 1..4); + let expected: Vec = items[1..4].iter().flat_map(|i| i.as_ssz_bytes()).collect(); + assert_eq!(data, expected); + assert_eq!(data.len(), 3 * 8); // 3 u64s, 8 bytes each. + } +} diff --git a/consensus/types/src/beacon_state_summary/chunking.rs b/consensus/types/src/beacon_state_summary/chunking.rs new file mode 100644 index 00000000000..8269151225b --- /dev/null +++ b/consensus/types/src/beacon_state_summary/chunking.rs @@ -0,0 +1,399 @@ +use std::ops::Range; + +use crate::{beacon_state_summary::BeaconStateSummary, core::EthSpec}; + +/// The `List`/`ProgressiveList` fields of `BeaconStateSummary` that get split into +/// [`BeaconStatePart`](super::BeaconStatePart) chunks, in the same order they're declared on +/// `BeaconStateSummary` — `chunk_index` in `beacon_state_parts_by_range` is a single flat `u64` +/// with no field selector, so client and server must agree on one global numbering, and this +/// declaration order is the simplest scheme consistent with that request shape. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ChunkedField { + HistoricalRoots, + Eth1DataVotes, + Validators, + Balances, + PreviousEpochParticipation, + CurrentEpochParticipation, + InactivityScores, + HistoricalSummaries, + PendingDeposits, + PendingPartialWithdrawals, + PendingConsolidations, + Builders, + BuilderPendingWithdrawals, + PayloadExpectedWithdrawals, +} + +impl ChunkedField { + /// All chunked fields, in `BeaconStateSummary`'s declaration order. + pub const ALL: [ChunkedField; 14] = [ + ChunkedField::HistoricalRoots, + ChunkedField::Eth1DataVotes, + ChunkedField::Validators, + ChunkedField::Balances, + ChunkedField::PreviousEpochParticipation, + ChunkedField::CurrentEpochParticipation, + ChunkedField::InactivityScores, + ChunkedField::HistoricalSummaries, + ChunkedField::PendingDeposits, + ChunkedField::PendingPartialWithdrawals, + ChunkedField::PendingConsolidations, + ChunkedField::Builders, + ChunkedField::BuilderPendingWithdrawals, + ChunkedField::PayloadExpectedWithdrawals, + ]; + + /// Max items per chunk for this field: the largest power of two `P` such that + /// `P * ssz_fixed_len(item) <= 0.5 MiB`, targeting Etan's `<0.5MB` per-chunk constraint. + /// + /// Confirmed against Etan's three known values (`validators` = 2^12, `balances` = 2^16, + /// `previous_epoch_participation` = 2^19) using each item type's real SSZ-encoded size from + /// this crate — all three match this formula exactly. The rest of the table is derived the + /// same way and hasn't been independently confirmed with Etan. + pub const fn items_per_chunk(self) -> usize { + match self { + ChunkedField::HistoricalRoots => 1 << 14, // Hash256, 32B + ChunkedField::Eth1DataVotes => 1 << 12, // Eth1Data, 72B + ChunkedField::Validators => 1 << 12, // Validator, 121B + ChunkedField::Balances => 1 << 16, // u64, 8B + ChunkedField::PreviousEpochParticipation => 1 << 19, // ParticipationFlags, 1B + ChunkedField::CurrentEpochParticipation => 1 << 19, // ParticipationFlags, 1B + ChunkedField::InactivityScores => 1 << 16, // u64, 8B + ChunkedField::HistoricalSummaries => 1 << 13, // HistoricalSummary, 64B + ChunkedField::PendingDeposits => 1 << 11, // PendingDeposit, 192B + ChunkedField::PendingPartialWithdrawals => 1 << 14, // PendingPartialWithdrawal, 24B + ChunkedField::PendingConsolidations => 1 << 15, // PendingConsolidation, 16B + ChunkedField::Builders => 1 << 12, // Builder, 93B + ChunkedField::BuilderPendingWithdrawals => 1 << 13, // BuilderPendingWithdrawal, 36B + ChunkedField::PayloadExpectedWithdrawals => 1 << 13, // Withdrawal, 44B + } + } + + /// SSZ tree-hash packing factor of this field's item type: how many items share one 32-byte + /// merkle leaf. `1` for `Container`/`List`/`Vector`-typed items (`Validator`, `Eth1Data`, + /// etc., which never pack), `4` for `u64` items (`Balances`, `InactivityScores`), `32` for + /// single-byte items (`ParticipationFlags`, which wraps a `u8`). Matches each type's real + /// `TreeHash::tree_hash_packing_factor()`. + /// + /// This matters for [`Self::first_chunk_size`]: the real `ProgressiveList` tree's levels + /// (`4^i`) are sized in merkle *leaves*, not items, so for a packed field the item-level fold + /// boundary has to be computed in leaf units first, then converted back, or it lands on the + /// wrong item index whenever the packing factor doesn't evenly divide the naive item-unit + /// fold (it doesn't, in general). + pub const fn packing_factor(self) -> usize { + match self { + ChunkedField::Balances | ChunkedField::InactivityScores => 4, // u64 + ChunkedField::PreviousEpochParticipation | ChunkedField::CurrentEpochParticipation => { + 32 // ParticipationFlags (wraps u8) + } + _ => 1, // Container-typed items never pack; plain-List fields don't use this anyway. + } + } + + /// Whether this field is backed by a `ProgressiveList` (EIP-7916) in `BeaconStateGloas`, as + /// opposed to a plain `List`. Checked directly against `consensus/types/src/state/beacon_state.rs`'s + /// Gloas field declarations: `historical_roots`, `eth1_data_votes`, and `historical_summaries` + /// stay as plain `List`; every other chunked field is `ProgressiveList`. This matters because + /// a plain `List`'s tree has no progressive folding, it's one flat `merkleize(pack(items), + /// limit)` tree, so a uniform `items_per_chunk`-sized stride starting at item 0 already + /// produces valid single-subtree chunks for those three fields; the irregular first chunk + /// below is only needed, and only correct, for the `ProgressiveList` fields. + pub const fn is_progressive(self) -> bool { + !matches!( + self, + ChunkedField::HistoricalRoots + | ChunkedField::Eth1DataVotes + | ChunkedField::HistoricalSummaries + ) + } + + /// Number of items in this field's first chunk. + /// + /// A `ProgressiveList`'s tree is a chain of perfectly-balanced binary subtrees, one per + /// level, with level `i` holding `4^i` items (1, 4, 16, 64, 256, ...). A uniform + /// `items_per_chunk`-sized stride only produces valid single-subtree chunks once it's + /// aligned to these level boundaries, so the first chunk is special: it folds together every + /// level that fits completely within `items_per_chunk` (e.g. for `items_per_chunk = 32`, + /// levels 0-2 sum to `1 + 4 + 16 = 21`), and is exactly that folded subtree's root. Every + /// level from there on is itself at least as large as `items_per_chunk` and an exact + /// power-of-two multiple of it, so every later chunk can just be a plain, round + /// `items_per_chunk`-sized slice, no further special-casing needed. Confirmed directly with + /// Etan Kissling (2026-10-06), who gave this exact rule. + /// + /// For a plain `List` field (see [`Self::is_progressive`]), there's no progressive folding to + /// account for, so the first chunk is just a normal, round `items_per_chunk`-sized slice like + /// every other chunk. + /// + /// The fold itself is computed in leaf units (dividing out [`Self::packing_factor`]) and + /// converted back to items at the end, since the real tree's levels are leaf-sized, not + /// item-sized (see [`Self::packing_factor`]'s docs). For unpacked fields (`packing_factor == + /// 1`) this is a no-op and matches the original item-unit fold exactly. + pub const fn first_chunk_size(self) -> usize { + if self.is_progressive() { + let packing_factor = self.packing_factor(); + let leaves_per_chunk = match self.items_per_chunk().checked_div(packing_factor) { + Some(leaves) => leaves, + None => 0, + }; + first_chunk_size_for(leaves_per_chunk).saturating_mul(packing_factor) + } else { + self.items_per_chunk() + } + } + + /// Number of chunks a field with `len` items splits into (0 items → 0 chunks). + /// + /// The first chunk covers `0..first_chunk_size()` (or the whole field, if it's shorter than + /// that); every chunk after that is a plain `items_per_chunk()`-sized slice. + pub fn num_chunks(self, len: usize) -> usize { + if len == 0 { + return 0; + } + let first_chunk_size = self.first_chunk_size(); + if len <= first_chunk_size { + 1 + } else { + 1_usize.saturating_add( + len.saturating_sub(first_chunk_size) + .div_ceil(self.items_per_chunk()), + ) + } + } + + /// Item-index range covered by the given 0-based chunk index within this field, given the + /// field's actual `len`. Returns `None` if `local_chunk_index >= num_chunks(len)`. + pub fn chunk_range(self, len: usize, local_chunk_index: usize) -> Option> { + if local_chunk_index >= self.num_chunks(len) { + return None; + } + let first_chunk_size = self.first_chunk_size(); + if local_chunk_index == 0 { + Some(0..first_chunk_size.min(len)) + } else { + let items_per_chunk = self.items_per_chunk(); + let start = first_chunk_size.saturating_add( + local_chunk_index + .saturating_sub(1) + .saturating_mul(items_per_chunk), + ); + let end = start.saturating_add(items_per_chunk).min(len); + Some(start..end) + } + } +} + +/// The largest `k` such that `4^0 + 4^1 + ... + 4^(k-1) <= items_per_chunk`, returning that sum. +const fn first_chunk_size_for(items_per_chunk: usize) -> usize { + let mut cumulative = 0usize; + let mut level_size = 1usize; + while cumulative.saturating_add(level_size) <= items_per_chunk { + cumulative = cumulative.saturating_add(level_size); + level_size = level_size.saturating_mul(4); + } + cumulative +} + +/// Maps a flat, global `chunk_index` (as used by `beacon_state_parts_by_range`) to the specific +/// field and item-index range within that field the chunk covers, given the actual lengths of +/// every chunked field (from a [`BeaconStateSummary`]'s `ListSummary`s). +/// +/// Chunks are laid out as a flat concatenation of each field's chunks, in [`ChunkedField::ALL`] +/// order: all of `historical_roots`'s chunks, then all of `eth1_data_votes`'s, and so on. Within +/// a single field, the first chunk is the irregular [`ChunkedField::first_chunk_size`]-sized one +/// and every chunk after that is a round [`ChunkedField::items_per_chunk`]-sized slice. +#[derive(Debug, Clone)] +pub struct ChunkLayout { + /// `(field, len, num_chunks, first_global_index)` for each chunked field, in declaration + /// order. `len` is kept so `locate` can clip the last chunk to the field's actual length. + per_field: Vec<(ChunkedField, usize, usize, u64)>, + total_chunks: u64, +} + +impl ChunkLayout { + /// Build a layout from each chunked field's length, in [`ChunkedField::ALL`] order. + pub fn new(lengths: [usize; 14]) -> Self { + let mut per_field = Vec::with_capacity(ChunkedField::ALL.len()); + let mut next_chunk_index: u64 = 0; + for (field, len) in ChunkedField::ALL.into_iter().zip(lengths) { + let num_chunks = field.num_chunks(len); + per_field.push((field, len, num_chunks, next_chunk_index)); + next_chunk_index = next_chunk_index.saturating_add(num_chunks as u64); + } + Self { + per_field, + total_chunks: next_chunk_index, + } + } + + /// Build a layout directly from a [`BeaconStateSummary`]'s `ListSummary` lengths. + pub fn from_summary(summary: &BeaconStateSummary) -> Self { + Self::new([ + summary.historical_roots.num_items as usize, + summary.eth1_data_votes.num_items as usize, + summary.validators.num_items as usize, + summary.balances.num_items as usize, + summary.previous_epoch_participation.num_items as usize, + summary.current_epoch_participation.num_items as usize, + summary.inactivity_scores.num_items as usize, + summary.historical_summaries.num_items as usize, + summary.pending_deposits.num_items as usize, + summary.pending_partial_withdrawals.num_items as usize, + summary.pending_consolidations.num_items as usize, + summary.builders.num_items as usize, + summary.builder_pending_withdrawals.num_items as usize, + summary.payload_expected_withdrawals.num_items as usize, + ]) + } + + /// Total number of chunks across every field, for validating a `(start_chunk, count)` request. + pub fn total_chunks(&self) -> u64 { + self.total_chunks + } + + /// Resolve a global `chunk_index` to the field it belongs to and the item-index range within + /// that field this chunk covers. Returns `None` if `chunk_index >= total_chunks()`. + pub fn locate(&self, chunk_index: u64) -> Option<(ChunkedField, Range)> { + for &(field, len, num_chunks, first_global_index) in &self.per_field { + let num_chunks = num_chunks as u64; + if chunk_index < first_global_index.saturating_add(num_chunks) { + let local_chunk_index = chunk_index.saturating_sub(first_global_index) as usize; + return field + .chunk_range(len, local_chunk_index) + .map(|range| (field, range)); + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn items_per_chunk_matches_etan_known_values() { + assert_eq!(ChunkedField::Validators.items_per_chunk(), 1 << 12); + assert_eq!(ChunkedField::Balances.items_per_chunk(), 1 << 16); + assert_eq!( + ChunkedField::PreviousEpochParticipation.items_per_chunk(), + 1 << 19 + ); + } + + #[test] + fn empty_field_has_zero_chunks() { + assert_eq!(ChunkedField::Balances.num_chunks(0), 0); + } + + #[test] + fn first_chunk_size_matches_etans_example() { + // Etan's own example (2026-10-06): items_per_chunk = 32 folds levels 0-2 + // (1 + 4 + 16 = 21) into the first chunk, since level 3 (64 items) doesn't fit. + assert_eq!(first_chunk_size_for(32), 21); + } + + #[test] + fn first_chunk_size_accounts_for_packing() { + // Balances is u64 (packing_factor = 4), items_per_chunk = 65536 items = 16384 leaves. + // Folding the leaf sequence (1, 4, 16, 64, 256, 1024, 4096) sums to 5461 leaves <= + // 16384; adding the next level (16384 leaves) would overflow it. So the fold stops at + // 5461 leaves, which is 5461 * 4 = 21844 items, NOT 21845 (the item-unit-only fold this + // field would get if packing were ignored, since 65536 items folded directly in item + // units gives the same numeric sequence coincidentally but 1 item off here). + assert_eq!(ChunkedField::Balances.packing_factor(), 4); + assert_eq!(ChunkedField::Balances.first_chunk_size(), 21844); + + // ParticipationFlags is a 1-byte type (packing_factor = 32), items_per_chunk = 524288 + // items = 16384 leaves, so it folds identically to Balances in leaf terms: 5461 leaves, + // but converts to 5461 * 32 = 174752 items. + assert_eq!( + ChunkedField::PreviousEpochParticipation.packing_factor(), + 32 + ); + assert_eq!( + ChunkedField::PreviousEpochParticipation.first_chunk_size(), + 174752 + ); + + // Container-typed fields (packing_factor = 1) are unaffected: the leaf-unit fold and the + // item-unit fold are the same computation. + assert_eq!(ChunkedField::Validators.packing_factor(), 1); + assert_eq!( + ChunkedField::Validators.first_chunk_size(), + first_chunk_size_for(ChunkedField::Validators.items_per_chunk()) + ); + } + + #[test] + fn plain_list_fields_have_no_irregular_first_chunk() { + // historical_roots, eth1_data_votes, and historical_summaries stay as a plain `List` in + // BeaconStateGloas (checked directly against beacon_state.rs), not `ProgressiveList`, so + // they have no progressive folding to account for: the first chunk should just be a + // normal, round items_per_chunk-sized slice, same as every other chunk. + assert!(!ChunkedField::HistoricalRoots.is_progressive()); + assert!(!ChunkedField::Eth1DataVotes.is_progressive()); + assert!(!ChunkedField::HistoricalSummaries.is_progressive()); + assert_eq!( + ChunkedField::HistoricalRoots.first_chunk_size(), + ChunkedField::HistoricalRoots.items_per_chunk() + ); + + // Every other chunked field is `ProgressiveList` and does need the irregular first chunk. + assert!(ChunkedField::Validators.is_progressive()); + assert!( + ChunkedField::Validators.first_chunk_size() + < ChunkedField::Validators.items_per_chunk() + ); + } + + #[test] + fn first_chunk_size_for_validators() { + // P = 4096: levels 0-5 (1+4+16+64+256+1024 = 1365) fit; level 6 (4096) doesn't. + assert_eq!(ChunkedField::Validators.first_chunk_size(), 1365); + } + + #[test] + fn num_chunks_accounts_for_irregular_first_chunk() { + let fc = ChunkedField::Validators.first_chunk_size(); + let per_chunk = ChunkedField::Validators.items_per_chunk(); + assert_eq!(ChunkedField::Validators.num_chunks(fc), 1); + assert_eq!(ChunkedField::Validators.num_chunks(fc + 1), 2); + assert_eq!(ChunkedField::Validators.num_chunks(fc + per_chunk), 2); + assert_eq!(ChunkedField::Validators.num_chunks(fc + per_chunk + 1), 3); + } + + #[test] + fn layout_locates_chunks_across_fields() { + // historical_roots: 0 items -> 0 chunks. + // eth1_data_votes: 1 item -> 1 chunk, global index 0, entirely within the first chunk. + // validators: first_chunk_size + items_per_chunk + 1 items -> 3 chunks, global indices + // 1, 2, 3: the irregular first chunk, one full round chunk, and one trailing 1-item + // chunk clipped to the field's actual length. + // everything else: 0 items -> 0 chunks. + let fc = ChunkedField::Validators.first_chunk_size(); + let per_chunk = ChunkedField::Validators.items_per_chunk(); + let validators_len = fc + per_chunk + 1; + let lengths = [0, 1, validators_len, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let layout = ChunkLayout::new(lengths); + assert_eq!(layout.total_chunks(), 4); + + let (field, range) = layout.locate(0).unwrap(); + assert_eq!(field, ChunkedField::Eth1DataVotes); + assert_eq!(range, 0..1); + + let (field, range) = layout.locate(1).unwrap(); + assert_eq!(field, ChunkedField::Validators); + assert_eq!(range, 0..fc); + + let (field, range) = layout.locate(2).unwrap(); + assert_eq!(field, ChunkedField::Validators); + assert_eq!(range, fc..(fc + per_chunk)); + + let (field, range) = layout.locate(3).unwrap(); + assert_eq!(field, ChunkedField::Validators); + assert_eq!(range, (fc + per_chunk)..validators_len); + + assert!(layout.locate(4).is_none()); + } +} diff --git a/consensus/types/src/beacon_state_summary/list_summary.rs b/consensus/types/src/beacon_state_summary/list_summary.rs new file mode 100644 index 00000000000..eb088ef4c19 --- /dev/null +++ b/consensus/types/src/beacon_state_summary/list_summary.rs @@ -0,0 +1,56 @@ +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::TreeHash; + +use crate::core::Hash256; + +/// Summary of an SSZ `List[T, N]` field: its merkleization root and length. +/// +/// Used by `BeaconStateSummary` to stand in for `List` fields of `BeaconState` while preserving +/// the same `hash_tree_root`. +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Default, Hash, Serialize, Deserialize, Encode, Decode, + TreeHash, +)] +pub struct ListSummary { + pub items_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub num_items: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + use tree_hash::TreeHash; + + ssz_and_tree_hash_tests!(ListSummary); + + /// `ListSummary { items_root, num_items }.tree_hash_root()` must exactly equal + /// `mix_in_length(items_root, num_items)` — the same final step every real SSZ `List`'s + /// `hash_tree_root` uses. This is the mathematical claim `ListSummary` depends on: a 2-field + /// container merkleizes as `hash(field0_root, field1_root)`, which is bit-for-bit the same + /// computation as `mix_in_length`'s `hash(items_root, length_as_a_chunk)`. + #[test] + fn matches_mix_in_length() { + let cases = [ + (Hash256::ZERO, 0u64), + (Hash256::repeat_byte(0xab), 1), + (Hash256::repeat_byte(0xff), 12_345), + (Hash256::repeat_byte(0x11), u64::MAX), + ]; + + for (items_root, num_items) in cases { + let summary = ListSummary { + items_root, + num_items, + }; + let expected = tree_hash::mix_in_length(&items_root, num_items as usize); + assert_eq!( + summary.tree_hash_root(), + expected, + "items_root={items_root:?}, num_items={num_items}" + ); + } + } +} diff --git a/consensus/types/src/beacon_state_summary/mod.rs b/consensus/types/src/beacon_state_summary/mod.rs new file mode 100644 index 00000000000..8f8026cbb52 --- /dev/null +++ b/consensus/types/src/beacon_state_summary/mod.rs @@ -0,0 +1,12 @@ +mod beacon_state_part; +mod beacon_state_snapshot; +mod beacon_state_summary; +mod chunk_proof; +mod chunking; +mod list_summary; + +pub use beacon_state_part::BeaconStatePart; +pub use beacon_state_snapshot::BeaconStateSnapshot; +pub use beacon_state_summary::BeaconStateSummary; +pub use chunking::{ChunkLayout, ChunkedField}; +pub use list_summary::ListSummary; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index f6d94331dee..85575215368 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -13,6 +13,7 @@ pub mod test_utils; pub mod attestation; +pub mod beacon_state_summary; pub mod block; pub mod builder; pub mod consolidation; @@ -32,6 +33,7 @@ pub mod withdrawal; // Temporary root level exports to maintain backwards compatibility for Lighthouse. pub use attestation::*; +pub use beacon_state_summary::*; pub use block::*; pub use builder::*; pub use consolidation::*;