diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index ac83d85fe..c10dbdbb3 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4705,6 +4705,12 @@ pub fn update_latest_messages( let beacon_block_root = data.beacon_block_root; let payload_present = rules == ForkRules::Gloas && data.index == 1; + // One reservation for the whole batch: an indexed attestation's indices + // are sorted, and growing the table one ascending index at a time would + // copy it many times over. + if let Some(&highest) = attesting_indices.iter().max() { + store.reserve_latest_messages(highest); + } for &index in attesting_indices { if store.is_equivocating(index) { continue; diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 14ac9116f..b6c776018 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -680,6 +680,42 @@ impl GossipSignatureBuffer { /// per index up to the highest seen. const MAX_PLAUSIBLE_VALIDATOR_INDEX: u64 = 1 << 26; +/// Spare entries [`Store::set_latest_message`] reserves past the index that +/// outgrew the dense vote table, so a registry growing by a few validators an +/// epoch reallocates the table rarely rather than on every new index. +const LATEST_MESSAGES_HEADROOM: usize = 1 << 12; + +/// The dense vote table's slot for validator `index`. +/// +/// The assertion is the tripwire for an index that did not come from a +/// validated attestation; see [`Store::set_latest_message`]. +fn latest_message_slot(index: u64) -> usize { + debug_assert!( + index < MAX_PLAUSIBLE_VALIDATOR_INDEX, + "validator index {index} would grow the dense vote table far past any registry" + ); + usize::try_from(index).expect("validator index fits in usize") +} + +/// Grows the vote table's capacity to hold `slot`, plus +/// [`LATEST_MESSAGES_HEADROOM`] spare entries; does nothing when it already +/// holds `slot`. +/// +/// Exact growth plus headroom rather than `resize`'s amortized doubling: +/// successive attestations reference validators in no particular order, so the +/// table is first sized by one high index and then doubled by a slightly higher one, +/// leaving about half of a registry-sized table unused for the life of the +/// process. +fn reserve_latest_message_slot(messages: &mut Vec>, slot: usize) { + if messages.capacity() <= slot { + let target = slot + .checked_add(1 + LATEST_MESSAGES_HEADROOM) + .expect("validator index plus headroom fits in usize"); + // `len <= capacity <= slot < target`, so this cannot underflow. + messages.reserve_exact(target - messages.len()); + } +} + /// How many epoch-boundary [`JustifiedBalances`] snapshots the store keeps. const JUSTIFIED_BALANCES_CAPACITY: usize = 8; @@ -4268,17 +4304,33 @@ impl Store { /// attestation (a member of a committee of the registry), never from raw /// input: a wild index would allocate a table to match. The assertion is /// the tripwire for that in debug builds and fixtures. + /// + /// A caller recording a batch should call + /// [`Store::reserve_latest_messages`] first. pub fn set_latest_message(&mut self, index: u64, message: LatestMessage) { - debug_assert!( - index < MAX_PLAUSIBLE_VALIDATOR_INDEX, - "validator index {index} would grow the dense vote table far past any registry" - ); - let slot = usize::try_from(index).expect("validator index fits in usize"); + let slot = latest_message_slot(index); let mut beacon = self.beacon.lock().unwrap(); - if beacon.latest_messages.len() <= slot { - beacon.latest_messages.resize(slot + 1, None); + let messages = &mut beacon.latest_messages; + if messages.len() <= slot { + reserve_latest_message_slot(messages, slot); + // Capacity now exceeds `slot`, so this cannot reallocate. + messages.resize(slot + 1, None); } - beacon.latest_messages[slot] = Some(message); + messages[slot] = Some(message); + } + + /// Makes room in the vote table for every index up to `highest`, so the + /// batch of [`Store::set_latest_message`] calls that follows reallocates + /// the table once at most. + /// + /// An aggregate lists its attesters in ascending index order. Grown one + /// index at a time, the table would be copied whenever an index passed the + /// headroom: hundreds of copies of a table sized to the registry, for a + /// single aggregate that spans it. + pub fn reserve_latest_messages(&mut self, highest: u64) { + let slot = latest_message_slot(highest); + let mut beacon = self.beacon.lock().unwrap(); + reserve_latest_message_slot(&mut beacon.latest_messages, slot); } /// Gloas: the payload timeliness committee's per-member votes on whether @@ -9115,6 +9167,76 @@ mod tests { assert_eq!(store.latest_message(u64::MAX), None); } + #[test] + fn the_latest_message_table_grows_with_headroom_rather_than_doubling() { + let mut store = Store::test_store(); + let message = LatestMessage { + epoch: 1, + slot: SLOTS_PER_EPOCH, + root: H256::from([1u8; 32]), + payload_present: false, + }; + let capacity = |store: &Store| store.beacon.lock().unwrap().latest_messages.capacity(); + + let headroom = u64::try_from(LATEST_MESSAGES_HEADROOM).unwrap(); + let slot = |index: u64| usize::try_from(index).unwrap(); + + // The first vote sizes the table to its index plus the headroom. + let first = 100_000; + store.set_latest_message(first, message); + let sized = capacity(&store); + assert!(sized > slot(first)); + assert!(sized <= slot(first) + 1 + LATEST_MESSAGES_HEADROOM); + + // The highest index inside the headroom fits without a reallocation. + store.set_latest_message(first + headroom, message); + assert_eq!(capacity(&store), sized); + + // Past the headroom, the table grows to the new index plus headroom. + let higher = first + headroom + 1_000; + store.set_latest_message(higher, message); + let grown = capacity(&store); + assert!(grown > slot(higher)); + assert!(grown <= slot(higher) + 1 + LATEST_MESSAGES_HEADROOM); + assert_eq!(store.latest_message(higher).map(|vote| vote.epoch), Some(1)); + } + + #[test] + fn a_reserved_batch_of_ascending_votes_never_moves_the_table() { + let mut store = Store::test_store(); + let message = LatestMessage { + epoch: 1, + slot: SLOTS_PER_EPOCH, + root: H256::from([1u8; 32]), + payload_present: false, + }; + let table = |store: &Store| { + let beacon = store.beacon.lock().unwrap(); + ( + beacon.latest_messages.as_ptr(), + beacon.latest_messages.capacity(), + ) + }; + + // An aggregate's attesters, ascending and spread wider apart than + // the headroom: grown index by index, each would reallocate. + let stride = u64::try_from(LATEST_MESSAGES_HEADROOM).unwrap() + 1; + let attesters: Vec = (1..=64).map(|i| i * stride).collect(); + let highest = *attesters.last().unwrap(); + + store.reserve_latest_messages(highest); + let reserved = table(&store); + assert!(reserved.1 > usize::try_from(highest).unwrap()); + for &index in &attesters { + store.set_latest_message(index, message); + assert_eq!(table(&store), reserved, "index {index} moved the table"); + } + assert_eq!( + store.latest_message(highest).map(|vote| vote.epoch), + Some(1) + ); + } + #[test] fn a_pow_block_is_looked_up_by_its_own_hash() { let mut store = Store::test_store();