From bd59c4d1179cc39483131416b418dbd4b07917ca Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Mon, 23 Sep 2024 21:20:45 +0800 Subject: [PATCH 01/16] integration everything --- fendermint/crypto/src/lib.rs | 2 +- fendermint/vm/topdown/Cargo.toml | 3 +- fendermint/vm/topdown/src/sync/mod.rs | 5 +- fendermint/vm/topdown/src/vote/error.rs | 25 +++ fendermint/vm/topdown/src/vote/gossip.rs | 17 ++ fendermint/vm/topdown/src/vote/mod.rs | 180 +++++++++++++----- .../vm/topdown/src/vote/operation/active.rs | 21 +- .../vm/topdown/src/vote/operation/mod.rs | 30 +-- .../vm/topdown/src/vote/operation/paused.rs | 24 ++- fendermint/vm/topdown/src/vote/payload.rs | 1 + fendermint/vm/topdown/src/vote/store.rs | 13 +- fendermint/vm/topdown/src/vote/tally.rs | 17 +- 12 files changed, 240 insertions(+), 98 deletions(-) create mode 100644 fendermint/vm/topdown/src/vote/error.rs create mode 100644 fendermint/vm/topdown/src/vote/gossip.rs diff --git a/fendermint/crypto/src/lib.rs b/fendermint/crypto/src/lib.rs index b73192e73..10a69b1f3 100644 --- a/fendermint/crypto/src/lib.rs +++ b/fendermint/crypto/src/lib.rs @@ -9,7 +9,7 @@ use base64::{alphabet, Engine}; use rand::Rng; use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; -pub use libsecp256k1::PublicKey; +pub use libsecp256k1::{PublicKey, RecoveryId, Signature}; /// A [`GeneralPurpose`] engine using the [`alphabet::STANDARD`] base64 alphabet /// padding bytes when writing but requireing no padding when reading. diff --git a/fendermint/vm/topdown/Cargo.toml b/fendermint/vm/topdown/Cargo.toml index 27a166bd3..de18ef1e9 100644 --- a/fendermint/vm/topdown/Cargo.toml +++ b/fendermint/vm/topdown/Cargo.toml @@ -30,6 +30,7 @@ thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } prometheus = { workspace = true } +arbitrary = { workspace = true } fendermint_vm_genesis = { path = "../genesis" } fendermint_vm_event = { path = "../event" } @@ -39,7 +40,7 @@ fendermint_crypto = { path = "../../crypto" } ipc-observability = { workspace = true } [dev-dependencies] -arbitrary = { workspace = true } + clap = { workspace = true } rand = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/fendermint/vm/topdown/src/sync/mod.rs b/fendermint/vm/topdown/src/sync/mod.rs index 0b456da5a..3c5d71d32 100644 --- a/fendermint/vm/topdown/src/sync/mod.rs +++ b/fendermint/vm/topdown/src/sync/mod.rs @@ -19,14 +19,13 @@ use std::time::Duration; use fendermint_vm_genesis::{Power, Validator}; +use crate::vote::payload::Vote; pub use syncer::fetch_topdown_events; #[derive(Clone)] pub enum TopDownSyncEvent { NodeSyncing, - NewParentView, - NewParentChainHead, - NewProposal, + NewProposal(Box), } /// Query the parent finality from the block chain state. diff --git a/fendermint/vm/topdown/src/vote/error.rs b/fendermint/vm/topdown/src/vote/error.rs new file mode 100644 index 000000000..aeb22de2f --- /dev/null +++ b/fendermint/vm/topdown/src/vote/error.rs @@ -0,0 +1,25 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::BlockHeight; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + #[error("the last finalized block has not been set")] + Uninitialized, + + #[error("failed to extend chain; height going backwards, current height {0}, got {1}")] + UnexpectedBlock(BlockHeight, BlockHeight), + + #[error("validator unknown or has no power")] + UnpoweredValidator, + + #[error("equivocation by validator")] + Equivocation, + + #[error("validator vote is invalidated")] + VoteCannotBeValidated, + + #[error("validator cannot sign vote")] + CannotSignVote, +} diff --git a/fendermint/vm/topdown/src/vote/gossip.rs b/fendermint/vm/topdown/src/vote/gossip.rs new file mode 100644 index 000000000..15305c05d --- /dev/null +++ b/fendermint/vm/topdown/src/vote/gossip.rs @@ -0,0 +1,17 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::vote::error::Error; +use crate::vote::payload::Vote; +use async_trait::async_trait; + +/// The gossip client communicates with the underlying gossip pub/sub network on various +/// subscribed topics. This client handles the event listening/forwarding and event publication. +#[async_trait] +pub trait GossipClient { + /// Attempts to poll if there are available vote. This method returns immediately. + /// If there is no vote, it returns None + fn try_poll_vote(&self) -> Result, Error>; + + async fn publish_vote(&self, vote: Vote) -> Result<(), Error>; +} diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index 6d9b2c6e3..99953963a 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -1,18 +1,24 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +pub mod error; +pub mod gossip; mod operation; -mod payload; -mod store; +pub mod payload; +pub mod store; mod tally; use crate::sync::TopDownSyncEvent; +use crate::vote::gossip::GossipClient; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; -use crate::vote::payload::Vote; +use crate::vote::payload::{PowerUpdates, Vote}; +use crate::vote::store::VoteStore; +use crate::vote::tally::VoteTally; use crate::BlockHeight; +use error::Error; use serde::{Deserialize, Serialize}; use std::time::Duration; -use tokio::sync::{broadcast, mpsc}; +use tokio::sync::{broadcast, mpsc, oneshot}; pub type Weight = u64; @@ -28,90 +34,157 @@ pub struct Config { voting_sleep_interval_sec: u64, } +/// The client to interact with the vote reactor pub struct VoteReactorClient { tx: mpsc::Sender, } -pub fn start_vote_reactor( +pub fn start_vote_reactor( config: Config, - gossip_rx: broadcast::Receiver, - gossip_tx: mpsc::Sender, + power_table: PowerUpdates, + last_finalized_height: BlockHeight, + gossip: G, + vote_store: V, internal_event_listener: broadcast::Receiver, -) -> VoteReactorClient { +) -> anyhow::Result { let (tx, rx) = mpsc::channel(config.req_channel_buffer_size); + let vote_tally = VoteTally::new(power_table, last_finalized_height, vote_store)?; tokio::spawn(async move { let sleep = Duration::new(config.voting_sleep_interval_sec, 0); let inner = VotingHandler { req_rx: rx, - gossip_rx, - gossip_tx, internal_event_listener, + vote_tally, config, + gossip, }; let mut machine = OperationStateMachine::new(inner); loop { - machine = machine.step(); + machine = machine.step().await; tokio::time::sleep(sleep).await; } }); - VoteReactorClient { tx } + Ok(VoteReactorClient { tx }) } -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum Error { - #[error("the last finalized block has not been set")] - Uninitialized, - - #[error("failed to extend chain; height going backwards, current height {0}, got {1}")] - UnexpectedBlock(BlockHeight, BlockHeight), - - #[error("validator unknown or has no power")] - UnpoweredValidator, - - #[error("equivocation by validator")] - Equivocation, +impl VoteReactorClient { + pub async fn query_operation_mode(&self) -> anyhow::Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(VoteReactorRequest::QueryOperationMode(tx)) + .await?; + Ok(rx.await?) + } - #[error("validator vote is invalidated")] - VoteCannotBeValidated, + pub async fn query_votes( + &self, + height: BlockHeight, + ) -> anyhow::Result, Error>> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(VoteReactorRequest::QueryVotes { + height, + reply_tx: tx, + }) + .await?; + Ok(rx.await?) + } - #[error("validator cannot sign vote")] - CannotSignVote, + pub async fn update_power_table(&self, updates: PowerUpdates) -> anyhow::Result<()> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(VoteReactorRequest::UpdatePowerTable { + updates, + reply_tx: tx, + }) + .await?; + Ok(rx.await?) + } } enum VoteReactorRequest { - QueryOperationMode, - QueryVotes(BlockHeight), + QueryOperationMode(oneshot::Sender), + QueryVotes { + height: BlockHeight, + reply_tx: oneshot::Sender, Error>>, + }, + UpdatePowerTable { + updates: PowerUpdates, + reply_tx: oneshot::Sender<()>, + }, } -struct VotingHandler { +struct VotingHandler { /// Handles the requests targeting the vote reactor, could be querying the /// vote tally status and etc. req_rx: mpsc::Receiver, - /// Receiver from gossip pub/sub, mostly listening to incoming votes - gossip_rx: broadcast::Receiver, - gossip_tx: mpsc::Sender, + /// Interface to gossip pub/sub for topdown voting + gossip: Gossip, /// Listens to internal events and handles the events accordingly internal_event_listener: broadcast::Receiver, + vote_tally: VoteTally, + config: Config, } -impl VotingHandler { - fn handle_request(&self, _req: VoteReactorRequest) {} +impl VotingHandler +where + G: GossipClient + Send + Sync + 'static, + V: VoteStore + Send + Sync + 'static, +{ + fn handle_request(&mut self, req: VoteReactorRequest, metrics: &OperationMetrics) { + match req { + VoteReactorRequest::QueryOperationMode(req_tx) => { + // ignore error + let _ = req_tx.send(metrics.clone()); + } + VoteReactorRequest::QueryVotes { height, reply_tx } => { + let _ = reply_tx.send(self.vote_tally.get_votes_at_height(height)); + } + VoteReactorRequest::UpdatePowerTable { updates, reply_tx } => { + self.vote_tally.update_power_table(updates); + let _ = reply_tx.send(()); + } + } + } - fn record_vote(&self, _vote: Vote) {} + async fn handle_event(&mut self, event: TopDownSyncEvent) { + match event { + TopDownSyncEvent::NewProposal(vote) => { + if let Err(e) = self.vote_tally.add_vote(*vote.clone()) { + tracing::error!(err = e.to_string(), "cannot self vote to tally"); + return; + } - fn handle_event(&self, _event: TopDownSyncEvent) {} + match self.gossip.publish_vote(*vote).await { + Ok(_) => {} + Err(e) => { + tracing::error!( + err = e.to_string(), + "cannot send to gossip sender, tx dropped" + ); + + // when this happens, we still keep the vote tally going as + // we can still receive other peers's votes. + } + } + } + _ => { + // ignore events we are not interested in + } + }; + } /// Process external request, such as RPC queries for debugging and status tracking. - fn process_external_request(&mut self, _metrics: &OperationMetrics) -> usize { + fn process_external_request(&mut self, metrics: &OperationMetrics) -> usize { let mut n = 0; while n < self.config.req_batch_processing_size { match self.req_rx.try_recv() { Ok(req) => { - self.handle_request(req); + self.handle_request(req, metrics); n += 1 } Err(mpsc::error::TryRecvError::Disconnected) => { @@ -126,21 +199,24 @@ impl VotingHandler { /// Handles vote tally gossip pab/sub incoming votes from other peers fn process_gossip_subscription_votes(&mut self) -> usize { - let mut n = 0; - while n < self.config.gossip_req_processing_size { - match self.gossip_rx.try_recv() { - Ok(vote) => { - self.record_vote(vote); - n += 1; + let mut vote_processed = 0; + while vote_processed < self.config.gossip_req_processing_size { + match self.gossip.try_poll_vote() { + Ok(Some(vote)) => { + if let Err(e) = self.vote_tally.add_vote(vote) { + tracing::error!(err = e.to_string(), "cannot add vote to tally"); + } else { + vote_processed += 1; + } } - Err(broadcast::error::TryRecvError::Empty) => break, - _ => { - tracing::warn!("gossip sender lagging or closed"); + Err(e) => { + tracing::warn!(err = e.to_string(), "cannot poll gossip vote"); break; - } + }, + _ => {} } } - n + vote_processed } /// Poll internal topdown syncer event broadcasted. diff --git a/fendermint/vm/topdown/src/vote/operation/active.rs b/fendermint/vm/topdown/src/vote/operation/active.rs index c70497fa8..96feefb8e 100644 --- a/fendermint/vm/topdown/src/vote/operation/active.rs +++ b/fendermint/vm/topdown/src/vote/operation/active.rs @@ -3,28 +3,33 @@ use crate::vote::operation::paused::PausedOperationMode; use crate::vote::operation::{ - OperationMetrics, OperationModeHandler, OperationStateMachine, PAUSED, + OperationMetrics, OperationStateMachine, ACTIVE, PAUSED, }; use crate::vote::TopDownSyncEvent; use crate::vote::VotingHandler; use std::fmt::{Display, Formatter}; +use crate::vote::gossip::GossipClient; +use crate::vote::store::VoteStore; /// In active mode, we observe a steady rate of topdown checkpoint commitments on chain. /// Our lookahead buffer is sliding continuously. As we acquire new finalised parent blocks, /// we broadcast individual signed votes for every epoch. -pub(crate) struct ActiveOperationMode { +pub(crate) struct ActiveOperationMode { pub(crate) metrics: OperationMetrics, - pub(crate) handler: VotingHandler, + pub(crate) handler: VotingHandler, } -impl Display for ActiveOperationMode { +impl Display for ActiveOperationMode { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "active") + write!(f, "{}", ACTIVE) } } -impl OperationModeHandler for ActiveOperationMode { - fn advance(mut self) -> OperationStateMachine { +impl ActiveOperationMode where + G: GossipClient + Send + Sync + 'static, + S: VoteStore + Send + Sync + 'static, +{ + pub(crate) async fn advance(mut self) -> OperationStateMachine { let mut n = self.handler.process_external_request(&self.metrics); tracing::debug!( num = n, @@ -50,7 +55,7 @@ impl OperationModeHandler for ActiveOperationMode { } // handle the polled event - self.handler.handle_event(v); + self.handler.handle_event(v).await; } OperationStateMachine::Active(self) diff --git a/fendermint/vm/topdown/src/vote/operation/mod.rs b/fendermint/vm/topdown/src/vote/operation/mod.rs index 578c4c30e..c45fb9515 100644 --- a/fendermint/vm/topdown/src/vote/operation/mod.rs +++ b/fendermint/vm/topdown/src/vote/operation/mod.rs @@ -4,10 +4,11 @@ mod active; mod paused; +use crate::vote::gossip::GossipClient; use crate::vote::operation::active::ActiveOperationMode; use crate::vote::operation::paused::PausedOperationMode; +use crate::vote::store::VoteStore; use crate::vote::VotingHandler; -use std::fmt::Display; pub type OperationMode = &'static str; pub const INITIALIZED: &str = "init"; @@ -34,24 +35,25 @@ pub const ACTIVE: &str = "active"; /// HardRecovery --> [*] : New checkpoints /// } /// TODO: Soft and Hard recovery mode to be added -pub enum OperationStateMachine { - Paused(PausedOperationMode), - Active(ActiveOperationMode), +pub enum OperationStateMachine { + Paused(PausedOperationMode), + Active(ActiveOperationMode), } /// Tracks the operation mdoe metrics for the voting system -pub(crate) struct OperationMetrics { +#[derive(Clone, Debug)] +pub struct OperationMetrics { current_mode: OperationMode, previous_mode: OperationMode, } -pub(crate) trait OperationModeHandler: Display { - fn advance(self) -> OperationStateMachine; -} - -impl OperationStateMachine { +impl OperationStateMachine +where + G: GossipClient + Send + Sync + 'static, + S: VoteStore + Send + Sync + 'static, +{ /// Always start with Paused operation mode, one needs to know the exact status from syncer. - pub fn new(handler: VotingHandler) -> OperationStateMachine { + pub fn new(handler: VotingHandler) -> OperationStateMachine { let metrics = OperationMetrics { current_mode: PAUSED, previous_mode: INITIALIZED, @@ -59,10 +61,10 @@ impl OperationStateMachine { Self::Paused(PausedOperationMode { metrics, handler }) } - pub fn step(self) -> Self { + pub async fn step(self) -> Self { match self { - OperationStateMachine::Paused(p) => p.advance(), - OperationStateMachine::Active(p) => p.advance(), + OperationStateMachine::Paused(p) => p.advance().await, + OperationStateMachine::Active(p) => p.advance().await, } } } diff --git a/fendermint/vm/topdown/src/vote/operation/paused.rs b/fendermint/vm/topdown/src/vote/operation/paused.rs index a66c01a48..e137e1c9c 100644 --- a/fendermint/vm/topdown/src/vote/operation/paused.rs +++ b/fendermint/vm/topdown/src/vote/operation/paused.rs @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::sync::TopDownSyncEvent; +use crate::vote::gossip::GossipClient; use crate::vote::operation::active::ActiveOperationMode; -use crate::vote::operation::{ - OperationMetrics, OperationModeHandler, OperationStateMachine, ACTIVE, -}; +use crate::vote::operation::{OperationMetrics, OperationStateMachine, ACTIVE, PAUSED}; +use crate::vote::store::VoteStore; use crate::vote::VotingHandler; use std::fmt::{Display, Formatter}; @@ -16,19 +16,23 @@ use std::fmt::{Display, Formatter}; /// Therefore, we still don’t know what the last committed topdown checkpoint is, /// so we refrain from watching the parent chain, and from gossiping /// any certified observations until we switch to active mode. -pub(crate) struct PausedOperationMode { +pub(crate) struct PausedOperationMode { pub(crate) metrics: OperationMetrics, - pub(crate) handler: VotingHandler, + pub(crate) handler: VotingHandler, } -impl Display for PausedOperationMode { +impl Display for PausedOperationMode { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "paused") + write!(f, "{}", PAUSED) } } -impl OperationModeHandler for PausedOperationMode { - fn advance(mut self) -> OperationStateMachine { +impl PausedOperationMode +where + G: GossipClient + Send + Sync + 'static, + S: VoteStore + Send + Sync + 'static, +{ + pub(crate) async fn advance(mut self) -> OperationStateMachine { let n = self.handler.process_external_request(&self.metrics); tracing::debug!( num = n, @@ -43,7 +47,7 @@ impl OperationModeHandler for PausedOperationMode { } // handle the polled event - self.handler.handle_event(v); + self.handler.handle_event(v).await; } self.metrics.mode_changed(ACTIVE); diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index e32c36312..fde896325 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::fmt::{Display, Formatter}; pub type PowerTable = HashMap; +pub type PowerUpdates = Vec<(ValidatorKey, Weight)>; /// The different versions of vote casted in topdown gossip pub-sub channel #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] diff --git a/fendermint/vm/topdown/src/vote/store.rs b/fendermint/vm/topdown/src/vote/store.rs index 8b64aed83..b84be8890 100644 --- a/fendermint/vm/topdown/src/vote/store.rs +++ b/fendermint/vm/topdown/src/vote/store.rs @@ -1,14 +1,15 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::vote::error::Error; use crate::vote::payload::{Ballot, PowerTable, Vote}; -use crate::vote::{Error, Weight}; +use crate::vote::Weight; use crate::BlockHeight; use fendermint_vm_genesis::ValidatorKey; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, HashMap}; -pub(crate) trait VoteStore { +pub trait VoteStore { /// Get the earliest block height of the votes stored fn earliest_vote_height(&self) -> Result, Error>; @@ -30,7 +31,7 @@ pub(crate) trait VoteStore { } #[derive(Default)] -pub(crate) struct InMemoryVoteStore { +pub struct InMemoryVoteStore { votes: BTreeMap>, } @@ -81,13 +82,17 @@ impl VoteStore for InMemoryVoteStore { } /// The aggregated votes from different validators. -pub(crate) struct VoteAgg<'a>(Vec<&'a Vote>); +pub struct VoteAgg<'a>(Vec<&'a Vote>); impl<'a> VoteAgg<'a> { pub fn new(votes: Vec<&'a Vote>) -> Self { Self(votes) } + pub fn into_owned(self) -> Vec { + self.0.into_iter().cloned().collect() + } + pub fn ballot_weights(&self, power_table: &PowerTable) -> Vec<(&Ballot, Weight)> { let mut votes: Vec<(&Ballot, Weight)> = Vec::new(); diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index b06d08a2b..a596b5b7d 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -1,9 +1,10 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::vote::payload::{Ballot, PowerTable, Vote}; +use crate::vote::error::Error; +use crate::vote::payload::{Ballot, PowerTable, PowerUpdates, Vote}; use crate::vote::store::VoteStore; -use crate::vote::{Error, Weight}; +use crate::vote::Weight; use crate::BlockHeight; use fendermint_vm_genesis::ValidatorKey; use std::collections::HashMap; @@ -74,6 +75,12 @@ impl VoteTally { self.last_finalized_height } + /// Returns the votes collected in the network at the target height + pub fn get_votes_at_height(&self, height: BlockHeight) -> Result, Error> { + let votes = self.votes.get_votes_at_height(height)?; + Ok(votes.into_owned()) + } + /// Add a vote we received. /// /// Returns `true` if this vote was added, `false` if it was ignored as a @@ -158,7 +165,7 @@ impl VoteTally { /// Overwrite the power table after it has changed to a new snapshot. /// /// This method expects absolute values, it completely replaces the existing powers. - pub fn set_power_table(&mut self, power_table: Vec<(ValidatorKey, Weight)>) { + pub fn set_power_table(&mut self, power_table: PowerUpdates) { let power_table = HashMap::from_iter(power_table); // We don't actually have to remove the votes of anyone who is no longer a validator, // we just have to make sure to handle the case when they are not in the power table. @@ -168,7 +175,7 @@ impl VoteTally { /// Update the power table after it has changed with changes. /// /// This method expects only the updated values, leaving everyone who isn't in it untouched - pub fn update_power_table(&mut self, power_updates: Vec<(ValidatorKey, Weight)>) { + pub fn update_power_table(&mut self, power_updates: PowerUpdates) { if power_updates.is_empty() { return; } @@ -186,10 +193,10 @@ impl VoteTally { #[cfg(test)] mod tests { + use crate::vote::error::Error; use crate::vote::payload::{CertifiedObservation, Observation, Vote}; use crate::vote::store::InMemoryVoteStore; use crate::vote::tally::VoteTally; - use crate::vote::Error; use arbitrary::{Arbitrary, Unstructured}; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; From c9939d3028f989f4c20b42847b7c004b259209ec Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Tue, 24 Sep 2024 13:35:29 +0800 Subject: [PATCH 02/16] more debug queries --- fendermint/vm/topdown/src/vote/mod.rs | 127 +++++++++++++----- .../vm/topdown/src/vote/operation/active.rs | 11 +- fendermint/vm/topdown/src/vote/payload.rs | 7 + fendermint/vm/topdown/src/vote/tally.rs | 12 +- 4 files changed, 114 insertions(+), 43 deletions(-) diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index 99953963a..af484827c 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -11,7 +11,7 @@ mod tally; use crate::sync::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; -use crate::vote::payload::{PowerUpdates, Vote}; +use crate::vote::payload::{Ballot, PowerUpdates, Vote, VoteTallyState}; use crate::vote::store::VoteStore; use crate::vote::tally::VoteTally; use crate::BlockHeight; @@ -39,7 +39,10 @@ pub struct VoteReactorClient { tx: mpsc::Sender, } -pub fn start_vote_reactor( +pub fn start_vote_reactor< + G: GossipClient + Send + Sync + 'static, + V: VoteStore + Send + Sync + 'static, +>( config: Config, power_table: PowerUpdates, last_finalized_height: BlockHeight, @@ -71,49 +74,89 @@ pub fn start_vote_reactor anyhow::Result { + async fn request) -> VoteReactorRequest>( + &self, + f: F, + ) -> anyhow::Result { let (tx, rx) = oneshot::channel(); - self.tx - .send(VoteReactorRequest::QueryOperationMode(tx)) - .await?; + self.tx.send(f(tx)).await?; Ok(rx.await?) } + /// Query the current operation mode of the vote tally state machine + pub async fn query_operation_mode(&self) -> anyhow::Result { + self.request(VoteReactorRequest::QueryOperationMode).await + } + + /// Query the current validator votes at the target block height pub async fn query_votes( &self, height: BlockHeight, ) -> anyhow::Result, Error>> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(VoteReactorRequest::QueryVotes { - height, - reply_tx: tx, - }) - .await?; - Ok(rx.await?) + self.request(|tx| VoteReactorRequest::QueryVotes { height, tx }) + .await + } + + /// Queries the vote tally to see if there are new quorum formed + pub async fn find_quorum(&self) -> anyhow::Result> { + self.request(VoteReactorRequest::FindQuorum).await } + /// Get the current vote tally state variables in vote tally + pub async fn query_vote_tally_state(&self) -> anyhow::Result { + self.request(VoteReactorRequest::QueryState).await + } + + /// Update power of some validators. If the weight is zero, the validator is removed + /// from the power table. pub async fn update_power_table(&self, updates: PowerUpdates) -> anyhow::Result<()> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(VoteReactorRequest::UpdatePowerTable { - updates, - reply_tx: tx, - }) - .await?; - Ok(rx.await?) + self.request(|tx| VoteReactorRequest::UpdatePowerTable { updates, tx }) + .await + } + + /// Completely over-write existing power table + pub async fn set_power_table(&self, updates: PowerUpdates) -> anyhow::Result<()> { + self.request(|tx| VoteReactorRequest::SetPowerTable { updates, tx }) + .await + } + + /// Signals that a new quorum is finalized and executed in the interpreter + pub async fn set_quorum_finalized( + &self, + height: BlockHeight, + ) -> anyhow::Result> { + self.request(|tx| VoteReactorRequest::SetQuorumFinalized { height, tx }) + .await } } enum VoteReactorRequest { + /// Query the current operation mode of the vote tally state machine QueryOperationMode(oneshot::Sender), + /// Query the current validator votes at the target block height QueryVotes { height: BlockHeight, - reply_tx: oneshot::Sender, Error>>, + tx: oneshot::Sender, Error>>, }, + /// Get the current vote tally state variables in vote tally + QueryState(oneshot::Sender), + /// Queries the vote tally to see if there are new quorum formed + FindQuorum(oneshot::Sender>), + /// Update power of some validators. If the weight is zero, the validator is removed + /// from the power table. UpdatePowerTable { updates: PowerUpdates, - reply_tx: oneshot::Sender<()>, + tx: oneshot::Sender<()>, + }, + /// Completely over-write existing power table + SetPowerTable { + updates: PowerUpdates, + tx: oneshot::Sender<()>, + }, + /// Signals that a new quorum is finalized and executed in the interpreter + SetQuorumFinalized { + height: BlockHeight, + tx: oneshot::Sender>, }, } @@ -137,16 +180,38 @@ where { fn handle_request(&mut self, req: VoteReactorRequest, metrics: &OperationMetrics) { match req { - VoteReactorRequest::QueryOperationMode(req_tx) => { + VoteReactorRequest::QueryOperationMode(tx) => { // ignore error - let _ = req_tx.send(metrics.clone()); + let _ = tx.send(metrics.clone()); } - VoteReactorRequest::QueryVotes { height, reply_tx } => { - let _ = reply_tx.send(self.vote_tally.get_votes_at_height(height)); + VoteReactorRequest::QueryVotes { height, tx } => { + let _ = tx.send(self.vote_tally.get_votes_at_height(height)); } - VoteReactorRequest::UpdatePowerTable { updates, reply_tx } => { + VoteReactorRequest::UpdatePowerTable { updates, tx } => { self.vote_tally.update_power_table(updates); - let _ = reply_tx.send(()); + let _ = tx.send(()); + } + VoteReactorRequest::FindQuorum(tx) => { + let quorum = self + .vote_tally + .find_quorum() + .inspect_err(|e| tracing::error!(err = e.to_string(), "cannot find quorum")) + .unwrap_or_default(); + let _ = tx.send(quorum); + } + VoteReactorRequest::SetPowerTable { updates, tx } => { + self.vote_tally.set_power_table(updates); + let _ = tx.send(()); + } + VoteReactorRequest::SetQuorumFinalized { height, tx } => { + let _ = tx.send(self.vote_tally.set_finalized(height)); + } + VoteReactorRequest::QueryState(tx) => { + let _ = tx.send(VoteTallyState { + last_finalized_height: self.vote_tally.last_finalized_height(), + quorum_threshold: self.vote_tally.quorum_threshold(), + power_table: self.vote_tally.power_table().clone(), + }); } } } @@ -212,7 +277,7 @@ where Err(e) => { tracing::warn!(err = e.to_string(), "cannot poll gossip vote"); break; - }, + } _ => {} } } diff --git a/fendermint/vm/topdown/src/vote/operation/active.rs b/fendermint/vm/topdown/src/vote/operation/active.rs index 96feefb8e..327f1f7dc 100644 --- a/fendermint/vm/topdown/src/vote/operation/active.rs +++ b/fendermint/vm/topdown/src/vote/operation/active.rs @@ -1,15 +1,13 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::vote::gossip::GossipClient; use crate::vote::operation::paused::PausedOperationMode; -use crate::vote::operation::{ - OperationMetrics, OperationStateMachine, ACTIVE, PAUSED, -}; +use crate::vote::operation::{OperationMetrics, OperationStateMachine, ACTIVE, PAUSED}; +use crate::vote::store::VoteStore; use crate::vote::TopDownSyncEvent; use crate::vote::VotingHandler; use std::fmt::{Display, Formatter}; -use crate::vote::gossip::GossipClient; -use crate::vote::store::VoteStore; /// In active mode, we observe a steady rate of topdown checkpoint commitments on chain. /// Our lookahead buffer is sliding continuously. As we acquire new finalised parent blocks, @@ -25,7 +23,8 @@ impl Display for ActiveOperationMode { } } -impl ActiveOperationMode where +impl ActiveOperationMode +where G: GossipClient + Send + Sync + 'static, S: VoteStore + Send + Sync + 'static, { diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index fde896325..193c9c518 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -16,6 +16,13 @@ use std::fmt::{Display, Formatter}; pub type PowerTable = HashMap; pub type PowerUpdates = Vec<(ValidatorKey, Weight)>; +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +pub struct VoteTallyState { + pub last_finalized_height: BlockHeight, + pub quorum_threshold: Weight, + pub power_table: PowerTable, +} + /// The different versions of vote casted in topdown gossip pub-sub channel #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] pub enum Vote { diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index a596b5b7d..4094a95e3 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -46,10 +46,6 @@ impl VoteTally { }) } - fn power_table(&self) -> &HashMap { - &self.power_table - } - /// Check that a validator key is currently part of the power table. fn has_power(&self, validator_key: &ValidatorKey) -> bool { // For consistency consider validators without power unknown. @@ -59,11 +55,15 @@ impl VoteTally { } } + pub fn power_table(&self) -> &HashMap { + &self.power_table + } + /// Calculate the minimum weight needed for a proposal to pass with the current membership. /// /// This is inclusive, that is, if the sum of weight is greater or equal to this, it should pass. /// The equivalent formula can be found in CometBFT [here](https://github.com/cometbft/cometbft/blob/a8991d63e5aad8be82b90329b55413e3a4933dc0/types/vote_set.go#L307). - fn quorum_threshold(&self) -> Weight { + pub fn quorum_threshold(&self) -> Weight { let total_weight: Weight = self.power_table.values().sum(); total_weight * 2 / 3 + 1 } @@ -71,7 +71,7 @@ impl VoteTally { /// Return the height of the first entry in the chain. /// /// This is the block that was finalized *in the ledger*. - fn last_finalized_height(&self) -> BlockHeight { + pub fn last_finalized_height(&self) -> BlockHeight { self.last_finalized_height } From 6e7bc5346f5ab9ccdb49deaf9ee702b7d3f2ab19 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Tue, 24 Sep 2024 14:48:00 +0800 Subject: [PATCH 03/16] fmt & clippy --- fendermint/vm/topdown/src/vote/tally.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index 4094a95e3..7e943a733 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -365,16 +365,15 @@ mod tests { .set_finalized(observation.ballot.parent_height() - 1) .unwrap(); - let mut count = 0; - for validator in &validators { + for (count, validator) in validators.iter().enumerate() { let certified = CertifiedObservation::sign(observation.clone(), &validator.0).unwrap(); let vote = Vote::v1(certified).unwrap(); vote_tally.add_vote(vote).unwrap(); + // only 3 validators vote if count == 2 { break; } - count += 1; } assert!(vote_tally.find_quorum().unwrap().is_none()); From 1fb3bee0dde7710c11aae106045c7e879c4e1680 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Tue, 24 Sep 2024 16:12:20 +0800 Subject: [PATCH 04/16] merge with upstream --- fendermint/vm/topdown/src/vote/payload.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index 95911fea3..10043aa01 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -5,7 +5,6 @@ use crate::vote::Weight; use crate::{BlockHash, BlockHeight, Bytes}; use anyhow::anyhow; use arbitrary::Arbitrary; -use ethers::core::k256::sha2; use fendermint_crypto::secp::RecoverableECDSASignature; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; @@ -71,10 +70,7 @@ pub struct CertifiedObservation { impl Vote { pub fn v1(obs: CertifiedObservation) -> anyhow::Result { let to_sign = fvm_ipld_encoding::to_vec(&obs.observed)?; - let (pk, _) = obs - .signature - .clone() - .recover(&to_sign)?; + let (pk, _) = obs.signature.clone().recover(&to_sign)?; Ok(Self::V1 { validator: ValidatorKey::new(pk), From ccf457ce32fdf1d01405753f9bc46ab8a3cd768c Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Tue, 24 Sep 2024 20:30:49 +0800 Subject: [PATCH 05/16] testing template --- fendermint/vm/topdown/src/vote/gossip.rs | 2 +- fendermint/vm/topdown/src/vote/mod.rs | 8 +- fendermint/vm/topdown/tests/vote_reactor.rs | 126 ++++++++++++++++++++ 3 files changed, 131 insertions(+), 5 deletions(-) create mode 100644 fendermint/vm/topdown/tests/vote_reactor.rs diff --git a/fendermint/vm/topdown/src/vote/gossip.rs b/fendermint/vm/topdown/src/vote/gossip.rs index 15305c05d..417ef5ce1 100644 --- a/fendermint/vm/topdown/src/vote/gossip.rs +++ b/fendermint/vm/topdown/src/vote/gossip.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; pub trait GossipClient { /// Attempts to poll if there are available vote. This method returns immediately. /// If there is no vote, it returns None - fn try_poll_vote(&self) -> Result, Error>; + fn try_poll_vote(&mut self) -> Result, Error>; async fn publish_vote(&self, vote: Vote) -> Result<(), Error>; } diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index af484827c..52ee87a5e 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -25,13 +25,13 @@ pub type Weight = u64; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Config { /// The reactor request channel buffer size - req_channel_buffer_size: usize, + pub req_channel_buffer_size: usize, /// The number of requests the reactor should process per run before handling other tasks - req_batch_processing_size: usize, + pub req_batch_processing_size: usize, /// The number of vote recording requests the reactor should process per run before handling other tasks - gossip_req_processing_size: usize, + pub gossip_req_processing_size: usize, /// The time to sleep for voting loop if nothing happens - voting_sleep_interval_sec: u64, + pub voting_sleep_interval_sec: u64, } /// The client to interact with the vote reactor diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs new file mode 100644 index 000000000..9460a6a21 --- /dev/null +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -0,0 +1,126 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +//! ```text +//! cargo test --release -p fendermint_vm_topdown --test smt_vote_reactor +//! ``` + +use async_trait::async_trait; +use libp2p::futures::AsyncReadExt; +use tokio::sync::broadcast; +use tokio::sync::broadcast::error::TryRecvError; +use fendermint_crypto::SecretKey; +use fendermint_vm_genesis::ValidatorKey; +use fendermint_vm_topdown::vote::error::Error; +use fendermint_vm_topdown::vote::gossip::GossipClient; +use fendermint_vm_topdown::vote::payload::{PowerTable, PowerUpdates, Vote}; +use fendermint_vm_topdown::vote::{Config, start_vote_reactor, Weight}; +use fendermint_vm_topdown::vote::store::InMemoryVoteStore; + +struct Validator { + sk: SecretKey, + weight: Weight, +} + +impl Validator { + fn validator_key(&self) -> ValidatorKey { + ValidatorKey::new(self.sk.public_key()) + } +} + +struct ChannelGossipClient { + tx: broadcast::Sender, + rxs: Vec>, +} + +#[async_trait] +impl GossipClient for ChannelGossipClient { + fn try_poll_vote(&mut self) -> Result, Error> { + for rx in self.rxs.iter_mut() { + match rx.try_recv() { + Ok(v) => return Ok(Some(v)), + Err(broadcast::error::TryRecvError::Empty) => continue, + _ => panic!("should not happen") + } + } + + Ok(None) + } + + async fn publish_vote(&self, vote: Vote) -> Result<(), Error> { + self.tx.send(vote).unwrap(); + Ok(()) + } +} + +fn default_config() -> Config { + Config{ + req_channel_buffer_size: 1024, + req_batch_processing_size: 10, + gossip_req_processing_size: 10, + voting_sleep_interval_sec: 1, + } +} + +fn gen_validators(weights: Vec) -> (Vec, Vec) { + let mut rng = rand::thread_rng(); + + let mut gossips: Vec = vec![]; + for _ in 0..weights.len() { + let (tx, rx) = broadcast::channel(100); + + let mut g = ChannelGossipClient{ tx, rxs: vec![] }; + + for existing in gossips.iter() { + g.rxs.push(existing.tx.subscribe()) + } + + for existing in gossips.iter_mut() { + existing.rxs.push(g.tx.subscribe()); + } + + gossips.push(g); + } + + let validators = weights + .into_iter() + .map(|w| Validator { sk: SecretKey::random(&mut rng), weight: w }) + .collect::>(); + + (validators, gossips) +} + +// fn gen_power_table(validators: &[Validator]) -> PowerTable { +// PowerTable::from_iter(validators.iter().map(|v| (v.validator_key(), v.weight))) +// } + +fn gen_power_updates(validators: &[Validator]) -> PowerUpdates { + validators.iter().map(|v| (v.validator_key(), v.weight)).collect() +} + +#[tokio::test] +async fn all_validators_active_mode() { + let config = default_config(); + + // 21 validators equal 100 weight + let (validators, gossips) = gen_validators(vec![100; 21]); + let power_updates = gen_power_updates(&validators); + let initial_finalized_height = 10; + + let (internal_event_tx, _) = broadcast::channel(1024); + + let node_clients = gossips + .into_iter() + .map(|gossip| { + start_vote_reactor( + config.clone(), + power_updates.clone(), + initial_finalized_height, + gossip, + InMemoryVoteStore::default(), + internal_event_tx.subscribe(), + ) + .unwrap() + }) + .collect::>(); +} \ No newline at end of file From 2233eb7aae46d8b0bb30a9545e92425978b65ffd Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Wed, 25 Sep 2024 09:48:12 +0800 Subject: [PATCH 06/16] more tests --- fendermint/vm/topdown/src/sync/mod.rs | 6 +- fendermint/vm/topdown/src/vote/mod.rs | 40 +++- .../vm/topdown/src/vote/operation/active.rs | 4 - .../vm/topdown/src/vote/operation/mod.rs | 4 +- .../vm/topdown/src/vote/operation/paused.rs | 2 +- fendermint/vm/topdown/src/vote/payload.rs | 21 +- fendermint/vm/topdown/src/vote/store.rs | 8 +- fendermint/vm/topdown/src/vote/tally.rs | 33 ++- fendermint/vm/topdown/tests/vote_reactor.rs | 195 +++++++++++++++--- 9 files changed, 262 insertions(+), 51 deletions(-) diff --git a/fendermint/vm/topdown/src/sync/mod.rs b/fendermint/vm/topdown/src/sync/mod.rs index 3c5d71d32..d6f1341e5 100644 --- a/fendermint/vm/topdown/src/sync/mod.rs +++ b/fendermint/vm/topdown/src/sync/mod.rs @@ -19,13 +19,13 @@ use std::time::Duration; use fendermint_vm_genesis::{Power, Validator}; -use crate::vote::payload::Vote; +use crate::vote::payload::Observation; pub use syncer::fetch_topdown_events; -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum TopDownSyncEvent { NodeSyncing, - NewProposal(Box), + NewProposal(Box), } /// Query the parent finality from the block chain state. diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index 52ee87a5e..d832f1b27 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -8,10 +8,11 @@ pub mod payload; pub mod store; mod tally; +use std::collections::HashMap; use crate::sync::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; -use crate::vote::payload::{Ballot, PowerUpdates, Vote, VoteTallyState}; +use crate::vote::payload::{Ballot, CertifiedObservation, PowerUpdates, Vote, VoteTallyState}; use crate::vote::store::VoteStore; use crate::vote::tally::VoteTally; use crate::BlockHeight; @@ -19,6 +20,8 @@ use error::Error; use serde::{Deserialize, Serialize}; use std::time::Duration; use tokio::sync::{broadcast, mpsc, oneshot}; +use fendermint_crypto::SecretKey; +use fendermint_vm_genesis::ValidatorKey; pub type Weight = u64; @@ -44,6 +47,7 @@ pub fn start_vote_reactor< V: VoteStore + Send + Sync + 'static, >( config: Config, + validator_key: SecretKey, power_table: PowerUpdates, last_finalized_height: BlockHeight, gossip: G, @@ -57,6 +61,7 @@ pub fn start_vote_reactor< let sleep = Duration::new(config.voting_sleep_interval_sec, 0); let inner = VotingHandler { + validator_key, req_rx: rx, internal_event_listener, vote_tally, @@ -80,7 +85,8 @@ impl VoteReactorClient { ) -> anyhow::Result { let (tx, rx) = oneshot::channel(); self.tx.send(f(tx)).await?; - Ok(rx.await?) + let r = rx.await?; + Ok(r) } /// Query the current operation mode of the vote tally state machine @@ -128,6 +134,12 @@ impl VoteReactorClient { self.request(|tx| VoteReactorRequest::SetQuorumFinalized { height, tx }) .await } + + pub async fn dump_votes( + &self, + ) -> anyhow::Result>, Error>> { + self.request(VoteReactorRequest::DumpAllVotes).await + } } enum VoteReactorRequest { @@ -138,6 +150,9 @@ enum VoteReactorRequest { height: BlockHeight, tx: oneshot::Sender, Error>>, }, + /// Dump all the votes that is currently stored in the vote tally. + /// This is generally a very expensive operation, but good for debugging, use with care + DumpAllVotes(oneshot::Sender>, Error>>), /// Get the current vote tally state variables in vote tally QueryState(oneshot::Sender), /// Queries the vote tally to see if there are new quorum formed @@ -161,6 +176,8 @@ enum VoteReactorRequest { } struct VotingHandler { + /// The validator key that is used to sign proposal produced for broadcasting + validator_key: SecretKey, /// Handles the requests targeting the vote reactor, could be querying the /// vote tally status and etc. req_rx: mpsc::Receiver, @@ -213,18 +230,29 @@ where power_table: self.vote_tally.power_table().clone(), }); } + VoteReactorRequest::DumpAllVotes(tx) => { + let _ = tx.send(self.vote_tally.dump_votes()); + } } } async fn handle_event(&mut self, event: TopDownSyncEvent) { match event { - TopDownSyncEvent::NewProposal(vote) => { - if let Err(e) = self.vote_tally.add_vote(*vote.clone()) { + TopDownSyncEvent::NewProposal(observation) => { + let vote = match CertifiedObservation::sign(*observation, &self.validator_key) { + Ok(v) => Vote::v1(ValidatorKey::new(self.validator_key.public_key()), v), + Err(e) => { + tracing::error!(err = e.to_string(), "cannot sign received proposal"); + return + } + }; + + if let Err(e) = self.vote_tally.add_vote(vote.clone()) { tracing::error!(err = e.to_string(), "cannot self vote to tally"); return; } - match self.gossip.publish_vote(*vote).await { + match self.gossip.publish_vote(vote).await { Ok(_) => {} Err(e) => { tracing::error!( @@ -278,7 +306,7 @@ where tracing::warn!(err = e.to_string(), "cannot poll gossip vote"); break; } - _ => {} + _ => break, } } vote_processed diff --git a/fendermint/vm/topdown/src/vote/operation/active.rs b/fendermint/vm/topdown/src/vote/operation/active.rs index 327f1f7dc..d639004f2 100644 --- a/fendermint/vm/topdown/src/vote/operation/active.rs +++ b/fendermint/vm/topdown/src/vote/operation/active.rs @@ -39,10 +39,6 @@ where n = self.handler.process_gossip_subscription_votes(); tracing::debug!(num = n, status = self.to_string(), "handled gossip votes"); - if n == 0 { - todo!("handle transition to soft recover") - } - while let Some(v) = self.handler.poll_internal_event() { // top down is now syncing, pause everything if matches!(v, TopDownSyncEvent::NodeSyncing) { diff --git a/fendermint/vm/topdown/src/vote/operation/mod.rs b/fendermint/vm/topdown/src/vote/operation/mod.rs index c45fb9515..bd5c4e406 100644 --- a/fendermint/vm/topdown/src/vote/operation/mod.rs +++ b/fendermint/vm/topdown/src/vote/operation/mod.rs @@ -43,8 +43,8 @@ pub enum OperationStateMachine { /// Tracks the operation mdoe metrics for the voting system #[derive(Clone, Debug)] pub struct OperationMetrics { - current_mode: OperationMode, - previous_mode: OperationMode, + pub current_mode: OperationMode, + pub previous_mode: OperationMode, } impl OperationStateMachine diff --git a/fendermint/vm/topdown/src/vote/operation/paused.rs b/fendermint/vm/topdown/src/vote/operation/paused.rs index e137e1c9c..33931e4e8 100644 --- a/fendermint/vm/topdown/src/vote/operation/paused.rs +++ b/fendermint/vm/topdown/src/vote/operation/paused.rs @@ -17,7 +17,7 @@ use std::fmt::{Display, Formatter}; /// so we refrain from watching the parent chain, and from gossiping /// any certified observations until we switch to active mode. pub(crate) struct PausedOperationMode { - pub(crate) metrics: OperationMetrics, + pub metrics: OperationMetrics, pub(crate) handler: VotingHandler, } diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index 10043aa01..f22f7a47f 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -68,7 +68,11 @@ pub struct CertifiedObservation { } impl Vote { - pub fn v1(obs: CertifiedObservation) -> anyhow::Result { + pub fn v1(validator_key: ValidatorKey, obs: CertifiedObservation) -> Self { + Self::V1 {validator: validator_key, payload: obs } + } + + pub fn v1_checked(obs: CertifiedObservation) -> anyhow::Result { let to_sign = fvm_ipld_encoding::to_vec(&obs.observed)?; let (pk, _) = obs.signature.clone().recover(&to_sign)?; @@ -98,7 +102,7 @@ impl TryFrom<&[u8]> for Vote { let version = bytes[0]; if version == 0 { - return Self::v1(CertifiedObservation::try_from(&bytes[1..])?); + return Self::v1_checked(CertifiedObservation::try_from(&bytes[1..])?); } Err(anyhow!("invalid vote version")) @@ -124,6 +128,19 @@ impl CertifiedObservation { } } +impl Observation { + pub fn new(local_hash: Bytes, parent_height: BlockHeight, parent_hash: Bytes, commitment: Bytes) -> Self { + Self { + local_hash, + ballot: Ballot { + parent_height, + parent_hash, + cumulative_effects_comm: commitment, + }, + } + } +} + impl Display for Ballot { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( diff --git a/fendermint/vm/topdown/src/vote/store.rs b/fendermint/vm/topdown/src/vote/store.rs index b84be8890..1f3c13ba4 100644 --- a/fendermint/vm/topdown/src/vote/store.rs +++ b/fendermint/vm/topdown/src/vote/store.rs @@ -89,6 +89,8 @@ impl<'a> VoteAgg<'a> { Self(votes) } + pub fn is_empty(&self) -> bool { self.0.is_empty() } + pub fn into_owned(self) -> Vec { self.0.into_iter().cloned().collect() } @@ -154,17 +156,17 @@ mod tests { let observation1 = random_observation(); votes.push( - Vote::v1(CertifiedObservation::sign(observation1.clone(), &validators[0].0).unwrap()) + Vote::v1_checked(CertifiedObservation::sign(observation1.clone(), &validators[0].0).unwrap()) .unwrap(), ); let observation2 = random_observation(); votes.push( - Vote::v1(CertifiedObservation::sign(observation2.clone(), &validators[1].0).unwrap()) + Vote::v1_checked(CertifiedObservation::sign(observation2.clone(), &validators[1].0).unwrap()) .unwrap(), ); votes.push( - Vote::v1(CertifiedObservation::sign(observation2.clone(), &validators[2].0).unwrap()) + Vote::v1_checked(CertifiedObservation::sign(observation2.clone(), &validators[2].0).unwrap()) .unwrap(), ); diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index 7e943a733..f23f1a247 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -81,6 +81,25 @@ impl VoteTally { Ok(votes.into_owned()) } + /// Dump all the votes that is currently stored in the vote tally. + /// This is generally a very expensive operation, but good for debugging, use with care + pub fn dump_votes(&self) -> Result>, Error> { + let mut r = HashMap::new(); + + let Some(latest) = self.votes.latest_vote_height()? else { + return Ok(r); + }; + + for h in self.last_finalized_height+1..=latest { + let votes = self.votes.get_votes_at_height(h)?; + if votes.is_empty() { + continue; + } + r.insert(h, votes.into_owned()); + } + Ok(r) + } + /// Add a vote we received. /// /// Returns `true` if this vote was added, `false` if it was ignored as a @@ -232,12 +251,12 @@ mod tests { let obs = random_observation(); let vote = - Vote::v1(CertifiedObservation::sign(obs.clone(), &validators[0].0).unwrap()).unwrap(); + Vote::v1_checked(CertifiedObservation::sign(obs.clone(), &validators[0].0).unwrap()).unwrap(); vote_tally.add_vote(vote).unwrap(); let mut obs2 = random_observation(); obs2.ballot.parent_height = obs.ballot.parent_height(); - let vote = Vote::v1(CertifiedObservation::sign(obs2, &validators[0].0).unwrap()).unwrap(); + let vote = Vote::v1_checked(CertifiedObservation::sign(obs2, &validators[0].0).unwrap()).unwrap(); assert_eq!(vote_tally.add_vote(vote), Err(Error::Equivocation)); } @@ -261,7 +280,7 @@ mod tests { for validator in validators { let certified = CertifiedObservation::sign(observation.clone(), &validator.0).unwrap(); - let vote = Vote::v1(certified).unwrap(); + let vote = Vote::v1_checked(certified).unwrap(); vote_tally.add_vote(vote).unwrap(); } @@ -299,14 +318,14 @@ mod tests { for validator in validators_grp1 { let certified = CertifiedObservation::sign(observation1.clone(), &validator.0).unwrap(); - let vote = Vote::v1(certified).unwrap(); + let vote = Vote::v1_checked(certified).unwrap(); vote_tally.add_vote(vote).unwrap(); } assert!(vote_tally.find_quorum().unwrap().is_none()); for validator in validators_grp2 { let certified = CertifiedObservation::sign(observation2.clone(), &validator.0).unwrap(); - let vote = Vote::v1(certified).unwrap(); + let vote = Vote::v1_checked(certified).unwrap(); vote_tally.add_vote(vote).unwrap(); } @@ -333,7 +352,7 @@ mod tests { for validator in validators { let certified = CertifiedObservation::sign(observation.clone(), &validator.0).unwrap(); - let vote = Vote::v1(certified).unwrap(); + let vote = Vote::v1_checked(certified).unwrap(); vote_tally.add_vote(vote).unwrap(); } @@ -367,7 +386,7 @@ mod tests { for (count, validator) in validators.iter().enumerate() { let certified = CertifiedObservation::sign(observation.clone(), &validator.0).unwrap(); - let vote = Vote::v1(certified).unwrap(); + let vote = Vote::v1_checked(certified).unwrap(); vote_tally.add_vote(vote).unwrap(); // only 3 validators vote diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs index 9460a6a21..7c3876291 100644 --- a/fendermint/vm/topdown/tests/vote_reactor.rs +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -2,18 +2,18 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! ```text -//! cargo test --release -p fendermint_vm_topdown --test smt_vote_reactor +//! cargo test --release -p fendermint_vm_topdown --test vote_reactor //! ``` use async_trait::async_trait; -use libp2p::futures::AsyncReadExt; use tokio::sync::broadcast; use tokio::sync::broadcast::error::TryRecvError; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; +use fendermint_vm_topdown::sync::TopDownSyncEvent; use fendermint_vm_topdown::vote::error::Error; use fendermint_vm_topdown::vote::gossip::GossipClient; -use fendermint_vm_topdown::vote::payload::{PowerTable, PowerUpdates, Vote}; +use fendermint_vm_topdown::vote::payload::{Observation, PowerUpdates, Vote}; use fendermint_vm_topdown::vote::{Config, start_vote_reactor, Weight}; use fendermint_vm_topdown::vote::store::InMemoryVoteStore; @@ -39,7 +39,7 @@ impl GossipClient for ChannelGossipClient { for rx in self.rxs.iter_mut() { match rx.try_recv() { Ok(v) => return Ok(Some(v)), - Err(broadcast::error::TryRecvError::Empty) => continue, + Err(TryRecvError::Empty) => continue, _ => panic!("should not happen") } } @@ -48,7 +48,7 @@ impl GossipClient for ChannelGossipClient { } async fn publish_vote(&self, vote: Vote) -> Result<(), Error> { - self.tx.send(vote).unwrap(); + let _ = self.tx.send(vote); Ok(()) } } @@ -67,7 +67,7 @@ fn gen_validators(weights: Vec) -> (Vec, Vec = vec![]; for _ in 0..weights.len() { - let (tx, rx) = broadcast::channel(100); + let (tx, _) = broadcast::channel(100); let mut g = ChannelGossipClient{ tx, rxs: vec![] }; @@ -99,28 +99,177 @@ fn gen_power_updates(validators: &[Validator]) -> PowerUpdates { } #[tokio::test] -async fn all_validators_active_mode() { +async fn simple_lifecycle() { let config = default_config(); // 21 validators equal 100 weight - let (validators, gossips) = gen_validators(vec![100; 21]); + let (validators, mut gossips) = gen_validators(vec![100; 1]); let power_updates = gen_power_updates(&validators); let initial_finalized_height = 10; - let (internal_event_tx, _) = broadcast::channel(1024); + let (internal_event_tx, _) = broadcast::channel(validators.len() + 1); - let node_clients = gossips - .into_iter() - .map(|gossip| { - start_vote_reactor( - config.clone(), - power_updates.clone(), - initial_finalized_height, - gossip, - InMemoryVoteStore::default(), - internal_event_tx.subscribe(), - ) - .unwrap() - }) - .collect::>(); + let client = start_vote_reactor( + config.clone(), + validators[0].sk.clone(), + power_updates.clone(), + initial_finalized_height, + gossips.pop().unwrap(), + InMemoryVoteStore::default(), + internal_event_tx.subscribe(), + ).unwrap(); + + assert_eq!(client.find_quorum().await.unwrap(), None); + + // now topdown sync published a new observation on parent height 100 + let parent_height = 100; + let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2,3,4]); + internal_event_tx.send(TopDownSyncEvent::NewProposal(Box::new(obs))).unwrap(); + + // wait for vote to be casted + while client.find_quorum().await.unwrap().is_none() {} + + let r = client.find_quorum().await.unwrap().unwrap(); + assert_eq!(r.parent_height(), parent_height); + + let r = client.query_votes(parent_height).await.unwrap().unwrap(); + assert_eq!(r.len(), 1); + + client.set_quorum_finalized(parent_height).await.unwrap().unwrap(); + + // now votes are cleared + assert_eq!(client.find_quorum().await.unwrap(), None); + let state = client.query_vote_tally_state().await.unwrap(); + assert_eq!(state.last_finalized_height, parent_height); +} + +#[tokio::test] +async fn waiting_for_quorum() { + let config = default_config(); + + // 21 validators equal 100 weight + let (validators, mut gossips) = gen_validators(vec![100; 5]); + let power_updates = gen_power_updates(&validators); + let initial_finalized_height = 10; + + let mut clients = vec![]; + let mut internal_txs = vec![]; + for i in 0..validators.len() { + let (internal_event_tx, _) = broadcast::channel(validators.len() + 1); + + let client = start_vote_reactor( + config.clone(), + validators[i].sk.clone(), + power_updates.clone(), + initial_finalized_height, + gossips.pop().unwrap(), + InMemoryVoteStore::default(), + internal_event_tx.subscribe(), + ).unwrap(); + + clients.push(client); + internal_txs.push(internal_event_tx); + } + + // now topdown sync published a new observation on parent height 100 + let parent_height1 = 100; + let obs1 = Observation::new(vec![100], parent_height1, vec![1, 2, 3], vec![2,3,4]); + let parent_height2 = 110; + let obs2 = Observation::new(vec![100], parent_height2, vec![1, 2, 3], vec![2,3,4]); + let parent_height3 = 120; + let obs3 = Observation::new(vec![100], parent_height3, vec![1, 2, 3], vec![2,3,4]); + + internal_txs[0].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); + internal_txs[1].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); + + internal_txs[2].send(TopDownSyncEvent::NewProposal(Box::new(obs2.clone()))).unwrap(); + internal_txs[3].send(TopDownSyncEvent::NewProposal(Box::new(obs2.clone()))).unwrap(); + + internal_txs[4].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); + + // ensure votes are received + for client in &clients { + while client.query_votes(parent_height1).await.unwrap().unwrap().len() != 2 {} + while client.query_votes(parent_height2).await.unwrap().unwrap().len() != 2 {} + while client.query_votes(parent_height3).await.unwrap().unwrap().len() != 1 {} + } + + // at this moment, no quorum should have ever formed + for client in &clients { + assert!(client.find_quorum().await.unwrap().is_none(), "should have no quorum"); + } + + // new observations made + internal_txs[3].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); + internal_txs[0].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); + internal_txs[1].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); + + // ensure every client receives the votes + for client in &clients { + while client.query_votes(parent_height3).await.unwrap().unwrap().len() != 4 {} + } + + for client in &clients { + let r = client.find_quorum().await.unwrap().unwrap(); + assert_eq!(r.parent_height(), parent_height3, "should have quorum"); + } + + // make observation on previous heights + internal_txs[3].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); + internal_txs[2].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); + + // ensure every client receives the votes + for client in &clients { + while client.query_votes(parent_height1).await.unwrap().unwrap().len() != 4 {} + } + + // but larger parent height wins + for client in &clients { + let r = client.find_quorum().await.unwrap().unwrap(); + assert_eq!(r.parent_height(), parent_height3, "should have formed quorum on larger height"); + } + + // finalize parent height 3 + for client in &clients { + client.set_quorum_finalized(parent_height3).await.unwrap().unwrap(); + assert!(client.dump_votes().await.unwrap().unwrap().is_empty(), "should have empty votes"); + } +} + +#[tokio::test] +async fn all_validator_in_sync() { + let config = default_config(); + + // 21 validators equal 100 weight + let (validators, mut gossips) = gen_validators(vec![100; 10]); + let power_updates = gen_power_updates(&validators); + let initial_finalized_height = 10; + + let (internal_event_tx, _) = broadcast::channel(validators.len() + 1); + + let mut node_clients = vec![]; + for i in 0..validators.len() { + let r = start_vote_reactor( + config.clone(), + validators[i].sk.clone(), + power_updates.clone(), + initial_finalized_height, + gossips.pop().unwrap(), + InMemoryVoteStore::default(), + internal_event_tx.subscribe(), + ) + .unwrap(); + node_clients.push(r); + } + + let parent_height = 100; + let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2,3,4]); + internal_event_tx.send(TopDownSyncEvent::NewProposal(Box::new(obs))).unwrap(); + + for n in node_clients { + while n.find_quorum().await.unwrap().is_none() {} + + let r = n.find_quorum().await.unwrap().unwrap(); + assert_eq!(r.parent_height(), parent_height) + } } \ No newline at end of file From 76350c60a09b1fc69336cac1326670603f73887a Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Wed, 25 Sep 2024 11:51:01 +0800 Subject: [PATCH 07/16] format code --- contracts/binding/src/lib.rs | 36 ++--- fendermint/vm/topdown/src/vote/mod.rs | 8 +- fendermint/vm/topdown/src/vote/payload.rs | 12 +- fendermint/vm/topdown/src/vote/store.rs | 22 ++- fendermint/vm/topdown/src/vote/tally.rs | 8 +- fendermint/vm/topdown/tests/vote_reactor.rs | 169 +++++++++++++++----- 6 files changed, 179 insertions(+), 76 deletions(-) diff --git a/contracts/binding/src/lib.rs b/contracts/binding/src/lib.rs index dd59524fb..b4fff9972 100644 --- a/contracts/binding/src/lib.rs +++ b/contracts/binding/src/lib.rs @@ -2,33 +2,27 @@ #[macro_use] mod convert; #[allow(clippy::all)] -pub mod checkpointing_facet; +pub mod i_diamond; +#[allow(clippy::all)] +pub mod diamond_loupe_facet; #[allow(clippy::all)] pub mod diamond_cut_facet; #[allow(clippy::all)] -pub mod diamond_loupe_facet; +pub mod ownership_facet; #[allow(clippy::all)] pub mod gateway_diamond; #[allow(clippy::all)] -pub mod gateway_getter_facet; -#[allow(clippy::all)] pub mod gateway_manager_facet; #[allow(clippy::all)] -pub mod gateway_messenger_facet; -#[allow(clippy::all)] -pub mod i_diamond; -#[allow(clippy::all)] -pub mod lib_gateway; -#[allow(clippy::all)] -pub mod lib_quorum; +pub mod gateway_getter_facet; #[allow(clippy::all)] -pub mod lib_staking; +pub mod checkpointing_facet; #[allow(clippy::all)] -pub mod lib_staking_change_log; +pub mod top_down_finality_facet; #[allow(clippy::all)] -pub mod ownership_facet; +pub mod xnet_messaging_facet; #[allow(clippy::all)] -pub mod register_subnet_facet; +pub mod gateway_messenger_facet; #[allow(clippy::all)] pub mod subnet_actor_checkpointing_facet; #[allow(clippy::all)] @@ -42,13 +36,19 @@ pub mod subnet_actor_pause_facet; #[allow(clippy::all)] pub mod subnet_actor_reward_facet; #[allow(clippy::all)] +pub mod subnet_registry_diamond; +#[allow(clippy::all)] +pub mod register_subnet_facet; +#[allow(clippy::all)] pub mod subnet_getter_facet; #[allow(clippy::all)] -pub mod subnet_registry_diamond; +pub mod lib_staking; #[allow(clippy::all)] -pub mod top_down_finality_facet; +pub mod lib_staking_change_log; #[allow(clippy::all)] -pub mod xnet_messaging_facet; +pub mod lib_gateway; +#[allow(clippy::all)] +pub mod lib_quorum; // The list of contracts need to convert FvmAddress to fvm_shared::Address fvm_address_conversion!(gateway_manager_facet); diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index d832f1b27..44b2c46d5 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -8,7 +8,6 @@ pub mod payload; pub mod store; mod tally; -use std::collections::HashMap; use crate::sync::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; @@ -17,11 +16,12 @@ use crate::vote::store::VoteStore; use crate::vote::tally::VoteTally; use crate::BlockHeight; use error::Error; +use fendermint_crypto::SecretKey; +use fendermint_vm_genesis::ValidatorKey; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::time::Duration; use tokio::sync::{broadcast, mpsc, oneshot}; -use fendermint_crypto::SecretKey; -use fendermint_vm_genesis::ValidatorKey; pub type Weight = u64; @@ -243,7 +243,7 @@ where Ok(v) => Vote::v1(ValidatorKey::new(self.validator_key.public_key()), v), Err(e) => { tracing::error!(err = e.to_string(), "cannot sign received proposal"); - return + return; } }; diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index f22f7a47f..b607288d1 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -69,7 +69,10 @@ pub struct CertifiedObservation { impl Vote { pub fn v1(validator_key: ValidatorKey, obs: CertifiedObservation) -> Self { - Self::V1 {validator: validator_key, payload: obs } + Self::V1 { + validator: validator_key, + payload: obs, + } } pub fn v1_checked(obs: CertifiedObservation) -> anyhow::Result { @@ -129,7 +132,12 @@ impl CertifiedObservation { } impl Observation { - pub fn new(local_hash: Bytes, parent_height: BlockHeight, parent_hash: Bytes, commitment: Bytes) -> Self { + pub fn new( + local_hash: Bytes, + parent_height: BlockHeight, + parent_hash: Bytes, + commitment: Bytes, + ) -> Self { Self { local_hash, ballot: Ballot { diff --git a/fendermint/vm/topdown/src/vote/store.rs b/fendermint/vm/topdown/src/vote/store.rs index 1f3c13ba4..098902b3b 100644 --- a/fendermint/vm/topdown/src/vote/store.rs +++ b/fendermint/vm/topdown/src/vote/store.rs @@ -89,7 +89,9 @@ impl<'a> VoteAgg<'a> { Self(votes) } - pub fn is_empty(&self) -> bool { self.0.is_empty() } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } pub fn into_owned(self) -> Vec { self.0.into_iter().cloned().collect() @@ -156,18 +158,24 @@ mod tests { let observation1 = random_observation(); votes.push( - Vote::v1_checked(CertifiedObservation::sign(observation1.clone(), &validators[0].0).unwrap()) - .unwrap(), + Vote::v1_checked( + CertifiedObservation::sign(observation1.clone(), &validators[0].0).unwrap(), + ) + .unwrap(), ); let observation2 = random_observation(); votes.push( - Vote::v1_checked(CertifiedObservation::sign(observation2.clone(), &validators[1].0).unwrap()) - .unwrap(), + Vote::v1_checked( + CertifiedObservation::sign(observation2.clone(), &validators[1].0).unwrap(), + ) + .unwrap(), ); votes.push( - Vote::v1_checked(CertifiedObservation::sign(observation2.clone(), &validators[2].0).unwrap()) - .unwrap(), + Vote::v1_checked( + CertifiedObservation::sign(observation2.clone(), &validators[2].0).unwrap(), + ) + .unwrap(), ); let agg = VoteAgg(votes.iter().collect()); diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index f23f1a247..f309c82c9 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -90,7 +90,7 @@ impl VoteTally { return Ok(r); }; - for h in self.last_finalized_height+1..=latest { + for h in self.last_finalized_height + 1..=latest { let votes = self.votes.get_votes_at_height(h)?; if votes.is_empty() { continue; @@ -251,12 +251,14 @@ mod tests { let obs = random_observation(); let vote = - Vote::v1_checked(CertifiedObservation::sign(obs.clone(), &validators[0].0).unwrap()).unwrap(); + Vote::v1_checked(CertifiedObservation::sign(obs.clone(), &validators[0].0).unwrap()) + .unwrap(); vote_tally.add_vote(vote).unwrap(); let mut obs2 = random_observation(); obs2.ballot.parent_height = obs.ballot.parent_height(); - let vote = Vote::v1_checked(CertifiedObservation::sign(obs2, &validators[0].0).unwrap()).unwrap(); + let vote = + Vote::v1_checked(CertifiedObservation::sign(obs2, &validators[0].0).unwrap()).unwrap(); assert_eq!(vote_tally.add_vote(vote), Err(Error::Equivocation)); } diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs index 7c3876291..310850ddf 100644 --- a/fendermint/vm/topdown/tests/vote_reactor.rs +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -6,16 +6,16 @@ //! ``` use async_trait::async_trait; -use tokio::sync::broadcast; -use tokio::sync::broadcast::error::TryRecvError; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; use fendermint_vm_topdown::sync::TopDownSyncEvent; use fendermint_vm_topdown::vote::error::Error; use fendermint_vm_topdown::vote::gossip::GossipClient; use fendermint_vm_topdown::vote::payload::{Observation, PowerUpdates, Vote}; -use fendermint_vm_topdown::vote::{Config, start_vote_reactor, Weight}; use fendermint_vm_topdown::vote::store::InMemoryVoteStore; +use fendermint_vm_topdown::vote::{start_vote_reactor, Config, Weight}; +use tokio::sync::broadcast; +use tokio::sync::broadcast::error::TryRecvError; struct Validator { sk: SecretKey, @@ -40,7 +40,7 @@ impl GossipClient for ChannelGossipClient { match rx.try_recv() { Ok(v) => return Ok(Some(v)), Err(TryRecvError::Empty) => continue, - _ => panic!("should not happen") + _ => panic!("should not happen"), } } @@ -54,7 +54,7 @@ impl GossipClient for ChannelGossipClient { } fn default_config() -> Config { - Config{ + Config { req_channel_buffer_size: 1024, req_batch_processing_size: 10, gossip_req_processing_size: 10, @@ -69,7 +69,7 @@ fn gen_validators(weights: Vec) -> (Vec, Vec) -> (Vec, Vec>(); (validators, gossips) @@ -95,7 +98,10 @@ fn gen_validators(weights: Vec) -> (Vec, Vec PowerUpdates { - validators.iter().map(|v| (v.validator_key(), v.weight)).collect() + validators + .iter() + .map(|v| (v.validator_key(), v.weight)) + .collect() } #[tokio::test] @@ -117,14 +123,17 @@ async fn simple_lifecycle() { gossips.pop().unwrap(), InMemoryVoteStore::default(), internal_event_tx.subscribe(), - ).unwrap(); + ) + .unwrap(); assert_eq!(client.find_quorum().await.unwrap(), None); // now topdown sync published a new observation on parent height 100 let parent_height = 100; - let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2,3,4]); - internal_event_tx.send(TopDownSyncEvent::NewProposal(Box::new(obs))).unwrap(); + let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2, 3, 4]); + internal_event_tx + .send(TopDownSyncEvent::NewProposal(Box::new(obs))) + .unwrap(); // wait for vote to be casted while client.find_quorum().await.unwrap().is_none() {} @@ -135,7 +144,11 @@ async fn simple_lifecycle() { let r = client.query_votes(parent_height).await.unwrap().unwrap(); assert_eq!(r.len(), 1); - client.set_quorum_finalized(parent_height).await.unwrap().unwrap(); + client + .set_quorum_finalized(parent_height) + .await + .unwrap() + .unwrap(); // now votes are cleared assert_eq!(client.find_quorum().await.unwrap(), None); @@ -165,7 +178,8 @@ async fn waiting_for_quorum() { gossips.pop().unwrap(), InMemoryVoteStore::default(), internal_event_tx.subscribe(), - ).unwrap(); + ) + .unwrap(); clients.push(client); internal_txs.push(internal_event_tx); @@ -173,40 +187,87 @@ async fn waiting_for_quorum() { // now topdown sync published a new observation on parent height 100 let parent_height1 = 100; - let obs1 = Observation::new(vec![100], parent_height1, vec![1, 2, 3], vec![2,3,4]); + let obs1 = Observation::new(vec![100], parent_height1, vec![1, 2, 3], vec![2, 3, 4]); let parent_height2 = 110; - let obs2 = Observation::new(vec![100], parent_height2, vec![1, 2, 3], vec![2,3,4]); + let obs2 = Observation::new(vec![100], parent_height2, vec![1, 2, 3], vec![2, 3, 4]); let parent_height3 = 120; - let obs3 = Observation::new(vec![100], parent_height3, vec![1, 2, 3], vec![2,3,4]); - - internal_txs[0].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); - internal_txs[1].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); - - internal_txs[2].send(TopDownSyncEvent::NewProposal(Box::new(obs2.clone()))).unwrap(); - internal_txs[3].send(TopDownSyncEvent::NewProposal(Box::new(obs2.clone()))).unwrap(); - - internal_txs[4].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); + let obs3 = Observation::new(vec![100], parent_height3, vec![1, 2, 3], vec![2, 3, 4]); + + internal_txs[0] + .send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))) + .unwrap(); + internal_txs[1] + .send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))) + .unwrap(); + + internal_txs[2] + .send(TopDownSyncEvent::NewProposal(Box::new(obs2.clone()))) + .unwrap(); + internal_txs[3] + .send(TopDownSyncEvent::NewProposal(Box::new(obs2.clone()))) + .unwrap(); + + internal_txs[4] + .send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))) + .unwrap(); // ensure votes are received for client in &clients { - while client.query_votes(parent_height1).await.unwrap().unwrap().len() != 2 {} - while client.query_votes(parent_height2).await.unwrap().unwrap().len() != 2 {} - while client.query_votes(parent_height3).await.unwrap().unwrap().len() != 1 {} + while client + .query_votes(parent_height1) + .await + .unwrap() + .unwrap() + .len() + != 2 + {} + while client + .query_votes(parent_height2) + .await + .unwrap() + .unwrap() + .len() + != 2 + {} + while client + .query_votes(parent_height3) + .await + .unwrap() + .unwrap() + .len() + != 1 + {} } // at this moment, no quorum should have ever formed for client in &clients { - assert!(client.find_quorum().await.unwrap().is_none(), "should have no quorum"); + assert!( + client.find_quorum().await.unwrap().is_none(), + "should have no quorum" + ); } // new observations made - internal_txs[3].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); - internal_txs[0].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); - internal_txs[1].send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))).unwrap(); + internal_txs[3] + .send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))) + .unwrap(); + internal_txs[0] + .send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))) + .unwrap(); + internal_txs[1] + .send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))) + .unwrap(); // ensure every client receives the votes for client in &clients { - while client.query_votes(parent_height3).await.unwrap().unwrap().len() != 4 {} + while client + .query_votes(parent_height3) + .await + .unwrap() + .unwrap() + .len() + != 4 + {} } for client in &clients { @@ -215,24 +276,46 @@ async fn waiting_for_quorum() { } // make observation on previous heights - internal_txs[3].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); - internal_txs[2].send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))).unwrap(); + internal_txs[3] + .send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))) + .unwrap(); + internal_txs[2] + .send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))) + .unwrap(); // ensure every client receives the votes for client in &clients { - while client.query_votes(parent_height1).await.unwrap().unwrap().len() != 4 {} + while client + .query_votes(parent_height1) + .await + .unwrap() + .unwrap() + .len() + != 4 + {} } // but larger parent height wins for client in &clients { let r = client.find_quorum().await.unwrap().unwrap(); - assert_eq!(r.parent_height(), parent_height3, "should have formed quorum on larger height"); + assert_eq!( + r.parent_height(), + parent_height3, + "should have formed quorum on larger height" + ); } // finalize parent height 3 for client in &clients { - client.set_quorum_finalized(parent_height3).await.unwrap().unwrap(); - assert!(client.dump_votes().await.unwrap().unwrap().is_empty(), "should have empty votes"); + client + .set_quorum_finalized(parent_height3) + .await + .unwrap() + .unwrap(); + assert!( + client.dump_votes().await.unwrap().unwrap().is_empty(), + "should have empty votes" + ); } } @@ -258,13 +341,15 @@ async fn all_validator_in_sync() { InMemoryVoteStore::default(), internal_event_tx.subscribe(), ) - .unwrap(); + .unwrap(); node_clients.push(r); } let parent_height = 100; - let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2,3,4]); - internal_event_tx.send(TopDownSyncEvent::NewProposal(Box::new(obs))).unwrap(); + let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2, 3, 4]); + internal_event_tx + .send(TopDownSyncEvent::NewProposal(Box::new(obs))) + .unwrap(); for n in node_clients { while n.find_quorum().await.unwrap().is_none() {} @@ -272,4 +357,4 @@ async fn all_validator_in_sync() { let r = n.find_quorum().await.unwrap().unwrap(); assert_eq!(r.parent_height(), parent_height) } -} \ No newline at end of file +} From 1c7ce517bbf6be293fbc8f8dfb2ee49f04b7426d Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Wed, 25 Sep 2024 12:30:46 +0800 Subject: [PATCH 08/16] more tests --- fendermint/vm/topdown/tests/vote_reactor.rs | 105 ++++++++++---------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs index 310850ddf..d5b571d6c 100644 --- a/fendermint/vm/topdown/tests/vote_reactor.rs +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -13,7 +13,8 @@ use fendermint_vm_topdown::vote::error::Error; use fendermint_vm_topdown::vote::gossip::GossipClient; use fendermint_vm_topdown::vote::payload::{Observation, PowerUpdates, Vote}; use fendermint_vm_topdown::vote::store::InMemoryVoteStore; -use fendermint_vm_topdown::vote::{start_vote_reactor, Config, Weight}; +use fendermint_vm_topdown::vote::{start_vote_reactor, Config, VoteReactorClient, Weight}; +use fendermint_vm_topdown::BlockHeight; use tokio::sync::broadcast; use tokio::sync::broadcast::error::TryRecvError; @@ -104,6 +105,17 @@ fn gen_power_updates(validators: &[Validator]) -> PowerUpdates { .collect() } +async fn ensure_votes_received( + clients: &[VoteReactorClient], + height_votes: Vec<(BlockHeight, usize)>, +) { + for client in clients { + for (height, votes) in &height_votes { + while client.query_votes(*height).await.unwrap().unwrap().len() != *votes {} + } + } +} + #[tokio::test] async fn simple_lifecycle() { let config = default_config(); @@ -144,23 +156,45 @@ async fn simple_lifecycle() { let r = client.query_votes(parent_height).await.unwrap().unwrap(); assert_eq!(r.len(), 1); + // now push another observation + let parent_height2 = 101; + let obs2 = Observation::new(vec![100], parent_height2, vec![1, 2, 3], vec![2, 3, 4]); + internal_event_tx + .send(TopDownSyncEvent::NewProposal(Box::new(obs2))) + .unwrap(); + client .set_quorum_finalized(parent_height) .await .unwrap() .unwrap(); - // now votes are cleared - assert_eq!(client.find_quorum().await.unwrap(), None); let state = client.query_vote_tally_state().await.unwrap(); assert_eq!(state.last_finalized_height, parent_height); + + let votes = client.query_votes(parent_height2).await.unwrap().unwrap(); + assert_eq!(votes.len(), 1); + let r = client.find_quorum().await.unwrap().unwrap(); + assert_eq!(r.parent_height(), parent_height2); + + client + .set_quorum_finalized(parent_height2) + .await + .unwrap() + .unwrap(); + + assert_eq!(client.find_quorum().await.unwrap(), None); + assert!( + client.dump_votes().await.unwrap().unwrap().is_empty(), + "should have no votes left" + ); } +/// This tests votes coming in the wrong block height order and it still works #[tokio::test] async fn waiting_for_quorum() { let config = default_config(); - // 21 validators equal 100 weight let (validators, mut gossips) = gen_validators(vec![100; 5]); let power_updates = gen_power_updates(&validators); let initial_finalized_height = 10; @@ -211,33 +245,15 @@ async fn waiting_for_quorum() { .send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))) .unwrap(); - // ensure votes are received - for client in &clients { - while client - .query_votes(parent_height1) - .await - .unwrap() - .unwrap() - .len() - != 2 - {} - while client - .query_votes(parent_height2) - .await - .unwrap() - .unwrap() - .len() - != 2 - {} - while client - .query_votes(parent_height3) - .await - .unwrap() - .unwrap() - .len() - != 1 - {} - } + ensure_votes_received( + &clients, + vec![ + (parent_height1, 2), + (parent_height2, 2), + (parent_height3, 1), + ], + ) + .await; // at this moment, no quorum should have ever formed for client in &clients { @@ -258,17 +274,7 @@ async fn waiting_for_quorum() { .send(TopDownSyncEvent::NewProposal(Box::new(obs3.clone()))) .unwrap(); - // ensure every client receives the votes - for client in &clients { - while client - .query_votes(parent_height3) - .await - .unwrap() - .unwrap() - .len() - != 4 - {} - } + ensure_votes_received(&clients, vec![(parent_height3, 4)]).await; for client in &clients { let r = client.find_quorum().await.unwrap().unwrap(); @@ -284,16 +290,7 @@ async fn waiting_for_quorum() { .unwrap(); // ensure every client receives the votes - for client in &clients { - while client - .query_votes(parent_height1) - .await - .unwrap() - .unwrap() - .len() - != 4 - {} - } + ensure_votes_received(&clients, vec![(parent_height1, 4)]).await; // but larger parent height wins for client in &clients { @@ -331,10 +328,10 @@ async fn all_validator_in_sync() { let (internal_event_tx, _) = broadcast::channel(validators.len() + 1); let mut node_clients = vec![]; - for i in 0..validators.len() { + for validator in &validators { let r = start_vote_reactor( config.clone(), - validators[i].sk.clone(), + validator.sk.clone(), power_updates.clone(), initial_finalized_height, gossips.pop().unwrap(), From 7fd001cc91f6c19652a144abe6493101f9b0d8fb Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Wed, 25 Sep 2024 15:47:08 +0800 Subject: [PATCH 09/16] wip --- fendermint/vm/topdown/src/lib.rs | 1 + fendermint/vm/topdown/src/proxy.rs | 24 ++ fendermint/vm/topdown/src/sync/mod.rs | 7 - fendermint/vm/topdown/src/syncer/error.rs | 18 ++ fendermint/vm/topdown/src/syncer/mod.rs | 53 ++++ fendermint/vm/topdown/src/syncer/payload.rs | 38 +++ fendermint/vm/topdown/src/syncer/reactor.rs | 261 ++++++++++++++++++ fendermint/vm/topdown/src/syncer/store.rs | 22 ++ fendermint/vm/topdown/src/vote/mod.rs | 2 +- .../vm/topdown/src/vote/operation/active.rs | 2 +- .../vm/topdown/src/vote/operation/paused.rs | 2 +- fendermint/vm/topdown/tests/vote_reactor.rs | 2 +- 12 files changed, 421 insertions(+), 11 deletions(-) create mode 100644 fendermint/vm/topdown/src/syncer/error.rs create mode 100644 fendermint/vm/topdown/src/syncer/mod.rs create mode 100644 fendermint/vm/topdown/src/syncer/payload.rs create mode 100644 fendermint/vm/topdown/src/syncer/reactor.rs create mode 100644 fendermint/vm/topdown/src/syncer/store.rs diff --git a/fendermint/vm/topdown/src/lib.rs b/fendermint/vm/topdown/src/lib.rs index d0fd7e6ca..d290edc25 100644 --- a/fendermint/vm/topdown/src/lib.rs +++ b/fendermint/vm/topdown/src/lib.rs @@ -13,6 +13,7 @@ pub mod voting; pub mod observe; pub mod vote; +pub mod syncer; use async_stm::Stm; use async_trait::async_trait; diff --git a/fendermint/vm/topdown/src/proxy.rs b/fendermint/vm/topdown/src/proxy.rs index 94a8e3177..0724adca9 100644 --- a/fendermint/vm/topdown/src/proxy.rs +++ b/fendermint/vm/topdown/src/proxy.rs @@ -1,6 +1,7 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use std::sync::Arc; use crate::observe::ParentRpcCalled; use crate::BlockHeight; use anyhow::anyhow; @@ -42,6 +43,29 @@ pub trait ParentQueryProxy { ) -> anyhow::Result>>; } +#[async_trait] +impl ParentQueryProxy for Arc

{ + async fn get_chain_head_height(&self) -> Result { + self.as_ref().get_chain_head_height().await + } + + async fn get_genesis_epoch(&self) -> Result { + self.as_ref().get_genesis_epoch().await + } + + async fn get_block_hash(&self, height: BlockHeight) -> Result { + self.as_ref().get_block_hash(height).await + } + + async fn get_top_down_msgs(&self, height: BlockHeight) -> Result>> { + self.as_ref().get_top_down_msgs(height).await + } + + async fn get_validator_changes(&self, height: BlockHeight) -> Result>> { + self.as_ref().get_validator_changes(height).await + } +} + /// The proxy to the subnet's parent pub struct IPCProviderProxy { ipc_provider: IpcProvider, diff --git a/fendermint/vm/topdown/src/sync/mod.rs b/fendermint/vm/topdown/src/sync/mod.rs index d6f1341e5..2cfe2a0f9 100644 --- a/fendermint/vm/topdown/src/sync/mod.rs +++ b/fendermint/vm/topdown/src/sync/mod.rs @@ -19,15 +19,8 @@ use std::time::Duration; use fendermint_vm_genesis::{Power, Validator}; -use crate::vote::payload::Observation; pub use syncer::fetch_topdown_events; -#[derive(Clone, Debug)] -pub enum TopDownSyncEvent { - NodeSyncing, - NewProposal(Box), -} - /// Query the parent finality from the block chain state. /// /// It returns `None` from queries until the ledger has been initialized. diff --git a/fendermint/vm/topdown/src/syncer/error.rs b/fendermint/vm/topdown/src/syncer/error.rs new file mode 100644 index 000000000..cbfbeed5c --- /dev/null +++ b/fendermint/vm/topdown/src/syncer/error.rs @@ -0,0 +1,18 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::BlockHeight; +use thiserror::Error; + +/// The errors for top down checkpointing +#[derive(Error, Debug, Eq, PartialEq, Clone)] +pub enum Error { + #[error("Incoming items are not order sequentially")] + NotSequential, + #[error("The parent view update with block height is not sequential")] + NonSequentialParentViewInsert, + #[error("Parent chain reorg detected")] + ParentChainReorgDetected, + #[error("Cannot query parent at height {1}: {0}")] + CannotQueryParent(String, BlockHeight), +} diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs new file mode 100644 index 000000000..1217afa45 --- /dev/null +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -0,0 +1,53 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::time::Duration; +use tokio::sync::mpsc; +use crate::BlockHeight; +use crate::vote::payload::Observation; + +pub mod payload; +pub mod reactor; +pub mod store; +pub mod error; + +#[derive(Clone, Debug)] +pub enum TopDownSyncEvent { + /// The fendermint node is syncing with peers + NodeSyncing, + /// The parent view store is full, this will pause the parent syncer + ParentViewStoreFull, + NewProposal(Box), +} + +pub struct ParentSyncerConfig { + pub request_channel_size: usize, + /// The number of blocks to delay before reporting a height as final on the parent chain. + /// To propose a certain number of epochs delayed from the latest height, we see to be + /// conservative and avoid other from rejecting the proposal because they don't see the + /// height as final yet. + pub chain_head_delay: BlockHeight, + /// Parent syncing cron period, in seconds + pub polling_interval: Duration, + /// Top down exponential back off retry base + pub exponential_back_off: Duration, + /// The max number of retries for exponential backoff before giving up + pub exponential_retry_limit: usize, + /// Max number of un-finalized parent blocks that should be stored in the store + pub max_store_blocks: BlockHeight, + /// Attempts to sync as many block as possible till the finalized chain head + pub sync_many: bool, +} + +pub struct ParentSyncerReactorClient { + tx: mpsc::Sender<()>, + +} + +pub fn start_parent_syncer(config: ParentSyncerConfig) -> anyhow::Result { + let (tx, rx) = mpsc::channel(config.request_channel_size); + + tokio::spawn(async move { + }); + Ok(ParentSyncerReactorClient { tx }) +} diff --git a/fendermint/vm/topdown/src/syncer/payload.rs b/fendermint/vm/topdown/src/syncer/payload.rs new file mode 100644 index 000000000..99338e8a0 --- /dev/null +++ b/fendermint/vm/topdown/src/syncer/payload.rs @@ -0,0 +1,38 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use ipc_api::cross::IpcEnvelope; +use ipc_api::staking::StakingChangeRequest; +use crate::{BlockHash, BlockHeight}; + +#[derive(Clone, Debug)] +pub struct ParentViewPayload { + pub parent_hash: BlockHash, + /// Encodes cross-net messages. + pub xnet_msgs: Vec, + /// Encodes validator membership change commands. + pub validator_changes: Vec, +} + +#[derive(Clone, Debug)] +pub struct ParentView { + pub parent_height: BlockHeight, + /// If the payload is None, this means the parent height is a null block + pub payload: Option, +} + +impl ParentView { + pub fn null_block(h: BlockHeight) -> Self { + Self { parent_height: h, payload: None } + } + + pub fn nonnull_block( + h: BlockHeight, + parent_hash: BlockHash, + xnet_msgs: Vec, + validator_changes: Vec + ) -> Self { + Self { parent_height: h, payload: Some(ParentViewPayload { parent_hash, xnet_msgs, validator_changes }) } + } +} + diff --git a/fendermint/vm/topdown/src/syncer/reactor.rs b/fendermint/vm/topdown/src/syncer/reactor.rs new file mode 100644 index 000000000..7989bdf41 --- /dev/null +++ b/fendermint/vm/topdown/src/syncer/reactor.rs @@ -0,0 +1,261 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT +//! The inner type of parent syncer + +use anyhow::anyhow; +use libp2p::futures::TryFutureExt; +use tokio::sync::broadcast; +use ipc_observability::emit; +use ipc_observability::serde::HexEncodableBlockHash; +use crate::{BlockHash, BlockHeight, IPCParentFinality, is_null_round_str}; +use crate::observe::ParentFinalityAcquired; +use crate::proxy::ParentQueryProxy; +use crate::syncer::error::Error; +use crate::syncer::{ParentSyncerConfig, TopDownSyncEvent}; +use crate::syncer::payload::ParentView; +use crate::syncer::store::ParentViewStore; +use tracing::instrument; + +struct ParentSyncerReactor { + config: ParentSyncerConfig, + parent_proxy: P, + store: S, + event_broadcast: broadcast::Sender, + last_finalized: IPCParentFinality, +} + +impl ParentSyncerReactor +where + S: ParentViewStore + Send + Sync + 'static, + P: Send + Sync + 'static + ParentQueryProxy +{ + async fn run(&mut self) { + loop { + if let Err(e) = self.sync().await { + tracing::error!(err = e.to_string(), "cannot sync with parent"); + } + tokio::time::sleep(self.config.polling_interval).await; + } + } + + /// Get the latest non null block data stored + async fn latest_nonnull_data(&self) -> anyhow::Result<(BlockHeight, BlockHash)> { + let Some(latest_height) = self.store.max_parent_view_height()? else { + return Ok((self.last_finalized.height, self.last_finalized.block_hash.clone())); + }; + + let start = self.last_finalized.height + 1; + for h in (start..=latest_height).rev() { + let Some(view) = self.store.get(h)? else { + continue; + }; + + let Some(payload) = view.payload else { + continue; + }; + + return Ok((h, payload.parent_hash)) + } + + // this means the votes stored are all null blocks, return last committed finality + Ok((self.last_finalized.height, self.last_finalized.block_hash.clone())) + } + + /// Insert the height into cache when we see a new non null block + async fn sync(&mut self) -> anyhow::Result<()> { + let Some(chain_head) = self.finalized_chain_head().await? else { + return Ok(()); + }; + + let (mut latest_height_fetched, mut first_non_null_parent_hash) = self.latest_nonnull_data().await?; + tracing::debug!(chain_head, latest_height_fetched, "syncing heights"); + + if latest_height_fetched > chain_head { + tracing::warn!( + chain_head, + latest_height_fetched, + "chain head went backwards, potential reorg detected from height" + ); + todo!("handle reorg, maybe just a warning???") + } + + if latest_height_fetched == chain_head { + tracing::debug!( + chain_head, + latest_height_fetched, + "the parent has yet to produce a new block" + ); + return Ok(()); + } + + loop { + if self.store_full()? { + tracing::debug!("exceeded cache size limit"); + let _ = self.event_broadcast.send(TopDownSyncEvent::ParentViewStoreFull); + break; + } + + first_non_null_parent_hash = match self + .poll_next(latest_height_fetched + 1, first_non_null_parent_hash) + .await + { + Ok(h) => h, + Err(Error::ParentChainReorgDetected) => { + tracing::warn!("potential reorg detected, clear cache and retry"); + todo!(); + // break; + } + Err(e) => return Err(anyhow!(e)), + }; + + latest_height_fetched += 1; + + if latest_height_fetched == chain_head { + tracing::debug!("reached the tip of the chain"); + break; + } else if !self.config.sync_many { + break; + } + } + + Ok(()) + } + + fn store_full(&self) -> anyhow::Result { + let Some(h) = self.store.max_parent_view_height()? else { + return Ok(false) + }; + Ok(h - self.last_finalized.height > self.config.max_store_blocks) + } + + async fn finalized_chain_head(&self) -> anyhow::Result> { + let parent_chain_head_height = self.parent_proxy.get_chain_head_height().await?; + // sanity check + if parent_chain_head_height < self.config.chain_head_delay { + tracing::debug!("latest height not more than the chain head delay"); + return Ok(None); + } + + // we consider the chain head finalized only after the `chain_head_delay` + Ok(Some( + parent_chain_head_height - self.config.chain_head_delay, + )) + } + + /// Poll the next block height. Returns finalized and executed block data. + async fn poll_next( + &mut self, + height: BlockHeight, + parent_block_hash: BlockHash, + ) -> Result { + tracing::debug!( + height, + parent_block_hash = hex::encode(&parent_block_hash), + "polling height with parent hash" + ); + + let block_hash_res = match self.parent_proxy.get_block_hash(height).await { + Ok(res) => res, + Err(e) => { + let err = e.to_string(); + if is_null_round_str(&err) { + tracing::debug!( + height, + "detected null round at height, inserted None to cache" + ); + + self.store.store(ParentView::null_block(height))?; + + emit(ParentFinalityAcquired { + source: "Parent syncer", + is_null: true, + block_height: height, + block_hash: None, + commitment_hash: None, + num_msgs: 0, + num_validator_changes: 0, + }); + + // Null block received, no block hash for the current height being polled. + // Return the previous parent hash as the non-null block hash. + return Ok(parent_block_hash); + } + return Err(Error::CannotQueryParent( + format!("get_block_hash: {e}"), + height, + )); + } + }; + + if block_hash_res.parent_block_hash != parent_block_hash { + tracing::warn!( + height, + parent_hash = hex::encode(&block_hash_res.parent_block_hash), + previous_hash = hex::encode(&parent_block_hash), + "parent block hash diff than previous hash", + ); + return Err(Error::ParentChainReorgDetected); + } + + let view = fetch_data(&self.parent_proxy, height, block_hash_res.block_hash).await?; + self.store.store(view.clone())?; + + let payload = view.payload.as_ref().unwrap(); + emit(ParentFinalityAcquired { + source: "Parent syncer", + is_null: false, + block_height: height, + block_hash: Some(HexEncodableBlockHash(payload.parent_hash.clone())), + // TODO Karel, Willes - when we introduce commitment hash, we should add it here + commitment_hash: None, + num_msgs: payload.xnet_msgs.len(), + num_validator_changes: payload.validator_changes.len(), + }); + + Ok(view.payload.unwrap().parent_hash) + } + +} + +#[instrument(skip(parent_proxy))] +async fn fetch_data

( + parent_proxy: &P, + height: BlockHeight, + block_hash: BlockHash, +) -> Result + where + P: ParentQueryProxy + Send + Sync + 'static, +{ + let changes_res = parent_proxy + .get_validator_changes(height) + .map_err(|e| Error::CannotQueryParent(format!("get_validator_changes: {e}"), height)); + + let topdown_msgs_res = parent_proxy + .get_top_down_msgs(height) + .map_err(|e| Error::CannotQueryParent(format!("get_top_down_msgs: {e}"), height)); + + let (changes_res, topdown_msgs_res) = tokio::join!(changes_res, topdown_msgs_res); + let (changes_res, topdown_msgs_res) = (changes_res?, topdown_msgs_res?); + + if changes_res.block_hash != block_hash { + tracing::warn!( + height, + change_set_hash = hex::encode(&changes_res.block_hash), + block_hash = hex::encode(&block_hash), + "change set block hash does not equal block hash", + ); + return Err(Error::ParentChainReorgDetected); + } + + if topdown_msgs_res.block_hash != block_hash { + tracing::warn!( + height, + topdown_msgs_hash = hex::encode(&topdown_msgs_res.block_hash), + block_hash = hex::encode(&block_hash), + "topdown messages block hash does not equal block hash", + ); + return Err(Error::ParentChainReorgDetected); + } + + Ok(ParentView::nonnull_block(height, block_hash, topdown_msgs_res.value, changes_res.value)) +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs new file mode 100644 index 000000000..32f59fba2 --- /dev/null +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -0,0 +1,22 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::BlockHeight; +use crate::syncer::error::Error; +use crate::syncer::payload::ParentView; + +/// Stores the parent view observed of the current node +pub trait ParentViewStore { + /// Store a newly observed parent view + fn store(&mut self, view: ParentView) -> Result<(), Error>; + + /// Get the parent view at the specified height + fn get(&self, height: BlockHeight) -> Result, Error>; + + /// Purge the parent view at the target height + fn purge(&mut self, height: BlockHeight) -> Result<(), Error>; + + fn minimal_parent_view_height(&self) -> Result, Error>; + + fn max_parent_view_height(&self) -> Result, Error>; +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index 44b2c46d5..51513a66b 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -8,7 +8,7 @@ pub mod payload; pub mod store; mod tally; -use crate::sync::TopDownSyncEvent; +use crate::syncer::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; use crate::vote::payload::{Ballot, CertifiedObservation, PowerUpdates, Vote, VoteTallyState}; diff --git a/fendermint/vm/topdown/src/vote/operation/active.rs b/fendermint/vm/topdown/src/vote/operation/active.rs index d639004f2..898d5762d 100644 --- a/fendermint/vm/topdown/src/vote/operation/active.rs +++ b/fendermint/vm/topdown/src/vote/operation/active.rs @@ -5,7 +5,7 @@ use crate::vote::gossip::GossipClient; use crate::vote::operation::paused::PausedOperationMode; use crate::vote::operation::{OperationMetrics, OperationStateMachine, ACTIVE, PAUSED}; use crate::vote::store::VoteStore; -use crate::vote::TopDownSyncEvent; +use crate::syncer::TopDownSyncEvent; use crate::vote::VotingHandler; use std::fmt::{Display, Formatter}; diff --git a/fendermint/vm/topdown/src/vote/operation/paused.rs b/fendermint/vm/topdown/src/vote/operation/paused.rs index 33931e4e8..ee86bbf79 100644 --- a/fendermint/vm/topdown/src/vote/operation/paused.rs +++ b/fendermint/vm/topdown/src/vote/operation/paused.rs @@ -1,7 +1,7 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::sync::TopDownSyncEvent; +use crate::syncer::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::active::ActiveOperationMode; use crate::vote::operation::{OperationMetrics, OperationStateMachine, ACTIVE, PAUSED}; diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs index d5b571d6c..ed75916d9 100644 --- a/fendermint/vm/topdown/tests/vote_reactor.rs +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -8,7 +8,7 @@ use async_trait::async_trait; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; -use fendermint_vm_topdown::sync::TopDownSyncEvent; +use fendermint_vm_topdown::syncer::TopDownSyncEvent; use fendermint_vm_topdown::vote::error::Error; use fendermint_vm_topdown::vote::gossip::GossipClient; use fendermint_vm_topdown::vote::payload::{Observation, PowerUpdates, Vote}; From 06e43e3f391bccfd8dad6cea03a0b01ee61736c6 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Wed, 25 Sep 2024 17:23:10 +0800 Subject: [PATCH 10/16] wip --- fendermint/vm/topdown/src/lib.rs | 2 +- fendermint/vm/topdown/src/proxy.rs | 14 ++- fendermint/vm/topdown/src/syncer/mod.rs | 68 ++++++++++++-- fendermint/vm/topdown/src/syncer/payload.rs | 19 ++-- .../src/syncer/{reactor.rs => poll.rs} | 89 +++++++++++++------ fendermint/vm/topdown/src/syncer/store.rs | 4 +- .../vm/topdown/src/vote/operation/active.rs | 2 +- 7 files changed, 148 insertions(+), 50 deletions(-) rename fendermint/vm/topdown/src/syncer/{reactor.rs => poll.rs} (82%) diff --git a/fendermint/vm/topdown/src/lib.rs b/fendermint/vm/topdown/src/lib.rs index d290edc25..05076ce1d 100644 --- a/fendermint/vm/topdown/src/lib.rs +++ b/fendermint/vm/topdown/src/lib.rs @@ -12,8 +12,8 @@ mod toggle; pub mod voting; pub mod observe; -pub mod vote; pub mod syncer; +pub mod vote; use async_stm::Stm; use async_trait::async_trait; diff --git a/fendermint/vm/topdown/src/proxy.rs b/fendermint/vm/topdown/src/proxy.rs index 0724adca9..882b11bb0 100644 --- a/fendermint/vm/topdown/src/proxy.rs +++ b/fendermint/vm/topdown/src/proxy.rs @@ -1,7 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use std::sync::Arc; use crate::observe::ParentRpcCalled; use crate::BlockHeight; use anyhow::anyhow; @@ -14,6 +13,7 @@ use ipc_api::subnet_id::SubnetID; use ipc_observability::emit; use ipc_provider::manager::{GetBlockHashResult, TopDownQueryPayload}; use ipc_provider::IpcProvider; +use std::sync::Arc; use std::time::Instant; use tracing::instrument; @@ -44,7 +44,7 @@ pub trait ParentQueryProxy { } #[async_trait] -impl ParentQueryProxy for Arc

{ +impl ParentQueryProxy for Arc

{ async fn get_chain_head_height(&self) -> Result { self.as_ref().get_chain_head_height().await } @@ -57,11 +57,17 @@ impl ParentQueryProxy for Arc

{ self.as_ref().get_block_hash(height).await } - async fn get_top_down_msgs(&self, height: BlockHeight) -> Result>> { + async fn get_top_down_msgs( + &self, + height: BlockHeight, + ) -> Result>> { self.as_ref().get_top_down_msgs(height).await } - async fn get_validator_changes(&self, height: BlockHeight) -> Result>> { + async fn get_validator_changes( + &self, + height: BlockHeight, + ) -> Result>> { self.as_ref().get_validator_changes(height).await } } diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index 1217afa45..b2420b18a 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -1,15 +1,19 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::proxy::ParentQueryProxy; +use crate::syncer::poll::ParentPoll; +use crate::syncer::store::ParentViewStore; +use crate::vote::payload::Observation; +use crate::{BlockHeight, IPCParentFinality}; use std::time::Duration; +use tokio::select; use tokio::sync::mpsc; -use crate::BlockHeight; -use crate::vote::payload::Observation; +pub mod error; pub mod payload; -pub mod reactor; +pub mod poll; pub mod store; -pub mod error; #[derive(Clone, Debug)] pub enum TopDownSyncEvent { @@ -22,6 +26,8 @@ pub enum TopDownSyncEvent { pub struct ParentSyncerConfig { pub request_channel_size: usize, + /// The event broadcast channel buffer size + pub broadcast_channel_size: usize, /// The number of blocks to delay before reporting a height as final on the parent chain. /// To propose a certain number of epochs delayed from the latest height, we see to be /// conservative and avoid other from rejecting the proposal because they don't see the @@ -39,15 +45,61 @@ pub struct ParentSyncerConfig { pub sync_many: bool, } +#[derive(Clone)] pub struct ParentSyncerReactorClient { - tx: mpsc::Sender<()>, - + tx: mpsc::Sender, } -pub fn start_parent_syncer(config: ParentSyncerConfig) -> anyhow::Result { - let (tx, rx) = mpsc::channel(config.request_channel_size); +pub fn start_parent_syncer( + config: ParentSyncerConfig, + proxy: P, + store: S, + last_finalized: IPCParentFinality, +) -> anyhow::Result +where + S: ParentViewStore + Send + Sync + 'static, + P: Send + Sync + 'static + ParentQueryProxy, +{ + let (tx, mut rx) = mpsc::channel(config.request_channel_size); tokio::spawn(async move { + let polling_interval = config.polling_interval; + let mut poller = ParentPoll::new(config, proxy, store, last_finalized); + + loop { + select! { + _ = tokio::time::sleep(polling_interval) => { + if let Err(e) = poller.try_poll().await { + tracing::error!(err = e.to_string(), "cannot sync with parent"); + } + } + req = rx.recv() => { + let Some(req) = req else { break }; + handle_request(req, &mut poller); + } + } + } + + tracing::warn!("parent syncer stopped") }); Ok(ParentSyncerReactorClient { tx }) } + +enum ParentSyncerRequest { + /// A new parent height is finalized + Finalized(BlockHeight), +} + +fn handle_request(req: ParentSyncerRequest, poller: &mut ParentPoll) + where + S: ParentViewStore + Send + Sync + 'static, + P: Send + Sync + 'static + ParentQueryProxy +{ + match req { + ParentSyncerRequest::Finalized(h) => { + if let Err(e) = poller.finalize(h) { + tracing::error!(height = h, err = e.to_string(), "cannot finalize parent viewer"); + } + }, + } +} diff --git a/fendermint/vm/topdown/src/syncer/payload.rs b/fendermint/vm/topdown/src/syncer/payload.rs index 99338e8a0..d501a19bd 100644 --- a/fendermint/vm/topdown/src/syncer/payload.rs +++ b/fendermint/vm/topdown/src/syncer/payload.rs @@ -1,9 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::{BlockHash, BlockHeight}; use ipc_api::cross::IpcEnvelope; use ipc_api::staking::StakingChangeRequest; -use crate::{BlockHash, BlockHeight}; #[derive(Clone, Debug)] pub struct ParentViewPayload { @@ -23,16 +23,25 @@ pub struct ParentView { impl ParentView { pub fn null_block(h: BlockHeight) -> Self { - Self { parent_height: h, payload: None } + Self { + parent_height: h, + payload: None, + } } pub fn nonnull_block( h: BlockHeight, parent_hash: BlockHash, xnet_msgs: Vec, - validator_changes: Vec + validator_changes: Vec, ) -> Self { - Self { parent_height: h, payload: Some(ParentViewPayload { parent_hash, xnet_msgs, validator_changes }) } + Self { + parent_height: h, + payload: Some(ParentViewPayload { + parent_hash, + xnet_msgs, + validator_changes, + }), + } } } - diff --git a/fendermint/vm/topdown/src/syncer/reactor.rs b/fendermint/vm/topdown/src/syncer/poll.rs similarity index 82% rename from fendermint/vm/topdown/src/syncer/reactor.rs rename to fendermint/vm/topdown/src/syncer/poll.rs index 7989bdf41..828577587 100644 --- a/fendermint/vm/topdown/src/syncer/reactor.rs +++ b/fendermint/vm/topdown/src/syncer/poll.rs @@ -2,21 +2,21 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! The inner type of parent syncer -use anyhow::anyhow; -use libp2p::futures::TryFutureExt; -use tokio::sync::broadcast; -use ipc_observability::emit; -use ipc_observability::serde::HexEncodableBlockHash; -use crate::{BlockHash, BlockHeight, IPCParentFinality, is_null_round_str}; use crate::observe::ParentFinalityAcquired; use crate::proxy::ParentQueryProxy; use crate::syncer::error::Error; -use crate::syncer::{ParentSyncerConfig, TopDownSyncEvent}; use crate::syncer::payload::ParentView; use crate::syncer::store::ParentViewStore; +use crate::syncer::{ParentSyncerConfig, TopDownSyncEvent}; +use crate::{is_null_round_str, BlockHash, BlockHeight, IPCParentFinality}; +use anyhow::anyhow; +use ipc_observability::emit; +use ipc_observability::serde::HexEncodableBlockHash; +use libp2p::futures::TryFutureExt; +use tokio::sync::broadcast; use tracing::instrument; -struct ParentSyncerReactor { +pub(crate) struct ParentPoll { config: ParentSyncerConfig, parent_proxy: P, store: S, @@ -24,24 +24,45 @@ struct ParentSyncerReactor { last_finalized: IPCParentFinality, } -impl ParentSyncerReactor +impl ParentPoll where S: ParentViewStore + Send + Sync + 'static, - P: Send + Sync + 'static + ParentQueryProxy + P: Send + Sync + 'static + ParentQueryProxy, { - async fn run(&mut self) { - loop { - if let Err(e) = self.sync().await { - tracing::error!(err = e.to_string(), "cannot sync with parent"); - } - tokio::time::sleep(self.config.polling_interval).await; + pub fn new( + config: ParentSyncerConfig, + proxy: P, + store: S, + last_finalized: IPCParentFinality, + ) -> Self { + let (tx, _) = broadcast::channel(config.broadcast_channel_size); + Self { + config, + parent_proxy: proxy, + store, + event_broadcast: tx, + last_finalized, + } + } + + /// The target block height is finalized + pub fn finalize(&mut self, height: BlockHeight) -> Result<(), Error> { + let Some(min_height) = self.store.minimal_parent_view_height()? else { + return Ok(()); + }; + for h in min_height..=height { + self.store.purge(h)?; } + Ok(()) } /// Get the latest non null block data stored async fn latest_nonnull_data(&self) -> anyhow::Result<(BlockHeight, BlockHash)> { let Some(latest_height) = self.store.max_parent_view_height()? else { - return Ok((self.last_finalized.height, self.last_finalized.block_hash.clone())); + return Ok(( + self.last_finalized.height, + self.last_finalized.block_hash.clone(), + )); }; let start = self.last_finalized.height + 1; @@ -54,20 +75,24 @@ where continue; }; - return Ok((h, payload.parent_hash)) + return Ok((h, payload.parent_hash)); } // this means the votes stored are all null blocks, return last committed finality - Ok((self.last_finalized.height, self.last_finalized.block_hash.clone())) + Ok(( + self.last_finalized.height, + self.last_finalized.block_hash.clone(), + )) } /// Insert the height into cache when we see a new non null block - async fn sync(&mut self) -> anyhow::Result<()> { + pub async fn try_poll(&mut self) -> anyhow::Result<()> { let Some(chain_head) = self.finalized_chain_head().await? else { return Ok(()); }; - let (mut latest_height_fetched, mut first_non_null_parent_hash) = self.latest_nonnull_data().await?; + let (mut latest_height_fetched, mut first_non_null_parent_hash) = + self.latest_nonnull_data().await?; tracing::debug!(chain_head, latest_height_fetched, "syncing heights"); if latest_height_fetched > chain_head { @@ -76,7 +101,7 @@ where latest_height_fetched, "chain head went backwards, potential reorg detected from height" ); - todo!("handle reorg, maybe just a warning???") + todo!("handle reorg, maybe just a warning???") } if latest_height_fetched == chain_head { @@ -91,7 +116,9 @@ where loop { if self.store_full()? { tracing::debug!("exceeded cache size limit"); - let _ = self.event_broadcast.send(TopDownSyncEvent::ParentViewStoreFull); + let _ = self + .event_broadcast + .send(TopDownSyncEvent::ParentViewStoreFull); break; } @@ -123,7 +150,7 @@ where fn store_full(&self) -> anyhow::Result { let Some(h) = self.store.max_parent_view_height()? else { - return Ok(false) + return Ok(false); }; Ok(h - self.last_finalized.height > self.config.max_store_blocks) } @@ -214,7 +241,6 @@ where Ok(view.payload.unwrap().parent_hash) } - } #[instrument(skip(parent_proxy))] @@ -223,8 +249,8 @@ async fn fetch_data

( height: BlockHeight, block_hash: BlockHash, ) -> Result - where - P: ParentQueryProxy + Send + Sync + 'static, +where + P: ParentQueryProxy + Send + Sync + 'static, { let changes_res = parent_proxy .get_validator_changes(height) @@ -257,5 +283,10 @@ async fn fetch_data

( return Err(Error::ParentChainReorgDetected); } - Ok(ParentView::nonnull_block(height, block_hash, topdown_msgs_res.value, changes_res.value)) -} \ No newline at end of file + Ok(ParentView::nonnull_block( + height, + block_hash, + topdown_msgs_res.value, + changes_res.value, + )) +} diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs index 32f59fba2..e9df04c29 100644 --- a/fendermint/vm/topdown/src/syncer/store.rs +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -1,9 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::BlockHeight; use crate::syncer::error::Error; use crate::syncer::payload::ParentView; +use crate::BlockHeight; /// Stores the parent view observed of the current node pub trait ParentViewStore { @@ -19,4 +19,4 @@ pub trait ParentViewStore { fn minimal_parent_view_height(&self) -> Result, Error>; fn max_parent_view_height(&self) -> Result, Error>; -} \ No newline at end of file +} diff --git a/fendermint/vm/topdown/src/vote/operation/active.rs b/fendermint/vm/topdown/src/vote/operation/active.rs index 898d5762d..c03a0bee3 100644 --- a/fendermint/vm/topdown/src/vote/operation/active.rs +++ b/fendermint/vm/topdown/src/vote/operation/active.rs @@ -1,11 +1,11 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::syncer::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::paused::PausedOperationMode; use crate::vote::operation::{OperationMetrics, OperationStateMachine, ACTIVE, PAUSED}; use crate::vote::store::VoteStore; -use crate::syncer::TopDownSyncEvent; use crate::vote::VotingHandler; use std::fmt::{Display, Formatter}; From 9f9f16bad2d8d77b1c21c4fd32a637d7b0e0c1d0 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 26 Sep 2024 22:47:55 +0800 Subject: [PATCH 11/16] wip --- fendermint/vm/topdown/src/lib.rs | 20 +++++++++++++++ fendermint/vm/topdown/src/proposal.rs | 30 +++++++++++++++++++++++ fendermint/vm/topdown/src/syncer/mod.rs | 29 +++++++++++++++------- fendermint/vm/topdown/src/syncer/poll.rs | 24 +++++++++--------- fendermint/vm/topdown/src/syncer/store.rs | 4 +-- 5 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 fendermint/vm/topdown/src/proposal.rs diff --git a/fendermint/vm/topdown/src/lib.rs b/fendermint/vm/topdown/src/lib.rs index 05076ce1d..3faa8b22b 100644 --- a/fendermint/vm/topdown/src/lib.rs +++ b/fendermint/vm/topdown/src/lib.rs @@ -14,6 +14,7 @@ pub mod voting; pub mod observe; pub mod syncer; pub mod vote; +pub(crate) mod proposal; use async_stm::Stm; use async_trait::async_trait; @@ -108,6 +109,25 @@ impl Config { } } +/// On-chain data structure representing a topdown checkpoint agreed to by a +/// majority of subnet validators. DAG-CBOR encoded, embedded in CertifiedCheckpoint. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Checkpoint { + /// Checkpoint version, expected to increment with schema changes. + version: u8, + /// The parent height we are forwarding our parent crosslink to. + target_height: u64, + /// The hash of the chain unit at that height. Usually a block hash, but could + /// be a different entity (e.g. tipset CID), depending on the parent chain + /// and our interface to it (e.g. if the parent is a Filecoin network, this + /// would be a tipset CID coerced into a block hash if we use the Eth API, + /// or the tipset CID as-is if we use the Filecoin API. + target_hash: [u8], + /// The commitment is an accumulated hash of all topdown effects since the genesis epoch + /// in the parent till the current parent block height(inclusive). + effects_commitment: Bytes, +} + /// The finality view for IPC parent at certain height. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct IPCParentFinality { diff --git a/fendermint/vm/topdown/src/proposal.rs b/fendermint/vm/topdown/src/proposal.rs new file mode 100644 index 000000000..42f83ec5f --- /dev/null +++ b/fendermint/vm/topdown/src/proposal.rs @@ -0,0 +1,30 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::{BlockHeight, Error}; +use crate::syncer::payload::ParentView; + +pub struct Proposal { + parent_height: BlockHeight, + +} + +pub struct ProposalMaker {} + +impl ProposalMaker { + /// Append a new parent view to the proposal maker. If there is a new proposal that can be + /// made, returns it. Else returns None. + pub fn append_new_view(&mut self, view: ParentView) -> Result, Error> { + todo!() + } + + pub fn get_proposal_at_height(&self, height: BlockHeight) -> Result, Error> { + todo!() + } + + /// Purge the proposals before the target height, inclusive + pub fn finalize(&mut self, height: BlockHeight) -> Result<(), Error> { + todo!() + } + +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index b2420b18a..588427284 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -5,7 +5,7 @@ use crate::proxy::ParentQueryProxy; use crate::syncer::poll::ParentPoll; use crate::syncer::store::ParentViewStore; use crate::vote::payload::Observation; -use crate::{BlockHeight, IPCParentFinality}; +use crate::{BlockHeight, Checkpoint, IPCParentFinality}; use std::time::Duration; use tokio::select; use tokio::sync::mpsc; @@ -19,8 +19,6 @@ pub mod store; pub enum TopDownSyncEvent { /// The fendermint node is syncing with peers NodeSyncing, - /// The parent view store is full, this will pause the parent syncer - ParentViewStoreFull, NewProposal(Box), } @@ -85,21 +83,34 @@ where Ok(ParentSyncerReactorClient { tx }) } +impl ParentSyncerReactorClient { + /// Marks the height as finalized. + /// There is no need to wait for ack from the reactor + pub async fn finalize_parent_height(&self, height: BlockHeight) -> anyhow::Result<()> { + self.tx.send(ParentSyncerRequest::Finalized(height)).await?; + Ok(()) + } +} + enum ParentSyncerRequest { /// A new parent height is finalized - Finalized(BlockHeight), + Finalized(Checkpoint), } fn handle_request(req: ParentSyncerRequest, poller: &mut ParentPoll) - where - S: ParentViewStore + Send + Sync + 'static, - P: Send + Sync + 'static + ParentQueryProxy +where + S: ParentViewStore + Send + Sync + 'static, + P: Send + Sync + 'static + ParentQueryProxy, { match req { ParentSyncerRequest::Finalized(h) => { if let Err(e) = poller.finalize(h) { - tracing::error!(height = h, err = e.to_string(), "cannot finalize parent viewer"); + tracing::error!( + height = h, + err = e.to_string(), + "cannot finalize parent viewer" + ); } - }, + } } } diff --git a/fendermint/vm/topdown/src/syncer/poll.rs b/fendermint/vm/topdown/src/syncer/poll.rs index 828577587..0705a9c99 100644 --- a/fendermint/vm/topdown/src/syncer/poll.rs +++ b/fendermint/vm/topdown/src/syncer/poll.rs @@ -1,6 +1,5 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -//! The inner type of parent syncer use crate::observe::ParentFinalityAcquired; use crate::proxy::ParentQueryProxy; @@ -8,7 +7,7 @@ use crate::syncer::error::Error; use crate::syncer::payload::ParentView; use crate::syncer::store::ParentViewStore; use crate::syncer::{ParentSyncerConfig, TopDownSyncEvent}; -use crate::{is_null_round_str, BlockHash, BlockHeight, IPCParentFinality}; +use crate::{is_null_round_str, BlockHash, BlockHeight, IPCParentFinality, Checkpoint}; use anyhow::anyhow; use ipc_observability::emit; use ipc_observability::serde::HexEncodableBlockHash; @@ -45,12 +44,14 @@ where } } - /// The target block height is finalized - pub fn finalize(&mut self, height: BlockHeight) -> Result<(), Error> { - let Some(min_height) = self.store.minimal_parent_view_height()? else { + /// The target block height is finalized, purge all the parent view before the target height + pub fn finalize(&mut self, checkpoint: Checkpoint) -> Result<(), Error> { + + + let Some(min_height) = self.store.min_parent_view_height()? else { return Ok(()); }; - for h in min_height..=height { + for h in min_height..=checkpoint.target_height { self.store.purge(h)?; } Ok(()) @@ -116,9 +117,6 @@ where loop { if self.store_full()? { tracing::debug!("exceeded cache size limit"); - let _ = self - .event_broadcast - .send(TopDownSyncEvent::ParentViewStoreFull); break; } @@ -191,7 +189,9 @@ where "detected null round at height, inserted None to cache" ); - self.store.store(ParentView::null_block(height))?; + self.proposal.new_view(ParentView::null_block(height))?; + + // self.store.store(ParentView::null_block(height))?; emit(ParentFinalityAcquired { source: "Parent syncer", @@ -225,7 +225,9 @@ where } let view = fetch_data(&self.parent_proxy, height, block_hash_res.block_hash).await?; - self.store.store(view.clone())?; + self.proposal.new_view(view.clone())?; + + // self.store.store(view.clone())?; let payload = view.payload.as_ref().unwrap(); emit(ParentFinalityAcquired { diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs index e9df04c29..c874ca278 100644 --- a/fendermint/vm/topdown/src/syncer/store.rs +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -8,7 +8,7 @@ use crate::BlockHeight; /// Stores the parent view observed of the current node pub trait ParentViewStore { /// Store a newly observed parent view - fn store(&mut self, view: ParentView) -> Result<(), Error>; + fn store(&mut self, view: ParentView) -> Result<(), Error> {} /// Get the parent view at the specified height fn get(&self, height: BlockHeight) -> Result, Error>; @@ -16,7 +16,7 @@ pub trait ParentViewStore { /// Purge the parent view at the target height fn purge(&mut self, height: BlockHeight) -> Result<(), Error>; - fn minimal_parent_view_height(&self) -> Result, Error>; + fn min_parent_view_height(&self) -> Result, Error>; fn max_parent_view_height(&self) -> Result, Error>; } From 370dc23db284fd68c9b0be41eabaa28b8d1def94 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Fri, 27 Sep 2024 20:42:41 +0800 Subject: [PATCH 12/16] skeleton code --- contracts/binding/src/lib.rs | 36 +++++----- fendermint/vm/topdown/src/lib.rs | 4 +- fendermint/vm/topdown/src/observation.rs | 80 +++++++++++++++++++++ fendermint/vm/topdown/src/proposal.rs | 30 -------- fendermint/vm/topdown/src/syncer/mod.rs | 23 +++--- fendermint/vm/topdown/src/syncer/payload.rs | 10 +-- fendermint/vm/topdown/src/syncer/poll.rs | 49 +++++++------ fendermint/vm/topdown/src/syncer/store.rs | 6 +- fendermint/vm/topdown/src/vote/mod.rs | 3 +- fendermint/vm/topdown/src/vote/payload.rs | 72 ++----------------- fendermint/vm/topdown/src/vote/store.rs | 10 +-- fendermint/vm/topdown/src/vote/tally.rs | 10 +-- fendermint/vm/topdown/tests/vote_reactor.rs | 15 ++-- 13 files changed, 168 insertions(+), 180 deletions(-) create mode 100644 fendermint/vm/topdown/src/observation.rs delete mode 100644 fendermint/vm/topdown/src/proposal.rs diff --git a/contracts/binding/src/lib.rs b/contracts/binding/src/lib.rs index b4fff9972..dd59524fb 100644 --- a/contracts/binding/src/lib.rs +++ b/contracts/binding/src/lib.rs @@ -2,27 +2,33 @@ #[macro_use] mod convert; #[allow(clippy::all)] -pub mod i_diamond; -#[allow(clippy::all)] -pub mod diamond_loupe_facet; +pub mod checkpointing_facet; #[allow(clippy::all)] pub mod diamond_cut_facet; #[allow(clippy::all)] -pub mod ownership_facet; +pub mod diamond_loupe_facet; #[allow(clippy::all)] pub mod gateway_diamond; #[allow(clippy::all)] +pub mod gateway_getter_facet; +#[allow(clippy::all)] pub mod gateway_manager_facet; #[allow(clippy::all)] -pub mod gateway_getter_facet; +pub mod gateway_messenger_facet; #[allow(clippy::all)] -pub mod checkpointing_facet; +pub mod i_diamond; #[allow(clippy::all)] -pub mod top_down_finality_facet; +pub mod lib_gateway; #[allow(clippy::all)] -pub mod xnet_messaging_facet; +pub mod lib_quorum; #[allow(clippy::all)] -pub mod gateway_messenger_facet; +pub mod lib_staking; +#[allow(clippy::all)] +pub mod lib_staking_change_log; +#[allow(clippy::all)] +pub mod ownership_facet; +#[allow(clippy::all)] +pub mod register_subnet_facet; #[allow(clippy::all)] pub mod subnet_actor_checkpointing_facet; #[allow(clippy::all)] @@ -36,19 +42,13 @@ pub mod subnet_actor_pause_facet; #[allow(clippy::all)] pub mod subnet_actor_reward_facet; #[allow(clippy::all)] -pub mod subnet_registry_diamond; -#[allow(clippy::all)] -pub mod register_subnet_facet; -#[allow(clippy::all)] pub mod subnet_getter_facet; #[allow(clippy::all)] -pub mod lib_staking; -#[allow(clippy::all)] -pub mod lib_staking_change_log; +pub mod subnet_registry_diamond; #[allow(clippy::all)] -pub mod lib_gateway; +pub mod top_down_finality_facet; #[allow(clippy::all)] -pub mod lib_quorum; +pub mod xnet_messaging_facet; // The list of contracts need to convert FvmAddress to fvm_shared::Address fvm_address_conversion!(gateway_manager_facet); diff --git a/fendermint/vm/topdown/src/lib.rs b/fendermint/vm/topdown/src/lib.rs index 3faa8b22b..eba6861da 100644 --- a/fendermint/vm/topdown/src/lib.rs +++ b/fendermint/vm/topdown/src/lib.rs @@ -11,10 +11,10 @@ pub mod proxy; mod toggle; pub mod voting; +pub(crate) mod observation; pub mod observe; pub mod syncer; pub mod vote; -pub(crate) mod proposal; use async_stm::Stm; use async_trait::async_trait; @@ -122,7 +122,7 @@ pub struct Checkpoint { /// and our interface to it (e.g. if the parent is a Filecoin network, this /// would be a tipset CID coerced into a block hash if we use the Eth API, /// or the tipset CID as-is if we use the Filecoin API. - target_hash: [u8], + target_hash: BlockHash, /// The commitment is an accumulated hash of all topdown effects since the genesis epoch /// in the parent till the current parent block height(inclusive). effects_commitment: Bytes, diff --git a/fendermint/vm/topdown/src/observation.rs b/fendermint/vm/topdown/src/observation.rs new file mode 100644 index 000000000..14f575336 --- /dev/null +++ b/fendermint/vm/topdown/src/observation.rs @@ -0,0 +1,80 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::syncer::error::Error; +use crate::syncer::store::ParentViewStore; +use crate::{BlockHash, BlockHeight, Checkpoint}; +use arbitrary::Arbitrary; +use serde::{Deserialize, Serialize}; +use std::fmt::{Display, Formatter}; + +/// The content that validators gossip among each other +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Arbitrary)] +pub struct ObservationCommitment { + /// The hash of the subnet's last committed block when this observation was made. + /// Used to discard stale observations that are, e.g. replayed by an attacker + /// at a later time. Also used to detect nodes that might be wrongly gossiping + /// whilst being out of sync. + local_hash: BlockHash, + pub(crate) ballot: Ballot, +} + +/// The actual content that validators should agree upon, or put in another way the content +/// that a quorum should be formed upon +#[derive(Serialize, Deserialize, Hash, Debug, Clone, Eq, PartialEq, Arbitrary)] +pub struct Ballot { + pub(crate) parent_height: u64, + /// The hash of the chain unit at that height. Usually a block hash, but could + /// be another entity (e.g. tipset CID), depending on the parent chain + /// and our interface to it. For example, if the parent is a Filecoin network, + /// this would be a tipset CID coerced into a block hash if queried through + /// the Eth API, or the tipset CID as-is if accessed through the Filecoin API. + pub(crate) parent_hash: crate::Bytes, + /// A rolling/cumulative commitment to topdown effects since the beginning of + /// time, including the ones in this block. + pub(crate) cumulative_effects_comm: crate::Bytes, +} + +/// check in the store to see if there is a new observation available +pub fn deduce_new_observation( + _store: &S, + _checkpoint: &Checkpoint, +) -> Result { + todo!() +} + +impl Display for Ballot { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Ballot(parent_height={}, parent_hash={}, commitment={})", + self.parent_height, + hex::encode(&self.parent_hash), + hex::encode(&self.cumulative_effects_comm), + ) + } +} + +impl Ballot { + pub fn parent_height(&self) -> BlockHeight { + self.parent_height + } +} + +impl ObservationCommitment { + pub fn new( + local_hash: crate::Bytes, + parent_height: BlockHeight, + parent_hash: crate::Bytes, + commitment: crate::Bytes, + ) -> Self { + Self { + local_hash, + ballot: Ballot { + parent_height, + parent_hash, + cumulative_effects_comm: commitment, + }, + } + } +} diff --git a/fendermint/vm/topdown/src/proposal.rs b/fendermint/vm/topdown/src/proposal.rs deleted file mode 100644 index 42f83ec5f..000000000 --- a/fendermint/vm/topdown/src/proposal.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::{BlockHeight, Error}; -use crate::syncer::payload::ParentView; - -pub struct Proposal { - parent_height: BlockHeight, - -} - -pub struct ProposalMaker {} - -impl ProposalMaker { - /// Append a new parent view to the proposal maker. If there is a new proposal that can be - /// made, returns it. Else returns None. - pub fn append_new_view(&mut self, view: ParentView) -> Result, Error> { - todo!() - } - - pub fn get_proposal_at_height(&self, height: BlockHeight) -> Result, Error> { - todo!() - } - - /// Purge the proposals before the target height, inclusive - pub fn finalize(&mut self, height: BlockHeight) -> Result<(), Error> { - todo!() - } - -} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index 588427284..3b111a298 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -1,11 +1,11 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::observation::ObservationCommitment; use crate::proxy::ParentQueryProxy; use crate::syncer::poll::ParentPoll; use crate::syncer::store::ParentViewStore; -use crate::vote::payload::Observation; -use crate::{BlockHeight, Checkpoint, IPCParentFinality}; +use crate::{BlockHeight, Checkpoint}; use std::time::Duration; use tokio::select; use tokio::sync::mpsc; @@ -19,7 +19,7 @@ pub mod store; pub enum TopDownSyncEvent { /// The fendermint node is syncing with peers NodeSyncing, - NewProposal(Box), + NewProposal(Box), } pub struct ParentSyncerConfig { @@ -52,7 +52,7 @@ pub fn start_parent_syncer( config: ParentSyncerConfig, proxy: P, store: S, - last_finalized: IPCParentFinality, + last_finalized: Checkpoint, ) -> anyhow::Result where S: ParentViewStore + Send + Sync + 'static, @@ -86,8 +86,8 @@ where impl ParentSyncerReactorClient { /// Marks the height as finalized. /// There is no need to wait for ack from the reactor - pub async fn finalize_parent_height(&self, height: BlockHeight) -> anyhow::Result<()> { - self.tx.send(ParentSyncerRequest::Finalized(height)).await?; + pub async fn finalize_parent_height(&self, cp: Checkpoint) -> anyhow::Result<()> { + self.tx.send(ParentSyncerRequest::Finalized(cp)).await?; Ok(()) } } @@ -103,13 +103,10 @@ where P: Send + Sync + 'static + ParentQueryProxy, { match req { - ParentSyncerRequest::Finalized(h) => { - if let Err(e) = poller.finalize(h) { - tracing::error!( - height = h, - err = e.to_string(), - "cannot finalize parent viewer" - ); + ParentSyncerRequest::Finalized(c) => { + let height = c.target_height; + if let Err(e) = poller.finalize(c) { + tracing::error!(height, err = e.to_string(), "cannot finalize parent viewer"); } } } diff --git a/fendermint/vm/topdown/src/syncer/payload.rs b/fendermint/vm/topdown/src/syncer/payload.rs index d501a19bd..5dd83f17e 100644 --- a/fendermint/vm/topdown/src/syncer/payload.rs +++ b/fendermint/vm/topdown/src/syncer/payload.rs @@ -6,7 +6,7 @@ use ipc_api::cross::IpcEnvelope; use ipc_api::staking::StakingChangeRequest; #[derive(Clone, Debug)] -pub struct ParentViewPayload { +pub struct ParentBlockViewPayload { pub parent_hash: BlockHash, /// Encodes cross-net messages. pub xnet_msgs: Vec, @@ -15,13 +15,13 @@ pub struct ParentViewPayload { } #[derive(Clone, Debug)] -pub struct ParentView { +pub struct ParentBlockView { pub parent_height: BlockHeight, /// If the payload is None, this means the parent height is a null block - pub payload: Option, + pub payload: Option, } -impl ParentView { +impl ParentBlockView { pub fn null_block(h: BlockHeight) -> Self { Self { parent_height: h, @@ -37,7 +37,7 @@ impl ParentView { ) -> Self { Self { parent_height: h, - payload: Some(ParentViewPayload { + payload: Some(ParentBlockViewPayload { parent_hash, xnet_msgs, validator_changes, diff --git a/fendermint/vm/topdown/src/syncer/poll.rs b/fendermint/vm/topdown/src/syncer/poll.rs index 0705a9c99..422394c5c 100644 --- a/fendermint/vm/topdown/src/syncer/poll.rs +++ b/fendermint/vm/topdown/src/syncer/poll.rs @@ -1,13 +1,14 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::observation::deduce_new_observation; use crate::observe::ParentFinalityAcquired; use crate::proxy::ParentQueryProxy; use crate::syncer::error::Error; -use crate::syncer::payload::ParentView; +use crate::syncer::payload::ParentBlockView; use crate::syncer::store::ParentViewStore; use crate::syncer::{ParentSyncerConfig, TopDownSyncEvent}; -use crate::{is_null_round_str, BlockHash, BlockHeight, IPCParentFinality, Checkpoint}; +use crate::{is_null_round_str, BlockHash, BlockHeight, Checkpoint}; use anyhow::anyhow; use ipc_observability::emit; use ipc_observability::serde::HexEncodableBlockHash; @@ -20,7 +21,7 @@ pub(crate) struct ParentPoll { parent_proxy: P, store: S, event_broadcast: broadcast::Sender, - last_finalized: IPCParentFinality, + last_finalized: Checkpoint, } impl ParentPoll @@ -28,12 +29,7 @@ where S: ParentViewStore + Send + Sync + 'static, P: Send + Sync + 'static + ParentQueryProxy, { - pub fn new( - config: ParentSyncerConfig, - proxy: P, - store: S, - last_finalized: IPCParentFinality, - ) -> Self { + pub fn new(config: ParentSyncerConfig, proxy: P, store: S, last_finalized: Checkpoint) -> Self { let (tx, _) = broadcast::channel(config.broadcast_channel_size); Self { config, @@ -46,8 +42,6 @@ where /// The target block height is finalized, purge all the parent view before the target height pub fn finalize(&mut self, checkpoint: Checkpoint) -> Result<(), Error> { - - let Some(min_height) = self.store.min_parent_view_height()? else { return Ok(()); }; @@ -61,28 +55,29 @@ where async fn latest_nonnull_data(&self) -> anyhow::Result<(BlockHeight, BlockHash)> { let Some(latest_height) = self.store.max_parent_view_height()? else { return Ok(( - self.last_finalized.height, - self.last_finalized.block_hash.clone(), + self.last_finalized.target_height, + self.last_finalized.target_hash.clone(), )); }; - let start = self.last_finalized.height + 1; + let start = self.last_finalized.target_height + 1; for h in (start..=latest_height).rev() { - let Some(view) = self.store.get(h)? else { + let Some(p) = self.store.get(h)? else { continue; }; - let Some(payload) = view.payload else { + // if parent hash of the proposal is null, it means the + let Some(p) = p.payload else { continue; }; - return Ok((h, payload.parent_hash)); + return Ok((h, p.parent_hash)); } // this means the votes stored are all null blocks, return last committed finality Ok(( - self.last_finalized.height, - self.last_finalized.block_hash.clone(), + self.last_finalized.target_height, + self.last_finalized.target_hash.clone(), )) } @@ -150,7 +145,7 @@ where let Some(h) = self.store.max_parent_view_height()? else { return Ok(false); }; - Ok(h - self.last_finalized.height > self.config.max_store_blocks) + Ok(h - self.last_finalized.target_height > self.config.max_store_blocks) } async fn finalized_chain_head(&self) -> anyhow::Result> { @@ -189,7 +184,7 @@ where "detected null round at height, inserted None to cache" ); - self.proposal.new_view(ParentView::null_block(height))?; + self.store.store(ParentBlockView::null_block(height))?; // self.store.store(ParentView::null_block(height))?; @@ -225,9 +220,13 @@ where } let view = fetch_data(&self.parent_proxy, height, block_hash_res.block_hash).await?; - self.proposal.new_view(view.clone())?; - // self.store.store(view.clone())?; + self.store.store(view.clone())?; + let commitment = deduce_new_observation(&self.store, &self.last_finalized)?; + // if there is an error, ignore, we can always try next loop + let _ = self + .event_broadcast + .send(TopDownSyncEvent::NewProposal(Box::new(commitment))); let payload = view.payload.as_ref().unwrap(); emit(ParentFinalityAcquired { @@ -250,7 +249,7 @@ async fn fetch_data

( parent_proxy: &P, height: BlockHeight, block_hash: BlockHash, -) -> Result +) -> Result where P: ParentQueryProxy + Send + Sync + 'static, { @@ -285,7 +284,7 @@ where return Err(Error::ParentChainReorgDetected); } - Ok(ParentView::nonnull_block( + Ok(ParentBlockView::nonnull_block( height, block_hash, topdown_msgs_res.value, diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs index c874ca278..0ef07ee6d 100644 --- a/fendermint/vm/topdown/src/syncer/store.rs +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -2,16 +2,16 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::syncer::error::Error; -use crate::syncer::payload::ParentView; +use crate::syncer::payload::ParentBlockView; use crate::BlockHeight; /// Stores the parent view observed of the current node pub trait ParentViewStore { /// Store a newly observed parent view - fn store(&mut self, view: ParentView) -> Result<(), Error> {} + fn store(&mut self, view: ParentBlockView) -> Result<(), Error>; /// Get the parent view at the specified height - fn get(&self, height: BlockHeight) -> Result, Error>; + fn get(&self, height: BlockHeight) -> Result, Error>; /// Purge the parent view at the target height fn purge(&mut self, height: BlockHeight) -> Result<(), Error>; diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index 51513a66b..ad44c6fa1 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -8,10 +8,11 @@ pub mod payload; pub mod store; mod tally; +use crate::observation::Ballot; use crate::syncer::TopDownSyncEvent; use crate::vote::gossip::GossipClient; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; -use crate::vote::payload::{Ballot, CertifiedObservation, PowerUpdates, Vote, VoteTallyState}; +use crate::vote::payload::{CertifiedObservation, PowerUpdates, Vote, VoteTallyState}; use crate::vote::store::VoteStore; use crate::vote::tally::VoteTally; use crate::BlockHeight; diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index b607288d1..22ae8ac20 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -1,16 +1,15 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::observation::{Ballot, ObservationCommitment}; use crate::vote::Weight; -use crate::{BlockHash, BlockHeight, Bytes}; +use crate::BlockHeight; use anyhow::anyhow; -use arbitrary::Arbitrary; use fendermint_crypto::secp::RecoverableECDSASignature; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::fmt::{Display, Formatter}; pub type PowerTable = HashMap; pub type PowerUpdates = Vec<(ValidatorKey, Weight)>; @@ -31,37 +30,10 @@ pub enum Vote { }, } -/// The actual content that validators should agree upon, or put in another way the content -/// that a quorum should be formed upon -#[derive(Serialize, Deserialize, Hash, Debug, Clone, Eq, PartialEq, Arbitrary)] -pub struct Ballot { - pub(crate) parent_height: u64, - /// The hash of the chain unit at that height. Usually a block hash, but could - /// be another entity (e.g. tipset CID), depending on the parent chain - /// and our interface to it. For example, if the parent is a Filecoin network, - /// this would be a tipset CID coerced into a block hash if queried through - /// the Eth API, or the tipset CID as-is if accessed through the Filecoin API. - pub(crate) parent_hash: Bytes, - /// A rolling/cumulative commitment to topdown effects since the beginning of - /// time, including the ones in this block. - pub(crate) cumulative_effects_comm: Bytes, -} - -/// The content that validators gossip among each other -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Arbitrary)] -pub struct Observation { - /// The hash of the subnet's last committed block when this observation was made. - /// Used to discard stale observations that are, e.g. replayed by an attacker - /// at a later time. Also used to detect nodes that might be wrongly gossiping - /// whilst being out of sync. - local_hash: BlockHash, - pub(crate) ballot: Ballot, -} - /// A self-certified observation made by a validator. #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] pub struct CertifiedObservation { - observed: Observation, + observed: ObservationCommitment, /// A "recoverable" ECDSA signature with the validator's secp256k1 private key over the /// CID of the DAG-CBOR encoded observation using a BLAKE2b-256 multihash. signature: RecoverableECDSASignature, @@ -121,7 +93,7 @@ impl TryFrom<&[u8]> for CertifiedObservation { } impl CertifiedObservation { - pub fn sign(ob: Observation, sk: &SecretKey) -> anyhow::Result { + pub fn sign(ob: ObservationCommitment, sk: &SecretKey) -> anyhow::Result { let to_sign = fvm_ipld_encoding::to_vec(&ob)?; let sig = RecoverableECDSASignature::sign(sk, to_sign.as_slice())?; Ok(Self { @@ -130,39 +102,3 @@ impl CertifiedObservation { }) } } - -impl Observation { - pub fn new( - local_hash: Bytes, - parent_height: BlockHeight, - parent_hash: Bytes, - commitment: Bytes, - ) -> Self { - Self { - local_hash, - ballot: Ballot { - parent_height, - parent_hash, - cumulative_effects_comm: commitment, - }, - } - } -} - -impl Display for Ballot { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Ballot(parent_height={}, parent_hash={}, commitment={})", - self.parent_height, - hex::encode(&self.parent_hash), - hex::encode(&self.cumulative_effects_comm), - ) - } -} - -impl Ballot { - pub fn parent_height(&self) -> BlockHeight { - self.parent_height - } -} diff --git a/fendermint/vm/topdown/src/vote/store.rs b/fendermint/vm/topdown/src/vote/store.rs index 098902b3b..10de5a238 100644 --- a/fendermint/vm/topdown/src/vote/store.rs +++ b/fendermint/vm/topdown/src/vote/store.rs @@ -1,8 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::observation::Ballot; use crate::vote::error::Error; -use crate::vote::payload::{Ballot, PowerTable, Vote}; +use crate::vote::payload::{PowerTable, Vote}; use crate::vote::Weight; use crate::BlockHeight; use fendermint_vm_genesis::ValidatorKey; @@ -121,7 +122,8 @@ impl<'a> VoteAgg<'a> { #[cfg(test)] mod tests { - use crate::vote::payload::{CertifiedObservation, Observation, Vote}; + use crate::observation::ObservationCommitment; + use crate::vote::payload::{CertifiedObservation, Vote}; use crate::vote::store::VoteAgg; use arbitrary::{Arbitrary, Unstructured}; use fendermint_crypto::SecretKey; @@ -136,13 +138,13 @@ mod tests { (sk, ValidatorKey::new(public_key)) } - fn random_observation() -> Observation { + fn random_observation() -> ObservationCommitment { let mut bytes = [0; 100]; let mut rng = rand::thread_rng(); rng.fill_bytes(&mut bytes); let mut unstructured = Unstructured::new(&bytes); - Observation::arbitrary(&mut unstructured).unwrap() + ObservationCommitment::arbitrary(&mut unstructured).unwrap() } #[test] diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index f309c82c9..cfce8c526 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -1,8 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::observation::Ballot; use crate::vote::error::Error; -use crate::vote::payload::{Ballot, PowerTable, PowerUpdates, Vote}; +use crate::vote::payload::{PowerTable, PowerUpdates, Vote}; use crate::vote::store::VoteStore; use crate::vote::Weight; use crate::BlockHeight; @@ -212,8 +213,9 @@ impl VoteTally { #[cfg(test)] mod tests { + use crate::observation::ObservationCommitment; use crate::vote::error::Error; - use crate::vote::payload::{CertifiedObservation, Observation, Vote}; + use crate::vote::payload::{CertifiedObservation, Vote}; use crate::vote::store::InMemoryVoteStore; use crate::vote::tally::VoteTally; use arbitrary::{Arbitrary, Unstructured}; @@ -228,13 +230,13 @@ mod tests { (sk, ValidatorKey::new(public_key)) } - fn random_observation() -> Observation { + fn random_observation() -> ObservationCommitment { let mut bytes = [0; 100]; let mut rng = rand::thread_rng(); rng.fill_bytes(&mut bytes); let mut unstructured = Unstructured::new(&bytes); - Observation::arbitrary(&mut unstructured).unwrap() + ObservationCommitment::arbitrary(&mut unstructured).unwrap() } #[test] diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs index ed75916d9..26ed0aead 100644 --- a/fendermint/vm/topdown/tests/vote_reactor.rs +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -8,10 +8,11 @@ use async_trait::async_trait; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; +use fendermint_vm_topdown::observation::ObservationCommitment; use fendermint_vm_topdown::syncer::TopDownSyncEvent; use fendermint_vm_topdown::vote::error::Error; use fendermint_vm_topdown::vote::gossip::GossipClient; -use fendermint_vm_topdown::vote::payload::{Observation, PowerUpdates, Vote}; +use fendermint_vm_topdown::vote::payload::{PowerUpdates, Vote}; use fendermint_vm_topdown::vote::store::InMemoryVoteStore; use fendermint_vm_topdown::vote::{start_vote_reactor, Config, VoteReactorClient, Weight}; use fendermint_vm_topdown::BlockHeight; @@ -142,7 +143,7 @@ async fn simple_lifecycle() { // now topdown sync published a new observation on parent height 100 let parent_height = 100; - let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2, 3, 4]); + let obs = ObservationCommitment::new(vec![100], parent_height, vec![1, 2, 3], vec![2, 3, 4]); internal_event_tx .send(TopDownSyncEvent::NewProposal(Box::new(obs))) .unwrap(); @@ -158,7 +159,7 @@ async fn simple_lifecycle() { // now push another observation let parent_height2 = 101; - let obs2 = Observation::new(vec![100], parent_height2, vec![1, 2, 3], vec![2, 3, 4]); + let obs2 = ObservationCommitment::new(vec![100], parent_height2, vec![1, 2, 3], vec![2, 3, 4]); internal_event_tx .send(TopDownSyncEvent::NewProposal(Box::new(obs2))) .unwrap(); @@ -221,11 +222,11 @@ async fn waiting_for_quorum() { // now topdown sync published a new observation on parent height 100 let parent_height1 = 100; - let obs1 = Observation::new(vec![100], parent_height1, vec![1, 2, 3], vec![2, 3, 4]); + let obs1 = ObservationCommitment::new(vec![100], parent_height1, vec![1, 2, 3], vec![2, 3, 4]); let parent_height2 = 110; - let obs2 = Observation::new(vec![100], parent_height2, vec![1, 2, 3], vec![2, 3, 4]); + let obs2 = ObservationCommitment::new(vec![100], parent_height2, vec![1, 2, 3], vec![2, 3, 4]); let parent_height3 = 120; - let obs3 = Observation::new(vec![100], parent_height3, vec![1, 2, 3], vec![2, 3, 4]); + let obs3 = ObservationCommitment::new(vec![100], parent_height3, vec![1, 2, 3], vec![2, 3, 4]); internal_txs[0] .send(TopDownSyncEvent::NewProposal(Box::new(obs1.clone()))) @@ -343,7 +344,7 @@ async fn all_validator_in_sync() { } let parent_height = 100; - let obs = Observation::new(vec![100], parent_height, vec![1, 2, 3], vec![2, 3, 4]); + let obs = ObservationCommitment::new(vec![100], parent_height, vec![1, 2, 3], vec![2, 3, 4]); internal_event_tx .send(TopDownSyncEvent::NewProposal(Box::new(obs))) .unwrap(); From cfd8e573540008a98d7cee94c6a00ab0002c0423 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Mon, 30 Sep 2024 16:09:39 +0800 Subject: [PATCH 13/16] integrate observation --- fendermint/vm/topdown/src/lib.rs | 37 ++++--- fendermint/vm/topdown/src/observation.rs | 113 +++++++++++++++++++- fendermint/vm/topdown/src/syncer/error.rs | 10 ++ fendermint/vm/topdown/src/syncer/mod.rs | 6 +- fendermint/vm/topdown/src/syncer/payload.rs | 21 +++- fendermint/vm/topdown/src/syncer/poll.rs | 16 +-- fendermint/vm/topdown/src/syncer/store.rs | 30 +++++- 7 files changed, 202 insertions(+), 31 deletions(-) diff --git a/fendermint/vm/topdown/src/lib.rs b/fendermint/vm/topdown/src/lib.rs index eba6861da..d547fe526 100644 --- a/fendermint/vm/topdown/src/lib.rs +++ b/fendermint/vm/topdown/src/lib.rs @@ -29,6 +29,7 @@ use std::time::Duration; pub use crate::cache::{SequentialAppendError, SequentialKeyCache, ValueIter}; pub use crate::error::Error; pub use crate::finality::CachedFinalityProvider; +use crate::observation::Ballot; pub use crate::toggle::Toggle; pub type BlockHeight = u64; @@ -112,20 +113,8 @@ impl Config { /// On-chain data structure representing a topdown checkpoint agreed to by a /// majority of subnet validators. DAG-CBOR encoded, embedded in CertifiedCheckpoint. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Checkpoint { - /// Checkpoint version, expected to increment with schema changes. - version: u8, - /// The parent height we are forwarding our parent crosslink to. - target_height: u64, - /// The hash of the chain unit at that height. Usually a block hash, but could - /// be a different entity (e.g. tipset CID), depending on the parent chain - /// and our interface to it (e.g. if the parent is a Filecoin network, this - /// would be a tipset CID coerced into a block hash if we use the Eth API, - /// or the tipset CID as-is if we use the Filecoin API. - target_hash: BlockHash, - /// The commitment is an accumulated hash of all topdown effects since the genesis epoch - /// in the parent till the current parent block height(inclusive). - effects_commitment: Bytes, +pub enum Checkpoint { + V1(Ballot) } /// The finality view for IPC parent at certain height. @@ -213,3 +202,23 @@ pub(crate) fn is_null_round_error(err: &anyhow::Error) -> bool { pub(crate) fn is_null_round_str(s: &str) -> bool { s.contains(NULL_ROUND_ERR_MSG) } + +impl Checkpoint { + pub fn target_height(&self) -> BlockHeight { + match self { + Checkpoint::V1(b) => b.parent_height + } + } + + pub fn target_hash(&self) -> &Bytes { + match self { + Checkpoint::V1(b) => &b.parent_hash + } + } + + pub fn cumulative_effects_comm(&self) -> &Bytes { + match self { + Checkpoint::V1(b) => &b.cumulative_effects_comm + } + } +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/observation.rs b/fendermint/vm/topdown/src/observation.rs index 14f575336..e6637d37c 100644 --- a/fendermint/vm/topdown/src/observation.rs +++ b/fendermint/vm/topdown/src/observation.rs @@ -1,12 +1,29 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use std::cmp::min; use crate::syncer::error::Error; use crate::syncer::store::ParentViewStore; -use crate::{BlockHash, BlockHeight, Checkpoint}; +use crate::{BlockHash, BlockHeight, Bytes, Checkpoint}; use arbitrary::Arbitrary; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; +use cid::Cid; +use fvm_ipld_encoding::DAG_CBOR; +use multihash::Code; +use multihash::MultihashDigest; + +use crate::syncer::payload::ParentBlockView; + +/// Default topdown observation height range +const DEFAULT_MAX_OBSERVATION_RANGE: BlockHeight = 100; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObservationConfig { + /// The max number of blocks one should make the topdown observation from the previous + /// committed checkpoint + pub max_observation_range: Option, +} /// The content that validators gossip among each other #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Arbitrary)] @@ -35,12 +52,45 @@ pub struct Ballot { pub(crate) cumulative_effects_comm: crate::Bytes, } -/// check in the store to see if there is a new observation available +/// Check in the store to see if there is a new observation available. +/// Caller should make sure: +/// - the store has votes since the last committed checkpoint +/// - the votes have at least 1 non-null block pub fn deduce_new_observation( - _store: &S, - _checkpoint: &Checkpoint, + store: &S, + checkpoint: &Checkpoint, + config: &ObservationConfig, ) -> Result { - todo!() + let Some(latest_height) = store.max_parent_view_height()? else { + tracing::info!("no observation yet as height not available"); + return Err(Error::BlockStoreEmpty); + }; + + if latest_height < checkpoint.target_height() { + tracing::info!("committed vote height more than latest parent view"); + return Err(Error::CommittedParentHeightNotPurged) + } + + let max_observation_height = checkpoint.target_height() + config.max_observation_range(); + let candidate_height = min(max_observation_height, latest_height); + tracing::debug!(max_observation_height, candidate_height, "propose observation height"); + + // aggregate commitment for the observation + let mut agg = LinearizedParentBlockView::from(checkpoint); + for h in checkpoint.target_height()+1..=candidate_height { + let Some(p) = store.get(h)? else { + tracing::debug!(height = h, "not parent block view"); + return Err(Error::MissingBlockView(h, candidate_height)); + }; + + agg.append(p)?; + } + + // TODO: integrate local hash + let observation = agg.into_commitment(vec![])?; + tracing::info!(height = observation.ballot.parent_height, "new observation derived"); + + Ok(observation) } impl Display for Ballot { @@ -78,3 +128,56 @@ impl ObservationCommitment { } } } + +impl ObservationConfig { + pub fn max_observation_range(&self) -> BlockHeight { + self.max_observation_range.unwrap_or(DEFAULT_MAX_OBSERVATION_RANGE) + } +} + +struct LinearizedParentBlockView { + parent_height: u64, + parent_hash: Option, + cumulative_effects_comm: Bytes, +} + +impl From<&Checkpoint> for LinearizedParentBlockView { + fn from(value: &Checkpoint) -> Self { + LinearizedParentBlockView { + parent_height: value.target_height(), + parent_hash: Some(value.target_hash().clone()), + cumulative_effects_comm: value.cumulative_effects_comm().clone(), + } + } +} + +impl LinearizedParentBlockView { + fn new_commitment(&mut self, to_append: Bytes) { + let bytes = [self.cumulative_effects_comm.as_slice(), to_append.as_slice()].concat(); + let cid = Cid::new_v1(DAG_CBOR, Code::Blake2b256.digest(&bytes)); + self.cumulative_effects_comm = cid.to_bytes(); + } + + pub fn append(&mut self, view: ParentBlockView) -> Result<(), Error>{ + if self.parent_height + 1 != view.parent_height { + return Err(Error::NotSequential) + } + + self.parent_height += 1; + + self.new_commitment(view.effects_commitment()?); + + if let Some(p) = view.payload { + self.parent_hash = Some(p.parent_hash); + } + + Ok(()) + } + + fn into_commitment(self, local_hash: BlockHash) -> Result { + let Some(hash) = self.parent_hash else { + return Err(Error::CannotCommitObservationAtNullBlock(self.parent_height)); + }; + Ok(ObservationCommitment::new(local_hash, self.parent_height, hash, self.cumulative_effects_comm)) + } +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/error.rs b/fendermint/vm/topdown/src/syncer/error.rs index cbfbeed5c..ca92175ab 100644 --- a/fendermint/vm/topdown/src/syncer/error.rs +++ b/fendermint/vm/topdown/src/syncer/error.rs @@ -15,4 +15,14 @@ pub enum Error { ParentChainReorgDetected, #[error("Cannot query parent at height {1}: {0}")] CannotQueryParent(String, BlockHeight), + #[error("Parent block view store is empty")] + BlockStoreEmpty, + #[error("Committed block height not purged yet")] + CommittedParentHeightNotPurged, + #[error("Cannot serialize parent block view payload to bytes")] + CannotSerializeParentBlockView, + #[error("Cannot create commitment at null parent block {0}")] + CannotCommitObservationAtNullBlock(BlockHeight), + #[error("Missing block view at height {0} for target observation height {0}")] + MissingBlockView(BlockHeight, BlockHeight), } diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index 3b111a298..1c722fad4 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -1,7 +1,7 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::observation::ObservationCommitment; +use crate::observation::{ObservationCommitment, ObservationConfig}; use crate::proxy::ParentQueryProxy; use crate::syncer::poll::ParentPoll; use crate::syncer::store::ParentViewStore; @@ -41,6 +41,8 @@ pub struct ParentSyncerConfig { pub max_store_blocks: BlockHeight, /// Attempts to sync as many block as possible till the finalized chain head pub sync_many: bool, + + pub observation: ObservationConfig, } #[derive(Clone)] @@ -104,7 +106,7 @@ where { match req { ParentSyncerRequest::Finalized(c) => { - let height = c.target_height; + let height = c.target_height(); if let Err(e) = poller.finalize(c) { tracing::error!(height, err = e.to_string(), "cannot finalize parent viewer"); } diff --git a/fendermint/vm/topdown/src/syncer/payload.rs b/fendermint/vm/topdown/src/syncer/payload.rs index 5dd83f17e..b68824e1e 100644 --- a/fendermint/vm/topdown/src/syncer/payload.rs +++ b/fendermint/vm/topdown/src/syncer/payload.rs @@ -1,9 +1,14 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::{BlockHash, BlockHeight}; +use cid::Cid; +use fvm_ipld_encoding::DAG_CBOR; +use multihash::Code; +use crate::{BlockHash, BlockHeight, Bytes}; use ipc_api::cross::IpcEnvelope; use ipc_api::staking::StakingChangeRequest; +use multihash::MultihashDigest; +use crate::syncer::error::Error; #[derive(Clone, Debug)] pub struct ParentBlockViewPayload { @@ -44,4 +49,18 @@ impl ParentBlockView { }), } } + + pub fn effects_commitment(&self) -> Result { + let Some(ref p) = self.payload else { + return Ok(Cid::default().to_bytes()); + }; + + let bytes = fvm_ipld_encoding::to_vec(&(&p.xnet_msgs, &p.validator_changes)) + .map_err(|e| { + tracing::error!(err = e.to_string(), "cannot serialize parent block view"); + Error::CannotSerializeParentBlockView + })?; + let cid = Cid::new_v1(DAG_CBOR, Code::Blake2b256.digest(&bytes)); + Ok(cid.to_bytes()) + } } diff --git a/fendermint/vm/topdown/src/syncer/poll.rs b/fendermint/vm/topdown/src/syncer/poll.rs index 422394c5c..01e6b260f 100644 --- a/fendermint/vm/topdown/src/syncer/poll.rs +++ b/fendermint/vm/topdown/src/syncer/poll.rs @@ -45,7 +45,7 @@ where let Some(min_height) = self.store.min_parent_view_height()? else { return Ok(()); }; - for h in min_height..=checkpoint.target_height { + for h in min_height..=checkpoint.target_height() { self.store.purge(h)?; } Ok(()) @@ -55,12 +55,12 @@ where async fn latest_nonnull_data(&self) -> anyhow::Result<(BlockHeight, BlockHash)> { let Some(latest_height) = self.store.max_parent_view_height()? else { return Ok(( - self.last_finalized.target_height, - self.last_finalized.target_hash.clone(), + self.last_finalized.target_height(), + self.last_finalized.target_hash().clone(), )); }; - let start = self.last_finalized.target_height + 1; + let start = self.last_finalized.target_height() + 1; for h in (start..=latest_height).rev() { let Some(p) = self.store.get(h)? else { continue; @@ -76,8 +76,8 @@ where // this means the votes stored are all null blocks, return last committed finality Ok(( - self.last_finalized.target_height, - self.last_finalized.target_hash.clone(), + self.last_finalized.target_height(), + self.last_finalized.target_hash().clone(), )) } @@ -145,7 +145,7 @@ where let Some(h) = self.store.max_parent_view_height()? else { return Ok(false); }; - Ok(h - self.last_finalized.target_height > self.config.max_store_blocks) + Ok(h - self.last_finalized.target_height() > self.config.max_store_blocks) } async fn finalized_chain_head(&self) -> anyhow::Result> { @@ -222,7 +222,7 @@ where let view = fetch_data(&self.parent_proxy, height, block_hash_res.block_hash).await?; self.store.store(view.clone())?; - let commitment = deduce_new_observation(&self.store, &self.last_finalized)?; + let commitment = deduce_new_observation(&self.store, &self.last_finalized, &self.config.observation)?; // if there is an error, ignore, we can always try next loop let _ = self .event_broadcast diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs index 0ef07ee6d..d4c7230df 100644 --- a/fendermint/vm/topdown/src/syncer/store.rs +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -3,7 +3,7 @@ use crate::syncer::error::Error; use crate::syncer::payload::ParentBlockView; -use crate::BlockHeight; +use crate::{BlockHeight, SequentialKeyCache}; /// Stores the parent view observed of the current node pub trait ParentViewStore { @@ -20,3 +20,31 @@ pub trait ParentViewStore { fn max_parent_view_height(&self) -> Result, Error>; } + +pub struct InMemoryParentViewStore { + inner: SequentialKeyCache, +} + +impl ParentViewStore for InMemoryParentViewStore { + fn store(&mut self, view: ParentBlockView) -> Result<(), Error> { + self.inner.append(view.parent_height, view) + .map_err(|_| Error::NonSequentialParentViewInsert) + } + + fn get(&self, height: BlockHeight) -> Result, Error> { + Ok(self.inner.get_value(height).cloned()) + } + + fn purge(&mut self, height: BlockHeight) -> Result<(), Error> { + self.inner.remove_key_below(height + 1); + Ok(()) + } + + fn min_parent_view_height(&self) -> Result, Error> { + Ok(self.inner.lower_bound()) + } + + fn max_parent_view_height(&self) -> Result, Error> { + Ok(self.inner.upper_bound()) + } +} \ No newline at end of file From 78a509fdf6085886ec14a2f23025f286689ab9e1 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Tue, 1 Oct 2024 15:51:02 +0800 Subject: [PATCH 14/16] align with previous PR --- fendermint/vm/topdown/src/observation.rs | 12 +++++++++++- fendermint/vm/topdown/src/syncer/mod.rs | 4 ++-- fendermint/vm/topdown/src/vote/store.rs | 4 ++-- fendermint/vm/topdown/src/vote/tally.rs | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/fendermint/vm/topdown/src/observation.rs b/fendermint/vm/topdown/src/observation.rs index eea36ffde..760580d40 100644 --- a/fendermint/vm/topdown/src/observation.rs +++ b/fendermint/vm/topdown/src/observation.rs @@ -1,7 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::{BlockHeight, Bytes}; +use crate::syncer::error::Error; +use crate::syncer::store::ParentViewStore; +use crate::{BlockHeight, Bytes, Checkpoint}; use anyhow::anyhow; use arbitrary::Arbitrary; use fendermint_crypto::secp::RecoverableECDSASignature; @@ -41,6 +43,14 @@ pub struct CertifiedObservation { signature: RecoverableECDSASignature, } +/// check in the store to see if there is a new observation available +pub fn deduce_new_observation( + _store: &S, + _checkpoint: &Checkpoint, +) -> Result { + todo!() +} + impl TryFrom<&[u8]> for CertifiedObservation { type Error = anyhow::Error; diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index 3b111a298..421c93c93 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -1,7 +1,7 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::observation::ObservationCommitment; +use crate::observation::Observation; use crate::proxy::ParentQueryProxy; use crate::syncer::poll::ParentPoll; use crate::syncer::store::ParentViewStore; @@ -19,7 +19,7 @@ pub mod store; pub enum TopDownSyncEvent { /// The fendermint node is syncing with peers NodeSyncing, - NewProposal(Box), + NewProposal(Box), } pub struct ParentSyncerConfig { diff --git a/fendermint/vm/topdown/src/vote/store.rs b/fendermint/vm/topdown/src/vote/store.rs index 3de09a6d9..5757b0926 100644 --- a/fendermint/vm/topdown/src/vote/store.rs +++ b/fendermint/vm/topdown/src/vote/store.rs @@ -138,13 +138,13 @@ mod tests { (sk, ValidatorKey::new(public_key)) } - fn random_observation() -> ObservationCommitment { + fn random_observation() -> Observation { let mut bytes = [0; 100]; let mut rng = rand::thread_rng(); rng.fill_bytes(&mut bytes); let mut unstructured = Unstructured::new(&bytes); - ObservationCommitment::arbitrary(&mut unstructured).unwrap() + Observation::arbitrary(&mut unstructured).unwrap() } #[test] diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index 5ed1beca0..376d9ff5b 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -230,13 +230,13 @@ mod tests { (sk, ValidatorKey::new(public_key)) } - fn random_observation() -> ObservationCommitment { + fn random_observation() -> Observation { let mut bytes = [0; 100]; let mut rng = rand::thread_rng(); rng.fill_bytes(&mut bytes); let mut unstructured = Unstructured::new(&bytes); - ObservationCommitment::arbitrary(&mut unstructured).unwrap() + Observation::arbitrary(&mut unstructured).unwrap() } #[test] From 7434eba9f2ab9d721d9f63b1bffaa0effcd8d09c Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 28 Nov 2024 14:20:39 +0800 Subject: [PATCH 15/16] update syncer --- fendermint/vm/topdown/src/observation.rs | 38 ++++-- fendermint/vm/topdown/src/syncer/mod.rs | 148 +++++++++++++++------- fendermint/vm/topdown/src/syncer/poll.rs | 122 ++++++++++-------- fendermint/vm/topdown/src/syncer/store.rs | 43 +++++-- 4 files changed, 233 insertions(+), 118 deletions(-) diff --git a/fendermint/vm/topdown/src/observation.rs b/fendermint/vm/topdown/src/observation.rs index 9e481029f..d59c9d9fa 100644 --- a/fendermint/vm/topdown/src/observation.rs +++ b/fendermint/vm/topdown/src/observation.rs @@ -32,13 +32,13 @@ pub struct ObservationConfig { /// The content that validators gossip among each other. #[derive(Serialize, Deserialize, Hash, Debug, Clone, Eq, PartialEq, Arbitrary)] pub struct Observation { - pub(crate) parent_height: u64, + pub(crate) parent_subnet_height: u64, /// The hash of the chain unit at that height. Usually a block hash, but could /// be another entity (e.g. tipset CID), depending on the parent chain /// and our interface to it. For example, if the parent is a Filecoin network, /// this would be a tipset CID coerced into a block hash if queried through /// the Eth API, or the tipset CID as-is if accessed through the Filecoin API. - pub(crate) parent_hash: Bytes, + pub(crate) parent_subnet_hash: Bytes, /// A rolling/cumulative commitment to topdown effects since the beginning of /// time, including the ones in this block. pub(crate) cumulative_effects_comm: Bytes, @@ -100,7 +100,7 @@ pub fn deduce_new_observation( let observation = agg.into_observation()?; tracing::info!( - height = observation.parent_height, + height = observation.parent_subnet_height, "new observation derived" ); @@ -120,6 +120,10 @@ impl CertifiedObservation { &self.observation } + pub fn observation_signature(&self) -> &RecoverableECDSASignature { + &self.observation_signature + } + pub fn ensure_valid(&self) -> anyhow::Result { let to_sign = fvm_ipld_encoding::to_vec(&self.observation)?; let (pk1, _) = self.observation_signature.recover(&to_sign)?; @@ -163,8 +167,8 @@ impl CertifiedObservation { impl Observation { pub fn new(parent_height: BlockHeight, parent_hash: Bytes, commitment: Bytes) -> Self { Self { - parent_height, - parent_hash, + parent_subnet_height: parent_height, + parent_subnet_hash: parent_hash, cumulative_effects_comm: commitment, } } @@ -175,8 +179,8 @@ impl Display for Observation { write!( f, "Observation(parent_height={}, parent_hash={}, commitment={})", - self.parent_height, - hex::encode(&self.parent_hash), + self.parent_subnet_height, + hex::encode(&self.parent_subnet_hash), hex::encode(&self.cumulative_effects_comm), ) } @@ -184,7 +188,7 @@ impl Display for Observation { impl Observation { pub fn parent_height(&self) -> BlockHeight { - self.parent_height + self.parent_subnet_height } } @@ -195,7 +199,7 @@ impl ObservationConfig { } } -struct LinearizedParentBlockView { +pub(crate) struct LinearizedParentBlockView { parent_height: u64, parent_hash: Option, cumulative_effects_comm: Bytes, @@ -211,13 +215,23 @@ impl From<&Checkpoint> for LinearizedParentBlockView { } } +impl From<&Observation> for LinearizedParentBlockView { + fn from(value: &Observation) -> Self { + LinearizedParentBlockView { + parent_height: value.parent_subnet_height, + parent_hash: Some(value.parent_subnet_hash.clone()), + cumulative_effects_comm: value.cumulative_effects_comm.clone(), + } + } +} + impl LinearizedParentBlockView { fn new_commitment(&mut self, to_append: Bytes) { let bytes = [ self.cumulative_effects_comm.as_slice(), to_append.as_slice(), ] - .concat(); + .concat(); let cid = Cid::new_v1(DAG_CBOR, Code::Blake2b256.digest(&bytes)); self.cumulative_effects_comm = cid.to_bytes(); } @@ -238,7 +252,7 @@ impl LinearizedParentBlockView { Ok(()) } - fn into_observation(self) -> Result { + pub fn into_observation(self) -> Result { let Some(hash) = self.parent_hash else { return Err(Error::CannotCommitObservationAtNullBlock( self.parent_height, @@ -250,4 +264,4 @@ impl LinearizedParentBlockView { self.cumulative_effects_comm, )) } -} +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index 691307367..3def2e614 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -1,20 +1,26 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::observation::{Observation, ObservationConfig}; -use crate::proxy::ParentQueryProxy; -use crate::syncer::poll::ParentPoll; +use crate::observation::{LinearizedParentBlockView, Observation, ObservationConfig}; use crate::syncer::store::ParentViewStore; use crate::{BlockHeight, Checkpoint}; +use anyhow::anyhow; +use async_trait::async_trait; +use ipc_api::cross::IpcEnvelope; +use ipc_api::staking::StakingChangeRequest; +use serde::Deserialize; +use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::select; -use tokio::sync::mpsc; +use tokio::sync::{broadcast, mpsc}; pub mod error; pub mod payload; pub mod poll; pub mod store; +pub type QuorumCertContent = (Observation, Vec, Vec); + #[derive(Clone, Debug)] pub enum TopDownSyncEvent { /// The fendermint node is syncing with peers @@ -22,6 +28,7 @@ pub enum TopDownSyncEvent { NewProposal(Box), } +#[derive(Debug, Clone, Deserialize)] pub struct ParentSyncerConfig { pub request_channel_size: usize, /// The event broadcast channel buffer size @@ -31,12 +38,10 @@ pub struct ParentSyncerConfig { /// conservative and avoid other from rejecting the proposal because they don't see the /// height as final yet. pub chain_head_delay: BlockHeight, - /// Parent syncing cron period, in seconds - pub polling_interval: Duration, - /// Top down exponential back off retry base - pub exponential_back_off: Duration, - /// The max number of retries for exponential backoff before giving up - pub exponential_retry_limit: usize, + /// Parent syncing cron period, in millis + pub polling_interval_millis: Duration, + /// Max number of requests to process in the reactor loop + pub max_requests_per_loop: usize, /// Max number of un-finalized parent blocks that should be stored in the store pub max_store_blocks: BlockHeight, /// Attempts to sync as many block as possible till the finalized chain head @@ -46,26 +51,37 @@ pub struct ParentSyncerConfig { } #[derive(Clone)] -pub struct ParentSyncerReactorClient { +pub struct ParentSyncerReactorClient { tx: mpsc::Sender, + checkpoint: Arc>, + store: S, } -pub fn start_parent_syncer( - config: ParentSyncerConfig, - proxy: P, - store: S, - last_finalized: Checkpoint, -) -> anyhow::Result -where - S: ParentViewStore + Send + Sync + 'static, - P: Send + Sync + 'static + ParentQueryProxy, -{ - let (tx, mut rx) = mpsc::channel(config.request_channel_size); +impl ParentSyncerReactorClient { + pub fn new( + request_channel_size: usize, + store: S, + ) -> (Self, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(request_channel_size); + let checkpoint = Arc::new(Mutex::new(Checkpoint::v1(0, vec![], vec![]))); + ( + Self { + tx, + checkpoint, + store, + }, + rx, + ) + } +} +pub fn start_polling_reactor( + mut rx: mpsc::Receiver, + mut poller: P, + config: ParentSyncerConfig, +) { + let polling_interval = config.polling_interval_millis; tokio::spawn(async move { - let polling_interval = config.polling_interval; - let mut poller = ParentPoll::new(config, proxy, store, last_finalized); - loop { select! { _ = tokio::time::sleep(polling_interval) => { @@ -75,41 +91,83 @@ where } req = rx.recv() => { let Some(req) = req else { break }; - handle_request(req, &mut poller); + match req { + ParentSyncerRequest::Finalized(cp) => { + if let Err(e) = poller.finalize(cp) { + tracing::error!(err = e.to_string(), "cannot finalize syncer") + } + }, + } } } } - - tracing::warn!("parent syncer stopped") }); - Ok(ParentSyncerReactorClient { tx }) } -impl ParentSyncerReactorClient { +/// Polls the parent block view +#[async_trait] +pub trait ParentPoller { + type Store: ParentViewStore + Send + Sync + 'static + Clone; + + fn subscribe(&self) -> broadcast::Receiver; + + fn store(&self) -> Self::Store; + + /// The target block height is finalized, purge all the parent view before the target height + fn finalize(&mut self, checkpoint: Checkpoint) -> anyhow::Result<()>; + + /// Try to poll the next parent height + async fn try_poll(&mut self) -> anyhow::Result<()>; +} + +impl ParentSyncerReactorClient { + fn set_checkpoint(&self, cp: Checkpoint) { + let mut checkpoint = self.checkpoint.lock().unwrap(); + *checkpoint = cp.clone(); + } /// Marks the height as finalized. /// There is no need to wait for ack from the reactor pub async fn finalize_parent_height(&self, cp: Checkpoint) -> anyhow::Result<()> { + self.set_checkpoint(cp.clone()); self.tx.send(ParentSyncerRequest::Finalized(cp)).await?; Ok(()) } -} -enum ParentSyncerRequest { - /// A new parent height is finalized - Finalized(Checkpoint), -} + pub fn prepare_quorum_cert_content( + &self, + end_height: BlockHeight, + ) -> anyhow::Result { + let latest_checkpoint = self.checkpoint.lock().unwrap().clone(); + + let mut xnet_msgs = vec![]; + let mut validator_changes = vec![]; + let mut linear = LinearizedParentBlockView::from(&latest_checkpoint); + + let start = latest_checkpoint.target_height() + 1; + for h in start..=end_height { + let Some(v) = self.store.get(h)? else { + return Err(anyhow!("parent block view store does not have data at {h}")); + }; -fn handle_request(req: ParentSyncerRequest, poller: &mut ParentPoll) -where - S: ParentViewStore + Send + Sync + 'static, - P: Send + Sync + 'static + ParentQueryProxy, -{ - match req { - ParentSyncerRequest::Finalized(c) => { - let height = c.target_height(); - if let Err(e) = poller.finalize(c) { - tracing::error!(height, err = e.to_string(), "cannot finalize parent viewer"); + if let Err(e) = linear.append(v.clone()) { + return Err(anyhow!("parent block view cannot be appended: {e}")); + } + + if let Some(payload) = v.payload { + xnet_msgs.extend(payload.xnet_msgs); + validator_changes.extend(payload.validator_changes); } } + + let ob = linear + .into_observation() + .map_err(|e| anyhow!("cannot convert linearized parent view into observation: {e}"))?; + + Ok((ob, xnet_msgs, validator_changes)) } } + +pub enum ParentSyncerRequest { + /// A new parent height is finalized + Finalized(Checkpoint), +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/poll.rs b/fendermint/vm/topdown/src/syncer/poll.rs index 125491fae..6bce1ae60 100644 --- a/fendermint/vm/topdown/src/syncer/poll.rs +++ b/fendermint/vm/topdown/src/syncer/poll.rs @@ -7,16 +7,18 @@ use crate::proxy::ParentQueryProxy; use crate::syncer::error::Error; use crate::syncer::payload::ParentBlockView; use crate::syncer::store::ParentViewStore; -use crate::syncer::{ParentSyncerConfig, TopDownSyncEvent}; +use crate::syncer::{ParentPoller, ParentSyncerConfig, TopDownSyncEvent}; use crate::{is_null_round_str, BlockHash, BlockHeight, Checkpoint}; use anyhow::anyhow; +use async_trait::async_trait; use ipc_observability::emit; use ipc_observability::serde::HexEncodableBlockHash; use libp2p::futures::TryFutureExt; use tokio::sync::broadcast; +use tokio::sync::broadcast::Receiver; use tracing::instrument; -pub(crate) struct ParentPoll { +pub struct ParentPoll { config: ParentSyncerConfig, parent_proxy: P, store: S, @@ -24,71 +26,44 @@ pub(crate) struct ParentPoll { last_finalized: Checkpoint, } -impl ParentPoll -where - S: ParentViewStore + Send + Sync + 'static, - P: Send + Sync + 'static + ParentQueryProxy, +#[async_trait] +impl ParentPoller for ParentPoll + where + S: ParentViewStore + Send + Sync + 'static + Clone, + P: Send + Sync + 'static + ParentQueryProxy, { - pub fn new(config: ParentSyncerConfig, proxy: P, store: S, last_finalized: Checkpoint) -> Self { - let (tx, _) = broadcast::channel(config.broadcast_channel_size); - Self { - config, - parent_proxy: proxy, - store, - event_broadcast: tx, - last_finalized, - } + type Store = S; + + fn subscribe(&self) -> Receiver { + self.event_broadcast.subscribe() + } + + fn store(&self) -> Self::Store { + self.store.clone() } /// The target block height is finalized, purge all the parent view before the target height - pub fn finalize(&mut self, checkpoint: Checkpoint) -> Result<(), Error> { + fn finalize(&mut self, checkpoint: Checkpoint) -> anyhow::Result<()> { let Some(min_height) = self.store.min_parent_view_height()? else { return Ok(()); }; for h in min_height..=checkpoint.target_height() { self.store.purge(h)?; } - Ok(()) - } - /// Get the latest non null block data stored - async fn latest_nonnull_data(&self) -> anyhow::Result<(BlockHeight, BlockHash)> { - let Some(latest_height) = self.store.max_parent_view_height()? else { - return Ok(( - self.last_finalized.target_height(), - self.last_finalized.target_hash().clone(), - )); - }; + self.last_finalized = checkpoint; - let start = self.last_finalized.target_height() + 1; - for h in (start..=latest_height).rev() { - let Some(p) = self.store.get(h)? else { - continue; - }; - - // if parent hash of the proposal is null, it means the - let Some(p) = p.payload else { - continue; - }; - - return Ok((h, p.parent_hash)); - } - - // this means the votes stored are all null blocks, return last committed finality - Ok(( - self.last_finalized.target_height(), - self.last_finalized.target_hash().clone(), - )) + Ok(()) } /// Insert the height into cache when we see a new non null block - pub async fn try_poll(&mut self) -> anyhow::Result<()> { + async fn try_poll(&mut self) -> anyhow::Result<()> { let Some(chain_head) = self.finalized_chain_head().await? else { return Ok(()); }; let (mut latest_height_fetched, mut first_non_null_parent_hash) = - self.latest_nonnull_data().await?; + self.latest_nonnull_data()?; tracing::debug!(chain_head, latest_height_fetched, "syncing heights"); if latest_height_fetched > chain_head { @@ -140,6 +115,53 @@ where Ok(()) } +} + +impl ParentPoll + where + S: ParentViewStore + Send + Sync + 'static, + P: Send + Sync + 'static + ParentQueryProxy, +{ + pub fn new(config: ParentSyncerConfig, proxy: P, store: S, last_finalized: Checkpoint) -> Self { + let (tx, _) = broadcast::channel(config.broadcast_channel_size); + Self { + config, + parent_proxy: proxy, + store, + event_broadcast: tx, + last_finalized, + } + } + + /// Get the latest non null block data stored + fn latest_nonnull_data(&self) -> anyhow::Result<(BlockHeight, BlockHash)> { + let Some(latest_height) = self.store.max_parent_view_height()? else { + return Ok(( + self.last_finalized.target_height(), + self.last_finalized.target_hash().clone(), + )); + }; + + let start = self.last_finalized.target_height() + 1; + for h in (start..=latest_height).rev() { + let Some(p) = self.store.get(h)? else { + continue; + }; + + // if parent hash of the proposal is null, it means the + let Some(p) = p.payload else { + continue; + }; + + return Ok((h, p.parent_hash)); + } + + // this means the votes stored are all null blocks, return last committed finality + Ok(( + self.last_finalized.target_height(), + self.last_finalized.target_hash().clone(), + )) + } fn store_full(&self) -> anyhow::Result { let Some(h) = self.store.max_parent_view_height()? else { @@ -251,8 +273,8 @@ async fn fetch_data

( height: BlockHeight, block_hash: BlockHash, ) -> Result -where - P: ParentQueryProxy + Send + Sync + 'static, + where + P: ParentQueryProxy + Send + Sync + 'static, { let changes_res = parent_proxy .get_validator_changes(height) @@ -291,4 +313,4 @@ where topdown_msgs_res.value, changes_res.value, )) -} +} \ No newline at end of file diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs index f38b4dccb..21015b89a 100644 --- a/fendermint/vm/topdown/src/syncer/store.rs +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -4,48 +4,69 @@ use crate::syncer::error::Error; use crate::syncer::payload::ParentBlockView; use crate::{BlockHeight, SequentialKeyCache}; +use std::sync::{Arc, RwLock}; /// Stores the parent view observed of the current node pub trait ParentViewStore { /// Store a newly observed parent view - fn store(&mut self, view: ParentBlockView) -> Result<(), Error>; + fn store(&self, view: ParentBlockView) -> Result<(), Error>; /// Get the parent view at the specified height fn get(&self, height: BlockHeight) -> Result, Error>; /// Purge the parent view at the target height - fn purge(&mut self, height: BlockHeight) -> Result<(), Error>; + fn purge(&self, height: BlockHeight) -> Result<(), Error>; fn min_parent_view_height(&self) -> Result, Error>; fn max_parent_view_height(&self) -> Result, Error>; } +#[derive(Clone)] pub struct InMemoryParentViewStore { - inner: SequentialKeyCache, + inner: Arc>>, +} + +impl Default for InMemoryParentViewStore { + fn default() -> Self { + Self::new() + } +} + +impl InMemoryParentViewStore { + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(SequentialKeyCache::sequential())), + } + } } impl ParentViewStore for InMemoryParentViewStore { - fn store(&mut self, view: ParentBlockView) -> Result<(), Error> { - self.inner + fn store(&self, view: ParentBlockView) -> Result<(), Error> { + let mut inner = self.inner.write().unwrap(); + inner .append(view.parent_height, view) .map_err(|_| Error::NonSequentialParentViewInsert) } fn get(&self, height: BlockHeight) -> Result, Error> { - Ok(self.inner.get_value(height).cloned()) + let inner = self.inner.read().unwrap(); + Ok(inner.get_value(height).cloned()) } - fn purge(&mut self, height: BlockHeight) -> Result<(), Error> { - self.inner.remove_key_below(height + 1); + fn purge(&self, height: BlockHeight) -> Result<(), Error> { + let mut inner = self.inner.write().unwrap(); + inner.remove_key_below(height + 1); Ok(()) } fn min_parent_view_height(&self) -> Result, Error> { - Ok(self.inner.lower_bound()) + let inner = self.inner.read().unwrap(); + Ok(inner.lower_bound()) } fn max_parent_view_height(&self) -> Result, Error> { - Ok(self.inner.upper_bound()) + let inner = self.inner.read().unwrap(); + Ok(inner.upper_bound()) } -} +} \ No newline at end of file From 74eca892648237f79b7a671624bd874a8ad45108 Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Tue, 18 Feb 2025 16:25:03 +0800 Subject: [PATCH 16/16] feat(node): integrating topdown cert into interpreters (#1187) Co-authored-by: cryptoAtwill --- Cargo.lock | 5 + Cargo.toml | 1 + contracts/contracts/errors/IPCErrors.sol | 2 +- .../contracts/gateway/GatewayGetterFacet.sol | 10 +- .../gateway/router/TopDownFinalityFacet.sol | 24 +- contracts/contracts/interfaces/IGateway.sol | 4 +- contracts/contracts/lib/LibGateway.sol | 18 +- .../contracts/lib/LibGatewayActorStorage.sol | 4 +- contracts/contracts/structs/CrossNet.sol | 5 +- contracts/test/IntegrationTestBase.sol | 18 +- contracts/test/helpers/SelectorLibrary.sol | 4 +- .../test/integration/GatewayDiamond.t.sol | 29 +- .../test/integration/L2GatewayDiamond.t.sol | 2 +- contracts/test/integration/MultiSubnet.t.sol | 28 +- .../linked-token/test/MultiSubnetTest.t.sol | 2 +- fendermint/app/settings/src/lib.rs | 13 +- fendermint/app/src/app.rs | 6 + fendermint/app/src/cmd/debug.rs | 43 +- fendermint/app/src/cmd/run.rs | 323 +++++----- fendermint/app/src/ipc.rs | 23 +- fendermint/crypto/Cargo.toml | 7 +- fendermint/crypto/src/lib.rs | 1 + fendermint/crypto/src/quorum.rs | 249 ++++++++ .../materializer/tests/docker_tests/layer2.rs | 6 +- fendermint/vm/genesis/src/lib.rs | 15 + fendermint/vm/interpreter/src/chain.rs | 178 +----- .../vm/interpreter/src/fvm/state/ipc.rs | 28 +- fendermint/vm/interpreter/src/fvm/topdown.rs | 28 +- fendermint/vm/message/Cargo.toml | 1 + .../vm/message/golden/chain/ipc_top_down.cbor | 2 +- .../vm/message/golden/chain/ipc_top_down.txt | 2 +- fendermint/vm/message/src/ipc.rs | 23 +- fendermint/vm/topdown/Cargo.toml | 3 +- fendermint/vm/topdown/src/convert.rs | 41 +- fendermint/vm/topdown/src/error.rs | 18 - fendermint/vm/topdown/src/finality/fetch.rs | 450 ------------- fendermint/vm/topdown/src/finality/mod.rs | 177 ------ fendermint/vm/topdown/src/finality/null.rs | 566 ----------------- fendermint/vm/topdown/src/launch.rs | 254 ++++++++ fendermint/vm/topdown/src/lib.rs | 247 +++----- fendermint/vm/topdown/src/observation.rs | 4 +- fendermint/vm/topdown/src/sync/mod.rs | 203 ------ fendermint/vm/topdown/src/sync/syncer.rs | 596 ------------------ fendermint/vm/topdown/src/sync/tendermint.rs | 47 -- fendermint/vm/topdown/src/syncer/mod.rs | 2 +- fendermint/vm/topdown/src/syncer/poll.rs | 18 +- fendermint/vm/topdown/src/syncer/store.rs | 2 +- fendermint/vm/topdown/src/toggle.rs | 130 ---- fendermint/vm/topdown/src/vote/error.rs | 9 + fendermint/vm/topdown/src/vote/mod.rs | 138 ++-- .../vm/topdown/src/vote/operation/paused.rs | 2 +- fendermint/vm/topdown/src/vote/payload.rs | 7 + fendermint/vm/topdown/src/vote/store.rs | 52 +- fendermint/vm/topdown/src/vote/tally.rs | 71 ++- fendermint/vm/topdown/src/voting.rs | 480 -------------- fendermint/vm/topdown/tests/smt_voting.rs | 508 --------------- fendermint/vm/topdown/tests/vote_reactor.rs | 102 +-- ipc/api/src/staking.rs | 6 +- .../src/commands/crossmsg/topdown_cross.rs | 2 +- ipc/provider/src/lib.rs | 6 +- ipc/provider/src/manager/evm/manager.rs | 4 +- ipc/provider/src/manager/subnet.rs | 4 +- ipld/resolver/src/behaviour/membership.rs | 19 +- ipld/resolver/src/client.rs | 8 +- ipld/resolver/src/lib.rs | 2 +- ipld/resolver/src/service.rs | 7 +- ipld/resolver/src/signed_record.rs | 2 + ipld/resolver/src/vote_record.rs | 5 + ipld/resolver/tests/smoke.rs | 13 +- 69 files changed, 1312 insertions(+), 3997 deletions(-) create mode 100644 fendermint/crypto/src/quorum.rs delete mode 100644 fendermint/vm/topdown/src/error.rs delete mode 100644 fendermint/vm/topdown/src/finality/fetch.rs delete mode 100644 fendermint/vm/topdown/src/finality/mod.rs delete mode 100644 fendermint/vm/topdown/src/finality/null.rs create mode 100644 fendermint/vm/topdown/src/launch.rs delete mode 100644 fendermint/vm/topdown/src/sync/mod.rs delete mode 100644 fendermint/vm/topdown/src/sync/syncer.rs delete mode 100644 fendermint/vm/topdown/src/sync/tendermint.rs delete mode 100644 fendermint/vm/topdown/src/toggle.rs delete mode 100644 fendermint/vm/topdown/src/voting.rs delete mode 100644 fendermint/vm/topdown/tests/smt_voting.rs diff --git a/Cargo.lock b/Cargo.lock index df58fc7d3..585d834f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3051,6 +3051,8 @@ dependencies = [ "fvm_ipld_encoding", "libsecp256k1", "multihash 0.18.1", + "num-rational", + "num-traits", "rand", "serde", "zeroize", @@ -3432,6 +3434,7 @@ dependencies = [ "fendermint_vm_actor_interface", "fendermint_vm_encoding", "fendermint_vm_message", + "fendermint_vm_topdown", "fvm_ipld_encoding", "fvm_shared", "hex", @@ -3524,8 +3527,10 @@ dependencies = [ "ipc_ipld_resolver", "libp2p", "multihash 0.18.1", + "num-rational", "num-traits", "prometheus", + "quickcheck", "rand", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 581161417..7caa50a25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,6 +127,7 @@ multihash = { version = "0.18.1", default-features = false, features = [ ] } num-bigint = "0.4" num-derive = "0.3" +num-rational = "0.4.1" num-traits = "0.2" num_enum = "0.7.2" paste = "1" diff --git a/contracts/contracts/errors/IPCErrors.sol b/contracts/contracts/errors/IPCErrors.sol index 7d73a8df4..6e33936ac 100644 --- a/contracts/contracts/errors/IPCErrors.sol +++ b/contracts/contracts/errors/IPCErrors.sol @@ -61,7 +61,7 @@ error NotValidator(address); error OldConfigurationNumber(); error PQDoesNotContainAddress(); error PQEmpty(); -error ParentFinalityAlreadyCommitted(); +error TopdownCheckpointAlreadyCommitted(); error PostboxNotExist(); error SignatureReplay(); error SubnetAlreadyKilled(); diff --git a/contracts/contracts/gateway/GatewayGetterFacet.sol b/contracts/contracts/gateway/GatewayGetterFacet.sol index 7dd47386e..2db734eac 100644 --- a/contracts/contracts/gateway/GatewayGetterFacet.sol +++ b/contracts/contracts/gateway/GatewayGetterFacet.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 pragma solidity ^0.8.23; -import {BottomUpCheckpoint, BottomUpMsgBatch, IpcEnvelope, ParentFinality} from "../structs/CrossNet.sol"; +import {BottomUpCheckpoint, BottomUpMsgBatch, IpcEnvelope, TopdownCheckpoint} from "../structs/CrossNet.sol"; import {QuorumInfo} from "../structs/Quorum.sol"; import {SubnetID, Subnet} from "../structs/Subnet.sol"; import {Membership} from "../structs/Subnet.sol"; @@ -71,13 +71,13 @@ contract GatewayGetterFacet { /// @notice Returns the parent chain finality information for a given block number. /// @param blockNumber The block number for which to retrieve parent-finality information. - function getParentFinality(uint256 blockNumber) external view returns (ParentFinality memory) { - return LibGateway.getParentFinality(blockNumber); + function getTopdownCheckpoint(uint256 blockNumber) external view returns (TopdownCheckpoint memory) { + return LibGateway.getTopdownCheckpoint(blockNumber); } /// @notice Gets the most recent parent-finality information from the parent. - function getLatestParentFinality() external view returns (ParentFinality memory) { - return LibGateway.getLatestParentFinality(); + function getLatestTopdownCheckpoint() external view returns (TopdownCheckpoint memory) { + return LibGateway.getLatestTopdownCheckpoint(); } /// @notice Returns the subnet with the given id. diff --git a/contracts/contracts/gateway/router/TopDownFinalityFacet.sol b/contracts/contracts/gateway/router/TopDownFinalityFacet.sol index b29f7e203..933b5a194 100644 --- a/contracts/contracts/gateway/router/TopDownFinalityFacet.sol +++ b/contracts/contracts/gateway/router/TopDownFinalityFacet.sol @@ -2,7 +2,7 @@ pragma solidity ^0.8.23; import {GatewayActorModifiers} from "../../lib/LibGatewayActorStorage.sol"; -import {ParentFinality} from "../../structs/CrossNet.sol"; +import {TopdownCheckpoint} from "../../structs/CrossNet.sol"; import {PermissionMode, Validator, ValidatorInfo, StakingChangeRequest, Membership} from "../../structs/Subnet.sol"; import {LibGateway} from "../../lib/LibGateway.sol"; @@ -16,17 +16,17 @@ contract TopDownFinalityFacet is GatewayActorModifiers { using LibValidatorTracking for ParentValidatorsTracker; using LibValidatorSet for ValidatorSet; - /// @notice commit the ipc parent finality into storage and returns the previous committed finality - /// This is useful to understand if the finalities are consistent or if there have been reorgs. - /// If there are no previous committed fainality, it will be default to zero values, i.e. zero height and block hash. - /// @param finality - the parent finality - /// @return hasCommittedBefore A flag that indicates if a finality record has been committed before. - /// @return previousFinality The previous finality information. - function commitParentFinality( - ParentFinality calldata finality - ) external systemActorOnly returns (bool hasCommittedBefore, ParentFinality memory previousFinality) { - previousFinality = LibGateway.commitParentFinality(finality); - hasCommittedBefore = previousFinality.height != 0; + /// @notice commit the ipc topdown checkpoint into storage and returns the previous committed checkpoint + /// This is useful to understand if the checkpoints are consistent or if there have been reorgs. + /// If there are no previous committed checkpoint, it will be default to zero values, i.e. zero height and block hash. + /// @param checkpoint - the topdown checkpoint + /// @return hasCommittedBefore A flag that indicates if a checkpoint record has been committed before. + /// @return previousCheckpoint The previous checkpoint information. + function commitTopdownCheckpoint( + TopdownCheckpoint calldata checkpoint + ) external systemActorOnly returns (bool hasCommittedBefore, TopdownCheckpoint memory previousCheckpoint) { + previousCheckpoint = LibGateway.commitTopdownCheckpoint(checkpoint); + hasCommittedBefore = previousCheckpoint.height != 0; } /// @notice Store the validator change requests from parent. diff --git a/contracts/contracts/interfaces/IGateway.sol b/contracts/contracts/interfaces/IGateway.sol index 1c807ea81..23f3e3ac0 100644 --- a/contracts/contracts/interfaces/IGateway.sol +++ b/contracts/contracts/interfaces/IGateway.sol @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 pragma solidity ^0.8.23; -import {BottomUpCheckpoint, BottomUpMsgBatch, IpcEnvelope, ParentFinality} from "../structs/CrossNet.sol"; +import {BottomUpCheckpoint, BottomUpMsgBatch, IpcEnvelope, TopdownCheckpoint} from "../structs/CrossNet.sol"; import {FullActivityRollup} from "../structs/Activity.sol"; import {SubnetID} from "../structs/Subnet.sol"; import {FvmAddress} from "../structs/FvmAddress.sol"; @@ -66,7 +66,7 @@ interface IGateway { function propagate(bytes32 msgCid) external payable; /// @notice commit the ipc parent finality into storage - function commitParentFinality(ParentFinality calldata finality) external; + function commitTopdownCheckpoint(TopdownCheckpoint calldata finality) external; /// @notice creates a new bottom-up checkpoint function createBottomUpCheckpoint( diff --git a/contracts/contracts/lib/LibGateway.sol b/contracts/contracts/lib/LibGateway.sol index e48c0dcee..403d24ee5 100644 --- a/contracts/contracts/lib/LibGateway.sol +++ b/contracts/contracts/lib/LibGateway.sol @@ -6,9 +6,9 @@ import {GatewayActorStorage, LibGatewayActorStorage} from "../lib/LibGatewayActo import {BURNT_FUNDS_ACTOR} from "../constants/Constants.sol"; import {SubnetID, Subnet, AssetKind, Asset} from "../structs/Subnet.sol"; import {SubnetActorGetterFacet} from "../subnet/SubnetActorGetterFacet.sol"; -import {CallMsg, IpcMsgKind, IpcEnvelope, OutcomeType, BottomUpMsgBatch, BottomUpMsgBatch, BottomUpCheckpoint, ParentFinality} from "../structs/CrossNet.sol"; +import {CallMsg, IpcMsgKind, IpcEnvelope, OutcomeType, BottomUpMsgBatch, BottomUpMsgBatch, BottomUpCheckpoint, TopdownCheckpoint} from "../structs/CrossNet.sol"; import {Membership} from "../structs/Subnet.sol"; -import {CannotSendCrossMsgToItself, MethodNotAllowed, MaxMsgsPerBatchExceeded, InvalidXnetMessage ,OldConfigurationNumber, NotRegisteredSubnet, InvalidActorAddress, ParentFinalityAlreadyCommitted, InvalidXnetMessageReason} from "../errors/IPCErrors.sol"; +import {CannotSendCrossMsgToItself, MethodNotAllowed, MaxMsgsPerBatchExceeded, InvalidXnetMessage ,OldConfigurationNumber, NotRegisteredSubnet, InvalidActorAddress, TopdownCheckpointAlreadyCommitted, InvalidXnetMessageReason} from "../errors/IPCErrors.sol"; import {CrossMsgHelper} from "../lib/CrossMsgHelper.sol"; import {FilAddress} from "fevmate/contracts/utils/FilAddress.sol"; import {SubnetIDHelper} from "../lib/SubnetIDHelper.sol"; @@ -117,27 +117,27 @@ library LibGateway { /// @notice obtain the ipc parent finality at certain block number /// @param blockNumber - the block number to obtain the finality - function getParentFinality(uint256 blockNumber) internal view returns (ParentFinality memory) { + function getTopdownCheckpoint(uint256 blockNumber) internal view returns (TopdownCheckpoint memory) { GatewayActorStorage storage s = LibGatewayActorStorage.appStorage(); return s.finalitiesMap[blockNumber]; } /// @notice obtain the latest committed ipc parent finality - function getLatestParentFinality() internal view returns (ParentFinality memory) { + function getLatestTopdownCheckpoint() internal view returns (TopdownCheckpoint memory) { GatewayActorStorage storage s = LibGatewayActorStorage.appStorage(); - return getParentFinality(s.latestParentHeight); + return getTopdownCheckpoint(s.latestParentHeight); } /// @notice commit the ipc parent finality into storage /// @param finality - the finality to be committed - function commitParentFinality( - ParentFinality calldata finality - ) internal returns (ParentFinality memory lastFinality) { + function commitTopdownCheckpoint( + TopdownCheckpoint calldata finality + ) internal returns (TopdownCheckpoint memory lastFinality) { GatewayActorStorage storage s = LibGatewayActorStorage.appStorage(); uint256 lastHeight = s.latestParentHeight; if (lastHeight >= finality.height) { - revert ParentFinalityAlreadyCommitted(); + revert TopdownCheckpointAlreadyCommitted(); } lastFinality = s.finalitiesMap[lastHeight]; diff --git a/contracts/contracts/lib/LibGatewayActorStorage.sol b/contracts/contracts/lib/LibGatewayActorStorage.sol index 4cb63536e..a8f41ecdc 100644 --- a/contracts/contracts/lib/LibGatewayActorStorage.sol +++ b/contracts/contracts/lib/LibGatewayActorStorage.sol @@ -3,7 +3,7 @@ pragma solidity ^0.8.23; import {NotSystemActor, NotEnoughFunds} from "../errors/IPCErrors.sol"; import {QuorumMap} from "../structs/Quorum.sol"; -import {BottomUpCheckpoint, BottomUpMsgBatch, IpcEnvelope, ParentFinality} from "../structs/CrossNet.sol"; +import {BottomUpCheckpoint, BottomUpMsgBatch, IpcEnvelope, TopdownCheckpoint} from "../structs/CrossNet.sol"; import {SubnetID, Subnet, ParentValidatorsTracker} from "../structs/Subnet.sol"; import {Membership} from "../structs/Subnet.sol"; import {AccountHelper} from "../lib/AccountHelper.sol"; @@ -60,7 +60,7 @@ struct GatewayActorStorage { /// SubnetID => Subnet mapping(bytes32 => Subnet) subnets; /// @notice The parent finalities. Key is the block number, value is the finality struct. - mapping(uint256 => ParentFinality) finalitiesMap; + mapping(uint256 => TopdownCheckpoint) finalitiesMap; /// @notice Postbox keeps track of all the cross-net messages triggered by /// an actor that need to be propagated further through the hierarchy. /// cross-net message id => CrossMsg diff --git a/contracts/contracts/structs/CrossNet.sol b/contracts/contracts/structs/CrossNet.sol index 1f5962346..ab4c528b2 100644 --- a/contracts/contracts/structs/CrossNet.sol +++ b/contracts/contracts/structs/CrossNet.sol @@ -9,9 +9,12 @@ uint64 constant MAX_MSGS_PER_BATCH = 10; uint256 constant BATCH_PERIOD = 100; /// @notice The parent finality for IPC parent at certain height. -struct ParentFinality { +struct TopdownCheckpoint { uint256 height; bytes32 blockHash; + /// The commiment of topdown effects (topdown messages + validator changes). + /// Current version is the CID. + bytes effectsCommitment; } /// @notice A bottom-up checkpoint type. diff --git a/contracts/test/IntegrationTestBase.sol b/contracts/test/IntegrationTestBase.sol index 19013b3d7..65053b095 100644 --- a/contracts/test/IntegrationTestBase.sol +++ b/contracts/test/IntegrationTestBase.sol @@ -7,7 +7,7 @@ import "../contracts/errors/IPCErrors.sol"; import {EMPTY_BYTES, METHOD_SEND} from "../contracts/constants/Constants.sol"; import {ConsensusType} from "../contracts/enums/ConsensusType.sol"; import {IDiamond} from "../contracts/interfaces/IDiamond.sol"; -import {IpcEnvelope, BottomUpCheckpoint, IpcMsgKind, ParentFinality, CallMsg} from "../contracts/structs/CrossNet.sol"; +import {IpcEnvelope, BottomUpCheckpoint, IpcMsgKind, TopdownCheckpoint, CallMsg} from "../contracts/structs/CrossNet.sol"; import {FvmAddress} from "../contracts/structs/FvmAddress.sol"; import {SubnetID, AssetKind, PermissionMode, PermissionMode, Subnet, Asset, IPCAddress, Validator} from "../contracts/structs/Subnet.sol"; import {SubnetIDHelper} from "../contracts/lib/SubnetIDHelper.sol"; @@ -782,10 +782,14 @@ contract IntegrationTestBase is Test, TestParams, TestRegistry, TestSubnetActor, weights[1] = 100; weights[2] = 100; - ParentFinality memory finality = ParentFinality({height: block.number, blockHash: bytes32(0)}); + TopdownCheckpoint memory finality = TopdownCheckpoint({ + height: block.number, + blockHash: bytes32(0), + effectsCommitment: new bytes(0) + }); vm.prank(FilAddress.SYSTEM_ACTOR); - gatewayDiamond.topDownFinalizer().commitParentFinality(finality); + gatewayDiamond.topDownFinalizer().commitTopdownCheckpoint(finality); } function setupWhiteListMethod(address caller, address src) public returns (bytes32) { @@ -829,11 +833,15 @@ contract IntegrationTestBase is Test, TestParams, TestRegistry, TestSubnetActor, weights[0] = weight; vm.deal(validator, 1); - ParentFinality memory finality = ParentFinality({height: block.number, blockHash: bytes32(0)}); + TopdownCheckpoint memory finality = TopdownCheckpoint({ + height: block.number, + blockHash: bytes32(0), + effectsCommitment: new bytes(0) + }); // uint64 n = gatewayDiamond.getter().getLastConfigurationNumber() + 1; vm.startPrank(FilAddress.SYSTEM_ACTOR); - gatewayDiamond.topDownFinalizer().commitParentFinality(finality); + gatewayDiamond.topDownFinalizer().commitTopdownCheckpoint(finality); vm.stopPrank(); } diff --git a/contracts/test/helpers/SelectorLibrary.sol b/contracts/test/helpers/SelectorLibrary.sol index 88ff96643..5d505584f 100644 --- a/contracts/test/helpers/SelectorLibrary.sol +++ b/contracts/test/helpers/SelectorLibrary.sol @@ -27,7 +27,7 @@ library SelectorLibrary { if (keccak256(abi.encodePacked(facetName)) == keccak256(abi.encodePacked("GatewayGetterFacet"))) { return abi.decode( - hex"000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000218789f83b0000000000000000000000000000000000000000000000000000000006c46853000000000000000000000000000000000000000000000000000000002da5794a00000000000000000000000000000000000000000000000000000000dd81b5cf0000000000000000000000000000000000000000000000000000000041b6a2e80000000000000000000000000000000000000000000000000000000038d6693200000000000000000000000000000000000000000000000000000000b3ab3f7400000000000000000000000000000000000000000000000000000000ac12d763000000000000000000000000000000000000000000000000000000004aa8f8a500000000000000000000000000000000000000000000000000000000ca41d5ce00000000000000000000000000000000000000000000000000000000444ead5100000000000000000000000000000000000000000000000000000000d6c5c39700000000000000000000000000000000000000000000000000000000544dddff000000000000000000000000000000000000000000000000000000006ad21bb000000000000000000000000000000000000000000000000000000000a517218f000000000000000000000000000000000000000000000000000000009704276600000000000000000000000000000000000000000000000000000000b1ba49b000000000000000000000000000000000000000000000000000000000f3229131000000000000000000000000000000000000000000000000000000000338150f0000000000000000000000000000000000000000000000000000000094074b03000000000000000000000000000000000000000000000000000000007edeac920000000000000000000000000000000000000000000000000000000006572c1a00000000000000000000000000000000000000000000000000000000c66c66a1000000000000000000000000000000000000000000000000000000003594c3c1000000000000000000000000000000000000000000000000000000009d3070b50000000000000000000000000000000000000000000000000000000042398a9a00000000000000000000000000000000000000000000000000000000fa34a400000000000000000000000000000000000000000000000000000000005d02968500000000000000000000000000000000000000000000000000000000599c7bd10000000000000000000000000000000000000000000000000000000005aff0b3000000000000000000000000000000000000000000000000000000008cfd78e70000000000000000000000000000000000000000000000000000000002e30f9a00000000000000000000000000000000000000000000000000000000a2b6715800000000000000000000000000000000000000000000000000000000", + hex"000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000218789f83b0000000000000000000000000000000000000000000000000000000006c46853000000000000000000000000000000000000000000000000000000002da5794a00000000000000000000000000000000000000000000000000000000dd81b5cf0000000000000000000000000000000000000000000000000000000041b6a2e80000000000000000000000000000000000000000000000000000000038d6693200000000000000000000000000000000000000000000000000000000b3ab3f7400000000000000000000000000000000000000000000000000000000ac12d763000000000000000000000000000000000000000000000000000000004aa8f8a500000000000000000000000000000000000000000000000000000000ca41d5ce00000000000000000000000000000000000000000000000000000000444ead5100000000000000000000000000000000000000000000000000000000d6c5c39700000000000000000000000000000000000000000000000000000000544dddff000000000000000000000000000000000000000000000000000000006ad21bb000000000000000000000000000000000000000000000000000000000a517218f000000000000000000000000000000000000000000000000000000009704276600000000000000000000000000000000000000000000000000000000b1ba49b000000000000000000000000000000000000000000000000000000000f322913100000000000000000000000000000000000000000000000000000000c17117e90000000000000000000000000000000000000000000000000000000094074b030000000000000000000000000000000000000000000000000000000006572c1a00000000000000000000000000000000000000000000000000000000c66c66a1000000000000000000000000000000000000000000000000000000003594c3c1000000000000000000000000000000000000000000000000000000009d3070b50000000000000000000000000000000000000000000000000000000042398a9a000000000000000000000000000000000000000000000000000000003c71caeb00000000000000000000000000000000000000000000000000000000fa34a400000000000000000000000000000000000000000000000000000000005d02968500000000000000000000000000000000000000000000000000000000599c7bd10000000000000000000000000000000000000000000000000000000005aff0b3000000000000000000000000000000000000000000000000000000008cfd78e70000000000000000000000000000000000000000000000000000000002e30f9a00000000000000000000000000000000000000000000000000000000a2b6715800000000000000000000000000000000000000000000000000000000", (bytes4[]) ); } @@ -55,7 +55,7 @@ library SelectorLibrary { if (keccak256(abi.encodePacked(facetName)) == keccak256(abi.encodePacked("TopDownFinalityFacet"))) { return abi.decode( - hex"000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000040df144610000000000000000000000000000000000000000000000000000000011196974000000000000000000000000000000000000000000000000000000008fbe0b7c00000000000000000000000000000000000000000000000000000000e49a547d00000000000000000000000000000000000000000000000000000000", + hex"000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000040df144610000000000000000000000000000000000000000000000000000000098ac2e7d000000000000000000000000000000000000000000000000000000008fbe0b7c00000000000000000000000000000000000000000000000000000000e49a547d00000000000000000000000000000000000000000000000000000000", (bytes4[]) ); } diff --git a/contracts/test/integration/GatewayDiamond.t.sol b/contracts/test/integration/GatewayDiamond.t.sol index 6a96a6c3b..7f269a6ac 100644 --- a/contracts/test/integration/GatewayDiamond.t.sol +++ b/contracts/test/integration/GatewayDiamond.t.sol @@ -12,7 +12,7 @@ import {IDiamond} from "../../contracts/interfaces/IDiamond.sol"; import {IDiamondLoupe} from "../../contracts/interfaces/IDiamondLoupe.sol"; import {IDiamondCut} from "../../contracts/interfaces/IDiamondCut.sol"; import {QuorumInfo} from "../../contracts/structs/Quorum.sol"; -import {IpcEnvelope, BottomUpMsgBatch, BottomUpCheckpoint, ParentFinality} from "../../contracts/structs/CrossNet.sol"; +import {IpcEnvelope, BottomUpMsgBatch, BottomUpCheckpoint, TopdownCheckpoint} from "../../contracts/structs/CrossNet.sol"; import {FvmAddress} from "../../contracts/structs/FvmAddress.sol"; import {SubnetID, Subnet, IPCAddress, Validator, StakingChange, StakingChangeRequest, Asset, StakingOperation} from "../../contracts/structs/Subnet.sol"; import {SubnetIDHelper} from "../../contracts/lib/SubnetIDHelper.sol"; @@ -951,7 +951,7 @@ contract GatewayActorDiamondTest is Test, IntegrationTestBase, SubnetWithNativeT ); } - function testGatewayDiamond_CommitParentFinality_Fails_NotSystemActor() public { + function testGatewayDiamond_CommitTopdownCheckpoint_Fails_NotSystemActor() public { address caller = vm.addr(100); FvmAddress[] memory validators = new FvmAddress[](1); @@ -962,9 +962,13 @@ contract GatewayActorDiamondTest is Test, IntegrationTestBase, SubnetWithNativeT vm.prank(caller); vm.expectRevert(NotSystemActor.selector); - ParentFinality memory finality = ParentFinality({height: block.number, blockHash: bytes32(0)}); + TopdownCheckpoint memory finality = TopdownCheckpoint({ + height: block.number, + blockHash: bytes32(0), + effectsCommitment: new bytes(0) + }); - gatewayDiamond.topDownFinalizer().commitParentFinality(finality); + gatewayDiamond.topDownFinalizer().commitTopdownCheckpoint(finality); } function testGatewayDiamond_applyFinality_works() public { @@ -1035,7 +1039,7 @@ contract GatewayActorDiamondTest is Test, IntegrationTestBase, SubnetWithNativeT vm.stopPrank(); } - function testGatewayDiamond_CommitParentFinality_Works_WithQuery() public { + function testGatewayDiamond_CommitTopdownCheckpoint_Works_WithQuery() public { FvmAddress[] memory validators = new FvmAddress[](2); validators[0] = FvmAddressHelper.from(vm.addr(100)); validators[1] = FvmAddressHelper.from(vm.addr(101)); @@ -1048,14 +1052,21 @@ contract GatewayActorDiamondTest is Test, IntegrationTestBase, SubnetWithNativeT // not the same as init committed parent finality height vm.roll(10); - ParentFinality memory finality = ParentFinality({height: block.number, blockHash: bytes32(0)}); + TopdownCheckpoint memory finality = TopdownCheckpoint({ + height: block.number, + blockHash: bytes32(0), + effectsCommitment: new bytes(0) + }); - gatewayDiamond.topDownFinalizer().commitParentFinality(finality); - ParentFinality memory committedFinality = gatewayDiamond.getter().getParentFinality(block.number); + gatewayDiamond.topDownFinalizer().commitTopdownCheckpoint(finality); + TopdownCheckpoint memory committedFinality = gatewayDiamond.getter().getTopdownCheckpoint(block.number); require(committedFinality.height == finality.height, "heights are not equal"); require(committedFinality.blockHash == finality.blockHash, "blockHash is not equal"); - require(gatewayDiamond.getter().getLatestParentFinality().height == block.number, "finality height not equal"); + require( + gatewayDiamond.getter().getLatestTopdownCheckpoint().height == block.number, + "finality height not equal" + ); vm.stopPrank(); } diff --git a/contracts/test/integration/L2GatewayDiamond.t.sol b/contracts/test/integration/L2GatewayDiamond.t.sol index 69409fae6..e70eb9cee 100644 --- a/contracts/test/integration/L2GatewayDiamond.t.sol +++ b/contracts/test/integration/L2GatewayDiamond.t.sol @@ -30,7 +30,7 @@ contract L2GatewayActorDiamondTest is Test, L2GatewayActorDiamond { using CrossMsgHelper for IpcEnvelope; using GatewayFacetsHelper for GatewayDiamond; - function testGatewayDiamond_CommitParentFinality_BigNumberOfMessages() public { + function testGatewayDiamond_CommitTopdownCheckpoint_BigNumberOfMessages() public { uint256 n = 2000; FvmAddress[] memory validators = new FvmAddress[](1); validators[0] = FvmAddressHelper.from(vm.addr(100)); diff --git a/contracts/test/integration/MultiSubnet.t.sol b/contracts/test/integration/MultiSubnet.t.sol index 53654136d..47b71c48f 100644 --- a/contracts/test/integration/MultiSubnet.t.sol +++ b/contracts/test/integration/MultiSubnet.t.sol @@ -4,7 +4,7 @@ pragma solidity ^0.8.23; import "forge-std/Test.sol"; import "../../contracts/errors/IPCErrors.sol"; import {EMPTY_BYTES, METHOD_SEND} from "../../contracts/constants/Constants.sol"; -import {IpcEnvelope, BottomUpMsgBatch, BottomUpCheckpoint, ParentFinality, IpcMsgKind, OutcomeType} from "../../contracts/structs/CrossNet.sol"; +import {IpcEnvelope, BottomUpMsgBatch, BottomUpCheckpoint, TopdownCheckpoint, IpcMsgKind, OutcomeType} from "../../contracts/structs/CrossNet.sol"; import {FvmAddress} from "../../contracts/structs/FvmAddress.sol"; import {SubnetID, Subnet, IPCAddress, Validator} from "../../contracts/structs/Subnet.sol"; import {SubnetIDHelper} from "../../contracts/lib/SubnetIDHelper.sol"; @@ -164,8 +164,8 @@ contract MultiSubnetTest is Test, IntegrationTestBase { IpcEnvelope[] memory msgs = new IpcEnvelope[](1); msgs[0] = expected; - // TODO: commitParentFinality doesn't not affect anything in this test. - commitParentFinality(nativeSubnet.gatewayAddr); + // TODO: commitTopdownCheckpoint doesn't not affect anything in this test. + commitTopdownCheckpoint(nativeSubnet.gatewayAddr); executeTopDownMsgs(msgs, nativeSubnet.id, nativeSubnet.gateway); @@ -293,8 +293,8 @@ contract MultiSubnetTest is Test, IntegrationTestBase { IpcEnvelope[] memory msgs = new IpcEnvelope[](1); msgs[0] = expected; - // TODO: commitParentFinality doesn't not affect anything in this test. - commitParentFinality(nativeSubnet.gatewayAddr); + // TODO: commitTopdownCheckpoint doesn't not affect anything in this test. + commitTopdownCheckpoint(nativeSubnet.gatewayAddr); vm.expectRevert(); executeTopDownMsgsRevert(msgs, nativeSubnet.id, nativeSubnet.gateway); @@ -330,7 +330,7 @@ contract MultiSubnetTest is Test, IntegrationTestBase { IpcEnvelope[] memory msgs = new IpcEnvelope[](1); msgs[0] = expected; - commitParentFinality(tokenSubnet.gatewayAddr); + commitTopdownCheckpoint(tokenSubnet.gatewayAddr); executeTopDownMsgs(msgs, tokenSubnet.id, tokenSubnet.gateway); @@ -577,7 +577,7 @@ contract MultiSubnetTest is Test, IntegrationTestBase { IpcEnvelope[] memory msgs = new IpcEnvelope[](1); msgs[0] = expected; - commitParentFinality(tokenSubnet.gatewayAddr); + commitTopdownCheckpoint(tokenSubnet.gatewayAddr); vm.expectRevert(); executeTopDownMsgsRevert(msgs, tokenSubnet.id, tokenSubnet.gateway); @@ -1136,7 +1136,7 @@ contract MultiSubnetTest is Test, IntegrationTestBase { IpcEnvelope[] memory msgs = new IpcEnvelope[](1); msgs[0] = xnetCallMsg; - commitParentFinality(nativeSubnet.gatewayAddr); + commitTopdownCheckpoint(nativeSubnet.gatewayAddr); executeTopDownMsgs(msgs, nativeSubnet.id, nativeSubnet.gateway); assertEq(address(recipient).balance, amount); @@ -1284,20 +1284,24 @@ contract MultiSubnetTest is Test, IntegrationTestBase { IpcEnvelope[] memory msgs = new IpcEnvelope[](1); msgs[0] = xnetCallMsg; - commitParentFinality(tokenSubnet.gatewayAddr); + commitTopdownCheckpoint(tokenSubnet.gatewayAddr); executeTopDownMsgs(msgs, tokenSubnet.id, tokenSubnet.gateway); assertEq(address(recipient).balance, amount); } - function commitParentFinality(address gateway) internal { + function commitTopdownCheckpoint(address gateway) internal { vm.roll(10); - ParentFinality memory finality = ParentFinality({height: block.number, blockHash: bytes32(0)}); + TopdownCheckpoint memory finality = TopdownCheckpoint({ + height: block.number, + blockHash: bytes32(0), + effectsCommitment: new bytes(0) + }); TopDownFinalityFacet gwTopDownFinalityFacet = TopDownFinalityFacet(address(gateway)); vm.prank(FilAddress.SYSTEM_ACTOR); - gwTopDownFinalityFacet.commitParentFinality(finality); + gwTopDownFinalityFacet.commitTopdownCheckpoint(finality); } function executeTopDownMsgs(IpcEnvelope[] memory msgs, SubnetID memory subnet, GatewayDiamond gw) internal { diff --git a/extras/linked-token/test/MultiSubnetTest.t.sol b/extras/linked-token/test/MultiSubnetTest.t.sol index 8b421d7e3..a98dbd311 100644 --- a/extras/linked-token/test/MultiSubnetTest.t.sol +++ b/extras/linked-token/test/MultiSubnetTest.t.sol @@ -30,7 +30,7 @@ import {SubnetActorCheckpointingFacet} from "@ipc/contracts/subnet/SubnetActorCh import {CheckpointingFacet} from "@ipc/contracts/gateway/router/CheckpointingFacet.sol"; import {FvmAddressHelper} from "@ipc/contracts/lib/FvmAddressHelper.sol"; import {Consensus, CompressedActivityRollup} from "@ipc/contracts/structs/Activity.sol"; -import {IpcEnvelope, BottomUpMsgBatch, BottomUpCheckpoint, ParentFinality, IpcMsgKind, ResultMsg, CallMsg} from "@ipc/contracts/structs/CrossNet.sol"; +import {IpcEnvelope, BottomUpMsgBatch, BottomUpCheckpoint, IpcMsgKind, ResultMsg, CallMsg} from "@ipc/contracts/structs/CrossNet.sol"; import {SubnetIDHelper} from "@ipc/contracts/lib/SubnetIDHelper.sol"; import {IERC20} from "@openzeppelin/contracts/token/ERC20/IERC20.sol"; import {CrossMsgHelper} from "@ipc/contracts/lib/CrossMsgHelper.sol"; diff --git a/fendermint/app/settings/src/lib.rs b/fendermint/app/settings/src/lib.rs index 566040dcb..7c1e3a546 100644 --- a/fendermint/app/settings/src/lib.rs +++ b/fendermint/app/settings/src/lib.rs @@ -164,22 +164,13 @@ pub struct TopDownSettings { /// conservative and avoid other from rejecting the proposal because they don't see the /// height as final yet. pub chain_head_delay: BlockHeight, - /// The number of blocks on top of `chain_head_delay` to wait before proposing a height - /// as final on the parent chain, to avoid slight disagreements between validators whether - /// a block is final, or not just yet. - pub proposal_delay: BlockHeight, /// The max number of blocks one should make the topdown proposal pub max_proposal_range: BlockHeight, - /// The max number of blocks to hold in memory for parent syncer - pub max_cache_blocks: Option, + /// The max number of blocks to hold in the parent view store for topdown syncer + pub parent_view_store_max_blocks: Option, /// Parent syncing cron period, in seconds #[serde_as(as = "DurationSeconds")] pub polling_interval: Duration, - /// Top down exponential back off retry base - #[serde_as(as = "DurationSeconds")] - pub exponential_back_off: Duration, - /// The max number of retries for exponential backoff before giving up - pub exponential_retry_limit: usize, /// The parent rpc http endpoint pub parent_http_endpoint: Url, /// Timeout for calls to the parent Ethereum API. diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index 427c70641..82b187292 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -40,6 +40,8 @@ use fendermint_vm_interpreter::{ }; use fendermint_vm_message::query::FvmQueryHeight; use fendermint_vm_snapshot::{SnapshotClient, SnapshotError}; +use fendermint_vm_topdown::launch::Toggle; +use fendermint_vm_topdown::TopdownClient; use fvm::engine::MultiEngine; use fvm_ipld_blockstore::Blockstore; use fvm_shared::chainid::ChainID; @@ -344,6 +346,10 @@ where Ok(ret) } + pub fn enable_topdown(&mut self, topdown: TopdownClient) { + self.chain_env.topdown_client = Toggle::enable(topdown); + } + /// Get a read-only view from the current FVM execution state, optionally passing a new BlockContext. /// This is useful to perform query commands targeting the latest state. Mutations from transactions /// will not be persisted. diff --git a/fendermint/app/src/cmd/debug.rs b/fendermint/app/src/cmd/debug.rs index 391f767b5..b66c0a103 100644 --- a/fendermint/app/src/cmd/debug.rs +++ b/fendermint/app/src/cmd/debug.rs @@ -1,15 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use anyhow::{anyhow, Context}; use fendermint_app_options::debug::{ DebugArgs, DebugCommands, DebugExportTopDownEventsArgs, DebugIpcCommands, }; -use fendermint_vm_topdown::proxy::IPCProviderProxy; -use ipc_provider::{ - config::subnet::{EVMSubnet, SubnetConfig}, - IpcProvider, -}; use crate::cmd; @@ -32,39 +26,6 @@ cmd! { } } -async fn export_topdown_events(args: &DebugExportTopDownEventsArgs) -> anyhow::Result<()> { - // Configuration for the child subnet on the parent network, - // based on how it's done in `run.rs` and the `genesis ipc from-parent` command. - let parent_provider = IpcProvider::new_with_subnet( - None, - ipc_provider::config::Subnet { - id: args - .subnet_id - .parent() - .ok_or_else(|| anyhow!("subnet is not a child"))?, - config: SubnetConfig::Fevm(EVMSubnet { - provider_http: args.parent_endpoint.clone(), - provider_timeout: None, - auth_token: args.parent_auth_token.clone(), - registry_addr: args.parent_registry, - gateway_addr: args.parent_gateway, - }), - }, - )?; - - let parent_proxy = IPCProviderProxy::new(parent_provider, args.subnet_id.clone()) - .context("failed to create provider proxy")?; - - let events = fendermint_vm_topdown::sync::fetch_topdown_events( - &parent_proxy, - args.start_block_height, - args.end_block_height, - ) - .await - .context("failed to fetch topdown events")?; - - let json = serde_json::to_string_pretty(&events)?; - std::fs::write(&args.events_file, json)?; - - Ok(()) +async fn export_topdown_events(_args: &DebugExportTopDownEventsArgs) -> anyhow::Result<()> { + todo!("integrate new RPC endpoints") } diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index a412b39a4..6074dc03b 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -1,10 +1,13 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::cmd::key::read_secret_key; +use crate::{cmd, options::run::RunArgs, settings::Settings}; use anyhow::{anyhow, bail, Context}; -use async_stm::atomically_or_err; +use async_trait::async_trait; use fendermint_abci::ApplicationService; -use fendermint_app::ipc::{AppParentFinalityQuery, AppVote}; +use fendermint_app::ipc::AppParentFinalityQuery; +use fendermint_app::observe::register_metrics as register_consensus_metrics; use fendermint_app::{App, AppConfig, AppStore, BitswapBlockstore}; use fendermint_app_settings::AccountKind; use fendermint_crypto::SecretKey; @@ -21,27 +24,36 @@ use fendermint_vm_interpreter::{ }; use fendermint_vm_resolver::ipld::IpldResolver; use fendermint_vm_snapshot::{SnapshotManager, SnapshotParams}; +use fendermint_vm_topdown::launch::{run_topdown, Toggle}; +use fendermint_vm_topdown::observation::ObservationConfig; use fendermint_vm_topdown::observe::register_metrics as register_topdown_metrics; -use fendermint_vm_topdown::proxy::{IPCProviderProxy, IPCProviderProxyWithLatency}; -use fendermint_vm_topdown::sync::launch_polling_syncer; -use fendermint_vm_topdown::voting::{publish_vote_loop, Error as VoteError, VoteTally}; -use fendermint_vm_topdown::{CachedFinalityProvider, IPCParentFinality, Toggle}; +use fendermint_vm_topdown::proxy::{ + IPCProviderProxy, IPCProviderProxyWithLatency, ParentQueryProxy, +}; +use fendermint_vm_topdown::syncer::poll::ParentPoll; +use fendermint_vm_topdown::syncer::store::{InMemoryParentViewStore, ParentViewStore}; +use fendermint_vm_topdown::syncer::{ParentPoller, ParentSyncerConfig, TopDownSyncEvent}; +use fendermint_vm_topdown::vote::error::Error; +use fendermint_vm_topdown::vote::gossip::{GossipReceiver, GossipSender}; +use fendermint_vm_topdown::vote::payload::Vote; +use fendermint_vm_topdown::vote::VoteConfig; +use fendermint_vm_topdown::{Checkpoint, TopdownClient}; use fvm_shared::address::{current_network, Address, Network}; -use ipc_ipld_resolver::{Event as ResolverEvent, VoteRecord}; +use ipc_api::subnet_id::SubnetID; +use ipc_ipld_resolver::{Event as ResolverEvent, SubnetVoteRecord}; use ipc_observability::observe::register_metrics as register_default_metrics; use ipc_provider::config::subnet::{EVMSubnet, SubnetConfig}; use ipc_provider::IpcProvider; use libp2p::identity::secp256k1; use libp2p::identity::Keypair; use std::sync::Arc; -use tokio::sync::broadcast::error::RecvError; +use std::time::Duration; +use tendermint_rpc::Client; +use tokio::sync::broadcast; +use tokio::sync::broadcast::Receiver; use tower::ServiceBuilder; use tracing::info; -use crate::cmd::key::read_secret_key; -use crate::{cmd, options::run::RunArgs, settings::Settings}; -use fendermint_app::observe::register_metrics as register_consensus_metrics; - cmd! { RunArgs(self, settings) { run(settings).await @@ -106,15 +118,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> { } }; - let validator_keypair = validator.as_ref().map(|(sk, _)| { - let mut bz = sk.serialize(); - let sk = libp2p::identity::secp256k1::SecretKey::try_from_bytes(&mut bz) - .expect("secp256k1 secret key"); - let kp = libp2p::identity::secp256k1::Keypair::from(sk); - libp2p::identity::Keypair::from(kp) - }); - - let validator_ctx = validator.map(|(sk, addr)| { + let validator_ctx = validator.clone().map(|(sk, addr)| { // For now we are using the validator key for submitting transactions. // This allows us to identify transactions coming from empowered validators, to give priority to protocol related transactions. let broadcaster = Broadcaster::new( @@ -165,12 +169,9 @@ async fn run(settings: Settings) -> anyhow::Result<()> { NamespaceBlockstore::new(db.clone(), ns.state_store).context("error creating state DB")?; let checkpoint_pool = CheckpointPool::new(); - let parent_finality_votes = VoteTally::empty(); - - let topdown_enabled = settings.topdown_enabled(); // If enabled, start a resolver that communicates with the application through the resolve pool. - if settings.resolver_enabled() { + let ipld_gossip_client = if settings.resolver_enabled() { let mut service = make_resolver_service(&settings, db.clone(), state_store.clone(), ns.bit_store)?; @@ -196,36 +197,13 @@ async fn run(settings: Settings) -> anyhow::Result<()> { own_subnet_id.clone(), ); - if topdown_enabled { - if let Some(key) = validator_keypair { - let parent_finality_votes = parent_finality_votes.clone(); - - tracing::info!("starting the parent finality vote gossip loop..."); - tokio::spawn(async move { - publish_vote_loop( - parent_finality_votes, - settings.ipc.vote_interval, - settings.ipc.vote_timeout, - key, - own_subnet_id, - client, - |height, block_hash| { - AppVote::ParentFinality(IPCParentFinality { height, block_hash }) - }, - ) - .await - }); - } - } else { - tracing::info!("parent finality vote gossip disabled"); - } - - tracing::info!("subscribing to gossip..."); + info!("subscribing to gossip..."); let rx = service.subscribe(); - let parent_finality_votes = parent_finality_votes.clone(); - tokio::spawn(async move { - dispatch_resolver_events(rx, parent_finality_votes, topdown_enabled).await; - }); + let topdown_gossip_rx = IPLDTopdownGossipReceiver { rx }; + let topdown_gossip_tx = IPLDTopdownGossipSender { + client, + subnet: own_subnet_id, + }; tracing::info!("starting the IPLD Resolver Service..."); tokio::spawn(async move { @@ -236,40 +214,11 @@ async fn run(settings: Settings) -> anyhow::Result<()> { tracing::info!("starting the IPLD Resolver..."); tokio::spawn(async move { resolver.run().await }); - } else { - tracing::info!("IPLD Resolver disabled.") - } - - let (parent_finality_provider, ipc_tuple) = if topdown_enabled { - info!("topdown finality enabled"); - let topdown_config = settings.ipc.topdown_config()?; - let mut config = fendermint_vm_topdown::Config::new( - topdown_config.chain_head_delay, - topdown_config.polling_interval, - topdown_config.exponential_back_off, - topdown_config.exponential_retry_limit, - ) - .with_proposal_delay(topdown_config.proposal_delay) - .with_max_proposal_range(topdown_config.max_proposal_range); - - if let Some(v) = topdown_config.max_cache_blocks { - info!(value = v, "setting max cache blocks"); - config = config.with_max_cache_blocks(v); - } - - let ipc_provider = { - let p = make_ipc_provider_proxy(&settings)?; - Arc::new(IPCProviderProxyWithLatency::new(p)) - }; - let finality_provider = - CachedFinalityProvider::uninitialized(config.clone(), ipc_provider.clone()).await?; - - let p = Arc::new(Toggle::enabled(finality_provider)); - (p, Some((ipc_provider, config))) + Some((topdown_gossip_tx, topdown_gossip_rx)) } else { - info!("topdown finality disabled"); - (Arc::new(Toggle::disabled()), None) + tracing::info!("IPLD Resolver disabled."); + None }; // Start a snapshot manager in the background. @@ -298,7 +247,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> { None }; - let app: App<_, _, AppStore, _> = App::new( + let mut app: App<_, _, AppStore, _> = App::new( AppConfig { app_namespace: ns.app, state_hist_namespace: ns.state_hist, @@ -310,29 +259,65 @@ async fn run(settings: Settings) -> anyhow::Result<()> { interpreter, ChainEnv { checkpoint_pool, - parent_finality_provider: parent_finality_provider.clone(), - parent_finality_votes: parent_finality_votes.clone(), + topdown_client: Toggle::::disable(), }, snapshots, )?; - if let Some((agent_proxy, config)) = ipc_tuple { + if settings.topdown_enabled() { + info!("topdown finality enabled"); + let app_parent_finality_query = AppParentFinalityQuery::new(app.clone()); - tokio::spawn(async move { - match launch_polling_syncer( - app_parent_finality_query, - config, - parent_finality_provider, - parent_finality_votes, - agent_proxy, - tendermint_client, - ) - .await - { - Ok(_) => {} - Err(e) => tracing::error!("cannot launch polling syncer: {e}"), - } - }); + + let topdown_config = settings.ipc.topdown_config()?; + let config = fendermint_vm_topdown::Config { + syncer: ParentSyncerConfig { + request_channel_size: 1024, + broadcast_channel_size: 1024, + chain_head_delay: topdown_config.chain_head_delay, + polling_interval_millis: Duration::from_millis(100), + max_requests_per_loop: 10, + max_store_blocks: topdown_config.parent_view_store_max_blocks.unwrap_or(2000), + sync_many: true, + observation: ObservationConfig { + max_observation_range: Some(topdown_config.max_proposal_range), + }, + }, + voting: VoteConfig { + req_channel_buffer_size: 1024, + }, + }; + + let parent_proxy = Arc::new(IPCProviderProxyWithLatency::new(make_ipc_provider_proxy( + &settings, + )?)); + let parent_view_store = InMemoryParentViewStore::new(); + + let gossip = ipld_gossip_client + .ok_or_else(|| anyhow!("topdown enabled but ipld is not, enable ipld first"))?; + + let client = run_topdown( + parent_view_store.clone(), + app_parent_finality_query, + config, + validator + .clone() + .ok_or_else(|| anyhow!("need validator key to run topdown"))? + .0, + gossip, + parent_proxy, + move |checkpoint, proxy, config| { + let poller_inner = + ParentPoll::new(config, proxy, parent_view_store, checkpoint.clone()); + TendermintAwareParentPoller { + client: tendermint_client.clone(), + inner: poller_inner, + } + }, + ) + .await?; + + app.enable_topdown(client); } // Start the metrics on a background thread. @@ -406,7 +391,7 @@ fn make_resolver_service( db: RocksDb, state_store: NamespaceBlockstore, bit_store_ns: String, -) -> anyhow::Result> { +) -> anyhow::Result> { // Blockstore for Bitswap. let bit_store = NamespaceBlockstore::new(db, bit_store_ns).context("error creating bit DB")?; @@ -509,65 +494,85 @@ fn to_address(sk: &SecretKey, kind: &AccountKind) -> anyhow::Result

{ } } -async fn dispatch_resolver_events( - mut rx: tokio::sync::broadcast::Receiver>, - parent_finality_votes: VoteTally, - topdown_enabled: bool, -) { - loop { - match rx.recv().await { - Ok(event) => match event { - ResolverEvent::ReceivedPreemptive(_, _) => {} - ResolverEvent::ReceivedVote(vote) => { - dispatch_vote(*vote, &parent_finality_votes, topdown_enabled).await; +struct IPLDTopdownGossipSender { + client: ipc_ipld_resolver::Client, + subnet: SubnetID, +} + +struct IPLDTopdownGossipReceiver { + rx: broadcast::Receiver>, +} + +#[async_trait] +impl GossipSender for IPLDTopdownGossipSender { + async fn publish_vote(&self, vote: Vote) -> Result<(), Error> { + let v = SubnetVoteRecord { + subnet: self.subnet.clone(), + vote, + }; + self.client + .publish_vote(v) + .map_err(|e| Error::CannotPublishVote(e.to_string())) + } +} + +#[async_trait] +impl GossipReceiver for IPLDTopdownGossipReceiver { + async fn recv_vote(&mut self) -> Result { + match self.rx.recv().await { + Ok(v) => match v { + ResolverEvent::ReceivedVote(v) => Ok(*v), + e => { + tracing::error!("unused event received"); + Err(Error::UnexpectedGossipEvent(format!("{e:?}"))) } }, - Err(RecvError::Lagged(n)) => { - tracing::warn!("the resolver service skipped {n} gossip events") - } - Err(RecvError::Closed) => { - tracing::error!("the resolver service stopped receiving gossip"); - return; - } + Err(e) => Err(Error::CannotReceiveVote(format!("{e}"))), } } } -async fn dispatch_vote( - vote: VoteRecord, - parent_finality_votes: &VoteTally, - topdown_enabled: bool, -) { - match vote.content { - AppVote::ParentFinality(f) => { - if !topdown_enabled { - tracing::debug!("ignoring vote; topdown disabled"); - return; - } - let res = atomically_or_err(|| { - parent_finality_votes.add_vote( - vote.public_key.clone(), - f.height, - f.block_hash.clone(), - ) - }) - .await; - - match res { - Err(e @ VoteError::Equivocation(_, _, _, _)) => { - tracing::warn!(error = e.to_string(), "failed to handle vote"); - } - Err(e @ ( - VoteError::Uninitialized // early vote, we're not ready yet - | VoteError::UnpoweredValidator(_) // maybe arrived too early or too late, or spam - | VoteError::UnexpectedBlock(_, _) // won't happen here - )) => { - tracing::debug!(error = e.to_string(), "failed to handle vote"); - } - _ => { - tracing::debug!("vote handled"); - } - }; +struct TendermintAwareParentPoller { + client: tendermint_rpc::HttpClient, + inner: ParentPoll, +} + +#[async_trait] +impl ParentPoller for TendermintAwareParentPoller +where + S: ParentViewStore + Send + Sync + 'static + Clone, + P: Send + Sync + 'static + ParentQueryProxy, +{ + type Store = S; + + fn subscribe(&self) -> Receiver { + self.inner.subscribe() + } + + fn store(&self) -> Self::Store { + self.inner.store() + } + + fn finalize(&mut self, checkpoint: Checkpoint) -> anyhow::Result<()> { + self.inner.finalize(checkpoint) + } + + async fn try_poll(&mut self) -> anyhow::Result<()> { + if self.is_syncing_peer().await? { + tracing::debug!("syncing with peer, skip parent finality syncing this round"); + return Ok(()); } + self.inner.try_poll().await + } +} + +impl TendermintAwareParentPoller { + async fn is_syncing_peer(&self) -> anyhow::Result { + let status: tendermint_rpc::endpoint::status::Response = self + .client + .status() + .await + .context("failed to get Tendermint status")?; + Ok(status.sync_info.catching_up) } } diff --git a/fendermint/app/src/ipc.rs b/fendermint/app/src/ipc.rs index eb5c22130..5c7ed93c6 100644 --- a/fendermint/app/src/ipc.rs +++ b/fendermint/app/src/ipc.rs @@ -9,19 +9,11 @@ use fendermint_vm_genesis::{Power, Validator}; use fendermint_vm_interpreter::fvm::state::ipc::GatewayCaller; use fendermint_vm_interpreter::fvm::state::{FvmExecState, FvmStateParams}; use fendermint_vm_interpreter::fvm::store::ReadOnlyBlockstore; -use fendermint_vm_topdown::sync::ParentFinalityStateQuery; -use fendermint_vm_topdown::IPCParentFinality; +use fendermint_vm_topdown::launch::LaunchQuery; use fvm_ipld_blockstore::Blockstore; use std::sync::Arc; -use serde::{Deserialize, Serialize}; - -/// All the things that can be voted on in a subnet. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum AppVote { - /// The validator considers a certain block final on the parent chain. - ParentFinality(IPCParentFinality), -} +use fendermint_vm_topdown::Checkpoint; /// Queries the LATEST COMMITTED parent finality from the storage pub struct AppParentFinalityQuery @@ -62,7 +54,7 @@ where } } -impl ParentFinalityStateQuery for AppParentFinalityQuery +impl LaunchQuery for AppParentFinalityQuery where S: KVStore + Codec @@ -72,10 +64,10 @@ where DB: KVWritable + KVReadable + 'static + Clone, SS: Blockstore + 'static + Clone, { - fn get_latest_committed_finality(&self) -> anyhow::Result> { + fn get_latest_checkpoint(&self) -> anyhow::Result> { self.with_exec_state(|mut exec_state| { self.gateway_caller - .get_latest_parent_finality(&mut exec_state) + .get_latest_topdown_checkpoint(&mut exec_state) }) } @@ -86,4 +78,9 @@ where .map(|(_, pt)| pt) }) } + + fn latest_chain_block(&self) -> anyhow::Result { + self.with_exec_state(|s| Ok(s.block_height() as fendermint_vm_topdown::BlockHeight)) + .map(|v| v.unwrap_or(1)) + } } diff --git a/fendermint/crypto/Cargo.toml b/fendermint/crypto/Cargo.toml index ddf29ee9b..5ba6bc2a5 100644 --- a/fendermint/crypto/Cargo.toml +++ b/fendermint/crypto/Cargo.toml @@ -16,10 +16,11 @@ rand = { workspace = true } zeroize = { workspace = true } multihash = { workspace = true, features = ["multihash-impl", "blake2b"] } serde = { workspace = true, optional = true } +fvm_ipld_encoding = { workspace = true, optional = true } +num-traits = { workspace = true } +num-rational = { workspace = true } -[dev-dependencies] -fvm_ipld_encoding = { workspace = true } [features] -default = ["with_serde"] +default = ["with_serde", "fvm_ipld_encoding"] with_serde = ["serde"] \ No newline at end of file diff --git a/fendermint/crypto/src/lib.rs b/fendermint/crypto/src/lib.rs index 10a69b1f3..57d5cb0fc 100644 --- a/fendermint/crypto/src/lib.rs +++ b/fendermint/crypto/src/lib.rs @@ -1,6 +1,7 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +pub mod quorum; pub mod secp; use base64::engine::GeneralPurpose; diff --git a/fendermint/crypto/src/quorum.rs b/fendermint/crypto/src/quorum.rs new file mode 100644 index 000000000..65ea111f3 --- /dev/null +++ b/fendermint/crypto/src/quorum.rs @@ -0,0 +1,249 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::secp::RecoverableECDSASignature; +use anyhow::anyhow; +use libsecp256k1::PublicKey; +use num_rational::Ratio; +use num_traits::Unsigned; + +/// The payload bytes that has been certified by a majority of signer. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "with_serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ECDSACertificate { + payload: T, + /// An array of nillable signatures of all active validators in deterministic order. + signatures: Vec>, +} + +impl ECDSACertificate { + pub fn new_of_size(payload: T, size: usize) -> Self { + Self { + payload, + signatures: vec![None; size], + } + } + + pub fn payload(&self) -> &T { + &self.payload + } + + #[inline] + fn quorum_threshold(total: W, threshold_ratio: Ratio) -> W + where + W: Unsigned + Copy, + { + total * *threshold_ratio.numer() / *threshold_ratio.denom() + W::one() + } +} + +#[cfg(feature = "with_serde")] +impl ECDSACertificate { + pub fn set_signature( + &mut self, + idx: usize, + pk: &PublicKey, + sig: RecoverableECDSASignature, + ) -> anyhow::Result<()> { + if !sig.verify(&fvm_ipld_encoding::to_vec(&self.payload)?, pk)? { + return Err(anyhow!("signature not match publick key")); + } + + self.signatures[idx] = Some(sig); + + Ok(()) + } + + /// Checks if a quorum is reached from external power table given the payload and sigatures + pub fn quorum_reached<'a, W, I>( + &self, + power_table: I, + threshold_ratio: Ratio, + ) -> anyhow::Result + where + W: Copy + Unsigned + PartialOrd, + I: Iterator, + { + let (total_weight, signed_weight) = self.calculate_weights::(power_table)?; + Ok(signed_weight >= Self::quorum_threshold::(total_weight, threshold_ratio)) + } + + pub fn calculate_weights<'a, W, I>(&self, power_table: I) -> anyhow::Result<(W, W)> + where + W: Copy + Unsigned, + I: Iterator, + { + let mut total_weight = W::zero(); + let mut total_pkeys = 0usize; + + let mut signed_weight = W::zero(); + + let payload_bytes = fvm_ipld_encoding::to_vec(&self.payload)?; + + for ((pk, weight), maybe_sig) in power_table.zip(self.signatures.iter()) { + total_weight = total_weight + weight; + total_pkeys += 1; + + let Some(ref sig) = maybe_sig else { + continue; + }; + + let (rec_pk, _) = sig.recover(payload_bytes.as_slice())?; + if *pk != rec_pk { + return Err(anyhow!("signature not signed by the public key")); + } + + signed_weight = signed_weight + weight; + } + + if total_pkeys != self.signatures.len() { + return Err(anyhow!( + "invalid number of public keys, expecting: {}, received: {}", + self.signatures.len(), + total_pkeys + )); + } + + Ok((total_weight, signed_weight)) + } +} + +#[cfg(test)] +mod tests { + use crate::quorum::ECDSACertificate; + use crate::secp::RecoverableECDSASignature; + use crate::SecretKey; + use num_rational::Ratio; + use rand::{random, thread_rng}; + + fn random_secret_keys(num: usize) -> Vec { + let mut rng = thread_rng(); + (0..num).map(|_| SecretKey::random(&mut rng)).collect() + } + + #[test] + fn test_quorum_all_signed_works() { + let sks = random_secret_keys(11); + + let payload = vec![10u8; 100]; + + let mut quorum = ECDSACertificate::new_of_size(payload.clone(), sks.len()); + let ratio = Ratio::new(2, 3); + for (i, sk) in sks.iter().enumerate() { + let sig = + RecoverableECDSASignature::sign(sk, &fvm_ipld_encoding::to_vec(&payload).unwrap()) + .unwrap(); + quorum.set_signature(i, &sk.public_key(), sig).unwrap(); + } + + let weights = sks + .iter() + .map(|sk| (sk.public_key(), 1u64)) + .collect::>(); + let is_ok = quorum + .quorum_reached::<_, _>(weights.iter().map(|(pk, weight)| (pk, *weight)), ratio) + .unwrap(); + assert!(is_ok); + } + + #[test] + fn test_no_quorum_works() { + let sks = random_secret_keys(11); + + let payload = vec![10u8; 100]; + let ratio = Ratio::new(2, 3); + + let mut quorum = ECDSACertificate::new_of_size(payload.clone(), sks.len()); + for (i, sk) in sks.iter().enumerate() { + let sig = + RecoverableECDSASignature::sign(sk, &fvm_ipld_encoding::to_vec(&payload).unwrap()) + .unwrap(); + if i % 3 == 0 { + quorum.set_signature(i, &sk.public_key(), sig).unwrap(); + } + } + + let weights = sks + .iter() + .map(|sk| (sk.public_key(), 1u64)) + .collect::>(); + let is_reached = quorum + .quorum_reached::<_, _>(weights.iter().map(|(pk, weight)| (pk, *weight)), ratio) + .unwrap(); + assert!(!is_reached); + } + + #[test] + fn test_calculate_weight_all_signed_works() { + let sks = random_secret_keys(11); + + let payload = vec![10u8; 100]; + + let mut quorum = ECDSACertificate::new_of_size(payload.clone(), sks.len()); + for (i, sk) in sks.iter().enumerate() { + let sig = + RecoverableECDSASignature::sign(sk, &fvm_ipld_encoding::to_vec(&payload).unwrap()) + .unwrap(); + quorum.set_signature(i, &sk.public_key(), sig).unwrap(); + } + + let mut total_expected = 0; + let weights = sks + .iter() + .map(|sk| { + let n = random::() % 100000; + total_expected += n; + (sk.public_key(), n) + }) + .collect::>(); + let (total, signed) = quorum + .calculate_weights::<_, _>(weights.iter().map(|(pk, weight)| (pk, *weight))) + .unwrap(); + + assert_eq!(total, signed); + assert_eq!(total, total_expected); + } + + #[test] + fn test_random_works() { + let sks = random_secret_keys(11); + + let payload = vec![10u8; 100]; + + let mut quorum = ECDSACertificate::new_of_size(payload.clone(), sks.len()); + let mut should_signs = vec![]; + for (i, sk) in sks.iter().enumerate() { + let sig = + RecoverableECDSASignature::sign(sk, &fvm_ipld_encoding::to_vec(&payload).unwrap()) + .unwrap(); + + let should_sign = random::(); + if should_sign { + quorum.set_signature(i, &sk.public_key(), sig).unwrap(); + } + should_signs.push(should_sign); + } + + let mut total_expected = 0; + let weights = sks + .iter() + .map(|sk| { + let n = random::() % 100000; + total_expected += n; + (sk.public_key(), n) + }) + .collect::>(); + let (total, signed) = quorum + .calculate_weights::<_, _>(weights.iter().map(|(pk, weight)| (pk, *weight))) + .unwrap(); + + let mut signed_expected = 0; + for (i, should_sign) in should_signs.iter().enumerate() { + if *should_sign { + signed_expected += weights[i].1; + } + } + assert_eq!(total, total_expected); + assert_eq!(signed, signed_expected); + } +} diff --git a/fendermint/testing/materializer/tests/docker_tests/layer2.rs b/fendermint/testing/materializer/tests/docker_tests/layer2.rs index acfd0f8f4..526068e51 100644 --- a/fendermint/testing/materializer/tests/docker_tests/layer2.rs +++ b/fendermint/testing/materializer/tests/docker_tests/layer2.rs @@ -11,7 +11,7 @@ use fendermint_materializer::{HasEthApi, ResourceId}; use fendermint_vm_actor_interface::init::builtin_actor_eth_addr; use fendermint_vm_actor_interface::ipc; use fendermint_vm_message::conv::from_fvm::to_eth_address; -use ipc_actors_abis::gateway_getter_facet::{GatewayGetterFacet, ParentFinality}; +use ipc_actors_abis::gateway_getter_facet::{GatewayGetterFacet, TopdownCheckpoint}; use ipc_actors_abis::subnet_actor_getter_facet::SubnetActorGetterFacet; use crate::with_testnet; @@ -71,8 +71,8 @@ async fn test_topdown_and_bottomup() { { let mut retry = 0; loop { - let finality: ParentFinality = england_gateway - .get_latest_parent_finality() + let finality: TopdownCheckpoint = england_gateway + .get_latest_topdown_checkpoint() .call() .await .context("failed to get parent finality")?; diff --git a/fendermint/vm/genesis/src/lib.rs b/fendermint/vm/genesis/src/lib.rs index ee48431ee..ba644a742 100644 --- a/fendermint/vm/genesis/src/lib.rs +++ b/fendermint/vm/genesis/src/lib.rs @@ -7,6 +7,7 @@ use anyhow::anyhow; use fvm_shared::bigint::{BigInt, Integer}; use serde::{Deserialize, Serialize}; use serde_with::serde_as; +use std::cmp::Ordering; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; @@ -173,6 +174,20 @@ impl ValidatorKey { } } +impl PartialOrd for ValidatorKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ValidatorKey { + fn cmp(&self, other: &Self) -> Ordering { + self.0 + .serialize_compressed() + .cmp(&other.0.serialize_compressed()) + } +} + impl TryFrom for tendermint::PublicKey { type Error = anyhow::Error; diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index d3fe2489e..68ef9c6eb 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -10,33 +10,26 @@ use crate::{ signed::{SignedMessageApplyRes, SignedMessageCheckRes, SyntheticMessage, VerifiableMessage}, CheckInterpreter, ExecInterpreter, ProposalInterpreter, QueryInterpreter, }; -use anyhow::{anyhow, bail, Context}; +use anyhow::{anyhow, Context}; use async_stm::atomically; use async_trait::async_trait; -use fendermint_tracing::emit; use fendermint_vm_actor_interface::ipc; -use fendermint_vm_event::ParentFinalityMissingQuorum; -use fendermint_vm_message::ipc::ParentFinality; +use fendermint_vm_genesis::ValidatorKey; use fendermint_vm_message::{ chain::ChainMessage, ipc::{BottomUpCheckpoint, CertifiedMessage, IpcMessage, SignedRelayedMessage}, }; use fendermint_vm_resolver::pool::{ResolveKey, ResolvePool}; -use fendermint_vm_topdown::proxy::IPCProviderProxyWithLatency; -use fendermint_vm_topdown::voting::{ValidatorKey, VoteTally}; -use fendermint_vm_topdown::{ - CachedFinalityProvider, IPCParentFinality, ParentFinalityProvider, ParentViewProvider, Toggle, -}; +use fendermint_vm_topdown::launch::Toggle; +use fendermint_vm_topdown::{Checkpoint, TopdownClient}; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::RawBytes; -use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use num_traits::Zero; use std::sync::Arc; /// A resolution pool for bottom-up and top-down checkpoints. pub type CheckpointPool = ResolvePool; -pub type TopDownFinalityProvider = Arc>>; /// These are the extra state items that the chain interpreter needs, /// a sort of "environment" supporting IPC. @@ -44,9 +37,8 @@ pub type TopDownFinalityProvider = Arc, } #[derive(Clone, Hash, PartialEq, Eq)] @@ -126,56 +118,12 @@ where CheckpointPoolItem::BottomUp(ckpt) => ChainMessage::Ipc(IpcMessage::BottomUpExec(ckpt)), }); - // Prepare top down proposals. - // Before we try to find a quorum, pause incoming votes. This is optional but if there are lots of votes coming in it might hold up proposals. - atomically(|| { - chain_env - .parent_finality_votes - .pause_votes_until_find_quorum() - }) - .await; - - // The pre-requisite for proposal is that there is a quorum of gossiped votes at that height. - // The final proposal can be at most as high as the quorum, but can be less if we have already, - // hit some limits such as how many blocks we can propose in a single step. - let finalities = atomically(|| { - let parent = chain_env.parent_finality_provider.next_proposal()?; - let quorum = chain_env - .parent_finality_votes - .find_quorum()? - .map(|(height, block_hash)| IPCParentFinality { height, block_hash }); - - Ok((parent, quorum)) - }) - .await; - - let maybe_finality = match finalities { - (Some(parent), Some(quorum)) => Some(if parent.height <= quorum.height { - parent - } else { - quorum - }), - (Some(parent), None) => { - emit!( - DEBUG, - ParentFinalityMissingQuorum { - block_height: parent.height, - block_hash: &hex::encode(&parent.block_hash), - } - ); - None + match chain_env.topdown_client.find_topdown_proposal().await { + Ok(Some(p)) => msgs.push(ChainMessage::Ipc(IpcMessage::TopDownExec(p))), + Ok(None) => {} + Err(e) => { + tracing::error!(err = e.to_string(), "cannot find topdown proposal"); } - (None, _) => { - // This is normal, the parent probably hasn't produced a block yet. - None - } - }; - - if let Some(finality) = maybe_finality { - msgs.push(ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { - height: finality.height as ChainEpoch, - block_hash: finality.block_hash, - }))) } // Append at the end - if we run out of block space, these are going to be reproposed in the next block. @@ -212,19 +160,16 @@ where return Ok(false); } } - ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { - height, - block_hash, - })) => { - let prop = IPCParentFinality { - height: height as u64, - block_hash, - }; - let is_final = - atomically(|| chain_env.parent_finality_provider.check_proposal(&prop)) - .await; - if !is_final { - return Ok(false); + ChainMessage::Ipc(IpcMessage::TopDownExec(p)) => { + let proposal_height = p.cert.payload().parent_height(); + match chain_env.topdown_client.validate_quorum_proposal(p).await { + Ok(_) => { + tracing::info!(proposal_height, "validated quorum proposal"); + } + Err(e) => { + tracing::error!(err = e.to_string(), "cannot validate quorum proposal"); + return Ok(false); + } } } ChainMessage::Signed(signed) => { @@ -308,57 +253,32 @@ where todo!("#197: implement BottomUp checkpoint execution") } IpcMessage::TopDownExec(p) => { - if !env.parent_finality_provider.is_enabled() { - bail!("cannot execute IPC top-down message: parent provider disabled"); - } - - // commit parent finality first - let finality = IPCParentFinality::new(p.height, p.block_hash); + let checkpoint = Checkpoint::from(p.cert.payload()); tracing::debug!( - finality = finality.to_string(), + checkpoint = checkpoint.to_string(), "chain interpreter received topdown exec proposal", ); - let (prev_height, prev_finality) = topdown::commit_finality( + let prev_checkpoint = topdown::commit_checkpoint( &self.gateway_caller, &mut state, - finality.clone(), - &env.parent_finality_provider, + checkpoint.clone(), ) .await .context("failed to commit finality")?; tracing::debug!( - previous_committed_height = prev_height, - previous_committed_finality = prev_finality + previous_committed_finality = prev_checkpoint .as_ref() .map(|f| format!("{f}")) .unwrap_or_else(|| String::from("None")), "chain interpreter committed topdown finality", ); - // The height range we pull top-down effects from. This _includes_ the proposed - // finality, as we assume that the interface we query publishes only fully - // executed blocks as the head of the chain. This is certainly the case for - // Ethereum-compatible JSON-RPC APIs, like Filecoin's. It should be the case - // too for future Filecoin light clients. - // - // Another factor to take into account is the chain_head_delay, which must be - // non-zero. So even in the case where deferred execution leaks through our - // query mechanism, it should not be problematic because we're guaranteed to - // be _at least_ 1 height behind. - let (execution_fr, execution_to) = (prev_height + 1, finality.height); - // error happens if we cannot get the validator set from ipc agent after retries - let validator_changes = env - .parent_finality_provider - .validator_changes_from(execution_fr, execution_to) - .await - .context("failed to fetch validator changes")?; + let validator_changes = p.effects.1; tracing::debug!( - from = execution_fr, - to = execution_to, msgs = validator_changes.len(), "chain interpreter received total validator changes" ); @@ -367,17 +287,10 @@ where .store_validator_changes(&mut state, validator_changes) .context("failed to store validator changes")?; - // error happens if we cannot get the cross messages from ipc agent after retries - let msgs = env - .parent_finality_provider - .top_down_msgs_from(execution_fr, execution_to) - .await - .context("failed to fetch top down messages")?; + let msgs = p.effects.0; tracing::debug!( number_of_messages = msgs.len(), - start = execution_fr, - end = execution_to, "chain interpreter received topdown msgs", ); @@ -387,30 +300,13 @@ where tracing::debug!("chain interpreter applied topdown msgs"); - let local_block_height = state.block_height() as u64; - let proposer = state - .block_producer() - .map(|id| hex::encode(id.serialize_compressed())); - let proposer_ref = proposer.as_deref(); - - atomically(|| { - env.parent_finality_provider - .set_new_finality(finality.clone(), prev_finality.clone())?; - - env.parent_finality_votes.set_finalized( - finality.height, - finality.block_hash.clone(), - proposer_ref, - Some(local_block_height), - )?; - - Ok(()) - }) - .await; + env.topdown_client + .parent_finalized(checkpoint.clone()) + .await?; tracing::debug!( - finality = finality.to_string(), - "chain interpreter has set new" + checkpoint = checkpoint.to_string(), + "chain interpreter has set new topdown checkpoint" ); Ok(((env, state), ChainMessageApplyRet::Ipc(ret))) @@ -440,17 +336,13 @@ where .0 .iter() .map(|v| { - let vk = ValidatorKey::from(v.public_key.0); + let vk = ValidatorKey::new(v.public_key.0); let w = v.power.0; (vk, w) }) .collect::>(); - atomically(|| { - env.parent_finality_votes - .update_power_table(power_updates.clone()) - }) - .await; + env.topdown_client.update_power_table(power_updates).await?; } Ok(((env, state), out)) diff --git a/fendermint/vm/interpreter/src/fvm/state/ipc.rs b/fendermint/vm/interpreter/src/fvm/state/ipc.rs index 9a91429b3..968604955 100644 --- a/fendermint/vm/interpreter/src/fvm/state/ipc.rs +++ b/fendermint/vm/interpreter/src/fvm/state/ipc.rs @@ -18,7 +18,7 @@ use fendermint_vm_actor_interface::{ use fendermint_vm_genesis::{Collateral, Power, PowerScale, Validator, ValidatorKey}; use fendermint_vm_message::conv::{from_eth, from_fvm}; use fendermint_vm_message::signed::sign_secp256k1; -use fendermint_vm_topdown::IPCParentFinality; +use fendermint_vm_topdown::Checkpoint; use ipc_actors_abis::checkpointing_facet::CheckpointingFacet; use ipc_actors_abis::gateway_getter_facet::GatewayGetterFacet; @@ -226,23 +226,23 @@ impl GatewayCaller { Ok(calldata) } - /// Commit the parent finality to the gateway and returns the previously committed finality. - /// None implies there is no previously committed finality. - pub fn commit_parent_finality( + /// Commit the parent checkpoint to the gateway and returns the previously committed checkpoint. + /// None implies there is no previously committed checkpoint. + pub fn commit_topdown_checkpoint( &self, state: &mut FvmExecState, - finality: IPCParentFinality, - ) -> anyhow::Result> { - let evm_finality = top_down_finality_facet::ParentFinality::try_from(finality)?; + checkpoint: Checkpoint, + ) -> anyhow::Result> { + let evm_finality = top_down_finality_facet::TopdownCheckpoint::try_from(checkpoint)?; - let (has_committed, prev_finality) = self + let (has_committed, prev_checkpoint) = self .topdown - .call(state, |c| c.commit_parent_finality(evm_finality))?; + .call(state, |c| c.commit_topdown_checkpoint(evm_finality))?; Ok(if !has_committed { None } else { - Some(IPCParentFinality::from(prev_finality)) + Some(Checkpoint::from(prev_checkpoint)) }) } @@ -294,14 +294,14 @@ impl GatewayCaller { Ok(r.into_return()) } - pub fn get_latest_parent_finality( + pub fn get_latest_topdown_checkpoint( &self, state: &mut FvmExecState, - ) -> anyhow::Result { + ) -> anyhow::Result { let r = self .getter - .call(state, |c| c.get_latest_parent_finality())?; - Ok(IPCParentFinality::from(r)) + .call(state, |c| c.get_latest_topdown_checkpoint())?; + Ok(Checkpoint::from(r)) } /// Get the Ethereum adresses of validators who signed a checkpoint. diff --git a/fendermint/vm/interpreter/src/fvm/topdown.rs b/fendermint/vm/interpreter/src/fvm/topdown.rs index 8c9e77b3b..6b9ef8c7a 100644 --- a/fendermint/vm/interpreter/src/fvm/topdown.rs +++ b/fendermint/vm/interpreter/src/fvm/topdown.rs @@ -1,41 +1,33 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT + //! Topdown finality related util functions -use crate::chain::TopDownFinalityProvider; use crate::fvm::state::ipc::GatewayCaller; use crate::fvm::state::FvmExecState; use crate::fvm::FvmApplyRet; use anyhow::Context; -use fendermint_vm_topdown::{BlockHeight, IPCParentFinality, ParentViewProvider}; +use fendermint_vm_topdown::Checkpoint; use fvm_ipld_blockstore::Blockstore; use ipc_api::cross::IpcEnvelope; use super::state::ipc::tokens_to_mint; -/// Commit the parent finality. Returns the height that the previous parent finality is committed and -/// the committed finality itself. If there is no parent finality committed, genesis epoch is returned. -pub async fn commit_finality( +/// Commit the topdown checkpoint. Returns the height that the previous parent checkpoint is committed and +/// the committed checkpoint itself. If there is no topdown checkpoint committed, genesis epoch is returned. +pub async fn commit_checkpoint( gateway_caller: &GatewayCaller, state: &mut FvmExecState, - finality: IPCParentFinality, - provider: &TopDownFinalityProvider, -) -> anyhow::Result<(BlockHeight, Option)> + checkpoint: Checkpoint, +) -> anyhow::Result> where DB: Blockstore + Sync + Send + Clone + 'static, { - let (prev_height, prev_finality) = - if let Some(prev_finality) = gateway_caller.commit_parent_finality(state, finality)? { - (prev_finality.height, Some(prev_finality)) - } else { - (provider.genesis_epoch()?, None) - }; + let prev_checkpoint = gateway_caller.commit_topdown_checkpoint(state, checkpoint)?; - tracing::debug!( - "commit finality parsed: prev_height {prev_height}, prev_finality: {prev_finality:?}" - ); + tracing::debug!("commit checkpoint parsed, prev_checkpoint: {prev_checkpoint:?}"); - Ok((prev_height, prev_finality)) + Ok(prev_checkpoint) } /// Execute the top down messages implicitly. Before the execution, mint to the gateway of the funds diff --git a/fendermint/vm/message/Cargo.toml b/fendermint/vm/message/Cargo.toml index 77d264fb2..f8e87103e 100644 --- a/fendermint/vm/message/Cargo.toml +++ b/fendermint/vm/message/Cargo.toml @@ -27,6 +27,7 @@ fvm_ipld_encoding = { workspace = true } ipc-api = { workspace = true } fendermint_crypto = { path = "../../crypto" } +fendermint_vm_topdown = { path = "../../vm/topdown" } fendermint_vm_encoding = { path = "../encoding" } fendermint_vm_actor_interface = { path = "../actor_interface" } fendermint_testing = { path = "../../testing", optional = true } diff --git a/fendermint/vm/message/golden/chain/ipc_top_down.cbor b/fendermint/vm/message/golden/chain/ipc_top_down.cbor index 212c6396c..3e37ed490 100644 --- a/fendermint/vm/message/golden/chain/ipc_top_down.cbor +++ b/fendermint/vm/message/golden/chain/ipc_top_down.cbor @@ -1 +1 @@ -a163497063a16b546f70446f776e45786563a2666865696768741ac0c004dd6a626c6f636b5f6861736889189600186418d418d10118b418a50c \ No newline at end of file +a163497063a16b546f70446f776e45786563a26463657274a2677061796c6f6164a374706172656e745f7375626e65745f6865696768741bb71f35f7c2df1f1c72706172656e745f7375626e65745f68617368890418a918771891187c187a1872187118927763756d756c61746976655f656666656374735f636f6d6d8118bd6a7369676e61747572657381f66765666665637473828080 \ No newline at end of file diff --git a/fendermint/vm/message/golden/chain/ipc_top_down.txt b/fendermint/vm/message/golden/chain/ipc_top_down.txt index 8ebe9328a..1f895acae 100644 --- a/fendermint/vm/message/golden/chain/ipc_top_down.txt +++ b/fendermint/vm/message/golden/chain/ipc_top_down.txt @@ -1 +1 @@ -Ipc(TopDownExec(ParentFinality { height: 3233809629, block_hash: [150, 0, 100, 212, 209, 1, 180, 165, 12] })) \ No newline at end of file +Ipc(TopDownExec(TopdownProposal { cert: ECDSACertificate { payload: Observation { parent_subnet_height: 13195324771461439260, parent_subnet_hash: [4, 169, 119, 145, 124, 122, 114, 113, 146], cumulative_effects_comm: [189] }, signatures: [None] }, effects: ([], []) })) \ No newline at end of file diff --git a/fendermint/vm/message/src/ipc.rs b/fendermint/vm/message/src/ipc.rs index 2e77d84c6..2156cd510 100644 --- a/fendermint/vm/message/src/ipc.rs +++ b/fendermint/vm/message/src/ipc.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT use cid::Cid; +use fendermint_vm_topdown::TopdownProposal; use fvm_shared::{ address::Address, clock::ChainEpoch, crypto::signature::Signature, econ::TokenAmount, }; @@ -27,7 +28,7 @@ pub enum IpcMessage { /// A top-down checkpoint parent finality proposal. This proposal should contain the latest parent /// state that to be checked and voted by validators. - TopDownExec(ParentFinality), + TopDownExec(TopdownProposal), } /// A message relayed by a user on the current subnet. @@ -94,19 +95,8 @@ pub struct BottomUpCheckpoint { pub bottom_up_messages: Cid, // TODO: Use TCid } -/// A proposal of the parent view that validators will be voting on. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct ParentFinality { - /// Block height of this proposal. - pub height: ChainEpoch, - /// The block hash of the parent, expressed as bytes - pub block_hash: Vec, -} - #[cfg(feature = "arb")] mod arb { - - use crate::ipc::ParentFinality; use fendermint_testing::arb::{ArbAddress, ArbCid, ArbSubnetID, ArbTokenAmount}; use fvm_shared::crypto::signature::Signature; use quickcheck::{Arbitrary, Gen}; @@ -186,13 +176,4 @@ mod arb { } } } - - impl Arbitrary for ParentFinality { - fn arbitrary(g: &mut Gen) -> Self { - Self { - height: u32::arbitrary(g).into(), - block_hash: Vec::arbitrary(g), - } - } - } } diff --git a/fendermint/vm/topdown/Cargo.toml b/fendermint/vm/topdown/Cargo.toml index 2251c20e5..aad255983 100644 --- a/fendermint/vm/topdown/Cargo.toml +++ b/fendermint/vm/topdown/Cargo.toml @@ -31,6 +31,8 @@ tokio = { workspace = true } tracing = { workspace = true } prometheus = { workspace = true } arbitrary = { workspace = true } +quickcheck = { workspace = true } +num-rational = { workspace = true } multihash = { workspace = true } @@ -43,7 +45,6 @@ fendermint_crypto = { path = "../../crypto" } ipc-observability = { workspace = true } [dev-dependencies] - clap = { workspace = true } rand = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/fendermint/vm/topdown/src/convert.rs b/fendermint/vm/topdown/src/convert.rs index 09ec76958..e1ec6d802 100644 --- a/fendermint/vm/topdown/src/convert.rs +++ b/fendermint/vm/topdown/src/convert.rs @@ -2,43 +2,46 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! Handles the type conversion to ethers contract types -use crate::IPCParentFinality; +use crate::Checkpoint; use anyhow::anyhow; -use ethers::types::U256; +use ethers::types::{Bytes, U256}; use ipc_actors_abis::{gateway_getter_facet, top_down_finality_facet}; -impl TryFrom for top_down_finality_facet::ParentFinality { +impl TryFrom for top_down_finality_facet::TopdownCheckpoint { type Error = anyhow::Error; - fn try_from(value: IPCParentFinality) -> Result { - if value.block_hash.len() != 32 { + fn try_from(value: Checkpoint) -> Result { + if value.target_hash().len() != 32 { return Err(anyhow!("invalid block hash length, expecting 32")); } let mut block_hash = [0u8; 32]; - block_hash.copy_from_slice(&value.block_hash[0..32]); + block_hash.copy_from_slice(&value.target_hash()[0..32]); Ok(Self { - height: U256::from(value.height), + height: U256::from(value.target_height()), block_hash, + effects_commitment: Bytes::from(value.cumulative_effects_comm().clone()), }) } } -impl From for IPCParentFinality { - fn from(value: gateway_getter_facet::ParentFinality) -> Self { - IPCParentFinality { - height: value.height.as_u64(), - block_hash: value.block_hash.to_vec(), - } +impl From for Checkpoint { + fn from(value: gateway_getter_facet::TopdownCheckpoint) -> Self { + Checkpoint::v1( + value.height.as_u64(), + value.block_hash.to_vec(), + value.effects_commitment.to_vec(), + ) } } -impl From for IPCParentFinality { - fn from(value: top_down_finality_facet::ParentFinality) -> Self { - IPCParentFinality { - height: value.height.as_u64(), - block_hash: value.block_hash.to_vec(), - } +impl From for Checkpoint { + fn from(value: top_down_finality_facet::TopdownCheckpoint) -> Self { + Checkpoint::v1( + value.height.as_u64(), + value.block_hash.to_vec(), + value.effects_commitment.to_vec(), + ) } } diff --git a/fendermint/vm/topdown/src/error.rs b/fendermint/vm/topdown/src/error.rs deleted file mode 100644 index eef6090d7..000000000 --- a/fendermint/vm/topdown/src/error.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::{BlockHeight, SequentialAppendError}; -use thiserror::Error; - -/// The errors for top down checkpointing -#[derive(Error, Debug, Eq, PartialEq, Clone)] -pub enum Error { - #[error("Incoming items are not order sequentially")] - NotSequential, - #[error("The parent view update with block height is not sequential: {0:?}")] - NonSequentialParentViewInsert(SequentialAppendError), - #[error("Parent chain reorg detected")] - ParentChainReorgDetected, - #[error("Cannot query parent at height {1}: {0}")] - CannotQueryParent(String, BlockHeight), -} diff --git a/fendermint/vm/topdown/src/finality/fetch.rs b/fendermint/vm/topdown/src/finality/fetch.rs deleted file mode 100644 index fb4203045..000000000 --- a/fendermint/vm/topdown/src/finality/fetch.rs +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::finality::null::FinalityWithNull; -use crate::finality::ParentViewPayload; -use crate::proxy::ParentQueryProxy; -use crate::{ - handle_null_round, BlockHash, BlockHeight, Config, Error, IPCParentFinality, - ParentFinalityProvider, ParentViewProvider, -}; -use async_stm::{Stm, StmResult}; -use ipc_api::cross::IpcEnvelope; -use ipc_api::staking::StakingChangeRequest; -use std::sync::Arc; - -/// The finality provider that performs io to the parent if not found in cache -#[derive(Clone)] -pub struct CachedFinalityProvider { - inner: FinalityWithNull, - config: Config, - /// The ipc client proxy that works as a back up if cache miss - parent_client: Arc, -} - -/// Exponential backoff for futures -macro_rules! retry { - ($wait:expr, $retires:expr, $f:expr) => {{ - let mut retries = $retires; - let mut wait = $wait; - - loop { - let res = $f; - if let Err(e) = &res { - // there is no point in retrying if the current block is null round - if crate::is_null_round_str(&e.to_string()) { - tracing::warn!( - "cannot query ipc parent_client due to null round, skip retry" - ); - break res; - } - - tracing::warn!( - error = e.to_string(), - retries, - wait = ?wait, - "cannot query ipc parent_client" - ); - - if retries > 0 { - retries -= 1; - - tokio::time::sleep(wait).await; - - wait *= 2; - continue; - } - } - - break res; - } - }}; -} - -#[async_trait::async_trait] -impl ParentViewProvider for CachedFinalityProvider { - fn genesis_epoch(&self) -> anyhow::Result { - self.inner.genesis_epoch() - } - - async fn validator_changes_from( - &self, - from: BlockHeight, - to: BlockHeight, - ) -> anyhow::Result> { - let mut v = vec![]; - for h in from..=to { - let mut r = self.validator_changes(h).await?; - tracing::debug!( - number_of_messages = r.len(), - height = h, - "fetched validator change set", - ); - v.append(&mut r); - } - - Ok(v) - } - - /// Get top down message in the range `from` to `to`, both inclusive. For the check to be valid, one - /// should not pass a height `to` that is a null block, otherwise the check is useless. In debug - /// mode, it will throw an error. - async fn top_down_msgs_from( - &self, - from: BlockHeight, - to: BlockHeight, - ) -> anyhow::Result> { - let mut v = vec![]; - for h in from..=to { - let mut r = self.top_down_msgs(h).await?; - tracing::debug!( - number_of_top_down_messages = r.len(), - height = h, - "obtained topdown messages", - ); - v.append(&mut r); - } - Ok(v) - } -} - -impl ParentFinalityProvider - for CachedFinalityProvider -{ - fn next_proposal(&self) -> Stm> { - self.inner.next_proposal() - } - - fn check_proposal(&self, proposal: &IPCParentFinality) -> Stm { - self.inner.check_proposal(proposal) - } - - fn set_new_finality( - &self, - finality: IPCParentFinality, - previous_finality: Option, - ) -> Stm<()> { - self.inner.set_new_finality(finality, previous_finality) - } -} - -impl CachedFinalityProvider { - /// Creates an uninitialized provider - /// We need this because `fendermint` has yet to be initialized and might - /// not be able to provide an existing finality from the storage. This provider requires an - /// existing committed finality. Providing the finality will enable other functionalities. - pub async fn uninitialized(config: Config, parent_client: Arc) -> anyhow::Result { - let genesis = parent_client.get_genesis_epoch().await?; - Ok(Self::new(config, genesis, None, parent_client)) - } - - /// Should always return the top down messages, only when ipc parent_client is down after exponential - /// retries - async fn validator_changes( - &self, - height: BlockHeight, - ) -> anyhow::Result> { - let r = self.inner.validator_changes(height).await?; - - if let Some(v) = r { - return Ok(v); - } - - let r = retry!( - self.config.exponential_back_off, - self.config.exponential_retry_limit, - self.parent_client - .get_validator_changes(height) - .await - .map(|r| r.value) - ); - - handle_null_round(r, Vec::new) - } - - /// Should always return the top down messages, only when ipc parent_client is down after exponential - /// retries - async fn top_down_msgs(&self, height: BlockHeight) -> anyhow::Result> { - let r = self.inner.top_down_msgs(height).await?; - - if let Some(v) = r { - return Ok(v); - } - - let r = retry!( - self.config.exponential_back_off, - self.config.exponential_retry_limit, - self.parent_client - .get_top_down_msgs(height) - .await - .map(|r| r.value) - ); - - handle_null_round(r, Vec::new) - } -} - -impl CachedFinalityProvider { - pub(crate) fn new( - config: Config, - genesis_epoch: BlockHeight, - committed_finality: Option, - parent_client: Arc, - ) -> Self { - let inner = FinalityWithNull::new(config.clone(), genesis_epoch, committed_finality); - Self { - inner, - config, - parent_client, - } - } - - pub fn block_hash(&self, height: BlockHeight) -> Stm> { - self.inner.block_hash_at_height(height) - } - - pub fn latest_height_in_cache(&self) -> Stm> { - self.inner.latest_height_in_cache() - } - - /// Get the latest height tracked in the provider, includes both cache and last committed finality - pub fn latest_height(&self) -> Stm> { - self.inner.latest_height() - } - - pub fn last_committed_finality(&self) -> Stm> { - self.inner.last_committed_finality() - } - - /// Clear the cache and set the committed finality to the provided value - pub fn reset(&self, finality: IPCParentFinality) -> Stm<()> { - self.inner.reset(finality) - } - - pub fn new_parent_view( - &self, - height: BlockHeight, - maybe_payload: Option, - ) -> StmResult<(), Error> { - self.inner.new_parent_view(height, maybe_payload) - } - - /// Returns the number of blocks cached. - pub fn cached_blocks(&self) -> Stm { - self.inner.cached_blocks() - } - - pub fn first_non_null_block(&self, height: BlockHeight) -> Stm> { - self.inner.first_non_null_block(height) - } -} - -#[cfg(test)] -mod tests { - use crate::finality::ParentViewPayload; - use crate::proxy::ParentQueryProxy; - use crate::{ - BlockHeight, CachedFinalityProvider, Config, IPCParentFinality, ParentViewProvider, - SequentialKeyCache, NULL_ROUND_ERR_MSG, - }; - use anyhow::anyhow; - use async_trait::async_trait; - use fvm_shared::address::Address; - use fvm_shared::econ::TokenAmount; - use ipc_api::cross::IpcEnvelope; - use ipc_api::staking::{StakingChange, StakingChangeRequest, StakingOperation}; - use ipc_api::subnet_id::SubnetID; - use ipc_provider::manager::{GetBlockHashResult, TopDownQueryPayload}; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - use std::time::Duration; - - /// Creates a mock of a new parent blockchain view. The key is the height and the value is the - /// block hash. If block hash is None, it means the current height is a null block. - macro_rules! new_parent_blocks { - ($($key:expr => $val:expr),* ,) => ( - hash_map!($($key => $val),*) - ); - ($($key:expr => $val:expr),*) => ({ - let mut map = SequentialKeyCache::sequential(); - $( map.append($key, $val).unwrap(); )* - map - }); - } - - struct TestParentProxy { - blocks: SequentialKeyCache>, - } - - #[async_trait] - impl ParentQueryProxy for TestParentProxy { - async fn get_chain_head_height(&self) -> anyhow::Result { - Ok(self.blocks.upper_bound().unwrap()) - } - - async fn get_genesis_epoch(&self) -> anyhow::Result { - Ok(self.blocks.lower_bound().unwrap() - 1) - } - - async fn get_block_hash(&self, height: BlockHeight) -> anyhow::Result { - let r = self.blocks.get_value(height).unwrap(); - if r.is_none() { - return Err(anyhow!(NULL_ROUND_ERR_MSG)); - } - - for h in (self.blocks.lower_bound().unwrap()..height).rev() { - let v = self.blocks.get_value(h).unwrap(); - if v.is_none() { - continue; - } - return Ok(GetBlockHashResult { - parent_block_hash: v.clone().unwrap().0, - block_hash: r.clone().unwrap().0, - }); - } - panic!("invalid testing data") - } - - async fn get_top_down_msgs( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - let r = self.blocks.get_value(height).cloned().unwrap(); - if r.is_none() { - return Err(anyhow!(NULL_ROUND_ERR_MSG)); - } - let r = r.unwrap(); - Ok(TopDownQueryPayload { - value: r.2, - block_hash: r.0, - }) - } - - async fn get_validator_changes( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - let r = self.blocks.get_value(height).cloned().unwrap(); - if r.is_none() { - return Err(anyhow!(NULL_ROUND_ERR_MSG)); - } - let r = r.unwrap(); - Ok(TopDownQueryPayload { - value: r.1, - block_hash: r.0, - }) - } - } - - fn new_provider( - blocks: SequentialKeyCache>, - ) -> CachedFinalityProvider { - let config = Config { - chain_head_delay: 2, - polling_interval: Default::default(), - exponential_back_off: Default::default(), - exponential_retry_limit: 0, - max_proposal_range: Some(1), - max_cache_blocks: None, - proposal_delay: None, - }; - let genesis_epoch = blocks.lower_bound().unwrap(); - let proxy = Arc::new(TestParentProxy { blocks }); - let committed_finality = IPCParentFinality { - height: genesis_epoch, - block_hash: vec![0; 32], - }; - - CachedFinalityProvider::new(config, genesis_epoch, Some(committed_finality), proxy) - } - - fn new_cross_msg(nonce: u64) -> IpcEnvelope { - let subnet_id = SubnetID::new(10, vec![Address::new_id(1000)]); - let mut msg = IpcEnvelope::new_fund_msg( - &subnet_id, - &Address::new_id(1), - &Address::new_id(2), - TokenAmount::from_atto(100), - ) - .unwrap(); - msg.nonce = nonce; - msg - } - - fn new_validator_changes(configuration_number: u64) -> StakingChangeRequest { - StakingChangeRequest { - configuration_number, - change: StakingChange { - op: StakingOperation::Deposit, - payload: vec![], - validator: Address::new_id(1), - }, - } - } - - #[tokio::test] - async fn test_retry() { - struct Test { - nums_run: AtomicUsize, - } - - impl Test { - async fn run(&self) -> Result<(), &'static str> { - self.nums_run.fetch_add(1, Ordering::SeqCst); - Err("mocked error") - } - } - - let t = Test { - nums_run: AtomicUsize::new(0), - }; - - let res = retry!(Duration::from_secs(1), 2, t.run().await); - assert!(res.is_err()); - // execute the first time, retries twice - assert_eq!(t.nums_run.load(Ordering::SeqCst), 3); - } - - #[tokio::test] - async fn test_query_topdown_msgs() { - let parent_blocks = new_parent_blocks!( - 100 => Some((vec![0; 32], vec![], vec![new_cross_msg(0)])), // genesis block - 101 => Some((vec![1; 32], vec![], vec![new_cross_msg(1)])), - 102 => Some((vec![2; 32], vec![], vec![new_cross_msg(2)])), - 103 => Some((vec![3; 32], vec![], vec![new_cross_msg(3)])), - 104 => None, - 105 => None, - 106 => Some((vec![6; 32], vec![], vec![new_cross_msg(6)])) - ); - let provider = new_provider(parent_blocks); - let messages = provider.top_down_msgs_from(100, 106).await.unwrap(); - - assert_eq!( - messages, - vec![ - new_cross_msg(0), - new_cross_msg(1), - new_cross_msg(2), - new_cross_msg(3), - new_cross_msg(6), - ] - ) - } - - #[tokio::test] - async fn test_query_validator_changes() { - let parent_blocks = new_parent_blocks!( - 100 => Some((vec![0; 32], vec![new_validator_changes(0)], vec![])), // genesis block - 101 => Some((vec![1; 32], vec![new_validator_changes(1)], vec![])), - 102 => Some((vec![2; 32], vec![], vec![])), - 103 => Some((vec![3; 32], vec![new_validator_changes(3)], vec![])), - 104 => None, - 105 => None, - 106 => Some((vec![6; 32], vec![new_validator_changes(6)], vec![])) - ); - let provider = new_provider(parent_blocks); - let messages = provider.validator_changes_from(100, 106).await.unwrap(); - - assert_eq!(messages.len(), 4) - } -} diff --git a/fendermint/vm/topdown/src/finality/mod.rs b/fendermint/vm/topdown/src/finality/mod.rs deleted file mode 100644 index c6cd2dc3d..000000000 --- a/fendermint/vm/topdown/src/finality/mod.rs +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -mod fetch; -mod null; - -use crate::error::Error; -use crate::BlockHash; -use async_stm::{abort, StmResult}; -use ipc_api::cross::IpcEnvelope; -use ipc_api::staking::StakingChangeRequest; - -pub use fetch::CachedFinalityProvider; - -pub(crate) type ParentViewPayload = (BlockHash, Vec, Vec); - -fn ensure_sequential u64>(msgs: &[T], f: F) -> StmResult<(), Error> { - if msgs.is_empty() { - return Ok(()); - } - - let first = msgs.first().unwrap(); - let mut nonce = f(first); - for msg in msgs.iter().skip(1) { - if nonce + 1 != f(msg) { - return abort(Error::NotSequential); - } - nonce += 1; - } - - Ok(()) -} - -pub(crate) fn validator_changes(p: &ParentViewPayload) -> Vec { - p.1.clone() -} - -pub(crate) fn topdown_cross_msgs(p: &ParentViewPayload) -> Vec { - p.2.clone() -} - -#[cfg(test)] -mod tests { - use crate::proxy::ParentQueryProxy; - use crate::{ - BlockHeight, CachedFinalityProvider, Config, IPCParentFinality, ParentFinalityProvider, - }; - use async_stm::atomically_or_err; - use async_trait::async_trait; - use ipc_api::cross::IpcEnvelope; - use ipc_api::staking::StakingChangeRequest; - use ipc_provider::manager::{GetBlockHashResult, TopDownQueryPayload}; - use std::sync::Arc; - use tokio::time::Duration; - - struct MockedParentQuery; - - #[async_trait] - impl ParentQueryProxy for MockedParentQuery { - async fn get_chain_head_height(&self) -> anyhow::Result { - Ok(1) - } - - async fn get_genesis_epoch(&self) -> anyhow::Result { - Ok(10) - } - - async fn get_block_hash(&self, _height: BlockHeight) -> anyhow::Result { - Ok(GetBlockHashResult::default()) - } - - async fn get_top_down_msgs( - &self, - _height: BlockHeight, - ) -> anyhow::Result>> { - Ok(TopDownQueryPayload { - value: vec![], - block_hash: vec![], - }) - } - - async fn get_validator_changes( - &self, - _height: BlockHeight, - ) -> anyhow::Result>> { - Ok(TopDownQueryPayload { - value: vec![], - block_hash: vec![], - }) - } - } - - fn mocked_agent_proxy() -> Arc { - Arc::new(MockedParentQuery) - } - - fn genesis_finality() -> IPCParentFinality { - IPCParentFinality { - height: 0, - block_hash: vec![0; 32], - } - } - - fn new_provider() -> CachedFinalityProvider { - let config = Config { - chain_head_delay: 20, - polling_interval: Duration::from_secs(10), - exponential_back_off: Duration::from_secs(10), - exponential_retry_limit: 10, - max_proposal_range: None, - max_cache_blocks: None, - proposal_delay: None, - }; - - CachedFinalityProvider::new(config, 10, Some(genesis_finality()), mocked_agent_proxy()) - } - - #[tokio::test] - async fn test_finality_works() { - let provider = new_provider(); - - atomically_or_err(|| { - // inject data - for i in 10..=100 { - provider.new_parent_view(i, Some((vec![1u8; 32], vec![], vec![])))?; - } - - let target_block = 120; - let finality = IPCParentFinality { - height: target_block, - block_hash: vec![1u8; 32], - }; - provider.set_new_finality(finality.clone(), Some(genesis_finality()))?; - - // all cache should be cleared - let r = provider.next_proposal()?; - assert!(r.is_none()); - - let f = provider.last_committed_finality()?; - assert_eq!(f, Some(finality)); - - Ok(()) - }) - .await - .unwrap(); - } - - #[tokio::test] - async fn test_check_proposal_works() { - let provider = new_provider(); - - atomically_or_err(|| { - let target_block = 100; - - // inject data - provider.new_parent_view(target_block, Some((vec![1u8; 32], vec![], vec![])))?; - provider.set_new_finality( - IPCParentFinality { - height: target_block - 1, - block_hash: vec![1u8; 32], - }, - Some(genesis_finality()), - )?; - - let finality = IPCParentFinality { - height: target_block, - block_hash: vec![1u8; 32], - }; - - assert!(provider.check_proposal(&finality).is_ok()); - - Ok(()) - }) - .await - .unwrap(); - } -} diff --git a/fendermint/vm/topdown/src/finality/null.rs b/fendermint/vm/topdown/src/finality/null.rs deleted file mode 100644 index 9a4a7beea..000000000 --- a/fendermint/vm/topdown/src/finality/null.rs +++ /dev/null @@ -1,566 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::finality::{ - ensure_sequential, topdown_cross_msgs, validator_changes, ParentViewPayload, -}; -use crate::{BlockHash, BlockHeight, Config, Error, IPCParentFinality, SequentialKeyCache}; -use async_stm::{abort, atomically, Stm, StmResult, TVar}; -use ipc_api::cross::IpcEnvelope; -use ipc_api::staking::StakingChangeRequest; -use std::cmp::min; - -use fendermint_tracing::emit; -use fendermint_vm_event::ParentFinalityCommitted; - -/// Finality provider that can handle null blocks -#[derive(Clone)] -pub struct FinalityWithNull { - config: Config, - genesis_epoch: BlockHeight, - /// Cached data that always syncs with the latest parent chain proactively - cached_data: TVar>>, - /// This is a in memory view of the committed parent finality. We need this as a starting point - /// for populating the cache - last_committed_finality: TVar>, -} - -impl FinalityWithNull { - pub fn new( - config: Config, - genesis_epoch: BlockHeight, - committed_finality: Option, - ) -> Self { - Self { - config, - genesis_epoch, - cached_data: TVar::new(SequentialKeyCache::sequential()), - last_committed_finality: TVar::new(committed_finality), - } - } - - pub fn genesis_epoch(&self) -> anyhow::Result { - Ok(self.genesis_epoch) - } - - pub async fn validator_changes( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - let r = atomically(|| self.handle_null_block(height, validator_changes, Vec::new)).await; - Ok(r) - } - - pub async fn top_down_msgs( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - let r = atomically(|| self.handle_null_block(height, topdown_cross_msgs, Vec::new)).await; - Ok(r) - } - - pub fn last_committed_finality(&self) -> Stm> { - self.last_committed_finality.read_clone() - } - - /// Clear the cache and set the committed finality to the provided value - pub fn reset(&self, finality: IPCParentFinality) -> Stm<()> { - self.cached_data.write(SequentialKeyCache::sequential())?; - self.last_committed_finality.write(Some(finality)) - } - - pub fn new_parent_view( - &self, - height: BlockHeight, - maybe_payload: Option, - ) -> StmResult<(), Error> { - if let Some((block_hash, validator_changes, top_down_msgs)) = maybe_payload { - self.parent_block_filled(height, block_hash, validator_changes, top_down_msgs) - } else { - self.parent_null_round(height) - } - } - - pub fn next_proposal(&self) -> Stm> { - let height = if let Some(h) = self.propose_next_height()? { - h - } else { - return Ok(None); - }; - - // safe to unwrap as we make sure null height will not be proposed - let block_hash = self.block_hash_at_height(height)?.unwrap(); - - let proposal = IPCParentFinality { height, block_hash }; - tracing::debug!(proposal = proposal.to_string(), "new proposal"); - Ok(Some(proposal)) - } - - pub fn check_proposal(&self, proposal: &IPCParentFinality) -> Stm { - if !self.check_height(proposal)? { - return Ok(false); - } - self.check_block_hash(proposal) - } - - pub fn set_new_finality( - &self, - finality: IPCParentFinality, - previous_finality: Option, - ) -> Stm<()> { - debug_assert!(previous_finality == self.last_committed_finality.read_clone()?); - - // the height to clear - let height = finality.height; - - self.cached_data.update(|mut cache| { - // only remove cache below height, but not at height, as we have delayed execution - cache.remove_key_below(height); - cache - })?; - - let hash = hex::encode(&finality.block_hash); - - self.last_committed_finality.write(Some(finality))?; - - // emit event only after successful write - emit!(ParentFinalityCommitted { - block_height: height, - block_hash: &hash - }); - - Ok(()) - } -} - -impl FinalityWithNull { - /// Returns the number of blocks cached. - pub(crate) fn cached_blocks(&self) -> Stm { - let cache = self.cached_data.read()?; - Ok(cache.size() as BlockHeight) - } - - pub(crate) fn block_hash_at_height(&self, height: BlockHeight) -> Stm> { - if let Some(f) = self.last_committed_finality.read()?.as_ref() { - if f.height == height { - return Ok(Some(f.block_hash.clone())); - } - } - - self.get_at_height(height, |i| i.0.clone()) - } - - pub(crate) fn latest_height_in_cache(&self) -> Stm> { - let cache = self.cached_data.read()?; - Ok(cache.upper_bound()) - } - - /// Get the latest height tracked in the provider, includes both cache and last committed finality - pub(crate) fn latest_height(&self) -> Stm> { - let h = if let Some(h) = self.latest_height_in_cache()? { - h - } else if let Some(p) = self.last_committed_finality()? { - p.height - } else { - return Ok(None); - }; - Ok(Some(h)) - } - - /// Get the first non-null block in the range of earliest cache block till the height specified, inclusive. - pub(crate) fn first_non_null_block(&self, height: BlockHeight) -> Stm> { - let cache = self.cached_data.read()?; - Ok(cache.lower_bound().and_then(|lower_bound| { - for h in (lower_bound..=height).rev() { - if let Some(Some(_)) = cache.get_value(h) { - return Some(h); - } - } - None - })) - } -} - -/// All the private functions -impl FinalityWithNull { - fn propose_next_height(&self) -> Stm> { - let latest_height = if let Some(h) = self.latest_height_in_cache()? { - h - } else { - tracing::debug!("no proposal yet as height not available"); - return Ok(None); - }; - - let last_committed_height = if let Some(h) = self.last_committed_finality.read_clone()? { - h.height - } else { - unreachable!("last committed finality will be available at this point"); - }; - - let max_proposal_height = last_committed_height + self.config.max_proposal_range(); - let candidate_height = min(max_proposal_height, latest_height); - tracing::debug!(max_proposal_height, candidate_height, "propose heights"); - - let first_non_null_height = if let Some(h) = self.first_non_null_block(candidate_height)? { - h - } else { - tracing::debug!(height = candidate_height, "no non-null block found before"); - return Ok(None); - }; - - tracing::debug!(first_non_null_height, candidate_height); - // an extra layer of delay - let maybe_proposal_height = - self.first_non_null_block(first_non_null_height - self.config.proposal_delay())?; - tracing::debug!( - delayed_height = maybe_proposal_height, - delay = self.config.proposal_delay() - ); - if let Some(proposal_height) = maybe_proposal_height { - // this is possible due to delayed execution as the proposed height's data cannot be - // executed because they have yet to be executed. - return if last_committed_height == proposal_height { - tracing::debug!( - last_committed_height, - proposal_height, - "no new blocks from cache, not proposing" - ); - Ok(None) - } else { - tracing::debug!(proposal_height, "new proposal height"); - Ok(Some(proposal_height)) - }; - } - - tracing::debug!(last_committed_height, "no non-null block after delay"); - Ok(None) - } - - fn handle_null_block T, D: Fn() -> T>( - &self, - height: BlockHeight, - f: F, - d: D, - ) -> Stm> { - let cache = self.cached_data.read()?; - Ok(cache.get_value(height).map(|v| { - if let Some(i) = v.as_ref() { - f(i) - } else { - tracing::debug!(height, "a null round detected, return default"); - d() - } - })) - } - - fn get_at_height T>( - &self, - height: BlockHeight, - f: F, - ) -> Stm> { - let cache = self.cached_data.read()?; - Ok(if let Some(Some(v)) = cache.get_value(height) { - Some(f(v)) - } else { - None - }) - } - - fn parent_block_filled( - &self, - height: BlockHeight, - block_hash: BlockHash, - validator_changes: Vec, - top_down_msgs: Vec, - ) -> StmResult<(), Error> { - if !top_down_msgs.is_empty() { - // make sure incoming top down messages are ordered by nonce sequentially - tracing::debug!(?top_down_msgs); - ensure_sequential(&top_down_msgs, |msg| msg.nonce)?; - }; - if !validator_changes.is_empty() { - tracing::debug!(?validator_changes, "validator changes"); - ensure_sequential(&validator_changes, |change| change.configuration_number)?; - } - - let r = self.cached_data.modify(|mut cache| { - let r = cache - .append(height, Some((block_hash, validator_changes, top_down_msgs))) - .map_err(Error::NonSequentialParentViewInsert); - (cache, r) - })?; - - if let Err(e) = r { - return abort(e); - } - - Ok(()) - } - - /// When there is a new parent view, but it is actually a null round, call this function. - fn parent_null_round(&self, height: BlockHeight) -> StmResult<(), Error> { - let r = self.cached_data.modify(|mut cache| { - let r = cache - .append(height, None) - .map_err(Error::NonSequentialParentViewInsert); - (cache, r) - })?; - - if let Err(e) = r { - return abort(e); - } - - Ok(()) - } - - fn check_height(&self, proposal: &IPCParentFinality) -> Stm { - let binding = self.last_committed_finality.read()?; - // last committed finality is not ready yet, we don't vote, just reject - let last_committed_finality = if let Some(f) = binding.as_ref() { - f - } else { - return Ok(false); - }; - - // the incoming proposal has height already committed, reject - if last_committed_finality.height >= proposal.height { - tracing::debug!( - last_committed = last_committed_finality.height, - proposed = proposal.height, - "proposed height already committed", - ); - return Ok(false); - } - - if let Some(latest_height) = self.latest_height_in_cache()? { - let r = latest_height >= proposal.height; - tracing::debug!( - is_true = r, - latest_height, - proposal = proposal.height.to_string(), - "incoming proposal height seen?" - ); - // requires the incoming height cannot be more advanced than our trusted parent node - Ok(r) - } else { - // latest height is not found, meaning we dont have any prefetched cache, we just be - // strict and vote no simply because we don't know. - tracing::debug!( - proposal = proposal.height.to_string(), - "reject proposal, no data in cache" - ); - Ok(false) - } - } - - fn check_block_hash(&self, proposal: &IPCParentFinality) -> Stm { - Ok( - if let Some(block_hash) = self.block_hash_at_height(proposal.height)? { - let r = block_hash == proposal.block_hash; - tracing::debug!(proposal = proposal.to_string(), is_same = r, "same hash?"); - r - } else { - tracing::debug!(proposal = proposal.to_string(), "reject, hash not found"); - false - }, - ) - } -} - -#[cfg(test)] -mod tests { - use super::FinalityWithNull; - use crate::finality::ParentViewPayload; - use crate::{BlockHeight, Config, IPCParentFinality}; - use async_stm::{atomically, atomically_or_err}; - - async fn new_provider( - mut blocks: Vec<(BlockHeight, Option)>, - ) -> FinalityWithNull { - let config = Config { - chain_head_delay: 2, - polling_interval: Default::default(), - exponential_back_off: Default::default(), - exponential_retry_limit: 0, - max_proposal_range: Some(6), - max_cache_blocks: None, - proposal_delay: Some(2), - }; - let committed_finality = IPCParentFinality { - height: blocks[0].0, - block_hash: vec![0; 32], - }; - - blocks.remove(0); - - let f = FinalityWithNull::new(config, 1, Some(committed_finality)); - for (h, p) in blocks { - atomically_or_err(|| f.new_parent_view(h, p.clone())) - .await - .unwrap(); - } - f - } - - #[tokio::test] - async fn test_happy_path() { - // max_proposal_range is 6. proposal_delay is 2 - let parent_blocks = vec![ - (100, Some((vec![0; 32], vec![], vec![]))), // last committed block - (101, Some((vec![1; 32], vec![], vec![]))), // cache start - (102, Some((vec![2; 32], vec![], vec![]))), - (103, Some((vec![3; 32], vec![], vec![]))), - (104, Some((vec![4; 32], vec![], vec![]))), // final delayed height + proposal height - (105, Some((vec![5; 32], vec![], vec![]))), - (106, Some((vec![6; 32], vec![], vec![]))), // max proposal height (last committed + 6), first non null block - (107, Some((vec![7; 32], vec![], vec![]))), // cache latest height - ]; - let provider = new_provider(parent_blocks).await; - - let f = IPCParentFinality { - height: 104, - block_hash: vec![4; 32], - }; - assert_eq!( - atomically(|| provider.next_proposal()).await, - Some(f.clone()) - ); - - // Test set new finality - atomically(|| { - let last = provider.last_committed_finality.read_clone()?; - provider.set_new_finality(f.clone(), last) - }) - .await; - - assert_eq!( - atomically(|| provider.last_committed_finality()).await, - Some(f.clone()) - ); - - // this ensures sequential insertion is still valid - atomically_or_err(|| provider.new_parent_view(108, None)) - .await - .unwrap(); - } - - #[tokio::test] - async fn test_not_enough_view() { - // max_proposal_range is 6. proposal_delay is 2 - let parent_blocks = vec![ - (100, Some((vec![0; 32], vec![], vec![]))), // last committed block - (101, Some((vec![1; 32], vec![], vec![]))), - (102, Some((vec![2; 32], vec![], vec![]))), - (103, Some((vec![3; 32], vec![], vec![]))), // delayed height + final height - (104, Some((vec![4; 32], vec![], vec![]))), - (105, Some((vec![4; 32], vec![], vec![]))), // cache latest height, first non null block - // max proposal height is 106 - ]; - let provider = new_provider(parent_blocks).await; - - assert_eq!( - atomically(|| provider.next_proposal()).await, - Some(IPCParentFinality { - height: 103, - block_hash: vec![3; 32] - }) - ); - } - - #[tokio::test] - async fn test_with_all_null_blocks() { - // max_proposal_range is 10. proposal_delay is 2 - let parent_blocks = vec![ - (102, Some((vec![2; 32], vec![], vec![]))), // last committed block - (103, None), - (104, None), - (105, None), - (106, None), - (107, None), - (108, None), - (109, None), - (110, Some((vec![4; 32], vec![], vec![]))), // cache latest height - // max proposal height is 112 - ]; - let mut provider = new_provider(parent_blocks).await; - provider.config.max_proposal_range = Some(8); - - assert_eq!(atomically(|| provider.next_proposal()).await, None); - } - - #[tokio::test] - async fn test_with_partially_null_blocks_i() { - // max_proposal_range is 10. proposal_delay is 2 - let parent_blocks = vec![ - (102, Some((vec![2; 32], vec![], vec![]))), // last committed block - (103, None), - (104, None), // we wont have a proposal because after delay, there is no more non-null proposal - (105, None), - (106, None), - (107, None), - (108, None), // delayed block - (109, Some((vec![8; 32], vec![], vec![]))), - (110, Some((vec![10; 32], vec![], vec![]))), // cache latest height, first non null block - // max proposal height is 112 - ]; - let mut provider = new_provider(parent_blocks).await; - provider.config.max_proposal_range = Some(10); - - assert_eq!(atomically(|| provider.next_proposal()).await, None); - } - - #[tokio::test] - async fn test_with_partially_null_blocks_ii() { - // max_proposal_range is 10. proposal_delay is 2 - let parent_blocks = vec![ - (102, Some((vec![2; 32], vec![], vec![]))), // last committed block - (103, Some((vec![3; 32], vec![], vec![]))), - (104, None), - (105, None), - (106, None), - (107, Some((vec![7; 32], vec![], vec![]))), // first non null after delay - (108, None), // delayed block - (109, None), - (110, Some((vec![10; 32], vec![], vec![]))), // cache latest height, first non null block - // max proposal height is 112 - ]; - let mut provider = new_provider(parent_blocks).await; - provider.config.max_proposal_range = Some(10); - - assert_eq!( - atomically(|| provider.next_proposal()).await, - Some(IPCParentFinality { - height: 107, - block_hash: vec![7; 32] - }) - ); - } - - #[tokio::test] - async fn test_with_partially_null_blocks_iii() { - let parent_blocks = vec![ - (102, Some((vec![2; 32], vec![], vec![]))), // last committed block - (103, Some((vec![3; 32], vec![], vec![]))), - (104, None), - (105, None), - (106, None), - (107, Some((vec![7; 32], vec![], vec![]))), // first non null delayed block, final - (108, None), // delayed block - (109, None), - (110, Some((vec![10; 32], vec![], vec![]))), // first non null block - (111, None), - (112, None), - // max proposal height is 122 - ]; - let mut provider = new_provider(parent_blocks).await; - provider.config.max_proposal_range = Some(20); - - assert_eq!( - atomically(|| provider.next_proposal()).await, - Some(IPCParentFinality { - height: 107, - block_hash: vec![7; 32] - }) - ); - } -} diff --git a/fendermint/vm/topdown/src/launch.rs b/fendermint/vm/topdown/src/launch.rs new file mode 100644 index 000000000..0ae1b212d --- /dev/null +++ b/fendermint/vm/topdown/src/launch.rs @@ -0,0 +1,254 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::proxy::ParentQueryProxy; +use crate::syncer::store::InMemoryParentViewStore; +use crate::syncer::{ + start_polling_reactor, ParentPoller, ParentSyncerConfig, ParentSyncerReactorClient, +}; +use crate::vote::gossip::{GossipReceiver, GossipSender}; +use crate::vote::payload::PowerUpdates; +use crate::vote::store::InMemoryVoteStore; +use crate::vote::{StartVoteReactorParams, VoteReactorClient}; +use crate::{BlockHeight, Checkpoint, Config, TopdownClient, TopdownProposal}; +use anyhow::anyhow; +use cid::Cid; +use fendermint_crypto::SecretKey; +use fendermint_vm_genesis::{Power, Validator, ValidatorKey}; +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + +/// Run the topdown checkpointing in the background. This consists of two processes: +/// - syncer: +/// - syncs with the parent through RPC endpoint to obtain: +/// - parent block hash/height +/// - topdown messages +/// - validator changes +/// - prepares for topdown observation to be braodcasted +/// - voting: +/// - signs/certifies and broadcast topdown observation to p2p peers +/// - listens to certified topdown observation from p2p +/// - aggregate peer certified observations into a quorum certificate for commitment in fendermint +pub async fn run_topdown( + store: InMemoryParentViewStore, + query: CheckpointQuery, + config: Config, + validator_key: SecretKey, + gossip: (GossipTx, GossipRx), + parent_client: ParentClient, + create_poller_fn: impl FnOnce(&Checkpoint, ParentClient, ParentSyncerConfig) -> Poller + + Send + + 'static, +) -> anyhow::Result +where + CheckpointQuery: LaunchQuery + Send + Sync + 'static, + GossipTx: GossipSender + Send + Sync + 'static, + GossipRx: GossipReceiver + Send + Sync + 'static, + Poller: ParentPoller + Send + Sync + 'static, + ParentClient: ParentQueryProxy + Send + Sync + 'static, +{ + let (syncer_client, syncer_rx) = + ParentSyncerReactorClient::new(config.syncer.request_channel_size, store); + let (voting_client, voting_rx) = VoteReactorClient::new(config.voting.req_channel_buffer_size); + + let syncer_client_cloned = syncer_client.clone(); + tokio::spawn(async move { + let query = Arc::new(query); + let checkpoint = query_starting_checkpoint(&query, &parent_client) + .await + .expect("should be able to query starting checkpoint"); + + let power_table = query_starting_committee(&query) + .await + .expect("should be able to query starting committee"); + let power_table = power_table + .into_iter() + .map(|v| { + let vk = ValidatorKey::new(v.public_key.0); + let w = v.power.0; + (vk, w) + }) + .collect::>(); + + let poller = create_poller_fn(&checkpoint, parent_client, config.syncer.clone()); + let internal_event_rx = poller.subscribe(); + + syncer_client_cloned + .finalize_parent_height(checkpoint.clone()) + .await + .expect("should be ok to set checkpoint"); + + start_polling_reactor(syncer_rx, poller, config.syncer); + VoteReactorClient::start_reactor( + voting_rx, + StartVoteReactorParams { + config: config.voting, + validator_key, + power_table, + last_finalized_height: checkpoint.target_height(), + latest_child_block: query + .latest_chain_block() + .expect("should query latest chain block"), + gossip_tx: gossip.0, + gossip_rx: gossip.1, + vote_store: InMemoryVoteStore::default(), + internal_event_listener: internal_event_rx, + }, + ) + .expect("cannot start vote reactor"); + + tracing::info!( + finality = checkpoint.to_string(), + "launching parent syncer with last committed checkpoint" + ); + }); + + Ok(TopdownClient { + syncer: syncer_client, + voting: voting_client, + }) +} + +/// Queries the starting finality for polling. First checks the committed finality, if none, that +/// means the chain has just started, then query from the parent to get the genesis epoch. +pub async fn query_starting_checkpoint( + query: &Arc, + parent_client: &P, +) -> anyhow::Result +where + T: LaunchQuery + Send + Sync + 'static, + P: ParentQueryProxy + Send + Sync + 'static, +{ + loop { + let mut checkpoint = match query.get_latest_checkpoint() { + Ok(Some(finality)) => finality, + Ok(None) => { + tracing::debug!("app not ready for query yet"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + Err(e) => { + tracing::warn!(error = e.to_string(), "cannot get committed finality"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + }; + tracing::info!( + checkpoint = checkpoint.to_string(), + "latest checkpoint committed" + ); + + // this means there are no previous committed finality yet, we fetch from parent to get + // the genesis epoch of the current subnet and its corresponding block hash. + if checkpoint.target_height() == 0 { + let genesis_epoch = parent_client.get_genesis_epoch().await?; + tracing::debug!(genesis_epoch = genesis_epoch, "obtained genesis epoch"); + let r = parent_client.get_block_hash(genesis_epoch).await?; + tracing::debug!( + block_hash = hex::encode(&r.block_hash), + "obtained genesis block hash", + ); + + checkpoint = Checkpoint::v1(genesis_epoch, r.block_hash, Cid::default().to_bytes()); + tracing::info!( + genesis_checkpoint = checkpoint.to_string(), + "no previous checkpoint committed, fetched from genesis epoch" + ); + } + + return Ok(checkpoint); + } +} + +/// Queries the starting finality for polling. First checks the committed finality, if none, that +/// means the chain has just started, then query from the parent to get the genesis epoch. +pub async fn query_starting_committee(query: &Arc) -> anyhow::Result>> +where + T: LaunchQuery + Send + Sync + 'static, +{ + loop { + match query.get_power_table() { + Ok(Some(power_table)) => return Ok(power_table), + Ok(None) => { + tracing::debug!("app not ready for query yet"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + Err(e) => { + tracing::warn!(error = e.to_string(), "cannot get comittee"); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + } + } +} + +/// Query the chain for bootstrapping topdown +/// +/// It returns `None` from queries until the ledger has been initialized. +pub trait LaunchQuery { + /// Get the latest committed checkpoint from the state + fn get_latest_checkpoint(&self) -> anyhow::Result>; + /// Get the current committee voting powers. + fn get_power_table(&self) -> anyhow::Result>>>; + /// Get the latest blockchain height, the local/child subnet chain + fn latest_chain_block(&self) -> anyhow::Result; +} + +/// Toggle is needed for initialization because cyclic dependencies in fendermint bootstrap process. +/// Fendermint's App owns TopdownClient, but TopdownClient needs App for chain state. +/// Also Toggle is needed to handle non ipc enabled setups. +#[derive(Clone)] +pub struct Toggle { + inner: Option, +} + +impl Toggle { + pub fn disable() -> Self { + Self { inner: None } + } + + pub fn enable(t: T) -> Self { + Self { inner: Some(t) } + } + + pub fn is_enabled(&self) -> bool { + self.inner.is_some() + } + + async fn perform_or_err< + 'a, + R, + F: Future>, + Fn: FnOnce(&'a T) -> F, + >( + &'a self, + f: Fn, + ) -> anyhow::Result { + let Some(ref inner) = self.inner else { + return Err(anyhow!("topdown not enabled")); + }; + f(inner).await + } +} + +impl Toggle { + pub async fn validate_quorum_proposal(&self, proposal: TopdownProposal) -> anyhow::Result<()> { + self.perform_or_err(|p| p.validate_quorum_proposal(proposal)) + .await + } + + pub async fn find_topdown_proposal(&self) -> anyhow::Result> { + self.perform_or_err(|p| p.find_topdown_proposal()).await + } + + pub async fn parent_finalized(&self, checkpoint: Checkpoint) -> anyhow::Result<()> { + self.perform_or_err(|p| p.parent_finalized(checkpoint)) + .await + } + + pub async fn update_power_table(&self, updates: PowerUpdates) -> anyhow::Result<()> { + self.perform_or_err(|p| p.update_power_table(updates)).await + } +} diff --git a/fendermint/vm/topdown/src/lib.rs b/fendermint/vm/topdown/src/lib.rs index 71730ba65..9490bc12a 100644 --- a/fendermint/vm/topdown/src/lib.rs +++ b/fendermint/vm/topdown/src/lib.rs @@ -2,35 +2,29 @@ // SPDX-License-Identifier: Apache-2.0, MIT mod cache; -mod error; -mod finality; -pub mod sync; pub mod convert; pub mod proxy; -mod toggle; -pub mod voting; +pub mod launch; pub mod observation; pub mod observe; pub mod syncer; pub mod vote; -use async_stm::Stm; -use async_trait::async_trait; use ethers::utils::hex; -use fvm_shared::clock::ChainEpoch; +use fendermint_crypto::quorum::ECDSACertificate; use ipc_api::cross::IpcEnvelope; use ipc_api::staking::StakingChangeRequest; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; -use std::time::Duration; pub use crate::cache::{SequentialAppendError, SequentialKeyCache, ValueIter}; -pub use crate::error::Error; -pub use crate::finality::CachedFinalityProvider; use crate::observation::Observation; -pub use crate::toggle::Toggle; +use crate::syncer::store::InMemoryParentViewStore; +use crate::syncer::{ParentSyncerConfig, ParentSyncerReactorClient}; +use crate::vote::payload::PowerUpdates; +use crate::vote::{VoteConfig, VoteReactorClient}; pub type BlockHeight = u64; pub type Bytes = Vec; @@ -38,76 +32,11 @@ pub type BlockHash = Bytes; /// The null round error message pub(crate) const NULL_ROUND_ERR_MSG: &str = "requested epoch was a null round"; -/// Default topdown proposal height range -pub(crate) const DEFAULT_MAX_PROPOSAL_RANGE: BlockHeight = 100; -pub(crate) const DEFAULT_MAX_CACHE_BLOCK: BlockHeight = 500; -pub(crate) const DEFAULT_PROPOSAL_DELAY: BlockHeight = 2; #[derive(Debug, Clone, Deserialize)] pub struct Config { - /// The number of blocks to delay before reporting a height as final on the parent chain. - /// To propose a certain number of epochs delayed from the latest height, we see to be - /// conservative and avoid other from rejecting the proposal because they don't see the - /// height as final yet. - pub chain_head_delay: BlockHeight, - /// Parent syncing cron period, in seconds - pub polling_interval: Duration, - /// Top down exponential back off retry base - pub exponential_back_off: Duration, - /// The max number of retries for exponential backoff before giving up - pub exponential_retry_limit: usize, - /// The max number of blocks one should make the topdown proposal - pub max_proposal_range: Option, - /// Max number of blocks that should be stored in cache - pub max_cache_blocks: Option, - pub proposal_delay: Option, -} - -impl Config { - pub fn new( - chain_head_delay: BlockHeight, - polling_interval: Duration, - exponential_back_off: Duration, - exponential_retry_limit: usize, - ) -> Self { - Self { - chain_head_delay, - polling_interval, - exponential_back_off, - exponential_retry_limit, - max_proposal_range: None, - max_cache_blocks: None, - proposal_delay: None, - } - } - - pub fn with_max_proposal_range(mut self, max_proposal_range: BlockHeight) -> Self { - self.max_proposal_range = Some(max_proposal_range); - self - } - - pub fn with_proposal_delay(mut self, proposal_delay: BlockHeight) -> Self { - self.proposal_delay = Some(proposal_delay); - self - } - - pub fn with_max_cache_blocks(mut self, max_cache_blocks: BlockHeight) -> Self { - self.max_cache_blocks = Some(max_cache_blocks); - self - } - - pub fn max_proposal_range(&self) -> BlockHeight { - self.max_proposal_range - .unwrap_or(DEFAULT_MAX_PROPOSAL_RANGE) - } - - pub fn proposal_delay(&self) -> BlockHeight { - self.proposal_delay.unwrap_or(DEFAULT_PROPOSAL_DELAY) - } - - pub fn max_cache_blocks(&self) -> BlockHeight { - self.max_cache_blocks.unwrap_or(DEFAULT_MAX_CACHE_BLOCK) - } + pub syncer: ParentSyncerConfig, + pub voting: VoteConfig, } /// On-chain data structure representing a topdown checkpoint agreed to by a @@ -117,102 +46,110 @@ pub enum Checkpoint { V1(Observation), } -/// The finality view for IPC parent at certain height. +/// Topdown proposal as part of fendermint proposal execution #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct IPCParentFinality { - /// The latest chain height - pub height: BlockHeight, - /// The block hash. For FVM, it is a Cid. For Evm, it is bytes32 as one can now potentially - /// deploy a subnet on EVM. - pub block_hash: BlockHash, +pub struct TopdownProposal { + pub cert: ECDSACertificate, + pub effects: (Vec, Vec), } -impl IPCParentFinality { - pub fn new(height: ChainEpoch, hash: BlockHash) -> Self { - Self { - height: height as BlockHeight, - block_hash: hash, +#[derive(Clone)] +pub struct TopdownClient { + syncer: ParentSyncerReactorClient, + voting: VoteReactorClient, +} + +impl TopdownClient { + pub async fn validate_quorum_proposal(&self, proposal: TopdownProposal) -> anyhow::Result<()> { + self.voting.check_quorum_cert(Box::new(proposal.cert)).await + } + + pub async fn find_topdown_proposal(&self) -> anyhow::Result> { + let Some(quorum_cert) = self.voting.find_quorum().await? else { + tracing::debug!("no quorum cert found"); + return Ok(None); + }; + + let end_height = quorum_cert.payload().parent_subnet_height; + let (ob, xnet_msgs, validator_changes) = + match self.syncer.prepare_quorum_cert_content(end_height) { + Ok(v) => v, + Err(e) => { + tracing::error!(err = e.to_string(), "cannot prepare quorum cert content"); + // return None, don't crash the app + return Ok(None); + } + }; + + if ob != *quorum_cert.payload() { + // could be due to the minor quorum, just return no proposal + tracing::warn!( + created = ob.to_string(), + expected = quorum_cert.payload().to_string(), + "block view observation created not match quorum cert" + ); + return Ok(None); } + + Ok(Some(TopdownProposal { + cert: quorum_cert, + effects: (xnet_msgs, validator_changes), + })) } -} -impl Display for IPCParentFinality { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "IPCParentFinality(height: {}, block_hash: {})", - self.height, - hex::encode(&self.block_hash) - ) + pub async fn parent_finalized(&self, checkpoint: Checkpoint) -> anyhow::Result<()> { + self.voting + .set_quorum_finalized(checkpoint.target_height()) + .await??; + self.syncer.finalize_parent_height(checkpoint).await?; + Ok(()) + } + + pub async fn update_power_table(&self, updates: PowerUpdates) -> anyhow::Result<()> { + self.voting.update_power_table(updates).await } } -#[async_trait] -pub trait ParentViewProvider { - /// Obtain the genesis epoch of the current subnet in the parent - fn genesis_epoch(&self) -> anyhow::Result; - /// Get the validator changes from and to height. - async fn validator_changes_from( - &self, - from: BlockHeight, - to: BlockHeight, - ) -> anyhow::Result>; - /// Get the top down messages from and to height. - async fn top_down_msgs_from( - &self, - from: BlockHeight, - to: BlockHeight, - ) -> anyhow::Result>; +pub(crate) fn is_null_round_str(s: &str) -> bool { + s.contains(NULL_ROUND_ERR_MSG) } -pub trait ParentFinalityProvider: ParentViewProvider { - /// Latest proposal for parent finality - fn next_proposal(&self) -> Stm>; - /// Check if the target proposal is valid - fn check_proposal(&self, proposal: &IPCParentFinality) -> Stm; - /// Called when finality is committed - fn set_new_finality( - &self, - finality: IPCParentFinality, - previous_finality: Option, - ) -> Stm<()>; +impl From<&Observation> for Checkpoint { + fn from(value: &Observation) -> Self { + Self::V1(value.clone()) + } } -/// If res is null round error, returns the default value from f() -pub(crate) fn handle_null_round T>( - res: anyhow::Result, - f: F, -) -> anyhow::Result { - match res { - Ok(t) => Ok(t), - Err(e) => { - if is_null_round_error(&e) { - Ok(f()) - } else { - Err(e) +impl Display for Checkpoint { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Checkpoint::V1(v) => { + write!( + f, + "Checkpoint(version = 1, height = {}, block_hash = {}, effects = {})", + v.parent_subnet_height, + hex::encode(&v.parent_subnet_hash), + hex::encode(&v.cumulative_effects_comm) + ) } } } } -pub(crate) fn is_null_round_error(err: &anyhow::Error) -> bool { - is_null_round_str(&err.to_string()) -} - -pub(crate) fn is_null_round_str(s: &str) -> bool { - s.contains(NULL_ROUND_ERR_MSG) -} - impl Checkpoint { + pub fn v1(height: BlockHeight, hash: BlockHash, effects: Bytes) -> Self { + Self::V1(Observation::new(height, hash, effects)) + } + pub fn target_height(&self) -> BlockHeight { match self { - Checkpoint::V1(b) => b.parent_height, + Checkpoint::V1(b) => b.parent_subnet_height, } } pub fn target_hash(&self) -> &Bytes { match self { - Checkpoint::V1(b) => &b.parent_hash, + Checkpoint::V1(b) => &b.parent_subnet_hash, } } @@ -222,3 +159,15 @@ impl Checkpoint { } } } + +impl quickcheck::Arbitrary for TopdownProposal { + fn arbitrary(g: &mut quickcheck::Gen) -> Self { + let observation = Observation::new(u64::arbitrary(g), Vec::arbitrary(g), Vec::arbitrary(g)); + let cert = ECDSACertificate::new_of_size(observation, 1); + + Self { + cert, + effects: (vec![], vec![]), + } + } +} diff --git a/fendermint/vm/topdown/src/observation.rs b/fendermint/vm/topdown/src/observation.rs index d59c9d9fa..cfa5d9814 100644 --- a/fendermint/vm/topdown/src/observation.rs +++ b/fendermint/vm/topdown/src/observation.rs @@ -231,7 +231,7 @@ impl LinearizedParentBlockView { self.cumulative_effects_comm.as_slice(), to_append.as_slice(), ] - .concat(); + .concat(); let cid = Cid::new_v1(DAG_CBOR, Code::Blake2b256.digest(&bytes)); self.cumulative_effects_comm = cid.to_bytes(); } @@ -264,4 +264,4 @@ impl LinearizedParentBlockView { self.cumulative_effects_comm, )) } -} \ No newline at end of file +} diff --git a/fendermint/vm/topdown/src/sync/mod.rs b/fendermint/vm/topdown/src/sync/mod.rs deleted file mode 100644 index 0092e8414..000000000 --- a/fendermint/vm/topdown/src/sync/mod.rs +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT -//! A constant running process that fetch or listener to parent state - -mod syncer; -mod tendermint; - -use crate::proxy::ParentQueryProxy; -use crate::sync::syncer::LotusParentSyncer; -use crate::sync::tendermint::TendermintAwareSyncer; -use crate::voting::VoteTally; -use crate::{CachedFinalityProvider, Config, IPCParentFinality, ParentFinalityProvider, Toggle}; -use anyhow::anyhow; -use async_stm::atomically; -use ethers::utils::hex; -use ipc_ipld_resolver::ValidatorKey; -use std::sync::Arc; -use std::time::Duration; - -use fendermint_vm_genesis::{Power, Validator}; - -use crate::observation::Observation; -pub use syncer::fetch_topdown_events; - -#[derive(Clone, Debug)] -pub enum TopDownSyncEvent { - NodeSyncing, - NewProposal(Box), -} - -/// Query the parent finality from the block chain state. -/// -/// It returns `None` from queries until the ledger has been initialized. -pub trait ParentFinalityStateQuery { - /// Get the latest committed finality from the state - fn get_latest_committed_finality(&self) -> anyhow::Result>; - /// Get the current committee voting powers. - fn get_power_table(&self) -> anyhow::Result>>>; -} - -/// Queries the starting finality for polling. First checks the committed finality, if none, that -/// means the chain has just started, then query from the parent to get the genesis epoch. -async fn query_starting_finality( - query: &Arc, - parent_client: &Arc

, -) -> anyhow::Result -where - T: ParentFinalityStateQuery + Send + Sync + 'static, - P: ParentQueryProxy + Send + Sync + 'static, -{ - loop { - let mut finality = match query.get_latest_committed_finality() { - Ok(Some(finality)) => finality, - Ok(None) => { - tracing::debug!("app not ready for query yet"); - tokio::time::sleep(Duration::from_secs(5)).await; - continue; - } - Err(e) => { - tracing::warn!(error = e.to_string(), "cannot get committed finality"); - tokio::time::sleep(Duration::from_secs(5)).await; - continue; - } - }; - tracing::info!(finality = finality.to_string(), "latest finality committed"); - - // this means there are no previous committed finality yet, we fetch from parent to get - // the genesis epoch of the current subnet and its corresponding block hash. - if finality.height == 0 { - let genesis_epoch = parent_client.get_genesis_epoch().await?; - tracing::debug!(genesis_epoch = genesis_epoch, "obtained genesis epoch"); - let r = parent_client.get_block_hash(genesis_epoch).await?; - tracing::debug!( - block_hash = hex::encode(&r.block_hash), - "obtained genesis block hash", - ); - - finality = IPCParentFinality { - height: genesis_epoch, - block_hash: r.block_hash, - }; - tracing::info!( - genesis_finality = finality.to_string(), - "no previous finality committed, fetched from genesis epoch" - ); - } - - return Ok(finality); - } -} - -/// Queries the starting finality for polling. First checks the committed finality, if none, that -/// means the chain has just started, then query from the parent to get the genesis epoch. -async fn query_starting_comittee(query: &Arc) -> anyhow::Result>> -where - T: ParentFinalityStateQuery + Send + Sync + 'static, -{ - loop { - match query.get_power_table() { - Ok(Some(power_table)) => return Ok(power_table), - Ok(None) => { - tracing::debug!("app not ready for query yet"); - tokio::time::sleep(Duration::from_secs(5)).await; - continue; - } - Err(e) => { - tracing::warn!(error = e.to_string(), "cannot get comittee"); - tokio::time::sleep(Duration::from_secs(5)).await; - continue; - } - } - } -} - -/// Start the polling parent syncer in the background -pub async fn launch_polling_syncer( - query: T, - config: Config, - view_provider: Arc>>, - vote_tally: VoteTally, - parent_client: Arc

, - tendermint_client: C, -) -> anyhow::Result<()> -where - T: ParentFinalityStateQuery + Send + Sync + 'static, - C: tendermint_rpc::Client + Send + Sync + 'static, - P: ParentQueryProxy + Send + Sync + 'static, -{ - if !view_provider.is_enabled() { - return Err(anyhow!("provider not enabled, enable to run syncer")); - } - - let query = Arc::new(query); - let finality = query_starting_finality(&query, &parent_client).await?; - - let power_table = query_starting_comittee(&query).await?; - let power_table = power_table - .into_iter() - .map(|v| { - let vk = ValidatorKey::from(v.public_key.0); - let w = v.power.0; - (vk, w) - }) - .collect::>(); - - atomically(|| { - view_provider.set_new_finality(finality.clone(), None)?; - - vote_tally.set_finalized(finality.height, finality.block_hash.clone(), None, None)?; - vote_tally.set_power_table(power_table.clone())?; - Ok(()) - }) - .await; - - tracing::info!( - finality = finality.to_string(), - "launching parent syncer with last committed finality" - ); - - start_syncing( - config, - view_provider, - vote_tally, - parent_client, - query, - tendermint_client, - ); - - Ok(()) -} - -/// Start the parent finality listener in the background -fn start_syncing( - config: Config, - view_provider: Arc>>, - vote_tally: VoteTally, - parent_proxy: Arc

, - query: Arc, - tendermint_client: C, -) where - T: ParentFinalityStateQuery + Send + Sync + 'static, - C: tendermint_rpc::Client + Send + Sync + 'static, - P: ParentQueryProxy + Send + Sync + 'static, -{ - let mut interval = tokio::time::interval(config.polling_interval); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - tokio::spawn(async move { - let lotus_syncer = - LotusParentSyncer::new(config, parent_proxy, view_provider, vote_tally, query) - .expect(""); - - let mut tendermint_syncer = TendermintAwareSyncer::new(lotus_syncer, tendermint_client); - - loop { - interval.tick().await; - - if let Err(e) = tendermint_syncer.sync().await { - tracing::error!(error = e.to_string(), "sync with parent encountered error"); - } - } - }); -} diff --git a/fendermint/vm/topdown/src/sync/syncer.rs b/fendermint/vm/topdown/src/sync/syncer.rs deleted file mode 100644 index ee4748058..000000000 --- a/fendermint/vm/topdown/src/sync/syncer.rs +++ /dev/null @@ -1,596 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT -//! The inner type of parent syncer - -use crate::finality::ParentViewPayload; -use crate::proxy::ParentQueryProxy; -use crate::sync::{query_starting_finality, ParentFinalityStateQuery}; -use crate::voting::{self, VoteTally}; -use crate::{ - is_null_round_str, BlockHash, BlockHeight, CachedFinalityProvider, Config, Error, Toggle, -}; -use anyhow::anyhow; -use async_stm::{atomically, atomically_or_err, StmError}; -use ethers::utils::hex; -use libp2p::futures::TryFutureExt; -use std::sync::Arc; -use tracing::instrument; - -use crate::observe::ParentFinalityAcquired; -use ipc_observability::{emit, serde::HexEncodableBlockHash}; - -/// Parent syncer that constantly poll parent. This struct handles lotus null blocks and deferred -/// execution. For ETH based parent, it should work out of the box as well. -pub(crate) struct LotusParentSyncer { - config: Config, - parent_proxy: Arc

, - provider: Arc>>, - vote_tally: VoteTally, - query: Arc, - - /// For testing purposes, we can sync one block at a time. - /// Not part of `Config` as it's a very niche setting; - /// if enabled it would slow down catching up with parent - /// history to a crawl, or one would have to increase - /// the polling frequence to where it's impractical after - /// we have caught up. - sync_many: bool, -} - -impl LotusParentSyncer -where - T: ParentFinalityStateQuery + Send + Sync + 'static, - P: ParentQueryProxy + Send + Sync + 'static, -{ - pub fn new( - config: Config, - parent_proxy: Arc

, - provider: Arc>>, - vote_tally: VoteTally, - query: Arc, - ) -> anyhow::Result { - Ok(Self { - config, - parent_proxy, - provider, - vote_tally, - query, - sync_many: true, - }) - } - - /// Insert the height into cache when we see a new non null block - pub async fn sync(&mut self) -> anyhow::Result<()> { - let chain_head = if let Some(h) = self.finalized_chain_head().await? { - h - } else { - return Ok(()); - }; - - let (mut latest_height_fetched, mut first_non_null_parent_hash) = - self.latest_cached_data().await; - tracing::debug!(chain_head, latest_height_fetched, "syncing heights"); - - if latest_height_fetched > chain_head { - tracing::warn!( - chain_head, - latest_height_fetched, - "chain head went backwards, potential reorg detected from height" - ); - return self.reset().await; - } - - if latest_height_fetched == chain_head { - tracing::debug!( - chain_head, - latest_height_fetched, - "the parent has yet to produce a new block" - ); - return Ok(()); - } - - loop { - if self.exceed_cache_size_limit().await { - tracing::debug!("exceeded cache size limit"); - break; - } - - first_non_null_parent_hash = match self - .poll_next(latest_height_fetched + 1, first_non_null_parent_hash) - .await - { - Ok(h) => h, - Err(Error::ParentChainReorgDetected) => { - tracing::warn!("potential reorg detected, clear cache and retry"); - self.reset().await?; - break; - } - Err(e) => return Err(anyhow!(e)), - }; - - latest_height_fetched += 1; - - if latest_height_fetched == chain_head { - tracing::debug!("reached the tip of the chain"); - break; - } else if !self.sync_many { - break; - } - } - - Ok(()) - } -} - -impl LotusParentSyncer -where - T: ParentFinalityStateQuery + Send + Sync + 'static, - P: ParentQueryProxy + Send + Sync + 'static, -{ - async fn exceed_cache_size_limit(&self) -> bool { - let max_cache_blocks = self.config.max_cache_blocks(); - atomically(|| self.provider.cached_blocks()).await > max_cache_blocks - } - - /// Get the latest data stored in the cache to pull the next block - async fn latest_cached_data(&self) -> (BlockHeight, BlockHash) { - // we are getting the latest height fetched in cache along with the first non null block - // that is stored in cache. - // we are doing two fetches in one `atomically` as if we get the data in two `atomically`, - // the cache might be updated in between the two calls. `atomically` should guarantee atomicity. - atomically(|| { - let latest_height = if let Some(h) = self.provider.latest_height()? { - h - } else { - unreachable!("guaranteed to have latest height, report bug please") - }; - - // first try to get the first non null block before latest_height + 1, i.e. from cache - let prev_non_null_height = - if let Some(height) = self.provider.first_non_null_block(latest_height)? { - tracing::debug!(height, "first non null block in cache"); - height - } else if let Some(p) = self.provider.last_committed_finality()? { - tracing::debug!( - height = p.height, - "first non null block not in cache, use latest finality" - ); - p.height - } else { - unreachable!("guaranteed to have last committed finality, report bug please") - }; - - let hash = if let Some(h) = self.provider.block_hash(prev_non_null_height)? { - h - } else { - unreachable!( - "guaranteed to have hash as the height {} is found", - prev_non_null_height - ) - }; - - Ok((latest_height, hash)) - }) - .await - } - - /// Poll the next block height. Returns finalized and executed block data. - async fn poll_next( - &mut self, - height: BlockHeight, - parent_block_hash: BlockHash, - ) -> Result { - tracing::debug!( - height, - parent_block_hash = hex::encode(&parent_block_hash), - "polling height with parent hash" - ); - - let block_hash_res = match self.parent_proxy.get_block_hash(height).await { - Ok(res) => res, - Err(e) => { - let err = e.to_string(); - if is_null_round_str(&err) { - tracing::debug!( - height, - "detected null round at height, inserted None to cache" - ); - - atomically_or_err::<_, Error, _>(|| { - self.provider.new_parent_view(height, None)?; - self.vote_tally - .add_block(height, None) - .map_err(map_voting_err)?; - Ok(()) - }) - .await?; - - emit(ParentFinalityAcquired { - source: "Parent syncer", - is_null: true, - block_height: height, - block_hash: None, - commitment_hash: None, - num_msgs: 0, - num_validator_changes: 0, - }); - - // Null block received, no block hash for the current height being polled. - // Return the previous parent hash as the non-null block hash. - return Ok(parent_block_hash); - } - return Err(Error::CannotQueryParent( - format!("get_block_hash: {e}"), - height, - )); - } - }; - - if block_hash_res.parent_block_hash != parent_block_hash { - tracing::warn!( - height, - parent_hash = hex::encode(&block_hash_res.parent_block_hash), - previous_hash = hex::encode(&parent_block_hash), - "parent block hash diff than previous hash", - ); - return Err(Error::ParentChainReorgDetected); - } - - let data = self.fetch_data(height, block_hash_res.block_hash).await?; - - tracing::debug!( - height, - staking_requests = data.1.len(), - cross_messages = data.2.len(), - "fetched data" - ); - - atomically_or_err::<_, Error, _>(|| { - // This is here so we see if there is abnormal amount of retries for some reason. - tracing::debug!(height, "adding data to the cache"); - - self.provider.new_parent_view(height, Some(data.clone()))?; - self.vote_tally - .add_block(height, Some(data.0.clone())) - .map_err(map_voting_err)?; - tracing::debug!(height, "non-null block pushed to cache"); - Ok(()) - }) - .await?; - - emit(ParentFinalityAcquired { - source: "Parent syncer", - is_null: false, - block_height: height, - block_hash: Some(HexEncodableBlockHash(data.0.clone())), - // TODO Karel, Willes - when we introduce commitment hash, we should add it here - commitment_hash: None, - num_msgs: data.2.len(), - num_validator_changes: data.1.len(), - }); - - Ok(data.0) - } - - async fn fetch_data( - &self, - height: BlockHeight, - block_hash: BlockHash, - ) -> Result { - fetch_data(self.parent_proxy.as_ref(), height, block_hash).await - } - - async fn finalized_chain_head(&self) -> anyhow::Result> { - let parent_chain_head_height = self.parent_proxy.get_chain_head_height().await?; - // sanity check - if parent_chain_head_height < self.config.chain_head_delay { - tracing::debug!("latest height not more than the chain head delay"); - return Ok(None); - } - - // we consider the chain head finalized only after the `chain_head_delay` - Ok(Some( - parent_chain_head_height - self.config.chain_head_delay, - )) - } - - /// Reset the cache in the face of a reorg - async fn reset(&self) -> anyhow::Result<()> { - let finality = query_starting_finality(&self.query, &self.parent_proxy).await?; - atomically(|| self.provider.reset(finality.clone())).await; - Ok(()) - } -} - -fn map_voting_err(e: StmError) -> StmError { - match e { - StmError::Abort(e) => { - tracing::error!( - error = e.to_string(), - "failed to append block to voting tally" - ); - StmError::Abort(Error::NotSequential) - } - StmError::Control(c) => StmError::Control(c), - } -} - -#[instrument(skip(parent_proxy))] -async fn fetch_data

( - parent_proxy: &P, - height: BlockHeight, - block_hash: BlockHash, -) -> Result -where - P: ParentQueryProxy + Send + Sync + 'static, -{ - let changes_res = parent_proxy - .get_validator_changes(height) - .map_err(|e| Error::CannotQueryParent(format!("get_validator_changes: {e}"), height)); - - let topdown_msgs_res = parent_proxy - .get_top_down_msgs(height) - .map_err(|e| Error::CannotQueryParent(format!("get_top_down_msgs: {e}"), height)); - - let (changes_res, topdown_msgs_res) = tokio::join!(changes_res, topdown_msgs_res); - let (changes_res, topdown_msgs_res) = (changes_res?, topdown_msgs_res?); - - if changes_res.block_hash != block_hash { - tracing::warn!( - height, - change_set_hash = hex::encode(&changes_res.block_hash), - block_hash = hex::encode(&block_hash), - "change set block hash does not equal block hash", - ); - return Err(Error::ParentChainReorgDetected); - } - - if topdown_msgs_res.block_hash != block_hash { - tracing::warn!( - height, - topdown_msgs_hash = hex::encode(&topdown_msgs_res.block_hash), - block_hash = hex::encode(&block_hash), - "topdown messages block hash does not equal block hash", - ); - return Err(Error::ParentChainReorgDetected); - } - - Ok((block_hash, changes_res.value, topdown_msgs_res.value)) -} - -pub async fn fetch_topdown_events

( - parent_proxy: &P, - start_height: BlockHeight, - end_height: BlockHeight, -) -> Result, Error> -where - P: ParentQueryProxy + Send + Sync + 'static, -{ - let mut events = Vec::new(); - for height in start_height..=end_height { - match parent_proxy.get_block_hash(height).await { - Ok(res) => { - let (block_hash, changes, msgs) = - fetch_data(parent_proxy, height, res.block_hash).await?; - - if !(changes.is_empty() && msgs.is_empty()) { - events.push((height, (block_hash, changes, msgs))); - } - } - Err(e) => { - if is_null_round_str(&e.to_string()) { - continue; - } else { - return Err(Error::CannotQueryParent( - format!("get_block_hash: {e}"), - height, - )); - } - } - } - } - Ok(events) -} - -#[cfg(test)] -mod tests { - use crate::proxy::ParentQueryProxy; - use crate::sync::syncer::LotusParentSyncer; - use crate::sync::ParentFinalityStateQuery; - use crate::voting::VoteTally; - use crate::{ - BlockHash, BlockHeight, CachedFinalityProvider, Config, IPCParentFinality, - SequentialKeyCache, Toggle, NULL_ROUND_ERR_MSG, - }; - use anyhow::anyhow; - use async_stm::atomically; - use async_trait::async_trait; - use fendermint_vm_genesis::{Power, Validator}; - use ipc_api::cross::IpcEnvelope; - use ipc_api::staking::StakingChangeRequest; - use ipc_provider::manager::{GetBlockHashResult, TopDownQueryPayload}; - use std::sync::Arc; - - /// How far behind the tip of the chain do we consider blocks final in the tests. - const FINALITY_DELAY: u64 = 2; - - struct TestParentFinalityStateQuery { - latest_finality: IPCParentFinality, - } - - impl ParentFinalityStateQuery for TestParentFinalityStateQuery { - fn get_latest_committed_finality(&self) -> anyhow::Result> { - Ok(Some(self.latest_finality.clone())) - } - fn get_power_table(&self) -> anyhow::Result>>> { - Ok(Some(vec![])) - } - } - - struct TestParentProxy { - blocks: SequentialKeyCache>, - } - - #[async_trait] - impl ParentQueryProxy for TestParentProxy { - async fn get_chain_head_height(&self) -> anyhow::Result { - Ok(self.blocks.upper_bound().unwrap()) - } - - async fn get_genesis_epoch(&self) -> anyhow::Result { - Ok(self.blocks.lower_bound().unwrap() - 1) - } - - async fn get_block_hash(&self, height: BlockHeight) -> anyhow::Result { - let r = self.blocks.get_value(height).unwrap(); - if r.is_none() { - return Err(anyhow!(NULL_ROUND_ERR_MSG)); - } - - for h in (self.blocks.lower_bound().unwrap()..height).rev() { - let v = self.blocks.get_value(h).unwrap(); - if v.is_none() { - continue; - } - return Ok(GetBlockHashResult { - parent_block_hash: v.clone().unwrap(), - block_hash: r.clone().unwrap(), - }); - } - panic!("invalid testing data") - } - - async fn get_top_down_msgs( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - Ok(TopDownQueryPayload { - value: vec![], - block_hash: self.blocks.get_value(height).cloned().unwrap().unwrap(), - }) - } - - async fn get_validator_changes( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - Ok(TopDownQueryPayload { - value: vec![], - block_hash: self.blocks.get_value(height).cloned().unwrap().unwrap(), - }) - } - } - - async fn new_syncer( - blocks: SequentialKeyCache>, - sync_many: bool, - ) -> LotusParentSyncer { - let config = Config { - chain_head_delay: FINALITY_DELAY, - polling_interval: Default::default(), - exponential_back_off: Default::default(), - exponential_retry_limit: 0, - max_proposal_range: Some(1), - max_cache_blocks: None, - proposal_delay: None, - }; - let genesis_epoch = blocks.lower_bound().unwrap(); - let proxy = Arc::new(TestParentProxy { blocks }); - let committed_finality = IPCParentFinality { - height: genesis_epoch, - block_hash: vec![0; 32], - }; - - let vote_tally = VoteTally::new( - vec![], - ( - committed_finality.height, - committed_finality.block_hash.clone(), - ), - ); - - let provider = CachedFinalityProvider::new( - config.clone(), - genesis_epoch, - Some(committed_finality.clone()), - proxy.clone(), - ); - let mut syncer = LotusParentSyncer::new( - config, - proxy, - Arc::new(Toggle::enabled(provider)), - vote_tally, - Arc::new(TestParentFinalityStateQuery { - latest_finality: committed_finality, - }), - ) - .unwrap(); - - // Some tests expect to sync one block at a time. - syncer.sync_many = sync_many; - - syncer - } - - /// Creates a mock of a new parent blockchain view. The key is the height and the value is the - /// block hash. If block hash is None, it means the current height is a null block. - macro_rules! new_parent_blocks { - ($($key:expr => $val:expr),* ,) => ( - hash_map!($($key => $val),*) - ); - ($($key:expr => $val:expr),*) => ({ - let mut map = SequentialKeyCache::sequential(); - $( map.append($key, $val).unwrap(); )* - map - }); - } - - #[tokio::test] - async fn happy_path() { - let parent_blocks = new_parent_blocks!( - 100 => Some(vec![0; 32]), // genesis block - 101 => Some(vec![1; 32]), - 102 => Some(vec![2; 32]), - 103 => Some(vec![3; 32]), - 104 => Some(vec![4; 32]), // after chain head delay, we fetch only to here - 105 => Some(vec![5; 32]), - 106 => Some(vec![6; 32]) // chain head - ); - - let mut syncer = new_syncer(parent_blocks, false).await; - - for h in 101..=104 { - syncer.sync().await.unwrap(); - let p = atomically(|| syncer.provider.latest_height()).await; - assert_eq!(p, Some(h)); - } - } - - #[tokio::test] - async fn with_non_null_block() { - let parent_blocks = new_parent_blocks!( - 100 => Some(vec![0; 32]), // genesis block - 101 => None, - 102 => None, - 103 => None, - 104 => Some(vec![4; 32]), - 105 => None, - 106 => None, - 107 => None, - 108 => Some(vec![5; 32]), - 109 => None, - 110 => None, - 111 => None - ); - - let mut syncer = new_syncer(parent_blocks, false).await; - - for h in 101..=109 { - syncer.sync().await.unwrap(); - assert_eq!( - atomically(|| syncer.provider.latest_height()).await, - Some(h) - ); - } - } -} diff --git a/fendermint/vm/topdown/src/sync/tendermint.rs b/fendermint/vm/topdown/src/sync/tendermint.rs deleted file mode 100644 index 22eb47e82..000000000 --- a/fendermint/vm/topdown/src/sync/tendermint.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT -//! The tendermint aware syncer - -use crate::proxy::ParentQueryProxy; -use crate::sync::syncer::LotusParentSyncer; -use crate::sync::ParentFinalityStateQuery; -use anyhow::Context; - -/// Tendermint aware syncer -pub(crate) struct TendermintAwareSyncer { - inner: LotusParentSyncer, - tendermint_client: C, -} - -impl TendermintAwareSyncer -where - T: ParentFinalityStateQuery + Send + Sync + 'static, - C: tendermint_rpc::Client + Send + Sync + 'static, - P: ParentQueryProxy + Send + Sync + 'static, -{ - pub fn new(inner: LotusParentSyncer, tendermint_client: C) -> Self { - Self { - inner, - tendermint_client, - } - } - - /// Sync with the parent, unless CometBFT is still catching up with the network, - /// in which case we'll get the changes from the subnet peers in the blocks. - pub async fn sync(&mut self) -> anyhow::Result<()> { - if self.is_syncing_peer().await? { - tracing::debug!("syncing with peer, skip parent finality syncing this round"); - return Ok(()); - } - self.inner.sync().await - } - - async fn is_syncing_peer(&self) -> anyhow::Result { - let status: tendermint_rpc::endpoint::status::Response = self - .tendermint_client - .status() - .await - .context("failed to get Tendermint status")?; - Ok(status.sync_info.catching_up) - } -} diff --git a/fendermint/vm/topdown/src/syncer/mod.rs b/fendermint/vm/topdown/src/syncer/mod.rs index 3def2e614..547a5db62 100644 --- a/fendermint/vm/topdown/src/syncer/mod.rs +++ b/fendermint/vm/topdown/src/syncer/mod.rs @@ -170,4 +170,4 @@ impl ParentSyncerReactorClient { pub enum ParentSyncerRequest { /// A new parent height is finalized Finalized(Checkpoint), -} \ No newline at end of file +} diff --git a/fendermint/vm/topdown/src/syncer/poll.rs b/fendermint/vm/topdown/src/syncer/poll.rs index 6bce1ae60..1e73eb1e3 100644 --- a/fendermint/vm/topdown/src/syncer/poll.rs +++ b/fendermint/vm/topdown/src/syncer/poll.rs @@ -28,9 +28,9 @@ pub struct ParentPoll { #[async_trait] impl ParentPoller for ParentPoll - where - S: ParentViewStore + Send + Sync + 'static + Clone, - P: Send + Sync + 'static + ParentQueryProxy, +where + S: ParentViewStore + Send + Sync + 'static + Clone, + P: Send + Sync + 'static + ParentQueryProxy, { type Store = S; @@ -118,9 +118,9 @@ impl ParentPoller for ParentPoll } impl ParentPoll - where - S: ParentViewStore + Send + Sync + 'static, - P: Send + Sync + 'static + ParentQueryProxy, +where + S: ParentViewStore + Send + Sync + 'static, + P: Send + Sync + 'static + ParentQueryProxy, { pub fn new(config: ParentSyncerConfig, proxy: P, store: S, last_finalized: Checkpoint) -> Self { let (tx, _) = broadcast::channel(config.broadcast_channel_size); @@ -273,8 +273,8 @@ async fn fetch_data

( height: BlockHeight, block_hash: BlockHash, ) -> Result - where - P: ParentQueryProxy + Send + Sync + 'static, +where + P: ParentQueryProxy + Send + Sync + 'static, { let changes_res = parent_proxy .get_validator_changes(height) @@ -313,4 +313,4 @@ async fn fetch_data

( topdown_msgs_res.value, changes_res.value, )) -} \ No newline at end of file +} diff --git a/fendermint/vm/topdown/src/syncer/store.rs b/fendermint/vm/topdown/src/syncer/store.rs index 21015b89a..20082dd32 100644 --- a/fendermint/vm/topdown/src/syncer/store.rs +++ b/fendermint/vm/topdown/src/syncer/store.rs @@ -69,4 +69,4 @@ impl ParentViewStore for InMemoryParentViewStore { let inner = self.inner.read().unwrap(); Ok(inner.upper_bound()) } -} \ No newline at end of file +} diff --git a/fendermint/vm/topdown/src/toggle.rs b/fendermint/vm/topdown/src/toggle.rs deleted file mode 100644 index c7dd10065..000000000 --- a/fendermint/vm/topdown/src/toggle.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::finality::ParentViewPayload; -use crate::{ - BlockHash, BlockHeight, CachedFinalityProvider, Error, IPCParentFinality, - ParentFinalityProvider, ParentViewProvider, -}; -use anyhow::anyhow; -use async_stm::{Stm, StmResult}; -use ipc_api::cross::IpcEnvelope; -use ipc_api::staking::StakingChangeRequest; - -/// The parent finality provider could have all functionalities disabled. -#[derive(Clone)] -pub struct Toggle

{ - inner: Option

, -} - -impl

Toggle

{ - pub fn disabled() -> Self { - Self { inner: None } - } - - pub fn enabled(inner: P) -> Self { - Self { inner: Some(inner) } - } - - pub fn is_enabled(&self) -> bool { - self.inner.is_some() - } - - fn perform_or_else(&self, f: F, other: T) -> Result - where - F: FnOnce(&P) -> Result, - { - match &self.inner { - Some(p) => f(p), - None => Ok(other), - } - } -} - -#[async_trait::async_trait] -impl ParentViewProvider for Toggle

{ - fn genesis_epoch(&self) -> anyhow::Result { - match self.inner.as_ref() { - Some(p) => p.genesis_epoch(), - None => Err(anyhow!("provider is toggled off")), - } - } - - async fn validator_changes_from( - &self, - from: BlockHeight, - to: BlockHeight, - ) -> anyhow::Result> { - match self.inner.as_ref() { - Some(p) => p.validator_changes_from(from, to).await, - None => Err(anyhow!("provider is toggled off")), - } - } - - async fn top_down_msgs_from( - &self, - from: BlockHeight, - to: BlockHeight, - ) -> anyhow::Result> { - match self.inner.as_ref() { - Some(p) => p.top_down_msgs_from(from, to).await, - None => Err(anyhow!("provider is toggled off")), - } - } -} - -impl ParentFinalityProvider for Toggle

{ - fn next_proposal(&self) -> Stm> { - self.perform_or_else(|p| p.next_proposal(), None) - } - - fn check_proposal(&self, proposal: &IPCParentFinality) -> Stm { - self.perform_or_else(|p| p.check_proposal(proposal), false) - } - - fn set_new_finality( - &self, - finality: IPCParentFinality, - previous_finality: Option, - ) -> Stm<()> { - self.perform_or_else(|p| p.set_new_finality(finality, previous_finality), ()) - } -} - -impl

Toggle> { - pub fn block_hash(&self, height: BlockHeight) -> Stm> { - self.perform_or_else(|p| p.block_hash(height), None) - } - - pub fn latest_height_in_cache(&self) -> Stm> { - self.perform_or_else(|p| p.latest_height_in_cache(), None) - } - - pub fn latest_height(&self) -> Stm> { - self.perform_or_else(|p| p.latest_height(), None) - } - - pub fn last_committed_finality(&self) -> Stm> { - self.perform_or_else(|p| p.last_committed_finality(), None) - } - - pub fn new_parent_view( - &self, - height: BlockHeight, - maybe_payload: Option, - ) -> StmResult<(), Error> { - self.perform_or_else(|p| p.new_parent_view(height, maybe_payload), ()) - } - - pub fn reset(&self, finality: IPCParentFinality) -> Stm<()> { - self.perform_or_else(|p| p.reset(finality), ()) - } - - pub fn cached_blocks(&self) -> Stm { - self.perform_or_else(|p| p.cached_blocks(), BlockHeight::MAX) - } - - pub fn first_non_null_block(&self, height: BlockHeight) -> Stm> { - self.perform_or_else(|p| p.first_non_null_block(height), None) - } -} diff --git a/fendermint/vm/topdown/src/vote/error.rs b/fendermint/vm/topdown/src/vote/error.rs index aeb22de2f..bd4f3f87c 100644 --- a/fendermint/vm/topdown/src/vote/error.rs +++ b/fendermint/vm/topdown/src/vote/error.rs @@ -22,4 +22,13 @@ pub enum Error { #[error("validator cannot sign vote")] CannotSignVote, + + #[error("cannot publish vote {0}")] + CannotPublishVote(String), + + #[error("receive gossip vote encountered error: {0}")] + CannotReceiveVote(String), + + #[error("received unexpected gossip event {0}")] + UnexpectedGossipEvent(String), } diff --git a/fendermint/vm/topdown/src/vote/mod.rs b/fendermint/vm/topdown/src/vote/mod.rs index 9441d4491..a2862e937 100644 --- a/fendermint/vm/topdown/src/vote/mod.rs +++ b/fendermint/vm/topdown/src/vote/mod.rs @@ -9,36 +9,86 @@ pub mod store; mod tally; use crate::observation::{CertifiedObservation, Observation}; -use crate::sync::TopDownSyncEvent; +use crate::syncer::TopDownSyncEvent; use crate::vote::gossip::{GossipReceiver, GossipSender}; use crate::vote::operation::{OperationMetrics, OperationStateMachine}; use crate::vote::payload::{PowerUpdates, Vote, VoteTallyState}; use crate::vote::store::VoteStore; use crate::vote::tally::VoteTally; use crate::BlockHeight; +use anyhow::anyhow; use error::Error; +use fendermint_crypto::quorum::ECDSACertificate; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use std::borrow::Borrow; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::{broadcast, mpsc, oneshot}; pub type Weight = u64; -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Config { +#[derive(Deserialize, Debug, Clone)] +pub struct VoteConfig { /// The reactor request channel buffer size pub req_channel_buffer_size: usize, } /// The client to interact with the vote reactor +#[derive(Clone)] pub struct VoteReactorClient { tx: mpsc::Sender, } +impl VoteReactorClient { + pub fn new(req_channel_buffer_size: usize) -> (Self, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(req_channel_buffer_size); + (Self { tx }, rx) + } + + pub fn start_reactor< + T: GossipSender + Send + Sync + 'static, + R: GossipReceiver + Send + Sync + 'static, + V: VoteStore + Send + Sync + 'static, + >( + rx: mpsc::Receiver, + params: StartVoteReactorParams, + ) -> anyhow::Result<()> { + let vote_tally = VoteTally::new( + params.power_table, + params.last_finalized_height, + params.vote_store, + )?; + + let validator_key = params.validator_key; + let internal_event_listener = params.internal_event_listener; + let latest_child_block = params.latest_child_block; + let gossip_tx = Arc::new(params.gossip_tx); + let gossip_rx = params.gossip_rx; + + tokio::spawn(async move { + let inner = VotingHandler { + validator_key, + req_rx: rx, + internal_event_listener, + vote_tally, + latest_child_block, + gossip_rx, + gossip_tx, + }; + let mut machine = OperationStateMachine::new(inner); + loop { + machine = machine.step().await; + } + }); + + Ok(()) + } +} + pub struct StartVoteReactorParams { - pub config: Config, + pub config: VoteConfig, pub validator_key: SecretKey, pub power_table: PowerUpdates, pub last_finalized_height: BlockHeight, @@ -49,45 +99,6 @@ pub struct StartVoteReactorParams { pub internal_event_listener: broadcast::Receiver, } -pub fn start_vote_reactor< - T: GossipSender + Send + Sync + 'static, - R: GossipReceiver + Send + Sync + 'static, - V: VoteStore + Send + Sync + 'static, ->( - params: StartVoteReactorParams, -) -> anyhow::Result { - let (tx, rx) = mpsc::channel(params.config.req_channel_buffer_size); - let vote_tally = VoteTally::new( - params.power_table, - params.last_finalized_height, - params.vote_store, - )?; - - let validator_key = params.validator_key; - let internal_event_listener = params.internal_event_listener; - let latest_child_block = params.latest_child_block; - let gossip_tx = Arc::new(params.gossip_tx); - let gossip_rx = params.gossip_rx; - - tokio::spawn(async move { - let inner = VotingHandler { - validator_key, - req_rx: rx, - internal_event_listener, - vote_tally, - latest_child_block, - gossip_rx, - gossip_tx, - }; - let mut machine = OperationStateMachine::new(inner); - loop { - machine = machine.step().await; - } - }); - - Ok(VoteReactorClient { tx }) -} - impl VoteReactorClient { async fn request) -> VoteReactorRequest>( &self, @@ -114,7 +125,7 @@ impl VoteReactorClient { } /// Queries the vote tally to see if there are new quorum formed - pub async fn find_quorum(&self) -> anyhow::Result> { + pub async fn find_quorum(&self) -> anyhow::Result>> { self.request(VoteReactorRequest::FindQuorum).await } @@ -158,9 +169,22 @@ impl VoteReactorClient { .await?; Ok(()) } + + pub async fn check_quorum_cert( + &self, + cert: Box>, + ) -> anyhow::Result<()> { + let is_reached = self + .request(|tx| VoteReactorRequest::CheckQuorumCert { tx, cert }) + .await?; + if !is_reached { + return Err(anyhow!("quorum not reached")); + } + Ok(()) + } } -enum VoteReactorRequest { +pub enum VoteReactorRequest { /// A new child subnet block is mined, this is the fendermint block NewLocalBlockMined(BlockHeight), /// Query the current operation mode of the vote tally state machine @@ -175,8 +199,12 @@ enum VoteReactorRequest { DumpAllVotes(oneshot::Sender>, Error>>), /// Get the current vote tally state variables in vote tally QueryState(oneshot::Sender), + CheckQuorumCert { + cert: Box>, + tx: oneshot::Sender, + }, /// Queries the vote tally to see if there are new quorum formed - FindQuorum(oneshot::Sender>), + FindQuorum(oneshot::Sender>>), /// Update power of some validators. If the weight is zero, the validator is removed /// from the power table. UpdatePowerTable { @@ -258,6 +286,22 @@ where VoteReactorRequest::NewLocalBlockMined(n) => { self.latest_child_block = n; } + VoteReactorRequest::CheckQuorumCert { cert, tx } => { + if !self.vote_tally.check_quorum_cert(cert.borrow()) { + let _ = tx.send(false); + } else { + let is_future = self.vote_tally.last_finalized_height() + < cert.payload().parent_subnet_height; + if !is_future { + tracing::error!( + finalized = self.vote_tally.last_finalized_height(), + cert = cert.payload().parent_subnet_height, + "cert block number lower than latest finalized height" + ); + } + let _ = tx.send(is_future); + } + } } } diff --git a/fendermint/vm/topdown/src/vote/operation/paused.rs b/fendermint/vm/topdown/src/vote/operation/paused.rs index fba97ff71..dea7b2fd7 100644 --- a/fendermint/vm/topdown/src/vote/operation/paused.rs +++ b/fendermint/vm/topdown/src/vote/operation/paused.rs @@ -1,11 +1,11 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::sync::TopDownSyncEvent; use crate::vote::gossip::{GossipReceiver, GossipSender}; use crate::vote::operation::active::ActiveOperationMode; use crate::vote::operation::{OperationMetrics, OperationMode, OperationStateMachine}; use crate::vote::store::VoteStore; +use crate::vote::TopDownSyncEvent; use crate::vote::VotingHandler; use std::fmt::{Display, Formatter}; use tokio::select; diff --git a/fendermint/vm/topdown/src/vote/payload.rs b/fendermint/vm/topdown/src/vote/payload.rs index a58b7e773..a5db8b62e 100644 --- a/fendermint/vm/topdown/src/vote/payload.rs +++ b/fendermint/vm/topdown/src/vote/payload.rs @@ -5,6 +5,7 @@ use crate::observation::{CertifiedObservation, Observation}; use crate::vote::Weight; use crate::BlockHeight; use anyhow::anyhow; +use fendermint_crypto::secp::RecoverableECDSASignature; use fendermint_vm_genesis::ValidatorKey; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -54,6 +55,12 @@ impl Vote { Self::V1 { payload, .. } => payload.observation(), } } + + pub fn observation_signature(&self) -> &RecoverableECDSASignature { + match self { + Self::V1 { payload, .. } => payload.observation_signature(), + } + } } impl TryFrom<&[u8]> for Vote { diff --git a/fendermint/vm/topdown/src/vote/store.rs b/fendermint/vm/topdown/src/vote/store.rs index 5757b0926..7a3492183 100644 --- a/fendermint/vm/topdown/src/vote/store.rs +++ b/fendermint/vm/topdown/src/vote/store.rs @@ -6,6 +6,7 @@ use crate::vote::error::Error; use crate::vote::payload::{PowerTable, Vote}; use crate::vote::Weight; use crate::BlockHeight; +use fendermint_crypto::quorum::ECDSACertificate; use fendermint_vm_genesis::ValidatorKey; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, HashMap}; @@ -83,11 +84,15 @@ impl VoteStore for InMemoryVoteStore { } /// The aggregated votes from different validators. -pub struct VoteAgg<'a>(Vec<&'a Vote>); +pub struct VoteAgg<'a>(HashMap); impl<'a> VoteAgg<'a> { pub fn new(votes: Vec<&'a Vote>) -> Self { - Self(votes) + let mut map = HashMap::new(); + for v in votes { + map.insert(v.voter(), v); + } + Self(map) } pub fn is_empty(&self) -> bool { @@ -95,16 +100,14 @@ impl<'a> VoteAgg<'a> { } pub fn into_owned(self) -> Vec { - self.0.into_iter().cloned().collect() + self.0.into_values().cloned().collect() } pub fn observation_weights(&self, power_table: &PowerTable) -> Vec<(&Observation, Weight)> { let mut votes: Vec<(&Observation, Weight)> = Vec::new(); - for v in self.0.iter() { - let validator = v.voter(); - - let power = power_table.get(&validator).cloned().unwrap_or(0); + for (validator, v) in self.0.iter() { + let power = power_table.get(validator).cloned().unwrap_or(0); if power == 0 { continue; } @@ -118,6 +121,35 @@ impl<'a> VoteAgg<'a> { votes } + + /// Generate a cert from the ordered validator keys and the target observation as payload + pub fn generate_cert( + &self, + ordered_validators: Vec<(&ValidatorKey, &Weight)>, + observation: &Observation, + ) -> Result, Error> { + let mut cert = ECDSACertificate::new_of_size(observation.clone(), ordered_validators.len()); + + for (idx, (validator, _)) in ordered_validators.into_iter().enumerate() { + let Some(vote) = self.0.get(validator) else { + continue; + }; + + if *vote.observation() == *observation { + cert.set_signature( + idx, + validator.public_key(), + vote.observation_signature().clone(), + ) + .map_err(|e| { + tracing::error!(err = e.to_string(), "cannot verify signature"); + Error::VoteCannotBeValidated + })?; + } + } + + Ok(cert) + } } #[cfg(test)] @@ -180,8 +212,10 @@ mod tests { .unwrap(), ); - let agg = VoteAgg(votes.iter().collect()); - let weights = agg.observation_weights(&HashMap::from_iter(powers)); + let agg = VoteAgg(HashMap::from_iter(votes.iter().map(|v| (v.voter(), v)))); + let mut weights = agg.observation_weights(&HashMap::from_iter(powers)); + weights.sort_by(|a, b| a.1.cmp(&b.1)); + assert_eq!(weights, vec![(&observation1, 1), (&observation2, 2),]) } } diff --git a/fendermint/vm/topdown/src/vote/tally.rs b/fendermint/vm/topdown/src/vote/tally.rs index 376d9ff5b..ac4d92370 100644 --- a/fendermint/vm/topdown/src/vote/tally.rs +++ b/fendermint/vm/topdown/src/vote/tally.rs @@ -7,7 +7,10 @@ use crate::vote::payload::{PowerTable, PowerUpdates, Vote}; use crate::vote::store::VoteStore; use crate::vote::Weight; use crate::BlockHeight; +use fendermint_crypto::quorum::ECDSACertificate; use fendermint_vm_genesis::ValidatorKey; +use num_rational::Ratio; +use std::cmp::Ordering; use std::collections::HashMap; /// VoteTally aggregates different votes received from various validators in the network @@ -24,6 +27,9 @@ pub(crate) struct VoteTally { /// The latest height that was voted to be finalized and committed to child blockchian last_finalized_height: BlockHeight, + + /// The quorum threshold ratio required for a quorum + quorum_ratio: Ratio, } impl VoteTally { @@ -44,6 +50,7 @@ impl VoteTally { power_table: HashMap::from_iter(power_table), votes: store, last_finalized_height, + quorum_ratio: Ratio::new(2, 3), }) } @@ -66,7 +73,7 @@ impl VoteTally { /// The equivalent formula can be found in CometBFT [here](https://github.com/cometbft/cometbft/blob/a8991d63e5aad8be82b90329b55413e3a4933dc0/types/vote_set.go#L307). pub fn quorum_threshold(&self) -> Weight { let total_weight: Weight = self.power_table.values().sum(); - total_weight * 2 / 3 + 1 + total_weight * self.quorum_ratio.numer() / self.quorum_ratio.denom() + 1 } /// Return the height of the first entry in the chain. @@ -82,6 +89,20 @@ impl VoteTally { Ok(votes.into_owned()) } + pub fn check_quorum_cert(&self, cert: &ECDSACertificate) -> bool { + let power_table = self + .ordered_validators() + .into_iter() + .map(|(v, w)| (v.public_key(), *w)); + match cert.quorum_reached(power_table, self.quorum_ratio) { + Ok(v) => v, + Err(e) => { + tracing::error!(err = e.to_string(), "check quorum encountered error"); + false + } + } + } + /// Dump all the votes that is currently stored in the vote tally. /// This is generally a very expensive operation, but good for debugging, use with care pub fn dump_votes(&self) -> Result>, Error> { @@ -143,7 +164,7 @@ impl VoteTally { } /// Find a block on the (from our perspective) finalized chain that gathered enough votes from validators. - pub fn find_quorum(&self) -> Result, Error> { + pub fn find_quorum(&self) -> Result>, Error> { let quorum_threshold = self.quorum_threshold(); let Some(max_height) = self.votes.latest_vote_height()? else { tracing::info!("vote store has no vote yet, skip finding quorum"); @@ -163,7 +184,8 @@ impl VoteTally { ); if weight >= quorum_threshold { - return Ok(Some(observation.clone())); + let cert = votes.generate_cert(self.ordered_validators(), observation)?; + return Ok(Some(cert)); } } @@ -177,7 +199,15 @@ impl VoteTally { /// /// After this operation the minimum item in the chain will the new finalized block. pub fn set_finalized(&mut self, block_height: BlockHeight) -> Result<(), Error> { - self.votes.purge_votes_at_height(block_height)?; + let start = if let Some(start) = self.votes.earliest_vote_height()? { + start + } else { + block_height + }; + for h in start..=block_height { + self.votes.purge_votes_at_height(h)?; + } + self.last_finalized_height = block_height; Ok(()) } @@ -209,6 +239,21 @@ impl VoteTally { } } } + + fn ordered_validators(&self) -> Vec<(&ValidatorKey, &Weight)> { + let mut sorted_powers = self.power_table.iter().collect::>(); + + sorted_powers.sort_by(|a, b| { + let cmp = b.1.cmp(a.1); + if cmp != Ordering::Equal { + cmp + } else { + b.0.cmp(a.0) + } + }); + + sorted_powers + } } #[cfg(test)] @@ -259,7 +304,7 @@ mod tests { vote_tally.add_vote(vote).unwrap(); let mut obs2 = random_observation(); - obs2.parent_height = obs.parent_height; + obs2.parent_subnet_height = obs.parent_subnet_height; let vote = Vote::v1_checked(CertifiedObservation::sign(obs2, 100, &validators[0].0).unwrap()) .unwrap(); @@ -281,7 +326,7 @@ mod tests { let observation = random_observation(); vote_tally - .set_finalized(observation.parent_height - 1) + .set_finalized(observation.parent_subnet_height - 1) .unwrap(); for validator in validators { @@ -292,7 +337,7 @@ mod tests { } let ob = vote_tally.find_quorum().unwrap().unwrap(); - assert_eq!(ob, observation); + assert_eq!(*ob.payload(), observation); } #[test] @@ -317,10 +362,10 @@ mod tests { let observation1 = random_observation(); let mut observation2 = observation1.clone(); - observation2.parent_hash = vec![1]; + observation2.parent_subnet_hash = vec![1]; vote_tally - .set_finalized(observation1.parent_height - 1) + .set_finalized(observation1.parent_subnet_height - 1) .unwrap(); for validator in validators_grp1 { @@ -356,7 +401,7 @@ mod tests { let observation = random_observation(); vote_tally - .set_finalized(observation.parent_height - 1) + .set_finalized(observation.parent_subnet_height - 1) .unwrap(); for validator in validators { @@ -367,7 +412,7 @@ mod tests { } let ob = vote_tally.find_quorum().unwrap().unwrap(); - assert_eq!(ob, observation); + assert_eq!(*ob.payload(), observation); let new_powers = (0..3) .map(|_| (random_validator_key().1.clone(), 1)) @@ -391,7 +436,7 @@ mod tests { let observation = random_observation(); vote_tally - .set_finalized(observation.parent_height - 1) + .set_finalized(observation.parent_subnet_height - 1) .unwrap(); for (count, validator) in validators.iter().enumerate() { @@ -413,6 +458,6 @@ mod tests { ]); let ob = vote_tally.find_quorum().unwrap().unwrap(); - assert_eq!(ob, observation); + assert_eq!(*ob.payload(), observation); } } diff --git a/fendermint/vm/topdown/src/voting.rs b/fendermint/vm/topdown/src/voting.rs deleted file mode 100644 index 793c2ab24..000000000 --- a/fendermint/vm/topdown/src/voting.rs +++ /dev/null @@ -1,480 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use async_stm::{abort, atomically_or_err, retry, Stm, StmResult, TVar}; -use serde::{de::DeserializeOwned, Serialize}; -use std::fmt::Display; -use std::hash::Hash; -use std::{fmt::Debug, time::Duration}; - -use crate::observe::{ - ParentFinalityCommitted, ParentFinalityPeerQuorumReached, ParentFinalityPeerVoteReceived, - ParentFinalityPeerVoteSent, -}; -use crate::{BlockHash, BlockHeight}; -use ipc_observability::{emit, serde::HexEncodableBlockHash}; - -// Usign this type because it's `Hash`, unlike the normal `libsecp256k1::PublicKey`. -pub use ipc_ipld_resolver::ValidatorKey; -use ipc_ipld_resolver::VoteRecord; - -pub type Weight = u64; - -#[derive(Debug, thiserror::Error)] -pub enum Error = BlockHash> { - #[error("the last finalized block has not been set")] - Uninitialized, - - #[error("failed to extend chain; expected block height {0}, got {1}")] - UnexpectedBlock(BlockHeight, BlockHeight), - - #[error("validator unknown or has no power: {0:?}")] - UnpoweredValidator(K), - - #[error( - "equivocation by validator {0:?} at height {1}; {} != {}", - hex::encode(.2), - hex::encode(.3) - )] - Equivocation(K, BlockHeight, V, V), -} - -/// Keep track of votes being gossiped about parent chain finality -/// and tally up the weights of the validators on the child subnet, -/// so that we can ask for proposals that are not going to be voted -/// down. -#[derive(Clone)] -pub struct VoteTally { - /// Current validator weights. These are the ones who will vote on the blocks, - /// so these are the weights which need to form a quorum. - power_table: TVar>, - - /// The *finalized mainchain* of the parent as observed by this node. - /// - /// These are assumed to be final because IIRC that's how the syncer works, - /// only fetching the info about blocks which are already sufficiently deep. - /// - /// When we want to propose, all we have to do is walk back this chain and - /// tally the votes we collected for the block hashes until we reach a quorum. - /// - /// The block hash is optional to allow for null blocks on Filecoin rootnet. - chain: TVar>>, - - /// Index votes received by height and hash, which makes it easy to look up - /// all the votes for a given block hash and also to verify that a validator - /// isn't equivocating by trying to vote for two different things at the - /// same height. - votes: TVar>>>, - - /// Adding votes can be paused if we observe that looking for a quorum takes too long - /// and is often retried due to votes being added. - pause_votes: TVar, -} - -impl VoteTally -where - K: Clone + Hash + Eq + Sync + Send + 'static + Debug + Display, - V: AsRef<[u8]> + Clone + Hash + Eq + Sync + Send + 'static, -{ - /// Create an uninitialized instance. Before blocks can be added to it - /// we will have to set the last finalized block. - /// - /// The reason this exists is so that we can delay initialization until - /// after the genesis block has been executed. - pub fn empty() -> Self { - Self { - power_table: TVar::default(), - chain: TVar::default(), - votes: TVar::default(), - pause_votes: TVar::new(false), - } - } - - /// Initialize the vote tally from the current power table - /// and the last finalized block from the ledger. - pub fn new(power_table: Vec<(K, Weight)>, last_finalized_block: (BlockHeight, V)) -> Self { - let (height, hash) = last_finalized_block; - Self { - power_table: TVar::new(im::HashMap::from_iter(power_table)), - chain: TVar::new(im::OrdMap::from_iter([(height, Some(hash))])), - votes: TVar::default(), - pause_votes: TVar::new(false), - } - } - - /// Check that a validator key is currently part of the power table. - pub fn has_power(&self, validator_key: &K) -> Stm { - let pt = self.power_table.read()?; - // For consistency consider validators without power unknown. - match pt.get(validator_key) { - None => Ok(false), - Some(weight) => Ok(*weight > 0), - } - } - - /// Calculate the minimum weight needed for a proposal to pass with the current membership. - /// - /// This is inclusive, that is, if the sum of weight is greater or equal to this, it should pass. - /// The equivalent formula can be found in CometBFT [here](https://github.com/cometbft/cometbft/blob/a8991d63e5aad8be82b90329b55413e3a4933dc0/types/vote_set.go#L307). - pub fn quorum_threshold(&self) -> Stm { - let total_weight: Weight = self.power_table.read().map(|pt| pt.values().sum())?; - - Ok(total_weight * 2 / 3 + 1) - } - - /// Return the height of the first entry in the chain. - /// - /// This is the block that was finalized *in the ledger*. - pub fn last_finalized_height(&self) -> Stm { - self.chain - .read() - .map(|c| c.get_min().map(|(h, _)| *h).unwrap_or_default()) - } - - /// Return the height of the last entry in the chain. - /// - /// This is the block that we can cast our vote on as final. - pub fn latest_height(&self) -> Stm { - self.chain - .read() - .map(|c| c.get_max().map(|(h, _)| *h).unwrap_or_default()) - } - - /// Get the hash of a block at the given height, if known. - pub fn block_hash(&self, height: BlockHeight) -> Stm> { - self.chain.read().map(|c| c.get(&height).cloned().flatten()) - } - - /// Add the next final block observed on the parent blockchain. - /// - /// Returns an error unless it's exactly the next expected height, - /// so the caller has to call this in every epoch. If the parent - /// chain produced no blocks in that epoch then pass `None` to - /// represent that null-round in the tally. - pub fn add_block( - &self, - block_height: BlockHeight, - block_hash: Option, - ) -> StmResult<(), Error> { - let mut chain = self.chain.read_clone()?; - - // Check that we are extending the chain. We could also ignore existing heights. - match chain.get_max() { - None => { - return abort(Error::Uninitialized); - } - Some((parent_height, _)) => { - if block_height != parent_height + 1 { - return abort(Error::UnexpectedBlock(parent_height + 1, block_height)); - } - } - } - - chain.insert(block_height, block_hash); - - self.chain.write(chain)?; - - Ok(()) - } - - /// Add a vote we received. - /// - /// Returns `true` if this vote was added, `false` if it was ignored as a - /// duplicate or a height we already finalized, and an error if it's an - /// equivocation or from a validator we don't know. - pub fn add_vote( - &self, - validator_key: K, - block_height: BlockHeight, - block_hash: V, - ) -> StmResult> { - if *self.pause_votes.read()? { - retry()?; - } - - let min_height = self.last_finalized_height()?; - - if block_height < min_height { - return Ok(false); - } - - if !self.has_power(&validator_key)? { - return abort(Error::UnpoweredValidator(validator_key)); - } - - let mut votes = self.votes.read_clone()?; - let votes_at_height = votes.entry(block_height).or_default(); - - for (bh, vs) in votes_at_height.iter() { - if *bh != block_hash && vs.contains(&validator_key) { - return abort(Error::Equivocation( - validator_key, - block_height, - block_hash, - bh.clone(), - )); - } - } - - let validator_pub_key = validator_key.to_string(); - - let votes_for_block = votes_at_height.entry(block_hash.clone()).or_default(); - - if votes_for_block.insert(validator_key).is_some() { - return Ok(false); - } - - self.votes.write(votes)?; - - emit(ParentFinalityPeerVoteReceived { - block_height, - validator: &validator_pub_key, - block_hash: HexEncodableBlockHash(block_hash.as_ref().to_vec()), - // TODO- this needs to be the commitment hash once implemented - commitment_hash: None, - }); - - Ok(true) - } - - /// Pause adding more votes until we are finished calling `find_quorum` which - /// automatically re-enables them. - pub fn pause_votes_until_find_quorum(&self) -> Stm<()> { - self.pause_votes.write(true) - } - - /// Find a block on the (from our perspective) finalized chain that gathered enough votes from validators. - pub fn find_quorum(&self) -> Stm> { - self.pause_votes.write(false)?; - - let quorum_threshold = self.quorum_threshold()?; - let chain = self.chain.read()?; - - let Some((finalized_height, _)) = chain.get_min() else { - tracing::debug!("finalized height not found"); - return Ok(None); - }; - - let votes = self.votes.read()?; - let power_table = self.power_table.read()?; - - let mut weight = 0; - let mut voters = im::HashSet::new(); - - for (block_height, block_hash) in chain.iter().rev() { - if block_height == finalized_height { - tracing::debug!( - block_height, - finalized_height, - "finalized height and block height equal, no new proposals" - ); - break; // This block is already finalized in the ledger, no need to propose it again. - } - let Some(block_hash) = block_hash else { - tracing::debug!(block_height, "null block found in vote proposal"); - continue; // Skip null blocks - }; - let Some(votes_at_height) = votes.get(block_height) else { - tracing::debug!(block_height, "no votes"); - continue; - }; - let Some(votes_for_block) = votes_at_height.get(block_hash) else { - tracing::debug!(block_height, "no votes for block"); - continue; // We could detect equovicating voters here. - }; - - for vk in votes_for_block { - if voters.insert(vk.clone()).is_none() { - // New voter, get their current weight; it might be 0 if they have been removed. - weight += power_table.get(vk).cloned().unwrap_or_default(); - tracing::debug!(weight, key = ?vk, "new voter"); - } - } - - tracing::debug!(weight, quorum_threshold, "showdown"); - - if weight >= quorum_threshold { - emit(ParentFinalityPeerQuorumReached { - block_height: *block_height, - block_hash: HexEncodableBlockHash(block_hash.as_ref().to_vec()), - // TODO - just placeholder - need to use real commitment once implemented - commitment_hash: None, - weight, - }); - - return Ok(Some((*block_height, block_hash.clone()))); - } - } - - Ok(None) - } - - /// Call when a new finalized block is added to the ledger, to clear out all preceding blocks. - /// - /// After this operation the minimum item in the chain will the new finalized block. - pub fn set_finalized( - &self, - parent_block_height: BlockHeight, - parent_block_hash: V, - proposer: Option<&str>, - local_block_height: Option, - ) -> Stm<()> { - self.chain.update(|chain| { - let (_, mut chain) = chain.split(&parent_block_height); - chain.insert(parent_block_height, Some(parent_block_hash.clone())); - chain - })?; - - self.votes - .update(|votes| votes.split(&parent_block_height).1)?; - - emit(ParentFinalityCommitted { - local_height: local_block_height, - parent_height: parent_block_height, - block_hash: HexEncodableBlockHash(parent_block_hash.as_ref().to_vec()), - proposer, - }); - - Ok(()) - } - - /// Overwrite the power table after it has changed to a new snapshot. - /// - /// This method expects absolute values, it completely replaces the existing powers. - pub fn set_power_table(&self, power_table: Vec<(K, Weight)>) -> Stm<()> { - let power_table = im::HashMap::from_iter(power_table); - // We don't actually have to remove the votes of anyone who is no longer a validator, - // we just have to make sure to handle the case when they are not in the power table. - self.power_table.write(power_table) - } - - /// Update the power table after it has changed with changes. - /// - /// This method expects only the updated values, leaving everyone who isn't in it untouched - pub fn update_power_table(&self, power_updates: Vec<(K, Weight)>) -> Stm<()> { - if power_updates.is_empty() { - return Ok(()); - } - // We don't actually have to remove the votes of anyone who is no longer a validator, - // we just have to make sure to handle the case when they are not in the power table. - self.power_table.update_mut(|pt| { - for (vk, w) in power_updates { - if w == 0 { - pt.remove(&vk); - } else { - *pt.entry(vk).or_default() = w; - } - } - }) - } -} - -/// Poll the vote tally for new finalized blocks and publish a vote about them if the validator is part of the power table. -pub async fn publish_vote_loop( - vote_tally: VoteTally, - // Throttle votes to maximum 1/interval - vote_interval: Duration, - // Publish a vote after a timeout even if it's the same as before. - vote_timeout: Duration, - key: libp2p::identity::Keypair, - subnet_id: ipc_api::subnet_id::SubnetID, - client: ipc_ipld_resolver::Client, - to_vote: F, -) where - F: Fn(BlockHeight, BlockHash) -> V, - V: Serialize + DeserializeOwned, -{ - let validator_key = ValidatorKey::from(key.public()); - - let mut vote_interval = tokio::time::interval(vote_interval); - vote_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - let mut prev = None; - - loop { - let prev_height = prev - .as_ref() - .map(|(height, _, _)| *height) - .unwrap_or_default(); - - let result = tokio::time::timeout( - vote_timeout, - atomically_or_err(|| { - let next_height = vote_tally.latest_height()?; - - if next_height == prev_height { - retry()?; - } - - let next_hash = match vote_tally.block_hash(next_height)? { - Some(next_hash) => next_hash, - None => retry()?, - }; - - let has_power = vote_tally.has_power(&validator_key)?; - - if has_power { - // Add our own vote to the tally directly rather than expecting a message from the gossip channel. - // TODO (ENG-622): I'm not sure gossip messages published by this node would be delivered to it, so this might be the only way. - // NOTE: We should not see any other error from this as we just checked that the validator had power, - // but for piece of mind let's return and log any potential errors, rather than ignore them. - vote_tally.add_vote(validator_key.clone(), next_height, next_hash.clone())?; - } - - Ok((next_height, next_hash, has_power)) - }), - ) - .await; - - let (next_height, next_hash, has_power) = match result { - Ok(Ok(vs)) => vs, - Err(_) => { - if let Some(ref vs) = prev { - tracing::debug!("vote timeout; re-publishing previous vote"); - vs.clone() - } else { - tracing::debug!("vote timeout, but no previous vote to re-publish"); - continue; - } - } - Ok(Err(e)) => { - tracing::error!( - error = e.to_string(), - "failed to get next height to vote on" - ); - continue; - } - }; - - if has_power && prev_height > 0 { - tracing::debug!(block_height = next_height, "publishing finality vote"); - - let vote = to_vote(next_height, next_hash.clone()); - - match VoteRecord::signed(&key, subnet_id.clone(), vote) { - Ok(vote) => { - if let Err(e) = client.publish_vote(vote) { - tracing::error!(error = e.to_string(), "failed to publish vote"); - } - - emit(ParentFinalityPeerVoteSent { - block_height: next_height, - block_hash: HexEncodableBlockHash(next_hash.clone()), - commitment_hash: None, - }); - } - Err(e) => { - tracing::error!(error = e.to_string(), "failed to sign vote"); - } - } - - // Throttle vote gossiping at periods of fast syncing. For example if we create a subnet contract on Friday - // and bring up a local testnet on Monday, all nodes would be ~7000 blocks behind a Lotus parent. CometBFT - // would be in-sync, and they could rapidly try to gossip votes on previous heights. GossipSub might not like - // that, and we can just cast our votes every now and then to finalize multiple blocks. - vote_interval.tick().await; - } - - prev = Some((next_height, next_hash, has_power)); - } -} diff --git a/fendermint/vm/topdown/tests/smt_voting.rs b/fendermint/vm/topdown/tests/smt_voting.rs deleted file mode 100644 index b605a79a1..000000000 --- a/fendermint/vm/topdown/tests/smt_voting.rs +++ /dev/null @@ -1,508 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -//! State Machine Test for the finality voting tally component. -//! -//! The test simulates random events that the tally can receive, such as votes received -//! over gossip, power table updates, block being executed, and tests that the tally -//! correctly identifies the blocks which are agreeable to the majority of validator. -//! -//! It can be executed the following way: -//! -//! ```text -//! cargo test --release -p fendermint_vm_topdown --test smt_voting -//! ``` - -use core::fmt; -use std::{ - cmp::{max, min}, - collections::BTreeMap, - fmt::Debug, -}; - -use arbitrary::Unstructured; -use async_stm::{atomically, atomically_or_err, Stm, StmResult}; -use fendermint_testing::{smt, state_machine_test}; -use fendermint_vm_topdown::{ - voting::{self, VoteTally, Weight}, - BlockHash, BlockHeight, -}; -use im::HashSet; -//use rand::{rngs::StdRng, SeedableRng}; - -/// Size of window of voting relative to the last cast vote. -const MAX_VOTE_DELTA: BlockHeight = 5; -/// Maximum number of blocks to finalize at a time. -const MAX_FINALIZED_DELTA: BlockHeight = 5; - -state_machine_test!(voting, 10000 ms, 65512 bytes, 200 steps, VotingMachine::new()); -//state_machine_test!(voting, 0xf7ac11a50000ffe8, 200 steps, VotingMachine::new()); - -/// Test key to make debugging more readable. -#[derive(Debug, Clone, Hash, Eq, PartialEq, PartialOrd, Ord)] -pub struct VotingKey(u64); - -impl fmt::Display for VotingKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "VotingKey({})", self.0) - } -} - -pub type VotingError = voting::Error; - -pub enum VotingCommand { - /// The tally observes the next block fo the chain. - ExtendChain(BlockHeight, Option), - /// One of the validators voted on a block. - AddVote(VotingKey, BlockHeight, BlockHash), - /// Update the power table. - UpdatePower(Vec<(VotingKey, Weight)>), - /// A certain height was finalized in the ledger. - BlockFinalized(BlockHeight, BlockHash), - /// Ask the tally for the highest agreeable block. - FindQuorum, -} - -// Debug format without block hashes which make it unreadable. -impl Debug for VotingCommand { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::ExtendChain(arg0, arg1) => f - .debug_tuple("ExtendChain") - .field(arg0) - .field(&arg1.is_some()) - .finish(), - Self::AddVote(arg0, arg1, _arg2) => { - f.debug_tuple("AddVote").field(arg0).field(arg1).finish() - } - Self::UpdatePower(arg0) => f.debug_tuple("UpdatePower").field(arg0).finish(), - Self::BlockFinalized(arg0, _arg1) => { - f.debug_tuple("BlockFinalized").field(arg0).finish() - } - Self::FindQuorum => write!(f, "FindQuorum"), - } - } -} - -/// Model state of voting -#[derive(Clone)] -pub struct VotingState { - /// We have a single parent chain that everybody observes, just at different heights. - /// There is no forking in this test because we assume that the syncing component - /// only downloads blocks which are final, and that reorgs don't happen. - /// - /// Null blocks are represented by `None`. - /// - /// The tally is currently unable to handle reorgs and rejects equivocations anyway. - /// - /// TODO (ENG-623): Decide what we want to achieve with Equivocation detection. - chain: Vec>, - /// All the validator keys to help pic random ones. - validator_keys: Vec, - /// All the validators with varying weights (can be zero). - validator_states: BTreeMap, - - last_finalized_block: BlockHeight, - last_chain_block: BlockHeight, -} - -impl VotingState { - pub fn can_extend(&self) -> bool { - self.last_chain_block < self.max_chain_height() - } - - pub fn can_finalize(&self) -> bool { - // We can finalize a block even if we haven't observed the votes, - // if the majority of validators vote for an actual block that - // proposed it for execution. - self.last_finalized_block < self.max_chain_height() - } - - pub fn next_chain_block(&self) -> Option<(BlockHeight, Option)> { - if self.can_extend() { - let h = self.last_chain_block + 1; - Some((h, self.block_hash(h))) - } else { - None - } - } - - pub fn max_chain_height(&self) -> BlockHeight { - self.chain.len() as BlockHeight - 1 - } - - pub fn block_hash(&self, h: BlockHeight) -> Option { - self.chain[h as usize].clone() - } - - pub fn has_quorum(&self, h: BlockHeight) -> bool { - if self.block_hash(h).is_none() { - return false; - } - - let mut total_weight: Weight = 0; - let mut vote_weight: Weight = 0; - - for vs in self.validator_states.values() { - total_weight += vs.weight; - if vs.highest_vote >= h { - vote_weight += vs.weight; - } - } - - let threshold = total_weight * 2 / 3; - - vote_weight > threshold - } -} - -#[derive(Clone, Debug)] -pub struct ValidatorState { - /// Current voting power (can be zero). - weight: Weight, - /// The heights this validator explicitly voted on. - votes: HashSet, - /// The highest vote *currently on the chain* the validator has voted for already. - /// Initially zero, meaning everyone voted on the initial finalized block. - highest_vote: BlockHeight, -} - -pub struct VotingMachine { - /// Runtime for executing async commands. - runtime: tokio::runtime::Runtime, -} - -impl VotingMachine { - pub fn new() -> Self { - Self { - runtime: tokio::runtime::Runtime::new().expect("create tokio runtime"), - } - } - - fn atomically_or_err(&self, f: F) -> Result - where - F: Fn() -> StmResult, - { - self.runtime.block_on(atomically_or_err(f)) - } - - fn atomically(&self, f: F) -> T - where - F: Fn() -> Stm, - { - self.runtime.block_on(atomically(f)) - } - - // For convenience in the command handler. - fn atomically_ok(&self, f: F) -> Result - where - F: Fn() -> Stm, - { - Ok(self.atomically(f)) - } -} - -impl Default for VotingMachine { - fn default() -> Self { - Self::new() - } -} - -impl smt::StateMachine for VotingMachine { - /// The System Under Test is the Vote Tally. - type System = VoteTally; - /// The model state is defined here in the test. - type State = VotingState; - /// Random commands we can apply in a step. - type Command = VotingCommand; - /// Result of command application on the system. - /// - /// The only return value we are interested in is the finality. - type Result = Result, voting::Error>; - - /// New random state. - fn gen_state(&self, u: &mut Unstructured) -> arbitrary::Result { - let chain_length = u.int_in_range(40..=60)?; - let mut chain = Vec::new(); - for i in 0..chain_length { - if i == 0 || u.ratio(9, 10)? { - let block_hash = u.bytes(32)?; - chain.push(Some(Vec::from(block_hash))); - } else { - chain.push(None); - } - } - - let validator_count = u.int_in_range(1..=5)?; - //let mut rng = StdRng::seed_from_u64(u.arbitrary()?); - let mut validator_states = BTreeMap::new(); - - for i in 0..validator_count { - let min_weight = if i == 0 { 1u64 } else { 0u64 }; - let weight = u.int_in_range(min_weight..=100)?; - - // A VotingKey is has a lot of wrapping... - // let secret_key = fendermint_crypto::SecretKey::random(&mut rng); - // let public_key = secret_key.public_key(); - // let public_key = libp2p::identity::secp256k1::PublicKey::try_from_bytes( - // &public_key.serialize_compressed(), - // ) - // .expect("secp256k1 public key"); - // let public_key = libp2p::identity::PublicKey::from(public_key); - // let validator_key = VotingKey::from(public_key); - - let validator_key = VotingKey(i); - - validator_states.insert( - validator_key, - ValidatorState { - weight, - votes: HashSet::default(), - highest_vote: 0, - }, - ); - } - - eprintln!("NEW STATE: {validator_states:?}"); - - Ok(VotingState { - chain, - validator_keys: validator_states.keys().cloned().collect(), - validator_states, - last_chain_block: 0, - last_finalized_block: 0, - }) - } - - /// New System Under Test. - fn new_system(&self, state: &Self::State) -> Self::System { - let power_table = state - .validator_states - .iter() - .filter(|(_, vs)| vs.weight > 0) - .map(|(vk, vs)| (vk.clone(), vs.weight)) - .collect(); - - let last_finalized_block = (0, state.block_hash(0).expect("first block is not null")); - - VoteTally::::new(power_table, last_finalized_block) - } - - /// New random command. - fn gen_command( - &self, - u: &mut Unstructured, - state: &Self::State, - ) -> arbitrary::Result { - let cmd = match u.int_in_range(0..=100)? { - // Add a block to the observed chain - i if i < 25 && state.can_extend() => { - let (height, hash) = state.next_chain_block().unwrap(); - VotingCommand::ExtendChain(height, hash) - } - // Add a new (or repeated) vote by a validator, extending its chain - i if i < 70 => { - let vk = u.choose(&state.validator_keys)?; - let high_vote = state.validator_states[vk].highest_vote; - let max_vote: BlockHeight = - min(state.max_chain_height(), high_vote + MAX_VOTE_DELTA); - let min_vote: BlockHeight = high_vote.saturating_sub(MAX_VOTE_DELTA); - - let mut vote_height = u.int_in_range(min_vote..=max_vote)?; - while state.block_hash(vote_height).is_none() { - vote_height -= 1; - } - let vote_hash = state - .block_hash(vote_height) - .expect("the first block not null"); - - VotingCommand::AddVote(vk.clone(), vote_height, vote_hash) - } - // Update the power table - i if i < 80 => { - // Move power from one validator to another (so we never have everyone be zero). - let vk1 = u.choose(&state.validator_keys)?; - let vk2 = u.choose(&state.validator_keys)?; - let w1 = state.validator_states[vk1].weight; - let w2 = state.validator_states[vk2].weight; - let delta = u.int_in_range(0..=w1)?; - - let updates = vec![(vk1.clone(), w1 - delta), (vk2.clone(), w2 + delta)]; - - VotingCommand::UpdatePower(updates) - } - // Finalize a block - i if i < 90 && state.can_finalize() => { - let min_fin = state.last_finalized_block + 1; - let max_fin = min( - state.max_chain_height(), - state.last_finalized_block + MAX_FINALIZED_DELTA, - ); - - let mut fin_height = u.int_in_range(min_fin..=max_fin)?; - while state.block_hash(fin_height).is_none() { - fin_height -= 1; - } - let fin_hash = state - .block_hash(fin_height) - .expect("the first block not null"); - - // Might be a duplicate, which doesn't happen in the real ledger, but it's okay. - VotingCommand::BlockFinalized(fin_height, fin_hash) - } - _ => VotingCommand::FindQuorum, - }; - Ok(cmd) - } - - /// Apply the command on the System Under Test. - fn run_command(&self, system: &mut Self::System, cmd: &Self::Command) -> Self::Result { - eprintln!("RUN CMD {cmd:?}"); - match cmd { - VotingCommand::ExtendChain(block_height, block_hash) => self.atomically_or_err(|| { - system - .add_block(*block_height, block_hash.clone()) - .map(|_| None) - }), - VotingCommand::AddVote(vk, block_height, block_hash) => self.atomically_or_err(|| { - system - .add_vote(vk.clone(), *block_height, block_hash.clone()) - .map(|_| None) - }), - - VotingCommand::UpdatePower(power_table) => { - self.atomically_ok(|| system.update_power_table(power_table.clone()).map(|_| None)) - } - - VotingCommand::BlockFinalized(block_height, block_hash) => self.atomically_ok(|| { - system - .set_finalized(*block_height, block_hash.clone(), None, None) - .map(|_| None) - }), - - VotingCommand::FindQuorum => self.atomically_ok(|| system.find_quorum()), - } - } - - /// Check that the result returned by the tally is correct. - fn check_result(&self, cmd: &Self::Command, pre_state: &Self::State, result: Self::Result) { - match cmd { - VotingCommand::ExtendChain(_, _) => { - result.expect("chain extension should succeed; not simulating unexpected heights"); - } - VotingCommand::AddVote(vk, h, _) => { - if *h < pre_state.last_finalized_block { - result.expect("old votes are ignored"); - } else if pre_state.validator_states[vk].weight == 0 { - result.expect_err("not accepting votes from validators with 0 power"); - } else { - result.expect("vote should succeed; not simulating equivocations"); - } - } - VotingCommand::FindQuorum => { - let result = result.expect("finding quorum should succeed"); - - let height = match result { - None => pre_state.last_finalized_block, - Some((height, hash)) => { - assert!( - pre_state.has_quorum(height), - "find: height {height} should have quorum" - ); - assert!( - height > pre_state.last_finalized_block, - "find: should be above last finalized" - ); - assert!( - height <= pre_state.last_chain_block, - "find: should not be beyond last chain" - ); - assert_eq!( - pre_state.block_hash(height), - Some(hash), - "find: should be correct hash" - ); - height - } - }; - - // Check that the first non-null block after the finalized one has no quorum. - let mut next = height + 1; - if next > pre_state.max_chain_height() || next > pre_state.last_chain_block { - return; - } - while next < pre_state.last_chain_block && pre_state.block_hash(next).is_none() { - next += 1; - } - assert!( - !pre_state.has_quorum(next), - "next block at {next} should not have quorum" - ) - } - other => { - assert!(result.is_ok(), "{other:?} should succeed: {result:?}"); - } - } - } - - /// Update the model state. - fn next_state(&self, cmd: &Self::Command, mut state: Self::State) -> Self::State { - match cmd { - VotingCommand::ExtendChain(h, _) => { - state.last_chain_block = *h; - for vs in state.validator_states.values_mut() { - if vs.votes.contains(h) { - vs.highest_vote = *h; - } - } - } - VotingCommand::AddVote(vk, h, _) => { - let vs = state - .validator_states - .get_mut(vk) - .expect("validator exists"); - - if vs.weight > 0 { - vs.votes.insert(*h); - - if *h <= state.last_chain_block { - vs.highest_vote = max(vs.highest_vote, *h); - } - } - } - VotingCommand::UpdatePower(pt) => { - for (vk, w) in pt { - state - .validator_states - .get_mut(vk) - .expect("validators exist") - .weight = *w; - } - } - VotingCommand::BlockFinalized(h, _) => { - state.last_finalized_block = *h; - state.last_chain_block = max(state.last_chain_block, state.last_finalized_block); - } - VotingCommand::FindQuorum => {} - } - state - } - - /// Compare the tally agains the updated model state. - fn check_system( - &self, - _cmd: &Self::Command, - post_state: &Self::State, - post_system: &Self::System, - ) -> bool { - let last_finalized_block = self.atomically(|| post_system.last_finalized_height()); - - assert_eq!( - last_finalized_block, post_state.last_finalized_block, - "last finalized blocks should match" - ); - - // Stop if we finalized everything. - last_finalized_block < post_state.max_chain_height() - } -} diff --git a/fendermint/vm/topdown/tests/vote_reactor.rs b/fendermint/vm/topdown/tests/vote_reactor.rs index d542bfbe9..abd471588 100644 --- a/fendermint/vm/topdown/tests/vote_reactor.rs +++ b/fendermint/vm/topdown/tests/vote_reactor.rs @@ -9,14 +9,12 @@ use async_trait::async_trait; use fendermint_crypto::SecretKey; use fendermint_vm_genesis::ValidatorKey; use fendermint_vm_topdown::observation::Observation; -use fendermint_vm_topdown::sync::TopDownSyncEvent; +use fendermint_vm_topdown::syncer::TopDownSyncEvent; use fendermint_vm_topdown::vote::error::Error; use fendermint_vm_topdown::vote::gossip::{GossipReceiver, GossipSender}; use fendermint_vm_topdown::vote::payload::{PowerUpdates, Vote}; use fendermint_vm_topdown::vote::store::InMemoryVoteStore; -use fendermint_vm_topdown::vote::{ - start_vote_reactor, Config, StartVoteReactorParams, VoteReactorClient, Weight, -}; +use fendermint_vm_topdown::vote::{StartVoteReactorParams, VoteConfig, VoteReactorClient, Weight}; use fendermint_vm_topdown::BlockHeight; use std::time::Duration; use tokio::sync::broadcast; @@ -66,8 +64,8 @@ impl GossipReceiver for ChannelGossipReceiver { } } -fn default_config() -> Config { - Config { +fn default_config() -> VoteConfig { + VoteConfig { req_channel_buffer_size: 1024, } } @@ -138,18 +136,22 @@ async fn simple_lifecycle() { let (internal_event_tx, _) = broadcast::channel(validators.len() + 1); + let (client, rx) = VoteReactorClient::new(1024); let (gossip_tx, gossip_rx) = gossips.pop().unwrap(); - let client = start_vote_reactor(StartVoteReactorParams { - config: config.clone(), - validator_key: validators[0].sk.clone(), - power_table: power_updates.clone(), - last_finalized_height: initial_finalized_height, - latest_child_block: 100, - gossip_rx, - vote_store: InMemoryVoteStore::default(), - internal_event_listener: internal_event_tx.subscribe(), - gossip_tx, - }) + VoteReactorClient::start_reactor( + rx, + StartVoteReactorParams { + config: config.clone(), + validator_key: validators[0].sk.clone(), + power_table: power_updates.clone(), + last_finalized_height: initial_finalized_height, + latest_child_block: 100, + gossip_tx, + gossip_rx, + vote_store: InMemoryVoteStore::default(), + internal_event_listener: internal_event_tx.subscribe(), + }, + ) .unwrap(); assert_eq!(client.find_quorum().await.unwrap(), None); @@ -165,7 +167,7 @@ async fn simple_lifecycle() { while client.find_quorum().await.unwrap().is_none() {} let r = client.find_quorum().await.unwrap().unwrap(); - assert_eq!(r.parent_height(), parent_height); + assert_eq!(r.payload().parent_height(), parent_height); let r = client.query_votes(parent_height).await.unwrap().unwrap(); assert_eq!(r.len(), 1); @@ -189,7 +191,7 @@ async fn simple_lifecycle() { let votes = client.query_votes(parent_height2).await.unwrap().unwrap(); assert_eq!(votes.len(), 1); let r = client.find_quorum().await.unwrap().unwrap(); - assert_eq!(r.parent_height(), parent_height2); + assert_eq!(r.payload().parent_height(), parent_height2); client .set_quorum_finalized(parent_height2) @@ -218,18 +220,22 @@ async fn waiting_for_quorum() { for i in 0..validators.len() { let (internal_event_tx, _) = broadcast::channel(validators.len() + 1); + let (client, rx) = VoteReactorClient::new(1024); let (gossip_tx, gossip_rx) = gossips.pop().unwrap(); - let client = start_vote_reactor(StartVoteReactorParams { - config: config.clone(), - validator_key: validators[i].sk.clone(), - power_table: power_updates.clone(), - last_finalized_height: initial_finalized_height, - latest_child_block: 100, - gossip_tx, - gossip_rx, - vote_store: InMemoryVoteStore::default(), - internal_event_listener: internal_event_tx.subscribe(), - }) + VoteReactorClient::start_reactor( + rx, + StartVoteReactorParams { + config: config.clone(), + validator_key: validators[i].sk.clone(), + power_table: power_updates.clone(), + last_finalized_height: initial_finalized_height, + latest_child_block: 100, + gossip_tx, + gossip_rx, + vote_store: InMemoryVoteStore::default(), + internal_event_listener: internal_event_tx.subscribe(), + }, + ) .unwrap(); clients.push(client); @@ -295,7 +301,11 @@ async fn waiting_for_quorum() { for client in &clients { let r = client.find_quorum().await.unwrap().unwrap(); - assert_eq!(r.parent_height(), parent_height3, "should have quorum"); + assert_eq!( + r.payload().parent_height(), + parent_height3, + "should have quorum" + ); } // make observation on previous heights @@ -313,7 +323,7 @@ async fn waiting_for_quorum() { for client in &clients { let r = client.find_quorum().await.unwrap().unwrap(); assert_eq!( - r.parent_height(), + r.payload().parent_height(), parent_height3, "should have formed quorum on larger height" ); @@ -346,18 +356,22 @@ async fn all_validator_in_sync() { let mut node_clients = vec![]; for validator in &validators { + let (r, rx) = VoteReactorClient::new(1024); let (gossip_tx, gossip_rx) = gossips.pop().unwrap(); - let r = start_vote_reactor(StartVoteReactorParams { - config: config.clone(), - validator_key: validator.sk.clone(), - power_table: power_updates.clone(), - last_finalized_height: initial_finalized_height, - latest_child_block: 100, - gossip_tx, - gossip_rx, - vote_store: InMemoryVoteStore::default(), - internal_event_listener: internal_event_tx.subscribe(), - }) + VoteReactorClient::start_reactor( + rx, + StartVoteReactorParams { + config: config.clone(), + validator_key: validator.sk.clone(), + power_table: power_updates.clone(), + last_finalized_height: initial_finalized_height, + latest_child_block: 100, + gossip_tx, + gossip_rx, + vote_store: InMemoryVoteStore::default(), + internal_event_listener: internal_event_tx.subscribe(), + }, + ) .unwrap(); node_clients.push(r); @@ -373,6 +387,6 @@ async fn all_validator_in_sync() { while n.find_quorum().await.unwrap().is_none() {} let r = n.find_quorum().await.unwrap().unwrap(); - assert_eq!(r.parent_height(), parent_height) + assert_eq!(r.payload().parent_height(), parent_height) } } diff --git a/ipc/api/src/staking.rs b/ipc/api/src/staking.rs index 723a10a76..4f0a93f76 100644 --- a/ipc/api/src/staking.rs +++ b/ipc/api/src/staking.rs @@ -13,7 +13,7 @@ use std::fmt::{Display, Formatter}; pub type ConfigurationNumber = u64; -#[derive(Clone, Debug, num_enum::TryFromPrimitive, Deserialize, Serialize)] +#[derive(Clone, Debug, num_enum::TryFromPrimitive, Deserialize, Serialize, PartialEq, Eq)] #[non_exhaustive] #[repr(u8)] pub enum StakingOperation { @@ -23,14 +23,14 @@ pub enum StakingOperation { SetFederatedPower = 3, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct StakingChangeRequest { pub configuration_number: ConfigurationNumber, pub change: StakingChange, } /// The change request to validator staking -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct StakingChange { pub op: StakingOperation, pub payload: Vec, diff --git a/ipc/cli/src/commands/crossmsg/topdown_cross.rs b/ipc/cli/src/commands/crossmsg/topdown_cross.rs index 3e100bc84..711575a66 100644 --- a/ipc/cli/src/commands/crossmsg/topdown_cross.rs +++ b/ipc/cli/src/commands/crossmsg/topdown_cross.rs @@ -72,7 +72,7 @@ impl CommandLineHandler for LatestParentFinality { let provider = get_ipc_provider(global)?; let subnet = SubnetID::from_str(&arguments.subnet)?; - println!("{}", provider.latest_parent_finality(&subnet).await?); + println!("{}", provider.latest_topdown_checkpoint(&subnet).await?); Ok(()) } } diff --git a/ipc/provider/src/lib.rs b/ipc/provider/src/lib.rs index 862a40cbe..af41e6563 100644 --- a/ipc/provider/src/lib.rs +++ b/ipc/provider/src/lib.rs @@ -735,11 +735,11 @@ impl IpcProvider { conn.manager().list_bootstrap_nodes(subnet).await } - /// Returns the latest finality from the parent committed in a child subnet. - pub async fn latest_parent_finality(&self, subnet: &SubnetID) -> anyhow::Result { + /// Returns the latest topdown checkpoint from the parent committed in a child subnet. + pub async fn latest_topdown_checkpoint(&self, subnet: &SubnetID) -> anyhow::Result { let conn = self.get_connection(subnet)?; - conn.manager().latest_parent_finality().await + conn.manager().latest_topdown_checkpoint().await } pub async fn set_federated_power( diff --git a/ipc/provider/src/manager/evm/manager.rs b/ipc/provider/src/manager/evm/manager.rs index 46e3228e3..3f26dbce4 100644 --- a/ipc/provider/src/manager/evm/manager.rs +++ b/ipc/provider/src/manager/evm/manager.rs @@ -241,14 +241,14 @@ impl TopDownFinalityQuery for EthSubnetManager { }) } - async fn latest_parent_finality(&self) -> Result { + async fn latest_topdown_checkpoint(&self) -> Result { tracing::info!("querying latest parent finality "); let contract = gateway_getter_facet::GatewayGetterFacet::new( self.ipc_contract_info.gateway_addr, Arc::new(self.ipc_contract_info.provider.clone()), ); - let finality = contract.get_latest_parent_finality().call().await?; + let finality = contract.get_latest_topdown_checkpoint().call().await?; Ok(finality.height.as_u64() as ChainEpoch) } } diff --git a/ipc/provider/src/manager/subnet.rs b/ipc/provider/src/manager/subnet.rs index d7715550e..e3e01a52d 100644 --- a/ipc/provider/src/manager/subnet.rs +++ b/ipc/provider/src/manager/subnet.rs @@ -253,8 +253,8 @@ pub trait TopDownFinalityQuery: Send + Sync { subnet_id: &SubnetID, epoch: ChainEpoch, ) -> Result>>; - /// Returns the latest parent finality committed in a child subnet - async fn latest_parent_finality(&self) -> Result; + /// Returns the latest topdown checkpoint committed in a child subnet + async fn latest_topdown_checkpoint(&self) -> Result; } /// The bottom up checkpoint manager that handles the bottom up relaying from child subnet to the parent diff --git a/ipld/resolver/src/behaviour/membership.rs b/ipld/resolver/src/behaviour/membership.rs index ae3d5c955..fa62323e9 100644 --- a/ipld/resolver/src/behaviour/membership.rs +++ b/ipld/resolver/src/behaviour/membership.rs @@ -5,13 +5,12 @@ use std::marker::PhantomData; use std::task::{Context, Poll}; use std::time::Duration; -use super::NetworkConfig; use crate::hash::blake2b_256; -use crate::observe; use crate::provider_cache::{ProviderDelta, SubnetProviderCache}; use crate::provider_record::{ProviderRecord, SignedProviderRecord}; -use crate::vote_record::{SignedVoteRecord, VoteRecord}; +use crate::vote_record::SubnetVoteRecord; use crate::Timestamp; +use crate::{observe, NetworkConfig}; use anyhow::anyhow; use ipc_api::subnet_id::SubnetID; use ipc_observability::emit; @@ -53,8 +52,8 @@ pub enum Event { /// to trigger a lookup by the discovery module to learn the address. Skipped(PeerId), - /// We received a [`VoteRecord`] in one of the subnets we are providing data for. - ReceivedVote(Box>), + /// We received a vote in one of the subnets we are providing data for. + ReceivedVote(Box), /// We received preemptive data published in a subnet we were interested in. ReceivedPreemptive(SubnetID, Vec), @@ -343,9 +342,9 @@ where } /// Publish the vote of the validator running the agent about a CID to a subnet. - pub fn publish_vote(&mut self, vote: SignedVoteRecord) -> anyhow::Result<()> { - let topic = self.voting_topic(&vote.record().subnet_id); - let data = vote.into_envelope().into_protobuf_encoding(); + pub fn publish_vote(&mut self, vote: SubnetVoteRecord) -> anyhow::Result<()> { + let topic = self.voting_topic(&vote.subnet); + let data = fvm_ipld_encoding::to_vec(&vote.vote)?; match self.inner.publish(topic, data) { Err(e) => { emit(observe::MembershipFailureEvent::PublishFailure( @@ -421,7 +420,7 @@ where ), } } else if self.voting_topics.contains(&msg.topic) { - match SignedVoteRecord::from_bytes(&msg.data).map(|r| r.into_record()) { + match fvm_ipld_encoding::from_slice(&msg.data) { Ok(record) => self.handle_vote_record(record), Err(e) => emit(observe::MembershipFailureEvent::GossipInvalidVoteRecord( msg.source, @@ -466,7 +465,7 @@ where } /// Raise an event to tell we received a new vote. - fn handle_vote_record(&mut self, record: VoteRecord) { + fn handle_vote_record(&mut self, record: V) { self.outbox.push_back(Event::ReceivedVote(Box::new(record))) } diff --git a/ipld/resolver/src/client.rs b/ipld/resolver/src/client.rs index 29e9eac55..2e62311ca 100644 --- a/ipld/resolver/src/client.rs +++ b/ipld/resolver/src/client.rs @@ -7,10 +7,8 @@ use libipld::Cid; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::oneshot; -use crate::{ - service::{Request, ResolveResult}, - vote_record::SignedVoteRecord, -}; +use crate::service::{Request, ResolveResult}; +use crate::vote_record::SubnetVoteRecord; /// A facade to the [`Service`] to provide a nicer interface than message passing would allow on its own. #[derive(Clone)] @@ -71,7 +69,7 @@ impl Client { } /// Publish a signed vote into a topic based on its subnet. - pub fn publish_vote(&self, vote: SignedVoteRecord) -> anyhow::Result<()> { + pub fn publish_vote(&self, vote: SubnetVoteRecord) -> anyhow::Result<()> { let req = Request::PublishVote(Box::new(vote)); self.send_request(req) } diff --git a/ipld/resolver/src/lib.rs b/ipld/resolver/src/lib.rs index 3d54127b3..a48a4d7b5 100644 --- a/ipld/resolver/src/lib.rs +++ b/ipld/resolver/src/lib.rs @@ -23,4 +23,4 @@ pub use behaviour::{ContentConfig, DiscoveryConfig, MembershipConfig, NetworkCon pub use client::{Client, Resolver}; pub use service::{Config, ConnectionConfig, Event, NoKnownPeers, Service}; pub use timestamp::Timestamp; -pub use vote_record::{ValidatorKey, VoteRecord}; +pub use vote_record::{SubnetVoteRecord, ValidatorKey}; diff --git a/ipld/resolver/src/service.rs b/ipld/resolver/src/service.rs index f5a1bb65e..627f1e67b 100644 --- a/ipld/resolver/src/service.rs +++ b/ipld/resolver/src/service.rs @@ -9,7 +9,6 @@ use crate::behaviour::{ }; use crate::client::Client; use crate::observe; -use crate::vote_record::{SignedVoteRecord, VoteRecord}; use anyhow::anyhow; use bloom::{BloomFilter, ASMS}; use ipc_api::subnet_id::SubnetID; @@ -36,6 +35,8 @@ use tokio::sync::broadcast; use tokio::sync::mpsc; use tokio::sync::oneshot::{self, Sender}; +use crate::vote_record::SubnetVoteRecord; + /// Result of attempting to resolve a CID. pub type ResolveResult = anyhow::Result<()>; @@ -91,7 +92,7 @@ pub(crate) enum Request { SetProvidedSubnets(Vec), AddProvidedSubnet(SubnetID), RemoveProvidedSubnet(SubnetID), - PublishVote(Box>), + PublishVote(Box>), PublishPreemptive(SubnetID, Vec), PinSubnet(SubnetID), UnpinSubnet(SubnetID), @@ -105,7 +106,7 @@ pub(crate) enum Request { #[derive(Clone, Debug)] pub enum Event { /// Received a vote about in a subnet about a CID. - ReceivedVote(Box>), + ReceivedVote(Box), /// Received raw pre-emptive data published to a pinned subnet. ReceivedPreemptive(SubnetID, Vec), } diff --git a/ipld/resolver/src/signed_record.rs b/ipld/resolver/src/signed_record.rs index 82e31d352..a29f46da9 100644 --- a/ipld/resolver/src/signed_record.rs +++ b/ipld/resolver/src/signed_record.rs @@ -65,10 +65,12 @@ where Ok(signed_record) } + #[allow(dead_code)] pub fn record(&self) -> &R { &self.record } + #[allow(dead_code)] pub fn envelope(&self) -> &SignedEnvelope { &self.envelope } diff --git a/ipld/resolver/src/vote_record.rs b/ipld/resolver/src/vote_record.rs index 3678d5e47..62ed27cc9 100644 --- a/ipld/resolver/src/vote_record.rs +++ b/ipld/resolver/src/vote_record.rs @@ -95,6 +95,11 @@ impl Record for VoteRecord { pub type SignedVoteRecord = SignedRecord>; +pub struct SubnetVoteRecord { + pub subnet: SubnetID, + pub vote: V, +} + impl VoteRecord where C: Serialize + DeserializeOwned, diff --git a/ipld/resolver/tests/smoke.rs b/ipld/resolver/tests/smoke.rs index 632df8dd2..40636a857 100644 --- a/ipld/resolver/tests/smoke.rs +++ b/ipld/resolver/tests/smoke.rs @@ -30,7 +30,7 @@ use fvm_shared::{address::Address, ActorID}; use ipc_api::subnet_id::SubnetID; use ipc_ipld_resolver::{ Client, Config, ConnectionConfig, ContentConfig, DiscoveryConfig, Event, MembershipConfig, - NetworkConfig, Resolver, Service, VoteRecord, + NetworkConfig, Resolver, Service, SubnetVoteRecord, }; use libp2p::{ core::{ @@ -221,15 +221,16 @@ async fn single_bootstrap_publish_receive_vote() { tokio::time::sleep(Duration::from_secs(2)).await; // Vote on some random CID. - let validator_key = Keypair::generate_secp256k1(); let cid = Cid::new_v1(IPLD_RAW, Code::Sha2_256.digest(b"foo")); - let vote = - VoteRecord::signed(&validator_key, subnet_id, TestVote(cid)).expect("failed to sign vote"); + let vote = TestVote(cid); // Pubilish vote cluster.agents[0] .client - .publish_vote(vote.clone()) + .publish_vote(SubnetVoteRecord { + vote: vote.clone(), + subnet: subnet_id.clone(), + }) .expect("failed to send vote"); // Receive vote. @@ -239,7 +240,7 @@ async fn single_bootstrap_publish_receive_vote() { .expect("error receiving vote"); if let Event::ReceivedVote(v) = event { - assert_eq!(&*v, vote.record()); + assert_eq!(*v, vote); } else { panic!("unexpected {event:?}") }