tests: 🏃 mock consensus can exercise staking component (#4001)

fixes #3966. fixes #3908. fixes _part of_ #3995.

this branch introduces the first steps towards mock consensus (#3588)
testing of the staking component (#3845).

this defines a validator after genesis, and then shows that it does
_not_ enter the consensus set. #3966 is addressed in this branch so that
the existing genesis validator correctly enters the consensus set, and
so that we can successfully progress to the second epoch.

subsequent changes will exercise delegating to this validator in the
`mock_consensus_can_define_and_delegate_to_a_validator`.

#### ✨ changes

* alters `with_penumbra_auto_app_state` so that it adds an allocation of
delegation tokens to the shielded pool component's genesis content.

* extends `generate_penumbra_validator` so that it generates a real
spend key, and returns an `Allocation` for the generated validator.
_(see #3966)_

* adds a new `mock_consensus_can_define_and_delegate_to_a_validator`
test that defines a post-genesis validator. _(see #3908)_

* defines a new `ConsensusIndexRead::get_consensus_set()` method, which
collects all of the identity keys returned by `consensus_set_stream`.

* lowers the events in
`penumbra_mock_consensus::block::Builder::execute()` to trace-level
events.

* `penumbra_mock_consensus::builder::Builder` will now log a warning if
values may be errantly rewritten by the builder methods.

* `TestNode::fast_forward` sets its `i` span field to `1..n`, rather
than `0..n-1`.

---

#### :link: related

* #4009
* #4010
* #4011
* #4017
* #4027
* #4028
* #4029
* #3966
* #3908
* #3588

---------

Co-authored-by: Henry de Valence <[email protected]>

crates/core/app/tests/common/mod.rs

@@ -1,7 +1,7 @@
 //! Shared integration testing facilities.
 
-// NB: Allow dead code, these are in fact shared by files in `tests/`.
-#![allow(dead_code)]
+// NB: Allow dead code, and unused imports. these are shared and consumed by files in `tests/`.
+#![allow(dead_code, unused_imports)]
 
 pub use self::test_node_builder_ext::BuilderExt;
 

crates/core/app/tests/common/test_node_builder_ext.rs

@@ -5,6 +5,8 @@ use {
         core::keys::v1::{GovernanceKey, IdentityKey},
         penumbra::core::component::stake::v1::Validator as PenumbraValidator,
     },
+    penumbra_shielded_pool::genesis::Allocation,
+    tracing::trace,
 };
 
 /// Penumbra-specific extensions to the mock consensus builder.
@@ -21,75 +23,59 @@ impl BuilderExt for Builder {
     type Error = anyhow::Error;
     fn with_penumbra_auto_app_state(self, app_state: AppState) -> Result<Self, Self::Error> {
         let Self { keyring, .. } = &self;
-
-        let app_state = if keyring.is_empty() {
-            // If there are no consensus keys to inject, pass along the provided app state.
-            app_state
-        } else {
-            // Otherwise, generate a penumbra validator for each entry in the keyring...
-            let validators = keyring
-                .verification_keys()
-                .cloned()
-                .map(generate_penumbra_validator)
-                .inspect(log_validator)
-                .collect::<Vec<_>>();
-            // ...and then inject these validators into the app state.
-            inject_penumbra_validators(app_state, validators)?
+        let mut content = match app_state {
+            AppState::Content(c) => c,
+            AppState::Checkpoint(_) => anyhow::bail!("checkpointed state is not supported"),
         };
 
+        for (consensus_vk, _) in keyring {
+            // Generate a penumbra validator with this consensus key, and a corresponding
+            // allocation of delegation tokens.
+            let (validator, allocation) = generate_penumbra_validator(consensus_vk);
+
+            // Add the validator to the staking component's genesis content.
+            trace!(?validator, "adding validator to staking genesis content");
+            content.stake_content.validators.push(validator);
+
+            // Add an allocation of delegation tokens to the shielded pool content.
+            trace!(
+                ?allocation,
+                "adding allocation to shielded pool genesis content"
+            );
+            content.shielded_pool_content.allocations.push(allocation);
+        }
+
         // Serialize the app state into bytes, and add it to the builder.
+        let app_state = AppState::Content(content);
         serde_json::to_vec(&app_state)
             .map_err(Self::Error::from)
             .map(|s| self.app_state(s))
     }
 }
 
-/// Injects the given collection of [`Validator`s][PenumbraValidator] into the app state.
-fn inject_penumbra_validators(
-    app_state: AppState,
-    validators: Vec<PenumbraValidator>,
-) -> Result<AppState, anyhow::Error> {
-    use AppState::{Checkpoint, Content};
-    match app_state {
-        Checkpoint(_) => anyhow::bail!("checkpoint app state isn't supported"),
-        Content(mut content) => {
-            // Inject the builder's validators into the staking component's genesis state...
-            let overwritten = std::mem::replace(&mut content.stake_content.validators, validators);
-            // ...and log a warning if this overwrote any validators already in the app state.
-            if !overwritten.is_empty() {
-                tracing::warn!(
-                    ?overwritten,
-                    "`with_penumbra_auto_app_state` overwrote validators in the given AppState"
-                )
-            }
-            Ok(Content(content))
-        }
-    }
-}
-
 /// Generates a [`Validator`][PenumbraValidator] given a consensus verification key.
 fn generate_penumbra_validator(
-    consensus_key: ed25519_consensus::VerificationKey,
-) -> PenumbraValidator {
-    /// A temporary stub for validator keys.
-    ///
-    /// An invalid key is intentionally provided here, until we have test coverage exercising the
-    /// use of these keys. Once we need it we will:
-    /// - generate a random signing key
-    /// - get its verification key
-    /// - use that for the identity key
-    /// - throw the signing key away
-    ///
-    /// NB: for now, we will use the same key for governance. See the documentation of
-    /// `GovernanceKey` for more information about cold storage of validator keys.
-    const INVALID_KEY_BYTES: [u8; 32] = [0; 32];
+    consensus_key: &ed25519_consensus::VerificationKey,
+) -> (PenumbraValidator, Allocation) {
+    use decaf377_rdsa::VerificationKey;
+    use penumbra_keys::keys::{SpendKey, SpendKeyBytes};
+    use penumbra_stake::DelegationToken;
+    use rand::Rng;
+    use rand_core::OsRng;
 
-    PenumbraValidator {
+    let seed = SpendKeyBytes(OsRng.gen());
+    let spend_key = SpendKey::from(seed.clone());
+    let validator_id_sk = spend_key.spend_auth_key();
+    let validator_id_vk = VerificationKey::from(validator_id_sk);
+
+    let v = PenumbraValidator {
         identity_key: Some(IdentityKey {
-            ik: INVALID_KEY_BYTES.to_vec().clone(),
+            ik: validator_id_vk.to_bytes().to_vec(),
         }),
+        // NB: for now, we will use the same key for governance. See the documentation of
+        // `GovernanceKey` for more information about cold storage of validator keys.
         governance_key: Some(GovernanceKey {
-            gk: INVALID_KEY_BYTES.to_vec().clone(),
+            gk: validator_id_vk.to_bytes().to_vec(),
         }),
         consensus_key: consensus_key.as_bytes().to_vec(),
         enabled: true,
@@ -98,7 +84,23 @@ fn generate_penumbra_validator(
         website: String::default(),
         description: String::default(),
         funding_streams: Vec::default(),
-    }
+    };
+
+    let (address, _) = spend_key
+        .full_viewing_key()
+        .incoming()
+        .payment_address(0u32.into());
+
+    let ik = penumbra_stake::IdentityKey(validator_id_vk);
+    let delegation_denom = DelegationToken::from(ik).denom();
+
+    let allocation = Allocation {
+        raw_amount: 1000u128.into(),
+        raw_denom: delegation_denom.to_string(),
+        address,
+    };
+
+    (v, allocation)
 }
 
 fn log_validator(

crates/core/app/tests/mock_consensus_staking.rs

@@ -0,0 +1,196 @@
+use {
+    self::common::BuilderExt,
+    anyhow::Context,
+    cnidarium::TempStorage,
+    decaf377_rdsa::{SigningKey, SpendAuth},
+    penumbra_app::server::consensus::Consensus,
+    penumbra_genesis::AppState,
+    penumbra_keys::test_keys,
+    penumbra_mock_client::MockClient,
+    penumbra_mock_consensus::TestNode,
+    penumbra_proto::DomainType,
+    penumbra_stake::{
+        component::validator_handler::ValidatorDataRead as _, validator::Validator, FundingStreams,
+        GovernanceKey, IdentityKey,
+    },
+    rand_core::OsRng,
+    tap::Tap,
+    tracing::{error_span, info, Instrument},
+};
+
+mod common;
+
+#[tokio::test]
+async fn mock_consensus_can_define_and_delegate_to_a_validator() -> anyhow::Result<()> {
+    // Install a test logger, acquire some temporary storage, and start the test node.
+    let guard = common::set_tracing_subscriber();
+    let storage = TempStorage::new().await?;
+
+    // Start the test node.
+    let mut node = {
+        let consensus = Consensus::new(storage.as_ref().clone());
+        let app_state = AppState::default();
+        TestNode::builder()
+            .single_validator()
+            .with_penumbra_auto_app_state(app_state)?
+            .init_chain(consensus)
+            .await
+    }?;
+
+    // Sync the mock client, using the test wallet's spend key, to the latest snapshot.
+    let client = MockClient::new(test_keys::SPEND_KEY.clone())
+        .with_sync_to_storage(&storage)
+        .await?
+        .tap(|c| info!(client.notes = %c.notes.len(), "mock client synced to test storage"));
+
+    // TODO(kate): get this number by querying the chain parameters.
+    const EPOCH_LENGTH: usize = 1000;
+
+    // Fast forward to the next epoch.
+    let snapshot_start = storage.latest_snapshot();
+    node.fast_forward(EPOCH_LENGTH)
+        .instrument(error_span!("fast forwarding test node to next epoch"))
+        .await
+        .context("fast forwarding {EPOCH_LENGTH} blocks")?;
+    let snapshot_end = storage.latest_snapshot();
+
+    // Retrieve the validator definition from the latest snapshot.
+    let existing_validator = match snapshot_end
+        .validator_definitions()
+        .tap(|_| info!("getting validator definitions"))
+        .await?
+        .as_slice()
+    {
+        [v] => v.clone(),
+        unexpected => panic!("there should be one validator, got: {unexpected:?}"),
+    };
+
+    let existing_validator_id = existing_validator.identity_key;
+
+    // Check that we are now in a new epoch.
+    {
+        use penumbra_sct::{component::clock::EpochRead, epoch::Epoch};
+        let Epoch { index: start, .. } = snapshot_start.get_current_epoch().await?;
+        let Epoch { index: end, .. } = snapshot_end.get_current_epoch().await?;
+        assert_eq!(start, 0, "we should start in the first epoch");
+        assert_eq!(end, 1, "we should now be in the second epoch");
+    }
+
+    // Show that the existing validator is and was active.
+    {
+        use penumbra_stake::{component::validator_handler::ValidatorDataRead, validator::State};
+        let start = snapshot_start
+            .get_validator_state(&existing_validator_id)
+            .await?;
+        let end = snapshot_end
+            .get_validator_state(&existing_validator_id)
+            .await?;
+        assert_eq!(start, Some(State::Active));
+        assert_eq!(end, Some(State::Active));
+    }
+
+    // Show that the validator was, and still is, in the consensus set.
+    {
+        use penumbra_stake::component::ConsensusIndexRead;
+        let start = snapshot_start.get_consensus_set().await?;
+        let end = snapshot_end.get_consensus_set().await?;
+        let expected = [existing_validator_id];
+        assert_eq!(
+            start, expected,
+            "validator should start in the consensus set"
+        );
+        assert_eq!(end, expected, "validator should stay in the consensus set");
+    }
+
+    // To define a validator, we need to define two keypairs: an identity key
+    // for the Penumbra application and a consensus key for cometbft.
+    let new_validator_id_sk = SigningKey::<SpendAuth>::new(OsRng);
+    let new_validator_id = IdentityKey(new_validator_id_sk.into());
+    let new_validator_consensus_sk = ed25519_consensus::SigningKey::new(OsRng);
+    let new_validator_consensus = new_validator_consensus_sk.verification_key();
+
+    // Insert the validator's consensus keypair into the keyring so it can be used to sign blocks.
+    /*
+    node.keyring_mut()
+        // Keyring should just be a BTreeMap rather than creating a new API
+        .insert(validator_consensus.clone(), validator_consensus_sk);
+     */
+
+    // Now define the validator's configuration data.
+    let new_validator = Validator {
+        identity_key: new_validator_id.clone(),
+        // TODO: upstream a direct conversion from ed25519_consensus
+        consensus_key: tendermint::PublicKey::from_raw_ed25519(&new_validator_consensus.to_bytes())
+            .expect("consensus key is valid"),
+        governance_key: GovernanceKey(new_validator_id_sk.into()),
+        enabled: true,
+        sequence_number: 0,
+        name: "test validator".to_string(),
+        website: String::default(),
+        description: String::default(),
+        funding_streams: FundingStreams::default(),
+    };
+
+    // Make a transaction that defines a new validator.
+    let plan = {
+        use {
+            penumbra_stake::validator,
+            penumbra_transaction::{ActionPlan, TransactionParameters, TransactionPlan},
+            rand_core::OsRng,
+        };
+        let bytes = new_validator.encode_to_vec();
+        let auth_sig = new_validator_id_sk.sign(OsRng, &bytes);
+        let action = ActionPlan::ValidatorDefinition(validator::Definition {
+            validator: new_validator.clone(),
+            auth_sig,
+        });
+        let mut plan = TransactionPlan {
+            actions: vec![action.into()],
+            // Now fill out the remaining parts of the transaction needed for verification:
+            memo: None,
+            detection_data: None, // We'll set this automatically below
+            transaction_parameters: TransactionParameters {
+                chain_id: TestNode::<()>::CHAIN_ID.to_string(),
+                ..Default::default()
+            },
+        };
+        plan.populate_detection_data(rand_core::OsRng, 0);
+        plan
+    };
+    let tx = client.witness_auth_build(&plan).await?;
+
+    // Execute the transaction, applying it to the chain state.
+    node.block().add_tx(tx.encode_to_vec()).execute().await?;
+    let post_tx_snapshot = storage.latest_snapshot();
+
+    // Show that the set of validators looks correct.
+    {
+        use penumbra_stake::{component::ConsensusIndexRead, validator::State};
+        let snapshot = post_tx_snapshot;
+        // The original validator should still be active.
+        assert_eq!(
+            snapshot.get_validator_state(&existing_validator_id).await?,
+            Some(State::Active),
+            "validator should be active"
+        );
+        // The new validator should be defined, but not yet active. It should not be inclueded in
+        // consensus yet.
+        assert_eq!(
+            snapshot.get_validator_state(&new_validator_id).await?,
+            Some(State::Defined),
+            "new validator definition should be defined but not active"
+        );
+        // The original validator should still be the only validator in the consensus set.
+        assert_eq!(
+            snapshot_start.get_consensus_set().await?.len(),
+            1,
+            "the new validator should not be part of the consensus set yet"
+        );
+    }
+
+    // The test passed. Free our temporary storage and drop our tracing subscriber.
+    Ok(())
+        .tap(|_| drop(node))
+        .tap(|_| drop(storage))
+        .tap(|_| drop(guard))
+}

crates/core/component/stake/src/component/action_handler/validator_definition.rs

@@ -1,14 +1,16 @@
+use crate::{
+    component::{
+        action_handler::ActionHandler, validator_handler::ValidatorDataRead,
+        validator_handler::ValidatorManager,
+    },
+    rate::RateData,
+    validator,
+};
 use anyhow::{ensure, Context, Result};
 use async_trait::async_trait;
 use cnidarium::StateWrite;
-
 use penumbra_proto::DomainType;
 
-use crate::{
-    component::action_handler::ActionHandler, component::validator_handler::ValidatorDataRead,
-    component::validator_handler::ValidatorManager, rate::RateData, validator,
-};
-
 #[async_trait]
 impl ActionHandler for validator::Definition {
     type CheckStatelessContext = ();