feat(contract-distribution): Merge intermediate versions of StoredChu…

…nkStateTransitionData (#12397)

We have added 3 versions of `StoredChunkStateTransitionData` for
contract distribution, where V1 and V2 are added while evolving the
implementation and only V3 will be used now. This PR addresses a TODO to
implement a migration to merge these three versions into a single one
(V1) to simplify the format and code. Since the column is stored only
for few blocks (the ones before the latest final block are deleted), the
migration should be pretty quick to run.

After this change, the code will only deal with
`StoredChunkStateTransitionData::V1`.

Also add contract accesses and deploys to entity-debug info while
modifying the related code (this is the reason for adding `Serialize` to
the types).

chain/chain/src/stateless_validation/state_transition_data.rs

@@ -116,7 +116,7 @@ mod tests {
     use near_primitives::block_header::{BlockHeader, BlockHeaderInnerLite, BlockHeaderV4};
     use near_primitives::hash::{hash, CryptoHash};
     use near_primitives::stateless_validation::stored_chunk_state_transition_data::{
-        StoredChunkStateTransitionData, StoredChunkStateTransitionDataV3,
+        StoredChunkStateTransitionData, StoredChunkStateTransitionDataV1,
     };
     use near_primitives::types::{BlockHeight, EpochId, ShardId};
     use near_primitives::utils::{get_block_shard_id, get_block_shard_id_rev, index_to_bytes};
@@ -204,7 +204,7 @@ mod tests {
             .set_ser(
                 DBCol::StateTransitionData,
                 &get_block_shard_id(&block_hash, shard_id),
-                &StoredChunkStateTransitionData::V3(StoredChunkStateTransitionDataV3 {
+                &StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
                     base_state: Default::default(),
                     receipts_hash: Default::default(),
                     contract_accesses: Default::default(),

chain/chain/src/store/mod.rs

@@ -25,7 +25,7 @@ use near_primitives::state_sync::{
 };
 use near_primitives::stateless_validation::contract_distribution::ContractUpdates;
 use near_primitives::stateless_validation::stored_chunk_state_transition_data::{
-    StoredChunkStateTransitionData, StoredChunkStateTransitionDataV3,
+    StoredChunkStateTransitionData, StoredChunkStateTransitionDataV1,
 };
 use near_primitives::transaction::{
     ExecutionOutcomeWithId, ExecutionOutcomeWithIdAndProof, ExecutionOutcomeWithProof,
@@ -2026,7 +2026,7 @@ impl<'a> ChainStoreUpdate<'a> {
             let ContractUpdates { contract_accesses, contract_deploys } = contract_updates;
             self.state_transition_data.insert(
                 (block_hash, shard_id),
-                StoredChunkStateTransitionData::V3(StoredChunkStateTransitionDataV3 {
+                StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
                     base_state: partial_storage.nodes,
                     receipts_hash: applied_receipts_hash,
                     contract_accesses: contract_accesses.into_iter().collect(),

chain/client/src/stateless_validation/state_witness_producer.rs

@@ -16,7 +16,6 @@ use near_primitives::stateless_validation::state_witness::{
 };
 use near_primitives::stateless_validation::stored_chunk_state_transition_data::{
     StoredChunkStateTransitionData, StoredChunkStateTransitionDataV1,
-    StoredChunkStateTransitionDataV2, StoredChunkStateTransitionDataV3,
 };
 use near_primitives::types::{AccountId, EpochId, ShardId};
 use near_primitives::validator_signer::ValidatorSigner;
@@ -276,26 +275,12 @@ impl Client {
                 }
                 Error::Other(message)
             })?;
-        let (base_state, receipts_hash, contract_accesses, contract_deploys) =
-            match stored_chunk_state_transition_data {
-                StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
-                    base_state,
-                    receipts_hash,
-                    contract_accesses,
-                })
-                | StoredChunkStateTransitionData::V2(StoredChunkStateTransitionDataV2 {
-                    base_state,
-                    receipts_hash,
-                    contract_accesses,
-                    ..
-                }) => (base_state, receipts_hash, contract_accesses, Default::default()),
-                StoredChunkStateTransitionData::V3(StoredChunkStateTransitionDataV3 {
-                    base_state,
-                    receipts_hash,
-                    contract_accesses,
-                    contract_deploys,
-                }) => (base_state, receipts_hash, contract_accesses, contract_deploys),
-            };
+        let StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
+            base_state,
+            receipts_hash,
+            contract_accesses,
+            contract_deploys,
+        }) = stored_chunk_state_transition_data;
         let contract_updates = ContractUpdates {
             contract_accesses: contract_accesses.into_iter().collect(),
             contract_deploys: contract_deploys.into_iter().map(|c| c.into()).collect(),

core/primitives/src/stateless_validation/contract_distribution.rs

@@ -5,7 +5,7 @@ use borsh::{BorshDeserialize, BorshSerialize};
 use bytesize::ByteSize;
 use near_crypto::{PublicKey, Signature};
 use near_primitives_core::code::ContractCode;
-use near_primitives_core::hash::CryptoHash;
+use near_primitives_core::hash::{hash, CryptoHash};
 use near_primitives_core::types::{AccountId, ShardId};
 use near_schema_checker_lib::ProtocolSchema;
 
@@ -319,6 +319,8 @@ impl
     PartialOrd,
     BorshSerialize,
     BorshDeserialize,
+    serde::Serialize,
+    serde::Deserialize,
     ProtocolSchema,
 )]
 pub struct CodeHash(pub CryptoHash);
@@ -336,9 +338,25 @@ impl Into<CryptoHash> for CodeHash {
 }
 
 /// Raw bytes of the (uncompiled) contract code.
-#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    BorshSerialize,
+    BorshDeserialize,
+    serde::Serialize,
+    serde::Deserialize,
+    ProtocolSchema,
+)]
 pub struct CodeBytes(pub Arc<[u8]>);
 
+impl CodeBytes {
+    pub fn hash(&self) -> CodeHash {
+        hash(self.0.as_ref()).into()
+    }
+}
+
 impl From<ContractCode> for CodeBytes {
     fn from(code: ContractCode) -> Self {
         Self(code.take_code().into())

core/primitives/src/stateless_validation/stored_chunk_state_transition_data.rs

@@ -7,21 +7,15 @@ use super::contract_distribution::{CodeBytes, CodeHash};
 
 /// Stored on disk for each chunk, including missing chunks, in order to
 /// produce a chunk state witness when needed.
-// TODO(#11099): Implement migration to combine these into single version
-// after the feature of excluding contracts from state witness is stabilized.
 #[derive(Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
 pub enum StoredChunkStateTransitionData {
     V1(StoredChunkStateTransitionDataV1),
-    V2(StoredChunkStateTransitionDataV2),
-    V3(StoredChunkStateTransitionDataV3),
 }
 
 impl StoredChunkStateTransitionData {
     pub fn contract_accesses(&self) -> &[CodeHash] {
         match &self {
-            Self::V1(_) => &[],
-            Self::V2(v2) => &v2.contract_accesses,
-            Self::V3(v3) => &v3.contract_accesses,
+            Self::V1(v1) => &v1.contract_accesses,
         }
     }
 }
@@ -39,38 +33,6 @@ pub struct StoredChunkStateTransitionDataV1 {
     pub receipts_hash: CryptoHash,
     /// The code-hashes of the contracts that are accessed (called) during the state transition.
     pub contract_accesses: Vec<CodeHash>,
-}
-
-#[derive(Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
-pub struct StoredChunkStateTransitionDataV2 {
-    /// The partial state that is needed to apply the state transition,
-    /// whether it is a new chunk state transition or a implicit missing chunk
-    /// state transition.
-    pub base_state: PartialState,
-    /// If this is a new chunk state transition, the hash of the receipts that
-    /// were used to apply the state transition. This is redundant information,
-    /// but is used to validate against `StateChunkWitness::exact_receipts_hash`
-    /// to ease debugging of why a state witness may be incorrect.
-    pub receipts_hash: CryptoHash,
-    /// The code-hashes of the contracts that are accessed (called) during the state transition.
-    pub contract_accesses: Vec<CodeHash>,
-    /// The code-hashes of the contracts that are deployed during the state transition.
-    pub contract_deploys: Vec<CodeHash>,
-}
-
-#[derive(Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
-pub struct StoredChunkStateTransitionDataV3 {
-    /// The partial state that is needed to apply the state transition,
-    /// whether it is a new chunk state transition or a implicit missing chunk
-    /// state transition.
-    pub base_state: PartialState,
-    /// If this is a new chunk state transition, the hash of the receipts that
-    /// were used to apply the state transition. This is redundant information,
-    /// but is used to validate against `StateChunkWitness::exact_receipts_hash`
-    /// to ease debugging of why a state witness may be incorrect.
-    pub receipts_hash: CryptoHash,
-    /// The code-hashes of the contracts that are accessed (called) during the state transition.
-    pub contract_accesses: Vec<CodeHash>,
     /// Contracts that are deployed during the state transition.
     pub contract_deploys: Vec<CodeBytes>,
 }

core/store/src/metadata.rs

@@ -2,7 +2,7 @@
 pub type DbVersion = u32;
 
 /// Current version of the database.
-pub const DB_VERSION: DbVersion = 42;
+pub const DB_VERSION: DbVersion = 43;
 
 /// Database version at which point DbKind was introduced.
 const DB_VERSION_WITH_KIND: DbVersion = 34;

core/store/src/migrations.rs

@@ -1,13 +1,14 @@
 use crate::metadata::DbKind;
 use crate::{DBCol, Store, StoreUpdate};
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use borsh::{BorshDeserialize, BorshSerialize};
 use near_primitives::challenge::PartialState;
 use near_primitives::epoch_manager::EpochSummary;
 use near_primitives::epoch_manager::AGGREGATOR_KEY;
 use near_primitives::hash::CryptoHash;
 use near_primitives::sharding::{ShardInfo, StateSyncInfo, StateSyncInfoV0};
 use near_primitives::state::FlatStateValue;
+use near_primitives::stateless_validation::contract_distribution::{CodeBytes, CodeHash};
 use near_primitives::stateless_validation::stored_chunk_state_transition_data::{
     StoredChunkStateTransitionData, StoredChunkStateTransitionDataV1,
 };
@@ -17,7 +18,7 @@ use near_primitives::types::{
     ValidatorKickoutReason, ValidatorStats,
 };
 use near_primitives::types::{BlockChunkValidatorStats, ChunkStats};
-use near_primitives::utils::get_outcome_id_block_hash;
+use near_primitives::utils::{get_block_shard_id_rev, get_outcome_id_block_hash};
 use near_primitives::version::ProtocolVersion;
 use std::collections::{BTreeMap, HashMap};
 use tracing::info;
@@ -365,7 +366,8 @@ pub fn migrate_39_to_40(store: &Store) -> anyhow::Result<()> {
 
 /// Migrates the database from version 40 to 41.
 ///
-/// The migraton replaces non-enum StoredChunkStateTransitionData struct with its enum version.
+/// The migraton replaces non-enum StoredChunkStateTransitionData struct with its enum version V1.
+/// NOTE: The data written by this migration is overriden by migrate_42_to_43 to a different format.
 pub fn migrate_40_to_41(store: &Store) -> anyhow::Result<()> {
     #[derive(BorshDeserialize)]
     pub struct DeprecatedStoredChunkStateTransitionData {
@@ -380,12 +382,13 @@ pub fn migrate_40_to_41(store: &Store) -> anyhow::Result<()> {
         let (key, old_value) = result?;
         let DeprecatedStoredChunkStateTransitionData { base_state, receipts_hash } =
             DeprecatedStoredChunkStateTransitionData::try_from_slice(&old_value)?;
-        let new_value =
-            borsh::to_vec(&StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
+        let new_value = borsh::to_vec(&DeprecatedStoredChunkStateTransitionDataEnum::V1(
+            DeprecatedStoredChunkStateTransitionDataV1 {
                 base_state,
                 receipts_hash,
                 contract_accesses: Default::default(),
-            }))?;
+            },
+        ))?;
         update.set(DBCol::StateTransitionData, &key, &new_value);
     }
     update.commit()?;
@@ -422,3 +425,89 @@ pub fn migrate_41_to_42(store: &Store) -> anyhow::Result<()> {
     update.commit()?;
     Ok(())
 }
+
+#[derive(BorshSerialize, BorshDeserialize)]
+enum DeprecatedStoredChunkStateTransitionDataEnum {
+    V1(DeprecatedStoredChunkStateTransitionDataV1),
+    V2(DeprecatedStoredChunkStateTransitionDataV2),
+    V3(DeprecatedStoredChunkStateTransitionDataV3),
+}
+
+#[derive(BorshSerialize, BorshDeserialize)]
+struct DeprecatedStoredChunkStateTransitionDataV1 {
+    base_state: PartialState,
+    receipts_hash: CryptoHash,
+    contract_accesses: Vec<CodeHash>,
+}
+
+#[derive(BorshSerialize, BorshDeserialize)]
+struct DeprecatedStoredChunkStateTransitionDataV2 {
+    base_state: PartialState,
+    receipts_hash: CryptoHash,
+    contract_accesses: Vec<CodeHash>,
+    // This field is ignored since it only contains code hashes.
+    _contract_deploys: Vec<CodeHash>,
+}
+
+#[derive(BorshSerialize, BorshDeserialize)]
+struct DeprecatedStoredChunkStateTransitionDataV3 {
+    base_state: PartialState,
+    receipts_hash: CryptoHash,
+    contract_accesses: Vec<CodeHash>,
+    contract_deploys: Vec<CodeBytes>,
+}
+
+/// Migrates the database from version 42 to 43.
+///
+/// Merges versions V1-V3 of StoredChunkStateTransitionData into a single version.
+pub fn migrate_42_to_43(store: &Store) -> anyhow::Result<()> {
+    let _span =
+        tracing::info_span!(target: "migrations", "Merging versions V1-V3 of StoredChunkStateTransitionData into single version").entered();
+    let mut update = store.store_update();
+    for result in store.iter(DBCol::StateTransitionData) {
+        let (key, old_value) = result?;
+
+        let old_data = DeprecatedStoredChunkStateTransitionDataEnum::try_from_slice(&old_value).map_err(|err| {
+            if let Ok((block_hash, shard_id)) = get_block_shard_id_rev(&key) {
+                anyhow!("Failed to parse StoredChunkStateTransitionData in DB. Block: {:?}, Shard: {:?}, Error: {:?}", block_hash, shard_id, err)
+            } else {
+                anyhow!("Failed to parse StoredChunkStateTransitionData in DB. Key: {:?}, Error: {:?}", key, err)
+            }
+        })?;
+        let (base_state, receipts_hash, contract_accesses, contract_deploys) = match old_data {
+            DeprecatedStoredChunkStateTransitionDataEnum::V1(
+                DeprecatedStoredChunkStateTransitionDataV1 {
+                    base_state,
+                    receipts_hash,
+                    contract_accesses,
+                },
+            )
+            | DeprecatedStoredChunkStateTransitionDataEnum::V2(
+                DeprecatedStoredChunkStateTransitionDataV2 {
+                    base_state,
+                    receipts_hash,
+                    contract_accesses,
+                    ..
+                },
+            ) => (base_state, receipts_hash, contract_accesses, vec![]),
+            DeprecatedStoredChunkStateTransitionDataEnum::V3(
+                DeprecatedStoredChunkStateTransitionDataV3 {
+                    base_state,
+                    receipts_hash,
+                    contract_accesses,
+                    contract_deploys,
+                },
+            ) => (base_state, receipts_hash, contract_accesses, contract_deploys),
+        };
+        let new_value =
+            borsh::to_vec(&StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
+                base_state,
+                receipts_hash,
+                contract_accesses,
+                contract_deploys,
+            }))?;
+        update.set(DBCol::StateTransitionData, &key, &new_value);
+    }
+    update.commit()?;
+    Ok(())
+}

nearcore/src/entity_debug.rs

@@ -25,7 +25,6 @@ use near_primitives::state::FlatStateValue;
 use near_primitives::state_sync::StateSyncDumpProgress;
 use near_primitives::stateless_validation::stored_chunk_state_transition_data::{
     StoredChunkStateTransitionData, StoredChunkStateTransitionDataV1,
-    StoredChunkStateTransitionDataV2, StoredChunkStateTransitionDataV3,
 };
 use near_primitives::transaction::{ExecutionOutcomeWithProof, SignedTransaction};
 use near_primitives::types::chunk_extra::ChunkExtra;
@@ -324,29 +323,23 @@ impl EntityDebugHandlerImpl {
                             &get_block_shard_id(&block_hash, shard_id),
                         )?
                         .ok_or_else(|| anyhow!("State transition not found"))?;
-                    let (base_state, receipts_hash) = match state_transition {
-                        StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
-                            base_state,
-                            receipts_hash,
-                            ..
-                        })
-                        | StoredChunkStateTransitionData::V2(StoredChunkStateTransitionDataV2 {
-                            base_state,
-                            receipts_hash,
-                            ..
-                        })
-                        | StoredChunkStateTransitionData::V3(StoredChunkStateTransitionDataV3 {
-                            base_state,
-                            receipts_hash,
-                            ..
-                        }) => (base_state, receipts_hash),
-                    };
+                    let StoredChunkStateTransitionData::V1(StoredChunkStateTransitionDataV1 {
+                        base_state,
+                        receipts_hash,
+                        contract_accesses,
+                        contract_deploys,
+                    }) = state_transition;
                     let mut serialized = EntityDataStruct::new();
                     serialized.add(
                         "base_state",
                         PartialStateParser::parse_and_serialize_partial_state(base_state),
                     );
                     serialized.add("receipts_hash", serialize_entity(&receipts_hash));
+                    serialized.add("contract_accesses", serialize_entity(&contract_accesses));
+                    // Add the hash of the deployed contract code instead of the actual code.
+                    let contract_deploy_hashes =
+                        contract_deploys.into_iter().map(|code| code.hash()).collect::<Vec<_>>();
+                    serialized.add("contract_deploys", serialize_entity(&contract_deploy_hashes));
                     state_transitions
                         .add(&shard_id.to_string(), EntityDataValue::Struct(serialized.into()));
                 }

nearcore/src/migrations.rs

@@ -88,6 +88,7 @@ impl<'a> near_store::StoreMigrator for Migrator<'a> {
             39 => near_store::migrations::migrate_39_to_40(store),
             40 => near_store::migrations::migrate_40_to_41(store),
             41 => near_store::migrations::migrate_41_to_42(store),
+            42 => near_store::migrations::migrate_42_to_43(store),
             DB_VERSION.. => unreachable!(),
         }
     }