state_witness: do not include new transactions with each state witness (

near#12270)

First and foremost this PR introduces an integration test that attempts
to push through a chunk that has a few invalid transactions. Previously
this would fail and make the entire chunk invalid.

A new protocol feature is introduced that adjusts the code in few
strategic places to remove the requirement for all transactions within a
chunk to be valid, instead delegating the responsibility of checking all
aspects of transaction validity to the moment when transactions get
converted to receipts.

Any transactions that are found to be invalid are simply ignored. At
this point all validators should agree equally as to which transactions
should be converted and which ones should be discarded.

chain/chain/Cargo.toml

@@ -81,6 +81,10 @@ no_cache = ["near-store/no_cache"]
 protocol_feature_reject_blocks_with_outdated_protocol_version = [
   "near-primitives/protocol_feature_reject_blocks_with_outdated_protocol_version",
 ]
+protocol_feature_relaxed_chunk_validation = [
+  "near-primitives/protocol_feature_relaxed_chunk_validation",
+  "node-runtime/protocol_feature_relaxed_chunk_validation",
+]
 
 nightly = [
   "near-async/nightly",
@@ -98,6 +102,7 @@ nightly = [
   "nightly_protocol",
   "node-runtime/nightly",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",
+  "protocol_feature_relaxed_chunk_validation",
 ]
 nightly_protocol = [
   "near-async/nightly_protocol",

chain/chain/src/chain.rs

@@ -3198,6 +3198,17 @@ impl Chain {
         prev_block_header: &BlockHeader,
         chunk: &ShardChunk,
     ) -> Result<(), Error> {
+        let protocol_version =
+            self.epoch_manager.get_epoch_protocol_version(block.header().epoch_id())?;
+
+        if checked_feature!(
+            "protocol_feature_relaxed_chunk_validation",
+            RelaxedChunkValidation,
+            protocol_version
+        ) {
+            return Ok(());
+        }
+
         if !validate_transactions_order(chunk.transactions()) {
             let merkle_paths =
                 Block::compute_chunk_headers_root(block.chunks().iter_deprecated()).1;
@@ -3214,8 +3225,6 @@ impl Chain {
             return Err(Error::InvalidChunkProofs(Box::new(chunk_proof)));
         }
 
-        let protocol_version =
-            self.epoch_manager.get_epoch_protocol_version(block.header().epoch_id())?;
         if checked_feature!("stable", AccessKeyNonceRange, protocol_version) {
             let transaction_validity_period = self.transaction_validity_period;
             for transaction in chunk.transactions() {

chain/chain/src/runtime/mod.rs

@@ -1400,11 +1400,18 @@ fn chunk_tx_gas_limit(
 fn calculate_transactions_size_limit(
     protocol_version: ProtocolVersion,
     runtime_config: &RuntimeConfig,
-    last_chunk_transactions_size: usize,
+    mut last_chunk_transactions_size: usize,
     transactions_gas_limit: Gas,
 ) -> u64 {
     // Checking feature WitnessTransactionLimits
     if ProtocolFeature::StatelessValidation.enabled(protocol_version) {
+        if near_primitives::checked_feature!(
+            "protocol_feature_relaxed_chunk_validation",
+            RelaxedChunkValidation,
+            protocol_version
+        ) {
+            last_chunk_transactions_size = 0;
+        }
         // Sum of transactions in the previous and current chunks should not exceed the limit.
         // Witness keeps transactions from both previous and current chunk, so we have to limit the sum of both.
         runtime_config

chain/chain/src/stateless_validation/chunk_validation.rs

@@ -18,6 +18,7 @@ use near_epoch_manager::EpochManagerAdapter;
 use near_pool::TransactionGroupIteratorWrapper;
 use near_primitives::apply::ApplyChunkReason;
 use near_primitives::block::Block;
+use near_primitives::checked_feature;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::merkle::merklize;
 use near_primitives::receipt::Receipt;
@@ -245,46 +246,54 @@ pub fn pre_validate_chunk_state_witness(
         )));
     }
 
-    // Verify that all proposed transactions are valid.
-    let new_transactions = &state_witness.new_transactions;
-    if !new_transactions.is_empty() {
-        let transactions_validation_storage_config = RuntimeStorageConfig {
-            state_root: state_witness.chunk_header.prev_state_root(),
-            use_flat_storage: true,
-            source: StorageDataSource::Recorded(PartialStorage {
-                nodes: state_witness.new_transactions_validation_state.clone(),
-            }),
-            state_patch: Default::default(),
-        };
+    let epoch_id = last_chunk_block.header().epoch_id();
+    let protocol_version = epoch_manager.get_epoch_protocol_version(&epoch_id)?;
+    if !checked_feature!(
+        "protocol_feature_relaxed_chunk_validation",
+        RelaxedChunkValidation,
+        protocol_version
+    ) {
+        // Verify that all proposed transactions are valid.
+        let new_transactions = &state_witness.new_transactions;
+        if !new_transactions.is_empty() {
+            let transactions_validation_storage_config = RuntimeStorageConfig {
+                state_root: state_witness.chunk_header.prev_state_root(),
+                use_flat_storage: true,
+                source: StorageDataSource::Recorded(PartialStorage {
+                    nodes: state_witness.new_transactions_validation_state.clone(),
+                }),
+                state_patch: Default::default(),
+            };
 
-        match validate_prepared_transactions(
-            chain,
-            runtime_adapter,
-            &state_witness.chunk_header,
-            transactions_validation_storage_config,
-            &new_transactions,
-            &state_witness.transactions,
-        ) {
-            Ok(result) => {
-                if result.transactions.len() != new_transactions.len() {
+            match validate_prepared_transactions(
+                chain,
+                runtime_adapter,
+                &state_witness.chunk_header,
+                transactions_validation_storage_config,
+                &new_transactions,
+                &state_witness.transactions,
+            ) {
+                Ok(result) => {
+                    if result.transactions.len() != new_transactions.len() {
+                        return Err(Error::InvalidChunkStateWitness(format!(
+                            "New transactions validation failed. \
+                         {} transactions out of {} proposed transactions were valid.",
+                            result.transactions.len(),
+                            new_transactions.len(),
+                        )));
+                    }
+                }
+                Err(error) => {
                     return Err(Error::InvalidChunkStateWitness(format!(
-                        "New transactions validation failed. {} transactions out of {} proposed transactions were valid.",
-                        result.transactions.len(),
-                        new_transactions.len(),
+                        "New transactions validation failed: {}",
+                        error,
                     )));
                 }
-            }
-            Err(error) => {
-                return Err(Error::InvalidChunkStateWitness(format!(
-                    "New transactions validation failed: {}",
-                    error,
-                )));
-            }
-        };
+            };
+        }
     }
 
     let main_transition_params = if last_chunk_block.header().is_genesis() {
-        let epoch_id = last_chunk_block.header().epoch_id();
         let shard_layout = epoch_manager.get_shard_layout(&epoch_id)?;
         let congestion_info = last_chunk_block
             .block_congestion_info()

chain/client/Cargo.toml

@@ -73,6 +73,11 @@ near-primitives = { workspace = true, features = ["clock", "solomon", "rand"] }
 near-actix-test-utils.workspace = true
 
 [features]
+protocol_feature_relaxed_chunk_validation = [
+  "near-chain/protocol_feature_relaxed_chunk_validation",
+  "near-primitives/protocol_feature_relaxed_chunk_validation",
+]
+
 # if enabled, we assert in most situations that are impossible unless some byzantine behavior is observed.
 byzantine_asserts = ["near-chain/byzantine_asserts"]
 shadow_chunk_validation = ["near-chain/shadow_chunk_validation"]
@@ -120,6 +125,7 @@ nightly = [
   "near-telemetry/nightly",
   "near-vm-runner/nightly",
   "nightly_protocol",
+  "protocol_feature_relaxed_chunk_validation",
 ]
 sandbox = [
   "near-client-primitives/sandbox",

chain/client/src/stateless_validation/state_witness_producer.rs

@@ -1,6 +1,6 @@
-use std::collections::HashMap;
-use std::sync::Arc;
-
+use super::partial_witness::partial_witness_actor::DistributeStateWitnessRequest;
+use crate::stateless_validation::chunk_validator::send_chunk_endorsement_to_block_producers;
+use crate::Client;
 use near_async::messaging::{CanSend, IntoSender};
 use near_chain::{BlockHeader, Chain, ChainStoreAccess};
 use near_chain_primitives::Error;
@@ -20,11 +20,8 @@ use near_primitives::stateless_validation::stored_chunk_state_transition_data::{
 use near_primitives::types::{AccountId, EpochId, ShardId};
 use near_primitives::validator_signer::ValidatorSigner;
 use near_primitives::version::ProtocolFeature;
-
-use crate::stateless_validation::chunk_validator::send_chunk_endorsement_to_block_producers;
-use crate::Client;
-
-use super::partial_witness::partial_witness_actor::DistributeStateWitnessRequest;
+use std::collections::HashMap;
+use std::sync::Arc;
 
 /// Result of collecting state transition data from the database to generate a state witness.
 /// Keep this private to this file.
@@ -126,6 +123,7 @@ impl Client {
         let chunk_header = chunk.cloned_header();
         let epoch_id =
             self.epoch_manager.get_epoch_id_from_prev_block(chunk_header.prev_block_hash())?;
+        let protocol_version = self.epoch_manager.get_epoch_protocol_version(&epoch_id)?;
         let prev_chunk = self.chain.get_chunk(&prev_chunk_header.chunk_hash())?;
         let StateTransitionData {
             main_transition,
@@ -135,17 +133,26 @@ impl Client {
             contract_updates,
         } = self.collect_state_transition_data(&chunk_header, prev_chunk_header)?;
 
-        let new_transactions = chunk.transactions().to_vec();
-        let new_transactions_validation_state = if new_transactions.is_empty() {
-            PartialState::default()
+        let (new_transactions, new_transactions_validation_state) = if checked_feature!(
+            "protocol_feature_relaxed_chunk_validation",
+            RelaxedChunkValidation,
+            protocol_version
+        ) {
+            (Vec::new(), PartialState::default())
         } else {
-            // With stateless validation chunk producer uses recording reads when validating transactions.
-            // The storage proof must be available here.
-            transactions_storage_proof.ok_or_else(|| {
-                let message = "Missing storage proof for transactions validation";
-                log_assert_fail!("{message}");
-                Error::Other(message.to_owned())
-            })?
+            let new_transactions = chunk.transactions().to_vec();
+            let new_transactions_validation_state = if new_transactions.is_empty() {
+                PartialState::default()
+            } else {
+                // With stateless validation chunk producer uses recording reads when validating
+                // transactions. The storage proof must be available here.
+                transactions_storage_proof.ok_or_else(|| {
+                    let message = "Missing storage proof for transactions validation";
+                    log_assert_fail!("{message}");
+                    Error::Other(message.to_owned())
+                })?
+            };
+            (new_transactions, new_transactions_validation_state)
         };
 
         let source_receipt_proofs =

core/primitives-core/Cargo.toml

@@ -38,13 +38,15 @@ protocol_feature_fix_staking_threshold = []
 protocol_feature_fix_contract_loading_cost = []
 protocol_feature_reject_blocks_with_outdated_protocol_version = []
 protocol_feature_nonrefundable_transfer_nep491 = []
+protocol_feature_relaxed_chunk_validation = []
 
 nightly = [
   "nightly_protocol",
   "protocol_feature_fix_contract_loading_cost",
   "protocol_feature_fix_staking_threshold",
   "protocol_feature_nonrefundable_transfer_nep491",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",
+  "protocol_feature_relaxed_chunk_validation",
 ]
 
 nightly_protocol = [

core/primitives-core/src/version.rs

@@ -183,6 +183,12 @@ pub enum ProtocolFeature {
     /// to sync the current epoch's state. This is not strictly a protocol feature, but is included
     /// here to coordinate among nodes
     CurrentEpochStateSync,
+    /// Relaxed validation of transactions included in a chunk.
+    ///
+    /// Chunks no longer become entirely invalid in case invalid transactions are included in the
+    /// chunk. Instead the transactions are discarded during their conversion to receipts.
+    #[cfg(feature = "protocol_feature_relaxed_chunk_validation")]
+    RelaxedChunkValidation,
 }
 
 impl ProtocolFeature {
@@ -262,6 +268,8 @@ impl ProtocolFeature {
             ProtocolFeature::ShuffleShardAssignments => 143,
             ProtocolFeature::CurrentEpochStateSync => 144,
             ProtocolFeature::SimpleNightshadeV4 => 145,
+            #[cfg(feature = "protocol_feature_relaxed_chunk_validation")]
+            ProtocolFeature::RelaxedChunkValidation => 146,
             ProtocolFeature::BandwidthScheduler => 147,
             // Place features that are not yet in Nightly below this line.
         }

core/primitives/Cargo.toml

@@ -67,6 +67,9 @@ protocol_feature_reject_blocks_with_outdated_protocol_version = [
 protocol_feature_nonrefundable_transfer_nep491 = [
   "near-primitives-core/protocol_feature_nonrefundable_transfer_nep491",
 ]
+protocol_feature_relaxed_chunk_validation = [
+  "near-primitives-core/protocol_feature_relaxed_chunk_validation",
+]
 
 nightly = [
   "near-fmt/nightly",
@@ -78,6 +81,7 @@ nightly = [
   "protocol_feature_fix_staking_threshold",
   "protocol_feature_nonrefundable_transfer_nep491",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",
+  "protocol_feature_relaxed_chunk_validation",
 ]
 
 nightly_protocol = [

core/primitives/src/errors.rs

@@ -56,8 +56,7 @@ impl From<InvalidTxError> for TxExecutionError {
 pub enum RuntimeError {
     /// An unexpected integer overflow occurred. The likely issue is an invalid state or the transition.
     UnexpectedIntegerOverflow(String),
-    /// An error happened during TX verification and account charging. It's likely the chunk is invalid.
-    /// and should be challenged.
+    /// An error happened during TX verification and account charging.
     InvalidTxError(InvalidTxError),
     /// Unexpected error which is typically related to the node storage corruption.
     /// It's possible the input state is invalid or malicious.

integration-tests/Cargo.toml

@@ -101,6 +101,9 @@ protocol_feature_reject_blocks_with_outdated_protocol_version = [
 protocol_feature_nonrefundable_transfer_nep491 = [
   "near-primitives/protocol_feature_nonrefundable_transfer_nep491",
 ]
+protocol_feature_relaxed_chunk_validation = [
+  "near-primitives-core/protocol_feature_relaxed_chunk_validation",
+]
 
 nightly = [
   "near-actix-test-utils/nightly",
@@ -131,6 +134,7 @@ nightly = [
   "protocol_feature_fix_contract_loading_cost",
   "protocol_feature_nonrefundable_transfer_nep491",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",
+  "protocol_feature_relaxed_chunk_validation",
   "testlib/nightly",
 ]
 nightly_protocol = [

integration-tests/src/test_loop/tests/mod.rs

@@ -1,19 +1,19 @@
-pub mod bandwidth_scheduler_protocol_upgrade;
+mod bandwidth_scheduler_protocol_upgrade;
 mod chunk_validator_kickout;
-pub mod congestion_control;
-pub mod congestion_control_genesis_bootstrap;
-pub mod contract_distribution_cross_shard;
-pub mod contract_distribution_simple;
+mod congestion_control;
+mod congestion_control_genesis_bootstrap;
+mod contract_distribution_cross_shard;
+mod contract_distribution_simple;
 mod create_delete_account;
-pub mod epoch_sync;
-pub mod fix_min_stake_ratio;
-pub mod in_memory_tries;
-pub mod max_receipt_size;
-pub mod multinode_stateless_validators;
-pub mod multinode_test_loop_example;
-pub mod protocol_upgrade;
+mod epoch_sync;
+mod fix_min_stake_ratio;
+mod in_memory_tries;
+mod max_receipt_size;
+mod multinode_stateless_validators;
+mod multinode_test_loop_example;
+mod protocol_upgrade;
 mod resharding_v3;
-pub mod simple_test_loop_example;
-pub mod state_sync;
-pub mod syncing;
-pub mod view_requests_to_archival_node;
+mod simple_test_loop_example;
+mod state_sync;
+mod syncing;
+mod view_requests_to_archival_node;