diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index e02c4bfed2e..522f7f95e1c 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -218,8 +218,8 @@ impl Client { /// Updates client's mutable validator signer. /// It will update all validator signers that synchronize with it. - pub(crate) fn update_validator_signer(&self, signer: Arc) -> bool { - self.validator_signer.update(Some(signer)) + pub(crate) fn update_validator_signer(&self, signer: Option>) -> bool { + self.validator_signer.update(signer) } } diff --git a/chain/client/src/client_actor.rs b/chain/client/src/client_actor.rs index 3b5c504d5fe..f71fefdc712 100644 --- a/chain/client/src/client_actor.rs +++ b/chain/client/src/client_actor.rs @@ -1208,11 +1208,10 @@ impl ClientActorInner { ); if update_result.validator_signer_updated { - let validator_signer = - self.client.validator_signer.get().expect("Validator signer just updated"); - - check_validator_tracked_shards(&self.client, validator_signer.validator_id()) - .expect("Could not check validator tracked shards"); + if let Some(validator_signer) = self.client.validator_signer.get() { + check_validator_tracked_shards(&self.client, validator_signer.validator_id()) + .expect("Could not check validator tracked shards"); + } // Request PeerManager to advertise tier1 proxies. // It is needed to advertise that our validator key changed. diff --git a/chain/client/src/config_updater.rs b/chain/client/src/config_updater.rs index 441d1bd4970..18832398ad3 100644 --- a/chain/client/src/config_updater.rs +++ b/chain/client/src/config_updater.rs @@ -33,7 +33,7 @@ impl ConfigUpdater { pub fn try_update( &mut self, update_client_config_fn: &dyn Fn(UpdateableClientConfig) -> bool, - update_validator_signer_fn: &dyn Fn(Arc) -> bool, + update_validator_signer_fn: &dyn Fn(Option>) -> bool, ) -> ConfigUpdaterResult { let mut update_result = ConfigUpdaterResult::default(); while let Ok(maybe_updateable_configs) = self.rx_config_update.try_recv() { diff --git a/core/chain-configs/src/lib.rs b/core/chain-configs/src/lib.rs index be2e1e45e03..614c5091ef1 100644 --- a/core/chain-configs/src/lib.rs +++ b/core/chain-configs/src/lib.rs @@ -33,7 +33,9 @@ pub use genesis_config::{ }; use near_primitives::types::{Balance, BlockHeightDelta, Gas, NumBlocks, NumSeats}; use num_rational::Rational32; -pub use updateable_config::{MutableConfigValue, MutableValidatorSigner, UpdateableClientConfig}; +pub use updateable_config::{ + MutableConfigValue, MutableValidatorSigner, UpdateableClientConfig, UpdateableValidatorSigner, +}; pub const GENESIS_CONFIG_FILENAME: &str = "genesis.json"; diff --git a/core/chain-configs/src/updateable_config.rs b/core/chain-configs/src/updateable_config.rs index ebace65d3ec..ef70d051964 100644 --- a/core/chain-configs/src/updateable_config.rs +++ b/core/chain-configs/src/updateable_config.rs @@ -110,4 +110,5 @@ pub struct UpdateableClientConfig { pub produce_chunk_add_transactions_time_limit: Option, } +pub type UpdateableValidatorSigner = Option>; pub type MutableValidatorSigner = MutableConfigValue>>; diff --git a/core/dyn-configs/src/lib.rs b/core/dyn-configs/src/lib.rs index 320e44dec8c..6b5bea44f7c 100644 --- a/core/dyn-configs/src/lib.rs +++ b/core/dyn-configs/src/lib.rs @@ -1,8 +1,7 @@ #![doc = include_str!("../README.md")] -use near_chain_configs::UpdateableClientConfig; +use near_chain_configs::{UpdateableClientConfig, UpdateableValidatorSigner}; use near_o11y::log_config::LogConfig; -use near_primitives::validator_signer::ValidatorSigner; use near_time::Clock; use std::path::PathBuf; use std::sync::Arc; @@ -18,7 +17,9 @@ pub struct UpdateableConfigs { /// Contents of the `config.json` corresponding to the mutable fields of `ClientConfig`. pub client_config: Option, /// Validator key hot loaded from file. - pub validator_signer: Option>, + /// `None` means that the validator key existence could not be determined. + /// `Some(None)` means that it was determined that the validator key does not exist. + pub validator_signer: Option, } /// Pushes the updates to listeners. diff --git a/nearcore/src/dyn_config.rs b/nearcore/src/dyn_config.rs index 9a480d0c17a..b237eaf8b84 100644 --- a/nearcore/src/dyn_config.rs +++ b/nearcore/src/dyn_config.rs @@ -34,10 +34,13 @@ pub fn read_updateable_configs( let updateable_client_config = config.as_ref().map(get_updateable_client_config); let validator_signer = if let Some(config) = config { - read_validator_key(home_dir, &config).unwrap_or_else(|err| { - errs.push(err); - None - }) + match read_validator_key(home_dir, &config) { + Ok(validator_key) => Some(validator_key), + Err(err) => { + errs.push(err); + None + } + } } else { None }; diff --git a/nightly/pytest-sanity.txt b/nightly/pytest-sanity.txt index fbfc4a75720..7168eeb5d84 100644 --- a/nightly/pytest-sanity.txt +++ b/nightly/pytest-sanity.txt @@ -112,6 +112,8 @@ pytest --timeout=240 sanity/switch_node_key.py pytest --timeout=240 sanity/switch_node_key.py --features nightly pytest --timeout=120 sanity/validator_switch_key_quick.py pytest --timeout=120 sanity/validator_switch_key_quick.py --features nightly +pytest --timeout=120 sanity/validator_remove_key_quick.py +pytest --timeout=120 sanity/validator_remove_key_quick.py --features nightly pytest --timeout=60 sanity/shadow_tracking.py pytest --timeout=60 sanity/shadow_tracking.py --features nightly pytest sanity/proxy_simple.py diff --git a/pytest/lib/cluster.py b/pytest/lib/cluster.py index e58e570c0a6..be5442b4208 100644 --- a/pytest/lib/cluster.py +++ b/pytest/lib/cluster.py @@ -608,6 +608,14 @@ def reset_validator_key(self, new_key): with open(os.path.join(self.node_dir, "validator_key.json"), 'w+') as f: json.dump(new_key.to_json(), f) + def remove_validator_key(self): + logger.info( + f"Removing validator_key.json file for node {self.ordinal}.") + self.validator_key = None + file_path = os.path.join(self.node_dir, "validator_key.json") + if os.path.exists(file_path): + os.remove(file_path) + def reset_node_key(self, new_key): self.node_key = new_key with open(os.path.join(self.node_dir, "node_key.json"), 'w+') as f: diff --git a/pytest/tests/sanity/validator_remove_key_quick.py b/pytest/tests/sanity/validator_remove_key_quick.py new file mode 100755 index 00000000000..28c1e4affc0 --- /dev/null +++ b/pytest/tests/sanity/validator_remove_key_quick.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Starts one validating node and one non-validating node +# Dynamically remove the key from the validator node +# and make sure that the network stalls. +# Dynamically reload the key at the validator node +# and make sure that the network progress again. + +import unittest +import sys +import pathlib + +sys.path.append(str(pathlib.Path(__file__).resolve().parents[2] / 'lib')) + +from configured_logger import logger +from cluster import start_cluster +from utils import wait_for_blocks +import state_sync_lib + +EPOCH_LENGTH = 20 +TIMEOUT = 100 + + +class ValidatorRemoveKeyQuickTest(unittest.TestCase): + + def test_validator_remove_key_quick(self): + logger.info("Validator remove key quick test") + validator_config, rpc_config = state_sync_lib.get_state_sync_configs_pair( + ) + + validator_config.update({ + "tracked_shards": [0], + "store.load_mem_tries_for_tracked_shards": True, + }) + + rpc_config.update({ + "tracked_shards": [0], + }) + + [validator, + rpc] = start_cluster(1, 1, 1, None, + [["epoch_length", EPOCH_LENGTH], + ["block_producer_kickout_threshold", 80], + ["chunk_producer_kickout_threshold", 80]], { + 0: validator_config, + 1: rpc_config + }) + + wait_for_blocks(rpc, target=EPOCH_LENGTH) + validator_key = validator.validator_key + validator.remove_validator_key() + wait_for_blocks(rpc, target=EPOCH_LENGTH * 2) + validator.reload_updateable_config() + validator.stop_checking_store() + try: + wait_for_blocks(rpc, count=5, timeout=10) + except: + pass + else: + self.fail('Blocks are not supposed to be produced') + + validator.reset_validator_key(validator_key) + validator.reload_updateable_config() + validator.stop_checking_store() + wait_for_blocks(rpc, count=EPOCH_LENGTH * 2) + + +if __name__ == '__main__': + unittest.main()