Skip to content

Commit

Permalink
fix: nonce issues (#310)
Browse files Browse the repository at this point in the history
* Internalize RwLock into ShiningState

* made &TestContext<'a> => TestContext

* Added near_fetch library for cached nonces

* Added KeyRotatingSigner

* tmp

* Working stress test

* Cleaning up

* Use Open Source Relayer for integration tests

* Fmt

* cleanup

* Fix wrong container image

* Working local testing

* Moved running local related testing functions over to local.rs

* Reduced LeaderNode::api to not require arguments

* Moved all containers init into env/

* Moved containers.rs and local.rs into env/

* Change feature flag to docker-test

* Fix docker test

* Consistent way to create CLI args

* Removed need for integration-tests/main.rs

* Update docs and cleanup

* Cleanup

* setup-env cmd

* Update GA pipeline

* More cleanup

* Use env::run for setup-env

* More cleanup

* Better setup-env logging

* prefer 127.0.0.1 over localhost to enforce IPv4

* Separate out docker image dep

* Rename local_url to pk_local_url

* tmp

* Minor fixes for local build

* Terraform

* Fix parsing

* Actually fix parsing

* Address comments

---------

Co-authored-by: Daniyar Itegulov <[email protected]>
  • Loading branch information
ChaoticTempest and itegulov authored Oct 19, 2023
1 parent aef8142 commit bc48a73
Show file tree
Hide file tree
Showing 17 changed files with 569 additions and 593 deletions.
625 changes: 303 additions & 322 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion infra/modules/leader/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ resource "google_cloud_run_v2_service" "leader" {
value_source {
secret_key_ref {
secret = var.account_creator_sk_secret_id
version = "latest"
version = "1"
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion infra/terraform-dev.tfvars
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ project = "pagoda-discovery-platform-dev"
docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery"

account_creator_id = "mpc-recovery-dev-creator.testnet"
account_creator_sk_secret_id = "mpc-account-creator-sk-dev"
account_creator_sk_secret_id = "mpc-recovery-account-creator-sk-dev"
oidc_providers_secret_id = "mpc-allowed-oidc-providers-dev"
fast_auth_partners_secret_id = "mpc-fast-auth-partners-dev"
signer_configs = [
Expand Down
1 change: 1 addition & 0 deletions integration-tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ near-jsonrpc-client = "0.6"
near-primitives = "0.17"
near-units = "0.2.0"
once_cell = "1"
rand = "0.7"
serde = "1"
serde_json = "1"
testcontainers = { version = "0.14", features = ["experimental"] }
Expand Down
47 changes: 27 additions & 20 deletions integration-tests/src/env/containers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use ed25519_dalek::{PublicKey as PublicKeyEd25519, Verifier};
use futures::{lock::Mutex, StreamExt};
use hyper::StatusCode;
use mpc_recovery::firewall::allowed::DelegateActionRelayer;
use mpc_recovery::logging;
use mpc_recovery::sign_node::oidc::OidcToken;
use mpc_recovery::{
msg::{
Expand Down Expand Up @@ -43,8 +44,9 @@ use tracing;
use std::fs;

use crate::env::{Context, LeaderNodeApi, SignerNodeApi};
use crate::util::{self, create_key_file, create_relayer_cofig_file};
use mpc_recovery::logging;
use crate::util::{
self, create_key_file, create_key_file_with_filepath, create_relayer_cofig_file,
};

static NETWORK_MUTEX: Lazy<Mutex<i32>> = Lazy::new(|| Mutex::new(0));

Expand Down Expand Up @@ -297,7 +299,7 @@ impl<'a> Relayer<'a> {
near_rpc: &str,
redis_full_address: &str,
relayer_account_id: &AccountId,
relayer_account_sk: &near_workspaces::types::SecretKey,
relayer_account_sks: &[near_workspaces::types::SecretKey],
creator_account_id: &AccountId,
social_db_id: &AccountId,
social_account_id: &AccountId,
Expand All @@ -312,14 +314,20 @@ impl<'a> Relayer<'a> {
.unwrap_or_else(|_| panic!("Failed to create {relayer_configs_path} directory"));

// Create dir for keys
let keys_path = format!("{relayer_configs_path}/account_keys");
std::fs::create_dir_all(&keys_path).expect("Failed to create account_keys directory");
let key_dir = format!("{relayer_configs_path}/account_keys");
std::fs::create_dir_all(&key_dir).expect("Failed to create account_keys directory");
let keys_absolute_path =
fs::canonicalize(&keys_path).expect("Failed to get absolute path for keys");
fs::canonicalize(&key_dir).expect("Failed to get absolute path for keys");

// Create JSON key files
create_key_file(relayer_account_id, relayer_account_sk, &keys_path)?;
create_key_file(social_account_id, social_account_sk, &keys_path)?;
create_key_file(social_account_id, social_account_sk, &key_dir)?;
let mut relayer_keyfiles = Vec::with_capacity(relayer_account_sks.len());
for (i, relayer_sk) in relayer_account_sks.iter().enumerate() {
let filename = format!("{i}-{relayer_account_id}");
let keypath = format!("{key_dir}/{filename}.json");
create_key_file_with_filepath(relayer_account_id, relayer_sk, &keypath)?;
relayer_keyfiles.push(format!("./account_keys/{filename}.json"));
}

// Create relayer config file
let config_file_name = "config.toml";
Expand All @@ -328,7 +336,7 @@ impl<'a> Relayer<'a> {
ip_address: [0, 0, 0, 0],
port: Self::CONTAINER_PORT,
relayer_account_id: relayer_account_id.clone(),
keys_filenames: vec![format!("./account_keys/{}.json", relayer_account_id)],
keys_filenames: relayer_keyfiles,
shared_storage_account_id: social_account_id.clone(),
shared_storage_keys_filename: format!("./account_keys/{}.json", social_account_id),
whitelisted_contracts: vec![creator_account_id.clone()],
Expand Down Expand Up @@ -643,7 +651,12 @@ impl<'a> LeaderNode<'a> {
near_rpc: ctx.relayer_ctx.sandbox.address.clone(),
near_root_account: ctx.relayer_ctx.worker.root_account()?.id().to_string(),
account_creator_id: account_creator.id().clone(),
account_creator_sk: Some(account_creator.secret_key().to_string()),
account_creator_sk: ctx
.relayer_ctx
.creator_account_keys
.iter()
.map(|k| k.to_string().parse())
.collect::<Result<Vec<_>, _>>()?,
fast_auth_partners: Some(
serde_json::json!([
{
Expand Down Expand Up @@ -765,15 +778,15 @@ impl LeaderNodeApi {

let frp_signature = match user_secret_key.sign(&user_credentials_request_digest) {
near_crypto::Signature::ED25519(k) => k,
_ => return Err(anyhow::anyhow!("Wrong signature type")),
_ => anyhow::bail!("Wrong signature type"),
};

let new_account_request = NewAccountRequest {
near_account_id: account_id.clone(),
create_account_options,
oidc_token: oidc_token.clone(),
user_credentials_frp_signature: frp_signature,
frp_public_key: user_pk.clone(),
frp_public_key: user_pk,
};

self.new_account(new_account_request).await
Expand All @@ -789,10 +802,7 @@ impl LeaderNodeApi {
frp_pk: &PublicKey,
) -> anyhow::Result<(StatusCode, SignResponse)> {
// Prepare SignRequest with add key delegate action
let (_, block_height, nonce) = self
.client
.access_key(account_id.clone(), recovery_pk.clone())
.await?;
let (_, block_height, nonce) = self.client.access_key(account_id, recovery_pk).await?;

let add_key_delegate_action = DelegateAction {
sender_id: account_id.clone(),
Expand Down Expand Up @@ -846,10 +856,7 @@ impl LeaderNodeApi {
frp_pk: &PublicKey,
) -> anyhow::Result<(StatusCode, SignResponse)> {
// Prepare SignRequest with add key delegate action
let (_, block_height, nonce) = self
.client
.access_key(account_id.clone(), recovery_pk.clone())
.await?;
let (_, block_height, nonce) = self.client.access_key(account_id, recovery_pk).await?;

let delete_key_delegate_action = DelegateAction {
sender_id: account_id.clone(),
Expand Down
9 changes: 7 additions & 2 deletions integration-tests/src/env/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ use aes_gcm::aead::consts::U32;
use aes_gcm::aead::generic_array::GenericArray;
use async_process::Child;
use mpc_recovery::firewall::allowed::DelegateActionRelayer;
use mpc_recovery::logging;
use mpc_recovery::relayer::NearRpcAndRelayerClient;
use multi_party_eddsa::protocols::ExpandedKeyPair;

use crate::env::{LeaderNodeApi, SignerNodeApi};
use crate::util;
use mpc_recovery::logging;

pub struct SignerNode {
pub address: String,
Expand Down Expand Up @@ -108,7 +108,12 @@ impl LeaderNode {
near_rpc: ctx.relayer_ctx.sandbox.local_address.clone(),
near_root_account: ctx.relayer_ctx.worker.root_account()?.id().to_string(),
account_creator_id: account_creator.id().clone(),
account_creator_sk: Some(account_creator.secret_key().to_string()),
account_creator_sk: ctx
.relayer_ctx
.creator_account_keys
.iter()
.map(|k| k.to_string().parse())
.collect::<Result<Vec<_>, _>>()?,
fast_auth_partners_filepath: None,
fast_auth_partners: Some(
serde_json::json!([
Expand Down
10 changes: 8 additions & 2 deletions integration-tests/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use near_crypto::KeyFile;
use near_units::parse_near;
use near_workspaces::{
network::{Sandbox, ValidatorKey},
types::SecretKey,
Account, Worker,
};

Expand Down Expand Up @@ -59,6 +60,7 @@ pub struct RelayerCtx<'a> {
pub relayer: containers::Relayer<'a>,
pub worker: Worker<Sandbox>,
pub creator_account: Account,
pub creator_account_keys: Vec<SecretKey>,
}

pub async fn initialize_relayer<'a>(
Expand Down Expand Up @@ -90,7 +92,11 @@ pub async fn initialize_relayer<'a>(
tracing::info!("Initializing relayer accounts...");
let relayer_account =
sandbox::create_account(&worker, "relayer", parse_near!("1000 N")).await?;
let relayer_account_keys = sandbox::gen_rotating_keys(&relayer_account, 5).await?;

let creator_account = sandbox::create_account(&worker, "creator", parse_near!("200 N")).await?;
let creator_account_keys = sandbox::gen_rotating_keys(&creator_account, 5).await?;

let social_account = sandbox::create_account(&worker, "social", parse_near!("1000 N")).await?;
tracing::info!(
"Relayer accounts initialized. Relayer account: {}, Creator account: {}, Social account: {}",
Expand All @@ -100,14 +106,13 @@ pub async fn initialize_relayer<'a>(
);

let redis = containers::Redis::run(docker_client, network).await?;

let relayer = containers::Relayer::run(
docker_client,
network,
&sandbox.address,
&redis.full_address,
relayer_account.id(),
relayer_account.secret_key(),
&relayer_account_keys,
creator_account.id(),
social_db.id(),
social_account.id(),
Expand All @@ -122,5 +127,6 @@ pub async fn initialize_relayer<'a>(
relayer,
worker,
creator_account,
creator_account_keys,
})
}
38 changes: 37 additions & 1 deletion integration-tests/src/sandbox.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use near_workspaces::{network::Sandbox, Account, Contract, Worker};
use near_workspaces::{network::Sandbox, types::SecretKey, AccessKey, Account, Contract, Worker};

const BATCH_COUNT_LIMIT: usize = 100;

pub async fn initialize_social_db(worker: &Worker<Sandbox>) -> anyhow::Result<Contract> {
tracing::info!("Initializing social DB contract...");
Expand Down Expand Up @@ -53,3 +55,37 @@ pub async fn create_account(
tracing::info!("Account created: {}", new_account.id());
Ok(new_account)
}

pub async fn gen_rotating_keys(account: &Account, amount: usize) -> anyhow::Result<Vec<SecretKey>> {
let mut keys = Vec::with_capacity(amount + 1);
keys.push(account.secret_key().clone());

// Each batch transaction has a limit of BATCH_COUNT_LIMIT actions.
let num_batches = amount / BATCH_COUNT_LIMIT + 1;
let rem_batches = amount % BATCH_COUNT_LIMIT;
let batch_counts = (0..num_batches).map(|i| {
if i == num_batches - 1 {
rem_batches
} else {
BATCH_COUNT_LIMIT
}
});

for batch_count in batch_counts {
let mut batch_tx = account.batch(account.id());
for _ in 0..batch_count {
let sk = SecretKey::from_seed(
near_workspaces::types::KeyType::ED25519,
&rand::Rng::sample_iter(rand::thread_rng(), &rand::distributions::Alphanumeric)
.take(10)
.map(char::from)
.collect::<String>(),
);
batch_tx = batch_tx.add_key(sk.public_key(), AccessKey::full_access());
keys.push(sk);
}
batch_tx.transact().await?.into_result()?;
}

Ok(keys)
}
18 changes: 14 additions & 4 deletions integration-tests/src/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,17 +79,27 @@ struct KeyFile {
pub fn create_key_file(
account_id: &AccountId,
account_sk: &SecretKey,
key_path: &str,
key_dir: &str,
) -> anyhow::Result<(), anyhow::Error> {
create_key_file_with_filepath(
account_id,
account_sk,
&format!("{key_dir}/{account_id}.json"),
)
}

pub fn create_key_file_with_filepath(
account_id: &AccountId,
account_sk: &SecretKey,
filepath: &str,
) -> anyhow::Result<(), anyhow::Error> {
let key_file = KeyFile {
account_id: account_id.to_string(),
public_key: account_sk.public_key().to_string(),
private_key: account_sk.to_string(),
};
let key_json_str = serde_json::to_string(&key_file).expect("Failed to serialize to JSON");
let key_json_file_path = format!("{key_path}/{account_id}.json");
let mut json_key_file =
File::create(key_json_file_path).expect("Failed to create JSON key file");
let mut json_key_file = File::create(filepath).expect("Failed to create JSON key file");
json_key_file
.write_all(key_json_str.as_bytes())
.expect("Failed to write to JSON key file");
Expand Down
Loading

0 comments on commit bc48a73

Please sign in to comment.