From d48856c73e7baae6f6cb1062eae16308492fe6f8 Mon Sep 17 00:00:00 2001 From: sword_smith Date: Mon, 5 Aug 2024 17:53:08 +0200 Subject: [PATCH 01/11] feat(mmr_auth_struct): `root_from_authentication_struct` algorithm Add a function to calculate a Merkle root from an authentication struct in a way that conductive to ZK calculations. And add a function to generate the relevant witness data for this calculation. The authentication structure is a list of digests that is identical to what can be found in `authentication_structure` field in the `MerkleTreeInclusionProof`. With appropriate witness structure, this authentication structure can be verified faster than verifying each individual leaf in an MMR, we hope. This `root_from_authentication_struct` function now needs to be implemented in `tasm-lib` to see if this actually gives shorter programs in Triton-VM. The purpose of this cryptography is to speed up the program that verifies that no double-spends are occurring. See #228. --- twenty-first/src/util_types/mmr.rs | 1 + .../mmr/mmr_authentication_struct.rs | 442 ++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 twenty-first/src/util_types/mmr/mmr_authentication_struct.rs diff --git a/twenty-first/src/util_types/mmr.rs b/twenty-first/src/util_types/mmr.rs index aad78c2c8..dc5017867 100644 --- a/twenty-first/src/util_types/mmr.rs +++ b/twenty-first/src/util_types/mmr.rs @@ -1,4 +1,5 @@ pub mod mmr_accumulator; +pub mod mmr_authentication_struct; pub mod mmr_membership_proof; pub mod mmr_successor_proof; pub mod mmr_trait; diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs new file mode 100644 index 000000000..89006c54b --- /dev/null +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -0,0 +1,442 @@ +use std::collections::HashSet; + +use itertools::Itertools; +use num_traits::One; + +use crate::bfe; +use crate::prelude::AlgebraicHasher; +use crate::prelude::BFieldCodec; +use crate::prelude::BFieldElement; +use crate::prelude::Digest; +use crate::prelude::Inverse; +use crate::prelude::MerkleTree; +use crate::prelude::Sponge; +use crate::prelude::Tip5; +use crate::prelude::XFieldElement; + +const ROOT_MT_INDEX: u64 = 1; + +pub struct MerkleAuthenticationStructAuthenticityWitness { + // All indices are Merkle tree node indices + nd_auth_struct_indices: Vec, + nd_sibling_indices: Vec<(u64, u64)>, + nd_siblings: Vec<(Digest, Digest)>, +} + +impl MerkleAuthenticationStructAuthenticityWitness { + /// Return the Merkle tree node indices of the digests required to prove + /// membership for the specified leaf indices + fn authentication_structure_mt_indices( + num_leafs: u64, + leaf_indices: &[u64], + ) -> impl ExactSizeIterator { + // The set of indices of nodes that need to be included in the authentications + // structure. In principle, every node of every authentication path is needed. + // The root is never needed. Hence, it is not considered below. + let mut node_is_needed = HashSet::new(); + + // The set of indices of nodes that can be computed from other nodes in the + // authentication structure or the leafs that are explicitly supplied during + // verification. Every node on the direct path from the leaf to the root can + // be computed by the very nature of “authentication path”. + let mut node_can_be_computed = HashSet::new(); + + for &leaf_index in leaf_indices { + assert!(num_leafs > leaf_index, "Leaf index must be less than number of leafs. Got leaf_index = {leaf_index}; num_leafs = {num_leafs}"); + + let mut node_index = leaf_index + num_leafs; + while node_index > ROOT_MT_INDEX { + let sibling_index = node_index ^ 1; + node_can_be_computed.insert(node_index); + node_is_needed.insert(sibling_index); + node_index /= 2; + } + } + + let set_difference = node_is_needed.difference(&node_can_be_computed).copied(); + set_difference.sorted_unstable().rev() + } + + pub fn root_from_authentication_struct( + &self, + tree_height: u32, + auth_struct: Vec, + indexed_leafs: Vec<(u64, Digest)>, + ) -> Digest { + fn digest_to_xfe(digest: Digest, challenge: XFieldElement) -> XFieldElement { + let leaf_xfe_lo = XFieldElement::new([digest.0[0], digest.0[1], digest.0[2]]); + let leaf_xfe_hi = + challenge * XFieldElement::new([digest.0[3], digest.0[4], BFieldElement::one()]); + + leaf_xfe_lo + leaf_xfe_hi + } + + fn node_index_to_bfe(node_index: u64) -> BFieldElement { + BFieldElement::new(node_index) + } + + // Sanity check + assert_eq!( + self.nd_auth_struct_indices.len(), + auth_struct.len(), + "Provided auth struct length must match that specified in receiver" + ); + + // Get challenges + let (alpha, gamma, delta) = { + let mut sponge = Tip5::init(); + sponge.pad_and_absorb_all(&indexed_leafs.encode()); + sponge.pad_and_absorb_all(&auth_struct.encode()); + let challenges = sponge.sample_scalars(3); + (challenges[0], challenges[1], challenges[2]) + }; + + // Accumulate `p` from public data + let mut p = XFieldElement::one(); + for i in (0..indexed_leafs.len()).rev() { + let leaf_index_as_bfe = node_index_to_bfe((1 << tree_height) ^ indexed_leafs[i].0); + let leaf_as_xfe = digest_to_xfe(indexed_leafs[i].1, alpha); + let fact = leaf_as_xfe - gamma + delta * leaf_index_as_bfe; + p *= fact; + } + + let mut prev = 0; + for i in (0..auth_struct.len()).rev() { + let auth_struct_index = self.nd_auth_struct_indices[i]; + assert!(auth_struct_index > prev); + prev = auth_struct_index; + + let auth_struct_index_as_bfe = node_index_to_bfe(auth_struct_index); + + let auth_str_elem_as_xfe = digest_to_xfe(auth_struct[i], alpha); + let fact = auth_str_elem_as_xfe - gamma + delta * auth_struct_index_as_bfe; + p *= fact; + } + + // Use secret data to invert `p` back and to calculate the root + let mut t = auth_struct + .first() + .copied() + .unwrap_or_else(|| indexed_leafs.first().unwrap().1); + let mut t_xfe = digest_to_xfe(t, alpha); + let mut parent_index_bfe = BFieldElement::one(); + for ((l, r), (left_index, right_index)) in self + .nd_siblings + .iter() + .zip_eq(self.nd_sibling_indices.clone()) + { + assert_eq!(left_index + 1, right_index); + + t = Tip5::hash_pair(*l, *r); + + let l_xfe = digest_to_xfe(*l, alpha); + let r_xfe = digest_to_xfe(*r, alpha); + t_xfe = digest_to_xfe(t, alpha); + + let left_index_bfe = node_index_to_bfe(left_index); + let right_index_bfe = node_index_to_bfe(right_index); + parent_index_bfe = left_index_bfe / bfe!(2); + + let fact1 = l_xfe - gamma + delta * left_index_bfe; + let fact2 = r_xfe - gamma + delta * right_index_bfe; + let fact_parent = t_xfe - gamma + delta * parent_index_bfe; + + p *= fact1.inverse() * fact2.inverse() * fact_parent; + } + + assert_eq!(t_xfe - gamma + delta, p); + assert!(parent_index_bfe.is_one()); + + t + } + + /// Return the authentication structure witness, authentication structure, + /// and the (leaf-index, leaf-digest) pairs. + pub fn new_from_merkle_tree( + tree: &MerkleTree, + mut revealed_leaf_indices: Vec, + ) -> (Self, Vec, Vec<(u64, Digest)>) { + revealed_leaf_indices.sort_unstable(); + revealed_leaf_indices.dedup(); + revealed_leaf_indices.reverse(); + let num_leafs: u64 = tree.num_leafs() as u64; + + let mut nd_auth_struct_indices = + Self::authentication_structure_mt_indices(num_leafs, &revealed_leaf_indices) + .collect_vec(); + if revealed_leaf_indices.is_empty() { + nd_auth_struct_indices = vec![ROOT_MT_INDEX]; + } + + let mut nd_sibling_indices = revealed_leaf_indices + .iter() + .map(|li| *li ^ num_leafs) + .chain(nd_auth_struct_indices.iter().copied()) + .filter(|idx| *idx != 1) + .map(|idx| (idx & (u64::MAX - 1), idx | 1u64)) + .unique() + .collect_vec(); + + if !nd_sibling_indices.is_empty() { + // TODO: I think we can use `PartialMerkleTree` to calculate all + // indices, and maybe also to get all the digests of `nd_siblings`. + let mut i = 0; + loop { + let elm = nd_sibling_indices[i]; + let parent = elm.0 >> 1; + if parent == 1 { + break; + } + let uncle = parent ^ 1; + + let new_pair = if parent & 1 == 0 { + (parent, uncle) + } else { + (uncle, parent) + }; + if !nd_sibling_indices.contains(&new_pair) { + nd_sibling_indices.push(new_pair); + } + + nd_sibling_indices.sort_by_key(|(left_idx, _right_idx)| *left_idx); + nd_sibling_indices.reverse(); + + i += 1; + } + } + + let nd_siblings = nd_sibling_indices + .iter() + .map(|&(l, r)| { + let l: usize = l.try_into().unwrap(); + let r: usize = r.try_into().unwrap(); + (tree.node(l).unwrap(), tree.node(r).unwrap()) + }) + .collect_vec(); + + let revealed_leafs = revealed_leaf_indices + .iter() + .map(|j| tree.node((*j + num_leafs) as usize).unwrap()) + .collect_vec(); + let indexed_leafs = revealed_leaf_indices + .clone() + .into_iter() + .zip_eq(revealed_leafs) + .collect_vec(); + + let auth_struct = nd_auth_struct_indices + .iter() + .map(|node_index| tree.node(*node_index as usize).unwrap()) + .collect_vec(); + + let auth_struct_witness = Self { + nd_auth_struct_indices, + nd_sibling_indices, + nd_siblings, + }; + + (auth_struct_witness, auth_struct, indexed_leafs) + } +} + +#[cfg(test)] +mod tests { + use crate::math::other::random_elements; + use crate::prelude::CpuParallel; + use crate::prelude::MerkleTree; + use proptest::collection::vec; + use proptest::prop_assert_eq; + use test_strategy::proptest; + + use super::*; + + #[proptest(cases = 20)] + fn root_from_authentication_struct_prop_test( + #[strategy(0..12u64)] tree_height: u64, + #[strategy(0usize..100)] _num_revealed_leafs: usize, + #[strategy(vec(0u64..1<<#tree_height, #_num_revealed_leafs))] revealed_leaf_indices: Vec< + u64, + >, + ) { + let num_leafs = 1u64 << tree_height; + let leafs: Vec = random_elements(num_leafs.try_into().unwrap()); + let tree = MerkleTree::::new::(&leafs).unwrap(); + + let (mmr_auth_struct, auth_struct, indexed_leafs) = + MerkleAuthenticationStructAuthenticityWitness::new_from_merkle_tree( + &tree, + revealed_leaf_indices, + ); + + let tree_height: u32 = tree_height.try_into().unwrap(); + let computed_root = mmr_auth_struct.root_from_authentication_struct( + tree_height, + auth_struct, + indexed_leafs, + ); + let expected_root = tree.root(); + prop_assert_eq!(expected_root, computed_root); + } + + fn prop_from_merkle_tree( + tree_height: usize, + leaf_indices: Vec, + nd_auth_struct_indices: Vec, + nd_sibling_indices: Vec<(u64, u64)>, + ) { + let leafs: Vec = random_elements(1 << tree_height); + let tree = MerkleTree::::new::(&leafs).unwrap(); + + let auth_struct = nd_auth_struct_indices + .iter() + .map(|i| tree.node(*i as usize).unwrap()) + .collect_vec(); + let revealed_leafs = leaf_indices + .iter() + .map(|i| tree.leaf(*i as usize).unwrap()) + .collect_vec(); + let revealed_leafs = leaf_indices + .into_iter() + .zip_eq(revealed_leafs) + .collect_vec(); + let nd_siblings = nd_sibling_indices + .iter() + .map(|(left_idx, right_idx)| { + ( + tree.node(*left_idx as usize).unwrap(), + tree.node(*right_idx as usize).unwrap(), + ) + }) + .collect_vec(); + + let mmr_auth_struct = MerkleAuthenticationStructAuthenticityWitness { + nd_auth_struct_indices, + nd_sibling_indices, + nd_siblings, + }; + let tree_height: u32 = tree_height.try_into().unwrap(); + let calculated_root = mmr_auth_struct.root_from_authentication_struct( + tree_height, + auth_struct, + revealed_leafs, + ); + assert_eq!(tree.root(), calculated_root); + } + + #[test] + fn root_from_authentication_struct_tree_height_0_no_revealed_leafs() { + let tree_height = 0; + let leaf_indices = vec![]; + let nd_auth_struct_indices = vec![1]; + let nd_sibling_indices = vec![]; + prop_from_merkle_tree( + tree_height, + leaf_indices, + nd_auth_struct_indices, + nd_sibling_indices, + ) + } + + #[test] + fn root_from_authentication_struct_tree_height_0_1_revealed() { + let tree_height = 0; + let leaf_indices = vec![0]; + let nd_auth_struct_indices = vec![]; + let nd_sibling_indices = vec![]; + prop_from_merkle_tree( + tree_height, + leaf_indices, + nd_auth_struct_indices, + nd_sibling_indices, + ) + } + + #[test] + fn root_from_authentication_struct_tree_height_1_1_revealed() { + let tree_height = 1; + let leaf_indices = vec![0u64]; + let nd_auth_struct_indices = vec![3]; + let nd_sibling_indices = vec![(2u64, 3u64)]; + prop_from_merkle_tree( + tree_height, + leaf_indices, + nd_auth_struct_indices, + nd_sibling_indices, + ) + } + + #[test] + fn root_from_authentication_struct_tree_height_1_2_revealed() { + let tree_height = 1; + let leaf_indices = vec![0u64, 1]; + let nd_auth_struct_indices = vec![]; + let nd_sibling_indices = vec![(2u64, 3u64)]; + prop_from_merkle_tree( + tree_height, + leaf_indices, + nd_auth_struct_indices, + nd_sibling_indices, + ) + } + + #[test] + fn root_from_authentication_struct_tree_height_2_0_revealed() { + let tree_height = 2; + let leaf_indices = vec![]; + let auth_struct_indices = vec![1]; + let nd_sibling_indices = vec![]; + prop_from_merkle_tree( + tree_height, + leaf_indices, + auth_struct_indices, + nd_sibling_indices, + ) + } + + #[test] + fn root_from_authentication_struct_tree_height_2_2_revealed() { + let tree_height = 2; + let leaf_indices = vec![0u64, 1]; + let auth_struct_indices = vec![3]; + let nd_sibling_indices = vec![(4u64, 5u64), (2, 3)]; + prop_from_merkle_tree( + tree_height, + leaf_indices, + auth_struct_indices, + nd_sibling_indices, + ) + } + + #[test] + fn root_from_authentication_struct_tree_height_4_4_revealed() { + let tree_height = 4; + let leaf_indices = vec![14u64, 12, 10, 8]; + let num_leafs = 1 << tree_height; + let auth_struct_indices = vec![ + num_leafs + 15, + num_leafs + 13, + num_leafs + 11, + num_leafs + 9, + 2, + ]; + let nd_sibling_indices_layer_height_0 = [(14u64, 15u64), (12, 13), (10, 11), (8, 9)] + .map(|(l, r)| (l + num_leafs, r + num_leafs)); + let nd_sibling_indices_layer_height_1 = [(14u64, 15u64), (12u64, 13u64)]; + let nd_sibling_indices_layer_height_2 = [(6u64, 7u64)]; + let nd_sibling_indices_layer_height_3 = [(2u64, 3u64)]; + let nd_sibling_indices = [ + nd_sibling_indices_layer_height_0.to_vec(), + nd_sibling_indices_layer_height_1.to_vec(), + nd_sibling_indices_layer_height_2.to_vec(), + nd_sibling_indices_layer_height_3.to_vec(), + ] + .concat(); + + prop_from_merkle_tree( + tree_height, + leaf_indices, + auth_struct_indices, + nd_sibling_indices, + ) + } +} From 4b2539cbb8ed493d58d48ad5f7032e32d4b2dbb4 Mon Sep 17 00:00:00 2001 From: sword_smith Date: Wed, 7 Aug 2024 15:35:42 +0200 Subject: [PATCH 02/11] refactor(mmr_authentication_struct): Encapsulate code for nd-sibling indices --- .../mmr/mmr_authentication_struct.rs | 83 ++++++++++--------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index 89006c54b..97b78325e 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -156,6 +156,50 @@ impl MerkleAuthenticationStructAuthenticityWitness { tree: &MerkleTree, mut revealed_leaf_indices: Vec, ) -> (Self, Vec, Vec<(u64, Digest)>) { + fn sibling_indices( + revealed_leaf_indices: &[u64], + nd_auth_struct_indices: &[u64], + num_leafs: u64, + ) -> Vec<(u64, u64)> { + let mut nd_sibling_indices = revealed_leaf_indices + .iter() + .map(|li| *li ^ num_leafs) + .chain(nd_auth_struct_indices.iter().copied()) + .filter(|idx| *idx != 1) + .map(|idx| (idx & (u64::MAX - 1), idx | 1u64)) + .unique() + .collect_vec(); + + if !nd_sibling_indices.is_empty() { + // TODO: I think we can use `PartialMerkleTree` to calculate all + // indices, and maybe also to get all the digests of `nd_siblings`. + let mut i = 0; + loop { + let elm = nd_sibling_indices[i]; + let parent = elm.0 >> 1; + if parent == 1 { + break; + } + let uncle = parent ^ 1; + + let new_pair = if parent & 1 == 0 { + (parent, uncle) + } else { + (uncle, parent) + }; + if !nd_sibling_indices.contains(&new_pair) { + nd_sibling_indices.push(new_pair); + } + + nd_sibling_indices.sort_by_key(|(left_idx, _right_idx)| *left_idx); + nd_sibling_indices.reverse(); + + i += 1; + } + } + + nd_sibling_indices + } revealed_leaf_indices.sort_unstable(); revealed_leaf_indices.dedup(); revealed_leaf_indices.reverse(); @@ -168,43 +212,8 @@ impl MerkleAuthenticationStructAuthenticityWitness { nd_auth_struct_indices = vec![ROOT_MT_INDEX]; } - let mut nd_sibling_indices = revealed_leaf_indices - .iter() - .map(|li| *li ^ num_leafs) - .chain(nd_auth_struct_indices.iter().copied()) - .filter(|idx| *idx != 1) - .map(|idx| (idx & (u64::MAX - 1), idx | 1u64)) - .unique() - .collect_vec(); - - if !nd_sibling_indices.is_empty() { - // TODO: I think we can use `PartialMerkleTree` to calculate all - // indices, and maybe also to get all the digests of `nd_siblings`. - let mut i = 0; - loop { - let elm = nd_sibling_indices[i]; - let parent = elm.0 >> 1; - if parent == 1 { - break; - } - let uncle = parent ^ 1; - - let new_pair = if parent & 1 == 0 { - (parent, uncle) - } else { - (uncle, parent) - }; - if !nd_sibling_indices.contains(&new_pair) { - nd_sibling_indices.push(new_pair); - } - - nd_sibling_indices.sort_by_key(|(left_idx, _right_idx)| *left_idx); - nd_sibling_indices.reverse(); - - i += 1; - } - } - + let nd_sibling_indices = + sibling_indices(&revealed_leaf_indices, &nd_auth_struct_indices, num_leafs); let nd_siblings = nd_sibling_indices .iter() .map(|&(l, r)| { From 2514703b7103bfee21eec9eaeef8b4fd7b0abe61 Mon Sep 17 00:00:00 2001 From: sword_smith Date: Wed, 7 Aug 2024 15:46:04 +0200 Subject: [PATCH 03/11] docs(mmr_authentication_struct): Add note about better way of finding nd-sibling indices --- .../src/util_types/mmr/mmr_authentication_struct.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index 97b78325e..2eb9d2547 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -156,11 +156,13 @@ impl MerkleAuthenticationStructAuthenticityWitness { tree: &MerkleTree, mut revealed_leaf_indices: Vec, ) -> (Self, Vec, Vec<(u64, Digest)>) { - fn sibling_indices( + fn nd_sibling_indices( revealed_leaf_indices: &[u64], nd_auth_struct_indices: &[u64], num_leafs: u64, ) -> Vec<(u64, u64)> { + // TODO: For a better way finding all nd-sibling indices, see the + // code for [`PartialMerkleTree`] in the `merkle_tree` module. let mut nd_sibling_indices = revealed_leaf_indices .iter() .map(|li| *li ^ num_leafs) @@ -171,8 +173,6 @@ impl MerkleAuthenticationStructAuthenticityWitness { .collect_vec(); if !nd_sibling_indices.is_empty() { - // TODO: I think we can use `PartialMerkleTree` to calculate all - // indices, and maybe also to get all the digests of `nd_siblings`. let mut i = 0; loop { let elm = nd_sibling_indices[i]; @@ -213,7 +213,7 @@ impl MerkleAuthenticationStructAuthenticityWitness { } let nd_sibling_indices = - sibling_indices(&revealed_leaf_indices, &nd_auth_struct_indices, num_leafs); + nd_sibling_indices(&revealed_leaf_indices, &nd_auth_struct_indices, num_leafs); let nd_siblings = nd_sibling_indices .iter() .map(|&(l, r)| { From 71aec1e1019e3d0c337ab4b5918a603c3af996c8 Mon Sep 17 00:00:00 2001 From: sword_smith Date: Wed, 7 Aug 2024 17:42:35 +0200 Subject: [PATCH 04/11] feat(mmr_authentication_struct): Derive from MMR & MMR MPs Add a function to derive an MMR authentication structure from an MMR accumulator and a list of indexed MMR membership proofs. --- .../mmr/mmr_authentication_struct.rs | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index 2eb9d2547..adca449c4 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::collections::HashSet; use itertools::Itertools; @@ -10,9 +11,15 @@ use crate::prelude::BFieldElement; use crate::prelude::Digest; use crate::prelude::Inverse; use crate::prelude::MerkleTree; +use crate::prelude::Mmr; +use crate::prelude::MmrMembershipProof; use crate::prelude::Sponge; use crate::prelude::Tip5; use crate::prelude::XFieldElement; +use crate::util_types::mmr::shared_advanced::get_peak_heights; +use crate::util_types::mmr::shared_basic::leaf_index_to_mt_index_and_peak_index; + +use super::mmr_accumulator::MmrAccumulator; const ROOT_MT_INDEX: u64 = 1; @@ -150,6 +157,144 @@ impl MerkleAuthenticationStructAuthenticityWitness { t } + /// Return the authentication structure authenticity witness, + /// authentication structure, and the (leaf-index, leaf-digest) pairs + /// from a list of MMR membership proofs. All MMR membership proofs must + /// belong under the same peak, i.e., be part of the same Merkle tree in + /// the list of Merkle trees that the MMR contains. + pub fn new_from_mmr_membership_proofs( + mmra: &MmrAccumulator, + indexed_mmr_mps: Vec<(u64, Digest, MmrMembershipProof)>, + ) -> (Self, Vec, Vec<(u64, Digest)>) { + // TODO: Consider rewriting this to return a list of authenticated + // authentication structs, one for each peak in question. + + // Verify that MMR leaf indices belong to the same peak + let num_mmr_leafs = mmra.num_leafs(); + let mt_and_peak_indices = indexed_mmr_mps + .iter() + .map(|(mmr_leaf_index, _leaf, _mmr_mp)| { + leaf_index_to_mt_index_and_peak_index(*mmr_leaf_index, num_mmr_leafs) + }) + .collect_vec(); + println!("mt_and_peak_indices\n{mt_and_peak_indices:?}"); + + assert!( + mt_and_peak_indices + .iter() + .map(|(_mt_index, peak_index)| peak_index) + .unique() + .count() + < 2 + ); + + // TODO: Also handle the trivial case where indexed_mmr_mps is the + // empty list. + let peak_index = mt_and_peak_indices[0].1; + let mt_indices = mt_and_peak_indices + .into_iter() + .map(|(mt_index, _peak_index)| mt_index) + .collect_vec(); + + let peak_index: usize = peak_index.try_into().unwrap(); + let height_of_local_mt = get_peak_heights(num_mmr_leafs)[peak_index]; + let num_leafs_in_local_mt = 1 << height_of_local_mt; + let local_mt_leaf_indices = mt_indices + .iter() + .map(|mt_index| mt_index - num_leafs_in_local_mt) + .collect_vec(); + + let mut nd_auth_struct_indices = Self::authentication_structure_mt_indices( + num_leafs_in_local_mt, + &local_mt_leaf_indices, + ) + .collect_vec(); + if mt_indices.is_empty() { + nd_auth_struct_indices = vec![ROOT_MT_INDEX]; + } + + let mut nd_left_indices = mt_indices + .iter() + .chain(nd_auth_struct_indices.iter()) + .filter(|idx| **idx != 1) + .map(|idx| idx & (!1)) + .unique() + .collect_vec(); + if !nd_left_indices.is_empty() { + let mut i = 0; + loop { + let parent = nd_left_indices[i] >> 1; + if parent == 1 { + break; + } + + let new_left_node = parent & (!1); + if !nd_left_indices.contains(&new_left_node) { + nd_left_indices.push(new_left_node); + } + + nd_left_indices.sort_unstable(); + nd_left_indices.reverse(); + + i += 1; + } + } + + let nd_sibling_indices = nd_left_indices + .into_iter() + .map(|idx| (idx, idx + 1)) + .collect_vec(); + + // Collect all node digests that can be calculated + let peak = mmra.peaks()[peak_index]; + let mut node_digests: HashMap = HashMap::default(); + node_digests.insert(ROOT_MT_INDEX, peak); + for ((_mmr_leaf_index, mut node, mmr_mp), mut mt_index) in indexed_mmr_mps + .clone() + .into_iter() + .zip_eq(mt_indices.clone()) + { + for ap_elem in mmr_mp.authentication_path.iter() { + node_digests.insert(mt_index, node); + node_digests.insert(mt_index ^ 1, *ap_elem); + node = if mt_index & 1 == 0 { + Tip5::hash_pair(node, *ap_elem) + } else { + Tip5::hash_pair(*ap_elem, node) + }; + + mt_index /= 2; + } + } + + println!("nd_sibling_indices: {nd_sibling_indices:?}"); + println!("node_digests keys: {:?}", node_digests.keys()); + let nd_siblings = nd_sibling_indices + .iter() + .map(|(left_idx, right_idx)| (node_digests[left_idx], node_digests[right_idx])) + .collect_vec(); + let auth_struct = nd_auth_struct_indices + .iter() + .map(|idx| node_digests[idx]) + .collect_vec(); + let indexed_leafs = local_mt_leaf_indices + .into_iter() + .zip_eq( + indexed_mmr_mps + .into_iter() + .map(|(_mmr_leaf_index, leaf, _mmr_mp)| leaf), + ) + .collect_vec(); + + let auth_struct_witness = Self { + nd_auth_struct_indices, + nd_sibling_indices, + nd_siblings, + }; + + (auth_struct_witness, auth_struct, indexed_leafs) + } + /// Return the authentication structure witness, authentication structure, /// and the (leaf-index, leaf-digest) pairs. pub fn new_from_merkle_tree( @@ -200,6 +345,7 @@ impl MerkleAuthenticationStructAuthenticityWitness { nd_sibling_indices } + revealed_leaf_indices.sort_unstable(); revealed_leaf_indices.dedup(); revealed_leaf_indices.reverse(); @@ -253,12 +399,80 @@ mod tests { use crate::math::other::random_elements; use crate::prelude::CpuParallel; use crate::prelude::MerkleTree; + use crate::util_types::mmr::mmr_accumulator::util::mmra_with_mps; use proptest::collection::vec; use proptest::prop_assert_eq; + use rand::random; use test_strategy::proptest; use super::*; + #[test] + fn auth_struct_from_mmr_mps_test_height_5_9_indices() { + let local_tree_height = 5; + let mmr_leaf_indices = [0, 1, 2, 16, 17, 18, 27, 29, 31]; + let indexed_leafs_input: Vec<(u64, Digest)> = mmr_leaf_indices + .iter() + .map(|idx| (*idx, random())) + .collect_vec(); + let (mmra, mmr_mps) = mmra_with_mps(1 << local_tree_height, indexed_leafs_input.clone()); + let indexed_mmr_mps = mmr_mps + .into_iter() + .zip_eq(indexed_leafs_input) + .map(|(mmr_mp, (idx, leaf))| (idx, leaf, mmr_mp)) + .collect_vec(); + + let (mmr_auth_struct, auth_struct, indexed_leafs) = + MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( + &mmra, + indexed_mmr_mps, + ); + + let tree_height: u32 = local_tree_height.try_into().unwrap(); + let computed_root = mmr_auth_struct.root_from_authentication_struct( + tree_height, + auth_struct, + indexed_leafs, + ); + + let peak_index = 0; + let expected_root = mmra.peaks()[peak_index]; + assert_eq!(expected_root, computed_root); + } + + #[test] + fn auth_struct_from_mmr_mps_test_height_4_2_indices() { + let local_tree_height = 4; + let mmr_leaf_indices = [0, 1]; + let indexed_leafs_input: Vec<(u64, Digest)> = mmr_leaf_indices + .iter() + .map(|idx| (*idx, random())) + .collect_vec(); + let (mmra, mmr_mps) = mmra_with_mps(1 << local_tree_height, indexed_leafs_input.clone()); + let indexed_mmr_mps = mmr_mps + .into_iter() + .zip_eq(indexed_leafs_input) + .map(|(mmr_mp, (idx, leaf))| (idx, leaf, mmr_mp)) + .collect_vec(); + + let (mmr_auth_struct, auth_struct, indexed_leafs) = + MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( + &mmra, + indexed_mmr_mps, + ); + + let tree_height: u32 = local_tree_height.try_into().unwrap(); + let computed_root = mmr_auth_struct.root_from_authentication_struct( + tree_height, + auth_struct, + indexed_leafs, + ); + + let peak_index = 0; + let expected_root = mmra.peaks()[peak_index]; + assert_eq!(expected_root, computed_root); + } + #[proptest(cases = 20)] fn root_from_authentication_struct_prop_test( #[strategy(0..12u64)] tree_height: u64, From 6c08b5d8956782b061e96e8d3611b9e2d285993b Mon Sep 17 00:00:00 2001 From: sword_smith Date: Thu, 8 Aug 2024 12:09:49 +0200 Subject: [PATCH 05/11] refactor(mmr_authnetication_struct): Cleanup MMR -> auth struct code --- .../mmr/mmr_authentication_struct.rs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index adca449c4..a11293cda 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -162,6 +162,9 @@ impl MerkleAuthenticationStructAuthenticityWitness { /// from a list of MMR membership proofs. All MMR membership proofs must /// belong under the same peak, i.e., be part of the same Merkle tree in /// the list of Merkle trees that the MMR contains. + /// + /// Panics if the input list of MMR-membership proofs is empty, or if they + /// do not all belong under the same peak. pub fn new_from_mmr_membership_proofs( mmra: &MmrAccumulator, indexed_mmr_mps: Vec<(u64, Digest, MmrMembershipProof)>, @@ -177,7 +180,6 @@ impl MerkleAuthenticationStructAuthenticityWitness { leaf_index_to_mt_index_and_peak_index(*mmr_leaf_index, num_mmr_leafs) }) .collect_vec(); - println!("mt_and_peak_indices\n{mt_and_peak_indices:?}"); assert!( mt_and_peak_indices @@ -187,9 +189,8 @@ impl MerkleAuthenticationStructAuthenticityWitness { .count() < 2 ); + assert!(!mt_and_peak_indices.is_empty(), ""); - // TODO: Also handle the trivial case where indexed_mmr_mps is the - // empty list. let peak_index = mt_and_peak_indices[0].1; let mt_indices = mt_and_peak_indices .into_iter() @@ -204,14 +205,11 @@ impl MerkleAuthenticationStructAuthenticityWitness { .map(|mt_index| mt_index - num_leafs_in_local_mt) .collect_vec(); - let mut nd_auth_struct_indices = Self::authentication_structure_mt_indices( + let nd_auth_struct_indices = Self::authentication_structure_mt_indices( num_leafs_in_local_mt, &local_mt_leaf_indices, ) .collect_vec(); - if mt_indices.is_empty() { - nd_auth_struct_indices = vec![ROOT_MT_INDEX]; - } let mut nd_left_indices = mt_indices .iter() @@ -220,7 +218,10 @@ impl MerkleAuthenticationStructAuthenticityWitness { .map(|idx| idx & (!1)) .unique() .collect_vec(); - if !nd_left_indices.is_empty() { + + // Fill nd-indices with all non-root indices for node values that can be + // derived from those from the auth-struct and the leafs. + { let mut i = 0; loop { let parent = nd_left_indices[i] >> 1; @@ -265,10 +266,11 @@ impl MerkleAuthenticationStructAuthenticityWitness { mt_index /= 2; } + + // Sanity check that MMR-MPs are valid + assert_eq!(peak, node, "Derived peak must match provided peak"); } - println!("nd_sibling_indices: {nd_sibling_indices:?}"); - println!("node_digests keys: {:?}", node_digests.keys()); let nd_siblings = nd_sibling_indices .iter() .map(|(left_idx, right_idx)| (node_digests[left_idx], node_digests[right_idx])) From 02799be5bb7f591ca8af49800dfcb91d72ffc2fa Mon Sep 17 00:00:00 2001 From: sword_smith Date: Thu, 8 Aug 2024 12:37:43 +0200 Subject: [PATCH 06/11] mmr_authentication_struct: Simplify derivation of nd-sibling indices The work to find these indices was already done in a helper function. No need to derive them again inside of the constructor-functions. --- .../mmr/mmr_authentication_struct.rs | 124 +++++------------- 1 file changed, 30 insertions(+), 94 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index a11293cda..1d38f5ae9 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -32,11 +32,13 @@ pub struct MerkleAuthenticationStructAuthenticityWitness { impl MerkleAuthenticationStructAuthenticityWitness { /// Return the Merkle tree node indices of the digests required to prove - /// membership for the specified leaf indices - fn authentication_structure_mt_indices( + /// membership for the specified leaf indices, as well as the node indices + /// that can be derived from the leaf indices and their authentication + /// path. + fn auth_struct_and_nd_indices( num_leafs: u64, leaf_indices: &[u64], - ) -> impl ExactSizeIterator { + ) -> (Vec, Vec<(u64, u64)>) { // The set of indices of nodes that need to be included in the authentications // structure. In principle, every node of every authentication path is needed. // The root is never needed. Hence, it is not considered below. @@ -61,7 +63,27 @@ impl MerkleAuthenticationStructAuthenticityWitness { } let set_difference = node_is_needed.difference(&node_can_be_computed).copied(); - set_difference.sorted_unstable().rev() + let set_union = node_is_needed + .union(&node_can_be_computed) + .sorted_unstable() + .rev(); + + let mut set_union = set_union.peekable(); + + let mut set_union_as_ordered_pairs = Vec::new(); + while set_union.peek().is_some() { + let right_index = *set_union.next().unwrap(); + + // Crashes on odd-length of input list, which is what we want, as + // this acts as a sanity check. + let left_index = *set_union.next().unwrap(); + set_union_as_ordered_pairs.push((left_index, right_index)); + } + + ( + set_difference.sorted_unstable().rev().collect(), + set_union_as_ordered_pairs, + ) } pub fn root_from_authentication_struct( @@ -205,46 +227,8 @@ impl MerkleAuthenticationStructAuthenticityWitness { .map(|mt_index| mt_index - num_leafs_in_local_mt) .collect_vec(); - let nd_auth_struct_indices = Self::authentication_structure_mt_indices( - num_leafs_in_local_mt, - &local_mt_leaf_indices, - ) - .collect_vec(); - - let mut nd_left_indices = mt_indices - .iter() - .chain(nd_auth_struct_indices.iter()) - .filter(|idx| **idx != 1) - .map(|idx| idx & (!1)) - .unique() - .collect_vec(); - - // Fill nd-indices with all non-root indices for node values that can be - // derived from those from the auth-struct and the leafs. - { - let mut i = 0; - loop { - let parent = nd_left_indices[i] >> 1; - if parent == 1 { - break; - } - - let new_left_node = parent & (!1); - if !nd_left_indices.contains(&new_left_node) { - nd_left_indices.push(new_left_node); - } - - nd_left_indices.sort_unstable(); - nd_left_indices.reverse(); - - i += 1; - } - } - - let nd_sibling_indices = nd_left_indices - .into_iter() - .map(|idx| (idx, idx + 1)) - .collect_vec(); + let (nd_auth_struct_indices, nd_sibling_indices) = + Self::auth_struct_and_nd_indices(num_leafs_in_local_mt, &local_mt_leaf_indices); // Collect all node digests that can be calculated let peak = mmra.peaks()[peak_index]; @@ -303,65 +287,17 @@ impl MerkleAuthenticationStructAuthenticityWitness { tree: &MerkleTree, mut revealed_leaf_indices: Vec, ) -> (Self, Vec, Vec<(u64, Digest)>) { - fn nd_sibling_indices( - revealed_leaf_indices: &[u64], - nd_auth_struct_indices: &[u64], - num_leafs: u64, - ) -> Vec<(u64, u64)> { - // TODO: For a better way finding all nd-sibling indices, see the - // code for [`PartialMerkleTree`] in the `merkle_tree` module. - let mut nd_sibling_indices = revealed_leaf_indices - .iter() - .map(|li| *li ^ num_leafs) - .chain(nd_auth_struct_indices.iter().copied()) - .filter(|idx| *idx != 1) - .map(|idx| (idx & (u64::MAX - 1), idx | 1u64)) - .unique() - .collect_vec(); - - if !nd_sibling_indices.is_empty() { - let mut i = 0; - loop { - let elm = nd_sibling_indices[i]; - let parent = elm.0 >> 1; - if parent == 1 { - break; - } - let uncle = parent ^ 1; - - let new_pair = if parent & 1 == 0 { - (parent, uncle) - } else { - (uncle, parent) - }; - if !nd_sibling_indices.contains(&new_pair) { - nd_sibling_indices.push(new_pair); - } - - nd_sibling_indices.sort_by_key(|(left_idx, _right_idx)| *left_idx); - nd_sibling_indices.reverse(); - - i += 1; - } - } - - nd_sibling_indices - } - revealed_leaf_indices.sort_unstable(); revealed_leaf_indices.dedup(); revealed_leaf_indices.reverse(); let num_leafs: u64 = tree.num_leafs() as u64; - let mut nd_auth_struct_indices = - Self::authentication_structure_mt_indices(num_leafs, &revealed_leaf_indices) - .collect_vec(); + let (mut nd_auth_struct_indices, nd_sibling_indices) = + Self::auth_struct_and_nd_indices(num_leafs, &revealed_leaf_indices); if revealed_leaf_indices.is_empty() { nd_auth_struct_indices = vec![ROOT_MT_INDEX]; } - let nd_sibling_indices = - nd_sibling_indices(&revealed_leaf_indices, &nd_auth_struct_indices, num_leafs); let nd_siblings = nd_sibling_indices .iter() .map(|&(l, r)| { From c2dd48d4131768c874c7f5662bedba0e63ac01ea Mon Sep 17 00:00:00 2001 From: sword_smith Date: Thu, 8 Aug 2024 15:34:16 +0200 Subject: [PATCH 07/11] mmr_authentication_struct: Build multiple witnesses from one MMR Add a struct for encapulating the Merkle authentication struct authenticity witness and one for encapsulating the auth struct, its withness, and the data, which are the indexed leaves. --- .../mmr/mmr_authentication_struct.rs | 300 ++++++++++++------ 1 file changed, 198 insertions(+), 102 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index 1d38f5ae9..9caacb85e 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -23,11 +23,24 @@ use super::mmr_accumulator::MmrAccumulator; const ROOT_MT_INDEX: u64 = 1; +/// A witness to facilitate the proving of the authenticity of a Merkle +/// authentication struct. +#[derive(Debug, Clone)] pub struct MerkleAuthenticationStructAuthenticityWitness { // All indices are Merkle tree node indices - nd_auth_struct_indices: Vec, - nd_sibling_indices: Vec<(u64, u64)>, - nd_siblings: Vec<(Digest, Digest)>, + pub nd_auth_struct_indices: Vec, + pub nd_sibling_indices: Vec<(u64, u64)>, + pub nd_siblings: Vec<(Digest, Digest)>, +} + +/// An authentication structure that can be used to prove membership of a list +/// of leaves in a Merkle tree, along with the indexed leaves in question, and +/// the witness necessary to prove membership in a ZK program. +#[derive(Debug, Clone)] +pub struct AuthenticatedMerkleAuthStruct { + pub auth_struct: Vec, + pub indexed_leafs: Vec<(u64, Digest)>, + pub witness: MerkleAuthenticationStructAuthenticityWitness, } impl MerkleAuthenticationStructAuthenticityWitness { @@ -190,95 +203,112 @@ impl MerkleAuthenticationStructAuthenticityWitness { pub fn new_from_mmr_membership_proofs( mmra: &MmrAccumulator, indexed_mmr_mps: Vec<(u64, Digest, MmrMembershipProof)>, - ) -> (Self, Vec, Vec<(u64, Digest)>) { - // TODO: Consider rewriting this to return a list of authenticated - // authentication structs, one for each peak in question. + ) -> HashMap { + #[derive(Clone, Debug)] + struct IndexedAuthenticatedMmrLeaf { + merkle_tree_node_index: u64, + merkle_tree_leaf_index: u64, + leaf_digest: Digest, + membership_proof: MmrMembershipProof, + } - // Verify that MMR leaf indices belong to the same peak + // Split indexed MMR-mps into a hashmap with one entry for each + // referenced peak in the MMR. let num_mmr_leafs = mmra.num_leafs(); - let mt_and_peak_indices = indexed_mmr_mps - .iter() - .map(|(mmr_leaf_index, _leaf, _mmr_mp)| { - leaf_index_to_mt_index_and_peak_index(*mmr_leaf_index, num_mmr_leafs) - }) - .collect_vec(); + let mut peak_index_to_indexed_mmr_mp: HashMap> = + HashMap::default(); + let peak_heights = get_peak_heights(num_mmr_leafs); + for (mmr_leaf_index, leaf, mmr_mp) in indexed_mmr_mps { + let (mt_index, peak_index) = + leaf_index_to_mt_index_and_peak_index(mmr_leaf_index, num_mmr_leafs); + let peak_index_as_usize: usize = peak_index.try_into().unwrap(); + let num_leafs_local_mt = 1 << peak_heights[peak_index_as_usize]; + let mt_leaf_index = mt_index - num_leafs_local_mt; + peak_index_to_indexed_mmr_mp + .entry(peak_index) + .or_default() + .push(IndexedAuthenticatedMmrLeaf { + merkle_tree_node_index: mt_index, + merkle_tree_leaf_index: mt_leaf_index, + leaf_digest: leaf, + membership_proof: mmr_mp, + }); + } - assert!( - mt_and_peak_indices + // Loop over all peaks and collect an authentication witness struct + // for each peak. + let mut peak_index_to_authenticated_auth_struct = HashMap::default(); + for (peak_index, indexed_mmr_mp_structs) in peak_index_to_indexed_mmr_mp { + let peak_index_as_usize: usize = peak_index.try_into().unwrap(); + let num_leafs_in_local_mt = 1 << peak_heights[peak_index_as_usize]; + let local_mt_leaf_indices = indexed_mmr_mp_structs .iter() - .map(|(_mt_index, peak_index)| peak_index) - .unique() - .count() - < 2 - ); - assert!(!mt_and_peak_indices.is_empty(), ""); - - let peak_index = mt_and_peak_indices[0].1; - let mt_indices = mt_and_peak_indices - .into_iter() - .map(|(mt_index, _peak_index)| mt_index) - .collect_vec(); - - let peak_index: usize = peak_index.try_into().unwrap(); - let height_of_local_mt = get_peak_heights(num_mmr_leafs)[peak_index]; - let num_leafs_in_local_mt = 1 << height_of_local_mt; - let local_mt_leaf_indices = mt_indices - .iter() - .map(|mt_index| mt_index - num_leafs_in_local_mt) - .collect_vec(); - - let (nd_auth_struct_indices, nd_sibling_indices) = - Self::auth_struct_and_nd_indices(num_leafs_in_local_mt, &local_mt_leaf_indices); - - // Collect all node digests that can be calculated - let peak = mmra.peaks()[peak_index]; - let mut node_digests: HashMap = HashMap::default(); - node_digests.insert(ROOT_MT_INDEX, peak); - for ((_mmr_leaf_index, mut node, mmr_mp), mut mt_index) in indexed_mmr_mps - .clone() - .into_iter() - .zip_eq(mt_indices.clone()) - { - for ap_elem in mmr_mp.authentication_path.iter() { - node_digests.insert(mt_index, node); - node_digests.insert(mt_index ^ 1, *ap_elem); - node = if mt_index & 1 == 0 { - Tip5::hash_pair(node, *ap_elem) - } else { - Tip5::hash_pair(*ap_elem, node) - }; - - mt_index /= 2; + .map(|x| x.merkle_tree_leaf_index) + .collect_vec(); + + let (nd_auth_struct_indices, nd_sibling_indices) = + Self::auth_struct_and_nd_indices(num_leafs_in_local_mt, &local_mt_leaf_indices); + let peak = mmra.peaks()[peak_index_as_usize]; + + let mut node_digests: HashMap = HashMap::default(); + node_digests.insert(ROOT_MT_INDEX, peak); + + // Loop over all indexed leafs for this peak + for indexed_mmr_mp in indexed_mmr_mp_structs.iter() { + let mut mt_node_index = indexed_mmr_mp.merkle_tree_node_index; + let mut node = indexed_mmr_mp.leaf_digest; + + // Loop over all authentication path elements for this indexed leaf + for ap_elem in indexed_mmr_mp.membership_proof.authentication_path.iter() { + node_digests.insert(mt_node_index, node); + node_digests.insert(mt_node_index ^ 1, *ap_elem); + node = if mt_node_index & 1 == 0 { + Tip5::hash_pair(node, *ap_elem) + } else { + Tip5::hash_pair(*ap_elem, node) + }; + + mt_node_index /= 2; + } + + // Sanity check that MMR-MPs are valid + assert_eq!(peak, node, "Derived peak must match provided peak"); } - - // Sanity check that MMR-MPs are valid - assert_eq!(peak, node, "Derived peak must match provided peak"); + let nd_siblings = nd_sibling_indices + .iter() + .map(|(left_idx, right_idx)| (node_digests[left_idx], node_digests[right_idx])) + .collect_vec(); + let auth_struct = nd_auth_struct_indices + .iter() + .map(|idx| node_digests[idx]) + .collect_vec(); + let indexed_leafs = indexed_mmr_mp_structs + .into_iter() + .map(|indexed_mmr_mp| { + ( + indexed_mmr_mp.merkle_tree_leaf_index, + indexed_mmr_mp.leaf_digest, + ) + }) + .collect_vec(); + + let witness = Self { + nd_auth_struct_indices, + nd_sibling_indices, + nd_siblings, + }; + + peak_index_to_authenticated_auth_struct.insert( + peak_index, + AuthenticatedMerkleAuthStruct { + auth_struct, + indexed_leafs, + witness, + }, + ); } - let nd_siblings = nd_sibling_indices - .iter() - .map(|(left_idx, right_idx)| (node_digests[left_idx], node_digests[right_idx])) - .collect_vec(); - let auth_struct = nd_auth_struct_indices - .iter() - .map(|idx| node_digests[idx]) - .collect_vec(); - let indexed_leafs = local_mt_leaf_indices - .into_iter() - .zip_eq( - indexed_mmr_mps - .into_iter() - .map(|(_mmr_leaf_index, leaf, _mmr_mp)| leaf), - ) - .collect_vec(); - - let auth_struct_witness = Self { - nd_auth_struct_indices, - nd_sibling_indices, - nd_siblings, - }; - - (auth_struct_witness, auth_struct, indexed_leafs) + peak_index_to_authenticated_auth_struct } /// Return the authentication structure witness, authentication structure, @@ -286,7 +316,7 @@ impl MerkleAuthenticationStructAuthenticityWitness { pub fn new_from_merkle_tree( tree: &MerkleTree, mut revealed_leaf_indices: Vec, - ) -> (Self, Vec, Vec<(u64, Digest)>) { + ) -> AuthenticatedMerkleAuthStruct { revealed_leaf_indices.sort_unstable(); revealed_leaf_indices.dedup(); revealed_leaf_indices.reverse(); @@ -322,13 +352,17 @@ impl MerkleAuthenticationStructAuthenticityWitness { .map(|node_index| tree.node(*node_index as usize).unwrap()) .collect_vec(); - let auth_struct_witness = Self { + let witness = Self { nd_auth_struct_indices, nd_sibling_indices, nd_siblings, }; - (auth_struct_witness, auth_struct, indexed_leafs) + AuthenticatedMerkleAuthStruct { + auth_struct, + indexed_leafs, + witness, + } } } @@ -345,6 +379,48 @@ mod tests { use super::*; + #[proptest(cases = 20)] + fn root_from_authentication_struct_mmr_prop_test( + #[strategy(0..u64::MAX / 2)] mmr_leaf_count: u64, + #[strategy(0usize..20)] _num_revealed_leafs: usize, + #[strategy(vec(0u64..#mmr_leaf_count, #_num_revealed_leafs))] + mmr_revealed_leaf_indices: Vec, + ) { + let indexed_leafs_input: Vec<(u64, Digest)> = mmr_revealed_leaf_indices + .iter() + .map(|idx| (*idx, random())) + .collect_vec(); + let (mmra, mmr_mps) = mmra_with_mps(mmr_leaf_count, indexed_leafs_input.clone()); + let indexed_mmr_mps = mmr_mps + .into_iter() + .zip_eq(indexed_leafs_input) + .map(|(mmr_mp, (idx, leaf))| (idx, leaf, mmr_mp)) + .collect_vec(); + + let authenticated_auth_structs = + MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( + &mmra, + indexed_mmr_mps, + ); + + let peak_heights = get_peak_heights(mmr_leaf_count); + for (peak_index, authentication_auth_struct) in authenticated_auth_structs { + let AuthenticatedMerkleAuthStruct { + auth_struct, + indexed_leafs, + witness, + } = &authentication_auth_struct; + let tree_height: u32 = peak_heights[peak_index as usize]; + let computed_root = witness.root_from_authentication_struct( + tree_height, + auth_struct.to_owned(), + indexed_leafs.to_owned(), + ); + + prop_assert_eq!(mmra.peaks()[peak_index as usize], computed_root); + } + } + #[test] fn auth_struct_from_mmr_mps_test_height_5_9_indices() { let local_tree_height = 5; @@ -360,17 +436,26 @@ mod tests { .map(|(mmr_mp, (idx, leaf))| (idx, leaf, mmr_mp)) .collect_vec(); - let (mmr_auth_struct, auth_struct, indexed_leafs) = + let authenticity_witnesses = MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( &mmra, indexed_mmr_mps, ); + assert!( + authenticity_witnesses.len().is_one(), + "All indices belong to first peak" + ); + let AuthenticatedMerkleAuthStruct { + auth_struct, + indexed_leafs, + witness, + } = &authenticity_witnesses[&0]; let tree_height: u32 = local_tree_height.try_into().unwrap(); - let computed_root = mmr_auth_struct.root_from_authentication_struct( + let computed_root = witness.root_from_authentication_struct( tree_height, - auth_struct, - indexed_leafs, + auth_struct.to_owned(), + indexed_leafs.to_owned(), ); let peak_index = 0; @@ -393,17 +478,26 @@ mod tests { .map(|(mmr_mp, (idx, leaf))| (idx, leaf, mmr_mp)) .collect_vec(); - let (mmr_auth_struct, auth_struct, indexed_leafs) = + let authenticity_witnesses = MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( &mmra, indexed_mmr_mps, ); + assert!( + authenticity_witnesses.len().is_one(), + "All indices belong to first peak" + ); + let AuthenticatedMerkleAuthStruct { + auth_struct, + indexed_leafs, + witness, + } = &authenticity_witnesses[&0]; let tree_height: u32 = local_tree_height.try_into().unwrap(); - let computed_root = mmr_auth_struct.root_from_authentication_struct( + let computed_root = witness.root_from_authentication_struct( tree_height, - auth_struct, - indexed_leafs, + auth_struct.to_owned(), + indexed_leafs.to_owned(), ); let peak_index = 0; @@ -423,18 +517,20 @@ mod tests { let leafs: Vec = random_elements(num_leafs.try_into().unwrap()); let tree = MerkleTree::::new::(&leafs).unwrap(); - let (mmr_auth_struct, auth_struct, indexed_leafs) = + let authenticated_auth_struct = MerkleAuthenticationStructAuthenticityWitness::new_from_merkle_tree( &tree, revealed_leaf_indices, ); - - let tree_height: u32 = tree_height.try_into().unwrap(); - let computed_root = mmr_auth_struct.root_from_authentication_struct( - tree_height, + let AuthenticatedMerkleAuthStruct { auth_struct, indexed_leafs, - ); + witness, + } = authenticated_auth_struct; + + let tree_height: u32 = tree_height.try_into().unwrap(); + let computed_root = + witness.root_from_authentication_struct(tree_height, auth_struct, indexed_leafs); let expected_root = tree.root(); prop_assert_eq!(expected_root, computed_root); } From 6ca061be403bf82d4ff06df555274bbb7b68ccc4 Mon Sep 17 00:00:00 2001 From: sword_smith Date: Thu, 8 Aug 2024 15:43:26 +0200 Subject: [PATCH 08/11] mmr_authentication_proof: Apply Alan's suggested name for this witness/proof --- .../mmr/mmr_authentication_struct.rs | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index 9caacb85e..fd8115a8c 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -26,7 +26,7 @@ const ROOT_MT_INDEX: u64 = 1; /// A witness to facilitate the proving of the authenticity of a Merkle /// authentication struct. #[derive(Debug, Clone)] -pub struct MerkleAuthenticationStructAuthenticityWitness { +pub struct AuthStructIntegrityProof { // All indices are Merkle tree node indices pub nd_auth_struct_indices: Vec, pub nd_sibling_indices: Vec<(u64, u64)>, @@ -40,10 +40,10 @@ pub struct MerkleAuthenticationStructAuthenticityWitness { pub struct AuthenticatedMerkleAuthStruct { pub auth_struct: Vec, pub indexed_leafs: Vec<(u64, Digest)>, - pub witness: MerkleAuthenticationStructAuthenticityWitness, + pub witness: AuthStructIntegrityProof, } -impl MerkleAuthenticationStructAuthenticityWitness { +impl AuthStructIntegrityProof { /// Return the Merkle tree node indices of the digests required to prove /// membership for the specified leaf indices, as well as the node indices /// that can be derived from the leaf indices and their authentication @@ -398,10 +398,7 @@ mod tests { .collect_vec(); let authenticated_auth_structs = - MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( - &mmra, - indexed_mmr_mps, - ); + AuthStructIntegrityProof::new_from_mmr_membership_proofs(&mmra, indexed_mmr_mps); let peak_heights = get_peak_heights(mmr_leaf_count); for (peak_index, authentication_auth_struct) in authenticated_auth_structs { @@ -437,10 +434,7 @@ mod tests { .collect_vec(); let authenticity_witnesses = - MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( - &mmra, - indexed_mmr_mps, - ); + AuthStructIntegrityProof::new_from_mmr_membership_proofs(&mmra, indexed_mmr_mps); assert!( authenticity_witnesses.len().is_one(), "All indices belong to first peak" @@ -479,10 +473,7 @@ mod tests { .collect_vec(); let authenticity_witnesses = - MerkleAuthenticationStructAuthenticityWitness::new_from_mmr_membership_proofs( - &mmra, - indexed_mmr_mps, - ); + AuthStructIntegrityProof::new_from_mmr_membership_proofs(&mmra, indexed_mmr_mps); assert!( authenticity_witnesses.len().is_one(), "All indices belong to first peak" @@ -518,10 +509,7 @@ mod tests { let tree = MerkleTree::::new::(&leafs).unwrap(); let authenticated_auth_struct = - MerkleAuthenticationStructAuthenticityWitness::new_from_merkle_tree( - &tree, - revealed_leaf_indices, - ); + AuthStructIntegrityProof::new_from_merkle_tree(&tree, revealed_leaf_indices); let AuthenticatedMerkleAuthStruct { auth_struct, indexed_leafs, @@ -566,7 +554,7 @@ mod tests { }) .collect_vec(); - let mmr_auth_struct = MerkleAuthenticationStructAuthenticityWitness { + let mmr_auth_struct = AuthStructIntegrityProof { nd_auth_struct_indices, nd_sibling_indices, nd_siblings, From 2744ab5c5c813b8376ce956de9f2cfb321d7535a Mon Sep 17 00:00:00 2001 From: sword_smith Date: Thu, 8 Aug 2024 15:49:17 +0200 Subject: [PATCH 09/11] mmr_authentication_struct: Add for empty MMR / no opened leafs --- .../util_types/mmr/mmr_authentication_struct.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index fd8115a8c..2a45aafb3 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -418,6 +418,22 @@ mod tests { } } + #[test] + fn auth_struct_on_empty_mmr() { + let empty_mmra = MmrAccumulator::init(vec![], 0); + let authenticated_auth_structs = + AuthStructIntegrityProof::new_from_mmr_membership_proofs(&empty_mmra, vec![]); + assert!(authenticated_auth_structs.is_empty()); + } + + #[test] + fn auth_struct_non_empty_mmr_empty_leaf_list() { + let mmra_10_leafs = MmrAccumulator::new_from_leafs(vec![Digest::default(); 10]); + let authenticated_auth_structs = + AuthStructIntegrityProof::new_from_mmr_membership_proofs(&mmra_10_leafs, vec![]); + assert!(authenticated_auth_structs.is_empty()); + } + #[test] fn auth_struct_from_mmr_mps_test_height_5_9_indices() { let local_tree_height = 5; From 365bdd9d1a1ace45a4bb0a07e8990605f296ffc7 Mon Sep 17 00:00:00 2001 From: sword_smith Date: Fri, 9 Aug 2024 13:32:25 +0200 Subject: [PATCH 10/11] test(mmr_authentication_struct): Verify panic on missing ND --- .../mmr/mmr_authentication_struct.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index 2a45aafb3..afc2f284d 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -434,6 +434,27 @@ mod tests { assert!(authenticated_auth_structs.is_empty()); } + #[should_panic] + #[test] + fn panics_on_missing_nd_digests() { + let tree_height = 3u32; + let num_leafs = 1u64 << tree_height; + let leafs: Vec = random_elements(num_leafs.try_into().unwrap()); + let tree = MerkleTree::::new::(&leafs).unwrap(); + + let authenticated_auth_struct = + AuthStructIntegrityProof::new_from_merkle_tree(&tree, vec![0]); + let AuthenticatedMerkleAuthStruct { + auth_struct, + indexed_leafs, + mut witness, + } = authenticated_auth_struct; + witness.nd_sibling_indices.clear(); + witness.nd_siblings.clear(); + + witness.root_from_authentication_struct(tree_height, auth_struct, indexed_leafs); + } + #[test] fn auth_struct_from_mmr_mps_test_height_5_9_indices() { let local_tree_height = 5; From dc84e272e1f26ae8de31223614276fe29a36ce7d Mon Sep 17 00:00:00 2001 From: sword_smith Date: Mon, 12 Aug 2024 19:47:57 +0200 Subject: [PATCH 11/11] style(mmr_authentication_struct): Fix some bad variable names --- .../mmr/mmr_authentication_struct.rs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs index afc2f284d..1f8bb9eb7 100644 --- a/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs +++ b/twenty-first/src/util_types/mmr/mmr_authentication_struct.rs @@ -125,7 +125,7 @@ impl AuthStructIntegrityProof { ); // Get challenges - let (alpha, gamma, delta) = { + let (alpha, beta, gamma) = { let mut sponge = Tip5::init(); sponge.pad_and_absorb_all(&indexed_leafs.encode()); sponge.pad_and_absorb_all(&auth_struct.encode()); @@ -136,22 +136,25 @@ impl AuthStructIntegrityProof { // Accumulate `p` from public data let mut p = XFieldElement::one(); for i in (0..indexed_leafs.len()).rev() { - let leaf_index_as_bfe = node_index_to_bfe((1 << tree_height) ^ indexed_leafs[i].0); + let node_index_as_bfe = node_index_to_bfe((1 << tree_height) ^ indexed_leafs[i].0); let leaf_as_xfe = digest_to_xfe(indexed_leafs[i].1, alpha); - let fact = leaf_as_xfe - gamma + delta * leaf_index_as_bfe; + let fact = leaf_as_xfe - beta + gamma * node_index_as_bfe; p *= fact; } let mut prev = 0; for i in (0..auth_struct.len()).rev() { let auth_struct_index = self.nd_auth_struct_indices[i]; + + // `auth_struct` must be sorted high-to-low by node-index. But since + // we're traversing in reverse order, the inequality is flipped. assert!(auth_struct_index > prev); prev = auth_struct_index; let auth_struct_index_as_bfe = node_index_to_bfe(auth_struct_index); let auth_str_elem_as_xfe = digest_to_xfe(auth_struct[i], alpha); - let fact = auth_str_elem_as_xfe - gamma + delta * auth_struct_index_as_bfe; + let fact = auth_str_elem_as_xfe - beta + gamma * auth_struct_index_as_bfe; p *= fact; } @@ -179,14 +182,15 @@ impl AuthStructIntegrityProof { let right_index_bfe = node_index_to_bfe(right_index); parent_index_bfe = left_index_bfe / bfe!(2); - let fact1 = l_xfe - gamma + delta * left_index_bfe; - let fact2 = r_xfe - gamma + delta * right_index_bfe; - let fact_parent = t_xfe - gamma + delta * parent_index_bfe; + let fact1 = l_xfe - beta + gamma * left_index_bfe; + let fact2 = r_xfe - beta + gamma * right_index_bfe; + let fact_parent = t_xfe - beta + gamma * parent_index_bfe; p *= fact1.inverse() * fact2.inverse() * fact_parent; } - assert_eq!(t_xfe - gamma + delta, p); + assert_eq!(t_xfe - beta + gamma, p); + assert!(parent_index_bfe.is_one()); t