From 5e4b29059710928abcb98efb54f77f9449547e0b Mon Sep 17 00:00:00 2001 From: marcbeunardeau88 Date: Tue, 28 Jan 2025 18:35:49 +0100 Subject: [PATCH 1/9] Saffron: index query conversions --- saffron/Cargo.toml | 2 +- saffron/src/main.rs | 2 +- saffron/src/utils.rs | 142 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/saffron/Cargo.toml b/saffron/Cargo.toml index 8b2ee4deba..df70b567e9 100644 --- a/saffron/Cargo.toml +++ b/saffron/Cargo.toml @@ -38,9 +38,9 @@ time = { version = "0.3", features = ["macros"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = [ "ansi", "env-filter", "fmt", "time" ] } +ark-std.workspace = true [dev-dependencies] -ark-std.workspace = true ctor = "0.2" proptest.workspace = true once_cell.workspace = true diff --git a/saffron/src/main.rs b/saffron/src/main.rs index 7e3dcdfba0..89294a91ac 100644 --- a/saffron/src/main.rs +++ b/saffron/src/main.rs @@ -11,7 +11,7 @@ use std::{ }; use tracing::debug; -const DEFAULT_SRS_SIZE: usize = 1 << 16; +pub const DEFAULT_SRS_SIZE: usize = 1 << 16; fn get_srs(cache: Option) -> (SRS, Radix2EvaluationDomain) { match cache { diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index d44606dc87..10a302a88a 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -1,5 +1,6 @@ use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; +use ark_std::rand::Rng; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. @@ -12,6 +13,11 @@ pub fn decode_into(buffer: &mut [u8], x: Fp) { buffer.copy_from_slice(&bytes); } +pub fn get_31_bytes(x: F) -> Vec { + let bytes = x.into_bigint().to_bytes_be(); + bytes[1..32].to_vec() +} + pub fn encode_as_field_elements(bytes: &[u8]) -> Vec { let n = (F::MODULUS_BIT_SIZE / 8) as usize; bytes @@ -44,6 +50,112 @@ pub fn encode_for_domain>( .collect() } +#[derive(Clone, Debug)] +/// Represents the bytes a user query +pub struct QueryBytes { + pub start: usize, + pub len: usize, +} + +/// For testing purposes +impl QueryBytes { + pub fn random(size: usize) -> Self { + let mut rng = ark_std::rand::thread_rng(); + let start = rng.gen_range(0..size); + QueryBytes { + start, + len: rng.gen_range(0..(size - start)), + } + } +} +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +/// We store the data in a vector of vector of field element +/// The inner vector represent polynomials +pub struct FieldElt { + /// the number of the polynomial the data point is attached too + pub poly_nb: usize, + /// the number of the root of unity the data point is attached too + pub eval_nb: usize, +} +/// Represents a query in term of Field element +#[derive(Debug)] +pub struct QueryField { + pub start: FieldElt, + /// how many bytes we need to trim from the first 31bytes chunk + /// we get from the first field element we decode + pub leftover_start: usize, + pub end: FieldElt, + /// how many bytes we need to trim from the last 31bytes chunk + /// we get from the last field element we decode + pub leftover_end: usize, +} + +impl QueryField { + pub fn is_valid(&self, nb_poly: usize) -> bool { + self.start.eval_nb < 1 << 16 + && self.end.eval_nb < 1 << 16 + && self.end.poly_nb < nb_poly + && self.start <= self.end + && self.leftover_end <= (F::MODULUS_BIT_SIZE as usize) / 8 + && self.leftover_start <= (F::MODULUS_BIT_SIZE as usize) / 8 + } + + pub fn apply(self, data: Vec>) -> Vec { + assert!(self.is_valid::(data.len()), "Invalid query"); + let mut answer: Vec = Vec::new(); + let mut field_elt = self.start; + while field_elt <= self.end { + if data[field_elt.poly_nb][field_elt.eval_nb] == F::zero() { + println!() + } + let mut to_append = get_31_bytes(data[field_elt.poly_nb][field_elt.eval_nb]); + answer.append(&mut to_append); + field_elt = field_elt.next().unwrap(); + } + let n = answer.len(); + // trimming the first and last 31bytes chunk + answer[(self.leftover_start)..(n - self.leftover_end)].to_vec() + } +} + +impl Iterator for FieldElt { + type Item = FieldElt; + fn next(&mut self) -> Option { + if self.eval_nb < (1 << 16) - 1 { + self.eval_nb += 1; + } else { + self.poly_nb += 1; + self.eval_nb = 0 + }; + Some(*self) + } +} + +impl Into for QueryBytes { + fn into(self) -> QueryField { + let n = 31 as usize; + let start_field_nb = self.start / n; + let start = FieldElt { + poly_nb: start_field_nb / (1 << 16), + eval_nb: start_field_nb % (1 << 16), + }; + let leftover_start = self.start % n; + + let byte_end = self.start + self.len; + let end_field_nb = byte_end / n; + let end = FieldElt { + poly_nb: end_field_nb / (1 << 16), + eval_nb: end_field_nb % (1 << 16), + }; + let leftover_end = n - byte_end % n; + QueryField { + start, + leftover_start, + end, + leftover_end, + } + } +} #[cfg(test)] mod tests { use super::*; @@ -117,4 +229,34 @@ mod tests { prop_assert_eq!(xs,ys); } } + + // check that appying a field query = applying a byte query + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_round_trip_query(xs in prop::collection::vec(any::(), 0..10 * Fp::size_in_bytes() *DOMAIN.size() ) + ) { + proptest! ( |query in prop::strategy::Just(QueryBytes::random(xs.len()))| + let chunked = encode_for_domain(&*DOMAIN, &xs); + let expected_answer = &xs[query.start..(query.start+query.len)]; + let field_query :QueryField = query.clone().into(); + let got_answer = field_query.apply(chunked); + prop_assert_eq!(expected_answer,got_answer); + ) + + } + } + + + } + proptest! { + #[test] + fn test_dependent_args(base in 0..100) { + let multiplied = (1..10).prop_map(|factor| base * factor); + prop_assume!(base > 0); + proptest!(|(multiplied in multiplied)| { + prop_assert!(base * multiplied != 0); + }); + } + } } From 76965e6c4027ee2ae33792a98853cf433ba733ac Mon Sep 17 00:00:00 2001 From: martyall Date: Tue, 28 Jan 2025 12:20:10 -0800 Subject: [PATCH 2/9] set up property based test to check multiple queries per data generation --- saffron/Cargo.toml | 3 +- saffron/src/blob.rs | 79 +----------------------- saffron/src/utils.rs | 139 +++++++++++++++++++++++++++++++------------ 3 files changed, 105 insertions(+), 116 deletions(-) diff --git a/saffron/Cargo.toml b/saffron/Cargo.toml index df70b567e9..75a057731f 100644 --- a/saffron/Cargo.toml +++ b/saffron/Cargo.toml @@ -38,9 +38,8 @@ time = { version = "0.3", features = ["macros"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = [ "ansi", "env-filter", "fmt", "time" ] } -ark-std.workspace = true - [dev-dependencies] +ark-std.workspace = true ctor = "0.2" proptest.workspace = true once_cell.workspace = true diff --git a/saffron/src/blob.rs b/saffron/src/blob.rs index ea982d2fad..2f86c36471 100644 --- a/saffron/src/blob.rs +++ b/saffron/src/blob.rs @@ -92,86 +92,13 @@ impl FieldBlob { } } -#[cfg(test)] -mod blob_test_utils { - use proptest::prelude::*; - - #[derive(Debug)] - pub struct BlobData(pub Vec); - - #[derive(Clone, Debug)] - pub enum DataSize { - Small, - Medium, - Large, - } - - impl DataSize { - const KB: usize = 1_000; - const MB: usize = 1_000_000; - - fn size_range_bytes(&self) -> (usize, usize) { - match self { - // Small: 1KB - 1MB - Self::Small => (Self::KB, Self::MB), - // Medium: 1MB - 10MB - Self::Medium => (Self::MB, 10 * Self::MB), - // Large: 10MB - 100MB - Self::Large => (10 * Self::MB, 100 * Self::MB), - } - } - } - - impl Arbitrary for DataSize { - type Parameters = (); - type Strategy = BoxedStrategy; - - fn arbitrary_with(_: ()) -> Self::Strategy { - prop_oneof![ - 6 => Just(DataSize::Small), // 60% chance - 3 => Just(DataSize::Medium), - 1 => Just(DataSize::Large) - ] - .boxed() - } - } - - impl Default for DataSize { - fn default() -> Self { - Self::Small - } - } - - impl Arbitrary for BlobData { - type Parameters = DataSize; - type Strategy = BoxedStrategy; - - fn arbitrary() -> Self::Strategy { - DataSize::arbitrary() - .prop_flat_map(|size| { - let (min, max) = size.size_range_bytes(); - prop::collection::vec(any::(), min..max) - }) - .prop_map(BlobData) - .boxed() - } - - fn arbitrary_with(size: Self::Parameters) -> Self::Strategy { - let (min, max) = size.size_range_bytes(); - prop::collection::vec(any::(), min..max) - .prop_map(BlobData) - .boxed() - } - } -} - #[cfg(test)] mod tests { use crate::{commitment::commit_to_field_elems, env}; use super::*; + use crate::utils::test_utils::*; use ark_poly::Radix2EvaluationDomain; - use blob_test_utils::*; use mina_curves::pasta::{Fp, Vesta}; use once_cell::sync::Lazy; use proptest::prelude::*; @@ -191,7 +118,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(20))] #[test] - fn test_round_trip_blob_encoding(BlobData(xs) in BlobData::arbitrary()) + fn test_round_trip_blob_encoding(UserData(xs) in UserData::arbitrary()) { let blob = FieldBlob::::encode(&*SRS, *DOMAIN, &xs); let bytes = rmp_serde::to_vec(&blob).unwrap(); let a = rmp_serde::from_slice(&bytes).unwrap(); @@ -206,7 +133,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(10))] #[test] - fn test_user_and_storage_provider_commitments_equal(BlobData(xs) in BlobData::arbitrary()) + fn test_user_and_storage_provider_commitments_equal(UserData(xs) in UserData::arbitrary()) { let elems = encode_for_domain(&*DOMAIN, &xs); let user_commitments = commit_to_field_elems(&*SRS, *DOMAIN, elems); let blob = FieldBlob::::encode(&*SRS, *DOMAIN, &xs); diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index 10a302a88a..7b5484b1f7 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -1,6 +1,5 @@ use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; -use ark_std::rand::Rng; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. @@ -57,17 +56,6 @@ pub struct QueryBytes { pub len: usize, } -/// For testing purposes -impl QueryBytes { - pub fn random(size: usize) -> Self { - let mut rng = ark_std::rand::thread_rng(); - let start = rng.gen_range(0..size); - QueryBytes { - start, - len: rng.gen_range(0..(size - start)), - } - } -} #[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] /// We store the data in a vector of vector of field element /// The inner vector represent polynomials @@ -156,6 +144,86 @@ impl Into for QueryBytes { } } } + +#[cfg(test)] +pub mod test_utils { + use proptest::prelude::*; + + #[derive(Debug, Clone)] + pub struct UserData(pub Vec); + + impl UserData { + pub fn len(&self) -> usize { + self.0.len() + } + } + + #[derive(Clone, Debug)] + pub enum DataSize { + Small, + Medium, + Large, + } + + impl DataSize { + const KB: usize = 1_000; + const MB: usize = 1_000_000; + + fn size_range_bytes(&self) -> (usize, usize) { + match self { + // Small: 1KB - 1MB + Self::Small => (Self::KB, Self::MB), + // Medium: 1MB - 10MB + Self::Medium => (Self::MB, 10 * Self::MB), + // Large: 10MB - 100MB + Self::Large => (10 * Self::MB, 100 * Self::MB), + } + } + } + + impl Arbitrary for DataSize { + type Parameters = (); + type Strategy = BoxedStrategy; + + fn arbitrary_with(_: ()) -> Self::Strategy { + prop_oneof![ + 6 => Just(DataSize::Small), // 60% chance + 3 => Just(DataSize::Medium), + 1 => Just(DataSize::Large) + ] + .boxed() + } + } + + impl Default for DataSize { + fn default() -> Self { + Self::Small + } + } + + impl Arbitrary for UserData { + type Parameters = DataSize; + type Strategy = BoxedStrategy; + + fn arbitrary() -> Self::Strategy { + DataSize::arbitrary() + .prop_flat_map(|size| { + let (min, max) = size.size_range_bytes(); + prop::collection::vec(any::(), min..max) + }) + .prop_map(UserData) + .boxed() + } + + fn arbitrary_with(size: Self::Parameters) -> Self::Strategy { + let (min, max) = size.size_range_bytes(); + prop::collection::vec(any::(), min..max) + .prop_map(UserData) + .boxed() + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -165,6 +233,7 @@ mod tests { use o1_utils::FieldHelpers; use once_cell::sync::Lazy; use proptest::prelude::*; + use test_utils::UserData; fn decode(x: Fp) -> Vec { let mut buffer = vec![0u8; Fp::size_in_bytes()]; @@ -215,7 +284,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(20))] #[test] - fn test_round_trip_encoding_to_field_elems(xs in prop::collection::vec(any::(), 0..=2 * Fp::size_in_bytes() * DOMAIN.size()) + fn test_round_trip_encoding_to_field_elems(UserData(xs) in UserData::arbitrary() ) { let chunked = encode_for_domain(&*DOMAIN, &xs); let elems = chunked @@ -230,33 +299,27 @@ mod tests { } } - // check that appying a field query = applying a byte query - proptest! { - #![proptest_config(ProptestConfig::with_cases(20))] - #[test] - fn test_round_trip_query(xs in prop::collection::vec(any::(), 0..10 * Fp::size_in_bytes() *DOMAIN.size() ) - ) { - proptest! ( |query in prop::strategy::Just(QueryBytes::random(xs.len()))| - let chunked = encode_for_domain(&*DOMAIN, &xs); - let expected_answer = &xs[query.start..(query.start+query.len)]; - let field_query :QueryField = query.clone().into(); - let got_answer = field_query.apply(chunked); - prop_assert_eq!(expected_answer,got_answer); - ) - - } - } - - - } proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] #[test] - fn test_dependent_args(base in 0..100) { - let multiplied = (1..10).prop_map(|factor| base * factor); - prop_assume!(base > 0); - proptest!(|(multiplied in multiplied)| { - prop_assert!(base * multiplied != 0); - }); + fn test_query( + (UserData(xs), queries) in UserData::arbitrary() + .prop_flat_map(|xs| { + let n = xs.len(); + let query_strategy = (0..(n - 1)).prop_flat_map(move |start| { + ((start + 1)..n).prop_map(move |end| QueryBytes { start, len: end - start}) + }); + let queries_strategy = prop::collection::vec(query_strategy, 10); + (Just(xs), queries_strategy) + }) + ) { + let chunked = encode_for_domain(&*DOMAIN, &xs); + for query in queries { + let expected = &xs[query.start..(query.start+query.len)]; + let field_query: QueryField = query.clone().into(); + let got_answer = field_query.apply(chunked.clone()); // Note: might need clone depending on your types + prop_assert_eq!(expected, got_answer); + } } } } From 482964f0e30ba6c01c7e102aa9f80c6d768c8c0b Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 14:41:44 -0800 Subject: [PATCH 3/9] remove function get_31_bytes, use existing decode method --- saffron/src/utils.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index 7b5484b1f7..6f92852935 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -1,5 +1,6 @@ use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; +use o1_utils::FieldHelpers; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. @@ -12,11 +13,6 @@ pub fn decode_into(buffer: &mut [u8], x: Fp) { buffer.copy_from_slice(&bytes); } -pub fn get_31_bytes(x: F) -> Vec { - let bytes = x.into_bigint().to_bytes_be(); - bytes[1..32].to_vec() -} - pub fn encode_as_field_elements(bytes: &[u8]) -> Vec { let n = (F::MODULUS_BIT_SIZE / 8) as usize; bytes @@ -92,17 +88,16 @@ impl QueryField { assert!(self.is_valid::(data.len()), "Invalid query"); let mut answer: Vec = Vec::new(); let mut field_elt = self.start; + let n = (F::MODULUS_BIT_SIZE / 8) as usize; + let m = F::size_in_bytes(); + let mut buffer = vec![0u8; m]; while field_elt <= self.end { - if data[field_elt.poly_nb][field_elt.eval_nb] == F::zero() { - println!() - } - let mut to_append = get_31_bytes(data[field_elt.poly_nb][field_elt.eval_nb]); - answer.append(&mut to_append); + decode_into(&mut buffer, data[field_elt.poly_nb][field_elt.eval_nb]); + answer.extend_from_slice(&buffer[(m - n)..m]); field_elt = field_elt.next().unwrap(); } - let n = answer.len(); // trimming the first and last 31bytes chunk - answer[(self.leftover_start)..(n - self.leftover_end)].to_vec() + answer[(self.leftover_start)..(answer.len() - self.leftover_end)].to_vec() } } @@ -230,7 +225,6 @@ mod tests { use ark_poly::Radix2EvaluationDomain; use ark_std::UniformRand; use mina_curves::pasta::Fp; - use o1_utils::FieldHelpers; use once_cell::sync::Lazy; use proptest::prelude::*; use test_utils::UserData; From ffa98c747a37ca628ff358fe6dc5408ca261fc80 Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 14:51:48 -0800 Subject: [PATCH 4/9] remove used of hard coded 31 (i.e. field elem byte length) --- saffron/src/utils.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index 6f92852935..841b514a09 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -1,3 +1,5 @@ +use std::marker::PhantomData; + use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; use o1_utils::FieldHelpers; @@ -57,25 +59,26 @@ pub struct QueryBytes { /// The inner vector represent polynomials pub struct FieldElt { /// the number of the polynomial the data point is attached too - pub poly_nb: usize, + poly_nb: usize, /// the number of the root of unity the data point is attached too - pub eval_nb: usize, + eval_nb: usize, } /// Represents a query in term of Field element #[derive(Debug)] -pub struct QueryField { - pub start: FieldElt, +pub struct QueryField { + start: FieldElt, /// how many bytes we need to trim from the first 31bytes chunk /// we get from the first field element we decode - pub leftover_start: usize, - pub end: FieldElt, + leftover_start: usize, + end: FieldElt, /// how many bytes we need to trim from the last 31bytes chunk /// we get from the last field element we decode - pub leftover_end: usize, + leftover_end: usize, + tag: PhantomData } -impl QueryField { - pub fn is_valid(&self, nb_poly: usize) -> bool { +impl QueryField { + pub fn is_valid(&self, nb_poly: usize) -> bool { self.start.eval_nb < 1 << 16 && self.end.eval_nb < 1 << 16 && self.end.poly_nb < nb_poly @@ -84,8 +87,8 @@ impl QueryField { && self.leftover_start <= (F::MODULUS_BIT_SIZE as usize) / 8 } - pub fn apply(self, data: Vec>) -> Vec { - assert!(self.is_valid::(data.len()), "Invalid query"); + pub fn apply(self, data: Vec>) -> Vec { + assert!(self.is_valid(data.len()), "Invalid query"); let mut answer: Vec = Vec::new(); let mut field_elt = self.start; let n = (F::MODULUS_BIT_SIZE / 8) as usize; @@ -114,9 +117,9 @@ impl Iterator for FieldElt { } } -impl Into for QueryBytes { - fn into(self) -> QueryField { - let n = 31 as usize; +impl Into> for QueryBytes { + fn into(self) -> QueryField { + let n = F::MODULUS_BIT_SIZE as usize / 8; let start_field_nb = self.start / n; let start = FieldElt { poly_nb: start_field_nb / (1 << 16), @@ -136,6 +139,7 @@ impl Into for QueryBytes { leftover_start, end, leftover_end, + tag: PhantomData, } } } @@ -310,7 +314,7 @@ mod tests { let chunked = encode_for_domain(&*DOMAIN, &xs); for query in queries { let expected = &xs[query.start..(query.start+query.len)]; - let field_query: QueryField = query.clone().into(); + let field_query: QueryField = query.clone().into(); let got_answer = field_query.apply(chunked.clone()); // Note: might need clone depending on your types prop_assert_eq!(expected, got_answer); } From d7fb04547a42cdd89923c00b13e1a058b0bc41c9 Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 15:04:04 -0800 Subject: [PATCH 5/9] add padded_field_length test helper --- saffron/src/utils.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index 841b514a09..d273e962ec 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -297,6 +297,25 @@ mod tests { } } + fn padded_field_length(xs: &[u8]) -> usize { + let m = Fp::MODULUS_BIT_SIZE as usize / 8; + let n = xs.len(); + let num_field_elems = (n + m - 1) / m; + let num_polys = (num_field_elems + DOMAIN.size() - 1) / DOMAIN.size(); + DOMAIN.size() * num_polys + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_padded_byte_length(UserData(xs) in UserData::arbitrary() + ) + { let chunked = encode_for_domain(&*DOMAIN, &xs); + let n = chunked.into_iter().flatten().count(); + prop_assert_eq!(n, padded_field_length(&xs)); + } + } + proptest! { #![proptest_config(ProptestConfig::with_cases(20))] #[test] From 416ed93c800bb52046061435ba8ef5c83d6cc0e5 Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 15:07:48 -0800 Subject: [PATCH 6/9] Force QueryField construction to come from smart constructor; throw an error when attempting to construct a malformed query --- Cargo.lock | 1 + saffron/Cargo.toml | 1 + saffron/src/utils.rs | 142 ++++++++++++++++++++++++++++++++----------- 3 files changed, 107 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 177f323176..bb6af2766c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2675,6 +2675,7 @@ dependencies = [ "serde", "serde_with", "sha3", + "thiserror", "time", "tracing", "tracing-subscriber", diff --git a/saffron/Cargo.toml b/saffron/Cargo.toml index 75a057731f..7d1d1f3c77 100644 --- a/saffron/Cargo.toml +++ b/saffron/Cargo.toml @@ -34,6 +34,7 @@ rmp-serde.workspace = true serde.workspace = true serde_with.workspace = true sha3.workspace = true +thiserror.workspace = true time = { version = "0.3", features = ["macros"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = [ "ansi", "env-filter", "fmt", "time" ] } diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index d273e962ec..cae23d823a 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -3,6 +3,7 @@ use std::marker::PhantomData; use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; use o1_utils::FieldHelpers; +use thiserror::Error; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. @@ -59,9 +60,11 @@ pub struct QueryBytes { /// The inner vector represent polynomials pub struct FieldElt { /// the number of the polynomial the data point is attached too - poly_nb: usize, + poly_index: usize, /// the number of the root of unity the data point is attached too - eval_nb: usize, + eval_index: usize, + domain_size: usize, + n_polys: usize, } /// Represents a query in term of Field element #[derive(Debug)] @@ -74,28 +77,21 @@ pub struct QueryField { /// how many bytes we need to trim from the last 31bytes chunk /// we get from the last field element we decode leftover_end: usize, - tag: PhantomData + tag: PhantomData, } impl QueryField { - pub fn is_valid(&self, nb_poly: usize) -> bool { - self.start.eval_nb < 1 << 16 - && self.end.eval_nb < 1 << 16 - && self.end.poly_nb < nb_poly - && self.start <= self.end - && self.leftover_end <= (F::MODULUS_BIT_SIZE as usize) / 8 - && self.leftover_start <= (F::MODULUS_BIT_SIZE as usize) / 8 - } - - pub fn apply(self, data: Vec>) -> Vec { - assert!(self.is_valid(data.len()), "Invalid query"); + pub fn apply(self, data: &[Vec]) -> Vec { let mut answer: Vec = Vec::new(); let mut field_elt = self.start; let n = (F::MODULUS_BIT_SIZE / 8) as usize; let m = F::size_in_bytes(); let mut buffer = vec![0u8; m]; while field_elt <= self.end { - decode_into(&mut buffer, data[field_elt.poly_nb][field_elt.eval_nb]); + decode_into( + &mut buffer, + data[field_elt.poly_index][field_elt.eval_index], + ); answer.extend_from_slice(&buffer[(m - n)..m]); field_elt = field_elt.next().unwrap(); } @@ -107,40 +103,80 @@ impl QueryField { impl Iterator for FieldElt { type Item = FieldElt; fn next(&mut self) -> Option { - if self.eval_nb < (1 << 16) - 1 { - self.eval_nb += 1; + if self.eval_index < (1 << 16) - 1 { + self.eval_index += 1; } else { - self.poly_nb += 1; - self.eval_nb = 0 + self.poly_index += 1; + self.eval_index = 0 }; Some(*self) } } -impl Into> for QueryBytes { - fn into(self) -> QueryField { - let n = F::MODULUS_BIT_SIZE as usize / 8; - let start_field_nb = self.start / n; - let start = FieldElt { - poly_nb: start_field_nb / (1 << 16), - eval_nb: start_field_nb % (1 << 16), - }; - let leftover_start = self.start % n; +#[derive(Debug, Error, Clone, PartialEq)] +pub enum QueryError { + #[error("Query out of bounds: poly_index {poly_index} eval_index {eval_index} n_polys {n_polys} domain_size {domain_size}")] + QueryOutOfBounds { + poly_index: usize, + eval_index: usize, + n_polys: usize, + domain_size: usize, + }, +} +impl QueryBytes { + pub fn into_query_field( + &self, + domain_size: usize, + n_polys: usize, + ) -> Result, QueryError> { + let n = (F::MODULUS_BIT_SIZE / 8) as usize; + let start = { + let start_field_nb = self.start / n; + FieldElt { + poly_index: start_field_nb / domain_size, + eval_index: start_field_nb % domain_size, + domain_size, + n_polys, + } + }; + if start.poly_index >= n_polys { + return Err(QueryError::QueryOutOfBounds { + poly_index: start.poly_index, + eval_index: start.eval_index, + n_polys, + domain_size, + }); + }; let byte_end = self.start + self.len; - let end_field_nb = byte_end / n; - let end = FieldElt { - poly_nb: end_field_nb / (1 << 16), - eval_nb: end_field_nb % (1 << 16), + let end = { + let end_field_nb = byte_end / n; + FieldElt { + poly_index: end_field_nb / domain_size, + eval_index: end_field_nb % domain_size, + domain_size, + n_polys, + } + }; + if end.poly_index >= n_polys { + return Err(QueryError::QueryOutOfBounds { + poly_index: end.poly_index, + eval_index: end.eval_index, + n_polys, + domain_size, + }); }; + + let leftover_start = self.start % n; let leftover_end = n - byte_end % n; - QueryField { + + Ok(QueryField { start, leftover_start, end, leftover_end, - tag: PhantomData, - } + tag: std::marker::PhantomData, + }) } } @@ -232,6 +268,7 @@ mod tests { use once_cell::sync::Lazy; use proptest::prelude::*; use test_utils::UserData; + use tracing::debug; fn decode(x: Fp) -> Vec { let mut buffer = vec![0u8; Fp::size_in_bytes()]; @@ -333,10 +370,41 @@ mod tests { let chunked = encode_for_domain(&*DOMAIN, &xs); for query in queries { let expected = &xs[query.start..(query.start+query.len)]; - let field_query: QueryField = query.clone().into(); - let got_answer = field_query.apply(chunked.clone()); // Note: might need clone depending on your types + let field_query: QueryField = query.into_query_field(DOMAIN.size(), chunked.len()).unwrap(); + let got_answer = field_query.apply(&chunked); // Note: might need clone depending on your types prop_assert_eq!(expected, got_answer); } } } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_for_invalid_query_length( + (UserData(xs), mut query) in UserData::arbitrary() + .prop_flat_map(|UserData(xs)| { + let padded_len = { + let m = Fp::MODULUS_BIT_SIZE as usize / 8; + padded_field_length(&xs) * m + }; + let query_strategy = (0..xs.len()).prop_map(move |start| { + // this is the last valid end point + let end = padded_len - 1; + QueryBytes { start, len: end - start } + }); + (Just(UserData(xs)), query_strategy) + }) + ) { + debug!("check that first query is valid"); + let chunked = encode_for_domain(&*DOMAIN, &xs); + let n_polys = chunked.len(); + let query_field = query.into_query_field::(DOMAIN.size(), n_polys); + prop_assert!(query_field.is_ok()); + debug!("check that extending query length by 1 is invalid"); + query.len += 1; + let query_field = query.into_query_field::(DOMAIN.size(), n_polys); + prop_assert!(query_field.is_err()); + + } + } } From af1b31b83c47ad259fe278b54c2cd82f1a4e530e Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 15:12:40 -0800 Subject: [PATCH 7/9] add nil query test --- saffron/src/utils.rs | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index cae23d823a..a2ebee4dfd 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -267,7 +267,7 @@ mod tests { use mina_curves::pasta::Fp; use once_cell::sync::Lazy; use proptest::prelude::*; - use test_utils::UserData; + use test_utils::{DataSize, UserData}; use tracing::debug; fn decode(x: Fp) -> Vec { @@ -371,7 +371,7 @@ mod tests { for query in queries { let expected = &xs[query.start..(query.start+query.len)]; let field_query: QueryField = query.into_query_field(DOMAIN.size(), chunked.len()).unwrap(); - let got_answer = field_query.apply(&chunked); // Note: might need clone depending on your types + let got_answer = field_query.apply(&chunked); prop_assert_eq!(expected, got_answer); } } @@ -407,4 +407,29 @@ mod tests { } } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_nil_query( + (UserData(xs), query) in UserData::arbitrary_with(DataSize::Small) + .prop_flat_map(|xs| { + let padded_len = { + let m = Fp::MODULUS_BIT_SIZE as usize / 8; + padded_field_length(&xs.0) * m + }; + let query_strategy = (0..padded_len).prop_map(move |start| { + QueryBytes { start, len: 0 } + }); + (Just(xs), query_strategy) + }) + ) { + let chunked = encode_for_domain(&*DOMAIN, &xs); + let n_polys = chunked.len(); + let field_query: QueryField = query.into_query_field(DOMAIN.size(), n_polys).unwrap(); + let got_answer = field_query.apply(&chunked); + prop_assert!(got_answer.is_empty()); + } + + } } From 3935f75c7de9d3b69dc5fbf589b2307b77c53bd6 Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 15:33:30 -0800 Subject: [PATCH 8/9] use an iterator which can return None, change apply to functional style --- saffron/src/utils.rs | 48 +++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index a2ebee4dfd..4c8920db3c 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -4,6 +4,7 @@ use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; use o1_utils::FieldHelpers; use thiserror::Error; +use tracing::instrument; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. @@ -59,9 +60,9 @@ pub struct QueryBytes { /// We store the data in a vector of vector of field element /// The inner vector represent polynomials pub struct FieldElt { - /// the number of the polynomial the data point is attached too + /// the index of the polynomial the data point is attached too poly_index: usize, - /// the number of the root of unity the data point is attached too + /// the index of the root of unity the data point is attached too eval_index: usize, domain_size: usize, n_polys: usize, @@ -70,46 +71,51 @@ pub struct FieldElt { #[derive(Debug)] pub struct QueryField { start: FieldElt, - /// how many bytes we need to trim from the first 31bytes chunk + /// how many bytes we need to trim from the first chunk /// we get from the first field element we decode leftover_start: usize, end: FieldElt, - /// how many bytes we need to trim from the last 31bytes chunk + /// how many bytes we need to trim from the last chunk /// we get from the last field element we decode leftover_end: usize, tag: PhantomData, } impl QueryField { + #[instrument(skip_all, level = "debug")] pub fn apply(self, data: &[Vec]) -> Vec { - let mut answer: Vec = Vec::new(); - let mut field_elt = self.start; let n = (F::MODULUS_BIT_SIZE / 8) as usize; let m = F::size_in_bytes(); let mut buffer = vec![0u8; m]; - while field_elt <= self.end { - decode_into( - &mut buffer, - data[field_elt.poly_index][field_elt.eval_index], - ); - answer.extend_from_slice(&buffer[(m - n)..m]); - field_elt = field_elt.next().unwrap(); - } - // trimming the first and last 31bytes chunk + let mut answer = Vec::new(); + self.start + .into_iter() + .take_while(|x| x <= &self.end) + .for_each(|x| { + let value = data[x.poly_index][x.eval_index]; + decode_into(&mut buffer, value); + answer.extend_from_slice(&buffer[(m - n)..m]); + }); + answer[(self.leftover_start)..(answer.len() - self.leftover_end)].to_vec() } } impl Iterator for FieldElt { type Item = FieldElt; - fn next(&mut self) -> Option { - if self.eval_index < (1 << 16) - 1 { + fn next(&mut self) -> Option { + let current = *self; + + if (self.eval_index + 1) < self.domain_size { self.eval_index += 1; - } else { + } else if (self.poly_index + 1) < self.n_polys { self.poly_index += 1; - self.eval_index = 0 - }; - Some(*self) + self.eval_index = 0; + } else { + return None; + } + + Some(current) } } From 47fa03d1543211ea99e1f6dd726d6a940b3cc64d Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 29 Jan 2025 15:34:17 -0800 Subject: [PATCH 9/9] satisfy linter; opaque FieldElt type; simplify control into_query_field" --- saffron/src/utils.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index 4c8920db3c..5f3c6d0c0f 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -59,7 +59,7 @@ pub struct QueryBytes { #[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] /// We store the data in a vector of vector of field element /// The inner vector represent polynomials -pub struct FieldElt { +struct FieldElt { /// the index of the polynomial the data point is attached too poly_index: usize, /// the index of the root of unity the data point is attached too @@ -146,14 +146,6 @@ impl QueryBytes { n_polys, } }; - if start.poly_index >= n_polys { - return Err(QueryError::QueryOutOfBounds { - poly_index: start.poly_index, - eval_index: start.eval_index, - n_polys, - domain_size, - }); - }; let byte_end = self.start + self.len; let end = { let end_field_nb = byte_end / n; @@ -164,7 +156,8 @@ impl QueryBytes { n_polys, } }; - if end.poly_index >= n_polys { + + if start.poly_index >= n_polys || end.poly_index >= n_polys { return Err(QueryError::QueryOutOfBounds { poly_index: end.poly_index, eval_index: end.eval_index, @@ -197,6 +190,10 @@ pub mod test_utils { pub fn len(&self) -> usize { self.0.len() } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } } #[derive(Clone, Debug)]