diff --git a/Cargo.lock b/Cargo.lock index f99bb2c77d..6a5e65cf6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2676,6 +2676,7 @@ dependencies = [ "serde", "serde_with", "sha3", + "thiserror", "time", "tracing", "tracing-subscriber", diff --git a/saffron/Cargo.toml b/saffron/Cargo.toml index 19b8bde121..24216c95d1 100644 --- a/saffron/Cargo.toml +++ b/saffron/Cargo.toml @@ -35,11 +35,11 @@ rmp-serde.workspace = true serde.workspace = true serde_with.workspace = true sha3.workspace = true +thiserror.workspace = true time = { version = "0.3", features = ["macros"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = [ "ansi", "env-filter", "fmt", "time" ] } - [dev-dependencies] ark-std.workspace = true ctor = "0.2" diff --git a/saffron/src/blob.rs b/saffron/src/blob.rs index c5b8d01fb8..2f86c36471 100644 --- a/saffron/src/blob.rs +++ b/saffron/src/blob.rs @@ -92,89 +92,16 @@ impl FieldBlob { } } -#[cfg(test)] -pub mod test_utils { - use proptest::prelude::*; - - #[derive(Debug)] - pub struct BlobData(pub Vec); - - #[derive(Clone, Debug)] - pub enum DataSize { - Small, - Medium, - Large, - } - - impl DataSize { - const KB: usize = 1_000; - const MB: usize = 1_000_000; - - fn size_range_bytes(&self) -> (usize, usize) { - match self { - // Small: 1KB - 1MB - Self::Small => (Self::KB, Self::MB), - // Medium: 1MB - 10MB - Self::Medium => (Self::MB, 10 * Self::MB), - // Large: 10MB - 100MB - Self::Large => (10 * Self::MB, 100 * Self::MB), - } - } - } - - impl Arbitrary for DataSize { - type Parameters = (); - type Strategy = BoxedStrategy; - - fn arbitrary_with(_: ()) -> Self::Strategy { - prop_oneof![ - 6 => Just(DataSize::Small), // 60% chance - 3 => Just(DataSize::Medium), - 1 => Just(DataSize::Large) - ] - .boxed() - } - } - - impl Default for DataSize { - fn default() -> Self { - Self::Small - } - } - - impl Arbitrary for BlobData { - type Parameters = DataSize; - type Strategy = BoxedStrategy; - - fn arbitrary() -> Self::Strategy { - DataSize::arbitrary() - .prop_flat_map(|size| { - let (min, max) = size.size_range_bytes(); - prop::collection::vec(any::(), min..max) - }) - .prop_map(BlobData) - .boxed() - } - - fn arbitrary_with(size: Self::Parameters) -> Self::Strategy { - let (min, max) = size.size_range_bytes(); - prop::collection::vec(any::(), min..max) - .prop_map(BlobData) - .boxed() - } - } -} - #[cfg(test)] mod tests { use crate::{commitment::commit_to_field_elems, env}; use super::*; + use crate::utils::test_utils::*; use ark_poly::Radix2EvaluationDomain; use mina_curves::pasta::{Fp, Vesta}; use once_cell::sync::Lazy; use proptest::prelude::*; - use test_utils::*; static SRS: Lazy> = Lazy::new(|| { if let Ok(srs) = std::env::var("SRS_FILEPATH") { @@ -191,7 +118,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(20))] #[test] - fn test_round_trip_blob_encoding(BlobData(xs) in BlobData::arbitrary()) + fn test_round_trip_blob_encoding(UserData(xs) in UserData::arbitrary()) { let blob = FieldBlob::::encode(&*SRS, *DOMAIN, &xs); let bytes = rmp_serde::to_vec(&blob).unwrap(); let a = rmp_serde::from_slice(&bytes).unwrap(); @@ -206,7 +133,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(10))] #[test] - fn test_user_and_storage_provider_commitments_equal(BlobData(xs) in BlobData::arbitrary()) + fn test_user_and_storage_provider_commitments_equal(UserData(xs) in UserData::arbitrary()) { let elems = encode_for_domain(&*DOMAIN, &xs); let user_commitments = commit_to_field_elems(&*SRS, *DOMAIN, elems); let blob = FieldBlob::::encode(&*SRS, *DOMAIN, &xs); diff --git a/saffron/src/main.rs b/saffron/src/main.rs index 7e3dcdfba0..89294a91ac 100644 --- a/saffron/src/main.rs +++ b/saffron/src/main.rs @@ -11,7 +11,7 @@ use std::{ }; use tracing::debug; -const DEFAULT_SRS_SIZE: usize = 1 << 16; +pub const DEFAULT_SRS_SIZE: usize = 1 << 16; fn get_srs(cache: Option) -> (SRS, Radix2EvaluationDomain) { match cache { diff --git a/saffron/src/proof.rs b/saffron/src/proof.rs index 8429c3fb88..1ccef567d8 100644 --- a/saffron/src/proof.rs +++ b/saffron/src/proof.rs @@ -109,10 +109,9 @@ where mod tests { use super::*; use crate::{ - blob::test_utils::*, commitment::{commit_to_field_elems, fold_commitments}, env, - utils::encode_for_domain, + utils::{encode_for_domain, test_utils::UserData}, }; use ark_poly::{EvaluationDomain, Radix2EvaluationDomain}; use ark_std::UniformRand; @@ -140,7 +139,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(5))] #[test] - fn test_storage_prove_verify(BlobData(data) in BlobData::arbitrary()) { + fn test_storage_prove_verify(UserData(data) in UserData::arbitrary()) { let mut rng = OsRng; let commitment = { let field_elems = encode_for_domain(&*DOMAIN, &data); diff --git a/saffron/src/utils.rs b/saffron/src/utils.rs index d44606dc87..5f3c6d0c0f 100644 --- a/saffron/src/utils.rs +++ b/saffron/src/utils.rs @@ -1,5 +1,10 @@ +use std::marker::PhantomData; + use ark_ff::{BigInteger, PrimeField}; use ark_poly::EvaluationDomain; +use o1_utils::FieldHelpers; +use thiserror::Error; +use tracing::instrument; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. @@ -44,15 +49,229 @@ pub fn encode_for_domain>( .collect() } +#[derive(Clone, Debug)] +/// Represents the bytes a user query +pub struct QueryBytes { + pub start: usize, + pub len: usize, +} + +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +/// We store the data in a vector of vector of field element +/// The inner vector represent polynomials +struct FieldElt { + /// the index of the polynomial the data point is attached too + poly_index: usize, + /// the index of the root of unity the data point is attached too + eval_index: usize, + domain_size: usize, + n_polys: usize, +} +/// Represents a query in term of Field element +#[derive(Debug)] +pub struct QueryField { + start: FieldElt, + /// how many bytes we need to trim from the first chunk + /// we get from the first field element we decode + leftover_start: usize, + end: FieldElt, + /// how many bytes we need to trim from the last chunk + /// we get from the last field element we decode + leftover_end: usize, + tag: PhantomData, +} + +impl QueryField { + #[instrument(skip_all, level = "debug")] + pub fn apply(self, data: &[Vec]) -> Vec { + let n = (F::MODULUS_BIT_SIZE / 8) as usize; + let m = F::size_in_bytes(); + let mut buffer = vec![0u8; m]; + let mut answer = Vec::new(); + self.start + .into_iter() + .take_while(|x| x <= &self.end) + .for_each(|x| { + let value = data[x.poly_index][x.eval_index]; + decode_into(&mut buffer, value); + answer.extend_from_slice(&buffer[(m - n)..m]); + }); + + answer[(self.leftover_start)..(answer.len() - self.leftover_end)].to_vec() + } +} + +impl Iterator for FieldElt { + type Item = FieldElt; + fn next(&mut self) -> Option { + let current = *self; + + if (self.eval_index + 1) < self.domain_size { + self.eval_index += 1; + } else if (self.poly_index + 1) < self.n_polys { + self.poly_index += 1; + self.eval_index = 0; + } else { + return None; + } + + Some(current) + } +} + +#[derive(Debug, Error, Clone, PartialEq)] +pub enum QueryError { + #[error("Query out of bounds: poly_index {poly_index} eval_index {eval_index} n_polys {n_polys} domain_size {domain_size}")] + QueryOutOfBounds { + poly_index: usize, + eval_index: usize, + n_polys: usize, + domain_size: usize, + }, +} + +impl QueryBytes { + pub fn into_query_field( + &self, + domain_size: usize, + n_polys: usize, + ) -> Result, QueryError> { + let n = (F::MODULUS_BIT_SIZE / 8) as usize; + let start = { + let start_field_nb = self.start / n; + FieldElt { + poly_index: start_field_nb / domain_size, + eval_index: start_field_nb % domain_size, + domain_size, + n_polys, + } + }; + let byte_end = self.start + self.len; + let end = { + let end_field_nb = byte_end / n; + FieldElt { + poly_index: end_field_nb / domain_size, + eval_index: end_field_nb % domain_size, + domain_size, + n_polys, + } + }; + + if start.poly_index >= n_polys || end.poly_index >= n_polys { + return Err(QueryError::QueryOutOfBounds { + poly_index: end.poly_index, + eval_index: end.eval_index, + n_polys, + domain_size, + }); + }; + + let leftover_start = self.start % n; + let leftover_end = n - byte_end % n; + + Ok(QueryField { + start, + leftover_start, + end, + leftover_end, + tag: std::marker::PhantomData, + }) + } +} + +#[cfg(test)] +pub mod test_utils { + use proptest::prelude::*; + + #[derive(Debug, Clone)] + pub struct UserData(pub Vec); + + impl UserData { + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + } + + #[derive(Clone, Debug)] + pub enum DataSize { + Small, + Medium, + Large, + } + + impl DataSize { + const KB: usize = 1_000; + const MB: usize = 1_000_000; + + fn size_range_bytes(&self) -> (usize, usize) { + match self { + // Small: 1KB - 1MB + Self::Small => (Self::KB, Self::MB), + // Medium: 1MB - 10MB + Self::Medium => (Self::MB, 10 * Self::MB), + // Large: 10MB - 100MB + Self::Large => (10 * Self::MB, 100 * Self::MB), + } + } + } + + impl Arbitrary for DataSize { + type Parameters = (); + type Strategy = BoxedStrategy; + + fn arbitrary_with(_: ()) -> Self::Strategy { + prop_oneof![ + 6 => Just(DataSize::Small), // 60% chance + 3 => Just(DataSize::Medium), + 1 => Just(DataSize::Large) + ] + .boxed() + } + } + + impl Default for DataSize { + fn default() -> Self { + Self::Small + } + } + + impl Arbitrary for UserData { + type Parameters = DataSize; + type Strategy = BoxedStrategy; + + fn arbitrary() -> Self::Strategy { + DataSize::arbitrary() + .prop_flat_map(|size| { + let (min, max) = size.size_range_bytes(); + prop::collection::vec(any::(), min..max) + }) + .prop_map(UserData) + .boxed() + } + + fn arbitrary_with(size: Self::Parameters) -> Self::Strategy { + let (min, max) = size.size_range_bytes(); + prop::collection::vec(any::(), min..max) + .prop_map(UserData) + .boxed() + } + } +} + #[cfg(test)] mod tests { use super::*; use ark_poly::Radix2EvaluationDomain; use ark_std::UniformRand; use mina_curves::pasta::Fp; - use o1_utils::FieldHelpers; use once_cell::sync::Lazy; use proptest::prelude::*; + use test_utils::{DataSize, UserData}; + use tracing::debug; fn decode(x: Fp) -> Vec { let mut buffer = vec![0u8; Fp::size_in_bytes()]; @@ -103,7 +322,7 @@ mod tests { proptest! { #![proptest_config(ProptestConfig::with_cases(20))] #[test] - fn test_round_trip_encoding_to_field_elems(xs in prop::collection::vec(any::(), 0..=2 * Fp::size_in_bytes() * DOMAIN.size()) + fn test_round_trip_encoding_to_field_elems(UserData(xs) in UserData::arbitrary() ) { let chunked = encode_for_domain(&*DOMAIN, &xs); let elems = chunked @@ -117,4 +336,103 @@ mod tests { prop_assert_eq!(xs,ys); } } + + fn padded_field_length(xs: &[u8]) -> usize { + let m = Fp::MODULUS_BIT_SIZE as usize / 8; + let n = xs.len(); + let num_field_elems = (n + m - 1) / m; + let num_polys = (num_field_elems + DOMAIN.size() - 1) / DOMAIN.size(); + DOMAIN.size() * num_polys + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_padded_byte_length(UserData(xs) in UserData::arbitrary() + ) + { let chunked = encode_for_domain(&*DOMAIN, &xs); + let n = chunked.into_iter().flatten().count(); + prop_assert_eq!(n, padded_field_length(&xs)); + } + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_query( + (UserData(xs), queries) in UserData::arbitrary() + .prop_flat_map(|xs| { + let n = xs.len(); + let query_strategy = (0..(n - 1)).prop_flat_map(move |start| { + ((start + 1)..n).prop_map(move |end| QueryBytes { start, len: end - start}) + }); + let queries_strategy = prop::collection::vec(query_strategy, 10); + (Just(xs), queries_strategy) + }) + ) { + let chunked = encode_for_domain(&*DOMAIN, &xs); + for query in queries { + let expected = &xs[query.start..(query.start+query.len)]; + let field_query: QueryField = query.into_query_field(DOMAIN.size(), chunked.len()).unwrap(); + let got_answer = field_query.apply(&chunked); + prop_assert_eq!(expected, got_answer); + } + } + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_for_invalid_query_length( + (UserData(xs), mut query) in UserData::arbitrary() + .prop_flat_map(|UserData(xs)| { + let padded_len = { + let m = Fp::MODULUS_BIT_SIZE as usize / 8; + padded_field_length(&xs) * m + }; + let query_strategy = (0..xs.len()).prop_map(move |start| { + // this is the last valid end point + let end = padded_len - 1; + QueryBytes { start, len: end - start } + }); + (Just(UserData(xs)), query_strategy) + }) + ) { + debug!("check that first query is valid"); + let chunked = encode_for_domain(&*DOMAIN, &xs); + let n_polys = chunked.len(); + let query_field = query.into_query_field::(DOMAIN.size(), n_polys); + prop_assert!(query_field.is_ok()); + debug!("check that extending query length by 1 is invalid"); + query.len += 1; + let query_field = query.into_query_field::(DOMAIN.size(), n_polys); + prop_assert!(query_field.is_err()); + + } + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(20))] + #[test] + fn test_nil_query( + (UserData(xs), query) in UserData::arbitrary_with(DataSize::Small) + .prop_flat_map(|xs| { + let padded_len = { + let m = Fp::MODULUS_BIT_SIZE as usize / 8; + padded_field_length(&xs.0) * m + }; + let query_strategy = (0..padded_len).prop_map(move |start| { + QueryBytes { start, len: 0 } + }); + (Just(xs), query_strategy) + }) + ) { + let chunked = encode_for_domain(&*DOMAIN, &xs); + let n_polys = chunked.len(); + let field_query: QueryField = query.into_query_field(DOMAIN.size(), n_polys).unwrap(); + let got_answer = field_query.apply(&chunked); + prop_assert!(got_answer.is_empty()); + } + + } }