From 0ea239e47160e22ab0edcb3b929306949e23e1e4 Mon Sep 17 00:00:00 2001 From: martyall Date: Wed, 15 Jan 2025 13:44:47 -0800 Subject: [PATCH] introduce FieldBlob type. use bash script for e2e test. implement round trip test as property-based-test --- Cargo.lock | 257 +---------------------------------- saffron/.gitignore | 2 + saffron/Cargo.toml | 7 +- saffron/fixtures/.gitignore | 1 - saffron/fixtures/lorem-bin | Bin 488 -> 0 bytes saffron/lorem.txt | Bin 465 -> 0 bytes saffron/src/cli.rs | 8 -- saffron/src/main.rs | 28 ++-- saffron/src/serialization.rs | 116 ++++++++++++---- saffron/test-encoding.sh | 15 ++ 10 files changed, 123 insertions(+), 311 deletions(-) delete mode 100644 saffron/fixtures/.gitignore delete mode 100644 saffron/fixtures/lorem-bin delete mode 100644 saffron/lorem.txt create mode 100755 saffron/test-encoding.sh diff --git a/Cargo.lock b/Cargo.lock index 4844e18284..c8f212d0ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,12 +491,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bytes" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" - [[package]] name = "cargo-spec" version = "0.5.0" @@ -1094,7 +1088,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "windows-sys 0.52.0", ] @@ -1187,95 +1181,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.58", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -1747,16 +1652,6 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.20" @@ -1926,17 +1821,6 @@ dependencies = [ "winapi 0.2.8", ] -[[package]] -name = "mio" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - [[package]] name = "mio-extras" version = "2.0.6" @@ -1945,7 +1829,7 @@ checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" dependencies = [ "lazycell", "log", - "mio 0.6.23", + "mio", "slab", ] @@ -2019,7 +1903,7 @@ dependencies = [ "fsevent-sys", "inotify", "libc", - "mio 0.6.23", + "mio", "mio-extras", "walkdir", "winapi 0.3.9", @@ -2069,16 +1953,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.4", - "libc", -] - [[package]] name = "o1-utils" version = "0.1.0" @@ -2287,29 +2161,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "redox_syscall 0.5.8", - "smallvec", - "windows-targets 0.52.0", -] - [[package]] name = "paste" version = "1.0.14" @@ -2367,18 +2218,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.29" @@ -2640,15 +2479,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_syscall" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" -dependencies = [ - "bitflags 2.4.2", -] - [[package]] name = "regex" version = "1.10.3" @@ -2778,15 +2608,10 @@ dependencies = [ "ark-ff", "ark-serialize", "ark-std", - "bytes", "clap 4.4.18", - "futures", "mina-curves", "o1-utils", "proptest", - "serde", - "tokio", - "tokio-util", ] [[package]] @@ -2798,12 +2623,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "secp256k1" version = "0.28.2" @@ -2925,15 +2744,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - [[package]] name = "slab" version = "0.4.9" @@ -2943,28 +2753,12 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - [[package]] name = "smawk" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" -[[package]] -name = "socket2" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "stacker" version = "0.1.15" @@ -3120,7 +2914,7 @@ checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if 1.0.0", "fastrand", - "redox_syscall 0.4.1", + "redox_syscall", "rustix", "windows-sys 0.52.0", ] @@ -3253,49 +3047,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tokio" -version = "1.38.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio 0.8.11", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.58", -] - -[[package]] -name = "tokio-util" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "0.5.11" diff --git a/saffron/.gitignore b/saffron/.gitignore index 402fd5d873..99b9bbeebd 100644 --- a/saffron/.gitignore +++ b/saffron/.gitignore @@ -1 +1,3 @@ proptest-regressions +fixtures/lorem.bin +fixtures/lorem-decoded.txt diff --git a/saffron/Cargo.toml b/saffron/Cargo.toml index 2ad0b56cd9..ef77f57cf7 100644 --- a/saffron/Cargo.toml +++ b/saffron/Cargo.toml @@ -19,15 +19,10 @@ path = "src/main.rs" [dependencies] anyhow = "1.0" ark-ff.workspace = true -ark-serialize.workspace = true -bytes = "1.0" +ark-serialize = { workspace = true, features = ["derive"]} clap = { workspace = true, features = ["derive"] } mina-curves.workspace = true o1-utils.workspace = true -futures = "0.3" -serde.workspace = true -tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.7", features = ["io"] } [dev-dependencies] diff --git a/saffron/fixtures/.gitignore b/saffron/fixtures/.gitignore deleted file mode 100644 index da2818c571..0000000000 --- a/saffron/fixtures/.gitignore +++ /dev/null @@ -1 +0,0 @@ -lorem-output.txt diff --git a/saffron/fixtures/lorem-bin b/saffron/fixtures/lorem-bin deleted file mode 100644 index dbb682d2dee20342c44a413087161c088b30d03a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 488 zcmY+BF>*s83`D8ZGgHZI2kvB z$->mC1W6CTSmIZl8cF9HnsjLhgD4pGouUVt8`bLQDn=%G@|{t9+S1fo3_y%{ z86mWtf=HHZ;TirseG1r|w$ld=3m{&5ZY`RISG^&~O(vJlzdQ%e{qQ7BPtaE=wMquB z6Ds7!?_wqsmMzr!Lhs|_tGa8 z4N}!ENWCphrsflpIk*zr4O$q*r8x$m^i_L?X&o`lmYA8n8Qf6hBjllWN-L_|nm`Kg zE(hKX+E_yymHYCDr_>AX&V|;OP0!FTorxyK-r%K3P5o5fO(aTawU=!} RHXsI$^~=@k@%;4vd;^A?r4s-E diff --git a/saffron/src/cli.rs b/saffron/src/cli.rs index 761567f9d7..6f539136a3 100644 --- a/saffron/src/cli.rs +++ b/saffron/src/cli.rs @@ -26,14 +26,6 @@ pub struct DecodeFileArgs { #[arg(long, short = 'o', value_name = "FILE", help = "output file")] pub output: String, - - #[arg( - long, - short = 'n', - value_name = "NUM", - help = "number of bytes to decode" - )] - pub length: usize } #[derive(Parser, Debug, Clone)] diff --git a/saffron/src/main.rs b/saffron/src/main.rs index e31c9dbec0..a8dfa711e6 100644 --- a/saffron/src/main.rs +++ b/saffron/src/main.rs @@ -1,7 +1,8 @@ use anyhow::Result; +use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use clap::Parser; use mina_curves::pasta::Fp; -use saffron::cli; +use saffron::{cli, serialization::FieldBlob}; use std::{ fs::File, io::{Read, Write}, @@ -11,15 +12,10 @@ fn decode_file(args: cli::DecodeFileArgs) -> Result<()> { let mut file = File::open(args.input)?; let mut buf = Vec::new(); file.read_to_end(&mut buf)?; - let xs = saffron::serialization::deserialize_vec::(&buf); - let bytes: Vec = xs - .into_iter() - .flat_map(|x| { - saffron::serialization::decode(x).as_slice()[1..32].to_vec() - }) - .collect(); + let blob: FieldBlob = FieldBlob::::deserialize_compressed(&buf[..])?; + let data = FieldBlob::::decode(blob); let mut writer = File::create(args.output)?; - writer.write_all(&bytes)?; + writer.write_all(&data)?; Ok(()) } @@ -27,17 +23,11 @@ fn encode_file(args: cli::EncodeFileArgs) -> Result<()> { let mut file = File::open(args.input)?; let mut buf = Vec::new(); file.read_to_end(&mut buf)?; - let xs = buf - .chunks(31) - .map(|chunk| { - let mut bytes = [0u8; 31]; - bytes[..chunk.len()].copy_from_slice(chunk); - saffron::serialization::encode(&bytes) - }) - .collect::>(); - let bytes = saffron::serialization::serialize_vec(&xs); + let blob = FieldBlob::::encode(&buf); + let mut bytes_to_write = Vec::with_capacity(buf.len()); + blob.serialize_compressed(&mut bytes_to_write)?; let mut writer = File::create(args.output)?; - writer.write_all(&bytes)?; + writer.write_all(&bytes_to_write)?; Ok(()) } diff --git a/saffron/src/serialization.rs b/saffron/src/serialization.rs index e1f46f0e19..292ccda8af 100644 --- a/saffron/src/serialization.rs +++ b/saffron/src/serialization.rs @@ -1,26 +1,91 @@ -use ark_ff::{BigInteger, Field, PrimeField}; -use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress}; +use ark_ff::{BigInteger, PrimeField}; +use ark_serialize::{ + CanonicalDeserialize, CanonicalSerialize, Compress, Read, SerializationError, Valid, Validate, + Write, +}; +use o1_utils::FieldHelpers; // For injectivity, you can only use this on inputs of length at most // 'F::MODULUS_BIT_SIZE / 8', e.g. for Vesta this is 31. -pub fn encode(bytes: &[u8]) -> Fp { +fn encode(bytes: &[u8]) -> Fp { Fp::from_be_bytes_mod_order(bytes) } -pub fn decode(x: Fp) -> Vec { +fn decode(x: Fp) -> Vec { x.into_bigint().to_bytes_be() } -pub fn serialize_vec(xs: &[F]) -> Vec { - let n = xs.serialized_size(Compress::Yes); - let mut writer = Vec::with_capacity(n); - xs.serialize_compressed(&mut writer) - .expect("Failed to serialize field elements"); - writer +// A FieldBlob represents the encoding of a Vec as a Vec where F is a prime field. +#[derive(Clone, Debug, PartialEq)] +pub struct FieldBlob { + pub n_bytes: usize, + pub data: Vec, } -pub fn deserialize_vec(bytes: &[u8]) -> Vec { - Vec::::deserialize_compressed(bytes).expect("Failed to deserialize field elements") +impl CanonicalSerialize for FieldBlob { + fn serialize_with_mode( + &self, + mut writer: W, + mode: Compress, + ) -> Result<(), SerializationError> { + self.n_bytes.serialize_with_mode(&mut writer, mode)?; + self.data.serialize_with_mode(&mut writer, mode)?; + Ok(()) + } + + fn serialized_size(&self, mode: Compress) -> usize { + self.n_bytes.serialized_size(mode) + self.data.serialized_size(mode) + } +} + +impl Valid for FieldBlob { + fn check(&self) -> Result<(), SerializationError> { + self.n_bytes.check()?; + self.data.check()?; + Ok(()) + } +} + +impl CanonicalDeserialize for FieldBlob { + fn deserialize_with_mode( + mut reader: R, + compress: Compress, + validate: Validate, + ) -> Result { + let n_bytes = usize::deserialize_with_mode(&mut reader, compress, validate)?; + let data = Vec::::deserialize_with_mode(&mut reader, compress, validate)?; + Ok(Self { n_bytes, data }) + } +} + +impl FieldBlob { + // Encode a bytestring as a list of field elements. + pub fn encode(bytes: &[u8]) -> FieldBlob { + let n = (F::MODULUS_BIT_SIZE / 8) as usize; + let data = bytes + .chunks(n) + .map(|chunk| { + let mut bytes = vec![0u8; n]; + bytes[..chunk.len()].copy_from_slice(chunk); + encode(&bytes) + }) + .collect::>(); + FieldBlob { + n_bytes: bytes.len(), + data, + } + } + + // Decode a list of field elements as a bytestring. + pub fn decode(blob: FieldBlob) -> Vec { + let n = (F::MODULUS_BIT_SIZE / 8) as usize; + let m = F::size_in_bytes(); + blob.data + .into_iter() + .flat_map(|x| decode(x).as_slice()[(m - n)..m].to_vec()) + .take(blob.n_bytes) + .collect() + } } #[cfg(test)] @@ -30,6 +95,7 @@ mod tests { use mina_curves::pasta::Fp; use proptest::prelude::*; + // Check that [u8] -> Fp -> [u8] is the identity function. proptest! { #[test] fn test_round_trip_from_bytes(xs in any::<[u8;31]>()) @@ -39,6 +105,7 @@ mod tests { } } + // Check that Fp -> [u8] -> Fp is the identity function. proptest! { #[test] fn test_round_trip_from_fp( @@ -50,18 +117,19 @@ mod tests { } } - fn fp_strategy() -> impl Strategy { - prop::strategy::Just(Fp::rand(&mut ark_std::rand::thread_rng())) - } - + // check that Vec -> FieldBlob -> Vec is the identity function proptest! { - #[test] - fn test_round_trip_vec( - xs in prop::collection::vec(fp_strategy(), 0..100) - ) { - let bytes = serialize_vec(&xs); - let ys = deserialize_vec(&bytes); - prop_assert_eq!(xs,ys); - } + #[test] + fn test_round_trip_blob_encoding( xs in any::>()) + { let blob = FieldBlob::::encode(&xs); + let mut buf = Vec::new(); + blob.serialize_compressed(&mut buf).unwrap(); + let a = FieldBlob::::deserialize_compressed(&buf[..]).unwrap(); + // check that ark-serialize is behaving as expected + prop_assert_eq!(blob.clone(), a); + let ys = FieldBlob::::decode(blob); + // check that we get the byte blob back again + prop_assert_eq!(xs,ys); + } } } diff --git a/saffron/test-encoding.sh b/saffron/test-encoding.sh new file mode 100755 index 0000000000..23d1095ac2 --- /dev/null +++ b/saffron/test-encoding.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Run encode and decode +cargo run --bin saffron encode -i fixtures/lorem.txt -o fixtures/lorem.bin +cargo run --bin saffron decode -i fixtures/lorem.bin -o fixtures/lorem-decoded.txt + +# Compare files +if cmp -s fixtures/lorem.txt fixtures/lorem-decoded.txt; then + echo "Files are identical" + exit 0 +else + echo "Files differ" + diff fixtures/lorem.txt fixtures/lorem-decoded.txt + exit 1 +fi \ No newline at end of file