fix: out of memory

plonky/src/api.rs

@@ -14,17 +14,22 @@ use std::path::Path;
 
 // generate a monomial_form SRS, and save it to a file
 #[time_profiler("plonk_setup")]
-pub fn setup(power: u32, srs_monomial_form: &str) -> Result<()> {
+pub fn setup<W: std::io::Write>(
+    power: u32,
+    /*srs_monomial_form: &str, */ writer: W,
+) -> Result<()> {
     let srs = plonk::gen_key_monomial_form(power)?;
+    srs.write(writer)?;
+    /*
     let path = Path::new(srs_monomial_form);
     assert!(
         !path.exists(),
         "duplicate srs_monomial_form file: {}",
         path.display()
     );
     let writer = std::fs::File::create(srs_monomial_form)?;
-    srs.write(writer)?;
     log::trace!("srs_monomial_form saved to {}", srs_monomial_form);
+    */
     Result::Ok(())
 }
 

recursion/src/compressor12/compressor12_exec.rs

@@ -3,13 +3,13 @@ use crate::compressor12_pil::CompressorPolName::a;
 use crate::io_utils::read_vec_from_file;
 use crate::pilcom::compile_pil_from_path;
 use algebraic::witness::{load_input_for_witness, WitnessCalculator};
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use fields::ff::PrimeField;
 use fields::field_gl::Fr as FGL;
 use num_traits::Zero;
 use starky::polsarray::{PolKind, PolsArray};
 use std::fs::File;
-use std::io::Write;
+use std::io::{BufReader, Read, Write};
 use std::path::Path;
 
 // exec phase:
@@ -23,7 +23,10 @@ pub fn exec(
     commit_file: &str,
 ) -> Result<()> {
     // 0. load exec_file,
-    let (adds_len, s_map_column_len, adds, s_map) = read_exec_file(exec_file)?;
+
+    let inputs_str = File::open(exec_file).map_err(|e| anyhow!("Read {}, {:?}", exec_file, e))?;
+    let reader = BufReader::new(inputs_str);
+    let (adds_len, s_map_column_len, adds, s_map) = read_exec_file(reader)?;
 
     // 1. Compiles a .pil file to its json form , and save it.
     // TODO: the pil_str has been compiled in plonk_setup#3
@@ -92,14 +95,15 @@ pub fn exec(
     }
 
     // 5. save cmPol to file.
-    cm_pols.save(commit_file)?;
+    let commit_writer = std::fs::File::create(commit_file)?;
+    cm_pols.save(commit_writer)?;
 
     log::trace!("files Generated Correctly");
     Result::Ok(())
 }
 
-fn read_exec_file(exec_file: &str) -> Result<(usize, usize, Vec<u64>, Vec<u64>)> {
-    let mut buff = read_vec_from_file(exec_file)?;
+fn read_exec_file<R: Read>(reader: R) -> Result<(usize, usize, Vec<u64>, Vec<u64>)> {
+    let mut buff = read_vec_from_file(reader)?;
 
     let mut new_buff = buff.split_off(2);
     let adds_len = buff[0] as usize;
@@ -118,6 +122,7 @@ fn read_exec_file(exec_file: &str) -> Result<(usize, usize, Vec<u64>, Vec<u64>)>
 mod test {
     use super::*;
     use crate::compressor12_setup::write_exec_file;
+    use std::io::BufWriter;
 
     #[test]
     fn test_write_and_read_exec_file() {
@@ -142,9 +147,12 @@ mod test {
             vec![3, 4, 5],
         ];
 
-        write_exec_file(&file_path, &target_adds, &target_s_map).unwrap();
+        let out = Vec::new();
+        let buf = BufWriter::new(out);
+        write_exec_file(buf, &target_adds, &target_s_map).unwrap();
 
-        let (adds_len, _s_map_column_len, _adds, _s_map) = read_exec_file(&file_path).unwrap();
+        let file = File::open(file_path).unwrap();
+        let (adds_len, _s_map_column_len, _adds, _s_map) = read_exec_file(file).unwrap();
 
         assert_eq!(adds_len, target_adds.len());
     }

recursion/src/compressor12/compressor12_setup.rs

@@ -2,7 +2,7 @@
 use crate::compressor12::plonk_setup::PlonkSetup;
 use crate::io_utils::write_vec_to_file;
 use crate::r1cs2plonk::PlonkAdd;
-use algebraic::reader::load_r1cs;
+use algebraic::reader::load_r1cs_from_bin;
 use anyhow::{anyhow, Result};
 use fields::field_gl::GL;
 use std::fs::File;
@@ -23,7 +23,8 @@ pub fn setup(
     force_n_bits: usize,
 ) -> Result<()> {
     // 0. readR1cs
-    let r1cs = load_r1cs::<GL>(r1cs_file);
+    let r1cs_reader = File::open(r1cs_file)?;
+    let (r1cs, _) = load_r1cs_from_bin::<_, GL>(r1cs_reader);
     let opts = Options {
         force_bits: force_n_bits,
     };
@@ -37,17 +38,20 @@ pub fn setup(
     write!(file, "{}", res.pil_str)?;
 
     // 3. write const pols file
-    res.const_pols.save(const_file)?;
+    let const_writer = std::fs::File::create(const_file)?;
+    res.const_pols.save(const_writer)?;
 
+    let file_writer =
+        File::create(exec_file).map_err(|e| anyhow!("Create {}, {:?}", exec_file, e))?;
     // 4. construct and save ExecFile: plonk additions + sMap -> BigUint64Array
-    write_exec_file(exec_file, &res.plonk_additions, &res.s_map)?;
+    write_exec_file(file_writer, &res.plonk_additions, &res.s_map)?;
 
     Ok(())
 }
 
 // construct and save ExecFile: plonk additions + sMap -> BigUint64Array
-pub(super) fn write_exec_file(
-    exec_file: &str,
+pub(super) fn write_exec_file<W: std::io::Write>(
+    exec_file_writer: W,
     adds: &[PlonkAdd],
     s_map: &[Vec<u64>],
 ) -> Result<()> {
@@ -77,5 +81,5 @@ pub(super) fn write_exec_file(
         }
     }
 
-    write_vec_to_file(exec_file, &buff)
+    write_vec_to_file(exec_file_writer, &buff)
 }

recursion/src/io_utils.rs

@@ -1,18 +1,13 @@
-use anyhow::{anyhow, Result};
-use std::fs::File;
-use std::io::Write;
+use anyhow::Result;
 
-pub fn write_vec_to_file(path: &str, vec: &[u64]) -> Result<()> {
-    let mut file = File::create(path).map_err(|e| anyhow!("Create {}, {:?}", path, e))?;
+pub fn write_vec_to_file<W: std::io::Write>(mut writer: W, vec: &[u64]) -> Result<()> {
     let input = serde_json::to_string(&vec)?;
-    write!(file, "{}", input)?;
+    write!(writer, "{}", input)?;
     Ok(())
 }
 
-pub fn read_vec_from_file(input_file: &str) -> Result<Vec<u64>> {
-    let inputs_str =
-        std::fs::read_to_string(input_file).map_err(|e| anyhow!("Read {}, {:?}", input_file, e))?;
-    let output: Vec<u64> = serde_json::from_str(&inputs_str)?;
+pub fn read_vec_from_file<R: std::io::Read>(reader: R) -> Result<Vec<u64>> {
+    let output = serde_json::from_reader(reader)?;
     Ok(output)
 }
 
@@ -25,9 +20,11 @@ mod test {
     fn test_read_write_vec_with_file() -> Result<()> {
         let target: Vec<u64> = vec![1, 2, 3, 4, 5, 1111112121, 2667022304383014929];
         let path = String::from("/tmp/vec_data.txt");
-        write_vec_to_file(&path, &target)?;
+        let file = std::fs::File::create(path.clone()).unwrap();
+        write_vec_to_file(file, &target)?;
 
-        let actual = read_vec_from_file(&path)?;
+        let file = std::fs::File::open(path).unwrap();
+        let actual = read_vec_from_file(file)?;
         assert_eq!(actual, target);
 
         Ok(())

starky/src/interpreter.rs

@@ -89,12 +89,6 @@ pub struct Block<T: FieldExtension> {
 }
 
 impl<T: FieldExtension> Block<T> {
-    fn codegen(&self, step: &str, codebuf: String) {
-        use std::io::Write;
-        let mut f = std::fs::File::create(format!("/tmp/{}_{}.rs", self.namespace, step)).unwrap();
-        write!(f, "{}", &codebuf).unwrap();
-    }
-
     /// parameters: ctx, i
     /// example:
     /// let block = compile_code();

starky/src/polsarray.rs

@@ -187,8 +187,7 @@ impl PolsArray {
         Ok(())
     }
 
-    pub fn save(&self, fileName: &str) -> Result<()> {
-        let mut writer = File::create(fileName)?;
+    pub fn save<W: Write>(&self, mut writer: W) -> Result<()> {
         let maxBufferSize = 1024 * 1024 * 32;
         let totalSize = self.nPols * self.n;
         let mut buff: Vec<u64> = vec![0u64; std::cmp::min(totalSize, maxBufferSize)];
@@ -246,10 +245,12 @@ pub mod tests {
         let pil = types::load_json::<PIL>("data/fib.pil.json").unwrap();
         let mut cp = PolsArray::new(&pil, PolKind::Constant);
         cp.load("data/fib.const").unwrap();
-        cp.save("data/fib.const.cp").unwrap();
+        let file = File::create("data/fib.const.cp").unwrap();
+        cp.save(file).unwrap();
 
         let mut cmp = PolsArray::new(&pil, PolKind::Commit);
         cmp.load("data/fib.exec").unwrap();
-        cmp.save("data/fib.exec.cp").unwrap();
+        let file = File::create("data/fib.exec.cp").unwrap();
+        cmp.save(file).unwrap();
     }
 }

starky/src/prove.rs

@@ -47,41 +47,43 @@ pub fn stark_prove(
     cm_pol.load(cm_pol_file)?;
 
     let stark_struct = load_json::<StarkStruct>(stark_struct)?;
+    let circom_file_writer = File::create(circom_file)?;
+    let zkin_writer = File::create(zkin)?;
     match stark_struct.verificationHashType.as_str() {
-        "BN128" => prove::<Fr_BN128, MerkleTreeBN128, TranscriptBN128>(
+        "BN128" => prove::<Fr_BN128, MerkleTreeBN128, TranscriptBN128, _>(
             &mut pil,
             const_pol,
             cm_pol,
             &stark_struct,
             false,
             norm_stage,
             skip_main,
-            circom_file,
-            zkin,
+            circom_file_writer,
+            zkin_writer,
             prover_addr,
         ),
-        "BLS12381" => prove::<Fr_BLS12381, MerkleTreeBLS12381, TranscriptBLS128>(
+        "BLS12381" => prove::<Fr_BLS12381, MerkleTreeBLS12381, TranscriptBLS128, _>(
             &mut pil,
             const_pol,
             cm_pol,
             &stark_struct,
             false,
             norm_stage,
             skip_main,
-            circom_file,
-            zkin,
+            circom_file_writer,
+            zkin_writer,
             prover_addr,
         ),
-        "GL" => prove::<FGL, MerkleTreeGL, TranscriptGL>(
+        "GL" => prove::<FGL, MerkleTreeGL, TranscriptGL, _>(
             &mut pil,
             const_pol,
             cm_pol,
             &stark_struct,
             agg_stage,
             norm_stage,
             skip_main,
-            circom_file,
-            zkin,
+            circom_file_writer,
+            zkin_writer,
             prover_addr,
         ),
         _ => panic!("Invalid hashtype {}", stark_struct.verificationHashType),
@@ -94,6 +96,7 @@ fn prove<
     F: PrimeField + Default,
     M: MerkleTree<MTNode = ElementDigest<4, F>> + Default,
     T: Transcript,
+    W: Write,
 >(
     pil: &mut PIL,
     const_pol: PolsArray,
@@ -102,8 +105,8 @@ fn prove<
     agg_stage: bool,
     norm_stage: bool,
     skip_main: bool,
-    circom_file: &str,
-    zkin: &str,
+    mut circom_file_writer: W,
+    mut zkin_writer: W,
     prover_addr: &str,
 ) -> Result<()> {
     let mut setup = StarkSetup::<M>::new(&const_pol, pil, stark_struct, None)?;
@@ -117,7 +120,6 @@ fn prove<
         stark_struct,
         prover_addr,
     )?;
-    log::debug!("generate the proof done");
 
     let result = stark_verify::<M, T>(
         &starkproof,
@@ -128,7 +130,6 @@ fn prove<
     )?;
 
     assert!(result);
-    log::debug!("verify the proof done");
 
     let opt = pil2circom::StarkOption {
         enable_input: false,
@@ -145,17 +146,15 @@ fn prove<
         &mut setup.program,
         &opt,
     )?;
-    let mut file = File::create(circom_file)?;
-    write!(file, "{}", str_ver)?;
-    log::debug!("generate circom done");
+    write!(circom_file_writer, "{}", str_ver)?;
 
     // if agg_stage {
     //     starkproof.rootC = None;
     // }
 
     let input = serde_json::to_string(&starkproof)?;
-    let mut file = File::create(zkin)?;
-    write!(file, "{}", input)?;
-    log::debug!("generate zkin done");
+    write!(zkin_writer, "{}", input)?;
+    drop(setup);
+    drop(starkproof);
     Ok(())
 }

zkit/src/main.rs

@@ -397,7 +397,18 @@ fn main() {
     env_logger::init();
     let start = Instant::now();
     let exec_result = match args.command {
-        Command::Setup(args) => setup(args.power, &args.srs_monomial_form),
+        Command::Setup(args) => {
+            let path = std::path::Path::new(&args.srs_monomial_form);
+            assert!(
+                !path.exists(),
+                "duplicate srs_monomial_form file: {}",
+                path.display()
+            );
+            let writer =
+                std::fs::File::create(&args.srs_monomial_form).expect("can not create file");
+            log::debug!("srs_monomial_form saved to {}", &args.srs_monomial_form);
+            setup(args.power, writer)
+        }
         Command::Compile(args) => circom_compiler(
             args.input,
             args.prime.to_lowercase(),