feat: add groth16 proving (0xEigenLabs#128)

* feat: add groth16 proving

* chore: add groth16 bls12381 test

* refactor: re-org module

* refactor: remove redundant test vectors

* refactor: remove redundant test vectors

* refactor: remove redundant test vectors

* refactor: remove redundant test vectors

* refactor: cargo fmt

* chore: add groth16 api interface

* chore: add groth16 command

* chore: fix regs

* chore: remove redundant script

* fix: unify curve name

---------

Co-authored-by: eigmax <[email protected]>

Cargo.toml

@@ -3,6 +3,8 @@ members = [
   "zkit",
   "plonky",
   "starky",
+  "algebraic",
+  "groth16",
   "dsl_compile"
 ]
-resolver = "2"
+resolver = "2"

algebraic/Cargo.toml

@@ -0,0 +1,49 @@
+[package]
+name = "algebraic"
+version = "0.0.1"
+edition = "2021"
+license = "Apache-2.0"
+description = "Eigen Algbraic based on R1CS"
+documentation = "eigen.cash"
+homepage = "eigen.cash"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+proptest = "1.1"
+hex-literal = "0.2.1"
+itertools = "0.8.1"
+log = "0.4.11"
+num-bigint = "0.3.3"
+num-traits = "0.2.8"
+serde = { version = "1.0", features = [ "derive" ] }
+serde_json = { version = "1.0", features = [ "arbitrary_precision" ] }
+hex = "*"
+wasmer = { version = "2.0", default-features = false }
+thiserror="1.0"
+fnv = { version = "1.0.3", default-features = false }
+num = { version = "0.4.0" }
+byteorder = "1"
+franklin-crypto = { git = "https://github.com/matter-labs/franklin-crypto", branch = "beta", features = [ "plonk" ], version = "0.0.5"}
+#franklin-crypto = { path = "../../franklin-crypto", features = [ "plonk" ], version = "0.0.5"}
+
+[dev-dependencies]
+env_logger = "0.10"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+rand = "0.4"
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+wasm-bindgen = { version = "0.2.51", features = ["serde-serialize"]  }
+wasm-bindgen-futures = "0.4.1"
+rand = { version="0.6.5", features = ["wasm-bindgen"] }
+
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+wasm-bindgen-test = "0.3"
+
+[features]
+default = ["franklin-crypto/multicore", "wasmer/default"]
+wasm = ["wasmer/js-default"]

algebraic/src/lib.rs

@@ -0,0 +1,64 @@
+#![allow(clippy::unit_arg)]
+
+#[macro_use]
+extern crate serde;
+#[macro_use]
+extern crate hex_literal;
+extern crate byteorder;
+extern crate franklin_crypto;
+extern crate itertools;
+extern crate num_bigint;
+extern crate num_traits;
+extern crate rand;
+
+pub mod circom_circuit;
+pub mod errors;
+pub mod field_gl;
+pub mod r1cs_file;
+pub mod reader;
+pub mod witness;
+
+pub mod utils;
+
+pub use bellman_ce::pairing::ff;
+pub use ff::*;
+pub use franklin_crypto::bellman as bellman_ce;
+
+#[cfg(test)]
+mod field_gl_test;
+
+#[cfg(target_arch = "wasm32")]
+extern crate wasm_bindgen;
+
+#[cfg(all(test, target_arch = "wasm32"))]
+extern crate wasm_bindgen_test;
+
+#[cfg(target_arch = "wasm32")]
+use wasm_bindgen::prelude::*;
+
+#[cfg(target_arch = "wasm32")]
+#[wasm_bindgen]
+extern "C" {
+    // Use `js_namespace` here to bind `console.log(..)` instead of just
+    // `log(..)`
+    #[wasm_bindgen(js_namespace = console)]
+    fn log(s: &str);
+
+    // The `console.log` is quite polymorphic, so we can bind it with multiple
+    // signatures. Note that we need to use `js_name` to ensure we always call
+    // `log` in JS.
+    #[wasm_bindgen(js_namespace = console, js_name = log)]
+    fn log_u32(a: u32);
+
+    // Multiple arguments too!
+    #[wasm_bindgen(js_namespace = console, js_name = log)]
+    fn log_many(a: &str, b: &str);
+}
+
+#[cfg(target_arch = "wasm32")]
+#[macro_export]
+macro_rules! console_log {
+    // Note that this is using the `log` function imported above during
+    // `bare_bones`
+    ($($t:tt)*) => (log(&format_args!($($t)*).to_string()))
+}

plonky/src/r1cs_file.rs → algebraic/src/r1cs_file.rs

@@ -244,6 +244,8 @@ pub fn from_reader<R: Read + Seek, E: ScalarEngine>(mut reader: R) -> Result<R1C
     }
     if !(header.prime_size
         == hex!("010000f093f5e1439170b97948e833285d588181b64550b829a031e1724e6430")
+        || header.prime_size
+            == hex!("01000000fffffffffe5bfeff02a4bd5305d8a10908d83933487d9d2953a7ed73")
         || header.prime_size == hex!("01000000ffffffff"))
     {
         return Err(Error::new(

algebraic/src/reader.rs

@@ -0,0 +1,232 @@
+use crate::errors::{EigenError, Result};
+use byteorder::{LittleEndian, ReadBytesExt};
+use itertools::Itertools;
+use std::collections::BTreeMap;
+use std::fs::{File, OpenOptions};
+use std::io::{BufReader, Read, Seek};
+use std::str;
+
+use crate::bellman_ce::{
+    kate_commitment::{Crs, CrsForLagrangeForm, CrsForMonomialForm},
+    pairing::Engine,
+    Field, PrimeField, PrimeFieldRepr, ScalarEngine,
+};
+
+use crate::circom_circuit::{CircuitJson, R1CS};
+
+/// get universal setup file by filename
+fn get_universal_setup_file_buff_reader(setup_file_name: &str) -> Result<BufReader<File>> {
+    let setup_file = File::open(setup_file_name).map_err(|e| {
+        EigenError::from(format!(
+            "Failed to open universal setup file {}, err: {}",
+            setup_file_name, e
+        ))
+    })?;
+    Ok(BufReader::with_capacity(1 << 29, setup_file))
+}
+
+/// load monomial form SRS by filename
+pub fn load_key_monomial_form<E: Engine>(filename: &str) -> Crs<E, CrsForMonomialForm> {
+    let mut buf_reader =
+        get_universal_setup_file_buff_reader(filename).expect("read key_monomial_form file err");
+    Crs::<E, CrsForMonomialForm>::read(&mut buf_reader).expect("read key_monomial_form err")
+}
+
+/// load optional lagrange form SRS by filename
+pub fn maybe_load_key_lagrange_form<E: Engine>(
+    option_filename: Option<String>,
+) -> Option<Crs<E, CrsForLagrangeForm>> {
+    match option_filename {
+        None => None,
+        Some(filename) => {
+            let mut buf_reader = get_universal_setup_file_buff_reader(&filename)
+                .expect("read key_lagrange_form file err");
+            let key_lagrange_form = Crs::<E, CrsForLagrangeForm>::read(&mut buf_reader)
+                .expect("read key_lagrange_form err");
+            Some(key_lagrange_form)
+        }
+    }
+}
+
+/// load witness file by filename with autodetect encoding (bin or json).
+pub fn load_witness_from_file<E: ScalarEngine>(filename: &str) -> Vec<E::Fr> {
+    if filename.ends_with("json") {
+        load_witness_from_json_file::<E>(filename)
+    } else {
+        load_witness_from_bin_file::<E>(filename)
+    }
+}
+
+/// load witness from json file by filename
+pub fn load_witness_from_json_file<E: ScalarEngine>(filename: &str) -> Vec<E::Fr> {
+    let reader = OpenOptions::new()
+        .read(true)
+        .open(filename)
+        .expect("unable to open.");
+    load_witness_from_json::<E, BufReader<File>>(BufReader::new(reader))
+}
+
+/// load witness from json by a reader
+fn load_witness_from_json<E: ScalarEngine, R: Read>(reader: R) -> Vec<E::Fr> {
+    let witness: Vec<String> = serde_json::from_reader(reader).expect("unable to read.");
+    witness
+        .into_iter()
+        .map(|x| E::Fr::from_str(&x).unwrap())
+        .collect::<Vec<E::Fr>>()
+}
+
+/// load witness from bin file by filename
+pub fn load_witness_from_bin_file<E: ScalarEngine>(filename: &str) -> Vec<E::Fr> {
+    let reader = OpenOptions::new()
+        .read(true)
+        .open(filename)
+        .expect("unable to open.");
+    load_witness_from_bin_reader::<E, BufReader<File>>(BufReader::new(reader))
+        .expect("read witness failed")
+}
+
+/// load witness from u8 array
+pub fn load_witness_from_array<E: ScalarEngine>(buffer: Vec<u8>) -> Result<Vec<E::Fr>> {
+    load_witness_from_bin_reader::<E, _>(buffer.as_slice())
+}
+
+/// load witness from u8 array by a reader
+pub fn load_witness_from_bin_reader<E: ScalarEngine, R: Read>(mut reader: R) -> Result<Vec<E::Fr>> {
+    let mut wtns_header = [0u8; 4];
+    reader.read_exact(&mut wtns_header)?;
+    if wtns_header != [119, 116, 110, 115] {
+        // python -c 'print([ord(c) for c in "wtns"])' => [119, 116, 110, 115]
+        return Err(EigenError::from("Invalid file header".to_string()));
+    }
+    let version = reader.read_u32::<LittleEndian>()?;
+    log::debug!("wtns version {}", version);
+    if version > 2 {
+        return Err(EigenError::from("unsupported file version".to_string()));
+    }
+    let num_sections = reader.read_u32::<LittleEndian>()?;
+    if num_sections != 2 {
+        return Err(EigenError::from("invalid num sections".to_string()));
+    }
+    // read the first section
+    let sec_type = reader.read_u32::<LittleEndian>()?;
+    if sec_type != 1 {
+        return Err(EigenError::from("invalid section type".to_string()));
+    }
+    let sec_size = reader.read_u64::<LittleEndian>()?;
+    if sec_size != 4 + 32 + 4 {
+        return Err(EigenError::from("invalid section len".to_string()));
+    }
+    let field_size = reader.read_u32::<LittleEndian>()?;
+    if field_size != 32 {
+        return Err(EigenError::from("invalid field byte size".to_string()));
+    }
+    let mut prime = vec![0u8; field_size as usize];
+    reader.read_exact(&mut prime)?;
+    if prime != hex!("010000f093f5e1439170b97948e833285d588181b64550b829a031e1724e6430") {
+        return Err(EigenError::from("invalid curve prime".to_string()));
+    }
+    let witness_len = reader.read_u32::<LittleEndian>()?;
+    log::debug!("witness len {}", witness_len);
+    let sec_type = reader.read_u32::<LittleEndian>()?;
+    if sec_type != 2 {
+        return Err(EigenError::from("invalid section type".to_string()));
+    }
+    let sec_size = reader.read_u64::<LittleEndian>()?;
+    if sec_size != (witness_len * field_size) as u64 {
+        return Err(EigenError::from(format!(
+            "Invalid witness section size {}",
+            sec_size
+        )));
+    }
+    let mut result = Vec::with_capacity(witness_len as usize);
+    for _ in 0..witness_len {
+        let mut repr = E::Fr::zero().into_repr();
+        repr.read_le(&mut reader)?;
+        result.push(E::Fr::from_repr(repr)?);
+    }
+    Ok(result)
+}
+
+/// load r1cs file by filename with autodetect encoding (bin or json)
+pub fn load_r1cs<E: ScalarEngine>(filename: &str) -> R1CS<E> {
+    if filename.ends_with("json") {
+        load_r1cs_from_json_file(filename)
+    } else {
+        let (r1cs, _wire_mapping) = load_r1cs_from_bin_file(filename);
+        r1cs
+    }
+}
+
+/// load r1cs from json file by filename
+fn load_r1cs_from_json_file<E: ScalarEngine>(filename: &str) -> R1CS<E> {
+    let reader = OpenOptions::new()
+        .read(true)
+        .open(filename)
+        .expect("unable to open.");
+    load_r1cs_from_json(BufReader::new(reader))
+}
+
+/// load r1cs from json by a reader
+fn load_r1cs_from_json<E: ScalarEngine, R: Read>(reader: R) -> R1CS<E> {
+    let circuit_json: CircuitJson = serde_json::from_reader(reader).expect("unable to read.");
+
+    let num_inputs = circuit_json.num_inputs + circuit_json.num_outputs + 1;
+    let num_aux = circuit_json.num_variables - num_inputs;
+
+    let convert_constraint = |lc: &BTreeMap<String, String>| {
+        lc.iter()
+            .map(|(index, coeff)| (index.parse().unwrap(), E::Fr::from_str(coeff).unwrap()))
+            .collect_vec()
+    };
+
+    let constraints = circuit_json
+        .constraints
+        .iter()
+        .map(|c| {
+            (
+                convert_constraint(&c[0]),
+                convert_constraint(&c[1]),
+                convert_constraint(&c[2]),
+            )
+        })
+        .collect_vec();
+
+    R1CS {
+        num_inputs,
+        num_aux,
+        num_variables: circuit_json.num_variables,
+        num_outputs: circuit_json.num_outputs,
+        constraints,
+        custom_gates: vec![],
+        custom_gates_uses: vec![],
+    }
+}
+
+/// load r1cs from bin file by filename
+fn load_r1cs_from_bin_file<E: ScalarEngine>(filename: &str) -> (R1CS<E>, Vec<usize>) {
+    let reader = OpenOptions::new()
+        .read(true)
+        .open(filename)
+        .expect(&format!("unable to open {}.", filename));
+    load_r1cs_from_bin(BufReader::new(reader))
+}
+
+/// load r1cs from bin by a reader
+pub fn load_r1cs_from_bin<R: Read + Seek, E: ScalarEngine>(reader: R) -> (R1CS<E>, Vec<usize>) {
+    let file = crate::r1cs_file::from_reader::<R, E>(reader).expect("unable to read.");
+    let num_inputs = (1 + file.header.n_pub_in + file.header.n_pub_out) as usize;
+    let num_variables = file.header.n_wires as usize;
+    let num_aux = num_variables - num_inputs;
+    (
+        R1CS {
+            num_aux,
+            num_inputs,
+            num_variables,
+            num_outputs: file.header.n_pub_out as usize,
+            constraints: file.constraints,
+            custom_gates: file.custom_gates,
+            custom_gates_uses: file.custom_gates_uses,
+        },
+        file.wire_mapping.iter().map(|e| *e as usize).collect_vec(),
+    )
+}

algebraic/src/utils.rs

@@ -0,0 +1,13 @@
+pub use num_bigint::BigUint;
+use num_traits::Num;
+use std::fmt::Display;
+
+//export some more funcs
+pub use franklin_crypto::plonk::circuit::bigint::bigint::{biguint_to_fe, fe_to_biguint};
+
+/// convert a hex integer representation ("0x...") to decimal representation
+pub fn repr_to_big<T: Display>(r: T) -> String {
+    BigUint::from_str_radix(&format!("{}", r)[2..], 16)
+        .unwrap()
+        .to_str_radix(10)
+}