From 929566b6bb604bcf14bd0c3ada87eeb4f033eac0 Mon Sep 17 00:00:00 2001 From: Stephen <81497928+eigmax@users.noreply.github.com> Date: Sat, 27 Jul 2024 01:57:02 +0800 Subject: [PATCH] fix: fix table exhausted (#271) * fix: fix table exhausted * fix: fix table exhausted * fix: fix table exhausted * fix: pil syntax * fix: pil syntax * fix: sytax * docs: add notes --- recursion/Cargo.toml | 9 ++- .../src/compressor12/compressor12_pil.rs | 10 +-- recursion/src/pilcom.rs | 4 +- recursion/src/pilcom/export.rs | 78 +++++++++++++++---- recursion/src/pilcom/expression_counter.rs | 12 +-- starkjs/README.md | 2 + starkjs/connection/connection.pil | 4 +- starkjs/connection/connection_main.pil | 6 +- starkjs/fibonacci/fibonacci.pil | 6 +- starkjs/permutation/permutation.pil | 4 +- starkjs/permutation/permutation_main.pil | 6 +- starkjs/plookup/plookup.pil | 4 +- starkjs/plookup/plookup_main.pil | 6 +- starkjs/poseidon/poseidong.pil | 12 +-- starkjs/simple_vm/simple_vm.pil | 2 +- starkjs/simple_vm/simple_vm_main.pil | 4 +- zkvm/Cargo.toml | 2 +- zkvm/src/lib.rs | 10 ++- 18 files changed, 113 insertions(+), 68 deletions(-) diff --git a/recursion/Cargo.toml b/recursion/Cargo.toml index 08e4dc6f..9014f6c5 100644 --- a/recursion/Cargo.toml +++ b/recursion/Cargo.toml @@ -30,9 +30,10 @@ fields = { path = "../fields", default-features = false } starky = { path = "../starky", default-features = false } algebraic = { path = "../algebraic", default-features = false } -powdr = { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false } -powdr-ast = { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false } -powdr-pil-analyzer = { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false } +powdr = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false } +powdr-ast = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false } +powdr-pil-analyzer = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false } +powdr-parser-util = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false } [dev-dependencies] env_logger = "0.10" @@ -51,4 +52,4 @@ wasm-bindgen-test = "0.3" [features] default = ["wasmer/singlepass", "starky/default"] wasm = ["wasmer/js-default"] -avx512 = ["fields/avx512", "starky/avx512", "powdr/starky-avx512"] \ No newline at end of file +avx512 = ["fields/avx512", "starky/avx512", "powdr/starky-avx512"] diff --git a/recursion/src/compressor12/compressor12_pil.rs b/recursion/src/compressor12/compressor12_pil.rs index e176aa18..4bdc6c7f 100644 --- a/recursion/src/compressor12/compressor12_pil.rs +++ b/recursion/src/compressor12/compressor12_pil.rs @@ -51,9 +51,9 @@ pub fn render(n_bits: usize, n_publics: usize) -> String { let mut res = String::from(""); res.push_str(&format!( r#" -constant %N = 2**{n_bits}; +let N: int = 2**{n_bits}; -namespace Global(%N); +namespace Global(N); pol constant L1; "# )); @@ -68,7 +68,7 @@ namespace Global(%N); res.push_str( r#" -namespace Compressor(%N); +namespace Compressor(N); pol constant S[12]; pol constant C[12]; pol constant PARTIAL; @@ -356,8 +356,8 @@ namespace Compressor(%N); EVPOL4 * (a[8]' - acc4_2 ) = 0; // Connection equations - {a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11]} connect - {S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[8], S[9], S[10], S[11]}; + [a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11]] connect + [S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[8], S[9], S[10], S[11]]; "#, ); diff --git a/recursion/src/pilcom.rs b/recursion/src/pilcom.rs index 95f2572d..cef495de 100644 --- a/recursion/src/pilcom.rs +++ b/recursion/src/pilcom.rs @@ -11,11 +11,11 @@ use std::path::Path; pub fn compile_pil_from_str(pil_str: &str) -> PIL { let analyze = powdr_pil_analyzer::analyze_string::(pil_str); - export(Rc::new(analyze)) + export(&Rc::new(analyze)) } pub fn compile_pil_from_path(pil_path: &str) -> PIL { let analyze = powdr_pil_analyzer::analyze_file::(Path::new(pil_path)); - export(Rc::new(analyze)) + export(&Rc::new(analyze)) } #[cfg(test)] diff --git a/recursion/src/pilcom/export.rs b/recursion/src/pilcom/export.rs index 090c6976..2a68f46e 100644 --- a/recursion/src/pilcom/export.rs +++ b/recursion/src/pilcom/export.rs @@ -1,16 +1,16 @@ -//! porting it from powdr use powdr::number::FieldElement; -use std::cmp; use std::collections::HashMap; -use std::path::PathBuf; +use std::{cmp, path::PathBuf}; use powdr_ast::analyzed::{ - AlgebraicBinaryOperator, AlgebraicExpression as Expression, AlgebraicUnaryOperator, Analyzed, - IdentityKind, PolyID, PolynomialType, StatementIdentifier, SymbolKind, + AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression as Expression, + AlgebraicUnaryOperation, AlgebraicUnaryOperator, Analyzed, IdentityKind, PolyID, + PolynomialType, StatementIdentifier, SymbolKind, }; +use powdr_parser_util::SourceRef; use starky::types::{ ConnectionIdentity, Expression as StarkyExpr, PermutationIdentity, PlookupIdentity, - PolIdentity, Public, Reference, PIL, + PolIdentity, Reference, PIL, }; use super::expression_counter::compute_intermediate_expression_ids; @@ -34,10 +34,14 @@ struct Exporter<'a, T> { /// polynomials. intermediate_poly_expression_ids: HashMap, number_q: u64, + /// A cache to improve computing the line from a file offset. + /// Comparison is by raw pointer value because the data comes + /// from Arcs and we assume the actual data is not cloned. + line_starts: HashMap<*const u8, Vec>, } -pub fn export(analyzed: std::rc::Rc>) -> PIL { - let mut exporter = Exporter::new(&analyzed); +pub fn export(analyzed: &Analyzed) -> PIL { + let mut exporter = Exporter::new(analyzed); let mut publics = Vec::new(); let mut pol_identities = Vec::new(); let mut plookup_identities = Vec::new(); @@ -69,7 +73,7 @@ pub fn export(analyzed: std::rc::Rc>) -> PIL { false, ); let id = publics.len(); - publics.push(Public { + publics.push(starky::types::Public { polType: polynomial_reference_type_to_type(&expr.op).to_string(), polId: expr.id.unwrap(), idx: pub_def.index as usize, @@ -79,9 +83,10 @@ pub fn export(analyzed: std::rc::Rc>) -> PIL { } StatementIdentifier::Identity(id) => { let identity = &analyzed.identities[*id]; + // PILCOM strips the path from filenames, we do the same here for compatibility let file_name = identity .source - .file + .file_name .as_deref() .and_then(|s| { PathBuf::from(s) @@ -90,7 +95,7 @@ pub fn export(analyzed: std::rc::Rc>) -> PIL { .map(String::from) }) .unwrap_or_default(); - let line = identity.source.line; + let line = exporter.line_of_source_ref(&identity.source); let selector_degree = if identity.kind == IdentityKind::Polynomial { 2 } else { @@ -160,7 +165,6 @@ fn symbol_kind_to_json_string(k: SymbolKind) -> &'static str { match k { SymbolKind::Poly(poly_type) => polynomial_type_to_json_string(poly_type), SymbolKind::Other() => panic!("Cannot translate \"other\" symbol to json."), - SymbolKind::Constant() => unreachable!(), } } @@ -185,6 +189,18 @@ fn polynomial_reference_type_to_type(t: &str) -> &'static str { } } +/// Makes names compatible with estark, which sometimes require that +/// there is exactly one `.` in the name. +fn fixup_name(name: &str) -> String { + if name.contains('.') { + name.to_string() + } else if let Some(last) = name.rfind("::") { + format!("{}.{}", &name[..last], &name[last + 1..]) + } else { + panic!("Witness or intermediate column is not inside a namespace: {name}"); + } +} + impl<'a, T: FieldElement> Exporter<'a, T> { fn new(analyzed: &'a Analyzed) -> Self { Self { @@ -192,6 +208,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { expressions: vec![], intermediate_poly_expression_ids: compute_intermediate_expression_ids(analyzed), number_q: 0, + line_starts: Default::default(), } } @@ -205,7 +222,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { panic!("Should be in intermediates") } SymbolKind::Poly(_) => Some(symbol.id), - SymbolKind::Other() | SymbolKind::Constant() => None, + SymbolKind::Other() => None, }?; let out = Reference { @@ -217,7 +234,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { elementType: None, len: symbol.length.map(|l| l as usize), }; - Some((name.clone(), out)) + Some((fixup_name(name), out)) }) .chain( self.analyzed @@ -236,7 +253,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { elementType: None, len: symbol.length.map(|l| l as usize), }; - (name.clone(), out) + (fixup_name(name), out) }), ) .collect::>() @@ -304,7 +321,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { ..DEFAULT_EXPR }, ), - Expression::BinaryOperation(left, op, right) => { + Expression::BinaryOperation(AlgebraicBinaryOperation { left, op, right }) => { let (deg_left, left) = self.expression_to_json(left); let (deg_right, right) = self.expression_to_json(right); let (op, degree) = match op { @@ -330,7 +347,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> { }, ) } - Expression::UnaryOperation(op, value) => { + Expression::UnaryOperation(AlgebraicUnaryOperation { op, expr: value }) => { let (deg, value) = self.expression_to_json(value); match op { AlgebraicUnaryOperator::Minus => ( @@ -366,4 +383,31 @@ impl<'a, T: FieldElement> Exporter<'a, T> { }; (1, poly) } + + fn line_of_source_ref(&mut self, source: &SourceRef) -> usize { + let Some(file_contents) = source.file_contents.as_ref() else { + return 0; + }; + let line_starts = self + .line_starts + .entry(file_contents.as_ptr()) + .or_insert_with(|| compute_line_starts(file_contents)); + offset_to_line_col(source.start, line_starts).0 + } +} + +fn compute_line_starts(source: &str) -> Vec { + std::iter::once(0) + .chain(source.match_indices('\n').map(|(i, _)| i + 1)) + .collect::>() +} + +/// Returns a tuple `(line, col)` given the file offset of line starts. +/// `line` is 1 based and `col` is 0 based. +fn offset_to_line_col(offset: usize, line_starts: &[usize]) -> (usize, usize) { + let line = match line_starts.binary_search(&offset) { + Ok(line) => line + 1, + Err(next_line) => next_line, + }; + (line, offset - line_starts[line - 1]) } diff --git a/recursion/src/pilcom/expression_counter.rs b/recursion/src/pilcom/expression_counter.rs index 9ed06d53..35313bd6 100644 --- a/recursion/src/pilcom/expression_counter.rs +++ b/recursion/src/pilcom/expression_counter.rs @@ -1,12 +1,8 @@ -//! porting it from powdr use std::collections::HashMap; -use powdr_ast::{ - analyzed::{ - Analyzed, Identity, PolynomialType, PublicDeclaration, StatementIdentifier, Symbol, - SymbolKind, - }, - parsed::SelectedExpressions, +use powdr_ast::analyzed::{ + Analyzed, Identity, PolynomialType, PublicDeclaration, SelectedExpressions, + StatementIdentifier, Symbol, SymbolKind, }; /// Computes expression IDs for each intermediate polynomial. @@ -43,7 +39,7 @@ trait ExpressionCounter { fn expression_count(&self) -> usize; } -impl ExpressionCounter for Identity { +impl ExpressionCounter for Identity> { fn expression_count(&self) -> usize { self.left.expression_count() + self.right.expression_count() } diff --git a/starkjs/README.md b/starkjs/README.md index a0d59234..41a13530 100644 --- a/starkjs/README.md +++ b/starkjs/README.md @@ -2,6 +2,8 @@ PIL compiler and Circom transpiler. The stark prover is [starky](../starky). +Note: since `powdr` syntax has been imcompatible with original `pil`, the scripts is no longer supported, the breaking changes happens at issue #271. + ## Run Example ### Arithmetization: Constraint Polynomial diff --git a/starkjs/connection/connection.pil b/starkjs/connection/connection.pil index bd4a70e6..e4a6c6a3 100644 --- a/starkjs/connection/connection.pil +++ b/starkjs/connection/connection.pil @@ -1,9 +1,9 @@ -namespace Connection(%N); +namespace Connection(N); pol constant S1, S2, S3; pol commit a,b,c; - public out = c(%N-1); + public out = c(N-1); {a, b, c} connect {S1, S2, S3}; diff --git a/starkjs/connection/connection_main.pil b/starkjs/connection/connection_main.pil index 9f11528f..21615a78 100644 --- a/starkjs/connection/connection_main.pil +++ b/starkjs/connection/connection_main.pil @@ -1,6 +1,6 @@ -constant %N = 2**10; +let N: int = 2**10; -namespace Global(%N); +namespace Global(N); pol constant L1; -include "connection.pil"; \ No newline at end of file +include "connection.pil"; diff --git a/starkjs/fibonacci/fibonacci.pil b/starkjs/fibonacci/fibonacci.pil index 8e48d701..104facf3 100644 --- a/starkjs/fibonacci/fibonacci.pil +++ b/starkjs/fibonacci/fibonacci.pil @@ -1,5 +1,5 @@ -constant %N = 2**10; -namespace Fibonacci(%N); +let N: int = 2**10; +namespace Fibonacci(N); pol constant L1, LLAST; pol commit l1,l2; @@ -7,7 +7,7 @@ namespace Fibonacci(%N); public in1 = l2(0); public in2 = l1(0); - public out = l1(%N-1); + public out = l1(N-1); (l2' - l1)*(1-LLAST) = 0; diff --git a/starkjs/permutation/permutation.pil b/starkjs/permutation/permutation.pil index 0028f7f9..c8725054 100644 --- a/starkjs/permutation/permutation.pil +++ b/starkjs/permutation/permutation.pil @@ -1,9 +1,9 @@ -namespace Permutation(%N); +namespace Permutation(N); pol commit a, b; pol commit c, d; pol commit selC, selD; - public out = d(%N-1); + public out = d(N-1); a is b; selC {c} is selD {d}; diff --git a/starkjs/permutation/permutation_main.pil b/starkjs/permutation/permutation_main.pil index 3a52cbfd..660f9322 100644 --- a/starkjs/permutation/permutation_main.pil +++ b/starkjs/permutation/permutation_main.pil @@ -1,6 +1,6 @@ -constant %N = 2**10; +let N: int = 2**10; -namespace Global(%N); +namespace Global(N); pol constant L1; -include "permutation.pil" \ No newline at end of file +include "permutation.pil" diff --git a/starkjs/plookup/plookup.pil b/starkjs/plookup/plookup.pil index 98cd02a3..2e161b88 100644 --- a/starkjs/plookup/plookup.pil +++ b/starkjs/plookup/plookup.pil @@ -1,11 +1,11 @@ -namespace Plookup(%N); +namespace Plookup(N); pol commit sel, a, b; pol constant SEL, A, B; pol commit cc; - public out = cc(%N-1); + public out = cc(N-1); sel {a, b', a*b'} in SEL { A, B, cc}; diff --git a/starkjs/plookup/plookup_main.pil b/starkjs/plookup/plookup_main.pil index 90ea4fe9..4e365c9b 100644 --- a/starkjs/plookup/plookup_main.pil +++ b/starkjs/plookup/plookup_main.pil @@ -1,7 +1,7 @@ -constant %N = 2**10; +let N: int = 2**10; -namespace Global(%N); +namespace Global(N); pol constant L1; -include "plookup.pil"; \ No newline at end of file +include "plookup.pil"; diff --git a/starkjs/poseidon/poseidong.pil b/starkjs/poseidon/poseidong.pil index 8e99169c..17468d5a 100644 --- a/starkjs/poseidon/poseidong.pil +++ b/starkjs/poseidon/poseidong.pil @@ -1,6 +1,6 @@ // come from this version: https://github.com/0xPolygonHermez/zkevm-proverjs/blob/v0.6.0.0/pil/poseidong.pil -constant %N = 2**10; -namespace PoseidonG(%N); +let N: int = 2**10; +namespace PoseidonG(N); pol constant LAST; // 0, 0, 0, ...0, 1, 0, ...., 0, 1, 0, .... pol constant LATCH; // 1, 0, 0, 0, ..., 0, 1, 0, 0, @@ -21,10 +21,10 @@ namespace PoseidonG(%N); public pin5 = in5(0); public pin6 = in6(0); public pin7 = in7(0); - public out0 = hash0(%N-1); - public out1 = hash1(%N-1); - public out2 = hash2(%N-1); - public out3 = hash3(%N-1); + public out0 = hash0(N-1); + public out1 = hash1(N-1); + public out2 = hash2(N-1); + public out3 = hash3(N-1); LINPUT * (in0 - :pin0) = 0; LINPUT * (in1 - :pin1) = 0; LINPUT * (in2 - :pin2) = 0; diff --git a/starkjs/simple_vm/simple_vm.pil b/starkjs/simple_vm/simple_vm.pil index 013cda21..312a4976 100644 --- a/starkjs/simple_vm/simple_vm.pil +++ b/starkjs/simple_vm/simple_vm.pil @@ -1,6 +1,6 @@ -namespace SimpleVM(%N); +namespace SimpleVM(N); pol constant ROM; pol commit inFree, sel_b, sel_c, sel_d, sel_ins, sel_A, sel_B, set_b, set_c, set_d, set_A, set_B, const, jmpz, inv_op, addr, free, position, PC; b'=set_b*(free-b) + b diff --git a/starkjs/simple_vm/simple_vm_main.pil b/starkjs/simple_vm/simple_vm_main.pil index d7a0d2d4..bd5a7913 100644 --- a/starkjs/simple_vm/simple_vm_main.pil +++ b/starkjs/simple_vm/simple_vm_main.pil @@ -1,6 +1,6 @@ -constant %N = 2**10; +let N: int = 2**10; -namespace Global(%N); +namespace Global(N); pol constant L1, LLAST; include "simple_vm.pil"; diff --git a/zkvm/Cargo.toml b/zkvm/Cargo.toml index 068e8b3f..495252ae 100644 --- a/zkvm/Cargo.toml +++ b/zkvm/Cargo.toml @@ -10,7 +10,7 @@ itertools = "0.12.0" # serialization log = "0.4.0" -powdr = { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false } +powdr = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false } starky = { path = "../starky" } recursion = { path = "../recursion" } diff --git a/zkvm/src/lib.rs b/zkvm/src/lib.rs index d71e9f9f..e0928fca 100644 --- a/zkvm/src/lib.rs +++ b/zkvm/src/lib.rs @@ -78,7 +78,7 @@ fn generate_verifier( agg_stage: false, }; if !setup.starkinfo.qs.is_empty() { - let pil_json = pil_export::(pil); + let pil_json = pil_export::(&pil); let str_ver = pil2circom::pil2circom( &pil_json, &setup.const_root, @@ -102,6 +102,7 @@ pub fn zkvm_execute_and_prove(task: &str, suite_json: String, output_path: &str) Path::new(output_path), force_overwrite, &Runtime::base().with_poseidon(), + false, with_bootloader, ) .ok_or_else(|| vec!["could not compile rust".to_string()]) @@ -140,7 +141,7 @@ pub fn zkvm_execute_and_prove(task: &str, suite_json: String, output_path: &str) log::debug!("Running powdr-riscv executor in trace mode for continuations..."); let start = Instant::now(); - let bootloader_inputs = rust_continuations_dry_run(&mut pipeline); + let bootloader_inputs = rust_continuations_dry_run(&mut pipeline, None); let duration = start.elapsed(); log::debug!("Trace executor took: {:?}", duration); @@ -169,6 +170,7 @@ pub fn zkvm_generate_chunks( Path::new(output_path), force_overwrite, &Runtime::base().with_poseidon(), + false, with_bootloader, ) .ok_or_else(|| vec!["could not compile rust".to_string()]) @@ -196,7 +198,7 @@ pub fn zkvm_generate_chunks( log::debug!("Running powdr-riscv executor in trace mode for continuations..."); let start = Instant::now(); - let bootloader_inputs = rust_continuations_dry_run(&mut pipeline); + let bootloader_inputs = rust_continuations_dry_run(&mut pipeline, None); let duration = start.elapsed(); log::debug!( @@ -281,7 +283,7 @@ where //let pipeline = pipeline.with_name(name); // now we should do - let parent_path = pipeline.output_dir().unwrap(); + let parent_path = pipeline.output_dir().as_ref().unwrap(); let chunk_dir = parent_path.join(name); //remove_dir_all(&chunk_dir).unwrap(); create_dir_all(&chunk_dir).unwrap();