fix: fix table exhausted (#271)

* fix: fix table exhausted

* fix: fix table exhausted

* fix: fix table exhausted

* fix: pil syntax

* fix: pil syntax

* fix: sytax

* docs: add notes

recursion/Cargo.toml

@@ -30,9 +30,10 @@ fields = { path = "../fields", default-features = false }
 starky = { path = "../starky", default-features = false }
 algebraic = { path = "../algebraic", default-features = false }
 
-powdr =              { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false }
-powdr-ast =          { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false }
-powdr-pil-analyzer = { git = "https://github.com/0xEigenLabs/powdr", branch = "binary-mux2", default-features = false }
+powdr =              { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false }
+powdr-ast =          { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false }
+powdr-pil-analyzer = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false }
+powdr-parser-util = { git = "https://github.com/0xEigenLabs/powdr", branch = "eigen/v1", default-features = false }
 
 [dev-dependencies]
 env_logger = "0.10"
@@ -51,4 +52,4 @@ wasm-bindgen-test = "0.3"
 [features]
 default = ["wasmer/singlepass", "starky/default"]
 wasm = ["wasmer/js-default"]
-avx512 = ["fields/avx512", "starky/avx512", "powdr/starky-avx512"]
+avx512 = ["fields/avx512", "starky/avx512", "powdr/starky-avx512"]

recursion/src/compressor12/compressor12_pil.rs

@@ -51,9 +51,9 @@ pub fn render(n_bits: usize, n_publics: usize) -> String {
     let mut res = String::from("");
     res.push_str(&format!(
         r#"
-constant %N = 2**{n_bits};
+let N: int = 2**{n_bits};
 
-namespace Global(%N);
+namespace Global(N);
     pol constant L1;
     "#
     ));
@@ -68,7 +68,7 @@ namespace Global(%N);
 
     res.push_str(
         r#"
-namespace Compressor(%N);
+namespace Compressor(N);
     pol constant S[12];
     pol constant C[12];
     pol constant PARTIAL;
@@ -356,8 +356,8 @@ namespace Compressor(%N);
     EVPOL4 * (a[8]' - acc4_2 ) = 0;
 
     // Connection equations
-    {a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11]} connect
-        {S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[8], S[9], S[10], S[11]};
+    [a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11]] connect
+        [S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[8], S[9], S[10], S[11]];
 
     "#,
     );

recursion/src/pilcom.rs

@@ -11,11 +11,11 @@ use std::path::Path;
 
 pub fn compile_pil_from_str(pil_str: &str) -> PIL {
     let analyze = powdr_pil_analyzer::analyze_string::<GoldilocksField>(pil_str);
-    export(Rc::new(analyze))
+    export(&Rc::new(analyze))
 }
 pub fn compile_pil_from_path(pil_path: &str) -> PIL {
     let analyze = powdr_pil_analyzer::analyze_file::<GoldilocksField>(Path::new(pil_path));
-    export(Rc::new(analyze))
+    export(&Rc::new(analyze))
 }
 
 #[cfg(test)]

recursion/src/pilcom/export.rs

@@ -1,16 +1,16 @@
-//! porting it from powdr
 use powdr::number::FieldElement;
-use std::cmp;
 use std::collections::HashMap;
-use std::path::PathBuf;
+use std::{cmp, path::PathBuf};
 
 use powdr_ast::analyzed::{
-    AlgebraicBinaryOperator, AlgebraicExpression as Expression, AlgebraicUnaryOperator, Analyzed,
-    IdentityKind, PolyID, PolynomialType, StatementIdentifier, SymbolKind,
+    AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression as Expression,
+    AlgebraicUnaryOperation, AlgebraicUnaryOperator, Analyzed, IdentityKind, PolyID,
+    PolynomialType, StatementIdentifier, SymbolKind,
 };
+use powdr_parser_util::SourceRef;
 use starky::types::{
     ConnectionIdentity, Expression as StarkyExpr, PermutationIdentity, PlookupIdentity,
-    PolIdentity, Public, Reference, PIL,
+    PolIdentity, Reference, PIL,
 };
 
 use super::expression_counter::compute_intermediate_expression_ids;
@@ -34,10 +34,14 @@ struct Exporter<'a, T> {
     /// polynomials.
     intermediate_poly_expression_ids: HashMap<u64, u64>,
     number_q: u64,
+    /// A cache to improve computing the line from a file offset.
+    /// Comparison is by raw pointer value because the data comes
+    /// from Arcs and we assume the actual data is not cloned.
+    line_starts: HashMap<*const u8, Vec<usize>>,
 }
 
-pub fn export<T: FieldElement>(analyzed: std::rc::Rc<Analyzed<T>>) -> PIL {
-    let mut exporter = Exporter::new(&analyzed);
+pub fn export<T: FieldElement>(analyzed: &Analyzed<T>) -> PIL {
+    let mut exporter = Exporter::new(analyzed);
     let mut publics = Vec::new();
     let mut pol_identities = Vec::new();
     let mut plookup_identities = Vec::new();
@@ -69,7 +73,7 @@ pub fn export<T: FieldElement>(analyzed: std::rc::Rc<Analyzed<T>>) -> PIL {
                     false,
                 );
                 let id = publics.len();
-                publics.push(Public {
+                publics.push(starky::types::Public {
                     polType: polynomial_reference_type_to_type(&expr.op).to_string(),
                     polId: expr.id.unwrap(),
                     idx: pub_def.index as usize,
@@ -79,9 +83,10 @@ pub fn export<T: FieldElement>(analyzed: std::rc::Rc<Analyzed<T>>) -> PIL {
             }
             StatementIdentifier::Identity(id) => {
                 let identity = &analyzed.identities[*id];
+                // PILCOM strips the path from filenames, we do the same here for compatibility
                 let file_name = identity
                     .source
-                    .file
+                    .file_name
                     .as_deref()
                     .and_then(|s| {
                         PathBuf::from(s)
@@ -90,7 +95,7 @@ pub fn export<T: FieldElement>(analyzed: std::rc::Rc<Analyzed<T>>) -> PIL {
                             .map(String::from)
                     })
                     .unwrap_or_default();
-                let line = identity.source.line;
+                let line = exporter.line_of_source_ref(&identity.source);
                 let selector_degree = if identity.kind == IdentityKind::Polynomial {
                     2
                 } else {
@@ -160,7 +165,6 @@ fn symbol_kind_to_json_string(k: SymbolKind) -> &'static str {
     match k {
         SymbolKind::Poly(poly_type) => polynomial_type_to_json_string(poly_type),
         SymbolKind::Other() => panic!("Cannot translate \"other\" symbol to json."),
-        SymbolKind::Constant() => unreachable!(),
     }
 }
 
@@ -185,13 +189,26 @@ fn polynomial_reference_type_to_type(t: &str) -> &'static str {
     }
 }
 
+/// Makes names compatible with estark, which sometimes require that
+/// there is exactly one `.` in the name.
+fn fixup_name(name: &str) -> String {
+    if name.contains('.') {
+        name.to_string()
+    } else if let Some(last) = name.rfind("::") {
+        format!("{}.{}", &name[..last], &name[last + 1..])
+    } else {
+        panic!("Witness or intermediate column is not inside a namespace: {name}");
+    }
+}
+
 impl<'a, T: FieldElement> Exporter<'a, T> {
     fn new(analyzed: &'a Analyzed<T>) -> Self {
         Self {
             analyzed,
             expressions: vec![],
             intermediate_poly_expression_ids: compute_intermediate_expression_ids(analyzed),
             number_q: 0,
+            line_starts: Default::default(),
         }
     }
 
@@ -205,7 +222,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> {
                         panic!("Should be in intermediates")
                     }
                     SymbolKind::Poly(_) => Some(symbol.id),
-                    SymbolKind::Other() | SymbolKind::Constant() => None,
+                    SymbolKind::Other() => None,
                 }?;
 
                 let out = Reference {
@@ -217,7 +234,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> {
                     elementType: None,
                     len: symbol.length.map(|l| l as usize),
                 };
-                Some((name.clone(), out))
+                Some((fixup_name(name), out))
             })
             .chain(
                 self.analyzed
@@ -236,7 +253,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> {
                             elementType: None,
                             len: symbol.length.map(|l| l as usize),
                         };
-                        (name.clone(), out)
+                        (fixup_name(name), out)
                     }),
             )
             .collect::<HashMap<String, Reference>>()
@@ -304,7 +321,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> {
                     ..DEFAULT_EXPR
                 },
             ),
-            Expression::BinaryOperation(left, op, right) => {
+            Expression::BinaryOperation(AlgebraicBinaryOperation { left, op, right }) => {
                 let (deg_left, left) = self.expression_to_json(left);
                 let (deg_right, right) = self.expression_to_json(right);
                 let (op, degree) = match op {
@@ -330,7 +347,7 @@ impl<'a, T: FieldElement> Exporter<'a, T> {
                     },
                 )
             }
-            Expression::UnaryOperation(op, value) => {
+            Expression::UnaryOperation(AlgebraicUnaryOperation { op, expr: value }) => {
                 let (deg, value) = self.expression_to_json(value);
                 match op {
                     AlgebraicUnaryOperator::Minus => (
@@ -366,4 +383,31 @@ impl<'a, T: FieldElement> Exporter<'a, T> {
         };
         (1, poly)
     }
+
+    fn line_of_source_ref(&mut self, source: &SourceRef) -> usize {
+        let Some(file_contents) = source.file_contents.as_ref() else {
+            return 0;
+        };
+        let line_starts = self
+            .line_starts
+            .entry(file_contents.as_ptr())
+            .or_insert_with(|| compute_line_starts(file_contents));
+        offset_to_line_col(source.start, line_starts).0
+    }
+}
+
+fn compute_line_starts(source: &str) -> Vec<usize> {
+    std::iter::once(0)
+        .chain(source.match_indices('\n').map(|(i, _)| i + 1))
+        .collect::<Vec<_>>()
+}
+
+/// Returns a tuple `(line, col)` given the file offset of line starts.
+/// `line` is 1 based and `col` is 0 based.
+fn offset_to_line_col(offset: usize, line_starts: &[usize]) -> (usize, usize) {
+    let line = match line_starts.binary_search(&offset) {
+        Ok(line) => line + 1,
+        Err(next_line) => next_line,
+    };
+    (line, offset - line_starts[line - 1])
 }

recursion/src/pilcom/expression_counter.rs

@@ -1,12 +1,8 @@
-//! porting it from powdr
 use std::collections::HashMap;
 
-use powdr_ast::{
-    analyzed::{
-        Analyzed, Identity, PolynomialType, PublicDeclaration, StatementIdentifier, Symbol,
-        SymbolKind,
-    },
-    parsed::SelectedExpressions,
+use powdr_ast::analyzed::{
+    Analyzed, Identity, PolynomialType, PublicDeclaration, SelectedExpressions,
+    StatementIdentifier, Symbol, SymbolKind,
 };
 
 /// Computes expression IDs for each intermediate polynomial.
@@ -43,7 +39,7 @@ trait ExpressionCounter {
     fn expression_count(&self) -> usize;
 }
 
-impl<Expr> ExpressionCounter for Identity<Expr> {
+impl<Expr> ExpressionCounter for Identity<SelectedExpressions<Expr>> {
     fn expression_count(&self) -> usize {
         self.left.expression_count() + self.right.expression_count()
     }

starkjs/README.md

@@ -2,6 +2,8 @@
 
 PIL compiler and Circom transpiler. The stark prover is [starky](../starky).
 
+Note: since `powdr` syntax has been imcompatible with original `pil`, the scripts is no longer supported, the breaking changes happens at issue #271.
+
 ## Run Example
 ### Arithmetization: Constraint Polynomial
 

starkjs/connection/connection.pil

@@ -1,9 +1,9 @@
 
 
-namespace Connection(%N);
+namespace Connection(N);
     pol constant S1, S2, S3;
     pol commit a,b,c;
-    public out = c(%N-1);
+    public out = c(N-1);
 
     {a, b, c} connect {S1, S2, S3};
 

starkjs/connection/connection_main.pil

@@ -1,6 +1,6 @@
-constant %N = 2**10;
+let N: int = 2**10;
 
-namespace Global(%N);
+namespace Global(N);
     pol constant L1;
 
-include "connection.pil";
+include "connection.pil";

starkjs/fibonacci/fibonacci.pil

@@ -1,13 +1,13 @@
-constant %N = 2**10;
-namespace Fibonacci(%N);
+let N: int = 2**10;
+namespace Fibonacci(N);
     pol constant L1, LLAST;
     pol commit l1,l2;
 
     //pol l2c = l2;
 
     public in1 = l2(0);
     public in2 = l1(0);
-    public out = l1(%N-1);
+    public out = l1(N-1);
 
     (l2' - l1)*(1-LLAST) = 0;
 

starkjs/permutation/permutation.pil

@@ -1,9 +1,9 @@
-namespace Permutation(%N);
+namespace Permutation(N);
 
     pol commit a, b;
     pol commit c, d;
     pol commit selC, selD;
-    public out = d(%N-1);
+    public out = d(N-1);
 
     a is b;
     selC {c} is selD {d};

starkjs/permutation/permutation_main.pil

@@ -1,6 +1,6 @@
-constant %N = 2**10;
+let N: int = 2**10;
 
-namespace Global(%N);
+namespace Global(N);
     pol constant L1;
 
-include "permutation.pil"
+include "permutation.pil"

starkjs/plookup/plookup.pil

@@ -1,11 +1,11 @@
-namespace Plookup(%N);
+namespace Plookup(N);
 
 
     pol commit sel, a, b;
 
     pol constant SEL, A, B;
     pol commit cc;
-    public out = cc(%N-1);
+    public out = cc(N-1);
 
     sel {a, b', a*b'} in SEL { A, B, cc};
 

starkjs/plookup/plookup_main.pil

@@ -1,7 +1,7 @@
 
-constant %N = 2**10;
+let N: int = 2**10;
 
-namespace Global(%N);
+namespace Global(N);
     pol constant L1;
 
-include "plookup.pil";
+include "plookup.pil";

starkjs/poseidon/poseidong.pil

@@ -1,6 +1,6 @@
 // come from this version: https://github.com/0xPolygonHermez/zkevm-proverjs/blob/v0.6.0.0/pil/poseidong.pil
-constant %N = 2**10;
-namespace PoseidonG(%N);
+let N: int = 2**10;
+namespace PoseidonG(N);
 
     pol constant LAST;   // 0, 0, 0, ...0, 1,      0, ...., 0, 1, 0, ....
     pol constant LATCH;  // 1, 0, 0, 0, ..., 0,    1, 0, 0,
@@ -21,10 +21,10 @@ namespace PoseidonG(%N);
     public pin5 = in5(0);
     public pin6 = in6(0);
     public pin7 = in7(0);
-    public out0 = hash0(%N-1);
-    public out1 = hash1(%N-1);
-    public out2 = hash2(%N-1);
-    public out3 = hash3(%N-1);
+    public out0 = hash0(N-1);
+    public out1 = hash1(N-1);
+    public out2 = hash2(N-1);
+    public out3 = hash3(N-1);
     LINPUT * (in0 - :pin0) = 0;
     LINPUT * (in1 - :pin1) = 0;
     LINPUT * (in2 - :pin2) = 0;