Merge branch 'feature/improved-merkle-masks-for-staged-ledger' into f…

…eature/merkle-mask-preloading

buildkite/scripts/connect-to-berkeley.sh

@@ -43,10 +43,10 @@ mina daemon \
   --libp2p-keypair "/root/libp2p-keys/key" \
 & # -background
 
-# Attempt to connect to the GraphQL client every 10s for up to 4 minutes
+# Attempt to connect to the GraphQL client every 30s for up to 12 minutes
 num_status_retries=24
 for ((i=1;i<=$num_status_retries;i++)); do
-  sleep 10s
+  sleep 30s
   set +e
   mina client status
   status_exit_code=$?

buildkite/scripts/connect-to-mainnet-on-compatible.sh

@@ -37,10 +37,10 @@ mina daemon \
 & # -background
 
 
-# Attempt to connect to the GraphQL client every 10s for up to 4 minutes
+# Attempt to connect to the GraphQL client every 30s for up to 12 minutes
 num_status_retries=24
 for ((i=1;i<=$num_status_retries;i++)); do
-  sleep 10s
+  sleep 30s
   set +e
   mina client status
   status_exit_code=$?

buildkite/scripts/run-snark-transaction-profiler.sh

@@ -8,13 +8,7 @@ export DEBIAN_FRONTEND=noninteractive
 apt-get update
 apt-get install -y git apt-transport-https ca-certificates tzdata curl python3
 
-case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
-  rampup|berkeley|release/2.0.0|develop)
-    TESTNET_NAME="berkeley"
-  ;;
-  *)
-    TESTNET_NAME="mainnet"
-esac
+TESTNET_NAME="berkeley"
 
 git config --global --add safe.directory /workdir
 

buildkite/scripts/version-linter.sh

@@ -19,7 +19,7 @@ git config --global --add safe.directory /workdir
 source buildkite/scripts/handle-fork.sh
 source buildkite/scripts/export-git-env-vars.sh
 
-pip3 install sexpdata
+pip3 install sexpdata==1.0.0
 
 base_branch=${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 pr_branch=origin/${BUILDKITE_BRANCH}

buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall

@@ -1,4 +1,6 @@
 let ContainerImages = ../../Constants/ContainerImages.dhall
+let B = ../../External/Buildkite.dhall
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 
 let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
@@ -32,6 +34,7 @@ Pipeline.build
           ],
           label = "delegation backend unit-tests",
           key = "delegation-backend-unit-tests",
+          soft_fail = Some (B/SoftFail.Boolean True),
           target = Size.Small,
           docker = None Docker.Type
         }

helm/archive-node/templates/db-bootstrap.yaml

@@ -9,7 +9,7 @@ spec:
       containers:
       {{- if .Values.archive.initFromDump }}
       - name: import-dump
-        image: gcr.io/o1labs-192920/postgresql-curl:latest
+        image: postgres:15-alpine
         env:
         - name: PGPASSWORD
           valueFrom:
@@ -19,6 +19,7 @@ spec:
         command: ["bash", "-c"]
         args:
         - 'sleep 30
+        && apk add curl
         && cd /tmp
         && curl https://storage.googleapis.com/mina-archive-dumps/{{ .Values.testnetName }}-archive-dump-$(date -Idate)_0000.sql.tar.gz -o {{ .Values.testnetName }}-archive-dump.tar.gz
         && tar -xvf {{ .Values.testnetName }}-archive-dump.tar.gz
@@ -37,7 +38,7 @@ spec:
         -c "ALTER DATABASE {{ .Values.postgresql.auth.database }} SET DEFAULT_TRANSACTION_ISOLATION TO SERIALIZABLE;"'
       {{- else }}
       - name: import-schema
-        image: gcr.io/o1labs-192920/postgresql-curl:latest
+        image: postgres:15-alpine
         env:
         - name: PGPASSWORD
           valueFrom:
@@ -47,6 +48,7 @@ spec:
         command: ["bash", "-c"]
         args:
         - 'sleep 30
+        && apk add curl
         && cd /tmp
         && {{ range .Values.archive.remoteSchemaAuxFiles }} curl -O {{.}} && {{ end }}
         psql

src/lib/crypto/kimchi_backend/common/dune

@@ -25,6 +25,7 @@
   base.caml
   ppx_inline_test.config
   bignum.bigint
+  zarith
   base.base_internalhash_types
   ;; local libraries
   tuple_lib

src/lib/crypto/kimchi_backend/common/field.ml

@@ -37,6 +37,8 @@ module type Input_intf = sig
 
   val is_square : t -> bool
 
+  val compare : t -> t -> int
+
   val equal : t -> t -> bool
 
   val print : t -> unit
@@ -177,24 +179,22 @@ module Make (F : Input_intf) :
             let of_sexpable = of_bigint
           end)
 
-      let to_bignum_bigint n =
-        let rec go i two_to_the_i acc =
-          if Int.equal i size_in_bits then acc
+      let to_bignum_bigint =
+        let zero = of_int 0 in
+        let one = of_int 1 in
+        fun n ->
+          if equal n zero then Bignum_bigint.zero
+          else if equal n one then Bignum_bigint.one
           else
-            let acc' =
-              if Bigint.test_bit n i then Bignum_bigint.(acc + two_to_the_i)
-              else acc
-            in
-            go (i + 1) Bignum_bigint.(two_to_the_i + two_to_the_i) acc'
-        in
-        go 0 Bignum_bigint.one Bignum_bigint.zero
+            Bytes.unsafe_to_string
+              ~no_mutation_while_string_reachable:(to_bytes n)
+            |> Z.of_bits |> Bignum_bigint.of_zarith_bigint
 
-      let hash_fold_t s x =
-        Bignum_bigint.hash_fold_t s (to_bignum_bigint (to_bigint x))
+      let hash_fold_t s x = Bignum_bigint.hash_fold_t s (to_bignum_bigint x)
 
       let hash = Hash.of_fold hash_fold_t
 
-      let compare t1 t2 = Bigint.compare (to_bigint t1) (to_bigint t2)
+      let compare = compare
 
       let equal = equal
 

src/lib/crypto/kimchi_backend/kimchi_backend.mli

@@ -11,8 +11,6 @@ module Kimchi_backend_common : sig
 
       val sexp_of_t : t -> Sexplib0.Sexp.t
 
-      val compare : t -> t -> int
-
       val bin_size_t : t Bin_prot.Size.sizer
 
       val bin_write_t : t Bin_prot.Write.writer
@@ -56,6 +54,8 @@ module Kimchi_backend_common : sig
 
       val is_square : t -> bool
 
+      val compare : t -> t -> int
+
       val equal : t -> t -> bool
 
       val print : t -> unit

src/lib/crypto/kimchi_bindings/stubs/src/arkworks/pasta_fp.rs

@@ -31,7 +31,7 @@ impl CamlFp {
     unsafe extern "C" fn ocaml_compare(x: ocaml::Raw, y: ocaml::Raw) -> i32 {
         let x = x.as_pointer::<Self>();
         let y = y.as_pointer::<Self>();
-        match x.as_ref().0.cmp(&y.as_ref().0) {
+        match x.as_ref().0.into_repr().cmp(&y.as_ref().0.into_repr()) {
             core::cmp::Ordering::Less => -1,
             core::cmp::Ordering::Equal => 0,
             core::cmp::Ordering::Greater => 1,
@@ -240,7 +240,7 @@ pub fn caml_pasta_fp_mut_square(mut x: ocaml::Pointer<CamlFp>) {
 #[ocaml_gen::func]
 #[ocaml::func]
 pub fn caml_pasta_fp_compare(x: ocaml::Pointer<CamlFp>, y: ocaml::Pointer<CamlFp>) -> ocaml::Int {
-    match x.as_ref().0.cmp(&y.as_ref().0) {
+    match x.as_ref().0.into_repr().cmp(&y.as_ref().0.into_repr()) {
         Less => -1,
         Equal => 0,
         Greater => 1,

src/lib/crypto/kimchi_bindings/stubs/src/arkworks/pasta_fq.rs

@@ -36,7 +36,7 @@ impl CamlFq {
     unsafe extern "C" fn ocaml_compare(x: ocaml::Raw, y: ocaml::Raw) -> i32 {
         let x = x.as_pointer::<Self>();
         let y = y.as_pointer::<Self>();
-        match x.as_ref().0.cmp(&y.as_ref().0) {
+        match x.as_ref().0.into_repr().cmp(&y.as_ref().0.into_repr()) {
             core::cmp::Ordering::Less => -1,
             core::cmp::Ordering::Equal => 0,
             core::cmp::Ordering::Greater => 1,
@@ -241,7 +241,7 @@ pub fn caml_pasta_fq_mut_square(mut x: ocaml::Pointer<CamlFq>) {
 #[ocaml_gen::func]
 #[ocaml::func]
 pub fn caml_pasta_fq_compare(x: ocaml::Pointer<CamlFq>, y: ocaml::Pointer<CamlFq>) -> ocaml::Int {
-    match x.as_ref().0.cmp(&y.as_ref().0) {
+    match x.as_ref().0.into_repr().cmp(&y.as_ref().0.into_repr()) {
         Less => -1,
         Equal => 0,
         Greater => 1,