Merge branch 'rampup' into dkijania/port_remove_snarkyjs_test_rampup

buildkite/scripts/connect-to-berkeley.sh

@@ -16,16 +16,17 @@ export DEBIAN_FRONTEND=noninteractive
 apt-get update
 apt-get install -y git apt-transport-https ca-certificates tzdata curl
 
-TESTNET_NAME="berkeley"
+TESTNET_VERSION_NAME="berkeley"
+TESTNET_NAME="testworld-2-0"
 
 git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-echo "Installing mina daemon package: mina-${TESTNET_NAME}=${MINA_DEB_VERSION}"
+echo "Installing mina daemon package: mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
 echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list
 apt-get update
-apt-get install --allow-downgrades -y "mina-${TESTNET_NAME}=${MINA_DEB_VERSION}"
+apt-get install --allow-downgrades -y "mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
 
 # Remove lockfile if present
 rm ~/.mina-config/.mina-lock ||:

buildkite/scripts/connect-to-testnet.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [[ $# -ne 3 ]]; then
+    echo "Usage: $0 '<testnet-name>' '<wait-between-polling-graphql>''<wait-after-final-check>'"
+    exit 1
+fi
+
+TESTNET_VERSION_NAME="berkeley"
+TESTNET_NAME=$1
+WAIT_BETWEEN_POLLING_GRAPHQL=$2
+WAIT_AFTER_FINAL_CHECK=$3
+
+case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
+  rampup|berkeley|release/2.0.0|develop)
+  ;;
+  *)
+    echo "Not pulling against rampup, not running the connect test"
+    exit 0 ;;
+esac
+
+# Don't prompt for answers during apt-get install
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y git apt-transport-https ca-certificates tzdata curl
+
+git config --global --add safe.directory /workdir
+
+source buildkite/scripts/export-git-env-vars.sh
+
+echo "Installing mina daemon package: mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
+echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list
+apt-get update
+apt-get install --allow-downgrades -y "mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
+
+# Remove lockfile if present
+rm ~/.mina-config/.mina-lock ||:
+
+mkdir -p /root/libp2p-keys/
+# Pre-generated random password for this quick test
+export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf
+mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key
+# Set permissions on the keypair so the daemon doesn't complain
+chmod -R 0700 /root/libp2p-keys/
+
+# Restart in the background
+mina daemon \
+  --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \
+  --libp2p-keypair "/root/libp2p-keys/key" \
+& # -background
+
+# Attempt to connect to the GraphQL client every 10s for up to 8 minutes
+num_status_retries=24
+for ((i=1;i<=$num_status_retries;i++)); do
+  sleep $WAIT_BETWEEN_POLLING_GRAPHQL
+  set +e
+  mina client status
+  status_exit_code=$?
+  set -e
+  if [ $status_exit_code -eq 0 ]; then
+    break
+  elif [ $i -eq $num_status_retries ]; then
+    exit $status_exit_code
+  fi
+done
+
+# Check that the daemon has connected to peers and is still up after 2 mins
+sleep $WAIT_AFTER_FINAL_CHECK
+mina client status
+if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then
+    echo "Found some peers"
+else
+    echo "No peers found"
+    exit 1
+fi
+

buildkite/src/Command/ConnectToTestnet.dhall

@@ -9,20 +9,24 @@ let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 
 let Cmd = ../Lib/Cmds.dhall in
 
-{ step = \(dependsOn : List Command.TaggedKey.Type) ->
-    Command.build
-      Command.Config::{
-        commands = [
-          Cmd.runInDocker
-            Cmd.Docker::{
-              image = (../Constants/ContainerImages.dhall).ubuntu2004
-            }
-            "./buildkite/scripts/connect-to-berkeley.sh"
-        ],
-        label = "Connect to Berkeley",
-        soft_fail = Some (B/SoftFail.Boolean True),
-        key = "connect-to-berkeley",
-        target = Size.Large,
-        depends_on = dependsOn
-      }
+{ step = \(dependsOn : List Command.TaggedKey.Type) 
+    -> \(testnet : Text) 
+    -> \(wait_between_graphql_poll : Text) 
+    -> \(wait_before_final_check: Text )
+    -> \(soft_fail: B/SoftFail)
+    -> Command.build
+        Command.Config::{
+          commands = [
+            Cmd.runInDocker
+              Cmd.Docker::{
+                image = (../Constants/ContainerImages.dhall).ubuntu2004
+              }
+              "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}"
+          ],
+          label = "Connect to ${testnet}",
+          soft_fail = Some soft_fail,
+          key = "connect-to-${testnet}",
+          target = Size.Large,
+          depends_on = dependsOn
+        }
 }

buildkite/src/Jobs/Test/ConnectToBerkeley.dhall

@@ -1,5 +1,6 @@
 let S = ../../Lib/SelectFiles.dhall
-
+let B = ../../External/Buildkite.dhall
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 let JobSpec = ../../Pipeline/JobSpec.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
 
@@ -14,14 +15,14 @@ in Pipeline.build Pipeline.Config::{
     JobSpec::{
     dirtyWhen = [
       S.strictlyStart (S.contains "src"),
-      S.exactly "buildkite/scripts/connect-to-berkeley" "sh",
+      S.exactly "buildkite/scripts/connect-to-testnet" "sh",
       S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall",
       S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall"
     ],
     path = "Test",
     name = "ConnectToBerkeley"
   },
   steps = [
-    ConnectToTestnet.step dependsOn
+    ConnectToTestnet.step dependsOn "berkeley" "40s" "2m" (B/SoftFail.Boolean True)
   ]
 }

buildkite/src/Jobs/Test/ConnectToTestworld-2-0.dhall

@@ -0,0 +1,29 @@
+let S = ../../Lib/SelectFiles.dhall
+let B = ../../External/Buildkite.dhall
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall
+
+let dependsOn = [
+  { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" }
+]
+
+in Pipeline.build Pipeline.Config::{
+  spec =
+    JobSpec::{
+    dirtyWhen = [
+      S.strictlyStart (S.contains "src"),
+      S.exactly "buildkite/scripts/connect-to-testnet" "sh",
+      S.exactly "buildkite/src/Jobs/Test/ConnectToTestworld-2-0" "dhall",
+      S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall"
+    ],
+    path = "Test",
+    name = "ConnectToTestworld-2-0"
+  },
+  steps = [
+    ConnectToTestnet.step dependsOn "testworld-2-0" "40s" "2m" (B/SoftFail.Boolean False)
+  ]
+}

src/lib/merkle_ledger/any_ledger.ml

@@ -113,8 +113,6 @@ module Make_base (Inputs : Inputs_intf) :
 
     module Addr = Location.Addr
 
-    let remove_accounts_exn (T ((module Base), t)) = Base.remove_accounts_exn t
-
     let merkle_path_at_index_exn (T ((module Base), t)) =
       Base.merkle_path_at_index_exn t
 
@@ -184,8 +182,6 @@ module Make_base (Inputs : Inputs_intf) :
 
     let to_list_sequential (T ((module Base), t)) = Base.to_list_sequential t
 
-    let make_space_for (T ((module Base), t)) = Base.make_space_for t
-
     let get_all_accounts_rooted_at_exn (T ((module Base), t)) =
       Base.get_all_accounts_rooted_at_exn t
 

src/lib/merkle_ledger/base_ledger_intf.ml

@@ -143,8 +143,6 @@ module type S = sig
 
   val get_hash_batch_exn : t -> Location.t list -> hash list
 
-  val remove_accounts_exn : t -> account_id list -> unit
-
   (** Triggers when the ledger has been detached and should no longer be
       accessed.
   *)

src/lib/merkle_ledger/database.ml

@@ -30,8 +30,6 @@ module Make (Inputs : Inputs_intf) :
   module Db_error = struct
     type t = Account_location_not_found | Out_of_leaves | Malformed_database
     [@@deriving sexp]
-
-    exception Db_exception of t
   end
 
   module Path = Merkle_path.Make (Hash)
@@ -198,8 +196,6 @@ module Make (Inputs : Inputs_intf) :
     assert (Addr.depth address <= mdb.depth) ;
     set_bin mdb (Location.Hash address) Hash.bin_size_t Hash.bin_write_t hash
 
-  let make_space_for _t _tot = ()
-
   let get_generic mdb location =
     assert (Location.is_generic location) ;
     get_raw mdb location
@@ -245,8 +241,6 @@ module Make (Inputs : Inputs_intf) :
       |> get_generic_batch mdb
       |> List.map ~f:(Option.bind ~f:parse_location)
 
-    let delete mdb key = delete_raw mdb (build_location key)
-
     let set mdb key location =
       set_raw mdb (build_location key)
         (Location.serialize ~ledger_depth:mdb.depth location)
@@ -361,9 +355,6 @@ module Make (Inputs : Inputs_intf) :
           Account_id.Stable.Latest.bin_size_t
           Account_id.Stable.Latest.bin_write_t account_id
 
-      let remove (mdb : t) (token_id : Token_id.t) : unit =
-        delete_raw mdb (build_location token_id)
-
       let all_owners (t : t) : (Token_id.t * Account_id.t) Sequence.t =
         let deduped_tokens =
           (* First get the sequence of unique tokens *)
@@ -439,18 +430,10 @@ module Make (Inputs : Inputs_intf) :
          most accounts are not going to be managers. *)
       Owner.set mdb (Account_id.derive_token_id ~owner:aid) aid
 
-    let remove mdb pk tid = update mdb pk ~f:(fun tids -> Set.remove tids tid)
-
     let _remove_several mdb pk rem_tids =
       update mdb pk ~f:(fun tids ->
           Set.diff tids (Token_id.Set.of_list rem_tids) )
 
-    let remove_account (mdb : t) (aid : Account_id.t) : unit =
-      let token = Account_id.token_id aid in
-      let key = Account_id.public_key aid in
-      remove mdb key token ;
-      Owner.remove mdb (Account_id.derive_token_id ~owner:aid)
-
     (** Generate a batch of database changes to add the given tokens. *)
     let add_batch_create mdb pks_to_tokens =
       let pks_to_all_tokens =
@@ -661,32 +644,6 @@ module Make (Inputs : Inputs_intf) :
 
   let merkle_root mdb = get_hash mdb Location.root_hash
 
-  let remove_accounts_exn t keys =
-    let locations =
-      (* if we don't have a location for all keys, raise an exception *)
-      let rec loop keys accum =
-        match keys with
-        | [] ->
-            accum (* no need to reverse *)
-        | key :: rest -> (
-            match Account_location.get t key with
-            | Ok loc ->
-                loop rest (loc :: accum)
-            | Error err ->
-                raise (Db_error.Db_exception err) )
-      in
-      loop keys []
-    in
-    (* N.B.: we're not using stack database here to make available newly-freed
-       locations *)
-    List.iter keys ~f:(Account_location.delete t) ;
-    List.iter keys ~f:(Tokens.remove_account t) ;
-    List.iter locations ~f:(fun loc -> delete_raw t loc) ;
-    (* recalculate hashes for each removed account *)
-    List.iter locations ~f:(fun loc ->
-        let hash_loc = Location.Hash (Location.to_path_exn loc) in
-        set_hash t hash_loc Hash.empty_account )
-
   let merkle_path mdb location =
     let location =
       if Location.is_account location then
@@ -713,7 +670,9 @@ module Make (Inputs : Inputs_intf) :
       List.map locations ~f:Location.merkle_path_dependencies_exn
     in
     let all_locs =
-      List.map list_of_dependencies ~f:(fun deps -> List.map ~f:fst deps |> expand_query) |> List.concat
+      List.map list_of_dependencies ~f:(fun deps ->
+          List.map ~f:fst deps |> expand_query )
+      |> List.concat
     in
     let hashes = get_hash_batch_exn mdb all_locs in
     snd @@ List.fold_map ~init:hashes ~f:compute_path list_of_dependencies

src/lib/merkle_ledger/null_ledger.ml

@@ -39,10 +39,6 @@ end = struct
 
   let create ~depth () = { uuid = Uuid_unix.create (); depth }
 
-  let remove_accounts_exn _t keys =
-    if List.is_empty keys then ()
-    else failwith "remove_accounts_exn: null ledgers cannot be mutated"
-
   let empty_hash_at_height =
     Empty_hashes.extensible_cache (module Hash) ~init_hash:Hash.empty_account
 
@@ -151,8 +147,6 @@ end = struct
 
   let to_list_sequential _t = []
 
-  let make_space_for _t _tot = ()
-
   let get_all_accounts_rooted_at_exn t addr =
     let first_node, last_node =
       Addr.Range.subtree_range ~ledger_depth:t.depth addr

src/lib/merkle_ledger/syncable_intf.ml

@@ -30,6 +30,4 @@ module type S = sig
   val get_all_accounts_rooted_at_exn : t -> addr -> (addr * account) list
 
   val merkle_root : t -> root_hash
-
-  val make_space_for : t -> int -> unit
 end

src/lib/merkle_ledger_tests/test_database.ml

@@ -430,28 +430,6 @@ let%test_module "test functor on in memory databases" =
                 Stdlib.List.compare_lengths accounts retrieved_accounts = 0 ) ;
               assert (List.equal Account.equal accounts retrieved_accounts) )
 
-      let%test_unit "removing accounts restores Merkle root" =
-        Test.with_instance (fun mdb ->
-            let num_accounts = 5 in
-            let account_ids = Account_id.gen_accounts num_accounts in
-            let balances =
-              Quickcheck.random_value
-                (Quickcheck.Generator.list_with_length num_accounts Balance.gen)
-            in
-            let accounts =
-              List.map2_exn account_ids balances ~f:Account.create
-            in
-            let merkle_root0 = MT.merkle_root mdb in
-            List.iter accounts ~f:(fun account ->
-                ignore @@ create_new_account_exn mdb account ) ;
-            let merkle_root1 = MT.merkle_root mdb in
-            (* adding accounts should change the Merkle root *)
-            assert (not (Hash.equal merkle_root0 merkle_root1)) ;
-            MT.remove_accounts_exn mdb account_ids ;
-            (* should see original Merkle root after removing the accounts *)
-            let merkle_root2 = MT.merkle_root mdb in
-            assert (Hash.equal merkle_root2 merkle_root0) )
-
       let%test_unit "fold over account balances" =
         Test.with_instance (fun mdb ->
             let num_accounts = 5 in