Problem: testground infra is not easy to setup (#1504)

* Problem: no simpler benchmark setup Solution: - most of the features of testground are not really used by us yet, so for a simple benchmark to get a TPS result, it's possible to do an alternative setup. the alternative implementation setup all the node data files in advance, then run the nodes in containers in a stateless manner, they don't need any coordinations at runtime, so we don't need those infrastructures to support the coordinations, just deploy the testplan image to the cluster like a stateless service. * match test_group_id * lint * Update testground/benchmark/benchmark/peer.py Co-authored-by: mmsqe <[email protected]> Signed-off-by: yihuang <[email protected]> * change docker-composer * cleanup * speficy name && disable strict * run load generator and jsonnet config quit wait for grpc service * fix lint * readme and flake * fix total * print tps * Update testground/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: yihuang <[email protected]> * Revert "print tps" This reverts commit 8ecd342. * kill directly --------- Signed-off-by: yihuang <[email protected]> Co-authored-by: mmsqe <[email protected]> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
crypto-org-chain · Jul 5, 2024 · aef58bb · aef58bb
1 parent 1a791bd
commit aef58bb
Show file tree

Hide file tree

Showing 11 changed files with 512 additions and 222 deletions.
diff --git a/flake.nix b/flake.nix
@@ -42,6 +42,10 @@
           apps = {
             cronosd = mkApp packages.cronosd;
             cronosd-testnet = mkApp packages.cronosd-testnet;
+            stateless-testcase = {
+              type = "app";
+              program = "${pkgs.testground-testcase}/bin/stateless-testcase";
+            };
           };
           defaultPackage = packages.cronosd;
           defaultApp = apps.cronosd;

diff --git a/testground/README.md b/testground/README.md
@@ -1,3 +1,5 @@
+# Testground
+
 [Testground documentation](https://docs.testground.ai/)
 
 ## Build Image
@@ -67,3 +69,28 @@ mounts:
     writable: true
 ```
 
+
+
+# Stateless Mode
+
+To simplify cluster setup, we are introducing a stateless mode.
+
+## Generate Data Files Locally
+
+You need to have a `cronosd` in `PATH`.
+
+```bash
+$ nix run github:crypto-org-chain/cronos#stateless-testcase gen /tmp/data/out 3 7
+```
+
+## Run In Local Docker
+
+```bash
+$ jsonnet -S testground/benchmark/compositions/docker-compose.jsonnet | docker-compose -f /dev/stdin up
+```
+
+It'll mount the data files to all the containers.
+
+## Run In Cluster
+
+TODO
diff --git a/testground/benchmark/benchmark/main.py b/testground/benchmark/benchmark/main.py
@@ -1,17 +1,14 @@
 import os
 import subprocess
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 import web3
 
 from .cli import ChainCommand
 from .context import Context
-from .peer import bootstrap
-from .sendtx import fund_test_accounts, sendtx
-from .utils import export_eth_account, wait_for_block, wait_for_port
-
-CRONOSD_PATH = "/bin/cronosd"
+from .peer import CONTAINER_CRONOSD_PATH, bootstrap
+from .sendtx import generate_load
+from .utils import wait_for_block, wait_for_port
 
 
 def influxdb_url():
@@ -21,15 +18,15 @@ def influxdb_url():
 def entrypoint(ctx: Context):
     ctx.init_common()
 
-    cli = ChainCommand(CRONOSD_PATH)
+    cli = ChainCommand(CONTAINER_CRONOSD_PATH)
 
     # build the genesis file collectively, and setup the network topology
     bootstrap(ctx, cli)
 
     # start the node
     logfile = Path(ctx.params.test_outputs_path) / "node.log"
     proc = subprocess.Popen(
-        [CRONOSD_PATH, "start"],
+        [CONTAINER_CRONOSD_PATH, "start"],
         stdout=open(logfile, "ab", buffering=0),
     )
 
@@ -39,21 +36,7 @@ def entrypoint(ctx: Context):
 
     test_finish_entry = f"finish-test-{ctx.params.test_group_id}"
     if not ctx.is_validator:
-        w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
-        assert w3.eth.chain_id == 777
-        genesis_account = export_eth_account(cli, "account")
-        accounts = fund_test_accounts(w3, genesis_account, ctx.params.num_accounts)
-        with ThreadPoolExecutor(max_workers=ctx.params.num_accounts) as executor:
-            futs = (
-                executor.submit(sendtx, w3, acct, ctx.params.num_txs)
-                for acct in accounts
-            )
-            for fut in as_completed(futs):
-                try:
-                    fut.result()
-                except Exception as e:
-                    print("test task failed", e)
-
+        generate_load(cli, ctx.params.num_accounts, ctx.params.num_txs)
         print("finish test", ctx.group_seq)
         ctx.sync.signal_and_wait(
             test_finish_entry, ctx.params.test_group_instance_count

diff --git a/testground/benchmark/benchmark/peer.py b/testground/benchmark/benchmark/peer.py
@@ -3,86 +3,129 @@
 from pathlib import Path
 from typing import List
 
+from .cli import ChainCommand
 from .context import Context
 from .network import get_data_ip
 from .topology import connect_all
 from .types import GenesisAccount, PeerPacket
 from .utils import patch_json, patch_toml
 
+VAL_ACCOUNT = "validator"
 VAL_INITIAL_AMOUNT = "100000000000000000000basecro"
 VAL_STAKED_AMOUNT = "10000000000000000000basecro"
-ACC_INITIAL_AMOUNT = "100000000000000000000basecro"
+ACC_INITIAL_AMOUNT = "100000000000000000000000basecro"
 MEMPOOL_SIZE = 50000
+DEFAULT_DENOM = "basecro"
+VALIDATOR_GROUP = "validators"
+FULLNODE_GROUP = "fullnodes"
+CONTAINER_CRONOSD_PATH = "/bin/cronosd"
 
 
 def bootstrap(ctx: Context, cli) -> PeerPacket:
-    ip = get_data_ip(ctx.params)
-    cli(
-        "init",
-        f"node{ctx.global_seq}",
-        chain_id=ctx.params.chain_id,
-        default_denom="basecro",
+    home = Path.home() / ".cronos"
+    peer = init_node(
+        cli,
+        home,
+        get_data_ip(ctx.params),
+        ctx.params.chain_id,
+        ctx.params.test_group_id,
+        ctx.group_seq,
     )
 
-    cli("keys", "add", "validator", keyring_backend="test")
-    cli("keys", "add", "account", keyring_backend="test")
-    validator_addr = cli(
-        "keys", "show", "validator", "--address", keyring_backend="test"
+    data = ctx.sync.publish_subscribe_simple(
+        "peers", peer.dict(), ctx.params.test_instance_count
+    )
+    peers: List[PeerPacket] = [PeerPacket.model_validate(item) for item in data]
+
+    if ctx.is_fullnode_leader:
+        # prepare genesis file and publish
+        genesis = gen_genesis(cli, home, peers)
+        ctx.sync.publish("genesis", genesis)
+    else:
+        genesis = ctx.sync.subscribe_simple("genesis", 1)[0]
+        (home / "config" / "genesis.json").write_text(json.dumps(genesis))
+        cli("genesis", "validate", home=home)
+
+    p2p_peers = connect_all(peer, peers)
+    patch_configs(home, ctx.params.test_group_id, p2p_peers)
+    return peer
+
+
+def init_node(
+    cli: ChainCommand,
+    home: Path,
+    ip: str,
+    chain_id: str,
+    group: str,
+    group_seq: int,
+) -> PeerPacket:
+    default_kwargs = {
+        "home": home,
+        "chain_id": chain_id,
+        "keyring_backend": "test",
+    }
+    cli(
+        "init",
+        f"{group}-{group_seq}",
+        default_denom=DEFAULT_DENOM,
+        **default_kwargs,
     )
-    account_addr = cli("keys", "show", "account", "--address", keyring_backend="test")
+    cli("keys", "add", VAL_ACCOUNT, **default_kwargs)
+    cli("keys", "add", "account", **default_kwargs)
+    validator_addr = cli("keys", "show", VAL_ACCOUNT, "--address", **default_kwargs)
+    account_addr = cli("keys", "show", "account", "--address", **default_kwargs)
     accounts = [
         GenesisAccount(address=validator_addr, balance=VAL_INITIAL_AMOUNT),
         GenesisAccount(address=account_addr, balance=ACC_INITIAL_AMOUNT),
     ]
 
-    node_id = cli("comet", "show-node-id")
+    node_id = cli("comet", "show-node-id", **default_kwargs)
     peer_id = f"{node_id}@{ip}:26656"
-    current = PeerPacket(
+    peer = PeerPacket(
         ip=str(ip),
         node_id=node_id,
         peer_id=peer_id,
         accounts=accounts,
     )
 
-    if ctx.is_validator:
-        current.gentx = gentx(cli, ctx.params.chain_id)
-
-    data = ctx.sync.publish_subscribe_simple(
-        "peers", current.dict(), ctx.params.test_instance_count
+    if group == VALIDATOR_GROUP:
+        peer.gentx = gentx(cli, **default_kwargs)
+
+    return peer
+
+
+def gen_genesis(cli: ChainCommand, leader_home: Path, peers: List[PeerPacket]):
+    for peer in peers:
+        for account in peer.accounts:
+            cli(
+                "genesis",
+                "add-genesis-account",
+                account.address,
+                account.balance,
+                home=leader_home,
+            )
+    collect_gen_tx(cli, peers, home=leader_home)
+    cli("genesis", "validate", home=leader_home)
+    return patch_json(
+        leader_home / "config" / "genesis.json",
+        {
+            "consensus.params.block.max_gas": "81500000",
+            "app_state.evm.params.evm_denom": "basecro",
+            "app_state.feemarket.params.no_base_fee": True,
+        },
     )
-    peers: List[PeerPacket] = [PeerPacket.model_validate(item) for item in data]
 
-    config_path = Path.home() / ".cronos" / "config"
-    if ctx.is_fullnode_leader:
-        # prepare genesis file and publish
-        for peer in peers:
-            for account in peer.accounts:
-                cli("genesis", "add-genesis-account", account.address, account.balance)
-        collect_gen_tx(cli, peers)
-        cli("genesis", "validate")
-        genesis = patch_json(
-            config_path / "genesis.json",
-            {
-                "consensus.params.block.max_gas": "81500000",
-                "app_state.evm.params.evm_denom": "basecro",
-                "app_state.feemarket.params.no_base_fee": True,
-            },
-        )
-        ctx.sync.publish("genesis", genesis)
-    else:
-        genesis = ctx.sync.subscribe_simple("genesis", 1)[0]
-        genesis_file = config_path / "genesis.json"
-        genesis_file.write_text(json.dumps(genesis))
-        cli("genesis", "validate")
 
+def patch_configs(home: Path, group: str, peers: str):
     # update persistent_peers and other configs in config.toml
     config_patch = {
-        "p2p.persistent_peers": connect_all(current, peers),
+        "p2p.persistent_peers": peers,
+        "p2p.addr_book_strict": False,
         "mempool.recheck": "false",
         "mempool.size": MEMPOOL_SIZE,
         "consensus.timeout_commit": "2s",
     }
-    if ctx.is_validator:
+    if group == VALIDATOR_GROUP:
         config_patch["tx_index.indexer"] = "null"
 
     app_patch = {
@@ -92,35 +135,33 @@ def bootstrap(ctx: Context, cli) -> PeerPacket:
         "mempool.max-txs": MEMPOOL_SIZE,
     }
 
-    patch_toml(config_path / "config.toml", config_patch)
-    patch_toml(config_path / "app.toml", app_patch)
+    patch_toml(home / "config" / "config.toml", config_patch)
+    patch_toml(home / "config" / "app.toml", app_patch)
 
-    return current
 
-
-def gentx(cli, chain_id):
+def gentx(cli, **kwargs):
     cli(
         "genesis",
         "add-genesis-account",
-        "validator",
+        VAL_ACCOUNT,
         VAL_INITIAL_AMOUNT,
-        keyring_backend="test",
+        **kwargs,
     )
-    output = Path("gentx.json")
-    cli(
-        "genesis",
-        "gentx",
-        "validator",
-        VAL_STAKED_AMOUNT,
-        min_self_delegation=1,
-        chain_id=chain_id,
-        output_document=output,
-        keyring_backend="test",
-    )
-    return json.loads(output.read_text())
+    with tempfile.TemporaryDirectory() as tmp:
+        output = Path(tmp) / "gentx.json"
+        cli(
+            "genesis",
+            "gentx",
+            VAL_ACCOUNT,
+            VAL_STAKED_AMOUNT,
+            min_self_delegation=1,
+            output_document=output,
+            **kwargs,
+        )
+        return json.loads(output.read_text())
 
 
-def collect_gen_tx(cli, peers):
+def collect_gen_tx(cli, peers, **kwargs):
     """
     save gentxs to file and call collect-gentxs
     leader node prepare genesis file and broadcast to other nodes
@@ -130,4 +171,4 @@ def collect_gen_tx(cli, peers):
         for i, peer in enumerate(peers):
             if peer.gentx is not None:
                 (tmpdir / f"gentx-{i}.json").write_text(json.dumps(peer.gentx))
-        cli("genesis", "collect-gentxs", gentx_dir=str(tmpdir))
+        cli("genesis", "collect-gentxs", gentx_dir=str(tmpdir), **kwargs)
diff --git a/testground/benchmark/benchmark/sendtx.py b/testground/benchmark/benchmark/sendtx.py
@@ -1,9 +1,10 @@
 import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import web3
 from eth_account import Account
 
-from .utils import send_transaction
+from .utils import export_eth_account, send_transaction
 
 TEST_AMOUNT = 1000000000000000000
 GAS_PRICE = 1000000000
@@ -54,3 +55,17 @@ def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
 
         if nonce % 100 == 0:
             print(f"{acct.address} sent {nonce} transactions")
+
+
+def generate_load(cli, num_accounts, num_txs, **kwargs):
+    w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
+    assert w3.eth.chain_id == 777
+    genesis_account = export_eth_account(cli, "account", **kwargs)
+    accounts = fund_test_accounts(w3, genesis_account, num_accounts)
+    with ThreadPoolExecutor(max_workers=num_accounts) as executor:
+        futs = (executor.submit(sendtx, w3, acct, num_txs) for acct in accounts)
+        for fut in as_completed(futs):
+            try:
+                fut.result()
+            except Exception as e:
+                print("test task failed", e)