Problem: testground stateless benchmark don't collect outputs (crypto…

…-org-chain#1524) * Problem: testground stateless benchmark don't collect outputs Solution: - tar the home directory into outputs volume * fix readme * fix readme * Update testground/benchmark/benchmark/stateless.py Signed-off-by: yihuang <[email protected]> * fix * log balance change after test --------- Signed-off-by: yihuang <[email protected]>
alienc0der · Jul 25, 2024 · 59c8b73 · 59c8b73
1 parent d545b24
commit 59c8b73
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 13 deletions.
diff --git a/testground/README.md b/testground/README.md
@@ -80,12 +80,22 @@ To simplify cluster setup, we are introducing a stateless mode.
 You need to have the `cronosd` in `PATH`.
 
 ```bash
-$ nix run github:crypto-org-chain/cronos#stateless-testcase gen /tmp/data/out 3 7
+$ nix run github:crypto-org-chain/cronos#stateless-testcase -- gen /tmp/data/out \
+  --validators 3 \
+  --fullnodes 7 \
+  --hostname_template "testplan-{index}" \
+  --num_accounts 10 \
+  --num_txs 1000
 ```
 
+* `validators`/`fullnodes` is the number of validators/full nodes.
+* `hostname_template` is the hostname of each node that can communicate.
+* `num_accounts` is the number of test accounts for each full node.
+* `num_txs` is the number of test transactions to be sent for each test account.
+
 ## Embed the data directory
 
-Patch the image to embed the data directory, it produce a local image:
+Embed the data directory into the image, it produce a new image:
 
 ```bash
 $ nix run github:crypto-org-chain/cronos#stateless-testcase patchimage cronos-testground:latest /tmp/data/out
@@ -94,10 +104,11 @@ $ nix run github:crypto-org-chain/cronos#stateless-testcase patchimage cronos-te
 ## Run In Local Docker
 
 ```bash
+$ mkdir /tmp/outputs
 $ jsonnet -S testground/benchmark/compositions/docker-compose.jsonnet | docker-compose -f /dev/stdin up
 ```
 
-It'll mount the data files to all the containers.
+It'll collect the node data files to the `/tmp/outputs` directory.
 
 ## Run In Cluster
 

diff --git a/testground/benchmark/benchmark/sendtx.py b/testground/benchmark/benchmark/sendtx.py
@@ -28,7 +28,9 @@ def fund_test_accounts(w3, from_account, num_accounts) -> [Account]:
 
 
 def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
-    print("test address", acct.address, "balance", w3.eth.get_balance(acct.address))
+    print(
+        "test begin, address", acct.address, "balance", w3.eth.get_balance(acct.address)
+    )
 
     initial_nonce = w3.eth.get_transaction_count(acct.address)
     nonce = initial_nonce
@@ -56,6 +58,10 @@ def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
         if nonce % 100 == 0:
             print(f"{acct.address} sent {nonce} transactions")
 
+    print(
+        "test end, address", acct.address, "balance", w3.eth.get_balance(acct.address)
+    )
+
 
 def generate_load(cli, num_accounts, num_txs, **kwargs):
     w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))

diff --git a/testground/benchmark/benchmark/stateless.py b/testground/benchmark/benchmark/stateless.py
@@ -3,6 +3,7 @@
 import shutil
 import socket
 import subprocess
+import tarfile
 import tempfile
 from pathlib import Path
 from typing import List
@@ -38,6 +39,8 @@ def gen(
         validators: int,
         fullnodes: int,
         hostname_template=HOSTNAME_TEMPLATE,
+        num_accounts=10,
+        num_txs=1000,
     ):
         outdir = Path(outdir)
         cli = ChainCommand(LOCAL_CRONOSD_PATH)
@@ -67,6 +70,15 @@ def gen(
                 peers, genesis, outdir, FULLNODE_GROUP, i, i + validators
             )
 
+        print("write config")
+        cfg = {
+            "validators": validators,
+            "fullnodes": fullnodes,
+            "num_accounts": num_accounts,
+            "num_txs": num_txs,
+        }
+        (outdir / "config.json").write_text(json.dumps(cfg))
+
     def patchimage(
         self,
         toimage,
@@ -89,21 +101,24 @@ def patchimage(
 
     def run(
         self,
-        outdir: str,
-        validators: int,
+        outdir: str = "/outputs",
+        datadir: str = "/data",
         cronosd=CONTAINER_CRONOSD_PATH,
         global_seq=None,
-        num_accounts=10,
-        num_txs=1000,
     ):
-        outdir = Path(outdir)
+        datadir = Path(datadir)
+
+        cfg = json.loads((datadir / "config.json").read_text())
+
         if global_seq is None:
             global_seq = node_index()
+
+        validators = cfg["validators"]
         group = VALIDATOR_GROUP if global_seq < validators else FULLNODE_GROUP
         group_seq = global_seq if group == VALIDATOR_GROUP else global_seq - validators
         print("node role", global_seq, group, group_seq)
 
-        home = outdir / group / str(group_seq)
+        home = datadir / group / str(group_seq)
 
         # start the node
         logfile = home / "node.log"
@@ -122,9 +137,17 @@ def run(
             # validators don't quit
             proc.wait()
         else:
-            generate_load(cli, num_accounts, num_txs, home=home)
-
+            generate_load(cli, cfg["num_accounts"], cfg["num_txs"], home=home)
             proc.kill()
+            proc.wait()
+
+            # collect outputs
+            outdir = Path(outdir)
+            if outdir.exists():
+                with tarfile.open(
+                    outdir / f"{group}_{group_seq}.tar.bz2", "x:bz2"
+                ) as tar:
+                    tar.add(home, arcname="data")
 
 
 def init_node_local(

diff --git a/testground/benchmark/compositions/docker-compose.jsonnet b/testground/benchmark/compositions/docker-compose.jsonnet
@@ -2,8 +2,11 @@ std.manifestYamlDoc({
   services: {
     ['testplan-' + i]: {
       image: 'cronos-testground:latest',
-      command: 'stateless-testcase run /data 3 --num_accounts=10 --num_txs=1000',
+      command: 'stateless-testcase run',
       container_name: 'testplan-' + i,
+      volumes: [
+        @'${OUTDIR:-/tmp/outputs}:/outputs',
+      ],
       environment: {
         JOB_COMPLETION_INDEX: i,
       },