feat: automated benchmark setup

.gitignore

@@ -1,4 +1,8 @@
+benchmark/machines.txt
+benchmark/benchmark.conf
+benchmark/inventory.ini
 benchmark/*.json
+benchmark/output
 benchmark/config/*
 !benchmark/config/README.md
 target

Cargo.toml

@@ -1,11 +1,11 @@
 [workspace]
 members = [
-    "benchmark/code",
-    "dog",
-    "examples/simple",
+  "benchmark/code",
+  "dog",
+  "examples/simple",
 
-    # Tests
-    "dog/tests",
+  # Tests
+  "dog/tests",
 ]
 resolver = "2"
 

benchmark/Dockerfile

@@ -1,6 +1,11 @@
-FROM rust AS builder
-WORKDIR /mnt
-RUN --mount=type=bind,source=./code,target=/mnt cargo build --release --target-dir /tmp
+FROM rust:latest AS builder
+WORKDIR /app
+COPY . .
+WORKDIR /app/benchmark
+RUN cargo build --release --target-dir /tmp --bin libp2p-dog-benchmark
 
 FROM debian:bookworm-slim
-COPY --from=builder /tmp/release/benchmark /usr/local/bin/benchmark
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /tmp/release/libp2p-dog-benchmark /usr/local/bin/benchmark

benchmark/README.md

@@ -1 +1,10 @@
 # DOG benchmark setup
+
+1. Spawn the machines you want and place their IPs in the `machines.txt` file (**PLEASE ADD A FINAL EMPTY LINE**).
+2. Run the command `./scripts/generate-inventory.sh --machines machines.txt --output inventory.ini` to generate the inventory file.
+3. Adjust the benchmark parameters (example in `sample-benchmark-config.conf`) in a file called `benchmark.conf`.
+4. Run the command `./scripts/generate-configs.sh --config benchmark.conf --machines machines.txt --output config`
+5. Initialize the machines with the command `ansible-playbook -i inventory.ini --forks $(cat machines.txt | wc -l) playbooks/init.yml`
+6. Build the benchmark container with the command `ansible-playbook -i inventory.ini --forks $(cat machines.txt | wc -l) playbooks/build.yml`
+7. Export the configuration on all machines with the command `ansible-playbook -i inventory.ini --forks $(cat machines.txt | wc -l) playbooks/config.yml`
+8. Run the benchmark with the command `ansible-playbook -i inventory.ini --forks $(cat machines.txt | wc -l) playbooks/start.yml`

benchmark/code/Cargo.toml

@@ -11,7 +11,14 @@ release = false
 
 [dependencies]
 clap = { workspace = true, features = ["derive"] }
-libp2p = { workspace = true, features = ["noise", "tcp", "yamux", "tokio", "macros", "gossipsub"] }
+libp2p = { workspace = true, features = [
+  "noise",
+  "tcp",
+  "yamux",
+  "tokio",
+  "macros",
+  "gossipsub",
+] }
 libp2p-dog = { path = "../../dog" }
 prometheus-client = { workspace = true }
 serde = { workspace = true, features = ["derive"] }

benchmark/code/src/args.rs

@@ -7,6 +7,9 @@ pub(crate) struct Args {
 
     #[arg(short, long)]
     pub config: String,
+
+    #[arg(short, long)]
+    pub start_timestamp: u64,
 }
 
 impl Args {

benchmark/code/src/config.rs

@@ -40,7 +40,6 @@ pub(crate) struct Node {
 #[derive(Deserialize)]
 pub(crate) struct Benchmark {
     pub protocol: Protocol,
-    pub start_timestamp_in_sec: u64,
     pub duration_in_sec: u64,
     pub tps: u64,
     pub tx_size_in_bytes: usize,

benchmark/code/src/main.rs

@@ -52,14 +52,19 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let transaction_timer = time::sleep(Duration::from_secs(u64::MAX)); // Wait for start timestamp
     tokio::pin!(transaction_timer);
 
-    let start_instant =
-        time::Instant::now() + Duration::from_secs(config.benchmark.start_timestamp_in_sec);
-
-    let stop_instant = time::Instant::now()
-        + Duration::from_secs(
-            config.benchmark.start_timestamp_in_sec + config.benchmark.duration_in_sec,
+    let start_instant = time::Instant::now()
+        + Duration::from_millis(
+            args.start_timestamp
+                .saturating_sub(std::time::UNIX_EPOCH.elapsed()?.as_millis() as u64),
         );
 
+    tracing::info!(
+        "Starting benchmark in {:?}",
+        start_instant - time::Instant::now(),
+    );
+
+    let stop_instant = start_instant + Duration::from_secs(config.benchmark.duration_in_sec);
+
     let dump_interval = Duration::from_millis(config.benchmark.dump_interval_in_ms);
     let dump_timer = time::sleep(Duration::from_secs(u64::MAX)); // Wait for start timestamp
     tokio::pin!(dump_timer);

benchmark/playbooks/build.yml

@@ -4,20 +4,18 @@
   tasks:
     - name: Copy source code to remote machine
       synchronize:
-        src: "{{ playbook_dir }}/../code"
+        src: "{{ playbook_dir }}/../../../libp2p-dog"
         dest: /home/{{ ansible_ssh_user }}
         mode: push
         delete: true
 
-    - name: Upload Dockerfile
-      copy:
-        src: "{{ playbook_dir }}/../Dockerfile"
-        dest: /home/{{ ansible_ssh_user }}
-
     - name: Build container
       shell: |
-        docker build -t dog-benchmark:latest -f /home/{{ ansible_ssh_user }}/Dockerfile .
-        docker save dog-benchmark:latest > container.tar
+        docker build \
+          -t dog-benchmark:latest \
+          -f /home/{{ ansible_ssh_user }}/libp2p-dog/benchmark/Dockerfile \
+          /home/{{ ansible_ssh_user }}/libp2p-dog
+        docker save dog-benchmark:latest > ~/container.tar
 
     - name: Copy container to local machine
       synchronize:

benchmark/playbooks/config.yml

@@ -0,0 +1,24 @@
+- name: Delete old config
+  hosts: all
+
+  vars:
+    ansible_ssh_user: root
+
+  tasks:
+    - name: Delete old config
+      shell: rm -rf /home/bob/config
+
+- name: Export configuration
+  hosts: all
+
+  vars:
+    playbook_dir: "{{ playbook_dir | default(playbook_dir | dirname) }}"
+
+  tasks:
+    - name: Create config directory
+      shell: mkdir -p /home/bob/config
+
+    - name: Export configuration
+      copy:
+        src: "{{ playbook_dir }}/../config/node-{{ inventory_hostname }}.toml"
+        dest: /home/bob/config/node.toml

benchmark/playbooks/start.yml

@@ -0,0 +1,32 @@
+- name: Start the join benchmark
+  hosts: all
+
+  tasks:
+    - name: Delete previous outputs
+      shell: rm -rf {{ playbook_dir }}/../output/*
+      delegate_to: localhost
+      run_once: true
+
+    - name: Get UTC timestamp
+      command: "gdate -u +%s%3N"
+      register: timestamp_output
+      delegate_to: localhost
+      run_once: true
+
+    - name: Execute benchmark
+      shell: |
+        mkdir -p /home/{{ ansible_user }}/output
+        rm -rf /home/{{ ansible_user }}/output/*
+        docker run --rm --network host \
+          -v /home/{{ ansible_user }}/config:/config \
+          -v /home/{{ ansible_user }}/output:/output \
+          -w /app \
+          dog-benchmark:latest /bin/bash -c \
+          "benchmark --dir /output --config /config/node.toml --start-timestamp {{ (timestamp_output.stdout | int) + 10000 }} > /output/benchmark.log 2>&1"
+
+    - name: Copy benchmark output to local machine
+      synchronize:
+        src: /home/{{ ansible_user }}/output
+        dest: "{{ playbook_dir }}/../output/output-{{ inventory_hostname }}"
+        mode: pull
+        delete: true

benchmark/sample-benchmark-config.conf

@@ -0,0 +1,9 @@
+NUM_NODES=5
+ID_PREFIX="node"
+PORT=27000
+PROTOCOL="dog" # dog, gossipsub
+DURATION=10 # in seconds
+TPS=5
+TX_SIZE=1024 # in bytes
+DUMP_INTERVAL=1000 # in milliseconds
+REGISTRY_PREFIX=""

benchmark/sample_config.toml → benchmark/sample-node-config.toml

@@ -2,14 +2,13 @@
 id = "node1"
 addr = "/ip4/127.0.0.1/tcp/12345"
 peers = [
-    "/ip4/127.0.0.1/tcp/12346",
-    "/ip4/127.0.0.1/tcp/12347",
-    "/ip4/127.0.0.1/tcp/12348",
+  "/ip4/127.0.0.1/tcp/12346",
+  "/ip4/127.0.0.1/tcp/12347",
+  "/ip4/127.0.0.1/tcp/12348",
 ]
 
 [benchmark]
 protocol = "dog"
-start_timestamp_in_sec = 1736850301
 duration_in_sec = 10
 tps = 1
 tx_size_in_bytes = 1024

benchmark/scripts/generate-configs.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+while [[ $# -gt 0 ]]; do
+    key="${1}"
+    case $key in
+        -c|--config)
+            CONFIG="${2}"
+            shift
+            shift
+            ;;
+        -m|--machines)
+            MACHINES="${2}"
+            shift
+            shift
+            ;;
+        -o|--output)
+            OUTPUT="${2}"
+            shift
+            shift
+            ;;
+        *)
+            echo "Unknown argument: ${1}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "${CONFIG}" ] || [ -z "${MACHINES}" ] || [ -z "${OUTPUT}" ]; then
+    echo "Usage: ${0} -c|--config <config> -m|--machines <machines> -o|--output <output>"
+    exit 1
+fi
+
+source $CONFIG
+
+NUM_MACHINES=$(grep -v '^$' ${MACHINES} | wc -l | tr -d ' ')
+if [ ${NUM_NODES} -ne ${NUM_MACHINES} ]; then
+    echo "Number of nodes (${NUM_NODES}) does not match number of machines (${NUM_MACHINES})"
+    exit 1
+fi
+
+echo "Generating configs for ${NUM_NODES} nodes"
+
+generate_node_config() {
+  output=${1}
+  id=${2}
+  addr=${3}
+  peers="${@:4}"
+
+  touch ${output}
+  > ${output}
+
+  echo "[node]" >> ${output}
+  echo "id = \"${id}\"" >> ${output}
+  echo "addr = \"${addr}\"" >> ${output}
+  echo "peers = [" >> ${output}
+  for peer in ${peers}; do
+    echo "  \"${peer}\"," >> ${output}
+  done
+  echo -e "]\n" >> ${output}
+
+  echo "[benchmark]" >> ${output}
+  echo "protocol = \"${PROTOCOL}\"" >> ${output}
+  echo "duration_in_sec = ${DURATION}" >> ${output}
+  echo "tps = ${TPS}" >> ${output}
+  echo "tx_size_in_bytes = ${TX_SIZE}" >> ${output}
+  echo "dump_interval_in_ms = ${DUMP_INTERVAL}" >> ${output}
+  echo "registry_prefix = \"${REGISTRY_PREFIX}\"" >> ${output}
+}
+
+rm -rf ${OUTPUT}/*-*.toml
+
+for i in $(seq 1 ${NUM_NODES}); do
+  echo "Generating config for node ${i}"
+
+  output="${OUTPUT}/${ID_PREFIX}-${i}.toml"
+  id="${ID_PREFIX}-${i}"
+  ip=$(sed -n "${i}p" ${MACHINES})
+  addr="/ip4/${ip}/tcp/${PORT}"
+  peers=""
+  for j in $(seq 1 ${NUM_NODES}); do
+    if [ ${i} -ne ${j} ]; then
+      peer="/ip4/$(sed -n "${j}p" ${MACHINES})/tcp/${PORT}"
+      peers="${peers} ${peer}"
+    fi
+  done
+
+  generate_node_config ${output} ${id} ${addr} ${peers}
+done
+
+echo "Done"

benchmark/scripts/generate-inventory.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+while [[ $# -gt 0 ]]; do
+    key="${1}"
+    case $key in
+        -m|--machines)
+            MACHINES="${2}"
+            shift
+            shift
+            ;;
+        -o|--output)
+            OUTPUT="${2}"
+            shift
+            shift
+            ;;
+        *)
+            echo "Unknown argument: ${1}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "${MACHINES}" ] || [ -z "${OUTPUT}" ]; then
+    echo "Usage: ${0} -m|--machines <machines> -o|--output <output>"
+    exit 1
+fi
+
+echo "Filling in the ansible inventory file"
+
+# check that the output file exists
+if [ ! -f "${OUTPUT}" ]; then
+    touch "${OUTPUT}"
+    echo "[machines]" > "${OUTPUT}"
+    echo "" >> "${OUTPUT}"
+    echo "[machines:vars]" >> "${OUTPUT}"
+    echo "ansible_ssh_user=bob" >> "${OUTPUT}"
+fi
+
+awk -v machines="${MACHINES}" '
+NR==FNR {
+    new_ips[NR] = $0;
+    next;
+}
+{
+    if ($0 ~ /^\[machines\]/) {
+        print;
+        for (i in new_ips) {
+            print i " ansible_host=" new_ips[i];
+        }
+        print ""; # Add an empty line after the entries
+        while (getline > 0 && $0 !~ /^\[machines:vars\]/) {}
+    }
+    if ($0 ~ /^\[machines:vars\]/ || FNR != 1) {
+        print;
+    }
+}' "${MACHINES}" "${OUTPUT}" > "${OUTPUT}.tmp"
+mv "${OUTPUT}.tmp" "${OUTPUT}"
+
+echo "Done"