Merge pull request #457 from luisgerhorst/bpf-spectre

RFC: Structure cicd/tcpsctpperf and add wrk2/nginx benchmark
loxilb-io · Dec 11, 2023 · 0e94b9c · 0e94b9c
2 parents edfc258 + 32add8d
commit 0e94b9c
Show file tree

Hide file tree

Showing 7 changed files with 276 additions and 29 deletions.
diff --git a/cicd/tcpsctpperf/config.sh b/cicd/tcpsctpperf/config.sh
@@ -9,47 +9,55 @@ echo "#########################################"
 
 spawn_docker_host --dock-type loxilb --dock-name llb1
 spawn_docker_host --dock-type host --dock-name l3h1
-spawn_docker_host --dock-type host --dock-name l3ep1
-spawn_docker_host --dock-type host --dock-name l3ep2
-spawn_docker_host --dock-type host --dock-name l3ep3
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  spawn_docker_host --dock-type host --dock-name l3ep$i
+done
 
 echo "#########################################"
 echo "Connecting and configuring  hosts"
 echo "#########################################"
 
-
 connect_docker_hosts l3h1 llb1
-connect_docker_hosts l3ep1 llb1
-connect_docker_hosts l3ep2 llb1
-connect_docker_hosts l3ep3 llb1
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  connect_docker_hosts l3ep$i llb1
+done
 
-sleep 5
+sleep 1
 
-#L3 config
+# L3 config
 config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254
-config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254
-config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254
-config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  config_docker_host --host1 l3ep$i --host2 llb1 --ptype phy --addr 31.31.$i.1/24 --gw 31.31.$i.254
+done
 config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24
-config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24
-config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24
-config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  config_docker_host --host1 llb1 --host2 l3ep$i --ptype phy --addr 31.31.$i.254/24
+done
 
-sleep 5
+sleep 1
 
-#Need to do this as netperf sctp doesn't work without this
+# Need to do this as netperf sctp doesn't work without this
 $hexec l3h1 ifconfig eth0 0
-$hexec l3ep1 ifconfig eth0 0
-$hexec l3ep2 ifconfig eth0 0
-$hexec l3ep3 ifconfig eth0 0
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  $hexec l3ep$i ifconfig eth0 0
+done
 
 for ((i=1,port=12865;i<=100;i++,port++))
 do
-  $dexec llb1 loxicmd create lb 20.20.20.1 --tcp=$port:$port  --endpoints=31.31.31.1:1 >> /dev/null
+  $dexec llb1 loxicmd create lb 20.20.20.1 --tcp=$port:$port --endpoints=31.31.1.1:1 >> /dev/null
 done
 
-$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=13866:13866  --endpoints=31.31.31.1:1 >> /dev/null
+# iperf3 --sctp will use tcp:13866 for control data, and sctp:13866 for the
+# benchmark data.
+$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=13866:13866 --endpoints=31.31.1.1:1 >> /dev/null
 for ((i=1,port=13866;i<=100;i++,port++))
 do
-  $dexec llb1 loxicmd create lb 20.20.20.1 --sctp=$port:$port  --endpoints=31.31.31.1:1 >> /dev/null
+  $dexec llb1 loxicmd create lb 20.20.20.1 --sctp=$port:$port --endpoints=31.31.1.1:1 >> /dev/null
 done
+
+$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=14000:14000 --endpoints=$(seq --sep , --format '31.31.%g.1:1' 1 $OSE_LOXILB_SERVERS) >> /dev/null
diff --git a/cicd/tcpsctpperf/rmconfig.sh b/cicd/tcpsctpperf/rmconfig.sh
@@ -3,15 +3,17 @@
 source ../common.sh
 
 disconnect_docker_hosts l3h1 llb1
-disconnect_docker_hosts l3ep1 llb1
-disconnect_docker_hosts l3ep2 llb1
-disconnect_docker_hosts l3ep3 llb1
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+    disconnect_docker_hosts l3ep$i llb1
+done
 
 delete_docker_host llb1
 delete_docker_host l3h1
-delete_docker_host l3ep1
-delete_docker_host l3ep2
-delete_docker_host l3ep3
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+    delete_docker_host l3ep$i
+done
 
 echo "#########################################"
 echo "Deleted testbed"

diff --git a/cicd/tcpsctpperf/validation-iperf b/cicd/tcpsctpperf/validation-iperf
@@ -0,0 +1,35 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM iperf
+}
+
+trap cleanup EXIT
+
+$hexec l3ep1 \
+  iperf -s -p 12865 --reportstyle c \
+  > "${dst}server.csv" \
+  &
+# $! only gives use the pid of sudo.
+
+sleep 1
+
+# --sum-only is not supported with CSV reportstyle.
+$hexec l3h1 \
+  iperf -c 20.20.20.1 -t $time -p 12865 -P $threads --reportstyle c \
+  > "${dst}client.csv"
+
+sudo pkill -SIGTERM iperf
+wait
diff --git a/cicd/tcpsctpperf/validation-iperf3-sctp b/cicd/tcpsctpperf/validation-iperf3-sctp
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM iperf3
+}
+
+trap cleanup EXIT
+
+$hexec l3ep1 \
+    iperf3 --server -p 13866 --json \
+    > "${dst}server.json" \
+    &
+# $! only gives use the pid of sudo.
+
+sleep 2
+
+$hexec l3h1 \
+    iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --sctp --json \
+    > "${dst}client.json"
+
+sudo pkill -SIGTERM iperf3
+wait
diff --git a/cicd/tcpsctpperf/validation-iperf3-tcp b/cicd/tcpsctpperf/validation-iperf3-tcp
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM iperf3
+}
+
+trap cleanup EXIT
+
+$hexec l3ep1 \
+    iperf3 --server -p 12865 --json \
+    > "${dst}server.json" \
+    &
+# $! only gives use the pid of sudo.
+
+sleep 1
+
+$hexec l3h1 \
+    iperf3 -c 20.20.20.1 -t $time -p 12865 -P $threads --json \
+    > "${dst}client.json"
+
+sudo pkill -SIGTERM iperf3
+wait
diff --git a/cicd/tcpsctpperf/validation-netperf b/cicd/tcpsctpperf/validation-netperf
@@ -0,0 +1,52 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM netserver
+    sudo pkill -SIGTERM netperf
+}
+
+trap cleanup EXIT
+trap cleanup SIGINT
+
+# Must be for TCP for this script.
+export OSE_NETPERF_TEST=${OSE_NETPERF_TEST:-TCP_CRR}
+export OSE_LATENCY_REQ_PAYLOAD_SIZE=${OSE_LATENCY_PAYLOAD_SIZE:-1}
+export OSE_LATENCY_PAYLOAD_SIZE=${OSE_LATENCY_PAYLOAD_SIZE:-1024}
+
+# Run as nobody to prevent /dev/null corruption
+# (https://github.com/HewlettPackard/netperf/issues/26).
+$hexec l3ep1 \
+    sudo -u nobody \
+    netserver -D -4 -p 12865 \
+    &
+# $! only gives use the pid of sudo.
+
+sleep 1 # await netserver, avoid 'could not establish control connection'
+
+for ((i=0,tport=12866;i<threads;i++,tport++))
+do
+    $hexec l3h1 \
+        netperf -L 10.10.10.1 -H 20.20.20.1 -t $OSE_NETPERF_TEST -l $time \
+        -- -P ,$tport -r ${OSE_LATENCY_REQ_PAYLOAD_SIZE},$OSE_LATENCY_PAYLOAD_SIZE \
+        > ${dst}client.$i.log \
+        &
+done
+
+sleep 10 # startup overhead
+sleep $time
+
+# netperf terminates by itself with -l.
+sudo pkill -SIGTERM netserver
+wait
diff --git a/cicd/tcpsctpperf/validation-wrk b/cicd/tcpsctpperf/validation-wrk
@@ -0,0 +1,82 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM nginx
+}
+
+trap cleanup EXIT
+
+OSE_NGINX_PAYLOAD=${OSE_NGINX_PAYLOAD:-1024}
+OSE_WRK_CONNECTIONS=${OSE_WRK_CONNECTIONS:-100}
+OSE_WRK_RATE=${OSE_WRK_RATE:-2000}
+
+# https://github.com/loxilb-io/loxilb/issues/449
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+    tmpd=$(mktemp --directory --suffix=-$(basename $0)-$i)
+    mkdir -p $tmpd/www
+    echo "Hello World from $i" > $tmpd/www/hello-world.txt
+    head --bytes=${OSE_LATENCY_PAYLOAD_SIZE} /dev/urandom > $tmpd/www/head-urandom
+
+    echo "
+worker_processes 1;
+daemon off;  # run in foreground
+events {}
+pid nginx.pid;
+error_log /dev/stderr;
+http {
+    access_log off;
+    client_body_temp_path .;
+    proxy_temp_path .;
+    fastcgi_temp_path .;
+    uwsgi_temp_path .;
+    scgi_temp_path .;
+    server {
+        server_name localhost;
+        listen 14000;
+        location / {
+            root www;
+        }
+    }
+}
+" > $tmpd/nginx.conf
+
+    $hexec l3ep$i \
+        nginx -p $tmpd -c nginx.conf \
+        > ${dst}nginx.log \
+        &
+done
+
+sleep $OSE_LOXILB_SERVERS
+
+set +e
+url=http://20.20.20.1:14000/hello-world.txt
+$hexec l3h1 \
+    curl $url
+
+url=http://20.20.20.1:14000/head-urandom
+$hexec l3h1 \
+    curl $url | wc --bytes
+
+$hexec l3h1 \
+    wrk --threads $threads --duration ${time}s --connections $OSE_WRK_CONNECTIONS --rate $OSE_WRK_RATE --latency $url \
+    > ${dst}latency.log
+ec=$?
+set -e
+
+sudo pkill -SIGTERM nginx
+wait
+
+exit $ec