From 80f433e69442f898357fa1668f7e81072b7626c9 Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Thu, 16 Nov 2023 18:17:44 +0100
Subject: [PATCH 01/10] cicd/common.sh: Support bullseye

---
 cicd/common.sh | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/cicd/common.sh b/cicd/common.sh
index 0a7635952..ce40e744d 100644
--- a/cicd/common.sh
+++ b/cicd/common.sh
@@ -181,9 +181,8 @@ get_llb_peerIP() {
 ## Deletes a docker host
 ## arg1 - hostname 
 delete_docker_host() {
-  id=`docker ps -f name=$1| grep -w $1 | cut  -d " "  -f 1 | grep -iv  "CONTAINER"`
-  if [ "$id" != "" ]; then
-    docker stop $1 2>&1 >> /dev/null
+  if docker stop $1 2>&1 >> /dev/null
+  then
     hd="true"
     ka=`docker ps -f name=ka_$1| grep -w ka_$1 | cut  -d " "  -f 1 | grep -iv  "CONTAINER"`
     loxilbs=( "${loxilbs[@]/$1}" )
@@ -196,9 +195,7 @@ delete_docker_host() {
     $hns del $1
     sudo rm -fr "$hexist/$1" 2>&1 >> /dev/null
   fi
-  if [ "$id" != "" ]; then
-    docker rm $1 2>&1 >> /dev/null
-  fi
+  docker rm $1 2>&1 >> /dev/null || true
 }
 
 ## Connects two docker hosts

From 72710b0b9831869347a331a7d2a3fe71ef1fc611 Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Fri, 17 Nov 2023 13:38:46 +0100
Subject: [PATCH 02/10] cicd/tcpsctpperf: set -eo pipefail to ease debugging

Not adding to rmconfig.sh, because it can also be used for cleanup
before running config.sh and then it is expected to fail.

set -u would also be nice, but it requires more modifications to the
scripts.
---
 cicd/tcpsctpperf/config.sh     | 1 +
 cicd/tcpsctpperf/validation.sh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/cicd/tcpsctpperf/config.sh b/cicd/tcpsctpperf/config.sh
index 6c2eabefc..370cde0af 100755
--- a/cicd/tcpsctpperf/config.sh
+++ b/cicd/tcpsctpperf/config.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -eo pipefail
 
 source ../common.sh
 
diff --git a/cicd/tcpsctpperf/validation.sh b/cicd/tcpsctpperf/validation.sh
index 84d670781..7617d3c90 100755
--- a/cicd/tcpsctpperf/validation.sh
+++ b/cicd/tcpsctpperf/validation.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -eo pipefail
 source ../common.sh
 if [ -z "$1" ]; then
     threads=50

From e87ee43cbaa62ea72fe1c8289c96eb68a318514c Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Wed, 22 Nov 2023 19:01:44 +0100
Subject: [PATCH 03/10] cicd/tcpsctpperf += validation-iperf/iperf3

---
 cicd/tcpsctpperf/validation-iperf  | 24 ++++++++++++++++++++++++
 cicd/tcpsctpperf/validation-iperf3 | 24 ++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100755 cicd/tcpsctpperf/validation-iperf
 create mode 100755 cicd/tcpsctpperf/validation-iperf3

diff --git a/cicd/tcpsctpperf/validation-iperf b/cicd/tcpsctpperf/validation-iperf
new file mode 100755
index 000000000..5b5e7b050
--- /dev/null
+++ b/cicd/tcpsctpperf/validation-iperf
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+$hexec l3ep1 \
+  iperf -s -p 12865 \
+  > "${dst}server.log" 2>&1 &
+# $! only gives use the pid of sudo.
+
+$hexec l3h1 \
+  iperf -c 20.20.20.1 -t $time -p 12865 -P $threads \
+  2>&1 | tee "${dst}client.log"
+
+sudo pkill -SIGTERM iperf
+wait
+cat "${dst}server.log"
diff --git a/cicd/tcpsctpperf/validation-iperf3 b/cicd/tcpsctpperf/validation-iperf3
new file mode 100755
index 000000000..3df370fe9
--- /dev/null
+++ b/cicd/tcpsctpperf/validation-iperf3
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+$hexec l3ep1 \
+    iperf3 --server -p 13866 \
+    > "${dst}server.log" 2>&1 &
+# $! only gives use the pid of sudo.
+
+$hexec l3h1 \
+    iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --sctp \
+    2>&1 | tee "${dst}client.log"
+
+sudo pkill -SIGTERM iperf3
+wait
+cat "${dst}server.log"

From a569c22fde41157419abea20361503ba73ed342d Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Fri, 24 Nov 2023 16:37:53 +0100
Subject: [PATCH 04/10] tcpsctpperf/iperf: CSV output

---
 cicd/tcpsctpperf/validation-iperf | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/cicd/tcpsctpperf/validation-iperf b/cicd/tcpsctpperf/validation-iperf
index 5b5e7b050..293603780 100755
--- a/cicd/tcpsctpperf/validation-iperf
+++ b/cicd/tcpsctpperf/validation-iperf
@@ -11,14 +11,15 @@ time=$2
 dst="$3"
 
 $hexec l3ep1 \
-  iperf -s -p 12865 \
-  > "${dst}server.log" 2>&1 &
+  iperf -s -p 12865 --reportstyle c \
+  > "${dst}server.csv" \
+  &
 # $! only gives use the pid of sudo.
 
+# --sum-only is not supported with CSV reportstyle.
 $hexec l3h1 \
-  iperf -c 20.20.20.1 -t $time -p 12865 -P $threads \
-  2>&1 | tee "${dst}client.log"
+  iperf -c 20.20.20.1 -t $time -p 12865 -P $threads --reportstyle c \
+  > "${dst}client.csv"
 
 sudo pkill -SIGTERM iperf
 wait
-cat "${dst}server.log"

From 9cf6b22f82201a8e13e3e0269ac5d87b3bfd9c84 Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Tue, 28 Nov 2023 15:29:06 +0100
Subject: [PATCH 05/10] tcpsctpperf/iperf3: tcp/sctp validation, json

---
 ...lidation-iperf3 => validation-iperf3-sctp} | 10 ++++----
 cicd/tcpsctpperf/validation-iperf3-tcp        | 24 +++++++++++++++++++
 2 files changed, 29 insertions(+), 5 deletions(-)
 rename cicd/tcpsctpperf/{validation-iperf3 => validation-iperf3-sctp} (52%)
 create mode 100755 cicd/tcpsctpperf/validation-iperf3-tcp

diff --git a/cicd/tcpsctpperf/validation-iperf3 b/cicd/tcpsctpperf/validation-iperf3-sctp
similarity index 52%
rename from cicd/tcpsctpperf/validation-iperf3
rename to cicd/tcpsctpperf/validation-iperf3-sctp
index 3df370fe9..7683d1f46 100755
--- a/cicd/tcpsctpperf/validation-iperf3
+++ b/cicd/tcpsctpperf/validation-iperf3-sctp
@@ -11,14 +11,14 @@ time=$2
 dst="$3"
 
 $hexec l3ep1 \
-    iperf3 --server -p 13866 \
-    > "${dst}server.log" 2>&1 &
+    iperf3 --server -p 13866 --json \
+    > "${dst}server.json" \
+    &
 # $! only gives use the pid of sudo.
 
 $hexec l3h1 \
-    iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --sctp \
-    2>&1 | tee "${dst}client.log"
+    iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --sctp --json \
+    > "${dst}client.json"
 
 sudo pkill -SIGTERM iperf3
 wait
-cat "${dst}server.log"
diff --git a/cicd/tcpsctpperf/validation-iperf3-tcp b/cicd/tcpsctpperf/validation-iperf3-tcp
new file mode 100755
index 000000000..9d6f92543
--- /dev/null
+++ b/cicd/tcpsctpperf/validation-iperf3-tcp
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+$hexec l3ep1 \
+    iperf3 --server -p 13866 --json \
+    > "${dst}server.json" \
+    &
+# $! only gives use the pid of sudo.
+
+$hexec l3h1 \
+    iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --json \
+    > "${dst}client.json"
+
+sudo pkill -SIGTERM iperf3
+wait

From a4d5ee065d12156ce572ce8572743a3e6344dabd Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Thu, 30 Nov 2023 18:48:05 +0100
Subject: [PATCH 06/10] tcpsctpperf += validation-netperf

---
 cicd/tcpsctpperf/validation-netperf | 43 +++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100755 cicd/tcpsctpperf/validation-netperf

diff --git a/cicd/tcpsctpperf/validation-netperf b/cicd/tcpsctpperf/validation-netperf
new file mode 100755
index 000000000..505b3ecb1
--- /dev/null
+++ b/cicd/tcpsctpperf/validation-netperf
@@ -0,0 +1,43 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+# TODO: Call this from bench-loxilb suite, parse netperf.*.log in tidy.py
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+# Must be for TCP for this script.
+export OSE_NETPERF_TEST=${OSE_NETPERF_TEST:-TCP_CRR}
+
+$hexec l3ep1 \
+    netserver -4 -p 12865 \
+    &
+# $! only gives use the pid of sudo.
+
+sleep 2 # await netserver, avoid 'could not establish control connection'
+
+for ((i=0,tport=12866;i<threads;i++,tport++))
+do
+    $hexec l3h1 \
+        netperf -L 10.10.10.1 -H 20.20.20.1 -t $OSE_NETPERF_TEST -l $time \
+        -- -P ,$tport \
+        > ${dst}client.$i.log \
+        &
+done
+
+sleep 10 # startup overhead
+sleep $time
+
+# netperf terminates by itself with -l.
+sudo pkill -SIGTERM netserver
+wait
+
+# netserver somehow corrupts /dev/null, so we have to create it again
+# https://github.com/HewlettPackard/netperf/pull/27
+sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3

From c9209ba1e884d3617bc89eb9b4b92a451d505321 Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Mon, 4 Dec 2023 16:06:48 +0100
Subject: [PATCH 07/10] tcpsctpperf += nginx/curl

---
 cicd/tcpsctpperf/validation-iperf3-tcp |  4 +-
 cicd/tcpsctpperf/validation-netperf    |  2 -
 cicd/tcpsctpperf/validation-wrk2       | 52 ++++++++++++++++++++++++++
 3 files changed, 54 insertions(+), 4 deletions(-)
 create mode 100755 cicd/tcpsctpperf/validation-wrk2

diff --git a/cicd/tcpsctpperf/validation-iperf3-tcp b/cicd/tcpsctpperf/validation-iperf3-tcp
index 9d6f92543..4c496251b 100755
--- a/cicd/tcpsctpperf/validation-iperf3-tcp
+++ b/cicd/tcpsctpperf/validation-iperf3-tcp
@@ -11,13 +11,13 @@ time=$2
 dst="$3"
 
 $hexec l3ep1 \
-    iperf3 --server -p 13866 --json \
+    iperf3 --server -p 12865 --json \
     > "${dst}server.json" \
     &
 # $! only gives use the pid of sudo.
 
 $hexec l3h1 \
-    iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --json \
+    iperf3 -c 20.20.20.1 -t $time -p 12865 -P $threads --json \
     > "${dst}client.json"
 
 sudo pkill -SIGTERM iperf3
diff --git a/cicd/tcpsctpperf/validation-netperf b/cicd/tcpsctpperf/validation-netperf
index 505b3ecb1..e065ecb27 100755
--- a/cicd/tcpsctpperf/validation-netperf
+++ b/cicd/tcpsctpperf/validation-netperf
@@ -4,8 +4,6 @@ set -x
 
 . "$(dirname $0)/../common.sh"
 
-# TODO: Call this from bench-loxilb suite, parse netperf.*.log in tidy.py
-
 set -u
 
 threads=$1
diff --git a/cicd/tcpsctpperf/validation-wrk2 b/cicd/tcpsctpperf/validation-wrk2
new file mode 100755
index 000000000..14008e8e0
--- /dev/null
+++ b/cicd/tcpsctpperf/validation-wrk2
@@ -0,0 +1,52 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+OSE_NGINX_PAYLOAD=${OSE_NGINX_PAYLOAD:-1024}
+
+tmpd=$(mktemp --directory --suffix=$(basename $0))
+mkdir -p $tmpd/www
+echo "Hello World" > $tmpd/www/hello-world.txt
+head --bytes=${OSE_NGINX_PAYLOAD} /dev/urandom > $tmpd/www/head-urandom
+
+echo "
+daemon off;  # run in foreground
+events {}
+pid nginx.pid;
+error_log /dev/stderr;
+http {
+    access_log /dev/stdout;
+    client_body_temp_path .;
+    proxy_temp_path .;
+    fastcgi_temp_path .;
+    uwsgi_temp_path .;
+    scgi_temp_path .;
+    server {
+        server_name localhost;
+        listen 12865;
+        location / {
+            root www;
+        }
+    }
+}
+" > $tmpd/nginx.conf
+
+$hexec l3ep1 \
+    nginx -p $tmpd -c nginx.conf \
+    > ${dst}nginx.log \
+    &
+# $! only gives use the pid of sudo.
+
+$hexec l3h1 \
+    curl http://20.20.20.1:12865/hello-world.txt
+
+sudo kill -SIGTERM $(cat $tmpd/nginx.pid)
+wait

From 137eb95046a0cf251a54d4100c3d32d81cf841b6 Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Tue, 5 Dec 2023 16:14:42 +0100
Subject: [PATCH 08/10] tcpsctpperf: wrk

---
 cicd/tcpsctpperf/config.sh              | 25 ++++----
 cicd/tcpsctpperf/validation-iperf       |  8 +++
 cicd/tcpsctpperf/validation-iperf3-sctp |  8 +++
 cicd/tcpsctpperf/validation-iperf3-tcp  |  8 +++
 cicd/tcpsctpperf/validation-netperf     | 16 +++--
 cicd/tcpsctpperf/validation-wrk         | 77 +++++++++++++++++++++++++
 cicd/tcpsctpperf/validation-wrk2        | 52 -----------------
 7 files changed, 125 insertions(+), 69 deletions(-)
 create mode 100755 cicd/tcpsctpperf/validation-wrk
 delete mode 100755 cicd/tcpsctpperf/validation-wrk2

diff --git a/cicd/tcpsctpperf/config.sh b/cicd/tcpsctpperf/config.sh
index 370cde0af..10750b841 100755
--- a/cicd/tcpsctpperf/config.sh
+++ b/cicd/tcpsctpperf/config.sh
@@ -10,8 +10,8 @@ echo "#########################################"
 spawn_docker_host --dock-type loxilb --dock-name llb1
 spawn_docker_host --dock-type host --dock-name l3h1
 spawn_docker_host --dock-type host --dock-name l3ep1
-spawn_docker_host --dock-type host --dock-name l3ep2
-spawn_docker_host --dock-type host --dock-name l3ep3
+# spawn_docker_host --dock-type host --dock-name l3ep2
+# spawn_docker_host --dock-type host --dock-name l3ep3
 
 echo "#########################################"
 echo "Connecting and configuring  hosts"
@@ -20,35 +20,34 @@ echo "#########################################"
 
 connect_docker_hosts l3h1 llb1
 connect_docker_hosts l3ep1 llb1
-connect_docker_hosts l3ep2 llb1
-connect_docker_hosts l3ep3 llb1
+# connect_docker_hosts l3ep2 llb1
+# connect_docker_hosts l3ep3 llb1
 
-sleep 5
+sleep 1
 
 #L3 config
 config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254
 config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254
-config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254
-config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254
+# config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254
+# config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254
 config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24
 config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24
-config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24
-config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24
+# config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24
+# config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24
 
-sleep 5
+sleep 1
 
 #Need to do this as netperf sctp doesn't work without this
 $hexec l3h1 ifconfig eth0 0
 $hexec l3ep1 ifconfig eth0 0
-$hexec l3ep2 ifconfig eth0 0
-$hexec l3ep3 ifconfig eth0 0
+# $hexec l3ep2 ifconfig eth0 0
+# $hexec l3ep3 ifconfig eth0 0
 
 for ((i=1,port=12865;i<=100;i++,port++))
 do
   $dexec llb1 loxicmd create lb 20.20.20.1 --tcp=$port:$port  --endpoints=31.31.31.1:1 >> /dev/null
 done
 
-$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=13866:13866  --endpoints=31.31.31.1:1 >> /dev/null
 for ((i=1,port=13866;i<=100;i++,port++))
 do
   $dexec llb1 loxicmd create lb 20.20.20.1 --sctp=$port:$port  --endpoints=31.31.31.1:1 >> /dev/null
diff --git a/cicd/tcpsctpperf/validation-iperf b/cicd/tcpsctpperf/validation-iperf
index 293603780..df8cf3341 100755
--- a/cicd/tcpsctpperf/validation-iperf
+++ b/cicd/tcpsctpperf/validation-iperf
@@ -10,6 +10,14 @@ threads=$1
 time=$2
 dst="$3"
 
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM iperf
+}
+
+trap cleanup EXIT
+
 $hexec l3ep1 \
   iperf -s -p 12865 --reportstyle c \
   > "${dst}server.csv" \
diff --git a/cicd/tcpsctpperf/validation-iperf3-sctp b/cicd/tcpsctpperf/validation-iperf3-sctp
index 7683d1f46..c0207a3f7 100755
--- a/cicd/tcpsctpperf/validation-iperf3-sctp
+++ b/cicd/tcpsctpperf/validation-iperf3-sctp
@@ -10,6 +10,14 @@ threads=$1
 time=$2
 dst="$3"
 
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM iperf3
+}
+
+trap cleanup EXIT
+
 $hexec l3ep1 \
     iperf3 --server -p 13866 --json \
     > "${dst}server.json" \
diff --git a/cicd/tcpsctpperf/validation-iperf3-tcp b/cicd/tcpsctpperf/validation-iperf3-tcp
index 4c496251b..911d94e1e 100755
--- a/cicd/tcpsctpperf/validation-iperf3-tcp
+++ b/cicd/tcpsctpperf/validation-iperf3-tcp
@@ -10,6 +10,14 @@ threads=$1
 time=$2
 dst="$3"
 
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM iperf3
+}
+
+trap cleanup EXIT
+
 $hexec l3ep1 \
     iperf3 --server -p 12865 --json \
     > "${dst}server.json" \
diff --git a/cicd/tcpsctpperf/validation-netperf b/cicd/tcpsctpperf/validation-netperf
index e065ecb27..43b07ab9d 100755
--- a/cicd/tcpsctpperf/validation-netperf
+++ b/cicd/tcpsctpperf/validation-netperf
@@ -10,6 +10,18 @@ threads=$1
 time=$2
 dst="$3"
 
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM netserver netperf
+
+    # netserver somehow corrupts /dev/null, so we have to create it again
+    # https://github.com/HewlettPackard/netperf/pull/27
+    sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3
+}
+
+trap cleanup EXIT
+
 # Must be for TCP for this script.
 export OSE_NETPERF_TEST=${OSE_NETPERF_TEST:-TCP_CRR}
 
@@ -35,7 +47,3 @@ sleep $time
 # netperf terminates by itself with -l.
 sudo pkill -SIGTERM netserver
 wait
-
-# netserver somehow corrupts /dev/null, so we have to create it again
-# https://github.com/HewlettPackard/netperf/pull/27
-sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3
diff --git a/cicd/tcpsctpperf/validation-wrk b/cicd/tcpsctpperf/validation-wrk
new file mode 100755
index 000000000..72a67c8d4
--- /dev/null
+++ b/cicd/tcpsctpperf/validation-wrk
@@ -0,0 +1,77 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+. "$(dirname $0)/../common.sh"
+
+set -u
+
+threads=$1
+time=$2
+dst="$3"
+
+cleanup() {
+    set +e
+
+    sudo pkill -SIGTERM nginx
+}
+
+trap cleanup EXIT
+
+OSE_NGINX_PAYLOAD=${OSE_NGINX_PAYLOAD:-1024}
+OSE_WRK_CONNECTIONS=${OSE_WRK_CONNECTIONS:-100}
+OSE_WRK_RATE=${OSE_WRK_RATE:-2000}
+
+# https://github.com/loxilb-io/loxilb/issues/449
+for i in $(seq 1 1)
+do
+    tmpd=$(mktemp --directory --suffix=-$(basename $0)-$i)
+    mkdir -p $tmpd/www
+    echo "Hello World from $i" > $tmpd/www/hello-world.txt
+    head --bytes=${OSE_NGINX_PAYLOAD} /dev/urandom > $tmpd/www/head-urandom
+
+    echo "
+worker_processes $threads;
+daemon off;  # run in foreground
+events {}
+pid nginx.pid;
+error_log /dev/stderr;
+http {
+    access_log off;
+    client_body_temp_path .;
+    proxy_temp_path .;
+    fastcgi_temp_path .;
+    uwsgi_temp_path .;
+    scgi_temp_path .;
+    server {
+        server_name localhost;
+        listen 12865;
+        location / {
+            root www;
+        }
+    }
+}
+" > $tmpd/nginx.conf
+
+    $hexec l3ep$i \
+        nginx -p $tmpd -c nginx.conf \
+        > ${dst}nginx.log \
+        &
+done
+
+set +e
+url=http://20.20.20.1:12865/head-urandom
+
+$hexec l3h1 \
+    curl $url | wc --bytes
+
+$hexec l3h1 \
+    wrk --threads $threads --duration ${time}s --connections $OSE_WRK_CONNECTIONS --rate $OSE_WRK_RATE --latency $url \
+    > ${dst}latency.log
+ec=$?
+set -e
+
+sudo pkill -SIGTERM nginx
+wait
+
+exit $ec
diff --git a/cicd/tcpsctpperf/validation-wrk2 b/cicd/tcpsctpperf/validation-wrk2
deleted file mode 100755
index 14008e8e0..000000000
--- a/cicd/tcpsctpperf/validation-wrk2
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-set -x
-
-. "$(dirname $0)/../common.sh"
-
-set -u
-
-threads=$1
-time=$2
-dst="$3"
-
-OSE_NGINX_PAYLOAD=${OSE_NGINX_PAYLOAD:-1024}
-
-tmpd=$(mktemp --directory --suffix=$(basename $0))
-mkdir -p $tmpd/www
-echo "Hello World" > $tmpd/www/hello-world.txt
-head --bytes=${OSE_NGINX_PAYLOAD} /dev/urandom > $tmpd/www/head-urandom
-
-echo "
-daemon off;  # run in foreground
-events {}
-pid nginx.pid;
-error_log /dev/stderr;
-http {
-    access_log /dev/stdout;
-    client_body_temp_path .;
-    proxy_temp_path .;
-    fastcgi_temp_path .;
-    uwsgi_temp_path .;
-    scgi_temp_path .;
-    server {
-        server_name localhost;
-        listen 12865;
-        location / {
-            root www;
-        }
-    }
-}
-" > $tmpd/nginx.conf
-
-$hexec l3ep1 \
-    nginx -p $tmpd -c nginx.conf \
-    > ${dst}nginx.log \
-    &
-# $! only gives use the pid of sudo.
-
-$hexec l3h1 \
-    curl http://20.20.20.1:12865/hello-world.txt
-
-sudo kill -SIGTERM $(cat $tmpd/nginx.pid)
-wait

From 070835203cf5604e0be21f8e77594f8a5d3e681d Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Thu, 7 Dec 2023 19:36:39 +0100
Subject: [PATCH 09/10] tcpsctpperf: OSE_LOXILB_SERVERS

---
 cicd/tcpsctpperf/config.sh              | 49 +++++++++++++++----------
 cicd/tcpsctpperf/rmconfig.sh            | 14 ++++---
 cicd/tcpsctpperf/validation-iperf       |  2 +
 cicd/tcpsctpperf/validation-iperf3-sctp |  2 +
 cicd/tcpsctpperf/validation-iperf3-tcp  |  2 +
 cicd/tcpsctpperf/validation-netperf     |  7 +++-
 cicd/tcpsctpperf/validation-wrk         | 15 +++++---
 7 files changed, 58 insertions(+), 33 deletions(-)

diff --git a/cicd/tcpsctpperf/config.sh b/cicd/tcpsctpperf/config.sh
index 10750b841..09c2bcefc 100755
--- a/cicd/tcpsctpperf/config.sh
+++ b/cicd/tcpsctpperf/config.sh
@@ -9,46 +9,55 @@ echo "#########################################"
 
 spawn_docker_host --dock-type loxilb --dock-name llb1
 spawn_docker_host --dock-type host --dock-name l3h1
-spawn_docker_host --dock-type host --dock-name l3ep1
-# spawn_docker_host --dock-type host --dock-name l3ep2
-# spawn_docker_host --dock-type host --dock-name l3ep3
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  spawn_docker_host --dock-type host --dock-name l3ep$i
+done
 
 echo "#########################################"
 echo "Connecting and configuring  hosts"
 echo "#########################################"
 
-
 connect_docker_hosts l3h1 llb1
-connect_docker_hosts l3ep1 llb1
-# connect_docker_hosts l3ep2 llb1
-# connect_docker_hosts l3ep3 llb1
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  connect_docker_hosts l3ep$i llb1
+done
 
 sleep 1
 
-#L3 config
+# L3 config
 config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254
-config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254
-# config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254
-# config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  config_docker_host --host1 l3ep$i --host2 llb1 --ptype phy --addr 31.31.$i.1/24 --gw 31.31.$i.254
+done
 config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24
-config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24
-# config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24
-# config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  config_docker_host --host1 llb1 --host2 l3ep$i --ptype phy --addr 31.31.$i.254/24
+done
 
 sleep 1
 
-#Need to do this as netperf sctp doesn't work without this
+# Need to do this as netperf sctp doesn't work without this
 $hexec l3h1 ifconfig eth0 0
-$hexec l3ep1 ifconfig eth0 0
-# $hexec l3ep2 ifconfig eth0 0
-# $hexec l3ep3 ifconfig eth0 0
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+  $hexec l3ep$i ifconfig eth0 0
+done
 
 for ((i=1,port=12865;i<=100;i++,port++))
 do
-  $dexec llb1 loxicmd create lb 20.20.20.1 --tcp=$port:$port  --endpoints=31.31.31.1:1 >> /dev/null
+  $dexec llb1 loxicmd create lb 20.20.20.1 --tcp=$port:$port --endpoints=31.31.1.1:1 >> /dev/null
 done
 
+# iperf3 --sctp will use tcp:13866 for control data, and sctp:13866 for the
+# benchmark data.
+$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=13866:13866 --endpoints=31.31.1.1:1 >> /dev/null
 for ((i=1,port=13866;i<=100;i++,port++))
 do
-  $dexec llb1 loxicmd create lb 20.20.20.1 --sctp=$port:$port  --endpoints=31.31.31.1:1 >> /dev/null
+  $dexec llb1 loxicmd create lb 20.20.20.1 --sctp=$port:$port --endpoints=31.31.1.1:1 >> /dev/null
 done
+
+$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=14000:14000 --endpoints=$(seq --sep , --format '31.31.%g.1:1' 1 $OSE_LOXILB_SERVERS) >> /dev/null
diff --git a/cicd/tcpsctpperf/rmconfig.sh b/cicd/tcpsctpperf/rmconfig.sh
index 9ea80577f..647c1c2d6 100755
--- a/cicd/tcpsctpperf/rmconfig.sh
+++ b/cicd/tcpsctpperf/rmconfig.sh
@@ -3,15 +3,17 @@
 source ../common.sh
 
 disconnect_docker_hosts l3h1 llb1
-disconnect_docker_hosts l3ep1 llb1
-disconnect_docker_hosts l3ep2 llb1
-disconnect_docker_hosts l3ep3 llb1
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+    disconnect_docker_hosts l3ep$i llb1
+done
 
 delete_docker_host llb1
 delete_docker_host l3h1
-delete_docker_host l3ep1
-delete_docker_host l3ep2
-delete_docker_host l3ep3
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
+do
+    delete_docker_host l3ep$i
+done
 
 echo "#########################################"
 echo "Deleted testbed"
diff --git a/cicd/tcpsctpperf/validation-iperf b/cicd/tcpsctpperf/validation-iperf
index df8cf3341..3c0665c8d 100755
--- a/cicd/tcpsctpperf/validation-iperf
+++ b/cicd/tcpsctpperf/validation-iperf
@@ -24,6 +24,8 @@ $hexec l3ep1 \
   &
 # $! only gives use the pid of sudo.
 
+sleep 1
+
 # --sum-only is not supported with CSV reportstyle.
 $hexec l3h1 \
   iperf -c 20.20.20.1 -t $time -p 12865 -P $threads --reportstyle c \
diff --git a/cicd/tcpsctpperf/validation-iperf3-sctp b/cicd/tcpsctpperf/validation-iperf3-sctp
index c0207a3f7..409b3af7c 100755
--- a/cicd/tcpsctpperf/validation-iperf3-sctp
+++ b/cicd/tcpsctpperf/validation-iperf3-sctp
@@ -24,6 +24,8 @@ $hexec l3ep1 \
     &
 # $! only gives use the pid of sudo.
 
+sleep 2
+
 $hexec l3h1 \
     iperf3 -c 20.20.20.1 -t $time -p 13866 -P $threads --sctp --json \
     > "${dst}client.json"
diff --git a/cicd/tcpsctpperf/validation-iperf3-tcp b/cicd/tcpsctpperf/validation-iperf3-tcp
index 911d94e1e..1354b3a83 100755
--- a/cicd/tcpsctpperf/validation-iperf3-tcp
+++ b/cicd/tcpsctpperf/validation-iperf3-tcp
@@ -24,6 +24,8 @@ $hexec l3ep1 \
     &
 # $! only gives use the pid of sudo.
 
+sleep 1
+
 $hexec l3h1 \
     iperf3 -c 20.20.20.1 -t $time -p 12865 -P $threads --json \
     > "${dst}client.json"
diff --git a/cicd/tcpsctpperf/validation-netperf b/cicd/tcpsctpperf/validation-netperf
index 43b07ab9d..332f27947 100755
--- a/cicd/tcpsctpperf/validation-netperf
+++ b/cicd/tcpsctpperf/validation-netperf
@@ -21,22 +21,25 @@ cleanup() {
 }
 
 trap cleanup EXIT
+trap cleanup SIGINT
 
 # Must be for TCP for this script.
 export OSE_NETPERF_TEST=${OSE_NETPERF_TEST:-TCP_CRR}
+export OSE_LATENCY_REQ_PAYLOAD_SIZE=${OSE_LATENCY_PAYLOAD_SIZE:-1}
+export OSE_LATENCY_PAYLOAD_SIZE=${OSE_LATENCY_PAYLOAD_SIZE:-1024}
 
 $hexec l3ep1 \
     netserver -4 -p 12865 \
     &
 # $! only gives use the pid of sudo.
 
-sleep 2 # await netserver, avoid 'could not establish control connection'
+sleep 1 # await netserver, avoid 'could not establish control connection'
 
 for ((i=0,tport=12866;i<threads;i++,tport++))
 do
     $hexec l3h1 \
         netperf -L 10.10.10.1 -H 20.20.20.1 -t $OSE_NETPERF_TEST -l $time \
-        -- -P ,$tport \
+        -- -P ,$tport -r ${OSE_LATENCY_REQ_PAYLOAD_SIZE},$OSE_LATENCY_PAYLOAD_SIZE \
         > ${dst}client.$i.log \
         &
 done
diff --git a/cicd/tcpsctpperf/validation-wrk b/cicd/tcpsctpperf/validation-wrk
index 72a67c8d4..71b70c5d2 100755
--- a/cicd/tcpsctpperf/validation-wrk
+++ b/cicd/tcpsctpperf/validation-wrk
@@ -23,15 +23,15 @@ OSE_WRK_CONNECTIONS=${OSE_WRK_CONNECTIONS:-100}
 OSE_WRK_RATE=${OSE_WRK_RATE:-2000}
 
 # https://github.com/loxilb-io/loxilb/issues/449
-for i in $(seq 1 1)
+for i in $(seq 1 $OSE_LOXILB_SERVERS)
 do
     tmpd=$(mktemp --directory --suffix=-$(basename $0)-$i)
     mkdir -p $tmpd/www
     echo "Hello World from $i" > $tmpd/www/hello-world.txt
-    head --bytes=${OSE_NGINX_PAYLOAD} /dev/urandom > $tmpd/www/head-urandom
+    head --bytes=${OSE_LATENCY_PAYLOAD_SIZE} /dev/urandom > $tmpd/www/head-urandom
 
     echo "
-worker_processes $threads;
+worker_processes 1;
 daemon off;  # run in foreground
 events {}
 pid nginx.pid;
@@ -45,7 +45,7 @@ http {
     scgi_temp_path .;
     server {
         server_name localhost;
-        listen 12865;
+        listen 14000;
         location / {
             root www;
         }
@@ -59,9 +59,14 @@ http {
         &
 done
 
+sleep $OSE_LOXILB_SERVERS
+
 set +e
-url=http://20.20.20.1:12865/head-urandom
+url=http://20.20.20.1:14000/hello-world.txt
+$hexec l3h1 \
+    curl $url
 
+url=http://20.20.20.1:14000/head-urandom
 $hexec l3h1 \
     curl $url | wc --bytes
 

From 10aa0c4a9b3126656e677199640b8e1bbfa6789b Mon Sep 17 00:00:00 2001
From: Luis Gerhorst <gerhorst@cs.fau.de>
Date: Fri, 8 Dec 2023 17:40:15 +0100
Subject: [PATCH 10/10] netperf: Run as nobody to not corrupt /dev/null

---
 cicd/tcpsctpperf/validation-netperf | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cicd/tcpsctpperf/validation-netperf b/cicd/tcpsctpperf/validation-netperf
index 332f27947..57cf6b718 100755
--- a/cicd/tcpsctpperf/validation-netperf
+++ b/cicd/tcpsctpperf/validation-netperf
@@ -13,11 +13,8 @@ dst="$3"
 cleanup() {
     set +e
 
-    sudo pkill -SIGTERM netserver netperf
-
-    # netserver somehow corrupts /dev/null, so we have to create it again
-    # https://github.com/HewlettPackard/netperf/pull/27
-    sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3
+    sudo pkill -SIGTERM netserver
+    sudo pkill -SIGTERM netperf
 }
 
 trap cleanup EXIT
@@ -28,8 +25,11 @@ export OSE_NETPERF_TEST=${OSE_NETPERF_TEST:-TCP_CRR}
 export OSE_LATENCY_REQ_PAYLOAD_SIZE=${OSE_LATENCY_PAYLOAD_SIZE:-1}
 export OSE_LATENCY_PAYLOAD_SIZE=${OSE_LATENCY_PAYLOAD_SIZE:-1024}
 
+# Run as nobody to prevent /dev/null corruption
+# (https://github.com/HewlettPackard/netperf/issues/26).
 $hexec l3ep1 \
-    netserver -4 -p 12865 \
+    sudo -u nobody \
+    netserver -D -4 -p 12865 \
     &
 # $! only gives use the pid of sudo.