Merge pull request distributed-system-analysis#491 from ndokos/wip-re…

…cord-iterations

Record iterations

agent/base

@@ -83,7 +83,9 @@ function debug_log {
 # Some standard global vars - try the config file first and fall back on hardwired defaults
 # which are valid today.
 if [[ -z "$_PBENCH_BENCH_TESTS" ]]; then
-    date=`date --utc "+%F_%H:%M:%S"`
+    if [ -z "$date" ] ;then
+        export date=`date --utc "+%F_%H:%M:%S"`
+    fi
     hostname=`hostname -s`
     full_hostname=`hostname`
 else

agent/bench-scripts/gold/pbench-dbench/test-02.txt

@@ -94,6 +94,7 @@ Iteration 2-48thread complete (2 of 2), with 1 pass and 0 failures
 /var/tmp/pbench-test-bench/pbench/dbench__1900-01-01_00:00:00/summary-result.txt
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/dbench__1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/gold/pbench-fio/test-04.txt

@@ -1105,6 +1105,7 @@ Iteration 6-randrw-1024KiB complete (6 of 6), with 1 pass and 6 failures
 /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/summary-result.txt
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/fio__1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/gold/pbench-fio/test-05.txt

@@ -561,6 +561,7 @@ Iteration 3-throughput-1024KiB complete (3 of 3), with 1 pass and 6 failures
 /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/summary-result.txt
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/fio__1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/gold/pbench-fio/test-06.txt

@@ -562,6 +562,7 @@ Iteration 3-throughput-1024KiB complete (3 of 3), with 1 pass and 6 failures
 /var/tmp/pbench-test-bench/pbench/fio__1900-01-01_00:00:00/summary-result.txt
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/fio__1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/gold/pbench-moongen/test-03.txt

@@ -76,6 +76,7 @@ Iteration 4-throughput-unidirec-1024B-1024flows-1pct_drop complete (4 of 4), wit
 /var/tmp/pbench-test-bench/pbench/moongen__1900-01-01_00:00:00/pbench-moongen.cmd
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/moongen__1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/gold/pbench-uperf/test-00.txt

@@ -17,6 +17,7 @@ Iteration 2-tcp_stream-64B-1i complete (2 of 2), with 1 pass and 0 failures
 /var/tmp/pbench-test-bench/pbench
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/uperf_test-00_1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/gold/pbench-uperf/test-01.txt

@@ -25,6 +25,7 @@ Iteration 2-tcp_stream-64B-1i complete (2 of 2), with 1 pass and 0 failures
 /var/tmp/pbench-test-bench/pbench
 /var/tmp/pbench-test-bench/pbench/pbench.log
 /var/tmp/pbench-test-bench/pbench/tmp
+/var/tmp/pbench-test-bench/pbench/tmp/uperf_test-01_1900-01-01_00:00:00.iterations
 /var/tmp/pbench-test-bench/pbench/tools-default
 /var/tmp/pbench-test-bench/pbench/tools-default/mpstat
 /var/tmp/pbench-test-bench/pbench/tools-default/sar

agent/bench-scripts/pbench-cyclictest

@@ -104,7 +104,9 @@ if [[ $? -ne 0 ]]; then
 	exit 1
 fi
 
-benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+benchmark_fullname="${benchmark}_${config}_${date}"
+benchmark_run_dir="$pbench_run/${benchmark_fullname}"
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt"
 benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html"
 if [ ! -z "$cpu" ]; then
@@ -137,6 +139,7 @@ for runtime in `echo $runtimes | sed -e s/,/" "/g`; do
 	if [ ! -z "$cpu" ]; then
 		iteration="$iteration-cpu_$cpu"
 	fi
+        echo $iteration >> $benchmark_iterations
 	benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result/"
 	result_file=$benchmark_results_dir/result.txt
 	benchmark_cmd_file=$benchmark_results_dir/cyclictest.cmd

agent/bench-scripts/pbench-dbench

@@ -240,14 +240,17 @@ while true; do
 done
 if [[ -z "$benchmark_run_dir" ]]; then
 	# We don't have an explicit run directory, construct one
-	benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+	benchmark_fullname="${benchmark}_${config}_${date}"
+	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
 else
 	# We have an explicit run directory provided by --run-dir, so warn
 	# the user if they also used --config
 	if [[ ! -z "$config" ]]; then
 		warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\""
 	fi
+	benchmark_fullname="$(basename $benchmark_run_dir)"
 fi
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 
 if [ -z "$clients" ]; then
 	clients=127.0.0.1
@@ -304,6 +307,7 @@ for thread_count in `echo $thread_counts | sed -e s/,/" "/g`; do
 		echo "Starting iteration $iteration ($count of $total_iterations)"
 		log "Starting iteration $iteration ($count of $total_iterations)"
 		iteration="${count}-${thread_count}thread"
+                echo $iteration >> $benchmark_iterations
 		iteration_dir="$benchmark_run_dir/$iteration"
 		result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
 		failures=0

agent/bench-scripts/pbench-fio

@@ -372,14 +372,17 @@ function fio_process_options() {
 		esac
 	done
 	if [ "$postprocess_only" = "n" ]; then
-		benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+		benchmark_fullname="${benchmark}_${config}_${date}"
+		benchmark_run_dir="$pbench_run/${benchmark_fullname}"
 	else
 		if [ -z "$run_dir" ]; then
-			echo "I need a directory if postprocessing an existing result (--run-dir=)"
-		else
-			benchmark_run_dir="$run_dir"
+			err_log "I need a directory if postprocessing an existing result (--run-dir=)"
+			exit 1
 		fi
+		benchmark_fullname="$(basename $run_dir)"
+		benchmark_run_dir="$run_dir"
 	fi
+	benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 
 	verify_tool_group $tool_group
 }
@@ -638,6 +641,8 @@ function fio_run_benchmark() {
 			for block_size in `echo $block_sizes | sed -e s/,/" "/g`; do
 				job_num=1
 				iteration="${count}-${test_type}-${block_size}KiB"
+
+                                echo $iteration >> $benchmark_iterations
 				iteration_dir=$benchmark_run_dir/$iteration
 				result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
 				failures=0

agent/bench-scripts/pbench-iozone

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 # pbench-iozone 0.01 initial version 
 # TODO
@@ -155,15 +156,17 @@ done
 
 verify_tool_group $tool_group
 
-benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+benchmark_fullname="${benchmark}_${config}_${date}"
+benchmark_run_dir="$pbench_run/${benchmark_fullname}"
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 benchmark_summary_file="$benchmark_run_dir/$benchmark-summary.txt"
 
 mkdir -p $benchmark_run_dir/.running
 export benchmark config
 pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg
 
 benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result"
-
+echo $iteration >> $benchmark_iterations
 benchmark_tools_dir="$benchmark_results_dir/tools-$tool_group"
 mkdir -p $benchmark_results_dir $benchmark_tools_dir
 

agent/bench-scripts/pbench-linpack

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 # This is a script to run the linpack benchmark
 
@@ -106,7 +107,9 @@ else
     exit 1
 fi
 
-benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+benchmark_fullname="${benchmark}_${config}_${date}"
+benchmark_run_dir="$pbench_run/${benchmark_fullname}"
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt"
 benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html"
 
@@ -118,6 +121,7 @@ result_file=$benchmark_results_dir/result.txt
 mkdir -p $benchmark_results_dir
 export benchmark config
 pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg
+echo $iteration >> $benchmark_iterations
 ## Run the benchmark and start/stop perf analysis tools
 pbench-start-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir
 pushd /usr/local/linpack_11.1.3/benchmarks/linpack

agent/bench-scripts/pbench-migrate

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 # This is a script to run the migrate benchmark
 # Author: Andrew Theurer
@@ -309,11 +310,14 @@ function do_migrate() {
 
 if [ "$postprocess" = "only" ]; then
 	# user provides directory for previous test
+	benchmark_fullname=$(basename $benchmark_data_dir)
 	benchmark_run_dir="$benchmark_data_dir"
 else
 	config="from_${src_host}_to_${dest_host}--$config"
-	benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+	benchmark_fullname="${benchmark}_${config}_${date}"
+	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
 fi
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt"
 benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html"
 
@@ -328,6 +332,7 @@ export benchmark config
 pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg
 for i in `seq 1 $loops`; do
 	iteration="$count-from-$src_host-to-$dest_host"
+	echo $iteration >> $benchmark_iterations
 	benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result"
 	if [ "$postprocess" != "only" ]; then
 		mkdir -p $benchmark_results_dir

agent/bench-scripts/pbench-moongen

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 # This is a script to run the moongen benchmark
 # Author: Andrew Theurer
@@ -305,14 +306,17 @@ while true; do
 done
 if [[ -z "$benchmark_run_dir" ]]; then
 	# We don't have an explicit run directory, construct one
-	benchmark_run_dir="$pbench_run/${benchmark_name}_${config}_$date"
+	benchmark_fullname="${benchmark_name}_${config}_${date}"
+	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
 else
 	# We have an explicit run directory provided by --run-dir, so warn
 	# the user if they also used --config
 	if [[ ! -z "$config" ]]; then
 		warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\""
 	fi
+	benchmark_fullname=$(basename $benchmark_run_dir)
 fi
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 mkdir -p $benchmark_run_dir/.running
 # save a copy of the command, in case the test needs to be reproduced or post-processed again
 echo "$script_name $pbench_cmd" >$benchmark_run_dir/$script_name.cmd
@@ -355,6 +359,7 @@ for rate in `echo $rates | sed -e s/,/" "/g`; do
 						else 
 							iteration="${count}-${test_type}-${traffic_direction}-${frame_size}B-${nr_flow}flows-${max_drop_pct}pct_drop"
 						fi
+                                                echo $iteration >> $benchmark_iterations
 						iteration_dir="$benchmark_run_dir/$iteration"
 						result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
 						failures=0

agent/bench-scripts/pbench-netperf

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 # This is a script to run the netperf benchmark
 # Note: This script is under development, and since netperf and uperf have very similar
@@ -51,7 +52,9 @@ spacing=25
 
 # Defaults
 
-benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+benchmark_fullname="${benchmark}_${config}_${date}"
+benchmark_run_dir="$pbench_run/${benchmark_fullname}"
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 protocols="tcp,udp"
 test_types="stream,maerts,bidirec,rr"
 message_sizes="1,64,1024,16384" # in bytes
@@ -410,6 +413,7 @@ for protocol in `echo $protocols | sed -e s/,/" "/g`; do
 			for instance in `echo $instances | sed -e s/,/" "/g`; do
 				if [ $count -ge $start_iteration_num ]; then
 					iteration="${count}-${protocol}_${test_type}-${message_size}B-${instance}i"
+					echo $iteration >> $benchmark_iterations
 					iteration_dir="$benchmark_run_dir/$iteration"
 					result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
 					failures=0

agent/bench-scripts/pbench-specjbb2005

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-
 
 # This is a script to run the specjbb2005 benchmark
 # Author: Andrew Theurer
@@ -158,7 +159,9 @@ if [ "$stop_warehouses" == "default" ]; then
 fi
 ## Ensure the right version of the benchmark is installed
 
-benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+benchmark_fullname="${benchmark}_${config}_${date}"
+benchmark_run_dir="$pbench_run/${benchmark_fullname}"
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 benchmark_summary_txt_file="$benchmark_run_dir/$benchmark-summary.txt"
 benchmark_summary_html_file="$benchmark_run_dir/$benchmark-summary.html"
 benchmark_results_dir="$benchmark_run_dir"
@@ -214,6 +217,7 @@ for i in `seq $start_warehouses $inc_warehouses $stop_warehouses`; do
 	# for each of the test iterations (each test with a specific number of warehouses), copy the raw result data for that iteration in iteraration directory
 	grep "test$i" SPECjbb.raw >$i/reference-result/SPECjbb.raw
 	new_iteration_name="$iteration-warehouses:$i"
+        echo $new_iteration_name >> $benchmark_iterations
 	mv $iteration $new_iteration_name
 	pbench-postprocess-tools --group=$tool_group --iteration=$new_iteration_name --dir=$benchmark_results_dir/$new_iteration_name/reference-result
 	let iteration=$iteration+1

agent/bench-scripts/pbench-uperf

@@ -366,14 +366,17 @@ while true; do
 done
 if [[ -z "$benchmark_run_dir" ]]; then
 	# We don't have an explicit run directory, construct one
-	benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+	benchmark_fullname="${benchmark}_${config}_${date}"
+	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
 else
 	# We have an explicit run directory provided by --run-dir, so warn
 	# the user if they also used --config
 	if [[ ! -z "$config" ]]; then
 		warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\""
 	fi
+	benchmark_fullname=$(basename $benchmark_run_dir)
 fi
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 
 # Verify the tool group exists
 verify_tool_group $tool_group
@@ -471,6 +474,7 @@ for protocol in `echo $protocols | sed -e s/,/" "/g`; do
 					echo "Starting iteration $iteration ($count of $total_iterations)"
 					log "Starting iteration $iteration ($count of $total_iterations)"
 					iteration="${count}-${protocol}_${test_type}-${message_size}B-${instance}i"
+                                        echo $iteration >> $benchmark_iterations
 					iteration_dir="$benchmark_run_dir/$iteration"
 					result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
 					failures=0

agent/bench-scripts/pbench-user-benchmark

@@ -75,14 +75,15 @@ done
 verify_tool_group $tool_group
 iteration="1" # there can be only 1 iteration for user-benchmark
 benchmark_bin="$@"
-benchmark_run_dir="$pbench_run/${benchmark}_${config}_$date"
+benchmark_fullname="${benchmark}_${config}_${date}"
+benchmark_run_dir="$pbench_run/${benchmark_fullname}"
+benchmark_iterations="$pbench_tmp/${benchmark_fullname}.iterations"
 # make the run dir available to the user script
 export benchmark_run_dir
 benchmark_results_dir="$benchmark_run_dir/$iteration/reference-result"
 mkdir -p $benchmark_results_dir/.running
 export benchmark_results_dir=$benchmark_results_dir
 
-
 # For now, since we only ever run one iteration here, don't bother
 # collecting sysinfo data before and after, just do it after.
 # pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir beg
@@ -100,6 +101,7 @@ trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT
 pbench-start-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir
 echo "Running $benchmark_bin"
 log "[$script_name] Running $benchmark_bin"
+echo $iteration >> $benchmark_iterations
 $benchmark_bin 2>&1 | tee $benchmark_results_dir/result.txt
 pbench-stop-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir
 pbench-postprocess-tools --group=$tool_group --iteration=$iteration --dir=$benchmark_results_dir

agent/bench-scripts/test-bin/ssh

@@ -2,10 +2,6 @@
 
 echo "$0 $*" >> $_testlog
 
-if [[ "$4" == "netstat" && "$5" == "-tlpn" ]]; then
-    echo "tcp        0      0 0.0.0.0:21000               0.0.0.0:*                   LISTEN      5830/uperf"
-fi
-
 while true ;do
     case $1 in
         -o)
@@ -21,6 +17,14 @@ while true ;do
 done
 remote=$1
 
+if [[ "$2" == "netstat" && "$3" == "-tlnp" ]]; then
+    echo "tcp        0      0 0.0.0.0:21000               0.0.0.0:*                   LISTEN      5830/uperf"
+fi
+
+if [[ "$2" == "hostname" && "$3" == "-s" ]]; then
+    echo "$remote"
+fi
+
 if [[ "$remote" == "fubar" ]] ;then
     exit 255
 else

agent/util-scripts/gold/pbench-metadata-log/test-01.txt

@@ -15,12 +15,6 @@ config:
 date: 1900-01-01_00:00:00
 rpm-version: 
 
-[tools]
-hosts: testhost 
-
-[tools/testhost]
-mpstat: --interval="10"
-
 [run]
 controller: testhost.example.com
 --- pbench tree state