testall

#!/bin/sh
# vim: filetype=sh
# Sources: MicroC, Professor Stephen A. Edwards, 3/26/2016

# ================================== #
# Regression Testing Script for Stop
# ================================== #

# Step through a list of files
# Compile, run, and check output of each expected-to-work test
# Compile and check the error of each expected-to-fail test

STOP="./stop.native"
TEST_FOLDER="tests"
LLI="lli"

ulimit -t 30

globallog=testall.log
rm -f $globallog
rm -f tests/*.ll
error=0
globalerror=0

keep=0

Usage() {
	echo "Usage: $ ./testall [options] [.stp files]"
	echo "-k	Keep intermediate files"
	echo "-h	Print this help"
	exit 1
}

SignalError() {
	if [ $error -eq 0 ] ; then
	echo "FAILED"
	error=1
	fi
	echo "  $1"
}

# Compare <outfile> <reffile> <difffile>
# Compares the outfile with reffile. Differences, if any, written to difffile
Compare() {
	generatedfiles="$generatedfiles $3"
	echo diff -b $1 $2 ">" $3 1>&2

	diff -b "$1" "$2" > "$3" 2>&1 || {
	SignalError "$1 differs"
	echo "FAILED $1 differs from $2" 1>&2
	}
}

# Run <args>
# Report the command, run it, and report any errors
Run() {
	echo $* 1>&2
	eval $* || {
	SignalError "$1 failed on $*"
	return 1
	}
}

# RunFail <args>
# Report the command, run it, and expect an error
RunFail() {
	echo $* 1>&2
	eval $* && {
	SignalError "failed: $* did not report an error"
	return 1
	}
	return 0
}
	
Check() {
	error=0
	basename=$(echo $1 | sed 's;.stp$;;' | sed 's;^.*/;;')
	reffile=$(echo $1 | sed 's;.stp$;;') 
	basedir="$(dirname "$1")/"

	echo -n "$basename..."

	echo 1>&2
	echo "##### Testing $basename" 1>&2

	generatedfiles=""

	generatedfiles="$generatedfiles ${basename}.ll ${basename}.out" &&
	Run "$STOP" $1 ">" "${basename}.ll" &&
	#lli "${basename}.ll" > "${basename}.out" &&
	Run "lli" "${basename}.ll" ">" "${basename}.out" &&
	Compare ${basename}.out ${reffile}.out ${basename}.diff

	# Report the status and clean up the generated files
	if [ $error -eq 0 ] ; then
	if [ $keep -eq 0 ] ; then
		rm -f $generatedfiles
	fi
	echo "OK"
	echo "##### SUCCESS" 1>&2
	else
	echo "##### FAILED" 1>&2
	globalerror=$error
	fi
}

CheckFail() {
	error=0
	basename=$(echo $1 | sed 's;.stp$;;' | sed 's;^.*/;;')
	reffile=$(echo $1 | sed 's;.stp$;;') 
	basedir="$(dirname "$1")/."

	echo -n "$basename..."

	echo 1>&2
	echo "##### Testing $basename" 1>&2

	generatedfiles=""

	generatedfiles="$generatedfiles ${basename}.err ${basename}.diff" &&
	RunFail "$STOP" "<" $1 ">" "${basename}.err" &&
	Compare ${basename}.err ${reffile}.err ${basename}.diff

	# Report the status and clean up the generated files
	if [ $error -eq 0 ] ; then
	if [ $keep -eq 0 ] ; then
		rm -f $generatedfiles
	fi
	echo "OK"
	echo "##### SUCCESS" 1>&2
	else
	echo "##### FAILED" 1>&2
	globalerror=$error
	fi
}

# Begin Script #
# ============ #

# Respond appropriately to supplied options
while getopts kh opt 
do
	case $opt in
	k) # Keep intermediate files
		keep=1
		;;
	h) # Help
		Usage
		;;
	esac
done

# Remove options, leave arguments
shift $((OPTIND - 1)) 

# Exit script if there is no stop.native executable
if [ ! -f ${STOP} ]; then
	echo "${STOP} not found" 1>&2
	exit 1
fi

# If the user supplies files, test only those files; 
# Otherwise, test all test files
if [ $# -ge 1 ]
then
	files="${TEST_FODLER}/$@"
else 
	files="${TEST_FOLDER}/test-*.stp ${TEST_FOLDER}/fail-*.stp"
fi

# Test the chosen files
for file in $files
do
	case $file in
	*test-*)
		Check $file 2>> $globallog
		;;
	*fail-*)
		# CheckFail $file 2>> $globallog
		;;
	*)
		echo "unkown file type $file"
		globalerror=1
		;;
	esac
done

exit $globalerror