feat: better test driver and more resilient tests

runtime/Makefile

@@ -108,7 +108,7 @@ CFILES += thirdparty/dist/lib/http_parser.o
 JSMNCFLAGS += -DJSMN_STATIC
 JSMNCFLAGS += -DJSMN_STRICT
 
-all: clean runtime
+all: runtime
 
 runtime:
 	@echo "Compiling runtime"

runtime/experiments/applications/imageresize/test/run.sh

@@ -16,7 +16,7 @@ fi
 
 if [ "$1" != "-d" ]; then
 	PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" &
-	sleep 1
+	sleep 2
 else
 	echo "Running under gdb"
 fi

runtime/experiments/applications/ocr/fivebyeight/run.sh

@@ -20,6 +20,22 @@ fi
 
 expected_result="$(cat ./expected_result.txt)"
 
+# Retry 5 times in case the runtime startup is slow
+retries=5
+for ((i = 0; i < retries; i++)); do
+	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null)
+	if [[ "$result" == "$expected_result" ]]; then
+		break
+	fi
+
+	if ((i == 4)); then
+		echo "Retries exhaused"
+		exit 1
+	fi
+
+	sleep 1
+done
+
 success_count=0
 total_count=50
 

runtime/experiments/applications/ocr/handwriting/run.sh

@@ -19,6 +19,23 @@ else
 fi
 
 expected_result="$(cat ./expected_result.txt)"
+
+# Retry 5 times in case the runtime startup is slow
+retries=5
+for ((i = 0; i < retries; i++)); do
+	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null)
+	if [[ "$result" == "$expected_result" ]]; then
+		break
+	fi
+
+	if ((i == 4)); then
+		echo "Retries exhaused"
+		exit 1
+	fi
+
+	sleep 1
+done
+
 success_count=0
 total_count=50
 

runtime/experiments/applications/ocr/hyde/run.sh

@@ -18,6 +18,23 @@ else
 fi
 
 expected_result="$(cat ./expected_result.txt)"
+
+# Retry 5 times in case the runtime startup is slow
+retries=5
+for ((i = 0; i < retries; i++)); do
+	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null)
+	if [[ "$result" == "$expected_result" ]]; then
+		break
+	fi
+
+	if ((i == 4)); then
+		echo "Retries exhaused"
+		exit 1
+	fi
+
+	sleep 1
+done
+
 success_count=0
 total_count=50
 

test.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Test Driver Script
+# TODO: Consider refactoring into Make infrastructure
 
 if [[ $0 != "./test.sh" ]]; then
 	echo "Must run in same directory as ./test.sh"
@@ -8,62 +9,199 @@ fi
 
 base_dir=$(pwd)
 
-# Awsm
+declare -ra tests=(
-cd "$base_dir/awsm" && cargo build --release || exit 1
+	ocr_hyde
-
+	ocr_handwriting
-# Sledge
+	ocr_fivebyeight
-cd "$base_dir/runtime" && make clean all || exit 1
+	ocr_by_word
+	ocr_by_font
+	ocr_by_dpi
+	ekf_by_iteration
+	ekf_one_iteration
+	image_classification
+	image_resize
+	lpd_by_resolution
+	lpd_by_plate_count
+	bimodal
+	concurrency
+	payload
+)
 
-# OCR Build
+declare -a failed_tests=()
-# FIXME: gocr incorectly reports up to date
-make clean gocr -B -C "$base_dir/runtime/tests" || exit 1
 
 # OCR Tests
-# FIXME: OCR tests seem to sporadically fail and then work on rerun. Don't bail on fail for now
+# FIXME: OCR tests seem to sporadically fail and then work on rerun.
-# cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh || exit 1
+ocr_hyde() {
-# cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh || exit 1
+	make gocr -C "$base_dir/runtime/tests" || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh || exit 1
+	pushd "$base_dir/runtime/experiments/applications/ocr/hyde" || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_hyde")
-# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh || exit 1
+	popd || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh || exit 1
+	return 0
-# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh || exit 1
+}
-# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh || exit 1
+
-# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh || exit 1
+ocr_handwriting() {
-cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh
+	make gocr -C "$base_dir/runtime/tests" || exit 1
-cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh
+	pushd "$base_dir/runtime/experiments/applications/ocr/handwriting" || exit 1
-cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh
+	./run.sh || failed_tests+=("ocr_handwriting")
-cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh
+	popd || exit 1
-cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh
+	return 0
-cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh
+}
-cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh
+
-cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh
+ocr_fivebyeight() {
-cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh
+	make gocr -C "$base_dir/runtime/tests" || exit 1
-
+	pushd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" || exit 1
-# EKF Build
+	./run.sh || failed_tests+=("ocr_fivebyeight")
-make clean tinyekf -B -C "$base_dir/runtime/tests" || exit 1
+	popd || exit 1
+	return 0
+}
+
+ocr_by_word() {
+	make gocr -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/ocr/by_word" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_by_word")
+	popd || exit 1
+	return 0
+}
+
+ocr_by_font() {
+	make gocr -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/ocr/by_font" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_by_font")
+	popd || exit 1
+	return 0
+}
+
+ocr_by_dpi() {
+	make gocr -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/ocr/by_dpi" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_by_dpi")
+	popd || exit 1
+	return 0
+}
 
 # EKF Tests
-cd "$base_dir/runtime/experiments/applications/ekf/by_iteration" && ./run.sh || exit 1
+ekf_by_iteration() {
-cd "$base_dir/runtime/experiments/applications/ekf/one_iteration" && ./run.sh || exit 1
+	make tinyekf -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/ekf/by_iteration" || exit 1
+	./run.sh || failed_tests+=("ocr_by_dpi")
+	popd || exit 1
+	return 0
+}
 
-# cifar10 Build
+ekf_one_iteration() {
-make clean cifar10 -B -C "$base_dir/runtime/tests" || exit 1
+	make tinyekf -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/ekf/one_iteration" || exit 1
+	./run.sh || failed_tests+=("ekf_one_iteration")
+	popd || exit 1
+	return 0
+}
 
 # cifar10 Tests
-cd "$base_dir/runtime/experiments/applications/imageclassification" && ./run.sh || exit 1
+image_classification() {
-
+	make cifar10 -C "$base_dir/runtime/tests" || exit 1
-# sod Build
+	pushd "$base_dir/runtime/experiments/applications/imageclassification" || exit 1
-make clean sod -B -C "$base_dir/runtime/tests" || exit 1
+	./run.sh || failed_tests+=("image_classification")
-
+	popd || exit 1
-# sod Tests
+	return 0
-# FIXME: sod sporadically fails. May be related to a file descriptor bug
+}
-# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh || exit 1
+
-# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh || exit 1
+image_resize() {
-# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh || exit 1
+	make sod -C "$base_dir/runtime/tests" || exit 1
-# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh || exit 1
+	pushd "$base_dir/runtime/experiments/applications/imageresize/test" || exit 1
-# cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh || exit 1
+	./install.sh || exit 1
-cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh
+	./run.sh || failed_tests+=("image_resize")
-cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh
+	popd || exit 1
-cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh
+	return 0
-cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh
+}
-cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh
+
+lpd_by_resolution() {
+	# Disabled... Seems to be dumping raw image data
+	return 0
+	make sod -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("lpd_by_resolution")
+	popd || exit 1
+	return 0
+}
+
+lpd_by_plate_count() {
+	# Disabled... Seems to be dumping raw image data
+	return 0
+	make sod -C "$base_dir/runtime/tests" || exit 1
+	pushd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" || exit 1
+	./run.sh || failed_tests+=("lpd_by_plate_count")
+	popd || exit 1
+	return 0
+}
+
+bimodal() {
+	echo "Bimodal"
+	# TODO: Make Dependency "fibonacci_wasm.so"
+	pushd "$base_dir/runtime/experiments/bimodal/" || exit 1
+	./run.sh || failed_tests+=("bimodal")
+	popd || exit 1
+	return 0
+}
+
+concurrency() {
+	echo "Concurrency"
+	# TODO: Make Dependency "empty_wasm.so"
+	pushd "$base_dir/runtime/experiments/concurrency/" || exit 1
+	./run.sh || failed_tests+=("concurrency")
+	popd || exit 1
+	return 0
+}
+
+payload() {
+	echo "Payload"
+	# TODO: Make Dependency "work1k_wasm.so" "work10k_wasm.so" "work100k_wasm.so" "work1m_wasm.so"
+	pushd "$base_dir/runtime/experiments/payload/" || exit 1
+	./run.sh || failed_tests+=("concurrency")
+	popd || exit 1
+	return 0
+}
+
+main() {
+	cd "$base_dir/awsm" && cargo build --release || exit 1
+	make all -C "$base_dir/runtime" || exit 1
+
+	# If no arguments are provided, run all tests
+	if (($# == 0)); then
+		for test in "${tests[@]}"; do
+			"$test"
+		done
+
+		local -i failure_count=${#failed_tests[@]}
+
+		if ((failure_count > 0)); then
+			printf "Failed Tests\n"
+			for test in "${failed_tests[@]}"; do
+				printf "\t%s\n" "$test"
+			done
+			exit 1
+		fi
+
+		exit 0
+	fi
+
+	# Otherwise, only run the tests passed as arguments
+	for test in "$@"; do
+		if [[ ! " ${tests[*]} " =~ " ${test} " ]]; then
+			printf "Error: %s is not a known test\n" "$test"
+		else
+			if "$test"; then
+				printf "%s: Pass\n" "$test"
+			else
+				printf "%s: Fail\n" "$test"
+
+			fi
+		fi
+
+	done
+}
+
+main "$@"