Merge pull request #196 from gwsystems/test-driver-improvements

feat: better test driver and more resilient tests

.github/workflows/main.yaml

@@ -76,10 +76,19 @@ jobs:
             fonts-dejavu \
             fonts-cascadia-code \
             fonts-roboto \
+            gnuplot \
             imagemagick \
             netpbm \
             pango1.0-tools \
             wamerican
+      - name: Cache Cargo
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            ./awsm/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('./awsm/Cargo.lock') }}
       - name: Compile sledge
         run: |
           make build 
@@ -92,6 +101,79 @@ jobs:
           LD_LIBRARY_PATH="$(pwd)/lib:$LD_LIBRARY_PATH"
           echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
           make build-validate
-      - name: Compile sample apps and run tests
+      # TODO:Cache assets before being copied to ./runtime/bin
+      - name: Cache gocr
+        uses: actions/cache@v2
+        with:
+          path: ./runtime/bin/gocr_wasm.so
+          key: ${{ runner.os }}-gocr2-${{ hashFiles('./runtime/tests/gocr') }}
+        if: success() || failure()
+      - name: Hyde
         run: |
-          ./test.sh
+          ./test.sh ocr_hyde
+        if: success() || failure()
+      - name: Handwriting
+        run: |
+          ./test.sh ocr_handwriting
+        if: success() || failure()
+      - name: Five by Eight
+        run: |
+          ./test.sh ocr_fivebyeight
+        if: success() || failure()
+      - name: OCR by Word
+        run: |
+          ./test.sh ocr_by_word
+        if: success() || failure()
+      - name: OCR by Font
+        run: |
+          ./test.sh ocr_by_font
+        if: success() || failure()
+      - name: OCR by DPI
+        run: |
+          ./test.sh ocr_by_dpi
+        if: success() || failure()
+      # TODO:Cache assets before being copied to ./runtime/bin
+      - name: Cache EKF
+        uses: actions/cache@v2
+        with:
+          path: ./runtime/bin/ekf_wasm.so
+          key: ${{ runner.os }}-gocr2-${{ hashFiles('./runtime/tests/TinyEKF') }}
+        if: success() || failure()
+      - name: EKF one iteration
+        run: |
+          ./test.sh ekf_one_iteration
+        if: success() || failure()
+      - name: EKF by Iteration
+        run: |
+          ./test.sh ekf_by_iteration
+        if: success() || failure()
+      # TODO:Cache assets before being copied to ./runtime/bin
+      - name: Image Classification
+        run: |
+          ./test.sh image_classification
+        if: success() || failure()
+      # TODO:Cache assets before being copied to ./runtime/bin
+      - name: Image Resize
+        run: |
+          ./test.sh image_resize
+        if: success() || failure()
+      - name: Image Resize by Resolution
+        run: |
+          ./test.sh image_resize_by_resolution
+        if: success() || failure()
+      - name: License Plate Detection by Plate Count
+        run: |
+          ./test.sh lpd_by_plate_count
+        if: success() || failure()
+      - name: Bimodal
+        run: |
+          ./test.sh bimodal
+        if: success() || failure()
+      - name: Concurrency
+        run: |
+          ./test.sh concurrency
+        if: success() || failure()
+      - name: Payload
+        run: |
+          ./test.sh payload
+        if: success() || failure()

runtime/Makefile

@@ -108,7 +108,7 @@ CFILES += thirdparty/dist/lib/http_parser.o
 JSMNCFLAGS += -DJSMN_STATIC
 JSMNCFLAGS += -DJSMN_STRICT
 
-all: clean runtime
+all: runtime
 
 runtime:
 	@echo "Compiling runtime"

runtime/compiletime/instr.c

@@ -116,56 +116,56 @@ i64_rem(int64_t a, int64_t b)
 uint32_t
 u32_trunc_f32(float f)
 {
-	assert(0 <= f && f <= UINT32_MAX);
+	assert(0 <= f && f <= (float)UINT32_MAX);
 	return (uint32_t)f;
 }
 
 int32_t
 i32_trunc_f32(float f)
 {
-	assert(INT32_MIN <= f && f <= INT32_MAX);
+	assert(INT32_MIN <= f && f <= (float)INT32_MAX);
 	return (int32_t)f;
 }
 
 uint32_t
 u32_trunc_f64(double f)
 {
-	assert(0 <= f && f <= UINT32_MAX);
+	assert(0 <= f && f <= (double)UINT32_MAX);
 	return (uint32_t)f;
 }
 
 int32_t
 i32_trunc_f64(double f)
 {
-	assert(INT32_MIN <= f && f <= INT32_MAX);
+	assert(INT32_MIN <= f && f <= (double)INT32_MAX);
 	return (int32_t)f;
 }
 
 uint64_t
 u64_trunc_f32(float f)
 {
-	assert(0 <= f && f <= UINT64_MAX);
+	assert(0 <= f && f <= (float)UINT64_MAX);
 	return (uint64_t)f;
 }
 
 int64_t
 i64_trunc_f32(float f)
 {
-	assert(INT64_MIN <= f && f <= INT64_MAX);
+	assert(INT64_MIN <= f && f <= (float)INT64_MAX);
 	return (int64_t)f;
 }
 
 uint64_t
 u64_trunc_f64(double f)
 {
-	assert(0 <= f && f <= UINT64_MAX);
+	assert(0 <= f && f <= (double)UINT64_MAX);
 	return (uint64_t)f;
 }
 
 int64_t
 i64_trunc_f64(double f)
 {
-	assert(INT64_MIN <= f && f <= INT64_MAX);
+	assert(INT64_MIN <= f && f <= (double)INT64_MAX);
 	return (int64_t)f;
 }
 

runtime/experiments/applications/imageresize/by_resolution/run.sh

@@ -12,7 +12,7 @@ log="$experiment_directory/log.csv"
 
 if [ "$1" != "-d" ]; then
 	SLEDGE_SANDBOX_PERF_LOG=$log PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" &
-	sleep 1
+	sleep 3
 else
 	echo "Running under gdb"
 fi
@@ -24,28 +24,43 @@ for ((i = 0; i < total_count; i++)); do
 	echo "$i"
 	ext="$RANDOM"
 
-	curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_small.jpg" --output "result_${ext}_small.png" localhost:10000 2> /dev/null 1> /dev/null
-	pixel_differences="$(compare -identify -metric AE "result_${ext}_small.png" expected_result_small.png null: 2>&1 > /dev/null)"
-	if [[ "$pixel_differences" != "0" ]]; then
-		echo "Small FAIL"
-		echo "$pixel_differences pixel differences detected"
-		exit 1
+	# Small
+	if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_small.jpg" --output "result_${ext}_small.png" localhost:10000 2> /dev/null 1> /dev/null; then
+		pixel_differences="$(compare -identify -metric AE "result_${ext}_small.png" expected_result_small.png null: 2>&1 > /dev/null)"
+		echo "Pixel Differences: $pixel_differences"
+		if [[ "$pixel_differences" != "0" ]]; then
+			echo "Small FAIL"
+			echo "$pixel_differences pixel differences detected"
+			exit 1
+		fi
+	else
+		echo "curl failed with ${?}. See man curl for meaning."
 	fi
 
-	curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_medium.jpg" --output "result_${ext}_medium.png" localhost:10001 2> /dev/null 1> /dev/null
-	pixel_differences="$(compare -identify -metric AE "result_${ext}_medium.png" expected_result_medium.png null: 2>&1 > /dev/null)"
-	if [[ "$pixel_differences" != "0" ]]; then
-		echo "Small FAIL"
-		echo "$pixel_differences pixel differences detected"
-		exit 1
+	# Medium
+	if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_medium.jpg" --output "result_${ext}_medium.png" localhost:10001 2> /dev/null 1> /dev/null; then
+		pixel_differences="$(compare -identify -metric AE "result_${ext}_medium.png" expected_result_medium.png null: 2>&1 > /dev/null)"
+		echo "Pixel Differences: $pixel_differences"
+		if [[ "$pixel_differences" != "0" ]]; then
+			echo "Medium FAIL"
+			echo "$pixel_differences pixel differences detected"
+			exit 1
+		fi
+	else
+		echo "curl failed with ${?}. See man curl for meaning."
 	fi
 
-	curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_large.jpg" --output "result_${ext}_large.png" localhost:10002 2> /dev/null 1> /dev/null
-	pixel_differences="$(compare -identify -metric AE "result_${ext}_large.png" expected_result_large.png null: 2>&1 > /dev/null)"
-	if [[ "$pixel_differences" != "0" ]]; then
-		echo "Small FAIL"
-		echo "$pixel_differences pixel differences detected"
-		exit 1
+	# Large
+	if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_large.jpg" --output "result_${ext}_large.png" localhost:10002 2> /dev/null 1> /dev/null; then
+		pixel_differences="$(compare -identify -metric AE "result_${ext}_large.png" expected_result_large.png null: 2>&1 > /dev/null)"
+		echo "Pixel Differences: $pixel_differences"
+		if [[ "$pixel_differences" != "0" ]]; then
+			echo "Large FAIL"
+			echo "$pixel_differences pixel differences detected"
+			exit 1
+		fi
+	else
+		echo "curl failed with ${?}. See man curl for meaning."
 	fi
 
 	success_count=$((success_count + 1))

runtime/experiments/applications/imageresize/test/run.sh

@@ -16,7 +16,7 @@ fi
 
 if [ "$1" != "-d" ]; then
 	PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" &
-	sleep 1
+	sleep 3
 else
 	echo "Running under gdb"
 fi
@@ -27,16 +27,20 @@ total_count=10
 for ((i = 0; i < total_count; i++)); do
 	echo "$i"
 	ext="$RANDOM"
-	curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@flower.jpg" --output "result_$ext.png" localhost:10000 2> /dev/null 1> /dev/null || exit 1
 
-	pixel_differences="$(compare -identify -metric AE "result_$ext.png" expected_result.png null: 2>&1 > /dev/null)"
+	if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@flower.jpg" --output "result_$ext.png" localhost:10000 2> /dev/null 1> /dev/null; then
 
-	if [[ "$pixel_differences" == "0" ]]; then
-		success_count=$((success_count + 1))
+		pixel_differences="$(compare -identify -metric AE "result_$ext.png" expected_result.png null: 2>&1 > /dev/null)"
+
+		if [[ "$pixel_differences" == "0" ]]; then
+			success_count=$((success_count + 1))
+		else
+			echo "FAIL"
+			echo "$pixel_differences pixel differences detected"
+			exit 1
+		fi
 	else
-		echo "FAIL"
-		echo "$pixel_differences pixel differences detected"
-		exit 1
+		echo "curl failed with ${?}. See man curl for meaning."
 	fi
 done
 

runtime/experiments/applications/ocr/by_dpi/run.sh

@@ -28,7 +28,6 @@ dpi_to_port[144]=10002
 total_count=100
 
 for ((i = 0; i < total_count; i++)); do
-	echo "$i"
 	words="$(shuf -n"$word_count" /usr/share/dict/american-english)"
 
 	for dpi in "${dpis[@]}"; do

runtime/experiments/applications/ocr/by_font/run.sh

@@ -22,7 +22,6 @@ fonts=("DejaVu Sans Mono" "Roboto" "Cascadia Code")
 total_count=10
 
 for ((i = 1; i <= total_count; i++)); do
-	echo "Test $i"
 	words="$(shuf -n"$word_count" /usr/share/dict/american-english)"
 
 	for font in "${fonts[@]}"; do

runtime/experiments/applications/ocr/by_word/run.sh

@@ -27,8 +27,6 @@ word_count_to_port["100_words.pnm"]=10002
 total_count=100
 
 for ((i = 0; i < total_count; i++)); do
-	echo "$i"
-
 	for word_count in "${word_counts[@]}"; do
 		echo "${word_count}"_words.pnm
 		words="$(shuf -n"$word_count" /usr/share/dict/american-english)"
@@ -37,10 +35,12 @@ for ((i = 0; i < total_count; i++)); do
 
 		result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary @"${word_count}"_words.pnm localhost:${word_count_to_port["$word_count"_words.pnm]} 2> /dev/null)
 
+		# If the OCR does not produce a guess, fail
+		[[ -z "$result" ]] && exit 1
+
 		diff -ywBZE --suppress-common-lines <(echo "$words") <(echo "$result")
 		echo "==============================================="
 	done
-
 done
 
 if [ "$1" != "-d" ]; then

runtime/experiments/applications/ocr/fivebyeight/run.sh

@@ -20,11 +20,26 @@ fi
 
 expected_result="$(cat ./expected_result.txt)"
 
+# Retry 5 times in case the runtime startup is slow
+retries=5
+for ((i = 0; i < retries; i++)); do
+	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null)
+	if [[ "$result" == "$expected_result" ]]; then
+		break
+	fi
+
+	if ((i == 4)); then
+		echo "Retries exhaused"
+		exit 1
+	fi
+
+	sleep 1
+done
+
 success_count=0
 total_count=50
 
 for ((i = 0; i < total_count; i++)); do
-	echo "$i"
 	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null)
 	# echo "$result"
 	if [[ "$result" == "$expected_result" ]]; then

runtime/experiments/applications/ocr/handwriting/run.sh

@@ -19,11 +19,27 @@ else
 fi
 
 expected_result="$(cat ./expected_result.txt)"
+
+# Retry 5 times in case the runtime startup is slow
+retries=5
+for ((i = 0; i < retries; i++)); do
+	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null)
+	if [[ "$result" == "$expected_result" ]]; then
+		break
+	fi
+
+	if ((i == 4)); then
+		echo "Retries exhaused"
+		exit 1
+	fi
+
+	sleep 1
+done
+
 success_count=0
 total_count=50
 
 for ((i = 0; i < total_count; i++)); do
-	echo "$i"
 	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null)
 	# echo "$result"
 	if [[ "$result" == "$expected_result" ]]; then

runtime/experiments/applications/ocr/hyde/run.sh

@@ -18,11 +18,27 @@ else
 fi
 
 expected_result="$(cat ./expected_result.txt)"
+
+# Retry 5 times in case the runtime startup is slow
+retries=5
+for ((i = 0; i < retries; i++)); do
+	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null)
+	if [[ "$result" == "$expected_result" ]]; then
+		break
+	fi
+
+	if ((i == 4)); then
+		echo "Retries exhaused"
+		exit 1
+	fi
+
+	sleep 1
+done
+
 success_count=0
 total_count=50
 
 for ((i = 0; i < total_count; i++)); do
-	echo "$i"
 	result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null)
 	# echo "$result"
 	if [[ "$result" == "$expected_result" ]]; then

runtime/experiments/bimodal/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if ! command -v hey; then
+	HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64
+	wget $HEY_URL -O hey
+	chmod +x hey
+
+	if [[ $(whoami) == "root" ]]; then
+		mv hey /usr/bin/hey
+	else
+		sudo mv hey /usr/bin/hey
+	fi
+fi

runtime/experiments/bimodal/run.sh

@@ -19,6 +19,11 @@ source get_result_count.sh || exit 1
 source panic.sh || exit 1
 source path_join.sh || exit 1
 
+if ! command -v hey; then
+	echo "hey is not present."
+	exit 1
+fi
+
 # Sends requests until the per-module perf window buffers are full
 # This ensures that Sledge has accurate estimates of execution time
 run_samples() {
@@ -46,13 +51,13 @@ run_samples() {
 	printf "Running Samples: "
 	hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || {
 		printf "[ERR]\n"
-		panic "fib40 samples failed"
+		panic "fib40 samples failed with $?"
 		return 1
 	}
 
 	hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || {
 		printf "[ERR]\n"
-		panic "fib10 samples failed"
+		panic "fib10 samples failed with $?"
 		return 1
 	}
 

runtime/experiments/concurrency/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if ! command -v hey; then
+	HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64
+	wget $HEY_URL -O hey
+	chmod +x hey
+
+	if [[ $(whoami) == "root" ]]; then
+		mv hey /usr/bin/hey
+	else
+		sudo mv hey /usr/bin/hey
+	fi
+fi

runtime/experiments/concurrency/run.sh

@@ -15,6 +15,11 @@ source get_result_count.sh || exit 1
 source panic.sh || exit 1
 source path_join.sh || exit 1
 
+if ! command -v hey; then
+	echo "hey is not present."
+	exit 1
+fi
+
 declare -gi iterations=10000
 declare -ga concurrency=(1 20 40 60 80 100)
 

runtime/experiments/payload/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if ! command -v hey; then
+	HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64
+	wget $HEY_URL -O hey
+	chmod +x hey
+
+	if [[ $(whoami) == "root" ]]; then
+		mv hey /usr/bin/hey
+	else
+		sudo mv hey /usr/bin/hey
+	fi
+fi

runtime/experiments/payload/run.sh

@@ -16,6 +16,11 @@ source "framework.sh" || exit 1
 source "get_result_count.sh" || exit 1
 source "generate_gnuplots.sh" || exit 1
 
+if ! command -v hey; then
+	echo "hey is not present."
+	exit 1
+fi
+
 # Experiment Globals and Setups
 declare -ar payloads=(1024 10240 102400 1048576)
 

runtime/tests/CMSIS_5_NN

@@ -1 +1 @@
-Subproject commit 503fc07b2248dc006cc5409ed964632ebd29352a
+Subproject commit 3507d3d0001db45eedbb65072f5fe7cad8bcb59b

runtime/tests/Makefile

@@ -4,33 +4,42 @@ TESTS=fibonacci empty work work1k work10k work100k work1m forever filesys sockse
 
 TESTSRT=$(TESTS:%=%_rt)
 
-.PHONY: all clean rttests
+.PHONY: all clean rttests tinyekf cifar10 gocr sod
 
 all: rttests tinyekf cifar10 gocr sod
-	@echo "Compilation done!"
+	@echo "Test Compilation done!"
 
 rttests: $(TESTSRT)
 
 clean:
+	@echo "Cleaning Test Applications"
 	@rm -rf ${TMP_DIR}
 	@rm -rf ${SLEDGE_BIN_DIR}/*_wasm.so
+	@make clean -C ./TinyEKF/extras/c/ -f wasm.mk
+	@make clean -C ./CMSIS_5_NN/ -f Makefile
+	@make clean -C ./gocr/src/ -f wasm.mk
+	@make clean -C ./sod/
 
 tinyekf:
-	make clean gps_ekf_fn.so -C ./TinyEKF/extras/c/ -f wasm.mk
-	cp ./TinyEKF/extras/c/gps_ekf_fn.so ${SLEDGE_BIN_DIR}/ekf_wasm.so
+	@echo "Making and Installing tinyekf"
+	@make gps_ekf_fn.so -C ./TinyEKF/extras/c/ -f wasm.mk
+	@cp ./TinyEKF/extras/c/gps_ekf_fn.so ${SLEDGE_BIN_DIR}/ekf_wasm.so
 
 cifar10:
-	make clean cifar10.so -C ./CMSIS_5_NN/ -f Makefile
-	cp ./CMSIS_5_NN/cifar10.so ${SLEDGE_BIN_DIR}/cifar10_wasm.so
+	@echo "Making and Installing cifar10"
+	@make cifar10.so -C ./CMSIS_5_NN/ -f Makefile
+	@cp ./CMSIS_5_NN/cifar10.so ${SLEDGE_BIN_DIR}/cifar10_wasm.so
 
 gocr:
-	make clean gocr.so -C ./gocr/src/ -f wasm.mk
-	cp ./gocr/src/gocr.so ${SLEDGE_BIN_DIR}/gocr_wasm.so
+	@echo "Making and Installing gocr"
+	@make gocr.so -C ./gocr/src/ -f wasm.mk
+	@cp ./gocr/src/gocr.so ${SLEDGE_BIN_DIR}/gocr_wasm.so
 
 sod:
-	make clean dir samples.so -C ./sod/
-	cp ./sod/bin/license_plate_detection.so ${SLEDGE_BIN_DIR}/lpd_wasm.so
-	cp ./sod/bin/resize_image.so ${SLEDGE_BIN_DIR}/resize_wasm.so
+	@echo "Making and Installing license_plate_detection and image_resize"
+	@make dir samples.so -C ./sod/
+	@cp ./sod/bin/license_plate_detection.so ${SLEDGE_BIN_DIR}/lpd_wasm.so
+	@cp ./sod/bin/resize_image.so ${SLEDGE_BIN_DIR}/resize_wasm.so
 
 %_rt:
 	@mkdir -p ${TMP_DIR}

runtime/tests/TinyEKF

@@ -1 +1 @@
-Subproject commit 58857dc898e3e2f4cbfd5d12e31e6e2ad273add4
+Subproject commit 76daf1da0546dd04896f20ae630cdeea0280c095

runtime/tests/gocr

@@ -1 +1 @@
-Subproject commit d74bcdad2d92fe909eccde19ad954c06d9f4c859
+Subproject commit 0976fd036462264f581a588fa6cfa46718c67716

runtime/tests/sockclient/client.c

@@ -1,10 +1,12 @@
 /* code from http://www.cs.rpi.edu/~moorthy/Courses/os98/Pgms/socket.html */
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <netdb.h>
 #include <string.h>
+#include <unistd.h>
 
 void
 error(char *msg)

runtime/tests/sockserver/server.c

@@ -2,8 +2,11 @@
    The port number is passed as an argument */
 /* code from: http://www.cs.rpi.edu/~moorthy/Courses/os98/Pgms/socket.html */
 #include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <unistd.h>
 #include <netinet/in.h>
 
 void

runtime/tests/sod

@@ -1 +1 @@
-Subproject commit 9728147fe396ab7946909ab0155c4c6dd77ea082
+Subproject commit 2f57d46df4074be9374f6021cef955b17702380e

runtime/tests/work/main.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #ifndef MAX_BUF
 #define MAX_BUF (1024 * 1024 * 1) // 1m

runtime/tests/work100k/main.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #define MAX_BUF 102400
 

runtime/tests/work10k/main.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #define MAX_BUF 10240
 

runtime/tests/work1k/main.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #define MAX_BUF 1024
 

runtime/tests/work1m/main.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #define MAX_BUF (1024 * 1024 * 1) // 1m
 

test.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Test Driver Script
+# TODO: Consider refactoring into Make infrastructure
 
 if [[ $0 != "./test.sh" ]]; then
 	echo "Must run in same directory as ./test.sh"
@@ -8,62 +9,231 @@ fi
 
 base_dir=$(pwd)
 
-# Awsm
-cd "$base_dir/awsm" && cargo build --release || exit 1
+declare -ra tests=(
+	ocr_hyde
+	ocr_handwriting
+	ocr_fivebyeight
+	ocr_by_word
+	ocr_by_font
+	ocr_by_dpi
+	ekf_by_iteration
+	ekf_one_iteration
+	image_classification
+	image_resize
+	image_resize_by_resolution
+	lpd_by_plate_count
+	bimodal
+	concurrency
+	payload
+)
 
-# Sledge
-cd "$base_dir/runtime" && make clean all || exit 1
-
-# OCR Build
-# FIXME: gocr incorectly reports up to date
-make clean gocr -B -C "$base_dir/runtime/tests" || exit 1
+declare -a failed_tests=()
 
 # OCR Tests
-# FIXME: OCR tests seem to sporadically fail and then work on rerun. Don't bail on fail for now
-# cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh || exit 1
-cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh
-cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh
-cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh
-cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh
-
-# EKF Build
-make clean tinyekf -B -C "$base_dir/runtime/tests" || exit 1
+# FIXME: OCR tests seem to sporadically fail and then work on rerun.
+ocr_hyde() {
+	# FIXME: This check is a hack because GitHub Actions is caching
+	# the *.so file in the destination file, not the subodule it built from
+	if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then
+		make gocr -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ocr/hyde" || exit 1
+	./run.sh || failed_tests+=("ocr_hyde")
+	popd || exit 1
+	return 0
+}
+
+ocr_handwriting() {
+	if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then
+		make gocr -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ocr/handwriting" || exit 1
+	./run.sh || failed_tests+=("ocr_handwriting")
+	popd || exit 1
+	return 0
+}
+
+ocr_fivebyeight() {
+	if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then
+		make gocr -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" || exit 1
+	./run.sh || failed_tests+=("ocr_fivebyeight")
+	popd || exit 1
+	return 0
+}
+
+ocr_by_word() {
+	if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then
+		make gocr -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ocr/by_word" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_by_word")
+	popd || exit 1
+	return 0
+}
+
+ocr_by_font() {
+	if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then
+		make gocr -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ocr/by_font" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_by_font")
+	popd || exit 1
+	return 0
+}
+
+ocr_by_dpi() {
+	if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then
+		make gocr -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ocr/by_dpi" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("ocr_by_dpi")
+	popd || exit 1
+	return 0
+}
 
 # EKF Tests
-cd "$base_dir/runtime/experiments/applications/ekf/by_iteration" && ./run.sh || exit 1
-cd "$base_dir/runtime/experiments/applications/ekf/one_iteration" && ./run.sh || exit 1
+ekf_by_iteration() {
+	if [[ ! -f "$base_dir/runtime/bin/ekf_wasm.so" ]]; then
+		make tinyekf -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ekf/by_iteration" || exit 1
+	./run.sh || failed_tests+=("ocr_by_dpi")
+	popd || exit 1
+	return 0
+}
 
-# cifar10 Build
-make clean cifar10 -B -C "$base_dir/runtime/tests" || exit 1
+ekf_one_iteration() {
+	if [[ ! -f "$base_dir/runtime/bin/ekf_wasm.so" ]]; then
+		make tinyekf -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/ekf/one_iteration" || exit 1
+	./run.sh || failed_tests+=("ekf_one_iteration")
+	popd || exit 1
+	return 0
+}
 
 # cifar10 Tests
-cd "$base_dir/runtime/experiments/applications/imageclassification" && ./run.sh || exit 1
-
-# sod Build
-make clean sod -B -C "$base_dir/runtime/tests" || exit 1
-
-# sod Tests
-# FIXME: sod sporadically fails. May be related to a file descriptor bug
-# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh || exit 1
-# cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh || exit 1
-cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh
-cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh
-cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh
-cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh
+image_classification() {
+	if [[ ! -f "$base_dir/runtime/bin/cifar10_wasm.so" ]]; then
+		make cifar10 -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/imageclassification" || exit 1
+	./run.sh || failed_tests+=("image_classification")
+	popd || exit 1
+	return 0
+}
+
+image_resize() {
+	if [[ ! -f "$base_dir/runtime/bin/resize_wasm.so" ]]; then
+		make sod -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/imageresize/test" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("image_resize")
+	popd || exit 1
+	return 0
+}
+
+image_resize_by_resolution() {
+	if [[ ! -f "$base_dir/runtime/bin/resize_wasm.so" ]]; then
+		make sod -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("image_resize_resolution")
+	popd || exit 1
+	return 0
+}
+
+lpd_by_plate_count() {
+	if [[ ! -f "$base_dir/runtime/bin/lpd_wasm.so" ]]; then
+		make sod -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" || exit 1
+	./run.sh || failed_tests+=("lpd_by_plate_count")
+	popd || exit 1
+	return 0
+}
+
+bimodal() {
+	echo "Bimodal"
+	if [[ ! -f "$base_dir/runtime/bin/fibonacci_wasm.so" ]]; then
+		make rttests -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/bimodal/" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("bimodal")
+	popd || exit 1
+	return 0
+}
+
+concurrency() {
+	echo "Concurrency"
+	if [[ ! -f "$base_dir/runtime/bin/empty_wasm.so" ]]; then
+		make rttests -C "$base_dir/runtime/tests" || exit 1
+	fi
+	pushd "$base_dir/runtime/experiments/concurrency/" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("concurrency")
+	popd || exit 1
+	return 0
+}
+
+payload() {
+	echo "Payload"
+	if [[ ! -f "$base_dir/runtime/bin/work1k_wasm.so" ]] \
+		|| [[ ! -f "$base_dir/runtime/bin/work10k_wasm.so" ]] \
+		|| [[ ! -f "$base_dir/runtime/bin/work100k_wasm.so" ]] \
+		|| [[ ! -f "$base_dir/runtime/bin/work1m_wasm.so" ]]; then
+		make rttests -C "$base_dir/runtime/tests" || exit 1
+	fi
+	# TODO: Make Dependency "work1k_wasm.so" "work10k_wasm.so" "work100k_wasm.so" "work1m_wasm.so"
+	pushd "$base_dir/runtime/experiments/payload/" || exit 1
+	./install.sh || exit 1
+	./run.sh || failed_tests+=("payload")
+	popd || exit 1
+	return 0
+}
+
+main() {
+	cd "$base_dir/awsm" && cargo build --release || exit 1
+	make all -C "$base_dir/runtime" || exit 1
+
+	if (($# == 0)); then
+		# If no arguments are provided, run all tests
+		for test in "${tests[@]}"; do
+			"$test"
+		done
+
+	else
+		# Otherwise, only run the tests passed as arguments
+		for test in "$@"; do
+			if [[ ! " ${tests[*]} " =~ " ${test} " ]]; then
+				printf "Error: %s is not a known test\n" "$test"
+				return 1
+			else
+				"$test"
+			fi
+		done
+
+	fi
+
+	local -i failure_count=${#failed_tests[@]}
+	if ((failure_count > 0)); then
+		printf "Failed Tests\n"
+		for test in "${failed_tests[@]}"; do
+			printf "\t%s\n" "$test"
+		done
+		exit 1
+	else
+		printf "All tests passed\n"
+		exit 0
+	fi
+}
+
+main "$@"