From cc4cc96cc310c1156431cffa4957468633153084 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 21 Apr 2021 18:07:11 +0000 Subject: [PATCH] feat: better test driver and more resilient tests --- runtime/Makefile | 2 +- .../applications/imageresize/test/run.sh | 2 +- .../applications/ocr/fivebyeight/run.sh | 16 ++ .../applications/ocr/handwriting/run.sh | 17 ++ .../experiments/applications/ocr/hyde/run.sh | 17 ++ test.sh | 240 ++++++++++++++---- 6 files changed, 241 insertions(+), 53 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index 2d24542..de58895 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -108,7 +108,7 @@ CFILES += thirdparty/dist/lib/http_parser.o JSMNCFLAGS += -DJSMN_STATIC JSMNCFLAGS += -DJSMN_STRICT -all: clean runtime +all: runtime runtime: @echo "Compiling runtime" diff --git a/runtime/experiments/applications/imageresize/test/run.sh b/runtime/experiments/applications/imageresize/test/run.sh index 501f344..be2390f 100755 --- a/runtime/experiments/applications/imageresize/test/run.sh +++ b/runtime/experiments/applications/imageresize/test/run.sh @@ -16,7 +16,7 @@ fi if [ "$1" != "-d" ]; then PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" & - sleep 1 + sleep 2 else echo "Running under gdb" fi diff --git a/runtime/experiments/applications/ocr/fivebyeight/run.sh b/runtime/experiments/applications/ocr/fivebyeight/run.sh index a97be2c..722ed7f 100755 --- a/runtime/experiments/applications/ocr/fivebyeight/run.sh +++ b/runtime/experiments/applications/ocr/fivebyeight/run.sh @@ -20,6 +20,22 @@ fi expected_result="$(cat ./expected_result.txt)" +# Retry 5 times in case the runtime startup is slow +retries=5 +for ((i = 0; i < retries; i++)); do + result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null) + if [[ "$result" == "$expected_result" ]]; then + break + fi + + if ((i == 4)); then + echo "Retries exhaused" + exit 1 + fi + + sleep 1 +done + success_count=0 total_count=50 diff --git a/runtime/experiments/applications/ocr/handwriting/run.sh b/runtime/experiments/applications/ocr/handwriting/run.sh index bc29711..6a2fa00 100755 --- a/runtime/experiments/applications/ocr/handwriting/run.sh +++ b/runtime/experiments/applications/ocr/handwriting/run.sh @@ -19,6 +19,23 @@ else fi expected_result="$(cat ./expected_result.txt)" + +# Retry 5 times in case the runtime startup is slow +retries=5 +for ((i = 0; i < retries; i++)); do + result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null) + if [[ "$result" == "$expected_result" ]]; then + break + fi + + if ((i == 4)); then + echo "Retries exhaused" + exit 1 + fi + + sleep 1 +done + success_count=0 total_count=50 diff --git a/runtime/experiments/applications/ocr/hyde/run.sh b/runtime/experiments/applications/ocr/hyde/run.sh index 67b7faa..a0c59b7 100755 --- a/runtime/experiments/applications/ocr/hyde/run.sh +++ b/runtime/experiments/applications/ocr/hyde/run.sh @@ -18,6 +18,23 @@ else fi expected_result="$(cat ./expected_result.txt)" + +# Retry 5 times in case the runtime startup is slow +retries=5 +for ((i = 0; i < retries; i++)); do + result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null) + if [[ "$result" == "$expected_result" ]]; then + break + fi + + if ((i == 4)); then + echo "Retries exhaused" + exit 1 + fi + + sleep 1 +done + success_count=0 total_count=50 diff --git a/test.sh b/test.sh index 3418edb..a3f9d79 100755 --- a/test.sh +++ b/test.sh @@ -1,5 +1,6 @@ #!/bin/bash # Test Driver Script +# TODO: Consider refactoring into Make infrastructure if [[ $0 != "./test.sh" ]]; then echo "Must run in same directory as ./test.sh" @@ -8,62 +9,199 @@ fi base_dir=$(pwd) -# Awsm -cd "$base_dir/awsm" && cargo build --release || exit 1 +declare -ra tests=( + ocr_hyde + ocr_handwriting + ocr_fivebyeight + ocr_by_word + ocr_by_font + ocr_by_dpi + ekf_by_iteration + ekf_one_iteration + image_classification + image_resize + lpd_by_resolution + lpd_by_plate_count + bimodal + concurrency + payload +) -# Sledge -cd "$base_dir/runtime" && make clean all || exit 1 - -# OCR Build -# FIXME: gocr incorectly reports up to date -make clean gocr -B -C "$base_dir/runtime/tests" || exit 1 +declare -a failed_tests=() # OCR Tests -# FIXME: OCR tests seem to sporadically fail and then work on rerun. Don't bail on fail for now -# cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh || exit 1 -cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh - -# EKF Build -make clean tinyekf -B -C "$base_dir/runtime/tests" || exit 1 +# FIXME: OCR tests seem to sporadically fail and then work on rerun. +ocr_hyde() { + make gocr -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ocr/hyde" || exit 1 + ./run.sh || failed_tests+=("ocr_hyde") + popd || exit 1 + return 0 +} + +ocr_handwriting() { + make gocr -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ocr/handwriting" || exit 1 + ./run.sh || failed_tests+=("ocr_handwriting") + popd || exit 1 + return 0 +} + +ocr_fivebyeight() { + make gocr -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" || exit 1 + ./run.sh || failed_tests+=("ocr_fivebyeight") + popd || exit 1 + return 0 +} + +ocr_by_word() { + make gocr -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ocr/by_word" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("ocr_by_word") + popd || exit 1 + return 0 +} + +ocr_by_font() { + make gocr -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ocr/by_font" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("ocr_by_font") + popd || exit 1 + return 0 +} + +ocr_by_dpi() { + make gocr -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ocr/by_dpi" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("ocr_by_dpi") + popd || exit 1 + return 0 +} # EKF Tests -cd "$base_dir/runtime/experiments/applications/ekf/by_iteration" && ./run.sh || exit 1 -cd "$base_dir/runtime/experiments/applications/ekf/one_iteration" && ./run.sh || exit 1 +ekf_by_iteration() { + make tinyekf -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ekf/by_iteration" || exit 1 + ./run.sh || failed_tests+=("ocr_by_dpi") + popd || exit 1 + return 0 +} -# cifar10 Build -make clean cifar10 -B -C "$base_dir/runtime/tests" || exit 1 +ekf_one_iteration() { + make tinyekf -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/ekf/one_iteration" || exit 1 + ./run.sh || failed_tests+=("ekf_one_iteration") + popd || exit 1 + return 0 +} # cifar10 Tests -cd "$base_dir/runtime/experiments/applications/imageclassification" && ./run.sh || exit 1 - -# sod Build -make clean sod -B -C "$base_dir/runtime/tests" || exit 1 - -# sod Tests -# FIXME: sod sporadically fails. May be related to a file descriptor bug -# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh || exit 1 -cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh -cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh -cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh -cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh -cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh +image_classification() { + make cifar10 -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/imageclassification" || exit 1 + ./run.sh || failed_tests+=("image_classification") + popd || exit 1 + return 0 +} + +image_resize() { + make sod -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/imageresize/test" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("image_resize") + popd || exit 1 + return 0 +} + +lpd_by_resolution() { + # Disabled... Seems to be dumping raw image data + return 0 + make sod -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("lpd_by_resolution") + popd || exit 1 + return 0 +} + +lpd_by_plate_count() { + # Disabled... Seems to be dumping raw image data + return 0 + make sod -C "$base_dir/runtime/tests" || exit 1 + pushd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" || exit 1 + ./run.sh || failed_tests+=("lpd_by_plate_count") + popd || exit 1 + return 0 +} + +bimodal() { + echo "Bimodal" + # TODO: Make Dependency "fibonacci_wasm.so" + pushd "$base_dir/runtime/experiments/bimodal/" || exit 1 + ./run.sh || failed_tests+=("bimodal") + popd || exit 1 + return 0 +} + +concurrency() { + echo "Concurrency" + # TODO: Make Dependency "empty_wasm.so" + pushd "$base_dir/runtime/experiments/concurrency/" || exit 1 + ./run.sh || failed_tests+=("concurrency") + popd || exit 1 + return 0 +} + +payload() { + echo "Payload" + # TODO: Make Dependency "work1k_wasm.so" "work10k_wasm.so" "work100k_wasm.so" "work1m_wasm.so" + pushd "$base_dir/runtime/experiments/payload/" || exit 1 + ./run.sh || failed_tests+=("concurrency") + popd || exit 1 + return 0 +} + +main() { + cd "$base_dir/awsm" && cargo build --release || exit 1 + make all -C "$base_dir/runtime" || exit 1 + + # If no arguments are provided, run all tests + if (($# == 0)); then + for test in "${tests[@]}"; do + "$test" + done + + local -i failure_count=${#failed_tests[@]} + + if ((failure_count > 0)); then + printf "Failed Tests\n" + for test in "${failed_tests[@]}"; do + printf "\t%s\n" "$test" + done + exit 1 + fi + + exit 0 + fi + + # Otherwise, only run the tests passed as arguments + for test in "$@"; do + if [[ ! " ${tests[*]} " =~ " ${test} " ]]; then + printf "Error: %s is not a known test\n" "$test" + else + if "$test"; then + printf "%s: Pass\n" "$test" + else + printf "%s: Fail\n" "$test" + + fi + fi + + done +} + +main "$@"