diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e816e1c..a89890a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -76,10 +76,19 @@ jobs: fonts-dejavu \ fonts-cascadia-code \ fonts-roboto \ + gnuplot \ imagemagick \ netpbm \ pango1.0-tools \ wamerican + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ./awsm/target + key: ${{ runner.os }}-cargo-${{ hashFiles('./awsm/Cargo.lock') }} - name: Compile sledge run: | make build @@ -92,6 +101,79 @@ jobs: LD_LIBRARY_PATH="$(pwd)/lib:$LD_LIBRARY_PATH" echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV make build-validate - - name: Compile sample apps and run tests + # TODO:Cache assets before being copied to ./runtime/bin + - name: Cache gocr + uses: actions/cache@v2 + with: + path: ./runtime/bin/gocr_wasm.so + key: ${{ runner.os }}-gocr2-${{ hashFiles('./runtime/tests/gocr') }} + if: success() || failure() + - name: Hyde run: | - ./test.sh + ./test.sh ocr_hyde + if: success() || failure() + - name: Handwriting + run: | + ./test.sh ocr_handwriting + if: success() || failure() + - name: Five by Eight + run: | + ./test.sh ocr_fivebyeight + if: success() || failure() + - name: OCR by Word + run: | + ./test.sh ocr_by_word + if: success() || failure() + - name: OCR by Font + run: | + ./test.sh ocr_by_font + if: success() || failure() + - name: OCR by DPI + run: | + ./test.sh ocr_by_dpi + if: success() || failure() + # TODO:Cache assets before being copied to ./runtime/bin + - name: Cache EKF + uses: actions/cache@v2 + with: + path: ./runtime/bin/ekf_wasm.so + key: ${{ runner.os }}-gocr2-${{ hashFiles('./runtime/tests/TinyEKF') }} + if: success() || failure() + - name: EKF one iteration + run: | + ./test.sh ekf_one_iteration + if: success() || failure() + - name: EKF by Iteration + run: | + ./test.sh ekf_by_iteration + if: success() || failure() + # TODO:Cache assets before being copied to ./runtime/bin + - name: Image Classification + run: | + ./test.sh image_classification + if: success() || failure() + # TODO:Cache assets before being copied to ./runtime/bin + - name: Image Resize + run: | + ./test.sh image_resize + if: success() || failure() + - name: Image Resize by Resolution + run: | + ./test.sh image_resize_by_resolution + if: success() || failure() + - name: License Plate Detection by Plate Count + run: | + ./test.sh lpd_by_plate_count + if: success() || failure() + - name: Bimodal + run: | + ./test.sh bimodal + if: success() || failure() + - name: Concurrency + run: | + ./test.sh concurrency + if: success() || failure() + - name: Payload + run: | + ./test.sh payload + if: success() || failure() diff --git a/runtime/Makefile b/runtime/Makefile index 2d24542..de58895 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -108,7 +108,7 @@ CFILES += thirdparty/dist/lib/http_parser.o JSMNCFLAGS += -DJSMN_STATIC JSMNCFLAGS += -DJSMN_STRICT -all: clean runtime +all: runtime runtime: @echo "Compiling runtime" diff --git a/runtime/compiletime/instr.c b/runtime/compiletime/instr.c index ad1f1de..2664d53 100644 --- a/runtime/compiletime/instr.c +++ b/runtime/compiletime/instr.c @@ -116,56 +116,56 @@ i64_rem(int64_t a, int64_t b) uint32_t u32_trunc_f32(float f) { - assert(0 <= f && f <= UINT32_MAX); + assert(0 <= f && f <= (float)UINT32_MAX); return (uint32_t)f; } int32_t i32_trunc_f32(float f) { - assert(INT32_MIN <= f && f <= INT32_MAX); + assert(INT32_MIN <= f && f <= (float)INT32_MAX); return (int32_t)f; } uint32_t u32_trunc_f64(double f) { - assert(0 <= f && f <= UINT32_MAX); + assert(0 <= f && f <= (double)UINT32_MAX); return (uint32_t)f; } int32_t i32_trunc_f64(double f) { - assert(INT32_MIN <= f && f <= INT32_MAX); + assert(INT32_MIN <= f && f <= (double)INT32_MAX); return (int32_t)f; } uint64_t u64_trunc_f32(float f) { - assert(0 <= f && f <= UINT64_MAX); + assert(0 <= f && f <= (float)UINT64_MAX); return (uint64_t)f; } int64_t i64_trunc_f32(float f) { - assert(INT64_MIN <= f && f <= INT64_MAX); + assert(INT64_MIN <= f && f <= (float)INT64_MAX); return (int64_t)f; } uint64_t u64_trunc_f64(double f) { - assert(0 <= f && f <= UINT64_MAX); + assert(0 <= f && f <= (double)UINT64_MAX); return (uint64_t)f; } int64_t i64_trunc_f64(double f) { - assert(INT64_MIN <= f && f <= INT64_MAX); + assert(INT64_MIN <= f && f <= (double)INT64_MAX); return (int64_t)f; } diff --git a/runtime/experiments/applications/imageresize/by_resolution/run.sh b/runtime/experiments/applications/imageresize/by_resolution/run.sh index b5a3862..3bfa988 100755 --- a/runtime/experiments/applications/imageresize/by_resolution/run.sh +++ b/runtime/experiments/applications/imageresize/by_resolution/run.sh @@ -12,7 +12,7 @@ log="$experiment_directory/log.csv" if [ "$1" != "-d" ]; then SLEDGE_SANDBOX_PERF_LOG=$log PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" & - sleep 1 + sleep 3 else echo "Running under gdb" fi @@ -24,28 +24,43 @@ for ((i = 0; i < total_count; i++)); do echo "$i" ext="$RANDOM" - curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_small.jpg" --output "result_${ext}_small.png" localhost:10000 2> /dev/null 1> /dev/null - pixel_differences="$(compare -identify -metric AE "result_${ext}_small.png" expected_result_small.png null: 2>&1 > /dev/null)" - if [[ "$pixel_differences" != "0" ]]; then - echo "Small FAIL" - echo "$pixel_differences pixel differences detected" - exit 1 + # Small + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_small.jpg" --output "result_${ext}_small.png" localhost:10000 2> /dev/null 1> /dev/null; then + pixel_differences="$(compare -identify -metric AE "result_${ext}_small.png" expected_result_small.png null: 2>&1 > /dev/null)" + echo "Pixel Differences: $pixel_differences" + if [[ "$pixel_differences" != "0" ]]; then + echo "Small FAIL" + echo "$pixel_differences pixel differences detected" + exit 1 + fi + else + echo "curl failed with ${?}. See man curl for meaning." fi - curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_medium.jpg" --output "result_${ext}_medium.png" localhost:10001 2> /dev/null 1> /dev/null - pixel_differences="$(compare -identify -metric AE "result_${ext}_medium.png" expected_result_medium.png null: 2>&1 > /dev/null)" - if [[ "$pixel_differences" != "0" ]]; then - echo "Small FAIL" - echo "$pixel_differences pixel differences detected" - exit 1 + # Medium + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_medium.jpg" --output "result_${ext}_medium.png" localhost:10001 2> /dev/null 1> /dev/null; then + pixel_differences="$(compare -identify -metric AE "result_${ext}_medium.png" expected_result_medium.png null: 2>&1 > /dev/null)" + echo "Pixel Differences: $pixel_differences" + if [[ "$pixel_differences" != "0" ]]; then + echo "Medium FAIL" + echo "$pixel_differences pixel differences detected" + exit 1 + fi + else + echo "curl failed with ${?}. See man curl for meaning." fi - curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_large.jpg" --output "result_${ext}_large.png" localhost:10002 2> /dev/null 1> /dev/null - pixel_differences="$(compare -identify -metric AE "result_${ext}_large.png" expected_result_large.png null: 2>&1 > /dev/null)" - if [[ "$pixel_differences" != "0" ]]; then - echo "Small FAIL" - echo "$pixel_differences pixel differences detected" - exit 1 + # Large + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_large.jpg" --output "result_${ext}_large.png" localhost:10002 2> /dev/null 1> /dev/null; then + pixel_differences="$(compare -identify -metric AE "result_${ext}_large.png" expected_result_large.png null: 2>&1 > /dev/null)" + echo "Pixel Differences: $pixel_differences" + if [[ "$pixel_differences" != "0" ]]; then + echo "Large FAIL" + echo "$pixel_differences pixel differences detected" + exit 1 + fi + else + echo "curl failed with ${?}. See man curl for meaning." fi success_count=$((success_count + 1)) diff --git a/runtime/experiments/applications/imageresize/test/run.sh b/runtime/experiments/applications/imageresize/test/run.sh index 501f344..4fab433 100755 --- a/runtime/experiments/applications/imageresize/test/run.sh +++ b/runtime/experiments/applications/imageresize/test/run.sh @@ -16,7 +16,7 @@ fi if [ "$1" != "-d" ]; then PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" & - sleep 1 + sleep 3 else echo "Running under gdb" fi @@ -27,16 +27,20 @@ total_count=10 for ((i = 0; i < total_count; i++)); do echo "$i" ext="$RANDOM" - curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@flower.jpg" --output "result_$ext.png" localhost:10000 2> /dev/null 1> /dev/null || exit 1 - pixel_differences="$(compare -identify -metric AE "result_$ext.png" expected_result.png null: 2>&1 > /dev/null)" + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@flower.jpg" --output "result_$ext.png" localhost:10000 2> /dev/null 1> /dev/null; then - if [[ "$pixel_differences" == "0" ]]; then - success_count=$((success_count + 1)) + pixel_differences="$(compare -identify -metric AE "result_$ext.png" expected_result.png null: 2>&1 > /dev/null)" + + if [[ "$pixel_differences" == "0" ]]; then + success_count=$((success_count + 1)) + else + echo "FAIL" + echo "$pixel_differences pixel differences detected" + exit 1 + fi else - echo "FAIL" - echo "$pixel_differences pixel differences detected" - exit 1 + echo "curl failed with ${?}. See man curl for meaning." fi done diff --git a/runtime/experiments/applications/ocr/by_dpi/run.sh b/runtime/experiments/applications/ocr/by_dpi/run.sh index 72df78d..7725cee 100755 --- a/runtime/experiments/applications/ocr/by_dpi/run.sh +++ b/runtime/experiments/applications/ocr/by_dpi/run.sh @@ -28,7 +28,6 @@ dpi_to_port[144]=10002 total_count=100 for ((i = 0; i < total_count; i++)); do - echo "$i" words="$(shuf -n"$word_count" /usr/share/dict/american-english)" for dpi in "${dpis[@]}"; do diff --git a/runtime/experiments/applications/ocr/by_font/run.sh b/runtime/experiments/applications/ocr/by_font/run.sh index ca2f8be..f706b27 100755 --- a/runtime/experiments/applications/ocr/by_font/run.sh +++ b/runtime/experiments/applications/ocr/by_font/run.sh @@ -22,7 +22,6 @@ fonts=("DejaVu Sans Mono" "Roboto" "Cascadia Code") total_count=10 for ((i = 1; i <= total_count; i++)); do - echo "Test $i" words="$(shuf -n"$word_count" /usr/share/dict/american-english)" for font in "${fonts[@]}"; do diff --git a/runtime/experiments/applications/ocr/by_word/run.sh b/runtime/experiments/applications/ocr/by_word/run.sh index 9276b0a..317c6fd 100755 --- a/runtime/experiments/applications/ocr/by_word/run.sh +++ b/runtime/experiments/applications/ocr/by_word/run.sh @@ -27,8 +27,6 @@ word_count_to_port["100_words.pnm"]=10002 total_count=100 for ((i = 0; i < total_count; i++)); do - echo "$i" - for word_count in "${word_counts[@]}"; do echo "${word_count}"_words.pnm words="$(shuf -n"$word_count" /usr/share/dict/american-english)" @@ -37,10 +35,12 @@ for ((i = 0; i < total_count; i++)); do result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary @"${word_count}"_words.pnm localhost:${word_count_to_port["$word_count"_words.pnm]} 2> /dev/null) + # If the OCR does not produce a guess, fail + [[ -z "$result" ]] && exit 1 + diff -ywBZE --suppress-common-lines <(echo "$words") <(echo "$result") echo "===============================================" done - done if [ "$1" != "-d" ]; then diff --git a/runtime/experiments/applications/ocr/fivebyeight/run.sh b/runtime/experiments/applications/ocr/fivebyeight/run.sh index a97be2c..96b9c86 100755 --- a/runtime/experiments/applications/ocr/fivebyeight/run.sh +++ b/runtime/experiments/applications/ocr/fivebyeight/run.sh @@ -20,11 +20,26 @@ fi expected_result="$(cat ./expected_result.txt)" +# Retry 5 times in case the runtime startup is slow +retries=5 +for ((i = 0; i < retries; i++)); do + result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null) + if [[ "$result" == "$expected_result" ]]; then + break + fi + + if ((i == 4)); then + echo "Retries exhaused" + exit 1 + fi + + sleep 1 +done + success_count=0 total_count=50 for ((i = 0; i < total_count; i++)); do - echo "$i" result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null) # echo "$result" if [[ "$result" == "$expected_result" ]]; then diff --git a/runtime/experiments/applications/ocr/handwriting/run.sh b/runtime/experiments/applications/ocr/handwriting/run.sh index bc29711..fefe210 100755 --- a/runtime/experiments/applications/ocr/handwriting/run.sh +++ b/runtime/experiments/applications/ocr/handwriting/run.sh @@ -19,11 +19,27 @@ else fi expected_result="$(cat ./expected_result.txt)" + +# Retry 5 times in case the runtime startup is slow +retries=5 +for ((i = 0; i < retries; i++)); do + result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null) + if [[ "$result" == "$expected_result" ]]; then + break + fi + + if ((i == 4)); then + echo "Retries exhaused" + exit 1 + fi + + sleep 1 +done + success_count=0 total_count=50 for ((i = 0; i < total_count; i++)); do - echo "$i" result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null) # echo "$result" if [[ "$result" == "$expected_result" ]]; then diff --git a/runtime/experiments/applications/ocr/hyde/run.sh b/runtime/experiments/applications/ocr/hyde/run.sh index 67b7faa..b059c22 100755 --- a/runtime/experiments/applications/ocr/hyde/run.sh +++ b/runtime/experiments/applications/ocr/hyde/run.sh @@ -18,11 +18,27 @@ else fi expected_result="$(cat ./expected_result.txt)" + +# Retry 5 times in case the runtime startup is slow +retries=5 +for ((i = 0; i < retries; i++)); do + result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null) + if [[ "$result" == "$expected_result" ]]; then + break + fi + + if ((i == 4)); then + echo "Retries exhaused" + exit 1 + fi + + sleep 1 +done + success_count=0 total_count=50 for ((i = 0; i < total_count; i++)); do - echo "$i" result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null) # echo "$result" if [[ "$result" == "$expected_result" ]]; then diff --git a/runtime/experiments/bimodal/install.sh b/runtime/experiments/bimodal/install.sh new file mode 100755 index 0000000..d5ec913 --- /dev/null +++ b/runtime/experiments/bimodal/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if ! command -v hey; then + HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64 + wget $HEY_URL -O hey + chmod +x hey + + if [[ $(whoami) == "root" ]]; then + mv hey /usr/bin/hey + else + sudo mv hey /usr/bin/hey + fi +fi diff --git a/runtime/experiments/bimodal/run.sh b/runtime/experiments/bimodal/run.sh index fa63cb7..a046470 100755 --- a/runtime/experiments/bimodal/run.sh +++ b/runtime/experiments/bimodal/run.sh @@ -19,6 +19,11 @@ source get_result_count.sh || exit 1 source panic.sh || exit 1 source path_join.sh || exit 1 +if ! command -v hey; then + echo "hey is not present." + exit 1 +fi + # Sends requests until the per-module perf window buffers are full # This ensures that Sledge has accurate estimates of execution time run_samples() { @@ -46,13 +51,13 @@ run_samples() { printf "Running Samples: " hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || { printf "[ERR]\n" - panic "fib40 samples failed" + panic "fib40 samples failed with $?" return 1 } hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || { printf "[ERR]\n" - panic "fib10 samples failed" + panic "fib10 samples failed with $?" return 1 } diff --git a/runtime/experiments/concurrency/install.sh b/runtime/experiments/concurrency/install.sh new file mode 100755 index 0000000..d5ec913 --- /dev/null +++ b/runtime/experiments/concurrency/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if ! command -v hey; then + HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64 + wget $HEY_URL -O hey + chmod +x hey + + if [[ $(whoami) == "root" ]]; then + mv hey /usr/bin/hey + else + sudo mv hey /usr/bin/hey + fi +fi diff --git a/runtime/experiments/concurrency/run.sh b/runtime/experiments/concurrency/run.sh index 0f69cd4..1f6b4ae 100755 --- a/runtime/experiments/concurrency/run.sh +++ b/runtime/experiments/concurrency/run.sh @@ -15,6 +15,11 @@ source get_result_count.sh || exit 1 source panic.sh || exit 1 source path_join.sh || exit 1 +if ! command -v hey; then + echo "hey is not present." + exit 1 +fi + declare -gi iterations=10000 declare -ga concurrency=(1 20 40 60 80 100) diff --git a/runtime/experiments/payload/install.sh b/runtime/experiments/payload/install.sh new file mode 100755 index 0000000..d5ec913 --- /dev/null +++ b/runtime/experiments/payload/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if ! command -v hey; then + HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64 + wget $HEY_URL -O hey + chmod +x hey + + if [[ $(whoami) == "root" ]]; then + mv hey /usr/bin/hey + else + sudo mv hey /usr/bin/hey + fi +fi diff --git a/runtime/experiments/payload/run.sh b/runtime/experiments/payload/run.sh index 6d758c7..a389976 100755 --- a/runtime/experiments/payload/run.sh +++ b/runtime/experiments/payload/run.sh @@ -16,6 +16,11 @@ source "framework.sh" || exit 1 source "get_result_count.sh" || exit 1 source "generate_gnuplots.sh" || exit 1 +if ! command -v hey; then + echo "hey is not present." + exit 1 +fi + # Experiment Globals and Setups declare -ar payloads=(1024 10240 102400 1048576) diff --git a/runtime/tests/CMSIS_5_NN b/runtime/tests/CMSIS_5_NN index 503fc07..3507d3d 160000 --- a/runtime/tests/CMSIS_5_NN +++ b/runtime/tests/CMSIS_5_NN @@ -1 +1 @@ -Subproject commit 503fc07b2248dc006cc5409ed964632ebd29352a +Subproject commit 3507d3d0001db45eedbb65072f5fe7cad8bcb59b diff --git a/runtime/tests/Makefile b/runtime/tests/Makefile index 65edc8f..61aa2c0 100644 --- a/runtime/tests/Makefile +++ b/runtime/tests/Makefile @@ -4,33 +4,42 @@ TESTS=fibonacci empty work work1k work10k work100k work1m forever filesys sockse TESTSRT=$(TESTS:%=%_rt) -.PHONY: all clean rttests +.PHONY: all clean rttests tinyekf cifar10 gocr sod all: rttests tinyekf cifar10 gocr sod - @echo "Compilation done!" + @echo "Test Compilation done!" rttests: $(TESTSRT) clean: + @echo "Cleaning Test Applications" @rm -rf ${TMP_DIR} @rm -rf ${SLEDGE_BIN_DIR}/*_wasm.so + @make clean -C ./TinyEKF/extras/c/ -f wasm.mk + @make clean -C ./CMSIS_5_NN/ -f Makefile + @make clean -C ./gocr/src/ -f wasm.mk + @make clean -C ./sod/ tinyekf: - make clean gps_ekf_fn.so -C ./TinyEKF/extras/c/ -f wasm.mk - cp ./TinyEKF/extras/c/gps_ekf_fn.so ${SLEDGE_BIN_DIR}/ekf_wasm.so + @echo "Making and Installing tinyekf" + @make gps_ekf_fn.so -C ./TinyEKF/extras/c/ -f wasm.mk + @cp ./TinyEKF/extras/c/gps_ekf_fn.so ${SLEDGE_BIN_DIR}/ekf_wasm.so cifar10: - make clean cifar10.so -C ./CMSIS_5_NN/ -f Makefile - cp ./CMSIS_5_NN/cifar10.so ${SLEDGE_BIN_DIR}/cifar10_wasm.so + @echo "Making and Installing cifar10" + @make cifar10.so -C ./CMSIS_5_NN/ -f Makefile + @cp ./CMSIS_5_NN/cifar10.so ${SLEDGE_BIN_DIR}/cifar10_wasm.so gocr: - make clean gocr.so -C ./gocr/src/ -f wasm.mk - cp ./gocr/src/gocr.so ${SLEDGE_BIN_DIR}/gocr_wasm.so + @echo "Making and Installing gocr" + @make gocr.so -C ./gocr/src/ -f wasm.mk + @cp ./gocr/src/gocr.so ${SLEDGE_BIN_DIR}/gocr_wasm.so sod: - make clean dir samples.so -C ./sod/ - cp ./sod/bin/license_plate_detection.so ${SLEDGE_BIN_DIR}/lpd_wasm.so - cp ./sod/bin/resize_image.so ${SLEDGE_BIN_DIR}/resize_wasm.so + @echo "Making and Installing license_plate_detection and image_resize" + @make dir samples.so -C ./sod/ + @cp ./sod/bin/license_plate_detection.so ${SLEDGE_BIN_DIR}/lpd_wasm.so + @cp ./sod/bin/resize_image.so ${SLEDGE_BIN_DIR}/resize_wasm.so %_rt: @mkdir -p ${TMP_DIR} diff --git a/runtime/tests/TinyEKF b/runtime/tests/TinyEKF index 58857dc..76daf1d 160000 --- a/runtime/tests/TinyEKF +++ b/runtime/tests/TinyEKF @@ -1 +1 @@ -Subproject commit 58857dc898e3e2f4cbfd5d12e31e6e2ad273add4 +Subproject commit 76daf1da0546dd04896f20ae630cdeea0280c095 diff --git a/runtime/tests/gocr b/runtime/tests/gocr index d74bcda..0976fd0 160000 --- a/runtime/tests/gocr +++ b/runtime/tests/gocr @@ -1 +1 @@ -Subproject commit d74bcdad2d92fe909eccde19ad954c06d9f4c859 +Subproject commit 0976fd036462264f581a588fa6cfa46718c67716 diff --git a/runtime/tests/sockclient/client.c b/runtime/tests/sockclient/client.c index 5ba32df..21230f0 100644 --- a/runtime/tests/sockclient/client.c +++ b/runtime/tests/sockclient/client.c @@ -1,10 +1,12 @@ /* code from http://www.cs.rpi.edu/~moorthy/Courses/os98/Pgms/socket.html */ #include +#include #include #include #include #include #include +#include void error(char *msg) diff --git a/runtime/tests/sockserver/server.c b/runtime/tests/sockserver/server.c index 3733187..0a68b09 100644 --- a/runtime/tests/sockserver/server.c +++ b/runtime/tests/sockserver/server.c @@ -2,8 +2,11 @@ The port number is passed as an argument */ /* code from: http://www.cs.rpi.edu/~moorthy/Courses/os98/Pgms/socket.html */ #include +#include +#include #include #include +#include #include void diff --git a/runtime/tests/sod b/runtime/tests/sod index 9728147..2f57d46 160000 --- a/runtime/tests/sod +++ b/runtime/tests/sod @@ -1 +1 @@ -Subproject commit 9728147fe396ab7946909ab0155c4c6dd77ea082 +Subproject commit 2f57d46df4074be9374f6021cef955b17702380e diff --git a/runtime/tests/work/main.c b/runtime/tests/work/main.c index bf42a16..600dca3 100644 --- a/runtime/tests/work/main.c +++ b/runtime/tests/work/main.c @@ -1,6 +1,7 @@ #include #include #include +#include #ifndef MAX_BUF #define MAX_BUF (1024 * 1024 * 1) // 1m diff --git a/runtime/tests/work100k/main.c b/runtime/tests/work100k/main.c index dd9cfa7..0ed61e3 100644 --- a/runtime/tests/work100k/main.c +++ b/runtime/tests/work100k/main.c @@ -1,6 +1,7 @@ #include #include #include +#include #define MAX_BUF 102400 diff --git a/runtime/tests/work10k/main.c b/runtime/tests/work10k/main.c index f456ccc..63e8978 100644 --- a/runtime/tests/work10k/main.c +++ b/runtime/tests/work10k/main.c @@ -1,6 +1,7 @@ #include #include #include +#include #define MAX_BUF 10240 diff --git a/runtime/tests/work1k/main.c b/runtime/tests/work1k/main.c index 1c76d4d..f6c6c95 100644 --- a/runtime/tests/work1k/main.c +++ b/runtime/tests/work1k/main.c @@ -1,6 +1,7 @@ #include #include #include +#include #define MAX_BUF 1024 diff --git a/runtime/tests/work1m/main.c b/runtime/tests/work1m/main.c index 0c40471..5ba8371 100644 --- a/runtime/tests/work1m/main.c +++ b/runtime/tests/work1m/main.c @@ -1,6 +1,7 @@ #include #include #include +#include #define MAX_BUF (1024 * 1024 * 1) // 1m diff --git a/test.sh b/test.sh index 3418edb..acb0ba6 100755 --- a/test.sh +++ b/test.sh @@ -1,5 +1,6 @@ #!/bin/bash # Test Driver Script +# TODO: Consider refactoring into Make infrastructure if [[ $0 != "./test.sh" ]]; then echo "Must run in same directory as ./test.sh" @@ -8,62 +9,231 @@ fi base_dir=$(pwd) -# Awsm -cd "$base_dir/awsm" && cargo build --release || exit 1 +declare -ra tests=( + ocr_hyde + ocr_handwriting + ocr_fivebyeight + ocr_by_word + ocr_by_font + ocr_by_dpi + ekf_by_iteration + ekf_one_iteration + image_classification + image_resize + image_resize_by_resolution + lpd_by_plate_count + bimodal + concurrency + payload +) -# Sledge -cd "$base_dir/runtime" && make clean all || exit 1 - -# OCR Build -# FIXME: gocr incorectly reports up to date -make clean gocr -B -C "$base_dir/runtime/tests" || exit 1 +declare -a failed_tests=() # OCR Tests -# FIXME: OCR tests seem to sporadically fail and then work on rerun. Don't bail on fail for now -# cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh || exit 1 -cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh -cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh - -# EKF Build -make clean tinyekf -B -C "$base_dir/runtime/tests" || exit 1 +# FIXME: OCR tests seem to sporadically fail and then work on rerun. +ocr_hyde() { + # FIXME: This check is a hack because GitHub Actions is caching + # the *.so file in the destination file, not the subodule it built from + if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then + make gocr -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ocr/hyde" || exit 1 + ./run.sh || failed_tests+=("ocr_hyde") + popd || exit 1 + return 0 +} + +ocr_handwriting() { + if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then + make gocr -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ocr/handwriting" || exit 1 + ./run.sh || failed_tests+=("ocr_handwriting") + popd || exit 1 + return 0 +} + +ocr_fivebyeight() { + if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then + make gocr -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" || exit 1 + ./run.sh || failed_tests+=("ocr_fivebyeight") + popd || exit 1 + return 0 +} + +ocr_by_word() { + if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then + make gocr -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ocr/by_word" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("ocr_by_word") + popd || exit 1 + return 0 +} + +ocr_by_font() { + if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then + make gocr -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ocr/by_font" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("ocr_by_font") + popd || exit 1 + return 0 +} + +ocr_by_dpi() { + if [[ ! -f "$base_dir/runtime/bin/gocr_wasm.so" ]]; then + make gocr -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ocr/by_dpi" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("ocr_by_dpi") + popd || exit 1 + return 0 +} # EKF Tests -cd "$base_dir/runtime/experiments/applications/ekf/by_iteration" && ./run.sh || exit 1 -cd "$base_dir/runtime/experiments/applications/ekf/one_iteration" && ./run.sh || exit 1 +ekf_by_iteration() { + if [[ ! -f "$base_dir/runtime/bin/ekf_wasm.so" ]]; then + make tinyekf -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ekf/by_iteration" || exit 1 + ./run.sh || failed_tests+=("ocr_by_dpi") + popd || exit 1 + return 0 +} -# cifar10 Build -make clean cifar10 -B -C "$base_dir/runtime/tests" || exit 1 +ekf_one_iteration() { + if [[ ! -f "$base_dir/runtime/bin/ekf_wasm.so" ]]; then + make tinyekf -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/ekf/one_iteration" || exit 1 + ./run.sh || failed_tests+=("ekf_one_iteration") + popd || exit 1 + return 0 +} # cifar10 Tests -cd "$base_dir/runtime/experiments/applications/imageclassification" && ./run.sh || exit 1 - -# sod Build -make clean sod -B -C "$base_dir/runtime/tests" || exit 1 - -# sod Tests -# FIXME: sod sporadically fails. May be related to a file descriptor bug -# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh || exit 1 -# cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh || exit 1 -cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh -cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh -cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh -cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh -cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh +image_classification() { + if [[ ! -f "$base_dir/runtime/bin/cifar10_wasm.so" ]]; then + make cifar10 -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/imageclassification" || exit 1 + ./run.sh || failed_tests+=("image_classification") + popd || exit 1 + return 0 +} + +image_resize() { + if [[ ! -f "$base_dir/runtime/bin/resize_wasm.so" ]]; then + make sod -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/imageresize/test" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("image_resize") + popd || exit 1 + return 0 +} + +image_resize_by_resolution() { + if [[ ! -f "$base_dir/runtime/bin/resize_wasm.so" ]]; then + make sod -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("image_resize_resolution") + popd || exit 1 + return 0 +} + +lpd_by_plate_count() { + if [[ ! -f "$base_dir/runtime/bin/lpd_wasm.so" ]]; then + make sod -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" || exit 1 + ./run.sh || failed_tests+=("lpd_by_plate_count") + popd || exit 1 + return 0 +} + +bimodal() { + echo "Bimodal" + if [[ ! -f "$base_dir/runtime/bin/fibonacci_wasm.so" ]]; then + make rttests -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/bimodal/" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("bimodal") + popd || exit 1 + return 0 +} + +concurrency() { + echo "Concurrency" + if [[ ! -f "$base_dir/runtime/bin/empty_wasm.so" ]]; then + make rttests -C "$base_dir/runtime/tests" || exit 1 + fi + pushd "$base_dir/runtime/experiments/concurrency/" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("concurrency") + popd || exit 1 + return 0 +} + +payload() { + echo "Payload" + if [[ ! -f "$base_dir/runtime/bin/work1k_wasm.so" ]] \ + || [[ ! -f "$base_dir/runtime/bin/work10k_wasm.so" ]] \ + || [[ ! -f "$base_dir/runtime/bin/work100k_wasm.so" ]] \ + || [[ ! -f "$base_dir/runtime/bin/work1m_wasm.so" ]]; then + make rttests -C "$base_dir/runtime/tests" || exit 1 + fi + # TODO: Make Dependency "work1k_wasm.so" "work10k_wasm.so" "work100k_wasm.so" "work1m_wasm.so" + pushd "$base_dir/runtime/experiments/payload/" || exit 1 + ./install.sh || exit 1 + ./run.sh || failed_tests+=("payload") + popd || exit 1 + return 0 +} + +main() { + cd "$base_dir/awsm" && cargo build --release || exit 1 + make all -C "$base_dir/runtime" || exit 1 + + if (($# == 0)); then + # If no arguments are provided, run all tests + for test in "${tests[@]}"; do + "$test" + done + + else + # Otherwise, only run the tests passed as arguments + for test in "$@"; do + if [[ ! " ${tests[*]} " =~ " ${test} " ]]; then + printf "Error: %s is not a known test\n" "$test" + return 1 + else + "$test" + fi + done + + fi + + local -i failure_count=${#failed_tests[@]} + if ((failure_count > 0)); then + printf "Failed Tests\n" + for test in "${failed_tests[@]}"; do + printf "\t%s\n" "$test" + done + exit 1 + else + printf "All tests passed\n" + exit 0 + fi +} + +main "$@"