diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 842bdf6..abb6e00 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -105,7 +105,7 @@ jobs: if: success() || failure() - name: Hyde run: | - ./test.sh ocr_hyde + make -f test.mk gocr__hyde if: success() || failure() - name: Upload Hyde Logs on Failure uses: actions/upload-artifact@v2 @@ -115,23 +115,23 @@ jobs: path: ./runtime/experiments/applications/ocr/hyde/res/**/ - name: Handwriting run: | - ./test.sh ocr_handwriting + make -f test.mk gocr__handwriting if: success() || failure() - name: Five by Eight run: | - ./test.sh ocr_fivebyeight + make -f test.mk gocr__fivebyeight if: success() || failure() # - name: OCR by Word # run: | - # ./test.sh ocr_by_word + # make -f test.mk gocr__by_word # if: success() || failure() # - name: OCR by Font # run: | - # ./test.sh ocr_by_font + # make -f test.mk gocr__by_font # if: success() || failure() # - name: OCR by DPI # run: | - # ./test.sh ocr_by_dpi + # make -f test.mk gocr__by_dpi # if: success() || failure() # TODO:Cache assets before being copied to ./runtime/bin - name: Cache EKF @@ -142,39 +142,39 @@ jobs: if: success() || failure() - name: EKF one iteration run: | - ./test.sh ekf_one_iteration + make -f test.mk ekf__one_iteration if: success() || failure() # - name: EKF by Iteration # run: | - # ./test.sh ekf_by_iteration + # make -f test.mk ekf__by_iteration # if: success() || failure() # TODO:Cache assets before being copied to ./runtime/bin - name: Image Classification run: | - ./test.sh image_classification + make -f test.mk cifar10__image_classification if: success() || failure() # TODO:Cache assets before being copied to ./runtime/bin - name: Image Resize run: | - ./test.sh image_resize + make -f test.mk sod__image_resize__test if: success() || failure() # - name: Image Resize by Resolution # run: | - # ./test.sh image_resize_by_resolution + # make -f test.mk sod__image_resize__by_resolution # if: success() || failure() # - name: License Plate Detection by Plate Count # run: | - # ./test.sh lpd_by_plate_count + # make -f test.mk sod__lpd_by_plate_count # if: success() || failure() # - name: Bimodal # run: | - # ./test.sh bimodal + # make -f test.mk fibonacci__bimodal # if: success() || failure() # - name: Concurrency # run: | - # ./test.sh concurrency + # make -f test.mk empty__concurrency # if: success() || failure() # - name: Payload # run: | - # ./test.sh payload + # make -f test.mk TODO # if: success() || failure() diff --git a/test.mk b/test.mk new file mode 100644 index 0000000..79fa67f --- /dev/null +++ b/test.mk @@ -0,0 +1,124 @@ +# This Makefile builds a variety of synthetic and real-world applications ported to SLEdge +# +# Some of the tests generate experimental results in the form of charts of *.csv files. Refer to +# the relevant directory for specifics. + +# gocr - GNU Optical Character Recognition +./runtime/bin/gocr_wasm.so: + make gocr.install -C ./runtime/tests + +PHONY: gocr__by_dpi +gocr__by_dpi: ./runtime/bin/gocr_wasm.so + # cd ./runtime/experiments/applications/ocr/by_dpi && ./install.sh + cd ./runtime/experiments/applications/ocr/by_dpi && ./run.sh + +PHONY: gocr__by_font +gocr__by_font: ./runtime/bin/gocr_wasm.so + cd ./runtime/experiments/applications/ocr/by_font && ./run.sh + +PHONY: gocr__by_word +gocr__by_word: ./runtime/bin/gocr_wasm.so + cd ./runtime/experiments/applications/ocr/by_word && ./run.sh + +PHONY: gocr__fivebyeight +gocr__fivebyeight: ./runtime/bin/gocr_wasm.so + cd ./runtime/experiments/applications/ocr/fivebyeight && ./run.sh + +PHONY: gocr__handwriting +gocr__handwriting: ./runtime/bin/gocr_wasm.so + cd ./runtime/experiments/applications/ocr/handwriting && ./run.sh + +PHONY: gocr__hyde +gocr__hyde: ./runtime/bin/gocr_wasm.so + cd ./runtime/experiments/applications/ocr/hyde && ./run.sh + +PHONY: gocr_all +gocr__all: \ + gocr__by_dpi \ + gocr__by_font \ + gocr__by_word \ + gocr__fivebyeight \ + gocr__handwriting \ + gocr__hyde + +# Extended Kalman Filter applied to binary GPS data +./runtime/bin/ekf_wasm.so: + make ekf.install -C ./runtime/tests + +PHONY: ekf__by_iteration +ekf__by_iteration: ./runtime/bin/ekf_wasm.so + cd ./runtime/experiments/applications/ekf/by_iteration && ./run.sh + +PHONY: ekf__one_iteration +ekf__one_iteration: ./runtime/bin/ekf_wasm.so + cd ./runtime/experiments/applications/ekf/one_iteration && ./run.sh + +PHONY: ekf__all +ekf__all: \ + ekf__by_iteration \ + ekf__one_iteration + +# CIFAR10-based Image Classification +./runtime/bin/cifar10_wasm.so: + make cifar10.install -C ./applications + +PHONY: cifar10__image_classification +cifar10__image_classification: ./runtime/bin/cifar10_wasm.so + cd ./runtime/experiments/applications/imageclassification && ./run.sh + +PHONY: cifar10__all +cifar10__all: \ + cifar10__image_classification + +# SOD Computer Vision / ML Applications +# https://sod.pixlab.io/ + +# SOD - Image Resize +./runtime/bin/resize_wasm.so: + make resize_image.install -C ./applications + +# Commented out command installs imagemagick. Requires password for sudo to install +PHONY: sod__image_resize__test +sod__image_resize__test: ./runtime/bin/resize_wasm.so + # cd ./runtime/experiments/applications/imageresize/test && ./install.sh + cd ./runtime/experiments/applications/imageresize/test && ./run.sh + +PHONY: sod__image_resize__by_resolution +sod__image_resize__by_resolution: ./runtime/bin/resize_wasm.so + # cd ./runtime/experiments/applications/imageresize/by_resolution && ./install.sh + cd ./runtime/experiments/applications/imageresize/by_resolution && ./run.sh + +# SOD - License Plate Detection +./runtime/bin/lpd_wasm.so: + make license_plate_detection.install -C ./applications + +PHONY: sod__lpd__by_plate_count +sod__lpd__by_plate_count: ./runtime/bin/lpd_wasm.so + cd ./runtime/experiments/applications/licenseplate/by_plate_count && ./run.sh + +PHONY: sod__all +sod__all: sod__image_resize__test sod__image_resize__by_resolution sod__lpd__by_plate_count + +# Scheduler Experiments with synthetic workloads +./runtime/bin/fibonacci_wasm.so: + make fibonacci.install -C ./applications + +PHONY: fibonacci__bimodal +fibonacci__bimodal: ./runtime/bin/fibonacci_wasm.so + cd ./runtime/experiments/bimodal/ && ./run.sh + +./runtime/bin/empty_wasm.so: + make empty.install -C ./applications + +PHONY: empty__concurrency +empty__concurrency: ./runtime/bin/empty_wasm.so + # ./runtime/experiments/concurrency/ && install.sh + ./runtime/experiments/concurrency/ && run.sh + +all: \ + gocr__all \ + ekf__all \ + cifar10__all \ + sod__all \ + fibonacci__bimodal \ + empty__concurrency diff --git a/test.sh b/test.sh deleted file mode 100755 index 5480f89..0000000 --- a/test.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/bin/bash -# Test Driver Script -# TODO: Consider refactoring into Make infrastructure - -if [[ $0 != "./test.sh" ]]; then - echo "Must run in same directory as ./test.sh" - exit 1 -fi - -base_dir=$(pwd) - -declare -ra tests=( - ocr_hyde - ocr_handwriting - ocr_fivebyeight - ocr_by_word - ocr_by_font - ocr_by_dpi - ekf_by_iteration - ekf_one_iteration - image_classification - image_resize - image_resize_by_resolution - lpd_by_plate_count - bimodal - concurrency -) - -declare -a failed_tests=() - -# OCR Tests -# FIXME: OCR tests seem to sporadically fail and then work on rerun. -ocr_hyde() { - make gocr.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ocr/hyde" || exit 1 - ./run.sh || failed_tests+=("ocr_hyde") - popd || exit 1 - return 0 -} - -ocr_handwriting() { - make gocr.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ocr/handwriting" || exit 1 - ./run.sh || failed_tests+=("ocr_handwriting") - popd || exit 1 - return 0 -} - -ocr_fivebyeight() { - make gocr.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" || exit 1 - ./run.sh || failed_tests+=("ocr_fivebyeight") - popd || exit 1 - return 0 -} - -ocr_by_word() { - make gocr.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ocr/by_word" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("ocr_by_word") - popd || exit 1 - return 0 -} - -ocr_by_font() { - make gocr.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ocr/by_font" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("ocr_by_font") - popd || exit 1 - return 0 -} - -ocr_by_dpi() { - make gocr.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ocr/by_dpi" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("ocr_by_dpi") - popd || exit 1 - return 0 -} - -# EKF Tests -ekf_by_iteration() { - make ekf.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ekf/by_iteration" || exit 1 - ./run.sh || failed_tests+=("ekf_by_iteration") - popd || exit 1 - return 0 -} - -ekf_one_iteration() { - make ekf.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/ekf/one_iteration" || exit 1 - ./run.sh || failed_tests+=("ekf_one_iteration") - popd || exit 1 - return 0 -} - -# cifar10 Tests -image_classification() { - make cifar10.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/imageclassification" || exit 1 - ./run.sh || failed_tests+=("image_classification") - popd || exit 1 - return 0 -} - -image_resize() { - make resize.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/imageresize/test" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("image_resize") - popd || exit 1 - return 0 -} - -image_resize_by_resolution() { - make resize.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("image_resize_resolution") - popd || exit 1 - return 0 -} - -lpd_by_plate_count() { - make lpd.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" || exit 1 - ./run.sh || failed_tests+=("lpd_by_plate_count") - popd || exit 1 - return 0 -} - -bimodal() { - echo "Bimodal" - make fibonacci.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/bimodal/" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("bimodal") - popd || exit 1 - return 0 -} - -concurrency() { - echo "Concurrency" - make empty.install -C "$base_dir/runtime/tests" || exit 1 - pushd "$base_dir/runtime/experiments/concurrency/" || exit 1 - # ./install.sh || exit 1 - ./run.sh || failed_tests+=("concurrency") - popd || exit 1 - return 0 -} - -main() { - cd "$base_dir/awsm" && cargo build --release || exit 1 - make all -C "$base_dir/runtime" || exit 1 - - if (($# == 0)); then - # If no arguments are provided, run all tests - for test in "${tests[@]}"; do - echo "[[$test]]" - "$test" - done - - else - # Otherwise, only run the tests passed as arguments - for test in "$@"; do - if [[ ! " ${tests[*]} " =~ " ${test} " ]]; then - printf "Error: %s is not a known test\n" "$test" - return 1 - else - "$test" - fi - done - - fi - - local -i failure_count=${#failed_tests[@]} - if ((failure_count > 0)); then - printf "Failed Tests\n" - for test in "${failed_tests[@]}"; do - printf "\t%s\n" "$test" - done - exit 1 - else - printf "All tests passed\n" - exit 0 - fi -} - -main "$@"