diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f69c904..9832a6f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -100,7 +100,7 @@ jobs: - name: Cache gocr uses: actions/cache@v2 with: - path: ./runtime/bin/gocr_wasm.so + path: ./runtime/bin/gocr.wasm.so key: ${{ runner.os }}-gocr2-${{ hashFiles('./applications/Makefile', './applications/wasmception_apps/gocr/**', './runtime/compiletime/**') }} if: success() || failure() - name: Hyde @@ -137,7 +137,7 @@ jobs: - name: Cache EKF uses: actions/cache@v2 with: - path: ./runtime/bin/ekf_wasm.so + path: ./runtime/bin/gps_ekf.wasm.so key: ${{ runner.os }}-gocr2-${{ hashFiles('./applications/Makefile', './applications/wasmception_apps/TinyEKF/**', './runtime/compiletime/**') }} if: success() || failure() - name: EKF one iteration diff --git a/README.md b/README.md index 931149f..4299f8b 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ cd /sledge/runtime make clean all ``` -There are a set of benchmarking applications in the `/sledge/applications` directory. Run the following to compile all benchmarks runtime tests using the aWsm compiler and then copy all resulting `_wasm.so` files to /sledge/runtime/bin. +There are a set of benchmarking applications in the `/sledge/applications` directory. Run the following to compile all benchmarks runtime tests using the aWsm compiler and then copy all resulting `.wasm.so` files to /sledge/runtime/bin. ```bash cd /sledge/applications/ @@ -61,7 +61,7 @@ An SLEdge serverless function consists of a shared library (\*.so) and a JSON co ```json { "name": "fibonacci", - "path": "fibonacci_wasm.so", + "path": "fibonacci.wasm.so", "port": 10000, "expected-execution-us": 600, "relative-deadline-us": 2000, diff --git a/applications/Makefile b/applications/Makefile index f040069..ef5899a 100644 --- a/applications/Makefile +++ b/applications/Makefile @@ -18,11 +18,11 @@ SLEDGE_COMPILETIME_SRC=${SLEDGE_RT_DIR}/compiletime/*.c all: \ cifar10.install \ empty.install \ - ekf.install \ + gps_ekf.install \ fibonacci.install \ gocr.install \ - lpd.install \ - resize.install + license_plate_detection.install \ + resize_image.install .PHONY: clean clean: @@ -42,11 +42,11 @@ dist: %.bc: %.wasm dist ${AWSMCC} ${AWSMFLAGS} $< -o $@ -%_wasm.so: %.bc ${SLEDGE_COMPILETIME_SRC} +%.wasm.so: %.bc ${SLEDGE_COMPILETIME_SRC} mkdir -p dist ${CC} ${CFLAGS} ${LDFLAGS} -I${SLEDGE_COMPILETIME_INC} $^ -o $@ -../runtime/bin/%_wasm.so: dist/%_wasm.so +../runtime/bin/%.wasm.so: dist/%.wasm.so cp $^ $@ # Fibonacci @@ -57,7 +57,7 @@ dist/fibonacci.wasm: ./wasmception_apps/fibonacci/fibonacci.wasm dist @cp ./wasmception_apps/fibonacci/fibonacci.wasm dist/fibonacci.wasm .PHONY: fibonacci.install -fibonacci.install: ../runtime/bin/fibonacci_wasm.so +fibonacci.install: ../runtime/bin/fibonacci.wasm.so # Empty ./wasmception_apps/empty/empty.wasm: @@ -67,7 +67,7 @@ dist/empty.wasm: ./wasmception_apps/empty/empty.wasm dist @cp ./wasmception_apps/empty/empty.wasm dist/empty.wasm .PHONY: empty.install -empty.install: ../runtime/bin/empty_wasm.so +empty.install: ../runtime/bin/empty.wasm.so # EKF ./wasmception_apps/TinyEKF/extras/c/gps_ekf_fn.wasm: @@ -76,8 +76,8 @@ empty.install: ../runtime/bin/empty_wasm.so dist/ekf.wasm: ./wasmception_apps/TinyEKF/extras/c/gps_ekf_fn.wasm dist @cp ./wasmception_apps/TinyEKF/extras/c/gps_ekf_fn.wasm dist/ekf.wasm -.PHONY: ekf.install -ekf.install: ../runtime/bin/ekf_wasm.so +.PHONY: gps_ekf.install +gps_ekf.install: ../runtime/bin/gps_ekf.wasm.so # CIFAR10 ./wasmception_apps/CMSIS_5_NN/cifar10.wasm: @@ -87,7 +87,7 @@ dist/cifar10.wasm: ./wasmception_apps/CMSIS_5_NN/cifar10.wasm dist @cp ./wasmception_apps/CMSIS_5_NN/cifar10.wasm dist/cifar10.wasm .PHONY: cifar10.install -cifar10.install: ../runtime/bin/cifar10_wasm.so +cifar10.install: ../runtime/bin/cifar10.wasm.so # GOCR ./wasmception_apps/gocr/src/gocr.wasm: @@ -97,7 +97,7 @@ dist/gocr.wasm: ./wasmception_apps/gocr/src/gocr.wasm dist @cp ./wasmception_apps/gocr/src/gocr.wasm dist/gocr.wasm .PHONY: gocr.install -gocr.install: ../runtime/bin/gocr_wasm.so +gocr.install: ../runtime/bin/gocr.wasm.so # LPD ./wasmception_apps/sod/bin/license_plate_detection.wasm: @@ -106,8 +106,8 @@ gocr.install: ../runtime/bin/gocr_wasm.so dist/lpd.wasm: ./wasmception_apps/sod/bin/license_plate_detection.wasm dist @cp ./wasmception_apps/sod/bin/license_plate_detection.wasm dist/lpd.wasm -.PHONY: lpd.install -lpd.install: ../runtime/bin/lpd_wasm.so +.PHONY: license_plate_detection.install +license_plate_detection.install: ../runtime/bin/license_plate_detection.wasm.so # Resize ./wasmception_apps/sod/bin/resize_image.wasm: @@ -116,5 +116,5 @@ lpd.install: ../runtime/bin/lpd_wasm.so dist/resize.wasm: ./wasmception_apps/sod/bin/resize_image.wasm dist @cp ./wasmception_apps/sod/bin/resize_image.wasm dist/resize.wasm -.PHONY: resize.install -resize.install: ../runtime/bin/resize_wasm.so +.PHONY: resize_image.install +resize_image.install: ../runtime/bin/resize_image.wasm.so diff --git a/test.mk b/test.mk index c9dabe4..ed98db3 100644 --- a/test.mk +++ b/test.mk @@ -4,32 +4,32 @@ # the relevant directory for specifics. # gocr - GNU Optical Character Recognition -./runtime/bin/gocr_wasm.so: +./runtime/bin/gocr.wasm.so: make gocr.install -C ./applications PHONY: gocr__by_dpi -gocr__by_dpi: ./runtime/bin/gocr_wasm.so +gocr__by_dpi: ./runtime/bin/gocr.wasm.so # cd ./tests/gocr/by_dpi && ./install.sh cd ./tests/gocr/by_dpi && ./run.sh PHONY: gocr__by_font -gocr__by_font: ./runtime/bin/gocr_wasm.so +gocr__by_font: ./runtime/bin/gocr.wasm.so cd ./tests/gocr/by_font && ./run.sh PHONY: gocr__by_word -gocr__by_word: ./runtime/bin/gocr_wasm.so +gocr__by_word: ./runtime/bin/gocr.wasm.so cd ./tests/gocr/by_word && ./run.sh PHONY: gocr__fivebyeight -gocr__fivebyeight: ./runtime/bin/gocr_wasm.so +gocr__fivebyeight: ./runtime/bin/gocr.wasm.so cd ./tests/gocr/fivebyeight && ./run.sh PHONY: gocr__handwriting -gocr__handwriting: ./runtime/bin/gocr_wasm.so +gocr__handwriting: ./runtime/bin/gocr.wasm.so cd ./tests/gocr/handwriting && ./run.sh PHONY: gocr__hyde -gocr__hyde: ./runtime/bin/gocr_wasm.so +gocr__hyde: ./runtime/bin/gocr.wasm.so cd ./tests/gocr/hyde && ./run.sh PHONY: gocr_all @@ -42,15 +42,15 @@ gocr__all: \ gocr__hyde # Extended Kalman Filter applied to binary GPS data -./runtime/bin/ekf_wasm.so: - make ekf.install -C ./applications +./runtime/bin/gps_ekf.wasm.so: + make gps_ekf.install -C ./applications PHONY: ekf__by_iteration -ekf__by_iteration: ./runtime/bin/ekf_wasm.so +ekf__by_iteration: ./runtime/bin/gps_ekf.wasm.so cd ./tests/TinyEKF/by_iteration && ./run.sh PHONY: ekf__one_iteration -ekf__one_iteration: ./runtime/bin/ekf_wasm.so +ekf__one_iteration: ./runtime/bin/gps_ekf.wasm.so cd ./tests/TinyEKF/one_iteration && ./run.sh PHONY: ekf__all @@ -59,11 +59,11 @@ ekf__all: \ ekf__one_iteration # CIFAR10-based Image Classification -./runtime/bin/cifar10_wasm.so: +./runtime/bin/cifar10.wasm.so: make cifar10.install -C ./applications PHONY: cifar10__image_classification -cifar10__image_classification: ./runtime/bin/cifar10_wasm.so +cifar10__image_classification: ./runtime/bin/cifar10.wasm.so cd ./tests/CMSIS_5_NN/imageclassification && ./run.sh PHONY: cifar10__all @@ -74,44 +74,44 @@ cifar10__all: \ # https://sod.pixlab.io/ # SOD - Image Resize -./runtime/bin/resize_wasm.so: - make resize.install -C ./applications +./runtime/bin/resize_image.wasm.so: + make resize_image.install -C ./applications # Commented out command installs imagemagick. Requires password for sudo to install PHONY: sod__image_resize__test -sod__image_resize__test: ./runtime/bin/resize_wasm.so +sod__image_resize__test: ./runtime/bin/resize_image.wasm.so # cd ./tests/sod/image_resize/test && ./install.sh cd ./tests/sod/image_resize/test && ./run.sh PHONY: sod__image_resize__by_resolution -sod__image_resize__by_resolution: ./runtime/bin/resize_wasm.so +sod__image_resize__by_resolution: ./runtime/bin/resize_image.wasm.so # cd ./tests/sod/image_resize/by_resolution && ./install.sh cd ./tests/sod/image_resize/by_resolution && ./run.sh # SOD - License Plate Detection -./runtime/bin/lpd_wasm.so: - make lpd.install -C ./applications +./runtime/bin/license_plate_detection.wasm.so: + make license_plate_detection.install -C ./applications PHONY: sod__lpd__by_plate_count -sod__lpd__by_plate_count: ./runtime/bin/lpd_wasm.so +sod__lpd__by_plate_count: ./runtime/bin/license_plate_detection.wasm.so cd ./tests/sod/lpd/by_plate_count && ./run.sh PHONY: sod__all sod__all: sod__image_resize__test sod__image_resize__by_resolution sod__lpd__by_plate_count # Scheduler Experiments with synthetic workloads -./runtime/bin/fibonacci_wasm.so: +./runtime/bin/fibonacci.wasm.so: make fibonacci.install -C ./applications PHONY: fibonacci__bimodal -fibonacci__bimodal: ./runtime/bin/fibonacci_wasm.so +fibonacci__bimodal: ./runtime/bin/fibonacci.wasm.so cd ./tests/fibonacci/bimodal/ && ./run.sh -./runtime/bin/empty_wasm.so: +./runtime/bin/empty.wasm.so: make empty.install -C ./applications PHONY: empty__concurrency -empty__concurrency: ./runtime/bin/empty_wasm.so +empty__concurrency: ./runtime/bin/empty.wasm.so # cd ./tests/empty/concurrency/ && ./install.sh cd ./tests/empty/concurrency/ && ./run.sh diff --git a/tests/CMSIS_5_NN/imageclassification/Makefile b/tests/CMSIS_5_NN/imageclassification/Makefile new file mode 100644 index 0000000..4ea2393 --- /dev/null +++ b/tests/CMSIS_5_NN/imageclassification/Makefile @@ -0,0 +1,26 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin + +.PHONY: run +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +.PHONY: debug +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" + +.PHONY: client +client: + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/airplane1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/automobile1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/bird1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/cat1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/deer1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/dog1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/frog1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/horse1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/ship1.bmp" "localhost:10000" + @curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@../../../applications/wasm_apps/CMSIS_5_NN/images/bmp/truck1.bmp" "localhost:10000" diff --git a/tests/CMSIS_5_NN/imageclassification/spec.json b/tests/CMSIS_5_NN/imageclassification/spec.json index 00241ea..e8f4a3e 100644 --- a/tests/CMSIS_5_NN/imageclassification/spec.json +++ b/tests/CMSIS_5_NN/imageclassification/spec.json @@ -1,7 +1,7 @@ [ { "name": "cifar10_rand", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -11,7 +11,7 @@ }, { "name": "cifar10_same", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/TinyEKF/by_iteration/Makefile b/tests/TinyEKF/by_iteration/Makefile new file mode 100644 index 0000000..619a7f7 --- /dev/null +++ b/tests/TinyEKF/by_iteration/Makefile @@ -0,0 +1,27 @@ +RUNTIME_DIR=../../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin +HOSTNAME=localhost + +all: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" + +client: + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@initial_state.dat" "${HOSTNAME}:10000" --output result1.dat + diff -s result1.dat one_iteration.dat + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@result1.dat" "${HOSTNAME}:10000" --output result2.dat + diff -s result2.dat two_iterations.dat + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@result2.dat" "${HOSTNAME}:10000" --output result3.dat + diff -s result3.dat three_iterations.dat + rm result*.dat diff --git a/tests/TinyEKF/by_iteration/run.sh b/tests/TinyEKF/by_iteration/run.sh index f7c6d84..093d09f 100755 --- a/tests/TinyEKF/by_iteration/run.sh +++ b/tests/TinyEKF/by_iteration/run.sh @@ -21,10 +21,7 @@ run_functional_tests() { local -ir total_count=50 local tmpfs_dir=/tmp/sledge_ekf_by_iteration - [[ -d "$tmpfs_dir" ]] && { - echo "$tmpfs_dir directory exists. Delete via rm -r $tmpfs_dir and rerun." - return 1 - } + rm -rf "$tmpfs_dir" mkdir "$tmpfs_dir" || { echo "Failed to create tmp directory" return 1 diff --git a/tests/TinyEKF/by_iteration/spec.json b/tests/TinyEKF/by_iteration/spec.json index feb133a..7fb3c44 100644 --- a/tests/TinyEKF/by_iteration/spec.json +++ b/tests/TinyEKF/by_iteration/spec.json @@ -1,7 +1,7 @@ [ { "name": "ekf_first_iter", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -11,7 +11,7 @@ }, { "name": "ekf_second_iter", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -21,7 +21,7 @@ }, { "name": "ekf_third_iter", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/TinyEKF/one_iteration/Makefile b/tests/TinyEKF/one_iteration/Makefile new file mode 100644 index 0000000..31344c2 --- /dev/null +++ b/tests/TinyEKF/one_iteration/Makefile @@ -0,0 +1,22 @@ +RUNTIME_DIR=../../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin +HOSTNAME=localhost + +all: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" + +client: + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@ekf_raw.dat" "${HOSTNAME}:10000" --output result.dat + diff -s result.dat expected_result.dat diff --git a/tests/TinyEKF/one_iteration/run.sh b/tests/TinyEKF/one_iteration/run.sh index 478a14a..0dc4160 100755 --- a/tests/TinyEKF/one_iteration/run.sh +++ b/tests/TinyEKF/one_iteration/run.sh @@ -53,7 +53,7 @@ experiment_client() { # Copy data if not here if [[ ! -f "$__run_sh__base_path/initial_state.dat" ]]; then pushd "$__run_sh__base_path" || exit 1 - pushd "../../../../tests/TinyEKF/extras/c/" || exit 1 + pushd "../../../applications/wasmception_apps/TinyEKF/extras/c/" || exit 1 cp ekf_raw.dat "$__run_sh__base_path/initial_state.dat" || exit 1 popd || exit 1 popd || exit 1 diff --git a/tests/TinyEKF/one_iteration/spec.json b/tests/TinyEKF/one_iteration/spec.json index 6324456..a9bab5e 100644 --- a/tests/TinyEKF/one_iteration/spec.json +++ b/tests/TinyEKF/one_iteration/spec.json @@ -1,6 +1,6 @@ { "name": "ekf", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/deadline_description/Makefile b/tests/deadline_description/Makefile new file mode 100644 index 0000000..b40cea0 --- /dev/null +++ b/tests/deadline_description/Makefile @@ -0,0 +1,13 @@ +SLEDGE_BINARY_DIR=../../runtime/bin +HOSTNAME=localhost + +default: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" diff --git a/tests/deadline_description/cifar10/template.json b/tests/deadline_description/cifar10/template.json index c1ec6fa..66caa3f 100644 --- a/tests/deadline_description/cifar10/template.json +++ b/tests/deadline_description/cifar10/template.json @@ -1,6 +1,6 @@ { "name": "cifar10", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/deadline_description/ekf/template.json b/tests/deadline_description/ekf/template.json index 6324456..a9bab5e 100644 --- a/tests/deadline_description/ekf/template.json +++ b/tests/deadline_description/ekf/template.json @@ -1,6 +1,6 @@ { "name": "ekf", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/deadline_description/gocr/template.json b/tests/deadline_description/gocr/template.json index 3bad5e2..59f6649 100644 --- a/tests/deadline_description/gocr/template.json +++ b/tests/deadline_description/gocr/template.json @@ -1,6 +1,6 @@ { "name": "gocr", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 360000, diff --git a/tests/deadline_description/lpd/template.json b/tests/deadline_description/lpd/template.json index 19dd519..e0e1afd 100644 --- a/tests/deadline_description/lpd/template.json +++ b/tests/deadline_description/lpd/template.json @@ -1,6 +1,6 @@ { "name": "lpd", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/deadline_description/resize/template.json b/tests/deadline_description/resize/template.json index 0a4aa1a..8f63222 100644 --- a/tests/deadline_description/resize/template.json +++ b/tests/deadline_description/resize/template.json @@ -1,6 +1,6 @@ { "name": "resize", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/deadline_description/spec.json b/tests/deadline_description/spec.json index 1c88f6c..50ede9d 100644 --- a/tests/deadline_description/spec.json +++ b/tests/deadline_description/spec.json @@ -1,7 +1,7 @@ [ { "name": "ekf", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -11,7 +11,7 @@ }, { "name": "resize", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -21,7 +21,7 @@ }, { "name": "lpd", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -31,7 +31,7 @@ }, { "name": "gocr", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10003, "expected-execution-us": 5000, "relative-deadline-us": 360000, @@ -41,7 +41,7 @@ }, { "name": "cifar10", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10004, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/empty/concurrency/Makefile b/tests/empty/concurrency/Makefile new file mode 100644 index 0000000..f29c7a2 --- /dev/null +++ b/tests/empty/concurrency/Makefile @@ -0,0 +1,20 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin +HOSTNAME=localhost + +default: run + +.PHONY: clean +clean: + rm -rf res/* + +.PHONY: run +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +.PHONY: debug +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" + +.PHONY: client +client: + curl -H 'Expect:' -H "Content-Type: text/plain" "${HOSTNAME}:10000" diff --git a/tests/empty/concurrency/README.md b/tests/empty/concurrency/README.md index 156efbd..be04a51 100644 --- a/tests/empty/concurrency/README.md +++ b/tests/empty/concurrency/README.md @@ -18,7 +18,7 @@ _How does increasing levels of concurrent client requests affect tail latency, t - You have a modern bash shell. My Linux environment shows version 4.4.20(1)-release - `hey` (https://github.com/rakyll/hey) is available in your PATH -- You have compiled `sledgert` and the `empty.so` test workload +- You have compiled `sledgert` and the `empty.wasm.so` test workload ## TODO diff --git a/tests/empty/concurrency/spec.json b/tests/empty/concurrency/spec.json index 0786d21..ee168c9 100644 --- a/tests/empty/concurrency/spec.json +++ b/tests/empty/concurrency/spec.json @@ -1,6 +1,6 @@ { "name": "empty", - "path": "empty_wasm.so", + "path": "empty.wasm.so", "port": 10000, "expected-execution-us": 500, "admissions-percentile": 70, diff --git a/tests/fibonacci/bimodal/Makefile b/tests/fibonacci/bimodal/Makefile new file mode 100644 index 0000000..2ea969a --- /dev/null +++ b/tests/fibonacci/bimodal/Makefile @@ -0,0 +1,47 @@ +RUNTIME_DIR=../../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin +SLEDGE_TESTS_DIR=${RUNTIME_DIR}/tests +HOSTNAME=localhost +DURATION_SEC=15 + +all: run + +clean: + make -C ${RUNTIME_DIR} clean + make -C ${SLEDGE_TESTS_DIR} clean + rm -f ${SLEDGE_BINARY_DIR}/fibonacci.wasm.so + +${SLEDGE_BINARY_DIR}/sledgert: + make -C ${RUNTIME_DIR} runtime + +.PHONY: sledgert +sledgert: ${SLEDGE_BINARY_DIR}/sledgert + +${SLEDGE_BINARY_DIR}/fibonacci.wasm.so: + make -C .. fibonacci.install + +.PHONY: fibonacci +fibonacci: ${SLEDGE_BINARY_DIR}/fibonacci.wasm.so + +run: sledgert fibonacci + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: sledgert fibonacci + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 \ + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" + +client-fib10-once: + echo "10" | http :10010 + +client-fib40-once: + echo "40" | http :10040 + +client-preempt: + (echo "40" | http :10040 &); echo "10" | http :10010 + +client-fib10-multi: + hey -z ${DURATION_SEC}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${HOSTNAME}:10010" diff --git a/tests/fibonacci/bimodal/run.sh b/tests/fibonacci/bimodal/run.sh index 2de7620..229dbc4 100755 --- a/tests/fibonacci/bimodal/run.sh +++ b/tests/fibonacci/bimodal/run.sh @@ -43,7 +43,7 @@ run_samples() { local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../../runtime/include/perf_window_t.h)" local -i perf_window_buffer_size if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then - printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n" + printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../../runtime/include/perf_window.h\n" printf "Defaulting to 16\n" perf_window_buffer_size=16 fi diff --git a/tests/fibonacci/bimodal/spec.json b/tests/fibonacci/bimodal/spec.json index de1e7c1..3d8be61 100644 --- a/tests/fibonacci/bimodal/spec.json +++ b/tests/fibonacci/bimodal/spec.json @@ -1,7 +1,7 @@ [ { "name": "fibonacci_10", - "path": "fibonacci_wasm.so", + "path": "fibonacci.wasm.so", "port": 10010, "expected-execution-us": 6000, "admissions-percentile": 70, @@ -12,7 +12,7 @@ }, { "name": "fibonacci_40", - "path": "fibonacci_wasm.so", + "path": "fibonacci.wasm.so", "port": 10040, "expected-execution-us": 10000000, "admissions-percentile": 70, diff --git a/tests/gocr/by_dpi/Makefile b/tests/gocr/by_dpi/Makefile new file mode 100644 index 0000000..6cac070 --- /dev/null +++ b/tests/gocr/by_dpi/Makefile @@ -0,0 +1,37 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin +HOSTNAME=localhost + +default: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" + +client-72dpi: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --dpi=72 --font=mono -qo tmp/72_dpi.png tmp/10_words.txt + pngtopnm tmp/72_dpi.png > tmp/72_dpi.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary @tmp/72_dpi.pnm "${HOSTNAME}:10000" + rm -rf tmp + +client-108dpi: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --dpi=108 --font=mono -qo tmp/108_dpi.png tmp/10_words.txt + pngtopnm tmp/108_dpi.png > tmp/108_dpi.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary @tmp/108_dpi.pnm "${HOSTNAME}:10001" + rm -rf tmp + +client-144dpi: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --dpi=144 --font=mono -qo tmp/144_dpi.png tmp/10_words.txt + pngtopnm tmp/144_dpi.png > tmp/144_dpi.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary @tmp/144_dpi.pnm "${HOSTNAME}:10002" + rm -rf tmp diff --git a/tests/gocr/by_dpi/spec.json b/tests/gocr/by_dpi/spec.json index a564a3a..a5fb6bb 100644 --- a/tests/gocr/by_dpi/spec.json +++ b/tests/gocr/by_dpi/spec.json @@ -1,7 +1,7 @@ [ { "name": "gocr_72_dpi", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 36000, @@ -11,7 +11,7 @@ }, { "name": "gocr_108_dpi", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 36000, @@ -21,7 +21,7 @@ }, { "name": "gocr_144_dpi", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 36000, diff --git a/tests/gocr/by_font/Makefile b/tests/gocr/by_font/Makefile new file mode 100644 index 0000000..92fa8f8 --- /dev/null +++ b/tests/gocr/by_font/Makefile @@ -0,0 +1,37 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin +HOSTNAME=localhost + +default: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" + +client-dejavu_sans_mono: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --font="DejaVu Sans Mono" -qo tmp/10_words.png tmp/10_words.txt + pngtopnm tmp/10_words.png > tmp/10_words.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@tmp/10_words.pnm" "${HOSTNAME}:10000" + rm -rf tmp + +client-roboto: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --font="Roboto" -qo tmp/10_words.png tmp/10_words.txt + pngtopnm tmp/10_words.png > tmp/10_words.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@tmp/10_words.pnm" "${HOSTNAME}:10000" + rm -rf tmp + +client-cascadia_code: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --font="Cascadia Code" -qo tmp/10_words.png tmp/10_words.txt + pngtopnm tmp/10_words.png > tmp/10_words.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@tmp/10_words.pnm" "${HOSTNAME}:10000" + rm -rf tmp diff --git a/tests/gocr/by_font/spec.json b/tests/gocr/by_font/spec.json index 07f0891..ace3711 100644 --- a/tests/gocr/by_font/spec.json +++ b/tests/gocr/by_font/spec.json @@ -1,7 +1,7 @@ [ { "name": "gocr_mono", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 36000, @@ -11,7 +11,7 @@ }, { "name": "gocr_urw_gothic", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 36000, @@ -21,7 +21,7 @@ }, { "name": "gocr_lobster_2", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 36000, diff --git a/tests/gocr/by_word/Makefile b/tests/gocr/by_word/Makefile new file mode 100644 index 0000000..7704675 --- /dev/null +++ b/tests/gocr/by_word/Makefile @@ -0,0 +1,21 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin +HOSTNAME=localhost + +default: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" + +client-10: + mkdir -p tmp + shuf -n10 /usr/share/dict/american-english > tmp/10_words.txt + pango-view --font=mono -qo tmp/10_words.png tmp/10_words.txt + pngtopnm tmp/10_words.png > tmp/10_words.pnm + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@tmp/10_words.pnm" "${HOSTNAME}:10000" + rm -rf tmp diff --git a/tests/gocr/by_word/spec.json b/tests/gocr/by_word/spec.json index 6bd60ca..b24b65b 100644 --- a/tests/gocr/by_word/spec.json +++ b/tests/gocr/by_word/spec.json @@ -1,7 +1,7 @@ [ { "name": "gocr_1_word", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 36000, @@ -11,7 +11,7 @@ }, { "name": "gocr_10_words", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 36000, @@ -21,7 +21,7 @@ }, { "name": "gocr_100_words", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 36000, diff --git a/tests/gocr/fivebyeight/Makefile b/tests/gocr/fivebyeight/Makefile new file mode 100644 index 0000000..b42ebc8 --- /dev/null +++ b/tests/gocr/fivebyeight/Makefile @@ -0,0 +1,16 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin +HOSTNAME=localhost + +default: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" + +client: + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" "${HOSTNAME}:10000" diff --git a/tests/gocr/fivebyeight/spec.json b/tests/gocr/fivebyeight/spec.json index a89a680..ea53b2d 100644 --- a/tests/gocr/fivebyeight/spec.json +++ b/tests/gocr/fivebyeight/spec.json @@ -1,6 +1,6 @@ { "name": "gocr", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 36000, diff --git a/tests/gocr/handwriting/Makefile b/tests/gocr/handwriting/Makefile new file mode 100644 index 0000000..345f6d1 --- /dev/null +++ b/tests/gocr/handwriting/Makefile @@ -0,0 +1,16 @@ +SLEDGE_BINARY_DIR=../../../runtime/bin +HOSTNAME=localhost + +default: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert --eval-command="run spec.json" + +client: + curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" "${HOSTNAME}:10000" diff --git a/tests/gocr/handwriting/spec.json b/tests/gocr/handwriting/spec.json index a89a680..ea53b2d 100644 --- a/tests/gocr/handwriting/spec.json +++ b/tests/gocr/handwriting/spec.json @@ -1,6 +1,6 @@ { "name": "gocr", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 36000, diff --git a/tests/gocr/hyde/Makefile b/tests/gocr/hyde/Makefile index 296acb5..3140d30 100644 --- a/tests/gocr/hyde/Makefile +++ b/tests/gocr/hyde/Makefile @@ -1,4 +1,4 @@ -SLEDGE_BINARY_DIR=../../../../bin +SLEDGE_BINARY_DIR=../../../runtime/bin HOSTNAME=localhost default: run diff --git a/tests/gocr/hyde/spec.json b/tests/gocr/hyde/spec.json index 3bad5e2..59f6649 100644 --- a/tests/gocr/hyde/spec.json +++ b/tests/gocr/hyde/spec.json @@ -1,6 +1,6 @@ { "name": "gocr", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 360000, diff --git a/tests/payload/.gitignore b/tests/payload/.gitignore deleted file mode 100644 index 64f722e..0000000 --- a/tests/payload/.gitignore +++ /dev/null @@ -1 +0,0 @@ -res diff --git a/tests/payload/body/.gitignore b/tests/payload/body/.gitignore deleted file mode 100644 index 2211df6..0000000 --- a/tests/payload/body/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.txt diff --git a/tests/payload/body/generate.sh b/tests/payload/body/generate.sh deleted file mode 100755 index e8b8a19..0000000 --- a/tests/payload/body/generate.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Generates payloads of 1KB, 10KB, 100KB, 1MB -for size in 1024 $((1024 * 10)) $((1024 * 100)) $((1024 * 1024)); do - # If the file exists, but is not the right size, wipe it - if [[ -f "$size.txt" ]] && (("$(wc -c "$size.txt" | cut -d\ -f1)" != size)); then - rm -rf "$size.txt" - fi - - # Regenerate the file if missing - if [[ ! -f "$size.txt" ]]; then - echo -n "Generating $size: " - for ((i = 0; i < size; i++)); do - printf 'a' >> $size.txt - done - echo "[OK]" - fi - -done diff --git a/tests/payload/edf_nopreemption.env b/tests/payload/edf_nopreemption.env deleted file mode 100644 index eeba531..0000000 --- a/tests/payload/edf_nopreemption.env +++ /dev/null @@ -1,2 +0,0 @@ -SLEDGE_SCHEDULER=EDF -SLEDGE_DISABLE_PREEMPTION=true diff --git a/tests/payload/edf_preemption.env b/tests/payload/edf_preemption.env deleted file mode 100644 index 302a324..0000000 --- a/tests/payload/edf_preemption.env +++ /dev/null @@ -1,3 +0,0 @@ -SLEDGE_SCHEDULER=EDF -SLEDGE_DISABLE_PREEMPTION=false -SLEDGE_SIGALRM_HANDLER=TRIAGED diff --git a/tests/payload/fifo_nopreemption.env b/tests/payload/fifo_nopreemption.env deleted file mode 100644 index a572a70..0000000 --- a/tests/payload/fifo_nopreemption.env +++ /dev/null @@ -1,2 +0,0 @@ -SLEDGE_SCHEDULER=FIFO -SLEDGE_DISABLE_PREEMPTION=true diff --git a/tests/payload/fifo_preemption.env b/tests/payload/fifo_preemption.env deleted file mode 100644 index eb1298f..0000000 --- a/tests/payload/fifo_preemption.env +++ /dev/null @@ -1,2 +0,0 @@ -SLEDGE_SCHEDULER=FIFO -SLEDGE_DISABLE_PREEMPTION=false diff --git a/tests/payload/install.sh b/tests/payload/install.sh deleted file mode 100755 index 0cbcfe8..0000000 --- a/tests/payload/install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if ! command -v hey > /dev/null; then - HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64 - wget $HEY_URL -O hey - chmod +x hey - - if [[ $(whoami) == "root" ]]; then - mv hey /usr/bin/hey - else - sudo mv hey /usr/bin/hey - fi -fi diff --git a/tests/payload/latency.gnuplot b/tests/payload/latency.gnuplot deleted file mode 100644 index 095aa47..0000000 --- a/tests/payload/latency.gnuplot +++ /dev/null @@ -1,20 +0,0 @@ -reset - -set term jpeg -set output "latency.jpg" - -set xlabel "Payload (bytes)" -set xrange [-5:1050000] - -set ylabel "Latency (ms)" -set yrange [0:] - -set key left top - - -set style histogram columnstacked - -plot 'latency.dat' using 1:2 title 'p50', \ - 'latency.dat' using 1:3 title 'p90', \ - 'latency.dat' using 1:4 title 'p99', \ - 'latency.dat' using 1:5 title 'p100', \ diff --git a/tests/payload/run.sh b/tests/payload/run.sh deleted file mode 100755 index e6ea9f0..0000000 --- a/tests/payload/run.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash -# This experiment is intended to document how the level of concurrent requests influences -# - latency -# - throughput -# - success/failure rate - -# Add bash_libraries directory to path -__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")" -__run_sh__bash_libraries_relative_path="../bash_libraries" -__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) -export PATH="$__run_sh__bash_libraries_absolute_path:$PATH" - -# Source libraries from bash_libraries directory -source path_join.sh || exit 1 -source framework.sh || exit 1 -source get_result_count.sh || exit 1 -source generate_gnuplots.sh || exit 1 -source percentiles_table.sh || exit 1 - -if ! command -v hey > /dev/null; then - echo "hey is not present." - exit 1 -fi - -# Experiment Globals and Setups -declare -ar payloads=(1024 10240 102400 1048576) - -declare -Ar ports=( - [1024]=10000 - [10240]=10001 - [102400]=10002 - [1048576]=10003 -) - -declare -ri iterations=10000 - -# If the one of the expected body files doesn't exist, trigger the generation script. -cd "$__run_sh__base_path/body" && ./generate.sh && cd "$OLDPWD" || exit - -run_samples() { - local hostname="$1" - - # Scrape the perf window size from the source if possible - local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window_t.h)" - local -i perf_window_buffer_size - if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then - printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n" - printf "Defaulting to 16\n" - perf_window_buffer_size=16 - fi - local -ir perf_window_buffer_size - - # Execute workloads long enough for runtime to learn excepted execution time - printf "Running Samples:\n" - for payload in "${payloads[@]}"; do - printf "\t%d Payload: " "$payload" - hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" "http://$hostname:${ports["$payload"]}" 1> /dev/null 2> /dev/null || { - printf "[ERR]\n" - panic "samples failed" - return 1 - } - printf "[OK]\n" - done - - return 0 -} - -run_experiments() { - if (($# != 2)); then - panic "invalid number of arguments \"$1\"" - return 1 - elif [[ ! -d "$2" ]]; then - panic "directory \"$2\" does not exist" - return 1 - fi - - local hostname="$1" - local results_directory="$2" - - # Execute the experiments - printf "Running Experiments:\n" - for payload in "${payloads[@]}"; do - printf "\t%d Payload: " "$payload" - hey -disable-compression -disable-keepalive -disable-redirects -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" "http://$hostname:${ports["$payload"]}" > "$results_directory/$payload.csv" 2> /dev/null || { - printf "[ERR]\n" - panic "$payload experiment failed" - return 1 - } - get_result_count "$results_directory/$payload.csv" || { - printf "[ERR]\n" - panic "$payload.csv unexpectedly has zero requests" - return 1 - } - printf "[OK]\n" - done - - return 0 -} - -process_results() { - if (($# != 1)); then - panic "invalid number of arguments ($#, expected 1)" - return 1 - elif ! [[ -d "$1" ]]; then - panic "directory $1 does not exist" - return 1 - fi - - local -r results_directory="$1" - - printf "Processing Results: " - - printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" - printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" - percentiles_table_header "$results_directory/latency.csv" "Payload" - - for payload in ${payloads[*]}; do - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / '"$iterations"' * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # We determine duration by looking at the timestamp of the last complete request - # TODO: Should this instead just use the client-side synthetic duration_sec value? - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - - # Throughput is calculated as the mean number of successful requests per second - throughput=$(echo "$oks/$duration" | bc) - printf "%d,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - percentiles_table_row "$results_directory/$payload-response.csv" "$results_directory/latency.csv" "$payload" - - # Delete scratch file used for sorting/counting - rm -rf "$results_directory/$payload-response.csv" - done - - # Transform csvs to dat files for gnuplot - for file in success latency throughput; do - printf "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" - done - - # Generate gnuplots - generate_gnuplots "$results_directory" "$__run_sh__base_path" || { - printf "[ERR]\n" - panic "failed to generate gnuplots" - } - - printf "[OK]\n" - return 0 -} - -# Expected Symbol used by the framework -experiment_client() { - local -r target_hostname="$1" - local -r results_directory="$2" - - run_samples "$target_hostname" || return 1 - run_experiments "$target_hostname" "$results_directory" || return 1 - process_results "$results_directory" || return 1 - - return 0 -} - -framework_init "$@" diff --git a/tests/payload/spec.json b/tests/payload/spec.json deleted file mode 100644 index 5e360b3..0000000 --- a/tests/payload/spec.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "name": "work1k", - "path": "work1k_wasm.so", - "port": 10000, - "expected-execution-us": 400, - "relative-deadline-us": 2000, - "http-req-size": 1548, - "http-resp-size": 1548, - "http-resp-content-type": "text/plain" - }, - { - "name": "work10k", - "path": "work10k_wasm.so", - "port": 10001, - "expected-execution-us": 600, - "relative-deadline-us": 2000, - "http-req-size": 10480, - "http-resp-size": 10480, - "http-resp-content-type": "text/plain" - }, - { - "name": "work100k", - "path": "work100k_wasm.so", - "port": 10002, - "expected-execution-us": 700, - "relative-deadline-us": 2000, - "http-req-size": 104800, - "http-resp-size": 104800, - "http-resp-content-type": "text/plain" - }, - { - "name": "work1m", - "path": "work1m_wasm.so", - "port": 10003, - "expected-execution-us": 2000, - "relative-deadline-us": 6000, - "http-req-size": 1048776, - "http-resp-size": 1048776, - "http-resp-content-type": "text/plain" - } -] diff --git a/tests/payload/success.gnuplot b/tests/payload/success.gnuplot deleted file mode 100644 index 3a28692..0000000 --- a/tests/payload/success.gnuplot +++ /dev/null @@ -1,12 +0,0 @@ -reset - -set term jpeg -set output "success.jpg" - -set xlabel "Payload (bytes)" -set xrange [-5:1050000] - -set ylabel "% 2XX" -set yrange [0:110] - -plot 'success.dat' using 1:2 title '2XX' diff --git a/tests/payload/throughput.gnuplot b/tests/payload/throughput.gnuplot deleted file mode 100644 index 23b7a68..0000000 --- a/tests/payload/throughput.gnuplot +++ /dev/null @@ -1,13 +0,0 @@ -reset - -set term jpeg -set output "throughput.jpg" - -# TODO: Axis shouldn't be linear -set xlabel "Payload (bytes)" -set xrange [-5:1050000] - -set ylabel "Requests/sec" -set yrange [0:] - -plot 'throughput.dat' using 1:2 title 'Reqs/sec' diff --git a/tests/sod/image_resize/by_resolution/Makefile b/tests/sod/image_resize/by_resolution/Makefile new file mode 100644 index 0000000..a96ec0d --- /dev/null +++ b/tests/sod/image_resize/by_resolution/Makefile @@ -0,0 +1,17 @@ +RUNTIME_DIR=../../../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin + +all: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" diff --git a/tests/sod/image_resize/by_resolution/spec.json b/tests/sod/image_resize/by_resolution/spec.json index 828402f..5fc6f7f 100644 --- a/tests/sod/image_resize/by_resolution/spec.json +++ b/tests/sod/image_resize/by_resolution/spec.json @@ -1,7 +1,7 @@ [ { "name": "resize_small", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -11,7 +11,7 @@ }, { "name": "resize_medium", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -21,7 +21,7 @@ }, { "name": "resize_large", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/sod/image_resize/test/Makefile b/tests/sod/image_resize/test/Makefile new file mode 100644 index 0000000..ad572f9 --- /dev/null +++ b/tests/sod/image_resize/test/Makefile @@ -0,0 +1,21 @@ +RUNTIME_DIR=../../../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin +HOSTNAME=localhost + +all: run + +clean: + rm -rf res/* + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" + +client: + curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@flower.jpg" --output "result.jpg" "${HOSTNAME}:10000" diff --git a/tests/sod/image_resize/test/run.sh b/tests/sod/image_resize/test/run.sh index edda91a..a766555 100755 --- a/tests/sod/image_resize/test/run.sh +++ b/tests/sod/image_resize/test/run.sh @@ -5,7 +5,7 @@ __run_sh__bash_libraries_relative_path="../../../bash_libraries" __run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) export PATH="$__run_sh__bash_libraries_absolute_path:$PATH" -__run_sh__project_base_relative_path="../../../.." +__run_sh__project_base_relative_path="../../../../" __run_sh__project_base_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__project_base_relative_path" && pwd) source csv_to_dat.sh || exit 1 @@ -22,12 +22,8 @@ experiment_client() { local -i success_count=0 local -ri total_count=10 - local tmpfs_dir=/tmp/sledge_imageresize_test/ - [[ -d "$tmpfs_dir" ]] && { - panic "tmpfs directory exists. Aborting" - return 1 - } - mkdir $tmpfs_dir + local tmpfs_dir=/tmp/sledge_image_resize_test/ + mkdir -p $tmpfs_dir for ((i = 0; i < total_count; i++)); do ext="$RANDOM" @@ -60,7 +56,7 @@ validate_dependencies curl compare # Copy Flower Image if not here if [[ ! -f "./flower.jpg" ]]; then - cp "$__run_sh__project_base_absolute_path/applications/sod/bin/flower.jpg" ./flower.jpg + cp "$__run_sh__project_base_absolute_path/applications/wasmception_apps/sod/samples/flower.jpg" ./flower.jpg fi framework_init "$@" diff --git a/tests/sod/image_resize/test/spec.json b/tests/sod/image_resize/test/spec.json index 14af43f..cbd18f7 100644 --- a/tests/sod/image_resize/test/spec.json +++ b/tests/sod/image_resize/test/spec.json @@ -1,6 +1,6 @@ { "name": "resize", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/sod/lpd/by_plate_count/Makefile b/tests/sod/lpd/by_plate_count/Makefile new file mode 100644 index 0000000..67dfc15 --- /dev/null +++ b/tests/sod/lpd/by_plate_count/Makefile @@ -0,0 +1,27 @@ +RUNTIME_DIR=../../../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin +HOSTNAME=localhost +IMAGE=./images/Cars0.png + +.PHONY: all +all: run + +.PHONY: clean +clean: + rm -rf res/* + +.PHONY: run +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +.PHONY: debug +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" + +.PHONY: client +client: + curl -H 'Expect:' -H "Content-Type: image/png" --data-binary "@${IMAGE}" "${HOSTNAME}:10000" diff --git a/tests/sod/lpd/by_plate_count/spec.json b/tests/sod/lpd/by_plate_count/spec.json index 7f47029..3f8b6f4 100644 --- a/tests/sod/lpd/by_plate_count/spec.json +++ b/tests/sod/lpd/by_plate_count/spec.json @@ -1,7 +1,7 @@ [ { "name": "lpd1", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10000, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -11,7 +11,7 @@ }, { "name": "lpd2", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10001, "expected-execution-us": 5000, "relative-deadline-us": 50000, @@ -21,7 +21,7 @@ }, { "name": "lpd4", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10002, "expected-execution-us": 5000, "relative-deadline-us": 50000, diff --git a/tests/speechtotext/spec.json b/tests/speechtotext/spec.json index af9724f..119a456 100644 --- a/tests/speechtotext/spec.json +++ b/tests/speechtotext/spec.json @@ -1,6 +1,6 @@ { "name": "hello_ps", - "path": "hello_ps_wasm.so", + "path": "hello_ps.wasm.so", "port": 10000, "relative-deadline-us": 50000, "http-req-size": 102400, diff --git a/tests/workload_mix/spec.json b/tests/workload_mix/spec.json index de1e7c1..3d8be61 100644 --- a/tests/workload_mix/spec.json +++ b/tests/workload_mix/spec.json @@ -1,7 +1,7 @@ [ { "name": "fibonacci_10", - "path": "fibonacci_wasm.so", + "path": "fibonacci.wasm.so", "port": 10010, "expected-execution-us": 6000, "admissions-percentile": 70, @@ -12,7 +12,7 @@ }, { "name": "fibonacci_40", - "path": "fibonacci_wasm.so", + "path": "fibonacci.wasm.so", "port": 10040, "expected-execution-us": 10000000, "admissions-percentile": 70, diff --git a/tests/workload_mix_realworld/spec.json b/tests/workload_mix_realworld/spec.json index 73d74cb..f69aa0d 100644 --- a/tests/workload_mix_realworld/spec.json +++ b/tests/workload_mix_realworld/spec.json @@ -1,7 +1,7 @@ [ { "name": "cifar10_1.5", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10018, "expected-execution-us": 2777, "relative-deadline-us": 4166, @@ -12,7 +12,7 @@ }, { "name": "cifar10_1.6", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10019, "expected-execution-us": 2777, "relative-deadline-us": 4443, @@ -23,7 +23,7 @@ }, { "name": "cifar10_1.7", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10020, "expected-execution-us": 2777, "relative-deadline-us": 4721, @@ -34,7 +34,7 @@ }, { "name": "cifar10_1.8", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10021, "expected-execution-us": 2777, "relative-deadline-us": 4999, @@ -45,7 +45,7 @@ }, { "name": "cifar10_1.9", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10022, "expected-execution-us": 2777, "relative-deadline-us": 5276, @@ -56,7 +56,7 @@ }, { "name": "cifar10_2.0", - "path": "cifar10_wasm.so", + "path": "cifar10.wasm.so", "port": 10023, "expected-execution-us": 2777, "relative-deadline-us": 5554, @@ -67,7 +67,7 @@ }, { "name": "ekf_1.5", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10000, "expected-execution-us": 158, "relative-deadline-us": 237, @@ -78,7 +78,7 @@ }, { "name": "ekf_1.6", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10001, "expected-execution-us": 158, "relative-deadline-us": 253, @@ -89,7 +89,7 @@ }, { "name": "ekf_1.7", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10002, "expected-execution-us": 158, "relative-deadline-us": 269, @@ -100,7 +100,7 @@ }, { "name": "ekf_1.8", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10003, "expected-execution-us": 158, "relative-deadline-us": 284, @@ -111,7 +111,7 @@ }, { "name": "ekf_1.9", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10004, "expected-execution-us": 158, "relative-deadline-us": 300, @@ -122,7 +122,7 @@ }, { "name": "ekf_2.0", - "path": "ekf_wasm.so", + "path": "gps_ekf.wasm.so", "port": 10005, "expected-execution-us": 158, "relative-deadline-us": 316, @@ -133,7 +133,7 @@ }, { "name": "gocr_1.5", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10006, "expected-execution-us": 461831, "relative-deadline-us": 692746, @@ -144,7 +144,7 @@ }, { "name": "gocr_1.6", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10007, "expected-execution-us": 461831, "relative-deadline-us": 738930, @@ -155,7 +155,7 @@ }, { "name": "gocr_1.7", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10008, "expected-execution-us": 461831, "relative-deadline-us": 785113, @@ -166,7 +166,7 @@ }, { "name": "gocr_1.8", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10009, "expected-execution-us": 461831, "relative-deadline-us": 831296, @@ -177,7 +177,7 @@ }, { "name": "gocr_1.9", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10010, "expected-execution-us": 461831, "relative-deadline-us": 877479, @@ -188,7 +188,7 @@ }, { "name": "gocr_2.0", - "path": "gocr_wasm.so", + "path": "gocr.wasm.so", "port": 10011, "expected-execution-us": 461831, "relative-deadline-us": 923662, @@ -199,7 +199,7 @@ }, { "name": "lpd_1.5", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10012, "expected-execution-us": 31597, "relative-deadline-us": 47396, @@ -210,7 +210,7 @@ }, { "name": "lpd_1.6", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10013, "expected-execution-us": 31597, "relative-deadline-us": 50555, @@ -221,7 +221,7 @@ }, { "name": "lpd_1.7", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10014, "expected-execution-us": 31597, "relative-deadline-us": 53715, @@ -232,7 +232,7 @@ }, { "name": "lpd_1.8", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10015, "expected-execution-us": 31597, "relative-deadline-us": 56875, @@ -243,7 +243,7 @@ }, { "name": "lpd_1.9", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10016, "expected-execution-us": 31597, "relative-deadline-us": 60034, @@ -254,7 +254,7 @@ }, { "name": "lpd_2.0", - "path": "lpd_wasm.so", + "path": "license_plate_detection.wasm.so", "port": 10017, "expected-execution-us": 31597, "relative-deadline-us": 63194, @@ -265,7 +265,7 @@ }, { "name": "resize_1.5", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10018, "expected-execution-us": 138903, "relative-deadline-us": 208354, @@ -276,7 +276,7 @@ }, { "name": "resize_1.6", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10019, "expected-execution-us": 138903, "relative-deadline-us": 222245, @@ -287,7 +287,7 @@ }, { "name": "resize_1.7", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10020, "expected-execution-us": 138903, "relative-deadline-us": 236135, @@ -298,7 +298,7 @@ }, { "name": "resize_1.8", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10021, "expected-execution-us": 138903, "relative-deadline-us": 250025, @@ -309,7 +309,7 @@ }, { "name": "resize_1.9", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10022, "expected-execution-us": 138903, "relative-deadline-us": 263916, @@ -320,7 +320,7 @@ }, { "name": "resize_2.0", - "path": "resize_wasm.so", + "path": "resize_image.wasm.so", "port": 10023, "expected-execution-us": 138903, "relative-deadline-us": 277806,