From 099aed1be22e36c011b4640e441556adc2a25e38 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Tue, 26 Apr 2022 22:07:10 -0400 Subject: [PATCH] test: other tests --- tests/TinyEKF/by_iteration/Makefile | 6 +- tests/TinyEKF/by_iteration/run.sh | 16 +- tests/TinyEKF/by_iteration/spec.json | 54 +-- tests/TinyEKF/one_iteration/Makefile | 2 +- tests/TinyEKF/one_iteration/run.sh | 2 +- tests/TinyEKF/one_iteration/spec.json | 18 +- tests/speechtotext/spec.json | 16 +- tests/stack_overflow/Makefile | 2 +- tests/stack_overflow/run.sh | 2 +- tests/stack_overflow/spec.json | 20 +- tests/traps/Makefile | 38 +- tests/traps/run.sh | 4 +- tests/traps/spec.json | 20 +- tests/workload_mix/Makefile | 13 + tests/workload_mix/run.sh | 16 +- tests/workload_mix/spec.json | 42 +- tests/workload_mix_realworld/Makefile | 13 + tests/workload_mix_realworld/run.sh | 17 +- tests/workload_mix_realworld/spec.json | 600 +++++++++++-------------- 19 files changed, 444 insertions(+), 457 deletions(-) create mode 100644 tests/workload_mix/Makefile create mode 100644 tests/workload_mix_realworld/Makefile diff --git a/tests/TinyEKF/by_iteration/Makefile b/tests/TinyEKF/by_iteration/Makefile index 619a7f7..aca1b41 100644 --- a/tests/TinyEKF/by_iteration/Makefile +++ b/tests/TinyEKF/by_iteration/Makefile @@ -18,10 +18,10 @@ debug: --eval-command="run spec.json" client: - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@initial_state.dat" "${HOSTNAME}:10000" --output result1.dat + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@initial_state.dat" "${HOSTNAME}:10000/ekf_first_iter" --output result1.dat diff -s result1.dat one_iteration.dat - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@result1.dat" "${HOSTNAME}:10000" --output result2.dat + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@result1.dat" "${HOSTNAME}:10000/ekf_second_iter" --output result2.dat diff -s result2.dat two_iterations.dat - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@result2.dat" "${HOSTNAME}:10000" --output result3.dat + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@result2.dat" "${HOSTNAME}:10000/ekf_third_iter" --output result3.dat diff -s result3.dat three_iterations.dat rm result*.dat diff --git a/tests/TinyEKF/by_iteration/run.sh b/tests/TinyEKF/by_iteration/run.sh index 093d09f..dc52dad 100755 --- a/tests/TinyEKF/by_iteration/run.sh +++ b/tests/TinyEKF/by_iteration/run.sh @@ -28,9 +28,9 @@ run_functional_tests() { } for ((i = 0; i < total_count; i++)); do - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@initial_state.dat" "$hostname":10000 2> /dev/null > "$tmpfs_dir/one_iteration_res.dat" - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@$tmpfs_dir/one_iteration_res.dat" "$hostname":10001 2> /dev/null > "$tmpfs_dir/two_iterations_res.dat" - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@$tmpfs_dir/two_iterations_res.dat" "$hostname":10002 2> /dev/null > "$tmpfs_dir/three_iterations_res.dat" + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@initial_state.dat" "$hostname":10000/ekf_first_iter 2> /dev/null > "$tmpfs_dir/one_iteration_res.dat" + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@$tmpfs_dir/one_iteration_res.dat" "$hostname":10000/ekf_second_iter 2> /dev/null > "$tmpfs_dir/two_iterations_res.dat" + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@$tmpfs_dir/two_iterations_res.dat" "$hostname":10000/ekf_third_iter 2> /dev/null > "$tmpfs_dir/three_iterations_res.dat" if diff -s "$tmpfs_dir/one_iteration_res.dat" one_iteration.dat > /dev/null \ && diff -s "$tmpfs_dir/two_iterations_res.dat" two_iterations.dat > /dev/null \ @@ -68,7 +68,7 @@ run_perf_tests() { done ((batch_id++)) - hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET -D "./${workload}.dat" "http://${hostname}:${port[$workload]}" > "$results_directory/${workload}_${batch_id}.csv" & + hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET -D "./${workload}.dat" "http://${hostname}:10000${path[$workload]}" > "$results_directory/${workload}_${batch_id}.csv" & done pids=$(pgrep hey | tr '\n' ' ') [[ -n $pids ]] && wait -f $pids @@ -142,10 +142,10 @@ if [[ ! -f "$__run_sh__base_path/initial_state.dat" ]]; then fi declare -a workloads=(initial_state one_iteration two_iterations) -declare -A port=( - [initial_state]=10000 - [one_iteration]=10001 - [two_iterations]=10002 +declare -A path=( + [initial_state]=/ekf_first_iter + [one_iteration]=/ekf_second_iter + [two_iterations]=/ekf_third_iter ) framework_init "$@" diff --git a/tests/TinyEKF/by_iteration/spec.json b/tests/TinyEKF/by_iteration/spec.json index 7fb3c44..8c7b55e 100644 --- a/tests/TinyEKF/by_iteration/spec.json +++ b/tests/TinyEKF/by_iteration/spec.json @@ -1,32 +1,32 @@ [ { - "name": "ekf_first_iter", - "path": "gps_ekf.wasm.so", + "name": "gwu", "port": 10000, - "expected-execution-us": 5000, - "relative-deadline-us": 50000, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream" - }, - { - "name": "ekf_second_iter", - "path": "gps_ekf.wasm.so", - "port": 10001, - "expected-execution-us": 5000, - "relative-deadline-us": 50000, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream" - }, - { - "name": "ekf_third_iter", - "path": "gps_ekf.wasm.so", - "port": 10002, - "expected-execution-us": 5000, - "relative-deadline-us": 50000, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream" + "routes": [ + { + "route": "/ekf_first_iter", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 5000, + "relative-deadline-us": 50000, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream" + }, + { + "route": "/ekf_second_iter", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 5000, + "relative-deadline-us": 50000, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream" + }, + { + "route": "/ekf_third_iter", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 5000, + "relative-deadline-us": 50000, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream" + } + ] } ] diff --git a/tests/TinyEKF/one_iteration/Makefile b/tests/TinyEKF/one_iteration/Makefile index 31344c2..37ed25f 100644 --- a/tests/TinyEKF/one_iteration/Makefile +++ b/tests/TinyEKF/one_iteration/Makefile @@ -18,5 +18,5 @@ debug: --eval-command="run spec.json" client: - curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@ekf_raw.dat" "${HOSTNAME}:10000" --output result.dat + curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@ekf_raw.dat" "${HOSTNAME}:10000/ekf" --output result.dat diff -s result.dat expected_result.dat diff --git a/tests/TinyEKF/one_iteration/run.sh b/tests/TinyEKF/one_iteration/run.sh index d0218c6..4b813b2 100755 --- a/tests/TinyEKF/one_iteration/run.sh +++ b/tests/TinyEKF/one_iteration/run.sh @@ -22,7 +22,7 @@ run_functional_tests() { expected_result="$(tr -d '\0' < ./expected_result.dat)" for ((i = 0; i < total_count; i++)); do - result="$(curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@ekf_raw.dat" "$hostname:10000" 2> /dev/null | tr -d '\0')" + result="$(curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@ekf_raw.dat" "$hostname:10000/ekf" 2> /dev/null | tr -d '\0')" if [[ "$expected_result" == "$result" ]]; then ((success_count++)) else diff --git a/tests/TinyEKF/one_iteration/spec.json b/tests/TinyEKF/one_iteration/spec.json index d571023..2ac8327 100644 --- a/tests/TinyEKF/one_iteration/spec.json +++ b/tests/TinyEKF/one_iteration/spec.json @@ -1,12 +1,16 @@ [ { - "name": "ekf", - "path": "gps_ekf.wasm.so", + "name": "gwu", "port": 10000, - "expected-execution-us": 5000, - "relative-deadline-us": 50000, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream" + "routes": [ + { + "route": "/ekf", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 5000, + "relative-deadline-us": 50000, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream" + } + ] } ] diff --git a/tests/speechtotext/spec.json b/tests/speechtotext/spec.json index cf6c2bc..227b6b7 100644 --- a/tests/speechtotext/spec.json +++ b/tests/speechtotext/spec.json @@ -1,11 +1,15 @@ [ { - "name": "hello_ps", - "path": "hello_ps.wasm.so", + "name": "gwu", "port": 10000, - "relative-deadline-us": 50000, - "http-req-size": 102400, - "http-resp-size": 1048576, - "http-resp-content-type": "image/jpeg" + "routes": [ + { + "route": "/hello_ps", + "path": "hello_ps.wasm.so", + "relative-deadline-us": 50000, + "http-resp-size": 1048576, + "http-resp-content-type": "image/jpeg" + } + ] } ] diff --git a/tests/stack_overflow/Makefile b/tests/stack_overflow/Makefile index 14e8ca9..9575abf 100644 --- a/tests/stack_overflow/Makefile +++ b/tests/stack_overflow/Makefile @@ -35,4 +35,4 @@ debug: sledgert stack_overflow --eval-command="run spec.json" client: - http :10000 + http :10000/stack_overflow diff --git a/tests/stack_overflow/run.sh b/tests/stack_overflow/run.sh index 1530559..e4edc8c 100755 --- a/tests/stack_overflow/run.sh +++ b/tests/stack_overflow/run.sh @@ -12,7 +12,7 @@ experiment_client() { local -r hostname="$1" for ((i = 1; i <= 10; i++)); do - http -p h "${hostname}:10000" | grep 500 || { + http -p h "${hostname}:10000/stack_overflow" | grep 500 || { echo "FAIL" return 1 } diff --git a/tests/stack_overflow/spec.json b/tests/stack_overflow/spec.json index a9a8695..e78789e 100644 --- a/tests/stack_overflow/spec.json +++ b/tests/stack_overflow/spec.json @@ -1,13 +1,17 @@ [ { - "name": "stack_overflow", - "path": "stack_overflow.wasm.so", + "name": "gwu", "port": 10000, - "expected-execution-us": 10000000, - "admissions-percentile": 70, - "relative-deadline-us": 20000000, - "http-req-size": 1024, - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" + "routes": [ + { + "route": "/stack_overflow", + "path": "stack_overflow.wasm.so", + "expected-execution-us": 10000000, + "admissions-percentile": 70, + "relative-deadline-us": 20000000, + "http-resp-size": 1024, + "http-resp-content-type": "text/plain" + } + ] } ] diff --git a/tests/traps/Makefile b/tests/traps/Makefile index 75542c1..e3285b0 100644 --- a/tests/traps/Makefile +++ b/tests/traps/Makefile @@ -35,30 +35,30 @@ debug: sledgert trap_divzero --eval-command="run spec.json" client-ok: - echo "1" | http :10000 + echo "1" | http :10000/divide client-trap: - echo "0" | http :10000 + echo "0" | http :10000/divide client-trap2: - echo "-1" | http :10000 + echo "-1" | http :10000/divide test: - echo "4" | http :10000 - echo "0" | http :10000 - echo "3" | http :10000 - echo "0" | http :10000 - echo "2" | http :10000 - echo "0" | http :10000 - echo "1" | http :10000 - echo "0" | http :10000 - echo "1" | http :10000 - echo "0" | http :10000 - echo "2" | http :10000 - echo "0" | http :10000 - echo "3" | http :10000 - echo "0" | http :10000 - echo "4" | http :10000 + echo "4" | http :10000/divide + echo "0" | http :10000/divide + echo "3" | http :10000/divide + echo "0" | http :10000/divide + echo "2" | http :10000/divide + echo "0" | http :10000/divide + echo "1" | http :10000/divide + echo "0" | http :10000/divide + echo "1" | http :10000/divide + echo "0" | http :10000/divide + echo "2" | http :10000/divide + echo "0" | http :10000/divide + echo "3" | http :10000/divide + echo "0" | http :10000/divide + echo "4" | http :10000/divide client-fib10-multi: - hey -z ${DURATION_SEC}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${HOSTNAME}:10010" + hey -z ${DURATION_SEC}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${HOSTNAME}:10010/divide" diff --git a/tests/traps/run.sh b/tests/traps/run.sh index fd3f9ef..c559105 100755 --- a/tests/traps/run.sh +++ b/tests/traps/run.sh @@ -12,11 +12,11 @@ experiment_client() { local -r hostname="$1" for ((i = 1; i <= 10; i++)); do - echo "$i" | http -p h "${hostname}:10000" | grep 200 || { + echo "$i" | http -p h "${hostname}:10000/divide" | grep 200 || { echo "FAIL" return 1 } - echo "0" | http -p h "${hostname}:10000" | grep 500 || { + echo "0" | http -p h "${hostname}:10000/divide" | grep 500 || { echo "FAIL" return 1 } diff --git a/tests/traps/spec.json b/tests/traps/spec.json index 19262ce..a7ed752 100644 --- a/tests/traps/spec.json +++ b/tests/traps/spec.json @@ -1,13 +1,17 @@ [ { - "name": "fibonacci_40", - "path": "trap_divzero.wasm.so", + "name": "gwu", "port": 10000, - "expected-execution-us": 10000000, - "admissions-percentile": 70, - "relative-deadline-us": 20000000, - "http-req-size": 1024, - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" + "routes": [ + { + "route": "/divide", + "path": "trap_divzero.wasm.so", + "expected-execution-us": 10000000, + "admissions-percentile": 70, + "relative-deadline-us": 20000000, + "http-resp-size": 64, + "http-resp-content-type": "text/plain" + } + ] } ] diff --git a/tests/workload_mix/Makefile b/tests/workload_mix/Makefile new file mode 100644 index 0000000..296c4fe --- /dev/null +++ b/tests/workload_mix/Makefile @@ -0,0 +1,13 @@ +RUNTIME_DIR=../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 \ + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" diff --git a/tests/workload_mix/run.sh b/tests/workload_mix/run.sh index 7b616a5..4079276 100755 --- a/tests/workload_mix/run.sh +++ b/tests/workload_mix/run.sh @@ -37,23 +37,23 @@ run_samples() { # Scrape the perf window size from the source if possible # TODO: Make a util function - local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window_t.h)" + local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../runtime/include/perf_window_t.h)" local -i perf_window_buffer_size if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then - printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n" + printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../runtime/include/perf_window.h\n" printf "Defaulting to 16\n" perf_window_buffer_size=16 fi local -ir perf_window_buffer_size printf "Running Samples: " - hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || { + hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10000/fibonacci_10" 1> /dev/null 2> /dev/null || { printf "[ERR]\n" panic "fibonacci_40 samples failed with $?" return 1 } - hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || { + hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10000/fibonacci_10" 1> /dev/null 2> /dev/null || { printf "[ERR]\n" panic "fibonacci_10 samples failed with $?" return 1 @@ -83,9 +83,9 @@ run_experiments() { local -a workloads=() - local -Ar port=( - [fibonacci_10]=10010 - [fibonacci_40]=10040 + local -Ar path=( + [fibonacci_10]=/fibonacci_10 + [fibonacci_40]=/fibonacci_40 ) local -Ar body=( @@ -140,7 +140,7 @@ run_experiments() { ((batch_id++)) for workload in "${workloads[@]}"; do if ((roll >= floor[$workload] && roll < floor[$workload] + length[$workload])); then - hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET -d "${body[$workload]}\n" "http://${hostname}:${port[$workload]}" > "$results_directory/${workload}_${batch_id}.csv" 2> /dev/null & + hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET -d "${body[$workload]}\n" "http://${hostname}:10000${path[$workload]}" > "$results_directory/${workload}_${batch_id}.csv" 2> /dev/null & break fi done diff --git a/tests/workload_mix/spec.json b/tests/workload_mix/spec.json index 3d8be61..ab5d351 100644 --- a/tests/workload_mix/spec.json +++ b/tests/workload_mix/spec.json @@ -1,24 +1,26 @@ [ { - "name": "fibonacci_10", - "path": "fibonacci.wasm.so", - "port": 10010, - "expected-execution-us": 6000, - "admissions-percentile": 70, - "relative-deadline-us": 20000, - "http-req-size": 1024, - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" - }, - { - "name": "fibonacci_40", - "path": "fibonacci.wasm.so", - "port": 10040, - "expected-execution-us": 10000000, - "admissions-percentile": 70, - "relative-deadline-us": 20000000, - "http-req-size": 1024, - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" + "name": "gwu", + "port": 10000, + "routes": [ + { + "route": "/fibonacci_10", + "path": "fibonacci.wasm.so", + "expected-execution-us": 6000, + "admissions-percentile": 70, + "relative-deadline-us": 20000, + "http-resp-size": 1024, + "http-resp-content-type": "text/plain" + }, + { + "route": "/fibonacci_40", + "path": "fibonacci.wasm.so", + "expected-execution-us": 10000000, + "admissions-percentile": 70, + "relative-deadline-us": 20000000, + "http-resp-size": 1024, + "http-resp-content-type": "text/plain" + } + ] } ] diff --git a/tests/workload_mix_realworld/Makefile b/tests/workload_mix_realworld/Makefile new file mode 100644 index 0000000..296c4fe --- /dev/null +++ b/tests/workload_mix_realworld/Makefile @@ -0,0 +1,13 @@ +RUNTIME_DIR=../../runtime/ +SLEDGE_BINARY_DIR=${RUNTIME_DIR}/bin + +run: + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} ${SLEDGE_BINARY_DIR}/sledgert spec.json + +debug: + SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 \ + LD_LIBRARY_PATH=${SLEDGE_BINARY_DIR} gdb ${SLEDGE_BINARY_DIR}/sledgert \ + --eval-command="handle SIGUSR1 noprint nostop" \ + --eval-command="handle SIGPIPE noprint nostop" \ + --eval-command="set pagination off" \ + --eval-command="run spec.json" diff --git a/tests/workload_mix_realworld/run.sh b/tests/workload_mix_realworld/run.sh index f06a46a..3815458 100755 --- a/tests/workload_mix_realworld/run.sh +++ b/tests/workload_mix_realworld/run.sh @@ -23,10 +23,11 @@ source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 source percentiles_table.sh || exit 1 -validate_dependencies hey jq +# validate_dependencies hey jq +validate_dependencies hey declare -a workloads=() -declare -A port=() +declare -A path=() # Example of stripping off multiple suffix from variable in format string_multiple # test="ekf_12223.23343" @@ -46,17 +47,13 @@ initialize_globals() { workload="${buffer[1]}" # Update workload mix structures workloads+=("$workload") - port+=(["$workload"]=$(get_port "$workload")) + path+=(["$workload"]=$(get_path "$workload")) done < "$__run_sh__base_path/mix.csv" } -get_port() { +get_path() { local name="$1" - { - echo "[" - cat ./spec.json - echo "]" - } | jq ".[] | select(.name == \"$name\") | .port" + cat spec.json | jq ".[] | select(.name==\"gwu\") | .routes[] | select(.route==\"/${name}\") | .route" } run_experiments() { @@ -125,7 +122,7 @@ run_experiments() { if ((roll >= floor[$workload] && roll < floor[$workload] + length[$workload])); then # We require word splitting on the value returned by the body associative array #shellcheck disable=SC2086 - hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET ${body[$shortname]} "http://${hostname}:${port[$workload]}" > /dev/null 2> /dev/null & + hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET ${body[$shortname]} "http://${hostname}:10000/${path[$shortname]}" > /dev/null 2> /dev/null & break fi done diff --git a/tests/workload_mix_realworld/spec.json b/tests/workload_mix_realworld/spec.json index f69aa0d..6e4ff48 100644 --- a/tests/workload_mix_realworld/spec.json +++ b/tests/workload_mix_realworld/spec.json @@ -1,332 +1,278 @@ [ { - "name": "cifar10_1.5", - "path": "cifar10.wasm.so", - "port": 10018, - "expected-execution-us": 2777, - "relative-deadline-us": 4166, - "http-req-size": 4096, - "http-resp-size": 128, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "cifar10_1.6", - "path": "cifar10.wasm.so", - "port": 10019, - "expected-execution-us": 2777, - "relative-deadline-us": 4443, - "http-req-size": 4096, - "http-resp-size": 128, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "cifar10_1.7", - "path": "cifar10.wasm.so", - "port": 10020, - "expected-execution-us": 2777, - "relative-deadline-us": 4721, - "http-req-size": 4096, - "http-resp-size": 128, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "cifar10_1.8", - "path": "cifar10.wasm.so", - "port": 10021, - "expected-execution-us": 2777, - "relative-deadline-us": 4999, - "http-req-size": 4096, - "http-resp-size": 128, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "cifar10_1.9", - "path": "cifar10.wasm.so", - "port": 10022, - "expected-execution-us": 2777, - "relative-deadline-us": 5276, - "http-req-size": 4096, - "http-resp-size": 128, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "cifar10_2.0", - "path": "cifar10.wasm.so", - "port": 10023, - "expected-execution-us": 2777, - "relative-deadline-us": 5554, - "http-req-size": 4096, - "http-resp-size": 128, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "ekf_1.5", - "path": "gps_ekf.wasm.so", + "name": "gwu", "port": 10000, - "expected-execution-us": 158, - "relative-deadline-us": 237, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream", - "admissions-percentile": 90 - }, - { - "name": "ekf_1.6", - "path": "gps_ekf.wasm.so", - "port": 10001, - "expected-execution-us": 158, - "relative-deadline-us": 253, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream", - "admissions-percentile": 90 - }, - { - "name": "ekf_1.7", - "path": "gps_ekf.wasm.so", - "port": 10002, - "expected-execution-us": 158, - "relative-deadline-us": 269, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream", - "admissions-percentile": 90 - }, - { - "name": "ekf_1.8", - "path": "gps_ekf.wasm.so", - "port": 10003, - "expected-execution-us": 158, - "relative-deadline-us": 284, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream", - "admissions-percentile": 90 - }, - { - "name": "ekf_1.9", - "path": "gps_ekf.wasm.so", - "port": 10004, - "expected-execution-us": 158, - "relative-deadline-us": 300, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream", - "admissions-percentile": 90 - }, - { - "name": "ekf_2.0", - "path": "gps_ekf.wasm.so", - "port": 10005, - "expected-execution-us": 158, - "relative-deadline-us": 316, - "http-req-size": 1024000, - "http-resp-size": 1024000, - "http-resp-content-type": "application/octet-stream", - "admissions-percentile": 90 - }, - { - "name": "gocr_1.5", - "path": "gocr.wasm.so", - "port": 10006, - "expected-execution-us": 461831, - "relative-deadline-us": 692746, - "http-req-size": 5335057, - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "gocr_1.6", - "path": "gocr.wasm.so", - "port": 10007, - "expected-execution-us": 461831, - "relative-deadline-us": 738930, - "http-req-size": 5335057, - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "gocr_1.7", - "path": "gocr.wasm.so", - "port": 10008, - "expected-execution-us": 461831, - "relative-deadline-us": 785113, - "http-req-size": 5335057, - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "gocr_1.8", - "path": "gocr.wasm.so", - "port": 10009, - "expected-execution-us": 461831, - "relative-deadline-us": 831296, - "http-req-size": 5335057, - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "gocr_1.9", - "path": "gocr.wasm.so", - "port": 10010, - "expected-execution-us": 461831, - "relative-deadline-us": 877479, - "http-req-size": 5335057, - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "gocr_2.0", - "path": "gocr.wasm.so", - "port": 10011, - "expected-execution-us": 461831, - "relative-deadline-us": 923662, - "http-req-size": 5335057, - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "lpd_1.5", - "path": "license_plate_detection.wasm.so", - "port": 10012, - "expected-execution-us": 31597, - "relative-deadline-us": 47396, - "http-req-size": 1002400, - "http-resp-size": 1048576, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "lpd_1.6", - "path": "license_plate_detection.wasm.so", - "port": 10013, - "expected-execution-us": 31597, - "relative-deadline-us": 50555, - "http-req-size": 1002400, - "http-resp-size": 1048576, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "lpd_1.7", - "path": "license_plate_detection.wasm.so", - "port": 10014, - "expected-execution-us": 31597, - "relative-deadline-us": 53715, - "http-req-size": 1002400, - "http-resp-size": 1048576, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "lpd_1.8", - "path": "license_plate_detection.wasm.so", - "port": 10015, - "expected-execution-us": 31597, - "relative-deadline-us": 56875, - "http-req-size": 1002400, - "http-resp-size": 1048576, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "lpd_1.9", - "path": "license_plate_detection.wasm.so", - "port": 10016, - "expected-execution-us": 31597, - "relative-deadline-us": 60034, - "http-req-size": 1002400, - "http-resp-size": 1048576, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "lpd_2.0", - "path": "license_plate_detection.wasm.so", - "port": 10017, - "expected-execution-us": 31597, - "relative-deadline-us": 63194, - "http-req-size": 1002400, - "http-resp-size": 1048576, - "http-resp-content-type": "text/plain", - "admissions-percentile": 90 - }, - { - "name": "resize_1.5", - "path": "resize_image.wasm.so", - "port": 10018, - "expected-execution-us": 138903, - "relative-deadline-us": 208354, - "http-req-size": 1524000, - "http-resp-size": 1524000, - "http-resp-content-type": "image/png", - "admissions-percentile": 90 - }, - { - "name": "resize_1.6", - "path": "resize_image.wasm.so", - "port": 10019, - "expected-execution-us": 138903, - "relative-deadline-us": 222245, - "http-req-size": 1524000, - "http-resp-size": 1524000, - "http-resp-content-type": "image/png", - "admissions-percentile": 90 - }, - { - "name": "resize_1.7", - "path": "resize_image.wasm.so", - "port": 10020, - "expected-execution-us": 138903, - "relative-deadline-us": 236135, - "http-req-size": 1524000, - "http-resp-size": 1524000, - "http-resp-content-type": "image/png", - "admissions-percentile": 90 - }, - { - "name": "resize_1.8", - "path": "resize_image.wasm.so", - "port": 10021, - "expected-execution-us": 138903, - "relative-deadline-us": 250025, - "http-req-size": 1524000, - "http-resp-size": 1524000, - "http-resp-content-type": "image/png", - "admissions-percentile": 90 - }, - { - "name": "resize_1.9", - "path": "resize_image.wasm.so", - "port": 10022, - "expected-execution-us": 138903, - "relative-deadline-us": 263916, - "http-req-size": 1524000, - "http-resp-size": 1524000, - "http-resp-content-type": "image/png", - "admissions-percentile": 90 - }, - { - "name": "resize_2.0", - "path": "resize_image.wasm.so", - "port": 10023, - "expected-execution-us": 138903, - "relative-deadline-us": 277806, - "http-req-size": 1524000, - "http-resp-size": 1524000, - "http-resp-content-type": "image/png", - "admissions-percentile": 90 + "routes": [ + { + "route": "/cifar10_1.5", + "path": "cifar10.wasm.so", + "expected-execution-us": 2777, + "relative-deadline-us": 4166, + "http-resp-size": 128, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/cifar10_1.6", + "path": "cifar10.wasm.so", + "expected-execution-us": 2777, + "relative-deadline-us": 4443, + "http-resp-size": 128, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/cifar10_1.7", + "path": "cifar10.wasm.so", + "expected-execution-us": 2777, + "relative-deadline-us": 4721, + "http-resp-size": 128, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/cifar10_1.8", + "path": "cifar10.wasm.so", + "expected-execution-us": 2777, + "relative-deadline-us": 4999, + "http-resp-size": 128, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/cifar10_1.9", + "path": "cifar10.wasm.so", + "expected-execution-us": 2777, + "relative-deadline-us": 5276, + "http-resp-size": 128, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/cifar10_2.0", + "path": "cifar10.wasm.so", + "expected-execution-us": 2777, + "relative-deadline-us": 5554, + "http-resp-size": 128, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/ekf_1.5", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 158, + "relative-deadline-us": 237, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream", + "admissions-percentile": 90 + }, + { + "route": "/ekf_1.6", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 158, + "relative-deadline-us": 253, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream", + "admissions-percentile": 90 + }, + { + "route": "/ekf_1.7", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 158, + "relative-deadline-us": 269, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream", + "admissions-percentile": 90 + }, + { + "route": "/ekf_1.8", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 158, + "relative-deadline-us": 284, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream", + "admissions-percentile": 90 + }, + { + "route": "/ekf_1.9", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 158, + "relative-deadline-us": 300, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream", + "admissions-percentile": 90 + }, + { + "route": "/ekf_2.0", + "path": "gps_ekf.wasm.so", + "expected-execution-us": 158, + "relative-deadline-us": 316, + "http-resp-size": 1024000, + "http-resp-content-type": "application/octet-stream", + "admissions-percentile": 90 + }, + { + "route": "/gocr_1.5", + "path": "gocr.wasm.so", + "expected-execution-us": 461831, + "relative-deadline-us": 692746, + "http-resp-size": 5335057, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/gocr_1.6", + "path": "gocr.wasm.so", + "expected-execution-us": 461831, + "relative-deadline-us": 738930, + "http-resp-size": 5335057, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/gocr_1.7", + "path": "gocr.wasm.so", + "expected-execution-us": 461831, + "relative-deadline-us": 785113, + "http-resp-size": 5335057, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/gocr_1.8", + "path": "gocr.wasm.so", + "expected-execution-us": 461831, + "relative-deadline-us": 831296, + "http-resp-size": 5335057, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/gocr_1.9", + "path": "gocr.wasm.so", + "expected-execution-us": 461831, + "relative-deadline-us": 877479, + "http-resp-size": 5335057, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/gocr_2.0", + "path": "gocr.wasm.so", + "expected-execution-us": 461831, + "relative-deadline-us": 923662, + "http-resp-size": 5335057, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/lpd_1.5", + "path": "license_plate_detection.wasm.so", + "expected-execution-us": 31597, + "relative-deadline-us": 47396, + "http-resp-size": 1048576, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/lpd_1.6", + "path": "license_plate_detection.wasm.so", + "expected-execution-us": 31597, + "relative-deadline-us": 50555, + "http-resp-size": 1048576, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/lpd_1.7", + "path": "license_plate_detection.wasm.so", + "expected-execution-us": 31597, + "relative-deadline-us": 53715, + "http-resp-size": 1048576, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/lpd_1.8", + "path": "license_plate_detection.wasm.so", + "expected-execution-us": 31597, + "relative-deadline-us": 56875, + "http-resp-size": 1048576, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/lpd_1.9", + "path": "license_plate_detection.wasm.so", + "expected-execution-us": 31597, + "relative-deadline-us": 60034, + "http-resp-size": 1048576, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/lpd_2.0", + "path": "license_plate_detection.wasm.so", + "expected-execution-us": 31597, + "relative-deadline-us": 63194, + "http-resp-size": 1048576, + "http-resp-content-type": "text/plain", + "admissions-percentile": 90 + }, + { + "route": "/resize_1.5", + "path": "resize_image.wasm.so", + "expected-execution-us": 138903, + "relative-deadline-us": 208354, + "http-resp-size": 1524000, + "http-resp-content-type": "image/png", + "admissions-percentile": 90 + }, + { + "route": "/resize_1.6", + "path": "resize_image.wasm.so", + "expected-execution-us": 138903, + "relative-deadline-us": 222245, + "http-resp-size": 1524000, + "http-resp-content-type": "image/png", + "admissions-percentile": 90 + }, + { + "route": "/resize_1.7", + "path": "resize_image.wasm.so", + "expected-execution-us": 138903, + "relative-deadline-us": 236135, + "http-resp-size": 1524000, + "http-resp-content-type": "image/png", + "admissions-percentile": 90 + }, + { + "route": "/resize_1.8", + "path": "resize_image.wasm.so", + "expected-execution-us": 138903, + "relative-deadline-us": 250025, + "http-resp-size": 1524000, + "http-resp-content-type": "image/png", + "admissions-percentile": 90 + }, + { + "route": "/resize_1.9", + "path": "resize_image.wasm.so", + "expected-execution-us": 138903, + "relative-deadline-us": 263916, + "http-resp-size": 1524000, + "http-resp-content-type": "image/png", + "admissions-percentile": 90 + }, + { + "route": "/resize_2.0", + "path": "resize_image.wasm.so", + "expected-execution-us": 138903, + "relative-deadline-us": 277806, + "http-resp-size": 1524000, + "http-resp-content-type": "image/png", + "admissions-percentile": 90 + } + ] } ]