From a1192e57f7e549ca890d7897f0a8f096b4cabd6e Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Mon, 24 May 2021 12:15:31 -0400 Subject: [PATCH] feat: image resize using framework --- .../imageresize/by_resolution/.gitignore | 1 + .../by_resolution/edf_nopreemption.env | 2 + .../by_resolution/edf_preemption.env | 3 + .../by_resolution/fifo_nopreemption.env | 2 + .../by_resolution/fifo_preemption.env | 2 + .../imageresize/by_resolution/run.sh | 241 +++++++++++++----- .../imageresize/by_resolution/test.sh | 15 ++ .../licenseplate/by_plate_count/run.sh | 2 - .../experiments/bash_libraries/framework.sh | 10 +- 9 files changed, 212 insertions(+), 66 deletions(-) create mode 100644 runtime/experiments/applications/imageresize/by_resolution/edf_nopreemption.env create mode 100644 runtime/experiments/applications/imageresize/by_resolution/edf_preemption.env create mode 100644 runtime/experiments/applications/imageresize/by_resolution/fifo_nopreemption.env create mode 100644 runtime/experiments/applications/imageresize/by_resolution/fifo_preemption.env create mode 100755 runtime/experiments/applications/imageresize/by_resolution/test.sh diff --git a/runtime/experiments/applications/imageresize/by_resolution/.gitignore b/runtime/experiments/applications/imageresize/by_resolution/.gitignore index f3c930c..172be53 100644 --- a/runtime/experiments/applications/imageresize/by_resolution/.gitignore +++ b/runtime/experiments/applications/imageresize/by_resolution/.gitignore @@ -1,2 +1,3 @@ result_*.png log.csv +res/* diff --git a/runtime/experiments/applications/imageresize/by_resolution/edf_nopreemption.env b/runtime/experiments/applications/imageresize/by_resolution/edf_nopreemption.env new file mode 100644 index 0000000..eeba531 --- /dev/null +++ b/runtime/experiments/applications/imageresize/by_resolution/edf_nopreemption.env @@ -0,0 +1,2 @@ +SLEDGE_SCHEDULER=EDF +SLEDGE_DISABLE_PREEMPTION=true diff --git a/runtime/experiments/applications/imageresize/by_resolution/edf_preemption.env b/runtime/experiments/applications/imageresize/by_resolution/edf_preemption.env new file mode 100644 index 0000000..302a324 --- /dev/null +++ b/runtime/experiments/applications/imageresize/by_resolution/edf_preemption.env @@ -0,0 +1,3 @@ +SLEDGE_SCHEDULER=EDF +SLEDGE_DISABLE_PREEMPTION=false +SLEDGE_SIGALRM_HANDLER=TRIAGED diff --git a/runtime/experiments/applications/imageresize/by_resolution/fifo_nopreemption.env b/runtime/experiments/applications/imageresize/by_resolution/fifo_nopreemption.env new file mode 100644 index 0000000..a572a70 --- /dev/null +++ b/runtime/experiments/applications/imageresize/by_resolution/fifo_nopreemption.env @@ -0,0 +1,2 @@ +SLEDGE_SCHEDULER=FIFO +SLEDGE_DISABLE_PREEMPTION=true diff --git a/runtime/experiments/applications/imageresize/by_resolution/fifo_preemption.env b/runtime/experiments/applications/imageresize/by_resolution/fifo_preemption.env new file mode 100644 index 0000000..eb1298f --- /dev/null +++ b/runtime/experiments/applications/imageresize/by_resolution/fifo_preemption.env @@ -0,0 +1,2 @@ +SLEDGE_SCHEDULER=FIFO +SLEDGE_DISABLE_PREEMPTION=false diff --git a/runtime/experiments/applications/imageresize/by_resolution/run.sh b/runtime/experiments/applications/imageresize/by_resolution/run.sh index 9c13af1..6c8e4a4 100755 --- a/runtime/experiments/applications/imageresize/by_resolution/run.sh +++ b/runtime/experiments/applications/imageresize/by_resolution/run.sh @@ -1,75 +1,190 @@ #!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -echo "$experiment_directory" -project_directory=$(cd ../../../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) -log="$experiment_directory/log.csv" - -if [ "$1" != "-d" ]; then - SLEDGE_SANDBOX_PERF_LOG=$log PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" & - sleep 3 -else - echo "Running under gdb" -fi - -success_count=0 -total_count=100 - -for ((i = 0; i < total_count; i++)); do - ext="$RANDOM" - - # Small - if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_small.jpg" --output "result_${ext}_small.png" localhost:10000 2> /dev/null 1> /dev/null; then - pixel_differences="$(compare -identify -metric AE "result_${ext}_small.png" expected_result_small.png null: 2>&1 > /dev/null)" - if [[ "$pixel_differences" != "0" ]]; then - echo "Small FAIL" - echo "$pixel_differences pixel differences detected" + +__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")" +__run_sh__bash_libraries_relative_path="../../../bash_libraries" +__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) +export PATH="$__run_sh__bash_libraries_absolute_path:$PATH" + +source csv_to_dat.sh || exit 1 +source framework.sh || exit 1 +# source generate_gnuplots.sh || exit 1 +source get_result_count.sh || exit 1 +source panic.sh || exit 1 +source path_join.sh || exit 1 + +declare -ar workloads=(small medium large) +declare -Ar port=( + [small]=10000 + [medium]=10001 + [large]=10002 +) + +# Validate that required tools are in path +declare -a required_binaries=(curl) +validate_dependencies() { + for required_binary in "${required_binaries[@]}"; do + if ! command -v "$required_binary" > /dev/null; then + echo "$required_binary is not present." exit 1 fi - else - echo "curl failed with ${?}. See man curl for meaning." - fi + done +} - # Medium - if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_medium.jpg" --output "result_${ext}_medium.png" localhost:10001 2> /dev/null 1> /dev/null; then - pixel_differences="$(compare -identify -metric AE "result_${ext}_medium.png" expected_result_medium.png null: 2>&1 > /dev/null)" - if [[ "$pixel_differences" != "0" ]]; then - echo "Medium FAIL" - echo "$pixel_differences pixel differences detected" - exit 1 +run_functional_tests() { + local hostname="$1" + local results_directory="$2" + + local -i success_count=0 + local -ir total_count=10 + + echo -n "Functional Tests: " + + for ((i = 0; i < total_count; i++)); do + ext="$RANDOM" + + # Small + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_small.jpg" --output "/tmp/result_${ext}_small.png" "${hostname}:10000" 2> /dev/null 1> /dev/null; then + pixel_differences="$(compare -identify -metric AE "/tmp/result_${ext}_small.png" expected_result_small.png null: 2>&1 > /dev/null)" + rm -f "/tmp/result_${ext}_small.png" + if [[ "$pixel_differences" != "0" ]]; then + echo "Small FAIL" >> "$results_directory/result.txt" + echo "$pixel_differences pixel differences detected" >> "$results_directory/result.txt" + continue + fi + else + echo "curl failed with ${?}. See man curl for meaning." >> "$results_directory/result.txt" + continue fi - else - echo "curl failed with ${?}. See man curl for meaning." - fi - # Large - if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_large.jpg" --output "result_${ext}_large.png" localhost:10002 2> /dev/null 1> /dev/null; then - pixel_differences="$(compare -identify -metric AE "result_${ext}_large.png" expected_result_large.png null: 2>&1 > /dev/null)" - if [[ "$pixel_differences" != "0" ]]; then - echo "Large FAIL" - echo "$pixel_differences pixel differences detected" - exit 1 + # Medium + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_medium.jpg" --output "/tmp/result_${ext}_medium.png" "${hostname}:10001" 2> /dev/null 1> /dev/null; then + pixel_differences="$(compare -identify -metric AE "/tmp/result_${ext}_medium.png" expected_result_medium.png null: 2>&1 > /dev/null)" + rm -f "/tmp/result_${ext}_medium.png" + if [[ "$pixel_differences" != "0" ]]; then + echo "Medium FAIL" >> "$results_directory/result.txt" + echo "$pixel_differences pixel differences detected" >> "$results_directory/result.txt" + continue + fi + else + echo "curl failed with ${?}. See man curl for meaning." >> "$results_directory/result.txt" + continue + fi + + # Large + if curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@shrinking_man_large.jpg" --output "/tmp/result_${ext}_large.png" "${hostname}:10002" 2> /dev/null 1> /dev/null; then + pixel_differences="$(compare -identify -metric AE "/tmp/result_${ext}_large.png" expected_result_large.png null: 2>&1 > /dev/null)" + rm -f "/tmp/result_${ext}_large.png" + if [[ "$pixel_differences" != "0" ]]; then + echo "Large FAIL" >> "$results_directory/result.txt" + echo "$pixel_differences pixel differences detected" >> "$results_directory/result.txt" + continue + fi + else + echo "curl failed with ${?}. See man curl for meaning." >> "$results_directory/result.txt" + continue fi - else - echo "curl failed with ${?}. See man curl for meaning." + + ((success_count++)) + done + + echo "$success_count / $total_count" >> "$results_directory/result.txt" + + echo "[OK]" +} + +run_perf_tests() { + local hostname="$1" + local results_directory="$2" + + local -ir total_iterations=100 + local -ir worker_max=10 + local -ir batch_size=10 + local -i batch_id=0 + + echo -n "Perf Tests: " + for workload in "${workloads[@]}"; do + batch_id=0 + for ((i = 0; i < total_iterations; i += batch_size)); do + # Block waiting for a worker to finish if we are at our max + while (($(pgrep --count hey) >= worker_max)); do + wait -n $(pgrep hey | tr '\n' ' ') + done + ((batch_id++)) + + hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET -D "shrinking_man_${workload}.jpg" "http://${hostname}:${port[$workload]}" > "$results_directory/${workload}_${batch_id}.csv" 2> /dev/null & + done + wait -f $(pgrep hey | tr '\n' ' ') + done + printf "[OK]\n" + + for workload in "${workloads[@]}"; do + tail --quiet -n +2 "$results_directory/${workload}"_*.csv >> "$results_directory/${workload}.csv" + rm "$results_directory/${workload}"_*.csv + done +} + +process_results() { + if (($# != 1)); then + error_msg "invalid number of arguments ($#, expected 1)" + return 1 + elif ! [[ -d "$1" ]]; then + error_msg "directory $1 does not exist" + return 1 fi - success_count=$((success_count + 1)) -done + local -r results_directory="$1" + + printf "Processing Results: " + + # Write headers to CSVs + printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" + + for workload in "${workloads[@]}"; do + + # Filter on 200s, subtract DNS time, convert from s to ms, and sort + awk -F, '$7 == 200 {print (($1 - $2) * 1000)}' < "$results_directory/$workload.csv" \ + | sort -g > "$results_directory/$workload-response.csv" + + oks=$(wc -l < "$results_directory/$workload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$workload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' < "$results_directory/$workload-response.csv" >> "$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + rm -rf "$results_directory/$workload-response.csv" + done + + # Transform csvs to dat files for gnuplot + csv_to_dat "$results_directory/latency.csv" + + printf "[OK]\n" + return 0 +} + +experiment_main() { + local -r hostname="$1" + local -r results_directory="$2" + + validate_dependencies -echo "$success_count / $total_count" -rm -f result_*.png + run_functional_tests "$hostname" "$results_directory" || return 1 + run_perf_tests "$hostname" "$results_directory" || return 1 + process_results "$results_directory" || return 1 -if [ "$1" != "-d" ]; then - sleep 5 - echo -n "Running Cleanup: " - pkill sledgert > /dev/null 2> /dev/null - echo "[DONE]" -fi +} -exit 0 +main "$@" diff --git a/runtime/experiments/applications/imageresize/by_resolution/test.sh b/runtime/experiments/applications/imageresize/by_resolution/test.sh new file mode 100755 index 0000000..7988c4e --- /dev/null +++ b/runtime/experiments/applications/imageresize/by_resolution/test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +declare image_buffer + +if image_buffer <(curl -H 'Expect:' -H "Content-Type: image/jpg" --output "result_small.png" "localhost:10000"); then + pixel_differences="$(compare -identify -metric AE "result_small.png" expected_result_small.png null: 2>&1 > /dev/null)" + if [[ "$pixel_differences" != "0" ]]; then + echo "Small FAIL" + echo "$pixel_differences pixel differences detected" + else + echo "OK" + fi +else + echo "curl failed with ${?}. See man curl for meaning." +fi diff --git a/runtime/experiments/applications/licenseplate/by_plate_count/run.sh b/runtime/experiments/applications/licenseplate/by_plate_count/run.sh index 173e2cb..7000b09 100755 --- a/runtime/experiments/applications/licenseplate/by_plate_count/run.sh +++ b/runtime/experiments/applications/licenseplate/by_plate_count/run.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -o xtrace - __run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")" __run_sh__bash_libraries_relative_path="../../../bash_libraries" __run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) diff --git a/runtime/experiments/bash_libraries/framework.sh b/runtime/experiments/bash_libraries/framework.sh index 0b64241..f2ea424 100644 --- a/runtime/experiments/bash_libraries/framework.sh +++ b/runtime/experiments/bash_libraries/framework.sh @@ -211,6 +211,11 @@ __framework_sh__start_runtime() { ;; esac + # Pad with a sleep to allow runtime to initialize before startup tasks run + # This should be improved adding some sort of ping/ping heartbeat to the runtime + # so the script can spin until initializaiton is complete + sleep 1 + printf "[OK]\n" return 0 } @@ -312,7 +317,9 @@ __framework_sh__unset_env_file() { __framework_sh__run_both() { local short_name shopt -s nullglob + local -i envfiles_found=0 for envfile in "$__framework_sh__application_directory"/*.env; do + ((envfiles_found++)) short_name="$(basename "${envfile/.env/}")" printf "Running %s\n" "$short_name" __framework_sh__load_env_file "$envfile" @@ -335,9 +342,10 @@ __framework_sh__run_both() { } __framework_sh__unset_env_file "$envfile" - done + ((envfiles_found == 0)) && echo "No *.env files found. Nothing to run!" + return 0 }