diff --git a/runtime/experiments/applications/ekf/by_iteration/run.sh b/runtime/experiments/applications/ekf/by_iteration/run.sh index 1c3a59f..281a15f 100755 --- a/runtime/experiments/applications/ekf/by_iteration/run.sh +++ b/runtime/experiments/applications/ekf/by_iteration/run.sh @@ -136,7 +136,7 @@ process_results() { return 0 } -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/ekf/one_iteration/run.sh b/runtime/experiments/applications/ekf/one_iteration/run.sh index 4c0acba..9679660 100755 --- a/runtime/experiments/applications/ekf/one_iteration/run.sh +++ b/runtime/experiments/applications/ekf/one_iteration/run.sh @@ -43,7 +43,7 @@ run_functional_tests() { fi } -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/imageclassification/run.sh b/runtime/experiments/applications/imageclassification/run.sh index 8913926..2a20ca6 100755 --- a/runtime/experiments/applications/imageclassification/run.sh +++ b/runtime/experiments/applications/imageclassification/run.sh @@ -29,7 +29,7 @@ run_functional_tests() { echo "${image}" >> "${results_directory}/cifar10_rand.txt" curl --data-binary "@${image}" -s "${hostname}:10000" >> "${results_directory}/cifar10_rand.txt" done - + echo "$same_image" >> "${results_directory}/cifar10_same.txt" curl --data-binary "@$same_image" -s "${hostname}:10001" >> "${results_directory}/cifar10_same.txt" @@ -56,7 +56,7 @@ run_perf_tests() { wait -n "$(pgrep hey | tr '\n' ' ')" done ((batch_id++)) - + if [ "$workload" = "cifar10_rand" ]; then get_random_image image else @@ -75,7 +75,6 @@ run_perf_tests() { done } - # Process the experimental results and generate human-friendly results for success rate, throughput, and latency process_results() { if (($# != 1)); then @@ -129,7 +128,7 @@ process_results() { return 0 } -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" local same_image # for the cifar10_same workload diff --git a/runtime/experiments/applications/imageresize/by_resolution/run.sh b/runtime/experiments/applications/imageresize/by_resolution/run.sh index 74da293..0ec1390 100755 --- a/runtime/experiments/applications/imageresize/by_resolution/run.sh +++ b/runtime/experiments/applications/imageresize/by_resolution/run.sh @@ -159,7 +159,7 @@ process_results() { return 0 } -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/imageresize/test/run.sh b/runtime/experiments/applications/imageresize/test/run.sh index d914a78..6d6fa73 100755 --- a/runtime/experiments/applications/imageresize/test/run.sh +++ b/runtime/experiments/applications/imageresize/test/run.sh @@ -15,7 +15,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/licenseplate/by_plate_count/run.sh b/runtime/experiments/applications/licenseplate/by_plate_count/run.sh index 01c67d3..866555d 100755 --- a/runtime/experiments/applications/licenseplate/by_plate_count/run.sh +++ b/runtime/experiments/applications/licenseplate/by_plate_count/run.sh @@ -131,7 +131,7 @@ run_perf_tests() { done } -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/ocr/by_dpi/run.sh b/runtime/experiments/applications/ocr/by_dpi/run.sh index 1a5ef51..99ebff2 100755 --- a/runtime/experiments/applications/ocr/by_dpi/run.sh +++ b/runtime/experiments/applications/ocr/by_dpi/run.sh @@ -12,7 +12,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -ri iteration_count=100 local -ri word_count=100 diff --git a/runtime/experiments/applications/ocr/by_font/run.sh b/runtime/experiments/applications/ocr/by_font/run.sh index 460cca0..f7949fa 100755 --- a/runtime/experiments/applications/ocr/by_font/run.sh +++ b/runtime/experiments/applications/ocr/by_font/run.sh @@ -12,7 +12,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -ri iteration_count=100 local -ri word_count=100 diff --git a/runtime/experiments/applications/ocr/by_word/run.sh b/runtime/experiments/applications/ocr/by_word/run.sh index b15e747..04a5289 100755 --- a/runtime/experiments/applications/ocr/by_word/run.sh +++ b/runtime/experiments/applications/ocr/by_word/run.sh @@ -13,7 +13,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -ir iteration_count=100 local -ra word_counts=(1 10 100) diff --git a/runtime/experiments/applications/ocr/fivebyeight/run.sh b/runtime/experiments/applications/ocr/fivebyeight/run.sh index e924027..4aa5a87 100755 --- a/runtime/experiments/applications/ocr/fivebyeight/run.sh +++ b/runtime/experiments/applications/ocr/fivebyeight/run.sh @@ -12,7 +12,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/ocr/handwriting/run.sh b/runtime/experiments/applications/ocr/handwriting/run.sh index 7f70036..566750b 100755 --- a/runtime/experiments/applications/ocr/handwriting/run.sh +++ b/runtime/experiments/applications/ocr/handwriting/run.sh @@ -12,7 +12,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/applications/ocr/hyde/run.sh b/runtime/experiments/applications/ocr/hyde/run.sh index bdd74a8..cbf98b9 100755 --- a/runtime/experiments/applications/ocr/hyde/run.sh +++ b/runtime/experiments/applications/ocr/hyde/run.sh @@ -12,7 +12,7 @@ source panic.sh || exit 1 source path_join.sh || exit 1 source validate_dependencies.sh || exit 1 -experiment_main() { +experiment_client() { local -r hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/bash_libraries/fn_exists.sh b/runtime/experiments/bash_libraries/fn_exists.sh new file mode 100644 index 0000000..0e98084 --- /dev/null +++ b/runtime/experiments/bash_libraries/fn_exists.sh @@ -0,0 +1,8 @@ +# shellcheck shell=bash +if [ -n "$__fn_exists_sh__" ]; then return; fi +__fn_exists_sh__=$(date) + +fn_exists() { + declare -f -F "$1" > /dev/null + return $? +} diff --git a/runtime/experiments/bash_libraries/framework.sh b/runtime/experiments/bash_libraries/framework.sh index 4ebfa3e..abc1b35 100644 --- a/runtime/experiments/bash_libraries/framework.sh +++ b/runtime/experiments/bash_libraries/framework.sh @@ -11,9 +11,10 @@ __framework_sh__=$(date) # main "$@" # # In your script, implement the following functions above main: -# - experiment_main +# - experiment_client # +source "fn_exists.sh" || exit 1 source "path_join.sh" || exit 1 source "panic.sh" || exit 1 @@ -28,14 +29,17 @@ __framework_sh__usage() { echo " -v,--valgrind Debug under Valgrind but do not run client" echo " -p,--perf Run under perf. Limited to running on a baremetal Linux host!" echo " -h,--help Display usage information" + echo " -n,--name=name Provide a unique name for this experimental run. Defaults to timestamp" } # Declares application level global state __framework_sh__initialize_globals() { # timestamp is used to name the results directory for a particular test run + # This can be manually overridden via the name argument # shellcheck disable=SC2155 # shellcheck disable=SC2034 declare -gir __framework_sh__timestamp=$(date +%s) + declare -g __framework_sh__experiment_name="$__framework_sh__timestamp" # Globals used by parse_arguments declare -g __framework_sh__target="" @@ -102,6 +106,15 @@ __framework_sh__parse_arguments() { __framework_sh__role=valgrind shift ;; + -n=* | --name=*) + if [[ -d "$__framework_sh__application_directory/res/${i#*=}/" ]]; then + echo "Experiment ${i#*=} already exists. Pick a unique name!" + exit 1 + fi + echo "Set experiment name to ${i#*=}" + __framework_sh__experiment_name="${i#*=}" + shift + ;; -e=* | --envfile=*) if [[ "$__framework_sh__role" == "client" ]]; then echo "Expected to be used with run by the server" @@ -175,27 +188,22 @@ __framework_sh__log_environment() { # $3 - JSON specification __framework_sh__start_runtime() { printf "Starting Runtime: " - if (($# != 3)); then + if (($# != 2)); then printf "[ERR]\n" panic "invalid number of arguments \"$1\"" return 1 - elif ! [[ -d "$1" ]]; then - printf "[ERR]\n" - panic "directory \"$1\" does not exist" - return 1 - elif ! [[ $2 =~ ^(foreground|background)$ ]]; then + elif ! [[ $1 =~ ^(foreground|background)$ ]]; then printf "[ERR]\n" - panic "expected foreground or background was \"$2\"" + panic "expected foreground or background was \"$1\"" return 1 - elif [[ ! -f "$3" || "$3" != *.json ]]; then + elif [[ ! -f "$2" || "$2" != *.json ]]; then printf "[ERR]\n" - panic "\"$3\" does not exist or is not a JSON" + panic "\"$2\" does not exist or is not a JSON" return 1 fi - local -r scheduler="$1" - local -r how_to_run="$2" - local -r specification="$3" + local -r how_to_run="$1" + local -r specification="$2" local -r log_name=log.txt local log="$RESULTS_DIRECTORY/${log_name}" @@ -208,6 +216,7 @@ __framework_sh__start_runtime() { ;; "foreground") sledgert "$specification" + fn_exists experiment_server_post && experiment_server_post ;; esac @@ -233,9 +242,9 @@ __framework_sh__run_server() { local -r how_to_run="$1" - __framework_sh__start_runtime "$RESULTS_DIRECTORY" "$how_to_run" "$__framework_sh__application_directory/spec.json" || { + __framework_sh__start_runtime "$how_to_run" "$__framework_sh__application_directory/spec.json" || { echo "__framework_sh__start_runtime RC: $?" - panic "Error calling __framework_sh__start_runtime $RESULTS_DIRECTORY $how_to_run $__framework_sh__application_directory/spec.json" + panic "Error calling __framework_sh__start_runtime $how_to_run $__framework_sh__application_directory/spec.json" return 1 } @@ -249,6 +258,7 @@ __framework_sh__run_perf() { fi perf record -g -s sledgert "$__framework_sh__application_directory/spec.json" + fn_exists experiment_server_post && experiment_server_post } __framework_sh__run_valgrind() { @@ -258,6 +268,7 @@ __framework_sh__run_valgrind() { fi valgrind --leak-check=full sledgert "$__framework_sh__application_directory/spec.json" + fn_exists experiment_server_post && experiment_server_post } # Starts the Sledge Runtime under GDB @@ -282,11 +293,13 @@ __framework_sh__run_debug() { --eval-command="run $__framework_sh__application_directory/spec.json" \ sledgert fi + + fn_exists experiment_server_post && experiment_server_post return 0 } __framework_sh__run_client() { - experiment_main "$__framework_sh__target" "$RESULTS_DIRECTORY" || return 1 + experiment_client "$__framework_sh__target" "$RESULTS_DIRECTORY" || return 1 return 0 } @@ -362,10 +375,10 @@ __framework_sh__create_and_export_results_directory() { # If we are running both client, and server, we need to namespace by scheduler since we run both variants case "$__framework_sh__role" in "both") - dir="$__framework_sh__application_directory/res/$__framework_sh__timestamp/$subdirectory" + dir="$__framework_sh__application_directory/res/$__framework_sh__experiment_name/$subdirectory" ;; "client" | "server" | "debug" | "perf" | "valgrind") - dir="$__framework_sh__application_directory/res/$__framework_sh__timestamp" + dir="$__framework_sh__application_directory/res/$__framework_sh__experiment_name" ;; *) panic "${FUNCNAME[0]} Unexpected $__framework_sh__role" @@ -383,8 +396,8 @@ __framework_sh__create_and_export_results_directory() { # Responsible for ensuring that the experiment file meets framework assumptions __framework_sh__validate_experiment() { - if [[ $(type -t experiment_main) != "function" ]]; then - panic "function experiment_main was not defined" + if [[ $(type -t experiment_client) != "function" ]]; then + panic "function experiment_client was not defined" return 1 fi @@ -438,5 +451,7 @@ __framework_sh__stop_runtime() { printf "[ERR]\npkill hey: %d\n" $? exit 1 } + + fn_exists experiment_server_post && experiment_server_post printf "[OK]\n" } diff --git a/runtime/experiments/bimodal/run.sh b/runtime/experiments/bimodal/run.sh index 051562c..0eb603f 100755 --- a/runtime/experiments/bimodal/run.sh +++ b/runtime/experiments/bimodal/run.sh @@ -250,7 +250,7 @@ process_results() { } # Expected Symbol used by the framework -experiment_main() { +experiment_client() { local -r target_hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/concurrency/run.sh b/runtime/experiments/concurrency/run.sh index 72bd0ae..3d5e4cc 100755 --- a/runtime/experiments/concurrency/run.sh +++ b/runtime/experiments/concurrency/run.sh @@ -176,7 +176,7 @@ process_results() { } # Expected Symbol used by the framework -experiment_main() { +experiment_client() { local -r target_hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/deadline_description/fifo_nopreemption.env b/runtime/experiments/deadline_description/fifo_nopreemption.env index d043589..7e95145 100644 --- a/runtime/experiments/deadline_description/fifo_nopreemption.env +++ b/runtime/experiments/deadline_description/fifo_nopreemption.env @@ -1,3 +1,4 @@ SLEDGE_SCHEDULER=FIFO SLEDGE_DISABLE_PREEMPTION=true SLEDGE_NWORKERS=1 +SLEDGE_SANDBOX_PERF_LOG=perf.log diff --git a/runtime/experiments/deadline_description/run.sh b/runtime/experiments/deadline_description/run.sh index 9ef24ef..eb808a9 100755 --- a/runtime/experiments/deadline_description/run.sh +++ b/runtime/experiments/deadline_description/run.sh @@ -22,27 +22,17 @@ profile() { local hostname="$1" local -r results_directory="$2" - echo "$results_directory/ekf/benchmark.csv" - # ekf - mkdir "$results_directory/ekf" - hey -disable-compression -disable-keepalive -disable-redirects -n 16 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./ekf/initial_state.dat" "http://${hostname}:10000" > /dev/null - hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./ekf/initial_state.dat" "http://${hostname}:10000" > "$results_directory/ekf/benchmark.csv" + hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./ekf/initial_state.dat" "http://${hostname}:10000" > /dev/null # Resize - mkdir "$results_directory/resize" - hey -disable-compression -disable-keepalive -disable-redirects -n 16 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./resize/shrinking_man_large.jpg" "http://${hostname}:10001" > /dev/null - hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./resize/shrinking_man_large.jpg" "http://${hostname}:10001" > "$results_directory/resize/benchmark.csv" + hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./resize/shrinking_man_large.jpg" "http://${hostname}:10001" > /dev/null # lpd - mkdir "$results_directory/lpd" - hey -disable-compression -disable-keepalive -disable-redirects -n 16 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./lpd/Cars0.png" "http://${hostname}:10002" > /dev/null - hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./lpd/Cars0.png" "http://${hostname}:10002" > "$results_directory/lpd/benchmark.csv" + hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./lpd/Cars0.png" "http://${hostname}:10002" > /dev/null # gocr - Hit error. Commented out temporarily - # mkdir "$results_directory/gocr" - # hey -disable-compression -disable-keepalive -disable-redirects -n 16 -c 1 -cpus 1 -t 0 -o csv -H 'Expect:' -H "Content-Type: text/plain" -m GET -D "./gocr/hyde.pnm" "http://${hostname}:10003" > /dev/null - # hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./gocr/hyde.pnm" "http://${hostname}:10003" > "$results_directory/gocr/benchmark.csv" + # hey -disable-compression -disable-keepalive -disable-redirects -n 256 -c 1 -cpus 1 -t 0 -o csv -m GET -D "./gocr/hyde.pnm" "http://${hostname}:10003" > /dev/null } get_baseline_execution() { @@ -120,9 +110,8 @@ process_results() { local results_directory="$1" for workload in "${workloads[@]}"; do - # Filter on 200s, subtract DNS time, convert from s to us, and sort - awk -F, '$7 == 200 {print (($1 - $2) * 1000000)}' < "$results_directory/$workload/benchmark.csv" \ - | sort -g > "$results_directory/$workload/response_times_sorted.csv" + mkdir "$results_directory/$workload" + awk -F, '$2 == "'"$workload"'" {printf("%.0f\n", $6 / $13)}' < "$results_directory/perf.log" | sort -g > "$results_directory/$workload/response_times_sorted.csv" done generate_spec "$results_directory" @@ -130,12 +119,16 @@ process_results() { return 0 } -experiment_main() { +experiment_server_post() { + mv "$__run_sh__base_path/perf.log" "$RESULTS_DIRECTORY/perf.log" + process_results "$RESULTS_DIRECTORY" +} + +experiment_client() { local -r hostname="$1" local -r results_directory="$2" profile "$hostname" "$results_directory" || return 1 - process_results "$results_directory" } main "$@" diff --git a/runtime/experiments/payload/run.sh b/runtime/experiments/payload/run.sh index 5876118..15ed115 100755 --- a/runtime/experiments/payload/run.sh +++ b/runtime/experiments/payload/run.sh @@ -173,7 +173,7 @@ process_results() { } # Expected Symbol used by the framework -experiment_main() { +experiment_client() { local -r target_hostname="$1" local -r results_directory="$2" diff --git a/runtime/experiments/workload_mix/run.sh b/runtime/experiments/workload_mix/run.sh index 1672cce..be3380a 100755 --- a/runtime/experiments/workload_mix/run.sh +++ b/runtime/experiments/workload_mix/run.sh @@ -214,7 +214,7 @@ process_results() { } # Expected Symbol used by the framework -experiment_main() { +experiment_client() { local -r target_hostname="$1" local -r results_directory="$2"