From 97e8c58cca1aa1a34d33c0605ca5f1c0d7f0ac76 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Tue, 13 Apr 2021 01:41:50 +0000 Subject: [PATCH 01/13] fix: Remove extra % in awk printf --- runtime/experiments/deadline/client.sh | 2 +- runtime/experiments/deadline/client2.sh | 2 +- runtime/experiments/deadline/client3.sh | 2 +- runtime/experiments/deadline/fix_calcs.sh | 2 +- runtime/experiments/deadline/fix_calcs2.sh | 2 +- runtime/experiments/deadline/run_relative.sh | 2 +- runtime/experiments/preemption/client.sh | 2 +- runtime/experiments/preemption/fix_results.sh | 2 +- runtime/experiments/preemption/run_relative.sh | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/runtime/experiments/deadline/client.sh b/runtime/experiments/deadline/client.sh index d1acfb1..76e4a5f 100755 --- a/runtime/experiments/deadline/client.sh +++ b/runtime/experiments/deadline/client.sh @@ -57,7 +57,7 @@ for ((i = 1; i < 2; i++)); do # Calculate Success Rate for csv awk -F, ' $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/deadline/client2.sh b/runtime/experiments/deadline/client2.sh index 2d9ca11..0dd9f65 100755 --- a/runtime/experiments/deadline/client2.sh +++ b/runtime/experiments/deadline/client2.sh @@ -60,7 +60,7 @@ for ((i = 0; i < 2; i++)); do awk -F, ' $7 == 200 {denom++} $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / denom * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/deadline/client3.sh b/runtime/experiments/deadline/client3.sh index 10e186c..e802e95 100755 --- a/runtime/experiments/deadline/client3.sh +++ b/runtime/experiments/deadline/client3.sh @@ -58,7 +58,7 @@ for ((i = 0; i < 1; i++)); do awk -F, ' $7 == 200 {denom++} $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)} + END{printf "'"$payload"',%3.5%\n", (ok / denom * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/deadline/fix_calcs.sh b/runtime/experiments/deadline/fix_calcs.sh index 7d8d5eb..c70ecaf 100755 --- a/runtime/experiments/deadline/fix_calcs.sh +++ b/runtime/experiments/deadline/fix_calcs.sh @@ -28,7 +28,7 @@ for ((i = 0; i < 2; i++)); do # Calculate Success Rate for csv awk -F, ' $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/deadline/fix_calcs2.sh b/runtime/experiments/deadline/fix_calcs2.sh index adff3c2..3a79b00 100755 --- a/runtime/experiments/deadline/fix_calcs2.sh +++ b/runtime/experiments/deadline/fix_calcs2.sh @@ -62,7 +62,7 @@ for ((i = 0; i < 2; i++)); do awk -F, ' $7 == 200 {denom++} $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / denom * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/deadline/run_relative.sh b/runtime/experiments/deadline/run_relative.sh index 0acc690..e286b55 100755 --- a/runtime/experiments/deadline/run_relative.sh +++ b/runtime/experiments/deadline/run_relative.sh @@ -74,7 +74,7 @@ for scheduler in ${schedulers[*]}; do # Calculate Success Rate for csv awk -F, ' $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/preemption/client.sh b/runtime/experiments/preemption/client.sh index e0b7584..da778b2 100755 --- a/runtime/experiments/preemption/client.sh +++ b/runtime/experiments/preemption/client.sh @@ -62,7 +62,7 @@ for payload in ${payloads[*]}; do # Calculate Success Rate for csv awk -F, ' $7 == 200 {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/preemption/fix_results.sh b/runtime/experiments/preemption/fix_results.sh index 80c6b1f..6eb4568 100755 --- a/runtime/experiments/preemption/fix_results.sh +++ b/runtime/experiments/preemption/fix_results.sh @@ -28,7 +28,7 @@ for payload in ${payloads[*]}; do # Calculate Success Rate for csv awk -F, ' $7 == 200 {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort diff --git a/runtime/experiments/preemption/run_relative.sh b/runtime/experiments/preemption/run_relative.sh index 0acc690..e286b55 100755 --- a/runtime/experiments/preemption/run_relative.sh +++ b/runtime/experiments/preemption/run_relative.sh @@ -74,7 +74,7 @@ for scheduler in ${schedulers[*]}; do # Calculate Success Rate for csv awk -F, ' $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort From 1a72791c1bad8783c3e992c62edefa12196a128f Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Tue, 13 Apr 2021 01:49:31 +0000 Subject: [PATCH 02/13] feat: first pass at new run format --- runtime/experiments/common.sh | 74 +++++- runtime/experiments/deadline/run.sh | 389 ++++++++++++++++++++++++---- 2 files changed, 406 insertions(+), 57 deletions(-) diff --git a/runtime/experiments/common.sh b/runtime/experiments/common.sh index bbfe879..7c0ef1e 100644 --- a/runtime/experiments/common.sh +++ b/runtime/experiments/common.sh @@ -1,5 +1,12 @@ #!/bin/bash +dump_bash_stack() { + echo "Call Stack:" + for func in "${FUNCNAME[@]}"; do + echo "$func" + done +} + log_environment() { if ! command -v git &> /dev/null; then echo "git could not be found" @@ -35,8 +42,40 @@ log_environment() { echo "*************" } +# Given a file, returns the number of results +# This assumes a *.csv file with a header +# $1 the file we want to check for results +# $2 an optional return nameref. If not set, writes results to STDOUT +get_result_count() { + if (($# != 1)); then + echo "${FUNCNAME[0]} error: insufficient parameters" + dump_bash_stack + elif [[ ! -f $1 ]]; then + echo "${FUNCNAME[0]} error: the file $1 does not exist" + dump_bash_stack + fi + + local -r file=$1 + + # Subtract one line for the header + local -i count=$(($(wc -l < "$file") - 1)) + + if (($# == 2)); then + local -n __result=$2 + __result=count + else + echo "$count" + fi + + if ((count > 0)); then + return 0 + else + return 1 + fi +} + kill_runtime() { - echo -n "Running Cleanup: " + echo -n "Killing Runtime: " pkill sledgert > /dev/null 2> /dev/null pkill hey > /dev/null 2> /dev/null echo "[DONE]" @@ -44,18 +83,20 @@ kill_runtime() { generate_gnuplots() { if ! command -v gnuplot &> /dev/null; then - echo "gnuplot could not be found" + echo "${FUNCNAME[0]} error: gnuplot could not be found in path" exit fi # shellcheck disable=SC2154 if [ -z "$results_directory" ]; then - echo "results_directory is unset or empty" - exit + echo "${FUNCNAME[0]} error: results_directory was unset or empty" + dump_bash_stack + exit 1 fi # shellcheck disable=SC2154 if [ -z "$experiment_directory" ]; then - echo "experiment_directory is unset or empty" - exit + echo "${FUNCNAME[0]} error: experiment_directory was unset or empty" + dump_bash_stack + exit 1 fi cd "$results_directory" || exit gnuplot ../../latency.gnuplot @@ -63,3 +104,24 @@ generate_gnuplots() { gnuplot ../../throughput.gnuplot cd "$experiment_directory" || exit } + +# Takes a variadic number of paths to *.csv files and converts to *.dat files in the same directory +csv_to_dat() { + if (($# == 0)); then + echo "${FUNCNAME[0]} error: insufficient parameters" + dump_bash_stack + fi + + for arg in "$@"; do + if ! [[ "$arg" =~ ".csv"$ ]]; then + echo "${FUNCNAME[0]} error: $arg is not a *.csv file" + dump_bash_stack + exit 1 + fi + done + + for file in "$@"; do + echo -n "#" > "${file/.csv/.dat}" + tr ',' ' ' < "$file" | column -t >> "${file/.csv/.dat}" + done +} diff --git a/runtime/experiments/deadline/run.sh b/runtime/experiments/deadline/run.sh index c69a3e5..4b176d9 100755 --- a/runtime/experiments/deadline/run.sh +++ b/runtime/experiments/deadline/run.sh @@ -3,78 +3,250 @@ source ../common.sh # This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate # Use -d flag if running under gdb +# TODO: GDB? Debug? +usage() { + echo "$0 [options...]" + echo "" + echo "Options:" + echo " -t,--target= Execute as client against remote URL" + echo " -s,--serve= Serve with scheduling policy, but do not run client" +} -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) +initialize_globals() { + # timestamp is used to name the results directory for a particular test run + # shellcheck disable=SC2155 + declare -gir timestamp=$(date +%s) -schedulers=(EDF FIFO) -for scheduler in ${schedulers[*]}; do + # shellcheck disable=SC2155 + declare -gr experiment_directory=$(pwd) - results_directory="$experiment_directory/res/$timestamp/$scheduler" - log=log.txt + # shellcheck disable=SC2155 + declare -gr binary_directory=$(cd ../../bin && pwd) - mkdir -p "$results_directory" - log_environment >> "$results_directory/$log" + # Scrape the perf window size from the source if possible + declare -gr perf_window_path="../../include/perf_window.h" + declare -gi perf_window_buffer_size + if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then + echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" + echo "Defaulting to 16" + declare -ir perf_window_buffer_size=16 + fi - # Start the runtime - if [ "$1" != "-d" ]; then - SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=$scheduler PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 - else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" + declare -gx target="" + declare -gx policy="" + declare -gx role="both" +} + +parse_arguments() { + for i in "$@"; do + case $i in + -t=* | --target=*) + if [[ "$role" == "server" ]]; then + echo "Cannot set target when server" + usage + exit 1 + fi + role=client + target="${i#*=}" + shift # past argument=value + ;; + -s=* | --serve=*) + if [[ "$role" == "client" ]]; then + echo "Cannot serve with target is set" + usage + exit 1 + fi + role=server + policy="${i#*=}" + if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" + usage + exit 1 + fi + shift # past argument=value + ;; + -h | --help) + usage + ;; + *) + echo "$1 is a not a valid option" + usage + exit 1 + ;; + esac + done + + # Set globals as read only + declare -r target + declare -r policy + declare -r role +} + +start_runtime() { + if (($# != 2)); then + echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + echo "${FUNCNAME[0]} error: expected EDF or FIFO was \"$1\"" + return 1 + elif ! [[ -d "$2" ]]; then + echo "${FUNCNAME[0]} error: \"$2\" does not exist" + return 1 fi - inputs=(40 10) - duration_sec=15 - offset=5 + local -r scheduler="$1" + local -r results_directory="$2" + + local -r log_name=log.txt + local log="$results_directory/${log_name}" + + log_environment >> "$log" + + SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=$scheduler PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$log" 2>> "$log" & + return $? +} + +# Seed enough work to fill the perf window buffers +run_samples() { + local hostname="${1:-localhost}" - # Execute workloads long enough for runtime to learn excepted execution time echo -n "Running Samples: " - for input in ${inputs[*]}; do - hey -z ${duration_sec}s -cpus 3 -t 0 -o csv -m GET -d "$input\n" http://localhost:$((10000 + input)) - done - echo "[DONE]" - sleep 5 + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" || { + echo "error" + return 1 + } + + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" || { + echo "error" + return 1 + } + + return 0 +} + +# $1 (results_directory) - a directory where we will store our results +# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost +run_experiments() { + if (($# < 1 || $# > 2)); then + echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" + exit + elif ! [[ -d "$1" ]]; then + echo "${FUNCNAME[0]} error: \"$2\" does not exist" + exit + elif (($# > 2)) && [[ ! $1 =~ ^(EDF|FIFO)$ ]]; then + echo "${FUNCNAME[0]} error: expected EDF or FIFO was \"$1\"" + exit + fi + + local results_directory="$1" + local hostname="${2:-localhost}" + + # The duration in seconds that the low priority task should run before the high priority task starts + local -ir offset=5 + + # The duration in seconds that we want the client to send requests + local -ir duration_sec=15 echo "Running Experiments" + # Run each separately - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40.csv" - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10.csv" + echo "Running fib40" + hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40.csv" || { + echo "error" + return 1 + } + get_result_count "$results_directory/fib40.csv" || { + echo "fib40 unexpectedly has zero requests" + return 1 + } + + echo "Running fib10" + hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10.csv" || { + echo "error" + return 1 + } + get_result_count "$results_directory/fib10.csv" || { + echo "fib10 unexpectedly has zero requests" + return 1 + } + + # Run concurrently + # The lower priority has offsets to ensure it runs the entire time the high priority is trying to run + # This asynchronously trigger jobs and then wait on their pids + local -a pids=() + + echo "Running fib40_con" + hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40_con.csv" & + pids+=($!) - # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run - hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40-con.csv" & sleep $offset - hey -z ${duration_sec}s -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10-con.csv" & - sleep $((duration_sec + offset + 15)) - # Stop the runtime if not in debug mode - [ "$1" != "-d" ] && kill_runtime + echo "Running fib10_con" + hey -z "${duration_sec}s" -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10_con.csv" & + pids+=($!) + + for ((i = 0; i < "${#pids[@]}"; i++)); do + wait -n "${pids[@]}" || { + echo "error" + return 1 + } + done + + get_result_count "$results_directory/fib40_con.csv" || { + echo "fib40_con unexpectedly has zero requests" + return 1 + } + get_result_count "$results_directory/fib10_con.csv" || { + echo "fib10_con has zero requests. This might be because fib40_con saturated the runtime" + } + + return 0 +} + +process_results() { + if (($# != 1)); then + echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" + exit + elif ! [[ -d "$1" ]]; then + echo "${FUNCNAME[0]} error: \"$1\" does not exist" + exit + fi + + local -r results_directory="$1" - # Generate *.csv and *.dat results - echo -n "Parsing Results: " + echo -n "Processing Results: " + # Write headers to CSVs printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - deadlines_ms=(2 2 3000 3000) - payloads=(fib10 fib10-con fib40 fib40-con) + # The four types of results that we are capturing. + # fib10 and fib 40 are run sequentially. + # fib10_con and fib40_con are run concurrently + local -ar payloads=(fib10 fib10_con fib40 fib40_con) - for ((i = 0; i < 4; i++)); do - # for payload in ${payloads[*]}; do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} + # The deadlines for each of the workloads + local -Ar deadlines_ms=( + [fib10]=2 + [fib40]=3000 + ) - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue + for payload in "${payloads[@]}"; do + # Strip the _con suffix when getting the deadline + local -i deadline=${deadlines_ms[${payload/_con/}]} + + # Get Number of Requests, subtracting the header + local -i requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) + ((requests == 0)) && { + echo "$payload unexpectedly has zero requests" + continue + } # Calculate Success Rate for csv awk -F, ' $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" # Filter on 200s, convery from s to ms, and sort @@ -111,14 +283,129 @@ for scheduler in ${schedulers[*]}; do done # Transform csvs to dat files for gnuplot - for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" - done + csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv" # Generate gnuplots. Commented out because we don't have *.gnuplots defined # generate_gnuplots +} + +run_server() { + if (($# != 2)); then + echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" + exit + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + echo "${FUNCNAME[0]} error: expected EDF or FIFO was \"$1\"" + exit + elif ! [[ -d "$2" ]]; then + echo "${FUNCNAME[0]} error: \"$2\" does not exist" + exit + fi + + local -r scheduler="$1" + local -r results_directory="$2" + + start_runtime "$scheduler" "$log" || { + echo "${FUNCNAME[0]} error" + return 1 + } +} + +run_client() { + results_directory="$experiment_directory/res/$timestamp" + mkdir -p "$results_directory" + + run_samples "$target" || { + echo "${FUNCNAME[0]} error" + exit 1 + } + + sleep 5 + + run_experiments "$target" || { + echo "${FUNCNAME[0]} error" + exit 1 + } + + sleep 1 + + process_results "$results_directory" || { + echo "${FUNCNAME[0]} error" + exit 1 + } - # Cleanup, if requires echo "[DONE]" -done + exit 0 + +} + +run_both() { + local -ar schedulers=(EDF FIFO) + for scheduler in "${schedulers[@]}"; do + results_directory="$experiment_directory/res/$timestamp/$scheduler" + mkdir -p "$results_directory" + start_runtime "$scheduler" "$results_directory" || { + echo "${FUNCNAME[0]} Error" + exit 1 + } + + sleep 1 + + run_samples || { + echo "${FUNCNAME[0]} Error" + kill_runtime + exit 1 + } + + sleep 1 + + run_experiments "$results_directory" || { + echo "${FUNCNAME[0]} Error" + kill_runtime + exit 1 + } + + sleep 1 + kill_runtime || { + echo "${FUNCNAME[0]} Error" + exit 1 + } + + process_results "$results_directory" || { + echo "${FUNCNAME[0]} Error" + exit 1 + } + + echo "[DONE]" + exit 0 + done +} + +main() { + initialize_globals + parse_arguments "$@" + + echo "$timestamp" + + echo "Target: $target" + echo "Policy: $policy" + echo "Role: $role" + + case $role in + both) + run_both + ;; + server) + results_directory="$experiment_directory/res/$timestamp" + mkdir -p "$results_directory" + start_runtime "$target" "$results_directory" + exit 0 + ;; + client) ;; + *) + echo "Invalid state" + exit 1 + ;; + esac +} + +main "$@" From e32339bbc13e24522c8f4b05aaeb30f5ef2a2b71 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Tue, 13 Apr 2021 16:09:11 -0400 Subject: [PATCH 03/13] feat: Complete deadline experiment cleanup --- Dockerfile.x86_64 | 6 +- install_perf.sh | 26 ++ runtime/experiments/common.sh | 38 +- runtime/experiments/deadline/client.sh | 105 ----- runtime/experiments/deadline/client2.sh | 109 ------ runtime/experiments/deadline/client3.sh | 107 ----- runtime/experiments/deadline/debug.sh | 19 - runtime/experiments/deadline/fix_calcs.sh | 77 ---- runtime/experiments/deadline/fix_calcs2.sh | 111 ------ runtime/experiments/deadline/perf.sh | 14 - runtime/experiments/deadline/run.sh | 392 +++++++++++++------ runtime/experiments/deadline/run_relative.sh | 124 ------ runtime/experiments/deadline/scratch.txt | 5 - runtime/experiments/deadline/server.sh | 8 - 14 files changed, 319 insertions(+), 822 deletions(-) create mode 100755 install_perf.sh delete mode 100755 runtime/experiments/deadline/client.sh delete mode 100755 runtime/experiments/deadline/client2.sh delete mode 100755 runtime/experiments/deadline/client3.sh delete mode 100755 runtime/experiments/deadline/debug.sh delete mode 100755 runtime/experiments/deadline/fix_calcs.sh delete mode 100755 runtime/experiments/deadline/fix_calcs2.sh delete mode 100755 runtime/experiments/deadline/perf.sh delete mode 100755 runtime/experiments/deadline/run_relative.sh delete mode 100644 runtime/experiments/deadline/scratch.txt delete mode 100755 runtime/experiments/deadline/server.sh diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index bb4d8fb..8439611 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -76,6 +76,10 @@ RUN tar xvfz wasmception.tar.gz -C /sledge/awsm/wasmception # RUN curl -sS -L -O $WASI_SDK_URL && dpkg -i wasi-sdk_8.0_amd64.deb && rm -f wasi-sdk_8.0_amd64.deb # ENV WASI_SDK=/opt/wasi-sdk +# PERF +ADD install_perf.sh /sledge/install_perf.sh +RUN ./sledge/install_perf.sh + # Create non-root user and add to sudoers ARG USERNAME=dev ARG USER_UID=1000 @@ -116,4 +120,4 @@ ENV PATH=/opt/sledge/bin:$PATH # TODO: Does the build process for the sample applications actually copy here? # TODO: Should we create a special SLEDGE_MODULE_PATH that is searched for these modules? -ENV LD_LIBRARY_PATH=/opt/sledge/bin:LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/opt/sledge/bin:$LD_LIBRARY_PATH diff --git a/install_perf.sh b/install_perf.sh new file mode 100755 index 0000000..cf45953 --- /dev/null +++ b/install_perf.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# If already installed, just return +[[ -x perf ]] && return 0 + +[[ "$(whoami)" != "root" ]] && { + echo "Expected to run as root" + exit 1 +} + +# Under WSL2, perf has to be installed from source +if grep --silent 'WSL2' <(uname -r); then + echo "WSL detected. perf must be built from source" + echo "WSL2 support is WIP and not currently functional" + exit 0 + + sudo apt-get install flex bison python3-dev liblzma-dev libnuma-dev zlib1g libperl-dev libgtk2.0-dev libslang2-dev systemtap-sdt-dev libelf-dev binutils-dev libbabeltrace-dev libdw-dev libunwind-dev libiberty-dev --yes + git clone --depth 1 https://github.com/microsoft/WSL2-Linux-Kernel ~/WSL2-Linux-Kernel + make -Wno-error -j8 -C ~/WSL2-Linux-Kernel/tools/perf + sudo cp ~/WSL2-Linux-Kernel/tools/perf/perf /usr/local/bin + # rm -rf ~/WSL2-Linux-Kernel +else + apt-get install "linux-tools-$(uname -r)" linux-tools-generic -y +fi + +exit 0 diff --git a/runtime/experiments/common.sh b/runtime/experiments/common.sh index 7c0ef1e..2663148 100644 --- a/runtime/experiments/common.sh +++ b/runtime/experiments/common.sh @@ -1,9 +1,20 @@ #!/bin/bash -dump_bash_stack() { +declare __common_did_dump_callstack=false + +error_msg() { + [[ "$__common_did_dump_callstack" == false ]] && { + printf "%.23s %s() in %s, line %s: %s\n" "$(date +%F.%T.%N)" "${FUNCNAME[1]}" "${BASH_SOURCE[1]##*/}" "${BASH_LINENO[0]}" "${@}" + __common_dump_callstack + __common_did_dump_callstack=true + } +} + +__common_dump_callstack() { echo "Call Stack:" - for func in "${FUNCNAME[@]}"; do - echo "$func" + # Skip the dump_bash_stack and error_msg_frames + for ((i = 2; i < ${#FUNCNAME[@]}; i++)); do + printf "\t%d - %s\n" "$((i - 2))" "${FUNCNAME[i]}" done } @@ -45,14 +56,17 @@ log_environment() { # Given a file, returns the number of results # This assumes a *.csv file with a header # $1 the file we want to check for results -# $2 an optional return nameref. If not set, writes results to STDOUT +# $2 an optional return nameref get_result_count() { if (($# != 1)); then - echo "${FUNCNAME[0]} error: insufficient parameters" - dump_bash_stack + error_msg "insufficient parameters. $#/1" + return 1 elif [[ ! -f $1 ]]; then - echo "${FUNCNAME[0]} error: the file $1 does not exist" - dump_bash_stack + error_msg "the file $1 does not exist" + return 1 + elif [[ ! -s $1 ]]; then + error_msg "the file $1 is size 0" + return 1 fi local -r file=$1 @@ -61,10 +75,8 @@ get_result_count() { local -i count=$(($(wc -l < "$file") - 1)) if (($# == 2)); then + # shellcheck disable=2034 local -n __result=$2 - __result=count - else - echo "$count" fi if ((count > 0)); then @@ -75,10 +87,10 @@ get_result_count() { } kill_runtime() { - echo -n "Killing Runtime: " + printf "Stopping Runtime: " pkill sledgert > /dev/null 2> /dev/null pkill hey > /dev/null 2> /dev/null - echo "[DONE]" + printf "[OK]\n" } generate_gnuplots() { diff --git a/runtime/experiments/deadline/client.sh b/runtime/experiments/deadline/client.sh deleted file mode 100755 index 76e4a5f..0000000 --- a/runtime/experiments/deadline/client.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -host=localhost -timestamp=$(date +%s) -experiment_directory=$(pwd) - -results_directory="$experiment_directory/res/$timestamp" -log=log.txt - -mkdir -p "$results_directory" -log_environment >> "$results_directory/$log" - -inputs=(40 10) -duration_sec=60 -offset=5 - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -for input in ${inputs[*]}; do - hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input)) -done -echo "[DONE]" -sleep 5 - -echo "Running Experiments" - -# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run -hey -n 1000 -c 1000 -cpus 6 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 > "$results_directory/fib40-con.csv" -sleep $offset -hey -n 25000 -c 1000000 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 > "$results_directory/fib10-con.csv" & -sleep $((duration_sec + offset + 45)) - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -deadlines_ms=(20 20000) -# durations_s=(60 70) -payloads=(fib10-con fib40-con) - -for ((i = 1; i < 2; i++)); do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - # duration=${durations_s[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - # throughput=$(echo "$oks/$duration" | bc) - # printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if requires -echo "[DONE]" diff --git a/runtime/experiments/deadline/client2.sh b/runtime/experiments/deadline/client2.sh deleted file mode 100755 index 0dd9f65..0000000 --- a/runtime/experiments/deadline/client2.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -host=192.168.1.13 -# host=localhost -timestamp=$(date +%s) -experiment_directory=$(pwd) - -results_directory="$experiment_directory/res/$timestamp" -log=log.txt - -mkdir -p "$results_directory" -log_environment >> "$results_directory/$log" - -inputs=(40 10) -duration_sec=30 -offset=5 - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -for input in ${inputs[*]}; do - hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input)) -done -echo "[DONE]" -sleep 5 - -echo "Running Experiments" - -# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run -hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 > "$results_directory/fib40-con.csv" & -sleep $offset -hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 > "$results_directory/fib10-con.csv" & -sleep $((duration_sec + offset + 15)) -sleep 30 - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -deadlines_ms=(20 20000) -payloads=(fib10-con fib40-con) -durations_s=(30 40) - -for ((i = 0; i < 2; i++)); do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - duration=${durations_s[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {denom++} - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / denom * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if requires -echo "[DONE]" diff --git a/runtime/experiments/deadline/client3.sh b/runtime/experiments/deadline/client3.sh deleted file mode 100755 index e802e95..0000000 --- a/runtime/experiments/deadline/client3.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -host=192.168.1.13 -# host=localhost -timestamp=$(date +%s) -experiment_directory=$(pwd) - -results_directory="$experiment_directory/res/$timestamp" -log=log.txt - -mkdir -p "$results_directory" -log_environment >> "$results_directory/$log" - -inputs=(10) -duration_sec=30 -offset=5 - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -hey -n 16 -c 4 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 -echo "[DONE]" -sleep 5 - -echo "Running Experiments" - -# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run -# hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" & -# sleep $offset -hey -z ${duration_sec}s -cpus 6 -c 400 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 > "$results_directory/fib10-con.csv" -# sleep $((duration_sec + offset + 15)) -# sleep 30 - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -deadlines_ms=(20 20000) -payloads=(fib10-con fib40-con) -durations_s=(30 40) - -for ((i = 0; i < 1; i++)); do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - duration=${durations_s[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {denom++} - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5%\n", (ok / denom * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if requires -echo "[DONE]" diff --git a/runtime/experiments/deadline/debug.sh b/runtime/experiments/deadline/debug.sh deleted file mode 100755 index f40fa45..0000000 --- a/runtime/experiments/deadline/debug.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" - -gdb --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert diff --git a/runtime/experiments/deadline/fix_calcs.sh b/runtime/experiments/deadline/fix_calcs.sh deleted file mode 100755 index c70ecaf..0000000 --- a/runtime/experiments/deadline/fix_calcs.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -experiment_directory=$(pwd) -results_directory="$experiment_directory/res/1606615320-fifo-adm" - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -deadlines_ms=(20 20000) -payloads=(fib10-con fib40-con) - -for ((i = 0; i < 2; i++)); do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if requires -echo "[DONE]" diff --git a/runtime/experiments/deadline/fix_calcs2.sh b/runtime/experiments/deadline/fix_calcs2.sh deleted file mode 100755 index 3a79b00..0000000 --- a/runtime/experiments/deadline/fix_calcs2.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -host=192.168.1.13 -# host=localhost -# timestamp=$(date +%s) -timestamp=1606697099 -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) - -results_directory="$experiment_directory/res/$timestamp" -log=log.txt - -mkdir -p "$results_directory" -log_environment >> "$results_directory/$log" - -inputs=(40 10) -duration_sec=60 -offset=5 - -# Execute workloads long enough for runtime to learn excepted execution time -# echo -n "Running Samples: " -# for input in ${inputs[*]}; do -# hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input)) -# done -# echo "[DONE]" -# sleep 5 - -# echo "Running Experiments" - -# # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run -# hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" & -# sleep $offset -# hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" & -# sleep $((duration_sec + offset + 15)) -# sleep 30 - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -deadlines_ms=(20 20000) -payloads=(fib10-con fib40-con) -durations_s=(60 70) - -for ((i = 0; i < 2; i++)); do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - duration=${durations_s[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {denom++} - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / denom * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if requires -echo "[DONE]" diff --git a/runtime/experiments/deadline/perf.sh b/runtime/experiments/deadline/perf.sh deleted file mode 100755 index c87504f..0000000 --- a/runtime/experiments/deadline/perf.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" - -SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=EDF perf record -g -s sledgert "$experiment_directory/spec.json" diff --git a/runtime/experiments/deadline/run.sh b/runtime/experiments/deadline/run.sh index 4b176d9..1112522 100755 --- a/runtime/experiments/deadline/run.sh +++ b/runtime/experiments/deadline/run.sh @@ -2,16 +2,24 @@ source ../common.sh # This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Success - The percentage of requests that complete by their deadlines +# TODO: Does this handle non-200s? +# Throughput - The mean number of successful requests per second +# Latency - the rount-trip resonse time (unit?) of successful requests at the p50, p90, p99, and p100 percetiles + # Use -d flag if running under gdb -# TODO: GDB? Debug? +# TODO: Just use ENV for policy and other runtime dynamic variables? usage() { echo "$0 [options...]" echo "" echo "Options:" echo " -t,--target= Execute as client against remote URL" echo " -s,--serve= Serve with scheduling policy, but do not run client" + echo " -d,--debug= Debug under GDB with scheduling policy, but do not run client" + echo " -p,--perf= Run under perf with scheduling policy, but do not run client" } +# Declares application level global state initialize_globals() { # timestamp is used to name the results directory for a particular test run # shellcheck disable=SC2155 @@ -24,19 +32,27 @@ initialize_globals() { declare -gr binary_directory=$(cd ../../bin && pwd) # Scrape the perf window size from the source if possible - declare -gr perf_window_path="../../include/perf_window.h" + local -r perf_window_path="../../include/perf_window.h" declare -gi perf_window_buffer_size if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" echo "Defaulting to 16" declare -ir perf_window_buffer_size=16 fi + declare -gir perf_window_buffer_size + + # Globals used by parse_arguments + declare -g target="" + declare -g policy="" + declare -g role="" - declare -gx target="" - declare -gx policy="" - declare -gx role="both" + # Configure environment variables + export PATH=$binary_directory:$PATH + export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH + export SLEDGE_NWORKERS=5 } +# Parses arguments from the user and sets associates global state parse_arguments() { for i in "$@"; do case $i in @@ -44,53 +60,94 @@ parse_arguments() { if [[ "$role" == "server" ]]; then echo "Cannot set target when server" usage - exit 1 + return 1 fi role=client target="${i#*=}" - shift # past argument=value + shift ;; -s=* | --serve=*) if [[ "$role" == "client" ]]; then - echo "Cannot serve with target is set" + echo "Cannot use -s,--serve with -t,--target" usage - exit 1 + return 1 fi role=server policy="${i#*=}" if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" usage - exit 1 + return 1 fi - shift # past argument=value + shift + ;; + -d=* | --debug=*) + if [[ "$role" == "client" ]]; then + echo "Cannot use -d,--debug with -t,--target" + usage + return 1 + fi + role=debug + policy="${i#*=}" + if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" + usage + return 1 + fi + shift + ;; + -p=* | --perf=*) + if [[ "$role" == "perf" ]]; then + echo "Cannot use -p,--perf with -t,--target" + usage + return 1 + fi + role=perf + policy="${i#*=}" + if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" + usage + return 1 + fi + shift ;; -h | --help) usage + exit 0 ;; *) echo "$1 is a not a valid option" usage - exit 1 + return 1 ;; esac done + # default to both if no arguments were passed + if [[ -z "$role" ]]; then + role="both" + fi + # Set globals as read only declare -r target declare -r policy declare -r role } +# Starts the Sledge Runtime start_runtime() { + printf "Starting Runtime: " if (($# != 2)); then - echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" + printf "[ERR]\n" + error_msg "invalid number of arguments \"$1\"" return 1 elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - echo "${FUNCNAME[0]} error: expected EDF or FIFO was \"$1\"" + printf "[ERR]\n" + error_msg "expected EDF or FIFO was \"$1\"" return 1 elif ! [[ -d "$2" ]]; then - echo "${FUNCNAME[0]} error: \"$2\" does not exist" + printf "[ERR]\n" + error_msg "directory \"$2\" does not exist" return 1 fi @@ -102,114 +159,132 @@ start_runtime() { log_environment >> "$log" - SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=$scheduler PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$log" 2>> "$log" & - return $? + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$experiment_directory/spec.json" >> "$log" 2>> "$log" & + + printf "[OK]\n" + return 0 } -# Seed enough work to fill the perf window buffers +# Sends requests until the per-module perf window buffers are full +# This ensures that Sledge has accurate estimates of execution time run_samples() { local hostname="${1:-localhost}" echo -n "Running Samples: " - hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" || { - echo "error" + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || { + error_msg "fib40 samples failed" return 1 } - hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" || { - echo "error" + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || { + error_msg "fib10 samples failed" return 1 } + echo "[OK]" return 0 } +# Execute the fib10 and fib40 experiments sequentially and concurrently # $1 (results_directory) - a directory where we will store our results # $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost run_experiments() { if (($# < 1 || $# > 2)); then - echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" - exit + error_msg "invalid number of arguments \"$1\"" + return 1 elif ! [[ -d "$1" ]]; then - echo "${FUNCNAME[0]} error: \"$2\" does not exist" - exit - elif (($# > 2)) && [[ ! $1 =~ ^(EDF|FIFO)$ ]]; then - echo "${FUNCNAME[0]} error: expected EDF or FIFO was \"$1\"" - exit + error_msg "directory \"$1\" does not exist" + return 1 fi local results_directory="$1" local hostname="${2:-localhost}" - # The duration in seconds that the low priority task should run before the high priority task starts - local -ir offset=5 - # The duration in seconds that we want the client to send requests local -ir duration_sec=15 - echo "Running Experiments" + # The duration in seconds that the low priority task should run before the high priority task starts + local -ir offset=5 + + printf "Running Experiments\n" # Run each separately - echo "Running fib40" - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40.csv" || { - echo "error" + printf "\tfib40: " + hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$hostname:10040" > "$results_directory/fib40.csv" 2> /dev/null || { + printf "[ERR]\n" + error_msg "fib40 failed" return 1 } get_result_count "$results_directory/fib40.csv" || { - echo "fib40 unexpectedly has zero requests" + printf "[ERR]\n" + error_msg "fib40 unexpectedly has zero requests" return 1 } + printf "[OK]\n" - echo "Running fib10" - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10.csv" || { - echo "error" + printf "\tfib10: " + hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$hostname:10010" > "$results_directory/fib10.csv" 2> /dev/null || { + printf "[ERR]\n" + error_msg "fib10 failed" return 1 } get_result_count "$results_directory/fib10.csv" || { - echo "fib10 unexpectedly has zero requests" + printf "[ERR]\n" + error_msg "fib10 unexpectedly has zero requests" return 1 } + printf "[OK]\n" # Run concurrently # The lower priority has offsets to ensure it runs the entire time the high priority is trying to run # This asynchronously trigger jobs and then wait on their pids - local -a pids=() + local fib40_con_PID + local fib10_con_PID - echo "Running fib40_con" - hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40_con.csv" & - pids+=($!) + hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40_con.csv" 2> /dev/null & + fib40_con_PID="$!" sleep $offset - echo "Running fib10_con" - hey -z "${duration_sec}s" -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10_con.csv" & - pids+=($!) - - for ((i = 0; i < "${#pids[@]}"; i++)); do - wait -n "${pids[@]}" || { - echo "error" - return 1 - } - done + hey -z "${duration_sec}s" -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10_con.csv" 2> /dev/null & + fib10_con_PID="$!" - get_result_count "$results_directory/fib40_con.csv" || { - echo "fib40_con unexpectedly has zero requests" + wait -f "$fib10_con_PID" || { + printf "\tfib10_con: [ERR]\n" + error_msg "failed to wait -f ${fib10_con_PID}" return 1 } get_result_count "$results_directory/fib10_con.csv" || { - echo "fib10_con has zero requests. This might be because fib40_con saturated the runtime" + printf "\tfib10_con: [ERR]\n" + error_msg "fib10_con has zero requests. This might be because fib40_con saturated the runtime" + return 1 } + printf "\tfib10_con: [OK]\n" + + wait -f "$fib40_con_PID" || { + printf "\tfib40_con: [ERR]\n" + error_msg "failed to wait -f ${fib40_con_PID}" + return 1 + } + get_result_count "$results_directory/fib40_con.csv" || { + printf "\tfib40_con: [ERR]\n" + error_msg "fib40_con has zero requests." + return 1 + } + printf "\tfib40_con: [OK]\n" return 0 } +# Process the experimental results and generate human-friendly results for success rate, throughput, and latency process_results() { if (($# != 1)); then - echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" - exit + error_msg "invalid number of arguments ($#, expected 1)" + return 1 elif ! [[ -d "$1" ]]; then - echo "${FUNCNAME[0]} error: \"$1\" does not exist" - exit + error_msg "directory $1 does not exist" + return 1 fi local -r results_directory="$1" @@ -227,6 +302,7 @@ process_results() { local -ar payloads=(fib10 fib10_con fib40 fib40_con) # The deadlines for each of the workloads + # TODO: Scrape these from spec.json local -Ar deadlines_ms=( [fib10]=2 [fib40]=3000 @@ -257,8 +333,11 @@ process_results() { oks=$(wc -l < "$results_directory/$payload-response.csv") ((oks == 0)) && continue # If all errors, skip line - # Get Latest Timestamp + # We determine duration by looking at the timestamp of the last complete request + # TODO: Should this instead just use the client-side synthetic duration_sec value? duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + + # Throughput is calculated as the mean number of successful requests per second throughput=$(echo "$oks/$duration" | bc) printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" @@ -290,122 +369,177 @@ process_results() { } run_server() { - if (($# != 2)); then - echo "${FUNCNAME[0]} error: invalid number of arguments \"$1\"" - exit + if (($# != 1)); then + error_msg "invalid number of arguments \"$1\"" + return 1 elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - echo "${FUNCNAME[0]} error: expected EDF or FIFO was \"$1\"" - exit - elif ! [[ -d "$2" ]]; then - echo "${FUNCNAME[0]} error: \"$2\" does not exist" - exit + error_msg "expected EDF or FIFO was \"$1\"" + return 1 fi local -r scheduler="$1" - local -r results_directory="$2" - start_runtime "$scheduler" "$log" || { - echo "${FUNCNAME[0]} error" + if [[ "$role" == "both" ]]; then + local results_directory="$experiment_directory/res/$timestamp/$scheduler" + elif [[ "$role" == "server" ]]; then + local results_directory="$experiment_directory/res/$timestamp" + else + error_msg "Unexpected $role" + return 1 + fi + + mkdir -p "$results_directory" + + start_runtime "$scheduler" "$results_directory" || { + echo "start_runtime RC: $?" + error_msg "Error calling start_runtime $scheduler $results_directory" return 1 } + + return 0 } -run_client() { - results_directory="$experiment_directory/res/$timestamp" - mkdir -p "$results_directory" +run_perf() { + if (($# != 1)); then + printf "[ERR]\n" + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + error_msg "expected EDF or FIFO was \"$1\"" + return 1 + fi - run_samples "$target" || { - echo "${FUNCNAME[0]} error" + [[ ! -x perf ]] && { + echo "perf is not present" exit 1 } - sleep 5 + SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$experiment_directory/spec.json" +} - run_experiments "$target" || { - echo "${FUNCNAME[0]} error" - exit 1 +# Starts the Sledge Runtime under GDB +run_debug() { + # shellcheck disable=SC2155 + local project_directory=$(cd ../.. && pwd) + if (($# != 1)); then + printf "[ERR]\n" + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + error_msg "expected EDF or FIFO was \"$1\"" + return 1 + fi + + local -r scheduler="$1" + + if [[ "$project_directory" != "/sledge/runtime" ]]; then + printf "It appears that you are not running in the container. Substituting path to match host environment\n" + SLEDGE_SCHEDULER="$scheduler" gdb \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="set substitute-path /sledge/runtime $project_directory" \ + --eval-command="run $experiment_directory/spec.json" \ + sledgert + else + SLEDGE_SCHEDULER="$scheduler" gdb \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="run $experiment_directory/spec.json" \ + sledgert + fi + return 0 +} + +run_client() { + if [[ "$role" == "both" ]]; then + local results_directory="$experiment_directory/res/$timestamp/$scheduler" + elif [[ "$role" == "client" ]]; then + local results_directory="$experiment_directory/res/$timestamp" + else + error_msg "${FUNCNAME[0]} Unexpected $role" + return 1 + fi + + mkdir -p "$results_directory" + + run_samples "$target" || { + error_msg "Error calling run_samples $target" + return 1 } - sleep 1 + run_experiments "$results_directory" || { + error_msg "Error calling run_experiments $results_directory" + return 1 + } process_results "$results_directory" || { - echo "${FUNCNAME[0]} error" - exit 1 + error_msg "Error calling process_results $results_directory" + return 1 } - echo "[DONE]" - exit 0 - + echo "[OK]" + return 0 } run_both() { local -ar schedulers=(EDF FIFO) for scheduler in "${schedulers[@]}"; do - results_directory="$experiment_directory/res/$timestamp/$scheduler" - mkdir -p "$results_directory" - start_runtime "$scheduler" "$results_directory" || { - echo "${FUNCNAME[0]} Error" - exit 1 - } - - sleep 1 + printf "Running %s\n" "$scheduler" - run_samples || { - echo "${FUNCNAME[0]} Error" - kill_runtime - exit 1 + run_server "$scheduler" || { + error_msg "Error calling run_server" + return 1 } - sleep 1 - - run_experiments "$results_directory" || { - echo "${FUNCNAME[0]} Error" + run_client || { + error_msg "Error calling run_client" kill_runtime - exit 1 + return 1 } - sleep 1 kill_runtime || { - echo "${FUNCNAME[0]} Error" - exit 1 - } - - process_results "$results_directory" || { - echo "${FUNCNAME[0]} Error" - exit 1 + error_msg "Error calling kill_runtime" + return 1 } - echo "[DONE]" - exit 0 done + + return 0 } main() { initialize_globals - parse_arguments "$@" - - echo "$timestamp" - - echo "Target: $target" - echo "Policy: $policy" - echo "Role: $role" + parse_arguments "$@" || { + exit 1 + } case $role in both) run_both ;; server) - results_directory="$experiment_directory/res/$timestamp" - mkdir -p "$results_directory" - start_runtime "$target" "$results_directory" - exit 0 + run_server "$policy" + ;; + debug) + run_debug "$policy" + ;; + perf) + run_perf "$policy" + ;; + client) + run_client ;; - client) ;; *) echo "Invalid state" - exit 1 + false ;; esac + + exit "$?" } main "$@" diff --git a/runtime/experiments/deadline/run_relative.sh b/runtime/experiments/deadline/run_relative.sh deleted file mode 100755 index e286b55..0000000 --- a/runtime/experiments/deadline/run_relative.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) - -schedulers=(EDF FIFO) -for scheduler in ${schedulers[*]}; do - - results_directory="$experiment_directory/res/$timestamp/$scheduler" - log=log.txt - - mkdir -p "$results_directory" - log_environment >> "$results_directory/$log" - - # Start the runtime - if [ "$1" != "-d" ]; then - SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=$scheduler PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 - else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" - fi - - inputs=(40 10) - duration_sec=15 - offset=5 - - # Execute workloads long enough for runtime to learn excepted execution time - echo -n "Running Samples: " - for input in ${inputs[*]}; do - hey -z ${duration_sec}s -cpus 3 -t 0 -o csv -m GET -d "$input\n" http://localhost:$((10000 + input)) - done - echo "[DONE]" - sleep 5 - - echo "Running Experiments" - # Run each separately - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40.csv" - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10.csv" - - # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run - hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40-con.csv" & - sleep $offset - hey -z ${duration_sec}s -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10-con.csv" & - sleep $((duration_sec + offset + 15)) - - # Stop the runtime if not in debug mode - [ "$1" != "-d" ] && kill_runtime - - # Generate *.csv and *.dat results - echo -n "Parsing Results: " - - printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" - printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" - printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - - deadlines_ms=(2 2 3000 3000) - payloads=(fib10 fib10-con fib40 fib40-con) - - for ((i = 0; i < 4; i++)); do - # for payload in ${payloads[*]}; do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f%,", $0 / '"$deadline"' * 100} - NR==p90 {printf "%1.4f%,", $0 / '"$deadline"' * 100} - NR==p99 {printf "%1.4f%,", $0 / '"$deadline"' * 100} - NR==p100 {printf "%1.4f%\n", $0 / '"$deadline"' * 100} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" - done - - # Transform csvs to dat files for gnuplot - for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" - done - - # Generate gnuplots. Commented out because we don't have *.gnuplots defined - # generate_gnuplots - - # Cleanup, if requires - echo "[DONE]" -done diff --git a/runtime/experiments/deadline/scratch.txt b/runtime/experiments/deadline/scratch.txt deleted file mode 100644 index 2f45190..0000000 --- a/runtime/experiments/deadline/scratch.txt +++ /dev/null @@ -1,5 +0,0 @@ - - -hey -n 200 -c 200 -t 0 -m GET -d "40\n" http://localhost:10040 - -hey -n 500 -c 500 -t 0 -m GET -d "10\n" http://localhost:10010 diff --git a/runtime/experiments/deadline/server.sh b/runtime/experiments/deadline/server.sh deleted file mode 100755 index 965228e..0000000 --- a/runtime/experiments/deadline/server.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) - -# Start the runtime - -PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" From 5e810df0a12ba18c286dd03a39980c51e283479f Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 12:33:43 -0400 Subject: [PATCH 04/13] chore: formatting nits --- .devcontainer/devcontainer.json | 53 +++++---- .editorconfig | 12 ++ .vscode/c_cpp_properties.json | 40 ++++--- .vscode/extensions.json | 22 ++-- .vscode/launch.json | 102 ++++++++--------- .vscode/settings.json | 190 +++++++++++++++++++------------- Dockerfile.aarch64 | 52 ++++----- devenv.sh | 6 +- 8 files changed, 261 insertions(+), 216 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a490b20..91f9060 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,32 +1,29 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: // https://github.com/microsoft/vscode-dev-containers/tree/v0.166.1/containers/docker-existing-dockerfile { - "name": "Dockerfile", - - // Sets the run context to one level up instead of the .devcontainer folder. - "context": "..", - - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerFile": "../Dockerfile.x86_64", - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": null - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "editorconfig.editorconfig", - "foxundermoon.shell-format", - "timonwong.shellcheck", - "dtsvet.vscode-wasm", - "13xforever.language-x86-64-assembly", - "ms-vscode.cpptools", - "ms-vscode.cpptools-themes", - "jeff-hykin.better-cpp-syntax" - ], - "workspaceMount": "source=${localWorkspaceFolder},target=/sledge,type=bind,consistency=cached", - "workspaceFolder": "/sledge", - "postCreateCommand": "make -C /sledge install && make -B -C /sledge/runtime/tests clean all", - "containerUser": "dev", + "name": "Dockerfile", + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerFile": "../Dockerfile.x86_64", + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": "bash" + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "editorconfig.editorconfig", + "foxundermoon.shell-format", + "timonwong.shellcheck", + "dtsvet.vscode-wasm", + "13xforever.language-x86-64-assembly", + "ms-vscode.cpptools", + "ms-vscode.cpptools-themes", + "jeff-hykin.better-cpp-syntax", + "mads-hartmann.bash-ide-vscode" + ], + "workspaceMount": "source=${localWorkspaceFolder},target=/sledge,type=bind,consistency=cached", + "workspaceFolder": "/sledge", + "postCreateCommand": "make -C /sledge install && make -B -C /sledge/runtime/tests clean all", + "containerUser": "dev", } diff --git a/.editorconfig b/.editorconfig index bfe019d..0234938 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,3 +23,15 @@ ignore = true [thirdparty/**] ignore = true + +[*.json] +indent_style = tab +indent_size = 4 + +[dockerfile] +indent_style = tab +indent_size = 4 + +[Dockerfile.*] +indent_style = tab +indent_size = 4 diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index a247cb5..f3e9c8a 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -1,19 +1,23 @@ { - "configurations": [ - { - "name": "Linux", - "intelliSenseMode": "clang-x64", - "includePath": [ - "/usr/include/", - "${workspaceFolder}/runtime/include/", - "${workspaceFolder}/runtime/thirdparty/ck/include/", - "${workspaceFolder}/runtime/thirdparty/http-parser/", - "${workspaceFolder}/runtime/thirdparty/jsmn/" - ], - "defines": ["USE_MEM_VM", "x86_64", "_GNU_SOURCE"], - "cStandard": "c17", - "compilerPath": "/usr/bin/clang" - } - ], - "version": 4 -} + "configurations": [ + { + "name": "Linux", + "intelliSenseMode": "clang-x64", + "includePath": [ + "/usr/include/", + "${workspaceFolder}/runtime/include/", + "${workspaceFolder}/runtime/thirdparty/ck/include/", + "${workspaceFolder}/runtime/thirdparty/http-parser/", + "${workspaceFolder}/runtime/thirdparty/jsmn/" + ], + "defines": [ + "USE_MEM_VM", + "x86_64", + "_GNU_SOURCE" + ], + "cStandard": "c17", + "compilerPath": "/usr/bin/clang" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json index dd5cc50..695c290 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,12 +1,12 @@ { - "recommendations": [ - "editorconfig.editorconfig", - "foxundermoon.shell-format", - "timonwong.shellcheck", - "dtsvet.vscode-wasm", - "13xforever.language-x86-64-assembly", - "ms-vscode.cpptools", - "ms-vscode.cpptools-themes", - "jeff-hykin.better-cpp-syntax" - ] -} + "recommendations": [ + "editorconfig.editorconfig", + "foxundermoon.shell-format", + "timonwong.shellcheck", + "dtsvet.vscode-wasm", + "13xforever.language-x86-64-assembly", + "ms-vscode.cpptools", + "ms-vscode.cpptools-themes", + "jeff-hykin.better-cpp-syntax" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index cc566d0..670e39e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,52 +1,52 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Hyde", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceFolder}/runtime/bin/sledgert", - "args": [ - "${workspaceFolder}/runtime/experiments/applications/ocr/hyde/spec.json" - ], - "stopAtEntry": false, - "cwd": "${workspaceFolder}", - "environment": [], - "externalConsole": false, - "MIMode": "gdb", - "envFile": "${workspaceFolder}/.env", - "setupCommands": [ - { - "description": "Enable pretty-printing for gdb", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - }, - { - "name": "Preemption", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceFolder}/runtime/bin/sledgert", - "args": [ - "${workspaceFolder}/runtime/experiments/preemption/spec.json" - ], - "stopAtEntry": false, - "cwd": "${workspaceFolder}", - "environment": [], - "externalConsole": false, - "MIMode": "gdb", - "envFile": "${workspaceFolder}/.env", - "setupCommands": [ - { - "description": "Enable pretty-printing for gdb", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - } - ] -} + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Hyde", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/runtime/bin/sledgert", + "args": [ + "${workspaceFolder}/runtime/experiments/applications/ocr/hyde/spec.json" + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "envFile": "${workspaceFolder}/.env", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "Preemption", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/runtime/bin/sledgert", + "args": [ + "${workspaceFolder}/runtime/experiments/preemption/spec.json" + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "envFile": "${workspaceFolder}/.env", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 8fd085a..25adc10 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,81 +1,113 @@ { - "files.associations": { - "*.inc": "cpp", - "arm_nnexamples_cifar10_parameter.h": "c", - "arm_nnexamples_cifar10_weights.h": "c", - "__hash_table": "cpp", - "__split_buffer": "cpp", - "__tree": "cpp", - "array": "cpp", - "bitset": "cpp", - "deque": "cpp", - "dynarray": "cpp", - "simd": "cpp", - "hash_map": "cpp", - "hash_set": "cpp", - "initializer_list": "cpp", - "iterator": "cpp", - "list": "cpp", - "map": "cpp", - "queue": "cpp", - "random": "cpp", - "regex": "cpp", - "set": "cpp", - "stack": "cpp", - "string": "cpp", - "string_view": "cpp", - "unordered_map": "cpp", - "unordered_set": "cpp", - "utility": "cpp", - "valarray": "cpp", - "vector": "cpp", - "__locale": "cpp", - "__config": "c", - "*.def": "c", - "mman.h": "c", - "types.h": "c", - "assert.h": "c", - "fstream": "c", - "locale": "c", - "*.tcc": "c", - "sandbox.h": "c", - "runtime.h": "c", - "panic.h": "c", - "ucontext.h": "c", - "stdlib.h": "c", - "pthread.h": "c", - "signal.h": "c", - "current_sandbox.h": "c", - "admissions_control.h": "c", - "sigval_t.h": "c", - "__sigval_t.h": "c", - "sigaction.h": "c", - "string.h": "c", - "errno.h": "c", - "siginfo_t.h": "c", - "features.h": "c" - }, - "files.exclude": { - "**/.git": true, - "**/.svn": true, - "**/.hg": true, - "**/CVS": true, - "**/.DS_Store": true, - "awsm/wasmception": true - }, - "C_Cpp.default.cStandard": "c17", - "C_Cpp.exclusionPolicy": "checkFilesAndFolders", - "C_Cpp.experimentalFeatures": "Enabled", - "C_Cpp.files.exclude": { - "awsm/wasmception": true, - "**/.vscode": true - }, - - "shellformat.flag": "-ln=bash -i 0 -bn -ci -sr -kp", - "terminal.integrated.profiles.linux": { - "bash": { - "path": "bash" - } - }, - "terminal.integrated.shell.linux": "bash" + "files.associations": { + "*.inc": "cpp", + "arm_nnexamples_cifar10_parameter.h": "c", + "arm_nnexamples_cifar10_weights.h": "c", + "__hash_table": "cpp", + "__split_buffer": "cpp", + "__tree": "cpp", + "array": "cpp", + "bitset": "cpp", + "deque": "cpp", + "dynarray": "cpp", + "simd": "cpp", + "hash_map": "cpp", + "hash_set": "cpp", + "initializer_list": "cpp", + "iterator": "cpp", + "list": "cpp", + "map": "cpp", + "queue": "cpp", + "random": "cpp", + "regex": "cpp", + "set": "cpp", + "stack": "cpp", + "string": "cpp", + "string_view": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "utility": "cpp", + "valarray": "cpp", + "vector": "cpp", + "__locale": "cpp", + "__config": "c", + "*.def": "c", + "mman.h": "c", + "types.h": "c", + "assert.h": "c", + "fstream": "c", + "locale": "c", + "*.tcc": "c", + "sandbox.h": "c", + "runtime.h": "c", + "panic.h": "c", + "ucontext.h": "c", + "stdlib.h": "c", + "pthread.h": "c", + "signal.h": "c", + "current_sandbox.h": "c", + "admissions_control.h": "c", + "sigval_t.h": "c", + "__sigval_t.h": "c", + "sigaction.h": "c", + "string.h": "c", + "errno.h": "c", + "siginfo_t.h": "c", + "features.h": "c" + }, + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "awsm/wasmception": true + }, + "C_Cpp.default.cStandard": "c17", + "C_Cpp.exclusionPolicy": "checkFilesAndFolders", + "C_Cpp.experimentalFeatures": "Enabled", + "C_Cpp.files.exclude": { + "awsm/wasmception": true, + "**/.vscode": true + }, + "shellformat.effectLanguages": [ + "shellscript", + "dockerfile", + "ignore", + "gitignore", + ], + "shellformat.path": "/usr/local/bin/shfmt", + "shellformat.flag": "-ln=bash -i 0 -bn -ci -sr -kp", + "terminal.integrated.shell.linux": "bash", + "[jsonc]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[shellscript]": { + "editor.defaultFormatter": "foxundermoon.shell-format" + }, + "[dockerfile]": { + "editor.defaultFormatter": "foxundermoon.shell-format" + }, + "[ignore]": { + "editor.defaultFormatter": "foxundermoon.shell-format" + }, + "files.watcherExclude": { + "**/.git/objects/**": true, + "**/.git/subtree-cache/**": true, + "**/.git/**": true, + "**/awsm/target/**": true, + "**/runtime/thirdparty/**": true, + "**/runtime/thirdparty/ck/**": true, + "**/runtime/thirdparty/http-parser/**": true, + "**/runtime/thirdparty/jsmn/**": true, + "**/runtime/thirdparty/dist/**": true, + "*.o": true, + "**/res/**": true, + "**/concurrency/res/**/**": true, + "**/deadline/res/**/**": true, + "**/preemption/res/**/**": true, + } } diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index 91af595..180edf2 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -6,32 +6,32 @@ FROM ubuntu:bionic # install some basic packages RUN apt-get update RUN apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - git \ - cmake \ - ca-certificates \ - libssl-dev \ - pkg-config \ - gcc \ - g++ \ - clang-8 \ - clang-tools-8 \ - llvm-8 \ - llvm-8-dev \ - libc++-dev \ - libc++abi-dev \ - lld-8 \ - lldb-8 \ - libclang-8-dev \ - libclang-common-8-dev \ - vim \ - binutils-dev \ - build-essential \ - automake \ - libtool \ - strace \ - less + build-essential \ + curl \ + git \ + cmake \ + ca-certificates \ + libssl-dev \ + pkg-config \ + gcc \ + g++ \ + clang-8 \ + clang-tools-8 \ + llvm-8 \ + llvm-8-dev \ + libc++-dev \ + libc++abi-dev \ + lld-8 \ + lldb-8 \ + libclang-8-dev \ + libclang-common-8-dev \ + vim \ + binutils-dev \ + build-essential \ + automake \ + libtool \ + strace \ + less RUN rm -rf /var/lib/apt/lists/* # set to use our installed clang version diff --git a/devenv.sh b/devenv.sh index 1b23e08..40f156e 100755 --- a/devenv.sh +++ b/devenv.sh @@ -132,11 +132,11 @@ envrun() { echo "Starting ${SYS_DOC_NAME}" docker run \ --privileged \ - --security-opt seccomp:unconfined \ + --security-opt seccomp:unconfined \ --name=${SYS_DOC_NAME} \ --detach \ - --mount type=bind,src="$(cd "$(dirname "${0}")" && pwd -P || exit 1),target=/${SYS_NAME}" \ - ${SYS_DOC_NAMETAG} /bin/sleep 99999999 > /dev/null + --mount type=bind,src="$(cd "$(dirname "${0}")" && pwd -P || exit 1),target=/${SYS_NAME}" \ + ${SYS_DOC_NAMETAG} /bin/sleep 99999999 > /dev/null fi echo "Running shell" From 0b9ca90fd0a3a63611e0cf62ef7cefff4a0f330c Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 12:34:13 -0400 Subject: [PATCH 05/13] feat: use unminimized dev container --- Dockerfile.x86_64 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 8439611..0022ae1 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -8,6 +8,15 @@ ARG WASMCEPTION_URL=https://github.com/gwsystems/wasmception/releases/download/v ARG SHFMT_URL=https://github.com/mvdan/sh/releases/download/v3.2.4/shfmt_v3.2.4_linux_amd64 ARG SHELLCHECK_URL=https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz +# Use bash, not sh +SHELL ["/bin/bash", "-c"] + +# We run the dev container interactively, so unminimize and install missing packages +RUN apt-get update && apt-get install -y --no-install-recommends \ + apt-utils \ + man-db \ + && yes | unminimize + # General GCC C/C++ Build toolchain # pkg-config, libtool - used by PocketSphinx # cmake - used by cmsis From db0c3504c0c28fb2c208aff6d0110fb59cf813ef Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 13:01:41 -0400 Subject: [PATCH 06/13] chore: Validate perf on baremetal server --- Dockerfile.x86_64 | 4 ---- runtime/experiments/deadline/.gitignore | 2 ++ runtime/experiments/deadline/run.sh | 10 ++++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 0022ae1..7d47065 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -85,10 +85,6 @@ RUN tar xvfz wasmception.tar.gz -C /sledge/awsm/wasmception # RUN curl -sS -L -O $WASI_SDK_URL && dpkg -i wasi-sdk_8.0_amd64.deb && rm -f wasi-sdk_8.0_amd64.deb # ENV WASI_SDK=/opt/wasi-sdk -# PERF -ADD install_perf.sh /sledge/install_perf.sh -RUN ./sledge/install_perf.sh - # Create non-root user and add to sudoers ARG USERNAME=dev ARG USER_UID=1000 diff --git a/runtime/experiments/deadline/.gitignore b/runtime/experiments/deadline/.gitignore index 64f722e..26cdcd9 100644 --- a/runtime/experiments/deadline/.gitignore +++ b/runtime/experiments/deadline/.gitignore @@ -1 +1,3 @@ res +perf.data +perf.data.old diff --git a/runtime/experiments/deadline/run.sh b/runtime/experiments/deadline/run.sh index 1112522..1be697c 100755 --- a/runtime/experiments/deadline/run.sh +++ b/runtime/experiments/deadline/run.sh @@ -16,7 +16,7 @@ usage() { echo " -t,--target= Execute as client against remote URL" echo " -s,--serve= Serve with scheduling policy, but do not run client" echo " -d,--debug= Debug under GDB with scheduling policy, but do not run client" - echo " -p,--perf= Run under perf with scheduling policy, but do not run client" + echo " -p,--perf= Run under perf with scheduling policy. Run on baremetal Linux host!" } # Declares application level global state @@ -410,10 +410,12 @@ run_perf() { return 1 fi - [[ ! -x perf ]] && { - echo "perf is not present" + if ! command -v perf; then + echo "perf is not present." exit 1 - } + fi + + local -r scheduler="$1" SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$experiment_directory/spec.json" } From 3112e856ae01b7c10e6fd05fd22b9312eedf5c91 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 13:32:41 -0400 Subject: [PATCH 07/13] chore: fix install script --- install_perf.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/install_perf.sh b/install_perf.sh index cf45953..e7705da 100755 --- a/install_perf.sh +++ b/install_perf.sh @@ -1,7 +1,10 @@ #!/bin/bash # If already installed, just return -[[ -x perf ]] && return 0 +command -v perf && { + echo "perf is already installed." + exit 0 +} [[ "$(whoami)" != "root" ]] && { echo "Expected to run as root" @@ -11,14 +14,11 @@ # Under WSL2, perf has to be installed from source if grep --silent 'WSL2' <(uname -r); then echo "WSL detected. perf must be built from source" - echo "WSL2 support is WIP and not currently functional" - exit 0 - sudo apt-get install flex bison python3-dev liblzma-dev libnuma-dev zlib1g libperl-dev libgtk2.0-dev libslang2-dev systemtap-sdt-dev libelf-dev binutils-dev libbabeltrace-dev libdw-dev libunwind-dev libiberty-dev --yes git clone --depth 1 https://github.com/microsoft/WSL2-Linux-Kernel ~/WSL2-Linux-Kernel make -Wno-error -j8 -C ~/WSL2-Linux-Kernel/tools/perf sudo cp ~/WSL2-Linux-Kernel/tools/perf/perf /usr/local/bin - # rm -rf ~/WSL2-Linux-Kernel + rm -rf ~/WSL2-Linux-Kernel else apt-get install "linux-tools-$(uname -r)" linux-tools-generic -y fi From 5693c65dd288f5e87b24adab7b5b941e24120463 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 13:37:32 -0400 Subject: [PATCH 08/13] fix: correct server arg in deadline driver --- runtime/experiments/deadline/run.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/runtime/experiments/deadline/run.sh b/runtime/experiments/deadline/run.sh index 1be697c..8d0f5cb 100755 --- a/runtime/experiments/deadline/run.sh +++ b/runtime/experiments/deadline/run.sh @@ -137,7 +137,7 @@ parse_arguments() { # Starts the Sledge Runtime start_runtime() { printf "Starting Runtime: " - if (($# != 2)); then + if (($# < 2 || $# > 3)); then printf "[ERR]\n" error_msg "invalid number of arguments \"$1\"" return 1 @@ -149,18 +149,31 @@ start_runtime() { printf "[ERR]\n" error_msg "directory \"$2\" does not exist" return 1 + elif ! [[ $3 =~ ^(foreground|background)$ ]]; then + printf "[ERR]\n" + error_msg "expected foreground or background was \"$3\"" + return 1 fi local -r scheduler="$1" local -r results_directory="$2" + local -r how_to_run="${3:-background}" local -r log_name=log.txt local log="$results_directory/${log_name}" log_environment >> "$log" - SLEDGE_SCHEDULER="$scheduler" \ - sledgert "$experiment_directory/spec.json" >> "$log" 2>> "$log" & + case "$how_to_run" in + "background") + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$experiment_directory/spec.json" >> "$log" 2>> "$log" & + ;; + "foreground") + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$experiment_directory/spec.json" + ;; + esac printf "[OK]\n" return 0 @@ -380,9 +393,11 @@ run_server() { local -r scheduler="$1" if [[ "$role" == "both" ]]; then - local results_directory="$experiment_directory/res/$timestamp/$scheduler" + local -r results_directory="$experiment_directory/res/$timestamp/$scheduler" + local -r how_to_run="background" elif [[ "$role" == "server" ]]; then - local results_directory="$experiment_directory/res/$timestamp" + local -r results_directory="$experiment_directory/res/$timestamp" + local -r how_to_run="foreground" else error_msg "Unexpected $role" return 1 @@ -390,7 +405,7 @@ run_server() { mkdir -p "$results_directory" - start_runtime "$scheduler" "$results_directory" || { + start_runtime "$scheduler" "$results_directory" "$how_to_run" || { echo "start_runtime RC: $?" error_msg "Error calling start_runtime $scheduler $results_directory" return 1 From 9204ab8f165d932e5f730590ac3db34d657cd27f Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 22:05:59 +0000 Subject: [PATCH 09/13] chore: remove results from watcher exclude --- .vscode/settings.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 25adc10..e59daf0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -105,9 +105,7 @@ "**/runtime/thirdparty/jsmn/**": true, "**/runtime/thirdparty/dist/**": true, "*.o": true, - "**/res/**": true, - "**/concurrency/res/**/**": true, - "**/deadline/res/**/**": true, - "**/preemption/res/**/**": true, + "*.bc": true, + "*.wasm": true, } } From d678e34ce3a544127570145a0543e7b5e37f4f7b Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 14 Apr 2021 22:06:44 +0000 Subject: [PATCH 10/13] test: centralize bimodal and refactor concurrency --- .../{deadline => bimodal}/.gitignore | 0 runtime/experiments/bimodal/README.md | 15 + runtime/experiments/bimodal/run.sh | 222 +++++++ .../{deadline => bimodal}/spec.json | 0 runtime/experiments/common.sh | 354 ++++++++++- runtime/experiments/concurrency/README.md | 10 - runtime/experiments/concurrency/debug.sh | 19 - runtime/experiments/concurrency/run.sh | 177 +++--- runtime/experiments/concurrency/spec.json | 26 +- runtime/experiments/deadline/README.md | 48 -- runtime/experiments/deadline/run.sh | 562 ------------------ runtime/experiments/preemption/.gitignore | 1 - runtime/experiments/preemption/README.md | 28 - runtime/experiments/preemption/backend.sh | 18 - runtime/experiments/preemption/client.sh | 111 ---- runtime/experiments/preemption/debug.sh | 20 - runtime/experiments/preemption/fix_results.sh | 81 --- runtime/experiments/preemption/perf.sh | 14 - runtime/experiments/preemption/run.sh | 134 ----- .../experiments/preemption/run_relative.sh | 124 ---- runtime/experiments/preemption/scratch.txt | 5 - runtime/experiments/preemption/spec.json | 30 - 22 files changed, 703 insertions(+), 1296 deletions(-) rename runtime/experiments/{deadline => bimodal}/.gitignore (100%) create mode 100644 runtime/experiments/bimodal/README.md create mode 100755 runtime/experiments/bimodal/run.sh rename runtime/experiments/{deadline => bimodal}/spec.json (100%) delete mode 100755 runtime/experiments/concurrency/debug.sh delete mode 100644 runtime/experiments/deadline/README.md delete mode 100755 runtime/experiments/deadline/run.sh delete mode 100644 runtime/experiments/preemption/.gitignore delete mode 100644 runtime/experiments/preemption/README.md delete mode 100755 runtime/experiments/preemption/backend.sh delete mode 100755 runtime/experiments/preemption/client.sh delete mode 100755 runtime/experiments/preemption/debug.sh delete mode 100755 runtime/experiments/preemption/fix_results.sh delete mode 100755 runtime/experiments/preemption/perf.sh delete mode 100755 runtime/experiments/preemption/run.sh delete mode 100755 runtime/experiments/preemption/run_relative.sh delete mode 100644 runtime/experiments/preemption/scratch.txt delete mode 100644 runtime/experiments/preemption/spec.json diff --git a/runtime/experiments/deadline/.gitignore b/runtime/experiments/bimodal/.gitignore similarity index 100% rename from runtime/experiments/deadline/.gitignore rename to runtime/experiments/bimodal/.gitignore diff --git a/runtime/experiments/bimodal/README.md b/runtime/experiments/bimodal/README.md new file mode 100644 index 0000000..0ad388d --- /dev/null +++ b/runtime/experiments/bimodal/README.md @@ -0,0 +1,15 @@ +# Bimodal Distribution + +This experiment drives a bimodal distribution of long-running low-priority and short-running high-priority workloads + +Relative Deadlines are tuned such that the scheduler should always preempt the low-priority workload for the high-priority workload if preemption is disabled. + +The two workloads are run separately as a baseline. They are then run concurrently, starting the low-priority long-running workload first such that the system begins execution and accumulates requests in the data structures. The high-priority short-running workload then begins. + +## Independent Variable + +The Scheduling Policy: EDF versus FIFO + +## Dependent Variables + +Latency of high priority workload diff --git a/runtime/experiments/bimodal/run.sh b/runtime/experiments/bimodal/run.sh new file mode 100755 index 0000000..2a84026 --- /dev/null +++ b/runtime/experiments/bimodal/run.sh @@ -0,0 +1,222 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Success - The percentage of requests that complete by their deadlines +# TODO: Does this handle non-200s? +# Throughput - The mean number of successful requests per second +# Latency - the rount-trip resonse time (unit?) of successful requests at the p50, p90, p99, and p100 percetiles + +# Sends requests until the per-module perf window buffers are full +# This ensures that Sledge has accurate estimates of execution time +run_samples() { + local hostname="${1:-localhost}" + + # Scrape the perf window size from the source if possible + local -r perf_window_path="../../include/perf_window.h" + local -i perf_window_buffer_size + if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then + echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" + echo "Defaulting to 16" + perf_window_buffer_size=16 + fi + local -ir perf_window_buffer_size + + echo -n "Running Samples: " + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || { + error_msg "fib40 samples failed" + return 1 + } + + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || { + error_msg "fib10 samples failed" + return 1 + } + + echo "[OK]" + return 0 +} + +# Execute the fib10 and fib40 experiments sequentially and concurrently +# $1 (results_directory) - a directory where we will store our results +# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost +run_experiments() { + if (($# < 1 || $# > 2)); then + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ -d "$1" ]]; then + error_msg "directory \"$1\" does not exist" + return 1 + fi + + local results_directory="$1" + local hostname="${2:-localhost}" + + # The duration in seconds that we want the client to send requests + local -ir duration_sec=15 + + # The duration in seconds that the low priority task should run before the high priority task starts + local -ir offset=5 + + printf "Running Experiments\n" + + # Run each separately + printf "\tfib40: " + hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$hostname:10040" > "$results_directory/fib40.csv" 2> /dev/null || { + printf "[ERR]\n" + error_msg "fib40 failed" + return 1 + } + get_result_count "$results_directory/fib40.csv" || { + printf "[ERR]\n" + error_msg "fib40 unexpectedly has zero requests" + return 1 + } + printf "[OK]\n" + + printf "\tfib10: " + hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$hostname:10010" > "$results_directory/fib10.csv" 2> /dev/null || { + printf "[ERR]\n" + error_msg "fib10 failed" + return 1 + } + get_result_count "$results_directory/fib10.csv" || { + printf "[ERR]\n" + error_msg "fib10 unexpectedly has zero requests" + return 1 + } + printf "[OK]\n" + + # Run concurrently + # The lower priority has offsets to ensure it runs the entire time the high priority is trying to run + # This asynchronously trigger jobs and then wait on their pids + local fib40_con_PID + local fib10_con_PID + + hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40_con.csv" 2> /dev/null & + fib40_con_PID="$!" + + sleep $offset + + hey -z "${duration_sec}s" -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10_con.csv" 2> /dev/null & + fib10_con_PID="$!" + + wait -f "$fib10_con_PID" || { + printf "\tfib10_con: [ERR]\n" + error_msg "failed to wait -f ${fib10_con_PID}" + return 1 + } + get_result_count "$results_directory/fib10_con.csv" || { + printf "\tfib10_con: [ERR]\n" + error_msg "fib10_con has zero requests. This might be because fib40_con saturated the runtime" + return 1 + } + printf "\tfib10_con: [OK]\n" + + wait -f "$fib40_con_PID" || { + printf "\tfib40_con: [ERR]\n" + error_msg "failed to wait -f ${fib40_con_PID}" + return 1 + } + get_result_count "$results_directory/fib40_con.csv" || { + printf "\tfib40_con: [ERR]\n" + error_msg "fib40_con has zero requests." + return 1 + } + printf "\tfib40_con: [OK]\n" + + return 0 +} + +# Process the experimental results and generate human-friendly results for success rate, throughput, and latency +process_results() { + if (($# != 1)); then + error_msg "invalid number of arguments ($#, expected 1)" + return 1 + elif ! [[ -d "$1" ]]; then + error_msg "directory $1 does not exist" + return 1 + fi + + local -r results_directory="$1" + + echo -n "Processing Results: " + + # Write headers to CSVs + printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" + printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" + printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" + + # The four types of results that we are capturing. + # fib10 and fib 40 are run sequentially. + # fib10_con and fib40_con are run concurrently + local -ar payloads=(fib10 fib10_con fib40 fib40_con) + + # The deadlines for each of the workloads + # TODO: Scrape these from spec.json + local -Ar deadlines_ms=( + [fib10]=2 + [fib40]=3000 + ) + + for payload in "${payloads[@]}"; do + # Strip the _con suffix when getting the deadline + local -i deadline=${deadlines_ms[${payload/_con/}]} + + # Get Number of Requests, subtracting the header + local -i requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) + ((requests == 0)) && { + echo "$payload unexpectedly has zero requests" + continue + } + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} + ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" + + # Filter on 200s, convert from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ + | sort -g > "$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l < "$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # We determine duration by looking at the timestamp of the last complete request + # TODO: Should this instead just use the client-side synthetic duration_sec value? + duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + + # Throughput is calculated as the mean number of successful requests per second + throughput=$(echo "$oks/$duration" | bc) + printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" + done + + # Transform csvs to dat files for gnuplot + csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv" + + # Generate gnuplots. Commented out because we don't have *.gnuplots defined + # generate_gnuplots +} + +main "$@" diff --git a/runtime/experiments/deadline/spec.json b/runtime/experiments/bimodal/spec.json similarity index 100% rename from runtime/experiments/deadline/spec.json rename to runtime/experiments/bimodal/spec.json diff --git a/runtime/experiments/common.sh b/runtime/experiments/common.sh index 2663148..102cfdf 100644 --- a/runtime/experiments/common.sh +++ b/runtime/experiments/common.sh @@ -86,6 +86,352 @@ get_result_count() { fi } +usage() { + echo "$0 [options...]" + echo "" + echo "Options:" + echo " -t,--target= Execute as client against remote URL" + echo " -s,--serve= Serve with scheduling policy, but do not run client" + echo " -d,--debug= Debug under GDB with scheduling policy, but do not run client" + echo " -p,--perf= Run under perf with scheduling policy. Run on baremetal Linux host!" +} + +# Parses arguments from the user and sets associates global state +parse_arguments() { + for i in "$@"; do + case $i in + -t=* | --target=*) + if [[ "$role" == "server" ]]; then + echo "Cannot set target when server" + usage + return 1 + fi + role=client + target="${i#*=}" + shift + ;; + -s=* | --serve=*) + if [[ "$role" == "client" ]]; then + echo "Cannot use -s,--serve with -t,--target" + usage + return 1 + fi + role=server + policy="${i#*=}" + if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" + usage + return 1 + fi + shift + ;; + -d=* | --debug=*) + if [[ "$role" == "client" ]]; then + echo "Cannot use -d,--debug with -t,--target" + usage + return 1 + fi + role=debug + policy="${i#*=}" + if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" + usage + return 1 + fi + shift + ;; + -p=* | --perf=*) + if [[ "$role" == "perf" ]]; then + echo "Cannot use -p,--perf with -t,--target" + usage + return 1 + fi + role=perf + policy="${i#*=}" + if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" + usage + return 1 + fi + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "$1 is a not a valid option" + usage + return 1 + ;; + esac + done + + # default to both if no arguments were passed + if [[ -z "$role" ]]; then + role="both" + fi + + # Set globals as read only + declare -r target + declare -r policy + declare -r role +} + +# Declares application level global state +initialize_globals() { + # timestamp is used to name the results directory for a particular test run + # shellcheck disable=SC2155 + # shellcheck disable=SC2034 + declare -gir timestamp=$(date +%s) + + # shellcheck disable=SC2155 + declare -gr experiment_directory=$(pwd) + + # shellcheck disable=SC2155 + declare -gr binary_directory=$(cd ../../bin && pwd) + + # Globals used by parse_arguments + declare -g target="" + declare -g policy="" + declare -g role="" + + # Configure environment variables + export PATH=$binary_directory:$PATH + export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH + export SLEDGE_NWORKERS=5 +} + +# $1 - Scheduler Variant (EDF|FIFO) +# $2 - Results Directory +# $3 - How to run (foreground|background) +# $4 - JSON specification +start_runtime() { + printf "Starting Runtime: " + if (($# != 4)); then + printf "[ERR]\n" + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + error_msg "expected EDF or FIFO was \"$1\"" + return 1 + elif ! [[ -d "$2" ]]; then + printf "[ERR]\n" + error_msg "directory \"$2\" does not exist" + return 1 + elif ! [[ $3 =~ ^(foreground|background)$ ]]; then + printf "[ERR]\n" + error_msg "expected foreground or background was \"$3\"" + return 1 + elif [[ ! -f "$4" || "$4" != *.json ]]; then + printf "[ERR]\n" + error_msg "\"$4\" does not exist or is not a JSON" + return 1 + fi + + local -r scheduler="$1" + local -r results_directory="$2" + local -r how_to_run="$3" + local -r specification="$4" + + local -r log_name=log.txt + local log="$results_directory/${log_name}" + + log_environment >> "$log" + + case "$how_to_run" in + "background") + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$specification" >> "$log" 2>> "$log" & + ;; + "foreground") + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$specification" + ;; + esac + + printf "[OK]\n" + return 0 +} + +run_server() { + if (($# != 1)); then + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + error_msg "expected EDF or FIFO was \"$1\"" + return 1 + fi + + local -r scheduler="$1" + + if [[ "$role" == "both" ]]; then + local -r results_directory="$experiment_directory/res/$timestamp/$scheduler" + local -r how_to_run="background" + elif [[ "$role" == "server" ]]; then + local -r results_directory="$experiment_directory/res/$timestamp" + local -r how_to_run="foreground" + else + error_msg "Unexpected $role" + return 1 + fi + + mkdir -p "$results_directory" + + start_runtime "$scheduler" "$results_directory" "$how_to_run" "$experiment_directory/spec.json" || { + echo "start_runtime RC: $?" + error_msg "Error calling start_runtime $scheduler $results_directory" + return 1 + } + + return 0 +} + +run_perf() { + if (($# != 1)); then + printf "[ERR]\n" + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + error_msg "expected EDF or FIFO was \"$1\"" + return 1 + fi + + if ! command -v perf; then + echo "perf is not present." + exit 1 + fi + + local -r scheduler="$1" + + SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$experiment_directory/spec.json" +} + +# Starts the Sledge Runtime under GDB +run_debug() { + # shellcheck disable=SC2155 + local project_directory=$(cd ../.. && pwd) + if (($# != 1)); then + printf "[ERR]\n" + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + error_msg "expected EDF or FIFO was \"$1\"" + return 1 + fi + + local -r scheduler="$1" + + if [[ "$project_directory" != "/sledge/runtime" ]]; then + printf "It appears that you are not running in the container. Substituting path to match host environment\n" + SLEDGE_SCHEDULER="$scheduler" gdb \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="set substitute-path /sledge/runtime $project_directory" \ + --eval-command="run $experiment_directory/spec.json" \ + sledgert + else + SLEDGE_SCHEDULER="$scheduler" gdb \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="run $experiment_directory/spec.json" \ + sledgert + fi + return 0 +} + +run_client() { + if [[ "$role" == "both" ]]; then + local results_directory="$experiment_directory/res/$timestamp/$scheduler" + elif [[ "$role" == "client" ]]; then + local results_directory="$experiment_directory/res/$timestamp" + else + error_msg "${FUNCNAME[0]} Unexpected $role" + return 1 + fi + + mkdir -p "$results_directory" + + run_samples "$target" || { + error_msg "Error calling run_samples $target" + return 1 + } + + run_experiments "$results_directory" || { + error_msg "Error calling run_experiments $results_directory" + return 1 + } + + process_results "$results_directory" || { + error_msg "Error calling process_results $results_directory" + return 1 + } + + return 0 +} + +run_both() { + local -ar schedulers=(EDF FIFO) + for scheduler in "${schedulers[@]}"; do + printf "Running %s\n" "$scheduler" + + run_server "$scheduler" || { + error_msg "Error calling run_server" + return 1 + } + + run_client || { + error_msg "Error calling run_client" + kill_runtime + return 1 + } + + kill_runtime || { + error_msg "Error calling kill_runtime" + return 1 + } + + done + + return 0 +} + +main() { + initialize_globals + parse_arguments "$@" || { + exit 1 + } + + case $role in + both) + run_both + ;; + server) + run_server "$policy" + ;; + debug) + run_debug "$policy" + ;; + perf) + run_perf "$policy" + ;; + client) + run_client + ;; + *) + echo "Invalid state" + false + ;; + esac + + exit "$?" +} + kill_runtime() { printf "Stopping Runtime: " pkill sledgert > /dev/null 2> /dev/null @@ -93,6 +439,8 @@ kill_runtime() { printf "[OK]\n" } +# Takes a variadic number of *.gnuplot filenames and generates the resulting images +# Assumes that the gnuplot definitions are in the experiment directory generate_gnuplots() { if ! command -v gnuplot &> /dev/null; then echo "${FUNCNAME[0]} error: gnuplot could not be found in path" @@ -111,9 +459,9 @@ generate_gnuplots() { exit 1 fi cd "$results_directory" || exit - gnuplot ../../latency.gnuplot - gnuplot ../../success.gnuplot - gnuplot ../../throughput.gnuplot + for gnuplot_file in "${@}"; do + gnuplot "$experiment_directory/$gnuplot_file.gnuplot" + done cd "$experiment_directory" || exit } diff --git a/runtime/experiments/concurrency/README.md b/runtime/experiments/concurrency/README.md index 2b89042..156efbd 100644 --- a/runtime/experiments/concurrency/README.md +++ b/runtime/experiments/concurrency/README.md @@ -20,16 +20,6 @@ _How does increasing levels of concurrent client requests affect tail latency, t - `hey` (https://github.com/rakyll/hey) is available in your PATH - You have compiled `sledgert` and the `empty.so` test workload -## To Execute - -1. Run `./run.sh` -2. View the results in the newest timestamped directory in `./res` - -## To Debug - -1. Run `./debug.sh` in a tab -2. Run `./run.sh -d` in a second tab - ## TODO - Harden scripts to validate assumptions diff --git a/runtime/experiments/concurrency/debug.sh b/runtime/experiments/concurrency/debug.sh deleted file mode 100755 index f40fa45..0000000 --- a/runtime/experiments/concurrency/debug.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" - -gdb --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert diff --git a/runtime/experiments/concurrency/run.sh b/runtime/experiments/concurrency/run.sh index 47bcb9e..ffdbcec 100755 --- a/runtime/experiments/concurrency/run.sh +++ b/runtime/experiments/concurrency/run.sh @@ -2,80 +2,101 @@ source ../common.sh # This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) -results_directory="$experiment_directory/res/$timestamp" -log=log.txt - -mkdir -p "$results_directory" - -log_environment >> "$results_directory/$log" - -# Start the runtime -if [ "$1" != "-d" ]; then - PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 -else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" -fi - -iterations=10000 - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -hey -n "$iterations" -c 3 -q 200 -o csv -m GET http://localhost:10000 -sleep 5 -echo "[DONE]" +declare -gi iterations=10000 +declare -ga concurrency=(1 20 40 60 80 100) + +run_samples() { + local hostname="${1:-localhost}" + + # Scrape the perf window size from the source if possible + local -r perf_window_path="../../include/perf_window.h" + local -i perf_window_buffer_size + if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then + echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" + echo "Defaulting to 16" + perf_window_buffer_size=16 + fi + local -ir perf_window_buffer_size + + printf "Running Samples: " + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -cpus 3 -o csv -m GET "http://${hostname}:10000" 1> /dev/null 2> /dev/null || { + printf "[ERR]\n" + error_msg "samples failed" + return 1 + } + + echo "[OK]" + return 0 +} # Execute the experiments -concurrency=(1 20 40 60 80 100) -echo "Running Experiments" -for conn in ${concurrency[*]}; do - printf "\t%d Concurrency: " "$conn" - hey -n "$iterations" -c "$conn" -cpus 2 -o csv -m GET http://localhost:10000 > "$results_directory/con$conn.csv" - echo "[DONE]" -done - -# Stop the runtime - -if [ "$1" != "-d" ]; then - sleep 5 - kill_runtime -fi - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv" -printf "Concurrency,Throughput\n" >> "$results_directory/throughput.csv" -printf "Con,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -for conn in ${concurrency[*]}; do - # Calculate Success Rate for csv - awk -F, ' +# $1 (results_directory) - a directory where we will store our results +# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost +run_experiments() { + if (($# < 1 || $# > 2)); then + error_msg "invalid number of arguments \"$1\"" + return 1 + elif ! [[ -d "$1" ]]; then + error_msg "directory \"$1\" does not exist" + return 1 + fi + + local results_directory="$1" + local hostname="${2:-localhost}" + + # Execute the experiments + echo "Running Experiments" + for conn in ${concurrency[*]}; do + printf "\t%d Concurrency: " "$conn" + hey -n "$iterations" -c "$conn" -cpus 2 -o csv -m GET "http://$hostname:10000" > "$results_directory/con$conn.csv" 2> /dev/null + echo "[OK]" + done +} + +process_results() { + if (($# != 1)); then + error_msg "invalid number of arguments ($#, expected 1)" + return 1 + elif ! [[ -d "$1" ]]; then + error_msg "directory $1 does not exist" + return 1 + fi + + local -r results_directory="$1" + + echo -n "Processing Results: " + + # Write headers to CSVs + printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv" + printf "Concurrency,Throughput\n" >> "$results_directory/throughput.csv" + printf "Con,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" + + for conn in ${concurrency[*]}; do + # Calculate Success Rate for csv (percent of requests resulting in 200) + awk -F, ' $7 == 200 {ok++} END{printf "'"$conn"',%3.5f\n", (ok / '"$iterations"' * 100)} ' < "$results_directory/con$conn.csv" >> "$results_directory/success.csv" - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/con$conn.csv" \ - | sort -g > "$results_directory/con$conn-response.csv" + # Filter on 200s, convert from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/con$conn.csv" \ + | sort -g > "$results_directory/con$conn-response.csv" + + # Get Number of 200s + oks=$(wc -l < "$results_directory/con$conn-response.csv") + ((oks == 0)) && continue # If all errors, skip line - # Get Number of 200s - oks=$(wc -l < "$results_directory/con$conn-response.csv") - ((oks == 0)) && continue # If all errors, skip line + # We determine duration by looking at the timestamp of the last complete request + # TODO: Should this instead just use the client-side synthetic duration_sec value? + duration=$(tail -n1 "$results_directory/con$conn.csv" | cut -d, -f8) - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/con$conn.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%d,%f\n" "$conn" "$throughput" >> "$results_directory/throughput.csv" + # Throughput is calculated as the mean number of successful requests per second + throughput=$(echo "$oks/$duration" | bc) + printf "%d,%f\n" "$conn" "$throughput" >> "$results_directory/throughput.csv" - # Generate Latency Data for csv - awk ' + # Generate Latency Data for csv + awk ' BEGIN { sum = 0 p50 = int('"$oks"' * 0.5) @@ -90,18 +111,22 @@ for conn in ${concurrency[*]}; do NR==p100 {printf "%1.4f\n", $0} ' < "$results_directory/con$conn-response.csv" >> "$results_directory/latency.csv" - # Delete scratch file used for sorting/counting - rm -rf "$results_directory/con$conn-response.csv" -done + # Delete scratch file used for sorting/counting + rm -rf "$results_directory/con$conn-response.csv" + done + + # Transform csvs to dat files for gnuplot + for file in success latency throughput; do + echo -n "#" > "$results_directory/$file.dat" + tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" + done + + # Generate gnuplots + generate_gnuplots latency success throughput -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done + # Cleanup, if requires + echo "[OK]" -# Generate gnuplots -generate_gnuplots +} -# Cleanup, if requires -echo "[DONE]" +main "$@" diff --git a/runtime/experiments/concurrency/spec.json b/runtime/experiments/concurrency/spec.json index 6128ca1..140ec1e 100644 --- a/runtime/experiments/concurrency/spec.json +++ b/runtime/experiments/concurrency/spec.json @@ -1,14 +1,16 @@ { - "active": true, - "name": "empty", - "path": "empty_wasm.so", - "port": 10000, - "relative-deadline-us": 50000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" + "active": true, + "name": "empty", + "path": "empty_wasm.so", + "port": 10000, + "expected-execution-us": 500, + "admissions-percentile": 70, + "relative-deadline-us": 50000, + "argsize": 1, + "http-req-headers": [], + "http-req-content-type": "text/plain", + "http-req-size": 1024, + "http-resp-headers": [], + "http-resp-size": 1024, + "http-resp-content-type": "text/plain" } diff --git a/runtime/experiments/deadline/README.md b/runtime/experiments/deadline/README.md deleted file mode 100644 index 2275b7c..0000000 --- a/runtime/experiments/deadline/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Admissions Control - -## Discussion of Implementation - -The admissions control subsystem seeks to ensure that the system does not accept more work than it can execute while meeting the relative deadline defined in a module's JSON specification. - -The system maintains an integral value expressing the capacity of the system as millionths of a worker core. This assumes that the runtime has "pinned" these workers to underlying processors and has no contention with other workloads. - -The system maintains a second integral value expressing the total accepted work. - -The module specification provides a relative deadline, an expected execution time, and a percentile target expressing the pXX latency that the admissions control system should use when making admissions decisions (tunable from 50% to 99%). Tuning this percentile expresses how conservative the system should be with regard to scheduling. Selecting a lower value, such as 50%, reserves less processor time and results in a higher likelihood that the relative deadline is not met. Selecting a higher value, such as 99%, reserves more processor time and provides a higher likelihood that that the relative deadline will be met. The provided expected execution time is assumed to match the percentile provided. - -Dividing the expected execution time by the relative deadline yields the fraction of a worker needed to meet the deadline. - -If the existing accepted workload plus the required work of this new workload is less than the system capacity, the workload is accepted, and the integral value expressing the total accepted work is increased. The resulting sandbox request is tagged with the fraction of a worker it was calculated to use, and when the request completes, the total accepted work is decreased by this amount. - -If the existing accepted workload plus the required work of this new workload is greater than the system capacity, the request is rejected and the runtime sends the client an HTTP 503 response. - -While the module specification provides an expected execution time, the system does not trust this value and only uses it in the absence of better information. Each sandbox is profiled as it runs through the system, and the end-to-end execution time of successful sandbox requests are added to a specialized performance window data structure that stores the last N execution times sorted in order of execution time. This structure optimizes for quick lookups of a specific ppXX percentile - -Once data is seeded into this data structure, the initial execution estimate provided in the module specification is ignored, and the pXX target is instead used to lookup the actual pXX performance metric. - -Future Work: - -Currently, the scheduler takes no actual when an executing sandbox exceeds its pXX execution time or deadline. - -In the case of the pXX workload, this means that a workload configured to target p50 during admissions control decisions with exceptionally poor p99 performance causes system-wide overheads that can cause other systems to miss their deadlines. - -Even worse, when executing beyond the relative deadline, the request might be too stale for the client. - -In the absolute worst case, one can imagine a client workload caught in an infinite loop that causes permanent head of line blocking because its deadline is earlier than the current time, such that nothing can possibly preempt the executing workload. - -## Question - -- Does Admissions Control guarantee that deadlines are met? - -## Independent Variable - -Deadline is disabled versus deadline is enabled - -## Invariants - -Single workload -Use FIFO policy - -## Dependent Variables - -End-to-end execution time of a workload measured from a client measured relative to its deadline diff --git a/runtime/experiments/deadline/run.sh b/runtime/experiments/deadline/run.sh deleted file mode 100755 index 8d0f5cb..0000000 --- a/runtime/experiments/deadline/run.sh +++ /dev/null @@ -1,562 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Success - The percentage of requests that complete by their deadlines -# TODO: Does this handle non-200s? -# Throughput - The mean number of successful requests per second -# Latency - the rount-trip resonse time (unit?) of successful requests at the p50, p90, p99, and p100 percetiles - -# Use -d flag if running under gdb -# TODO: Just use ENV for policy and other runtime dynamic variables? -usage() { - echo "$0 [options...]" - echo "" - echo "Options:" - echo " -t,--target= Execute as client against remote URL" - echo " -s,--serve= Serve with scheduling policy, but do not run client" - echo " -d,--debug= Debug under GDB with scheduling policy, but do not run client" - echo " -p,--perf= Run under perf with scheduling policy. Run on baremetal Linux host!" -} - -# Declares application level global state -initialize_globals() { - # timestamp is used to name the results directory for a particular test run - # shellcheck disable=SC2155 - declare -gir timestamp=$(date +%s) - - # shellcheck disable=SC2155 - declare -gr experiment_directory=$(pwd) - - # shellcheck disable=SC2155 - declare -gr binary_directory=$(cd ../../bin && pwd) - - # Scrape the perf window size from the source if possible - local -r perf_window_path="../../include/perf_window.h" - declare -gi perf_window_buffer_size - if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then - echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" - echo "Defaulting to 16" - declare -ir perf_window_buffer_size=16 - fi - declare -gir perf_window_buffer_size - - # Globals used by parse_arguments - declare -g target="" - declare -g policy="" - declare -g role="" - - # Configure environment variables - export PATH=$binary_directory:$PATH - export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH - export SLEDGE_NWORKERS=5 -} - -# Parses arguments from the user and sets associates global state -parse_arguments() { - for i in "$@"; do - case $i in - -t=* | --target=*) - if [[ "$role" == "server" ]]; then - echo "Cannot set target when server" - usage - return 1 - fi - role=client - target="${i#*=}" - shift - ;; - -s=* | --serve=*) - if [[ "$role" == "client" ]]; then - echo "Cannot use -s,--serve with -t,--target" - usage - return 1 - fi - role=server - policy="${i#*=}" - if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then - echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" - usage - return 1 - fi - shift - ;; - -d=* | --debug=*) - if [[ "$role" == "client" ]]; then - echo "Cannot use -d,--debug with -t,--target" - usage - return 1 - fi - role=debug - policy="${i#*=}" - if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then - echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" - usage - return 1 - fi - shift - ;; - -p=* | --perf=*) - if [[ "$role" == "perf" ]]; then - echo "Cannot use -p,--perf with -t,--target" - usage - return 1 - fi - role=perf - policy="${i#*=}" - if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then - echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" - usage - return 1 - fi - shift - ;; - -h | --help) - usage - exit 0 - ;; - *) - echo "$1 is a not a valid option" - usage - return 1 - ;; - esac - done - - # default to both if no arguments were passed - if [[ -z "$role" ]]; then - role="both" - fi - - # Set globals as read only - declare -r target - declare -r policy - declare -r role -} - -# Starts the Sledge Runtime -start_runtime() { - printf "Starting Runtime: " - if (($# < 2 || $# > 3)); then - printf "[ERR]\n" - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - printf "[ERR]\n" - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - elif ! [[ -d "$2" ]]; then - printf "[ERR]\n" - error_msg "directory \"$2\" does not exist" - return 1 - elif ! [[ $3 =~ ^(foreground|background)$ ]]; then - printf "[ERR]\n" - error_msg "expected foreground or background was \"$3\"" - return 1 - fi - - local -r scheduler="$1" - local -r results_directory="$2" - local -r how_to_run="${3:-background}" - - local -r log_name=log.txt - local log="$results_directory/${log_name}" - - log_environment >> "$log" - - case "$how_to_run" in - "background") - SLEDGE_SCHEDULER="$scheduler" \ - sledgert "$experiment_directory/spec.json" >> "$log" 2>> "$log" & - ;; - "foreground") - SLEDGE_SCHEDULER="$scheduler" \ - sledgert "$experiment_directory/spec.json" - ;; - esac - - printf "[OK]\n" - return 0 -} - -# Sends requests until the per-module perf window buffers are full -# This ensures that Sledge has accurate estimates of execution time -run_samples() { - local hostname="${1:-localhost}" - - echo -n "Running Samples: " - hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || { - error_msg "fib40 samples failed" - return 1 - } - - hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || { - error_msg "fib10 samples failed" - return 1 - } - - echo "[OK]" - return 0 -} - -# Execute the fib10 and fib40 experiments sequentially and concurrently -# $1 (results_directory) - a directory where we will store our results -# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost -run_experiments() { - if (($# < 1 || $# > 2)); then - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ -d "$1" ]]; then - error_msg "directory \"$1\" does not exist" - return 1 - fi - - local results_directory="$1" - local hostname="${2:-localhost}" - - # The duration in seconds that we want the client to send requests - local -ir duration_sec=15 - - # The duration in seconds that the low priority task should run before the high priority task starts - local -ir offset=5 - - printf "Running Experiments\n" - - # Run each separately - printf "\tfib40: " - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$hostname:10040" > "$results_directory/fib40.csv" 2> /dev/null || { - printf "[ERR]\n" - error_msg "fib40 failed" - return 1 - } - get_result_count "$results_directory/fib40.csv" || { - printf "[ERR]\n" - error_msg "fib40 unexpectedly has zero requests" - return 1 - } - printf "[OK]\n" - - printf "\tfib10: " - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$hostname:10010" > "$results_directory/fib10.csv" 2> /dev/null || { - printf "[ERR]\n" - error_msg "fib10 failed" - return 1 - } - get_result_count "$results_directory/fib10.csv" || { - printf "[ERR]\n" - error_msg "fib10 unexpectedly has zero requests" - return 1 - } - printf "[OK]\n" - - # Run concurrently - # The lower priority has offsets to ensure it runs the entire time the high priority is trying to run - # This asynchronously trigger jobs and then wait on their pids - local fib40_con_PID - local fib10_con_PID - - hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" > "$results_directory/fib40_con.csv" 2> /dev/null & - fib40_con_PID="$!" - - sleep $offset - - hey -z "${duration_sec}s" -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:10010" > "$results_directory/fib10_con.csv" 2> /dev/null & - fib10_con_PID="$!" - - wait -f "$fib10_con_PID" || { - printf "\tfib10_con: [ERR]\n" - error_msg "failed to wait -f ${fib10_con_PID}" - return 1 - } - get_result_count "$results_directory/fib10_con.csv" || { - printf "\tfib10_con: [ERR]\n" - error_msg "fib10_con has zero requests. This might be because fib40_con saturated the runtime" - return 1 - } - printf "\tfib10_con: [OK]\n" - - wait -f "$fib40_con_PID" || { - printf "\tfib40_con: [ERR]\n" - error_msg "failed to wait -f ${fib40_con_PID}" - return 1 - } - get_result_count "$results_directory/fib40_con.csv" || { - printf "\tfib40_con: [ERR]\n" - error_msg "fib40_con has zero requests." - return 1 - } - printf "\tfib40_con: [OK]\n" - - return 0 -} - -# Process the experimental results and generate human-friendly results for success rate, throughput, and latency -process_results() { - if (($# != 1)); then - error_msg "invalid number of arguments ($#, expected 1)" - return 1 - elif ! [[ -d "$1" ]]; then - error_msg "directory $1 does not exist" - return 1 - fi - - local -r results_directory="$1" - - echo -n "Processing Results: " - - # Write headers to CSVs - printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" - printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" - printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - - # The four types of results that we are capturing. - # fib10 and fib 40 are run sequentially. - # fib10_con and fib40_con are run concurrently - local -ar payloads=(fib10 fib10_con fib40 fib40_con) - - # The deadlines for each of the workloads - # TODO: Scrape these from spec.json - local -Ar deadlines_ms=( - [fib10]=2 - [fib40]=3000 - ) - - for payload in "${payloads[@]}"; do - # Strip the _con suffix when getting the deadline - local -i deadline=${deadlines_ms[${payload/_con/}]} - - # Get Number of Requests, subtracting the header - local -i requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && { - echo "$payload unexpectedly has zero requests" - continue - } - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # We determine duration by looking at the timestamp of the last complete request - # TODO: Should this instead just use the client-side synthetic duration_sec value? - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - - # Throughput is calculated as the mean number of successful requests per second - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" - done - - # Transform csvs to dat files for gnuplot - csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv" - - # Generate gnuplots. Commented out because we don't have *.gnuplots defined - # generate_gnuplots -} - -run_server() { - if (($# != 1)); then - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - fi - - local -r scheduler="$1" - - if [[ "$role" == "both" ]]; then - local -r results_directory="$experiment_directory/res/$timestamp/$scheduler" - local -r how_to_run="background" - elif [[ "$role" == "server" ]]; then - local -r results_directory="$experiment_directory/res/$timestamp" - local -r how_to_run="foreground" - else - error_msg "Unexpected $role" - return 1 - fi - - mkdir -p "$results_directory" - - start_runtime "$scheduler" "$results_directory" "$how_to_run" || { - echo "start_runtime RC: $?" - error_msg "Error calling start_runtime $scheduler $results_directory" - return 1 - } - - return 0 -} - -run_perf() { - if (($# != 1)); then - printf "[ERR]\n" - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - printf "[ERR]\n" - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - fi - - if ! command -v perf; then - echo "perf is not present." - exit 1 - fi - - local -r scheduler="$1" - - SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$experiment_directory/spec.json" -} - -# Starts the Sledge Runtime under GDB -run_debug() { - # shellcheck disable=SC2155 - local project_directory=$(cd ../.. && pwd) - if (($# != 1)); then - printf "[ERR]\n" - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - printf "[ERR]\n" - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - fi - - local -r scheduler="$1" - - if [[ "$project_directory" != "/sledge/runtime" ]]; then - printf "It appears that you are not running in the container. Substituting path to match host environment\n" - SLEDGE_SCHEDULER="$scheduler" gdb \ - --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert - else - SLEDGE_SCHEDULER="$scheduler" gdb \ - --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert - fi - return 0 -} - -run_client() { - if [[ "$role" == "both" ]]; then - local results_directory="$experiment_directory/res/$timestamp/$scheduler" - elif [[ "$role" == "client" ]]; then - local results_directory="$experiment_directory/res/$timestamp" - else - error_msg "${FUNCNAME[0]} Unexpected $role" - return 1 - fi - - mkdir -p "$results_directory" - - run_samples "$target" || { - error_msg "Error calling run_samples $target" - return 1 - } - - run_experiments "$results_directory" || { - error_msg "Error calling run_experiments $results_directory" - return 1 - } - - process_results "$results_directory" || { - error_msg "Error calling process_results $results_directory" - return 1 - } - - echo "[OK]" - return 0 -} - -run_both() { - local -ar schedulers=(EDF FIFO) - for scheduler in "${schedulers[@]}"; do - printf "Running %s\n" "$scheduler" - - run_server "$scheduler" || { - error_msg "Error calling run_server" - return 1 - } - - run_client || { - error_msg "Error calling run_client" - kill_runtime - return 1 - } - - kill_runtime || { - error_msg "Error calling kill_runtime" - return 1 - } - - done - - return 0 -} - -main() { - initialize_globals - parse_arguments "$@" || { - exit 1 - } - - case $role in - both) - run_both - ;; - server) - run_server "$policy" - ;; - debug) - run_debug "$policy" - ;; - perf) - run_perf "$policy" - ;; - client) - run_client - ;; - *) - echo "Invalid state" - false - ;; - esac - - exit "$?" -} - -main "$@" diff --git a/runtime/experiments/preemption/.gitignore b/runtime/experiments/preemption/.gitignore deleted file mode 100644 index 64f722e..0000000 --- a/runtime/experiments/preemption/.gitignore +++ /dev/null @@ -1 +0,0 @@ -res diff --git a/runtime/experiments/preemption/README.md b/runtime/experiments/preemption/README.md deleted file mode 100644 index c2160ef..0000000 --- a/runtime/experiments/preemption/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Preemption - -## Question - -- How do mixed criticality workloads perform under the Sledge scheduler policies? -- How does the latency of a high criticality workload that triggers preemption on a system under load compare to being the only workload on the system? -- What is the slowdown on the low priority workload? -- How does this affect aggregate throughput? - -## Setup - -The system is configured with admission control disabled. - -The driver script drives a bimodal distribution of long-running low-priority and short-running high-priority workloads - -Relative Deadlines are tuned such that the scheduler should always preempt the low-priority workload for the high-priority workload. - -A driver script runs the two workloads separately as a baseline - -It then runs them concurrently, starting the low-priority long-running workload first such that the system begins execution and accumulates requests in the data structures. The high-priority short-running workload then begins. - -## Independent Variable - -The Scheduling Policy: EDF versus FIFO - -## Dependent Variables - -Latency of high priority workload diff --git a/runtime/experiments/preemption/backend.sh b/runtime/experiments/preemption/backend.sh deleted file mode 100755 index 8d2ba89..0000000 --- a/runtime/experiments/preemption/backend.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) - -results_directory="$experiment_directory/res/$timestamp/$scheduler" -log=log.txt - -mkdir -p "$results_directory" -log_environment >> "$results_directory/$log" - -# Start the runtime -PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" | tee -a "$results_directory/$log" diff --git a/runtime/experiments/preemption/client.sh b/runtime/experiments/preemption/client.sh deleted file mode 100755 index da778b2..0000000 --- a/runtime/experiments/preemption/client.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Modified to target a remote host - -timestamp=$(date +%s) -experiment_directory=$(pwd) -host=192.168.1.13 - -results_directory="$experiment_directory/res/$timestamp" - -mkdir -p "$results_directory" - -# Start the runtime -inputs=(40 10) -duration_sec=30 -offset=5 - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -for input in ${inputs[*]}; do - hey -n 45 -c 4 -t 0 -o csv -m GET -d "$input\n" http://"$host":$((10000 + input)) -done -echo "[DONE]" -sleep 30 - -echo "Running Experiments" -# Run each separately -hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$host:10010" > "$results_directory/fib10.csv" -echo "fib(10) Complete" -sleep 60 - -hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" > "$results_directory/fib40.csv" -echo "fib(40) Complete" -sleep 120 - -# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run -hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" > "$results_directory/fib40-con.csv" & -sleep $offset -hey -z ${duration_sec}s -cpus 3 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$host:10010" > "$results_directory/fib10-con.csv" & -sleep $((duration_sec + offset + 15)) -echo "fib(10) & fib(40) Complete" - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -durations_s=(15 15 15 25) -payloads=(fib10 fib10-con fib40 fib40-con) - -for payload in ${payloads[*]}; do - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - duration=${durations_s[$i]} - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if required -echo "[DONE]" diff --git a/runtime/experiments/preemption/debug.sh b/runtime/experiments/preemption/debug.sh deleted file mode 100755 index 4ef1936..0000000 --- a/runtime/experiments/preemption/debug.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" -export SLEDGE_SCHEDULER="EDF" - -gdb --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert diff --git a/runtime/experiments/preemption/fix_results.sh b/runtime/experiments/preemption/fix_results.sh deleted file mode 100755 index 6eb4568..0000000 --- a/runtime/experiments/preemption/fix_results.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Modified to target a remote host - -timestamp=1606608313-FIFO -experiment_directory=$(pwd) -results_directory="$experiment_directory/res/$timestamp" - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p998,p999,p100\n" >> "$results_directory/latency.csv" - -durations_s=(15 15 15 25) -payloads=(fib10 fib10-con fib40 fib40-con) - -for payload in ${payloads[*]}; do - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - duration=${durations_s[$i]} - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p998 = int('"$oks"' * 0.998) - p999 = int('"$oks"' * 0.999) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p998 {printf "%1.4f,", $0} - NR==p999 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots. Commented out because we don't have *.gnuplots defined -# generate_gnuplots - -# Cleanup, if required -echo "[DONE]" diff --git a/runtime/experiments/preemption/perf.sh b/runtime/experiments/preemption/perf.sh deleted file mode 100755 index c87504f..0000000 --- a/runtime/experiments/preemption/perf.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" - -SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=EDF perf record -g -s sledgert "$experiment_directory/spec.json" diff --git a/runtime/experiments/preemption/run.sh b/runtime/experiments/preemption/run.sh deleted file mode 100755 index b549e21..0000000 --- a/runtime/experiments/preemption/run.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -source ../common.sh - -# Validate dependencies -declare -a -r dependencies=(awk hey wc) -for dependency in "${dependencies[@]}"; do - if ! command -v "$dependency" &> /dev/null; then - echo "$dependency could not be found" - exit - fi -done - -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) - -schedulers=(EDF FIFO) -for scheduler in ${schedulers[*]}; do - - results_directory="$experiment_directory/res/$timestamp/$scheduler" - log=log.txt - - mkdir -p "$results_directory" - log_environment >> "$results_directory/$log" - - # Start the runtime - if [ "$1" != "-d" ]; then - SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=$scheduler PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 - else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" - fi - - inputs=(40 10) - duration_sec=15 - offset=5 - - # Execute workloads long enough for runtime to learn excepted execution time - echo -n "Running Samples: " - for input in ${inputs[*]}; do - hey -z ${duration_sec}s -cpus 3 -t 0 -o csv -m GET -d "$input\n" http://localhost:$((10000 + input)) - done - echo "[DONE]" - sleep 5 - - echo "Running Experiments" - # Run each separately - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40.csv" - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10.csv" - - # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run - hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40-con.csv" & - sleep $offset - hey -z ${duration_sec}s -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10-con.csv" & - sleep $((duration_sec + offset + 15)) - - # Stop the runtime if not in debug mode - [ "$1" != "-d" ] && kill_runtime - - # Generate *.csv and *.dat results - echo -n "Parsing Results: " - - printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" - printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" - printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - - deadlines_ms=(2 2 3000 3000) - payloads=(fib10 fib10-con fib40 fib40-con) - - for ((i = 0; i < 4; i++)); do - # for payload in ${payloads[*]}; do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" - done - - # Transform csvs to dat files for gnuplot - for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" - done - - # Generate gnuplots. Commented out because we don't have *.gnuplots defined - # generate_gnuplots - - # Cleanup, if requires - echo "[DONE]" -done diff --git a/runtime/experiments/preemption/run_relative.sh b/runtime/experiments/preemption/run_relative.sh deleted file mode 100755 index e286b55..0000000 --- a/runtime/experiments/preemption/run_relative.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) - -schedulers=(EDF FIFO) -for scheduler in ${schedulers[*]}; do - - results_directory="$experiment_directory/res/$timestamp/$scheduler" - log=log.txt - - mkdir -p "$results_directory" - log_environment >> "$results_directory/$log" - - # Start the runtime - if [ "$1" != "-d" ]; then - SLEDGE_NWORKERS=5 SLEDGE_SCHEDULER=$scheduler PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 - else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" - fi - - inputs=(40 10) - duration_sec=15 - offset=5 - - # Execute workloads long enough for runtime to learn excepted execution time - echo -n "Running Samples: " - for input in ${inputs[*]}; do - hey -z ${duration_sec}s -cpus 3 -t 0 -o csv -m GET -d "$input\n" http://localhost:$((10000 + input)) - done - echo "[DONE]" - sleep 5 - - echo "Running Experiments" - # Run each separately - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40.csv" - hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10.csv" - - # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run - hey -z $((duration_sec + 2 * offset))s -cpus 2 -c 100 -t 0 -o csv -m GET -d "40\n" http://localhost:10040 > "$results_directory/fib40-con.csv" & - sleep $offset - hey -z ${duration_sec}s -cpus 2 -c 100 -t 0 -o csv -m GET -d "10\n" http://localhost:10010 > "$results_directory/fib10-con.csv" & - sleep $((duration_sec + offset + 15)) - - # Stop the runtime if not in debug mode - [ "$1" != "-d" ] && kill_runtime - - # Generate *.csv and *.dat results - echo -n "Parsing Results: " - - printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" - printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" - printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - - deadlines_ms=(2 2 3000 3000) - payloads=(fib10 fib10-con fib40 fib40-con) - - for ((i = 0; i < 4; i++)); do - # for payload in ${payloads[*]}; do - payload=${payloads[$i]} - deadline=${deadlines_ms[$i]} - - # Get Number of Requests - requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && continue - - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$payload"'," - } - NR==p50 {printf "%1.4f%,", $0 / '"$deadline"' * 100} - NR==p90 {printf "%1.4f%,", $0 / '"$deadline"' * 100} - NR==p99 {printf "%1.4f%,", $0 / '"$deadline"' * 100} - NR==p100 {printf "%1.4f%\n", $0 / '"$deadline"' * 100} - ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - # rm -rf "$results_directory/$payload-response.csv" - done - - # Transform csvs to dat files for gnuplot - for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" - done - - # Generate gnuplots. Commented out because we don't have *.gnuplots defined - # generate_gnuplots - - # Cleanup, if requires - echo "[DONE]" -done diff --git a/runtime/experiments/preemption/scratch.txt b/runtime/experiments/preemption/scratch.txt deleted file mode 100644 index 2f45190..0000000 --- a/runtime/experiments/preemption/scratch.txt +++ /dev/null @@ -1,5 +0,0 @@ - - -hey -n 200 -c 200 -t 0 -m GET -d "40\n" http://localhost:10040 - -hey -n 500 -c 500 -t 0 -m GET -d "10\n" http://localhost:10010 diff --git a/runtime/experiments/preemption/spec.json b/runtime/experiments/preemption/spec.json deleted file mode 100644 index 0bf91bf..0000000 --- a/runtime/experiments/preemption/spec.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "active": true, - "name": "fibonacci_10", - "path": "fibonacci_wasm.so", - "port": 10010, - "expected-execution-us": 600, - "relative-deadline-us": 2000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" -}, -{ - "active": true, - "name": "fibonacci_40", - "path": "fibonacci_wasm.so", - "port": 10040, - "expected-execution-us": 550000, - "relative-deadline-us": 300000000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" -} From 7e62eceb0eca14f0bb5bbd1b391cb76307e172b4 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Thu, 15 Apr 2021 19:49:41 +0000 Subject: [PATCH 11/13] refactor: concurrency and payload use framework --- runtime/experiments/applications/.gitignore | 1 - runtime/experiments/applications/debug.sh | 19 - .../experiments/applications/latency.gnuplot | 20 - runtime/experiments/applications/run.sh | 111 ---- runtime/experiments/applications/spec.json | 42 -- .../experiments/applications/success.gnuplot | 12 - runtime/experiments/applications/tests.sh | 10 - .../applications/throughput.gnuplot | 13 - .../experiments/bash_libraries/csv_to_dat.sh | 29 ++ .../experiments/bash_libraries/error_msg.sh | 7 + .../experiments/bash_libraries/framework.sh | 410 +++++++++++++++ .../bash_libraries/generate_gnuplots.sh | 36 ++ .../bash_libraries/get_result_count.sh | 38 ++ runtime/experiments/bash_libraries/panic.sh | 24 + .../experiments/bash_libraries/path_join.sh | 14 + runtime/experiments/bimodal/run.sh | 107 ++-- runtime/experiments/common.sh | 487 ------------------ runtime/experiments/concurrency/run.sh | 109 +++- runtime/experiments/payload/body/generate.sh | 22 +- runtime/experiments/payload/debug.sh | 19 - runtime/experiments/payload/run.sh | 267 ++++++---- 21 files changed, 894 insertions(+), 903 deletions(-) delete mode 100644 runtime/experiments/applications/.gitignore delete mode 100755 runtime/experiments/applications/debug.sh delete mode 100644 runtime/experiments/applications/latency.gnuplot delete mode 100755 runtime/experiments/applications/run.sh delete mode 100644 runtime/experiments/applications/spec.json delete mode 100644 runtime/experiments/applications/success.gnuplot delete mode 100755 runtime/experiments/applications/tests.sh delete mode 100644 runtime/experiments/applications/throughput.gnuplot create mode 100644 runtime/experiments/bash_libraries/csv_to_dat.sh create mode 100644 runtime/experiments/bash_libraries/error_msg.sh create mode 100644 runtime/experiments/bash_libraries/framework.sh create mode 100644 runtime/experiments/bash_libraries/generate_gnuplots.sh create mode 100644 runtime/experiments/bash_libraries/get_result_count.sh create mode 100644 runtime/experiments/bash_libraries/panic.sh create mode 100644 runtime/experiments/bash_libraries/path_join.sh delete mode 100644 runtime/experiments/common.sh delete mode 100755 runtime/experiments/payload/debug.sh diff --git a/runtime/experiments/applications/.gitignore b/runtime/experiments/applications/.gitignore deleted file mode 100644 index 64f722e..0000000 --- a/runtime/experiments/applications/.gitignore +++ /dev/null @@ -1 +0,0 @@ -res diff --git a/runtime/experiments/applications/debug.sh b/runtime/experiments/applications/debug.sh deleted file mode 100755 index f40fa45..0000000 --- a/runtime/experiments/applications/debug.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" - -gdb --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert diff --git a/runtime/experiments/applications/latency.gnuplot b/runtime/experiments/applications/latency.gnuplot deleted file mode 100644 index 095aa47..0000000 --- a/runtime/experiments/applications/latency.gnuplot +++ /dev/null @@ -1,20 +0,0 @@ -reset - -set term jpeg -set output "latency.jpg" - -set xlabel "Payload (bytes)" -set xrange [-5:1050000] - -set ylabel "Latency (ms)" -set yrange [0:] - -set key left top - - -set style histogram columnstacked - -plot 'latency.dat' using 1:2 title 'p50', \ - 'latency.dat' using 1:3 title 'p90', \ - 'latency.dat' using 1:4 title 'p99', \ - 'latency.dat' using 1:5 title 'p100', \ diff --git a/runtime/experiments/applications/run.sh b/runtime/experiments/applications/run.sh deleted file mode 100755 index 04843d1..0000000 --- a/runtime/experiments/applications/run.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -source ../common.sh - -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb - -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) -results_directory="$experiment_directory/res/$timestamp" -log=log.txt - -mkdir -p "$results_directory" - -log_environment >> "$results_directory/$log" - -# Start the runtime -if [ "$1" != "-d" ]; then - PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 -else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" -fi -payloads=(fivebyeight/5x8 handwriting/handwrt1 hyde/hyde) -ports=(10000 10001 10002) -iterations=1000 - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -for i in {0..2}; do - hey -n 200 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/${payloads[$i]}.pnm" "http://localhost:${ports[$i]}" -done -sleep 1 -echo "[DONE]" - -# Execute the experiments -echo "Running Experiments" -for i in {0..2}; do - printf "\t%s Payload: " "${payloads[$i]}" - file=$(echo "${payloads[$i]}" | awk -F/ '{print $2}').csv - hey -n "$iterations" -c 3 -cpus 2 -o csv -m GET -D "$experiment_directory/${payloads[$i]}.pnm" "http://localhost:${ports[$i]}" > "$results_directory/$file" - echo "[DONE]" -done - -# Stop the runtime - -if [ "$1" != "-d" ]; then - sleep 5 - kill_runtime -fi - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv" -printf "Concurrency,Throughput\n" >> "$results_directory/throughput.csv" -printf "Con,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -for payload in ${payloads[*]}; do - # Calculate Success Rate for csv - file=$(echo "$payload" | awk -F/ '{print $2}') - awk -F, ' - $7 == 200 {ok++} - END{printf "'"$file"',%3.5f\n", (ok / '"$iterations"' * 100)} - ' < "$results_directory/$file.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$file.csv" \ - | sort -g > "$results_directory/$file-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$file-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$file.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%s,%f\n" "$file" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' - BEGIN { - sum = 0 - p50 = int('"$oks"' * 0.5) - p90 = int('"$oks"' * 0.9) - p99 = int('"$oks"' * 0.99) - p100 = '"$oks"' - printf "'"$file"'," - } - NR==p50 {printf "%1.4f,", $0} - NR==p90 {printf "%1.4f,", $0} - NR==p99 {printf "%1.4f,", $0} - NR==p100 {printf "%1.4f\n", $0} - ' < "$results_directory/$file-response.csv" >> "$results_directory/latency.csv" - - # Delete scratch file used for sorting/counting - rm -rf "$results_directory/$file-response.csv" -done - -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done - -# Generate gnuplots -generate_gnuplots - -# Cleanup, if requires -echo "[DONE]" diff --git a/runtime/experiments/applications/spec.json b/runtime/experiments/applications/spec.json deleted file mode 100644 index 8867897..0000000 --- a/runtime/experiments/applications/spec.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "active": true, - "name": "gocr", - "path": "gocr.aso", - "port": 10000, - "relative-deadline-us": 50000000000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024000, - "http-resp-headers": [], - "http-resp-size": 1024000, - "http-resp-content-type": "text/plain" -}, -{ - "active": true, - "name": "gocr", - "path": "gocr.aso", - "port": 10001, - "relative-deadline-us": 50000000000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024000, - "http-resp-headers": [], - "http-resp-size": 1024000, - "http-resp-content-type": "text/plain" -}, -{ - "active": true, - "name": "gocr", - "path": "gocr.aso", - "port": 10002, - "relative-deadline-us": 50000000000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 5335057, - "http-resp-headers": [], - "http-resp-size": 5335057, - "http-resp-content-type": "text/plain" -} diff --git a/runtime/experiments/applications/success.gnuplot b/runtime/experiments/applications/success.gnuplot deleted file mode 100644 index 9a55bce..0000000 --- a/runtime/experiments/applications/success.gnuplot +++ /dev/null @@ -1,12 +0,0 @@ -reset - -set term jpeg -set output "success.jpg" - -set xlabel "Connections" -set xrange [-5:105] - -set ylabel "% 2XX" -set yrange [0:110] - -plot 'success.dat' using 1:2 title '2XX' diff --git a/runtime/experiments/applications/tests.sh b/runtime/experiments/applications/tests.sh deleted file mode 100755 index b7ce938..0000000 --- a/runtime/experiments/applications/tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -cd handwriting || exit -./run.sh -cd .. || exit -cd hyde || exit -./run.sh -cd .. || exit -cd fivebyeight || exit -./run.sh -cd .. diff --git a/runtime/experiments/applications/throughput.gnuplot b/runtime/experiments/applications/throughput.gnuplot deleted file mode 100644 index dc11747..0000000 --- a/runtime/experiments/applications/throughput.gnuplot +++ /dev/null @@ -1,13 +0,0 @@ -reset - -set term jpeg -set output "throughput.jpg" - -# TODO: Axis shouldn't be linear -set xlabel "Connections" -set xrange [-5:105] - -set ylabel "Requests/sec" -set yrange [0:] - -plot 'throughput.dat' using 1:2 title 'Reqs/sec' diff --git a/runtime/experiments/bash_libraries/csv_to_dat.sh b/runtime/experiments/bash_libraries/csv_to_dat.sh new file mode 100644 index 0000000..d6951a3 --- /dev/null +++ b/runtime/experiments/bash_libraries/csv_to_dat.sh @@ -0,0 +1,29 @@ +# shellcheck shell=bash +if [ -n "$__csv_to_dat_sh__" ]; then return; fi +__csv_to_dat_sh__=$(date) + +source "panic.sh" || exit 1 + +# Takes a variadic number of paths to *.csv files and converts to *.dat files in the same directory +csv_to_dat() { + if (($# == 0)); then + panic "insufficient parameters" + return 1 + fi + + for arg in "$@"; do + if ! [[ "$arg" =~ ".csv"$ ]]; then + panic "$arg is not a *.csv file" + return 1 + fi + if [[ ! -f "$arg" ]]; then + panic "$arg does not exit" + return 1 + fi + done + + for file in "$@"; do + echo -n "#" > "${file/.csv/.dat}" + tr ',' ' ' < "$file" | column -t >> "${file/.csv/.dat}" + done +} diff --git a/runtime/experiments/bash_libraries/error_msg.sh b/runtime/experiments/bash_libraries/error_msg.sh new file mode 100644 index 0000000..c30585a --- /dev/null +++ b/runtime/experiments/bash_libraries/error_msg.sh @@ -0,0 +1,7 @@ +# shellcheck shell=bash +if [ -n "$__error_msg_sh__" ]; then return; fi +__error_msg_sh__=$(date) + +error_msg() { + printf "%.23s %s() at %s:%s - %s\n" "$(date +%F.%T.%N)" "${FUNCNAME[0]}" "$(realpath "${BASH_SOURCE[0]##*/}")" "${BASH_LINENO[0]}" "${@}" +} diff --git a/runtime/experiments/bash_libraries/framework.sh b/runtime/experiments/bash_libraries/framework.sh new file mode 100644 index 0000000..946aeae --- /dev/null +++ b/runtime/experiments/bash_libraries/framework.sh @@ -0,0 +1,410 @@ +# shellcheck shell=bash +if [ -n "$__framework_sh__" ]; then return; fi +__framework_sh__=$(date) + +# +# This framework simplifies the scripting of experiments +# +# To use, import the framework source file and pass all arguments to the provided main function +# source "framework.sh" +# +# main "$@" +# +# In your script, implement the following functions above main: +# - experiment_main +# + +source "path_join.sh" || exit 1 +source "panic.sh" || exit 1 + +__framework_sh__usage() { + echo "$0 [options...]" + echo "" + echo "Options:" + echo " -t,--target= Execute as client against remote URL" + echo " -s,--serve= Serve with scheduling policy, but do not run client" + echo " -d,--debug= Debug under GDB with scheduling policy, but do not run client" + echo " -p,--perf= Run under perf with scheduling policy. Run on baremetal Linux host!" +} + +# Declares application level global state +__framework_sh__initialize_globals() { + # timestamp is used to name the results directory for a particular test run + # shellcheck disable=SC2155 + # shellcheck disable=SC2034 + declare -gir __framework_sh__timestamp=$(date +%s) + + # Globals used by parse_arguments + declare -g __framework_sh__target="" + declare -g __framework_sh__policy="" + declare -g __framework_sh__role="" + + # Configure environment variables + # shellcheck disable=SC2155 + declare -gr __framework_sh__application_directory="$(dirname "$(realpath "$0"))")" + local -r binary_directory="$(cd "$__framework_sh__application_directory" && cd ../../bin && pwd)" + export PATH=$binary_directory:$PATH + export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH + export SLEDGE_NWORKERS=5 +} + +# Parses arguments from the user and sets associates global state +__framework_sh__parse_arguments() { + for i in "$@"; do + case $i in + -t=* | --target=*) + if [[ "$__framework_sh__role" == "server" ]]; then + echo "Cannot set target when server" + __framework_sh__usage + return 1 + fi + __framework_sh__role=client + __framework_sh__target="${i#*=}" + shift + ;; + -s=* | --serve=*) + if [[ "$__framework_sh__role" == "client" ]]; then + echo "Cannot use -s,--serve with -t,--target" + __framework_sh__usage + return 1 + fi + __framework_sh__role=server + __framework_sh__policy="${i#*=}" + if [[ ! $__framework_sh__policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$__framework_sh__policy\" is not a valid policy. EDF or FIFO allowed" + __framework_sh__usage + return 1 + fi + shift + ;; + -d=* | --debug=*) + if [[ "$__framework_sh__role" == "client" ]]; then + echo "Cannot use -d,--debug with -t,--target" + __framework_sh__usage + return 1 + fi + __framework_sh__role=debug + __framework_sh__policy="${i#*=}" + if [[ ! $__framework_sh__policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$__framework_sh__policy\" is not a valid policy. EDF or FIFO allowed" + __framework_sh__usage + return 1 + fi + shift + ;; + -p=* | --perf=*) + if [[ "$__framework_sh__role" == "perf" ]]; then + echo "Cannot use -p,--perf with -t,--target" + __framework_sh__usage + return 1 + fi + __framework_sh__role=perf + __framework_sh__policy="${i#*=}" + if [[ ! $__framework_sh__policy =~ ^(EDF|FIFO)$ ]]; then + echo "\"$__framework_sh__policy\" is not a valid policy. EDF or FIFO allowed" + __framework_sh__usage + return 1 + fi + shift + ;; + -h | --help) + __framework_sh__usage + exit 0 + ;; + *) + echo "$1 is a not a valid option" + __framework_sh__usage + return 1 + ;; + esac + done + + # default to both if no arguments were passed + if [[ -z "$__framework_sh__role" ]]; then + __framework_sh__role="both" + __framework_sh__target="localhost" + fi + + # Set globals as read only + declare -r __framework_sh__target + declare -r __framework_sh__policy + declare -r __framework_sh__role +} + +# Log hardware and software info for the execution +__framework_sh__log_environment() { + if ! command -v git &> /dev/null; then + echo "git could not be found" + exit + fi + echo "*******" + echo "* Git *" + echo "*******" + git log | head -n 1 | cut -d' ' -f2 + git status + echo "" + + echo "************" + echo "* Makefile *" + echo "************" + cat "$(path_join "$__framework_sh__application_directory" ../../Makefile)" + echo "" + + echo "**********" + echo "* Run.sh *" + echo "**********" + cat "$(path_join "$__framework_sh__application_directory" ./run.sh)" + echo "" + + echo "************" + echo "* Hardware *" + echo "************" + lscpu + echo "" + + echo "*************" + echo "* Execution *" + echo "*************" +} + +# $1 - Scheduler Variant (EDF|FIFO) +# $2 - Results Directory +# $3 - How to run (foreground|background) +# $4 - JSON specification +__framework_sh__start_runtime() { + printf "Starting Runtime: " + if (($# != 4)); then + printf "[ERR]\n" + panic "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + panic "expected EDF or FIFO was \"$1\"" + return 1 + elif ! [[ -d "$2" ]]; then + printf "[ERR]\n" + panic "directory \"$2\" does not exist" + return 1 + elif ! [[ $3 =~ ^(foreground|background)$ ]]; then + printf "[ERR]\n" + panic "expected foreground or background was \"$3\"" + return 1 + elif [[ ! -f "$4" || "$4" != *.json ]]; then + printf "[ERR]\n" + panic "\"$4\" does not exist or is not a JSON" + return 1 + fi + + local -r scheduler="$1" + local -r results_directory="$2" + local -r how_to_run="$3" + local -r specification="$4" + + local -r log_name=log.txt + local log="$results_directory/${log_name}" + + __framework_sh__log_environment >> "$log" + + case "$how_to_run" in + "background") + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$specification" >> "$log" 2>> "$log" & + ;; + "foreground") + SLEDGE_SCHEDULER="$scheduler" \ + sledgert "$specification" + ;; + esac + + printf "[OK]\n" + return 0 +} + +__framework_sh__run_server() { + if (($# != 2)); then + panic "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + panic "expected EDF or FIFO was \"$1\"" + return 1 + elif ! [[ $2 =~ ^(foreground|background)$ ]]; then + printf "[ERR]\n" + panic "expected foreground or background was \"$3\"" + return 1 + fi + + local -r scheduler="$1" + local -r how_to_run="$2" + + __framework_sh__start_runtime "$scheduler" "$RESULTS_DIRECTORY" "$how_to_run" "$__framework_sh__application_directory/spec.json" || { + echo "__framework_sh__start_runtime RC: $?" + panic "Error calling __framework_sh__start_runtime $scheduler $RESULTS_DIRECTORY $how_to_run $__framework_sh__application_directory/spec.json" + return 1 + } + + return 0 +} + +__framework_sh__run_perf() { + if (($# != 1)); then + printf "[ERR]\n" + panic "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + panic "expected EDF or FIFO was \"$1\"" + return 1 + fi + + if ! command -v perf; then + echo "perf is not present." + exit 1 + fi + + local -r scheduler="$1" + + SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$__framework_sh__application_directory/spec.json" +} + +# Starts the Sledge Runtime under GDB +__framework_sh__run_debug() { + # shellcheck disable=SC2155 + local project_directory=$(cd ../.. && pwd) + if (($# != 1)); then + printf "[ERR]\n" + panic "invalid number of arguments \"$1\"" + return 1 + elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then + printf "[ERR]\n" + panic "expected EDF or FIFO was \"$1\"" + return 1 + fi + + local -r scheduler="$1" + + if [[ "$project_directory" != "/sledge/runtime" ]]; then + printf "It appears that you are not running in the container. Substituting path to match host environment\n" + SLEDGE_SCHEDULER="$scheduler" gdb \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="set substitute-path /sledge/runtime $project_directory" \ + --eval-command="run $__framework_sh__application_directory/spec.json" \ + sledgert + else + SLEDGE_SCHEDULER="$scheduler" gdb \ + --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="run $__framework_sh__application_directory/spec.json" \ + sledgert + fi + return 0 +} + +__framework_sh__run_client() { + experiment_main "$__framework_sh__target" "$RESULTS_DIRECTORY" || { + panic "Error calling process_results $RESULTS_DIRECTORY" + return 1 + } + + return 0 +} + +__framework_sh__run_both() { + local -ar schedulers=(EDF FIFO) + for scheduler in "${schedulers[@]}"; do + printf "Running %s\n" "$scheduler" + + __framework_sh__run_server "$scheduler" background || { + panic "Error calling __framework_sh__run_server" + return 1 + } + + __framework_sh__run_client || { + panic "Error calling __framework_sh__run_client" + __framework_sh__stop_runtime + return 1 + } + + __framework_sh__stop_runtime || { + panic "Error calling __framework_sh__stop_runtime" + return 1 + } + + done + + return 0 +} + +__framework_sh__create_and_export_results_directory() { + local dir="" + + # If we are running both client, and server, we need to namespace by scheduler since we run both variants + case "$__framework_sh__role" in + "both") + dir="$__framework_sh__application_directory/res/$__framework_sh__timestamp/$scheduler" + ;; + "client" | "server" | "debug" | "perf") + dir="$__framework_sh__application_directory/res/$__framework_sh__timestamp" + ;; + *) + panic "${FUNCNAME[0]} Unexpected $__framework_sh__role" + return 1 + ;; + esac + + mkdir -p "$dir" || { + panic "mkdir -p $dir" + return 1 + } + + export RESULTS_DIRECTORY="$dir" +} + +# Responsible for ensuring that the experiment file meets framework assumptions +__framework_sh__validate_experiment() { + if [[ $(type -t experiment_main) != "function" ]]; then + panic "function experiment_main was not defined" + return 1 + fi + +} + +main() { + __framework_sh__validate_experiment || exit 1 + __framework_sh__initialize_globals || exit 1 + __framework_sh__parse_arguments "$@" || exit 1 + __framework_sh__create_and_export_results_directory || exit 1 + + case $__framework_sh__role in + both) + __framework_sh__run_both + ;; + server) + __framework_sh__run_server "$__framework_sh__policy" foreground + ;; + debug) + __framework_sh__run_debug "$__framework_sh__policy" + ;; + perf) + __framework_sh__run_perf "$__framework_sh__policy" + ;; + client) + __framework_sh__run_client + ;; + *) + echo "Invalid state" + false + ;; + esac + + exit "$?" +} + +__framework_sh__stop_runtime() { + printf "Stopping Runtime: " + pkill sledgert > /dev/null 2> /dev/null + pkill hey > /dev/null 2> /dev/null + printf "[OK]\n" +} diff --git a/runtime/experiments/bash_libraries/generate_gnuplots.sh b/runtime/experiments/bash_libraries/generate_gnuplots.sh new file mode 100644 index 0000000..45826ce --- /dev/null +++ b/runtime/experiments/bash_libraries/generate_gnuplots.sh @@ -0,0 +1,36 @@ +# shellcheck shell=bash +if [ -n "$__generate_gnuplots_sh__" ]; then return; fi +__generate_gnuplots_sh__=$(date) + +source "panic.sh" || exit 1 + +# Runs all *.gnuplot files found gnuplot_directory from results_directory +# Outputting resulting diagrams in results_directory +# $1 - results_directory containing the data file referenced in the gnuplot file +# $2 - gnuplot_directory containing the *.gnuplot specification files +generate_gnuplots() { + local -r results_directory="$1" + local -r experiment_directory="$2" + + if ! command -v gnuplot &> /dev/null; then + panic "gnuplot could not be found in path" + return 1 + fi + # shellcheck disable=SC2154 + if [ -z "$results_directory" ]; then + panic "results_directory was unset or empty" + return 1 + fi + # shellcheck disable=SC2154 + if [ -z "$experiment_directory" ]; then + panic "error: EXPERIMENT_DIRECTORY was unset or empty" + return 1 + fi + cd "$results_directory" || exit + + shopt -s nullglob + for gnuplot_file in "$experiment_directory"/*.gnuplot; do + gnuplot "$gnuplot_file" + done + cd "$experiment_directory" || exit +} diff --git a/runtime/experiments/bash_libraries/get_result_count.sh b/runtime/experiments/bash_libraries/get_result_count.sh new file mode 100644 index 0000000..d7bd501 --- /dev/null +++ b/runtime/experiments/bash_libraries/get_result_count.sh @@ -0,0 +1,38 @@ +# shellcheck shell=bash +if [ -n "$__get_result_count_sh__" ]; then return; fi +__get_result_count_sh__=$(date) + +source "panic.sh" || exit 1 + +# Given a file, returns the number of results +# This assumes a *.csv file with a header +# $1 the file we want to check for results +# $2 an optional return nameref +get_result_count() { + if (($# != 1)); then + panic "insufficient parameters. $#/1" + return 1 + elif [[ ! -f $1 ]]; then + panic "the file $1 does not exist" + return 1 + elif [[ ! -s $1 ]]; then + panic "the file $1 is size 0" + return 1 + fi + + local -r file=$1 + + # Subtract one line for the header + local -i count=$(($(wc -l < "$file") - 1)) + + if (($# == 2)); then + # shellcheck disable=2034 + local -n __result=$2 + fi + + if ((count > 0)); then + return 0 + else + return 1 + fi +} diff --git a/runtime/experiments/bash_libraries/panic.sh b/runtime/experiments/bash_libraries/panic.sh new file mode 100644 index 0000000..8533823 --- /dev/null +++ b/runtime/experiments/bash_libraries/panic.sh @@ -0,0 +1,24 @@ +# shellcheck shell=bash +if [ -n "$__panic_sh__" ]; then return; fi +__panic_sh__=$(date) + +source "error_msg.sh" || exit 1 + +declare __common_did_dump_callstack=false + +__common_dump_callstack() { + echo "Call Stack:" + # Skip the dump_bash_stack and error_msg_frames + for ((i = 2; i < ${#FUNCNAME[@]}; i++)); do + printf "\t%d - %s\n" "$((i - 2))" "${FUNCNAME[i]} (${BASH_SOURCE[i + 1]}:${BASH_LINENO[i]})" + done +} + +# Public API +panic() { + error_msg "${@}" + [[ "$__common_did_dump_callstack" == false ]] && { + __common_dump_callstack + __common_did_dump_callstack=true + } +} diff --git a/runtime/experiments/bash_libraries/path_join.sh b/runtime/experiments/bash_libraries/path_join.sh new file mode 100644 index 0000000..fbb549b --- /dev/null +++ b/runtime/experiments/bash_libraries/path_join.sh @@ -0,0 +1,14 @@ +# shellcheck shell=bash + +if [ -n "$__path_join_sh__" ]; then return; fi +__path_join_sh__=$(date) + +path_join() { + local base=$1 + local relative=$2 + + relative_path="$(dirname "$relative")" + file_name="$(basename "$relative")" + absolute_path="$(cd "$base" && cd "$relative_path" && pwd)" + echo "$absolute_path/$file_name" +} diff --git a/runtime/experiments/bimodal/run.sh b/runtime/experiments/bimodal/run.sh index 2a84026..fa63cb7 100755 --- a/runtime/experiments/bimodal/run.sh +++ b/runtime/experiments/bimodal/run.sh @@ -1,5 +1,4 @@ #!/bin/bash -source ../common.sh # This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate # Success - The percentage of requests that complete by their deadlines @@ -7,50 +6,77 @@ source ../common.sh # Throughput - The mean number of successful requests per second # Latency - the rount-trip resonse time (unit?) of successful requests at the p50, p90, p99, and p100 percetiles +# Add bash_libraries directory to path +__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")" +__run_sh__bash_libraries_relative_path="../bash_libraries" +__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) +export PATH="$__run_sh__bash_libraries_absolute_path:$PATH" + +source csv_to_dat.sh || exit 1 +source framework.sh || exit 1 +# source generate_gnuplots.sh || exit 1 +source get_result_count.sh || exit 1 +source panic.sh || exit 1 +source path_join.sh || exit 1 + # Sends requests until the per-module perf window buffers are full # This ensures that Sledge has accurate estimates of execution time run_samples() { - local hostname="${1:-localhost}" + if (($# != 1)); then + panic "invalid number of arguments \"$1\"" + return 1 + elif [[ -z "$1" ]]; then + panic "hostname \"$1\" was empty" + return 1 + fi + + local hostname="${1}" # Scrape the perf window size from the source if possible - local -r perf_window_path="../../include/perf_window.h" + # TODO: Make a util function + local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window.h)" local -i perf_window_buffer_size if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then - echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" - echo "Defaulting to 16" + printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n" + printf "Defaulting to 16\n" perf_window_buffer_size=16 fi local -ir perf_window_buffer_size - echo -n "Running Samples: " + printf "Running Samples: " hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || { - error_msg "fib40 samples failed" + printf "[ERR]\n" + panic "fib40 samples failed" return 1 } hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || { - error_msg "fib10 samples failed" + printf "[ERR]\n" + panic "fib10 samples failed" return 1 } - echo "[OK]" + printf "[OK]\n" return 0 } # Execute the fib10 and fib40 experiments sequentially and concurrently -# $1 (results_directory) - a directory where we will store our results -# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost +# $1 (hostname) +# $2 (results_directory) - a directory where we will store our results run_experiments() { - if (($# < 1 || $# > 2)); then - error_msg "invalid number of arguments \"$1\"" + if (($# != 2)); then + panic "invalid number of arguments \"$1\"" return 1 - elif ! [[ -d "$1" ]]; then - error_msg "directory \"$1\" does not exist" + elif [[ -z "$1" ]]; then + panic "hostname \"$1\" was empty" + return 1 + elif [[ ! -d "$2" ]]; then + panic "directory \"$2\" does not exist" return 1 fi - local results_directory="$1" - local hostname="${2:-localhost}" + local hostname="$1" + local results_directory="$2" # The duration in seconds that we want the client to send requests local -ir duration_sec=15 @@ -64,12 +90,12 @@ run_experiments() { printf "\tfib40: " hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$hostname:10040" > "$results_directory/fib40.csv" 2> /dev/null || { printf "[ERR]\n" - error_msg "fib40 failed" + panic "fib40 failed" return 1 } get_result_count "$results_directory/fib40.csv" || { printf "[ERR]\n" - error_msg "fib40 unexpectedly has zero requests" + panic "fib40 unexpectedly has zero requests" return 1 } printf "[OK]\n" @@ -77,12 +103,12 @@ run_experiments() { printf "\tfib10: " hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$hostname:10010" > "$results_directory/fib10.csv" 2> /dev/null || { printf "[ERR]\n" - error_msg "fib10 failed" + panic "fib10 failed" return 1 } get_result_count "$results_directory/fib10.csv" || { printf "[ERR]\n" - error_msg "fib10 unexpectedly has zero requests" + panic "fib10 unexpectedly has zero requests" return 1 } printf "[OK]\n" @@ -103,24 +129,24 @@ run_experiments() { wait -f "$fib10_con_PID" || { printf "\tfib10_con: [ERR]\n" - error_msg "failed to wait -f ${fib10_con_PID}" + panic "failed to wait -f ${fib10_con_PID}" return 1 } get_result_count "$results_directory/fib10_con.csv" || { printf "\tfib10_con: [ERR]\n" - error_msg "fib10_con has zero requests. This might be because fib40_con saturated the runtime" + panic "fib10_con has zero requests. This might be because fib40_con saturated the runtime" return 1 } printf "\tfib10_con: [OK]\n" wait -f "$fib40_con_PID" || { printf "\tfib40_con: [ERR]\n" - error_msg "failed to wait -f ${fib40_con_PID}" + panic "failed to wait -f ${fib40_con_PID}" return 1 } get_result_count "$results_directory/fib40_con.csv" || { printf "\tfib40_con: [ERR]\n" - error_msg "fib40_con has zero requests." + panic "fib40_con has zero requests." return 1 } printf "\tfib40_con: [OK]\n" @@ -140,7 +166,7 @@ process_results() { local -r results_directory="$1" - echo -n "Processing Results: " + printf "Processing Results: " # Write headers to CSVs printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" @@ -163,14 +189,7 @@ process_results() { # Strip the _con suffix when getting the deadline local -i deadline=${deadlines_ms[${payload/_con/}]} - # Get Number of Requests, subtracting the header - local -i requests=$(($(wc -l < "$results_directory/$payload.csv") - 1)) - ((requests == 0)) && { - echo "$payload unexpectedly has zero requests" - continue - } - - # Calculate Success Rate for csv + # Calculate Success Rate for csv (percent of requests that return 200 within deadline) awk -F, ' $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} @@ -216,7 +235,25 @@ process_results() { csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv" # Generate gnuplots. Commented out because we don't have *.gnuplots defined - # generate_gnuplots + # generate_gnuplots "$results_directory" "$__run_sh__base_path" || { + # printf "[ERR]\n" + # panic "failed to generate gnuplots" + # } + + printf "[OK]\n" + return 0 +} + +# Expected Symbol used by the framework +experiment_main() { + local -r target_hostname="$1" + local -r results_directory="$2" + + run_samples "$target_hostname" || return 1 + run_experiments "$target_hostname" "$results_directory" || return 1 + process_results "$results_directory" || return 1 + + return 0 } main "$@" diff --git a/runtime/experiments/common.sh b/runtime/experiments/common.sh deleted file mode 100644 index 102cfdf..0000000 --- a/runtime/experiments/common.sh +++ /dev/null @@ -1,487 +0,0 @@ -#!/bin/bash - -declare __common_did_dump_callstack=false - -error_msg() { - [[ "$__common_did_dump_callstack" == false ]] && { - printf "%.23s %s() in %s, line %s: %s\n" "$(date +%F.%T.%N)" "${FUNCNAME[1]}" "${BASH_SOURCE[1]##*/}" "${BASH_LINENO[0]}" "${@}" - __common_dump_callstack - __common_did_dump_callstack=true - } -} - -__common_dump_callstack() { - echo "Call Stack:" - # Skip the dump_bash_stack and error_msg_frames - for ((i = 2; i < ${#FUNCNAME[@]}; i++)); do - printf "\t%d - %s\n" "$((i - 2))" "${FUNCNAME[i]}" - done -} - -log_environment() { - if ! command -v git &> /dev/null; then - echo "git could not be found" - exit - fi - echo "*******" - echo "* Git *" - echo "*******" - git log | head -n 1 | cut -d' ' -f2 - git status - echo "" - - echo "************" - echo "* Makefile *" - echo "************" - cat ../../Makefile - echo "" - - echo "**********" - echo "* Run.sh *" - echo "**********" - cat run.sh - echo "" - - echo "************" - echo "* Hardware *" - echo "************" - lscpu - echo "" - - echo "*************" - echo "* Execution *" - echo "*************" -} - -# Given a file, returns the number of results -# This assumes a *.csv file with a header -# $1 the file we want to check for results -# $2 an optional return nameref -get_result_count() { - if (($# != 1)); then - error_msg "insufficient parameters. $#/1" - return 1 - elif [[ ! -f $1 ]]; then - error_msg "the file $1 does not exist" - return 1 - elif [[ ! -s $1 ]]; then - error_msg "the file $1 is size 0" - return 1 - fi - - local -r file=$1 - - # Subtract one line for the header - local -i count=$(($(wc -l < "$file") - 1)) - - if (($# == 2)); then - # shellcheck disable=2034 - local -n __result=$2 - fi - - if ((count > 0)); then - return 0 - else - return 1 - fi -} - -usage() { - echo "$0 [options...]" - echo "" - echo "Options:" - echo " -t,--target= Execute as client against remote URL" - echo " -s,--serve= Serve with scheduling policy, but do not run client" - echo " -d,--debug= Debug under GDB with scheduling policy, but do not run client" - echo " -p,--perf= Run under perf with scheduling policy. Run on baremetal Linux host!" -} - -# Parses arguments from the user and sets associates global state -parse_arguments() { - for i in "$@"; do - case $i in - -t=* | --target=*) - if [[ "$role" == "server" ]]; then - echo "Cannot set target when server" - usage - return 1 - fi - role=client - target="${i#*=}" - shift - ;; - -s=* | --serve=*) - if [[ "$role" == "client" ]]; then - echo "Cannot use -s,--serve with -t,--target" - usage - return 1 - fi - role=server - policy="${i#*=}" - if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then - echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" - usage - return 1 - fi - shift - ;; - -d=* | --debug=*) - if [[ "$role" == "client" ]]; then - echo "Cannot use -d,--debug with -t,--target" - usage - return 1 - fi - role=debug - policy="${i#*=}" - if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then - echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" - usage - return 1 - fi - shift - ;; - -p=* | --perf=*) - if [[ "$role" == "perf" ]]; then - echo "Cannot use -p,--perf with -t,--target" - usage - return 1 - fi - role=perf - policy="${i#*=}" - if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then - echo "\"$policy\" is not a valid policy. EDF or FIFO allowed" - usage - return 1 - fi - shift - ;; - -h | --help) - usage - exit 0 - ;; - *) - echo "$1 is a not a valid option" - usage - return 1 - ;; - esac - done - - # default to both if no arguments were passed - if [[ -z "$role" ]]; then - role="both" - fi - - # Set globals as read only - declare -r target - declare -r policy - declare -r role -} - -# Declares application level global state -initialize_globals() { - # timestamp is used to name the results directory for a particular test run - # shellcheck disable=SC2155 - # shellcheck disable=SC2034 - declare -gir timestamp=$(date +%s) - - # shellcheck disable=SC2155 - declare -gr experiment_directory=$(pwd) - - # shellcheck disable=SC2155 - declare -gr binary_directory=$(cd ../../bin && pwd) - - # Globals used by parse_arguments - declare -g target="" - declare -g policy="" - declare -g role="" - - # Configure environment variables - export PATH=$binary_directory:$PATH - export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH - export SLEDGE_NWORKERS=5 -} - -# $1 - Scheduler Variant (EDF|FIFO) -# $2 - Results Directory -# $3 - How to run (foreground|background) -# $4 - JSON specification -start_runtime() { - printf "Starting Runtime: " - if (($# != 4)); then - printf "[ERR]\n" - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - printf "[ERR]\n" - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - elif ! [[ -d "$2" ]]; then - printf "[ERR]\n" - error_msg "directory \"$2\" does not exist" - return 1 - elif ! [[ $3 =~ ^(foreground|background)$ ]]; then - printf "[ERR]\n" - error_msg "expected foreground or background was \"$3\"" - return 1 - elif [[ ! -f "$4" || "$4" != *.json ]]; then - printf "[ERR]\n" - error_msg "\"$4\" does not exist or is not a JSON" - return 1 - fi - - local -r scheduler="$1" - local -r results_directory="$2" - local -r how_to_run="$3" - local -r specification="$4" - - local -r log_name=log.txt - local log="$results_directory/${log_name}" - - log_environment >> "$log" - - case "$how_to_run" in - "background") - SLEDGE_SCHEDULER="$scheduler" \ - sledgert "$specification" >> "$log" 2>> "$log" & - ;; - "foreground") - SLEDGE_SCHEDULER="$scheduler" \ - sledgert "$specification" - ;; - esac - - printf "[OK]\n" - return 0 -} - -run_server() { - if (($# != 1)); then - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - fi - - local -r scheduler="$1" - - if [[ "$role" == "both" ]]; then - local -r results_directory="$experiment_directory/res/$timestamp/$scheduler" - local -r how_to_run="background" - elif [[ "$role" == "server" ]]; then - local -r results_directory="$experiment_directory/res/$timestamp" - local -r how_to_run="foreground" - else - error_msg "Unexpected $role" - return 1 - fi - - mkdir -p "$results_directory" - - start_runtime "$scheduler" "$results_directory" "$how_to_run" "$experiment_directory/spec.json" || { - echo "start_runtime RC: $?" - error_msg "Error calling start_runtime $scheduler $results_directory" - return 1 - } - - return 0 -} - -run_perf() { - if (($# != 1)); then - printf "[ERR]\n" - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - printf "[ERR]\n" - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - fi - - if ! command -v perf; then - echo "perf is not present." - exit 1 - fi - - local -r scheduler="$1" - - SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$experiment_directory/spec.json" -} - -# Starts the Sledge Runtime under GDB -run_debug() { - # shellcheck disable=SC2155 - local project_directory=$(cd ../.. && pwd) - if (($# != 1)); then - printf "[ERR]\n" - error_msg "invalid number of arguments \"$1\"" - return 1 - elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then - printf "[ERR]\n" - error_msg "expected EDF or FIFO was \"$1\"" - return 1 - fi - - local -r scheduler="$1" - - if [[ "$project_directory" != "/sledge/runtime" ]]; then - printf "It appears that you are not running in the container. Substituting path to match host environment\n" - SLEDGE_SCHEDULER="$scheduler" gdb \ - --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert - else - SLEDGE_SCHEDULER="$scheduler" gdb \ - --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert - fi - return 0 -} - -run_client() { - if [[ "$role" == "both" ]]; then - local results_directory="$experiment_directory/res/$timestamp/$scheduler" - elif [[ "$role" == "client" ]]; then - local results_directory="$experiment_directory/res/$timestamp" - else - error_msg "${FUNCNAME[0]} Unexpected $role" - return 1 - fi - - mkdir -p "$results_directory" - - run_samples "$target" || { - error_msg "Error calling run_samples $target" - return 1 - } - - run_experiments "$results_directory" || { - error_msg "Error calling run_experiments $results_directory" - return 1 - } - - process_results "$results_directory" || { - error_msg "Error calling process_results $results_directory" - return 1 - } - - return 0 -} - -run_both() { - local -ar schedulers=(EDF FIFO) - for scheduler in "${schedulers[@]}"; do - printf "Running %s\n" "$scheduler" - - run_server "$scheduler" || { - error_msg "Error calling run_server" - return 1 - } - - run_client || { - error_msg "Error calling run_client" - kill_runtime - return 1 - } - - kill_runtime || { - error_msg "Error calling kill_runtime" - return 1 - } - - done - - return 0 -} - -main() { - initialize_globals - parse_arguments "$@" || { - exit 1 - } - - case $role in - both) - run_both - ;; - server) - run_server "$policy" - ;; - debug) - run_debug "$policy" - ;; - perf) - run_perf "$policy" - ;; - client) - run_client - ;; - *) - echo "Invalid state" - false - ;; - esac - - exit "$?" -} - -kill_runtime() { - printf "Stopping Runtime: " - pkill sledgert > /dev/null 2> /dev/null - pkill hey > /dev/null 2> /dev/null - printf "[OK]\n" -} - -# Takes a variadic number of *.gnuplot filenames and generates the resulting images -# Assumes that the gnuplot definitions are in the experiment directory -generate_gnuplots() { - if ! command -v gnuplot &> /dev/null; then - echo "${FUNCNAME[0]} error: gnuplot could not be found in path" - exit - fi - # shellcheck disable=SC2154 - if [ -z "$results_directory" ]; then - echo "${FUNCNAME[0]} error: results_directory was unset or empty" - dump_bash_stack - exit 1 - fi - # shellcheck disable=SC2154 - if [ -z "$experiment_directory" ]; then - echo "${FUNCNAME[0]} error: experiment_directory was unset or empty" - dump_bash_stack - exit 1 - fi - cd "$results_directory" || exit - for gnuplot_file in "${@}"; do - gnuplot "$experiment_directory/$gnuplot_file.gnuplot" - done - cd "$experiment_directory" || exit -} - -# Takes a variadic number of paths to *.csv files and converts to *.dat files in the same directory -csv_to_dat() { - if (($# == 0)); then - echo "${FUNCNAME[0]} error: insufficient parameters" - dump_bash_stack - fi - - for arg in "$@"; do - if ! [[ "$arg" =~ ".csv"$ ]]; then - echo "${FUNCNAME[0]} error: $arg is not a *.csv file" - dump_bash_stack - exit 1 - fi - done - - for file in "$@"; do - echo -n "#" > "${file/.csv/.dat}" - tr ',' ' ' < "$file" | column -t >> "${file/.csv/.dat}" - done -} diff --git a/runtime/experiments/concurrency/run.sh b/runtime/experiments/concurrency/run.sh index ffdbcec..0f69cd4 100755 --- a/runtime/experiments/concurrency/run.sh +++ b/runtime/experiments/concurrency/run.sh @@ -1,20 +1,41 @@ #!/bin/bash -source ../common.sh # This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Add bash_libraries directory to path +__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")" +__run_sh__bash_libraries_relative_path="../bash_libraries" +__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) +export PATH="$__run_sh__bash_libraries_absolute_path:$PATH" + +source csv_to_dat.sh || exit 1 +source framework.sh || exit 1 +source generate_gnuplots.sh || exit 1 +source get_result_count.sh || exit 1 +source panic.sh || exit 1 +source path_join.sh || exit 1 + declare -gi iterations=10000 declare -ga concurrency=(1 20 40 60 80 100) run_samples() { - local hostname="${1:-localhost}" + if (($# != 1)); then + panic "invalid number of arguments \"$1\"" + return 1 + elif [[ -z "$1" ]]; then + panic "hostname \"$1\" was empty" + return 1 + fi + + local hostname="$1" # Scrape the perf window size from the source if possible - local -r perf_window_path="../../include/perf_window.h" + # TODO: Make a util function + local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window.h)" local -i perf_window_buffer_size if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then - echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h" - echo "Defaulting to 16" + printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n" + printf "Defaulting to 16\n" perf_window_buffer_size=16 fi local -ir perf_window_buffer_size @@ -22,50 +43,64 @@ run_samples() { printf "Running Samples: " hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -cpus 3 -o csv -m GET "http://${hostname}:10000" 1> /dev/null 2> /dev/null || { printf "[ERR]\n" - error_msg "samples failed" + panic "samples failed" return 1 } - echo "[OK]" + printf "[OK]\n" return 0 } # Execute the experiments -# $1 (results_directory) - a directory where we will store our results -# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost +# $1 (hostname) +# $2 (results_directory) - a directory where we will store our results run_experiments() { - if (($# < 1 || $# > 2)); then - error_msg "invalid number of arguments \"$1\"" + if (($# != 2)); then + panic "invalid number of arguments \"$1\"" return 1 - elif ! [[ -d "$1" ]]; then - error_msg "directory \"$1\" does not exist" + elif [[ -z "$1" ]]; then + panic "hostname \"$1\" was empty" + return 1 + elif [[ ! -d "$2" ]]; then + panic "directory \"$2\" does not exist" return 1 fi - local results_directory="$1" - local hostname="${2:-localhost}" + local hostname="$1" + local results_directory="$2" # Execute the experiments - echo "Running Experiments" + printf "Running Experiments:\n" for conn in ${concurrency[*]}; do printf "\t%d Concurrency: " "$conn" - hey -n "$iterations" -c "$conn" -cpus 2 -o csv -m GET "http://$hostname:10000" > "$results_directory/con$conn.csv" 2> /dev/null - echo "[OK]" + hey -n "$iterations" -c "$conn" -cpus 2 -o csv -m GET "http://$hostname:10000" > "$results_directory/con$conn.csv" 2> /dev/null || { + printf "[ERR]\n" + panic "experiment failed" + return 1 + } + get_result_count "$results_directory/con$conn.csv" || { + printf "[ERR]\n" + panic "con$conn.csv unexpectedly has zero requests" + return 1 + } + printf "[OK]\n" done + + return 0 } process_results() { if (($# != 1)); then - error_msg "invalid number of arguments ($#, expected 1)" + panic "invalid number of arguments ($#, expected 1)" return 1 elif ! [[ -d "$1" ]]; then - error_msg "directory $1 does not exist" + panic "directory $1 does not exist" return 1 fi local -r results_directory="$1" - echo -n "Processing Results: " + printf "Processing Results: " # Write headers to CSVs printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv" @@ -73,6 +108,13 @@ process_results() { printf "Con,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" for conn in ${concurrency[*]}; do + + if [[ ! -f "$results_directory/con$conn.csv" ]]; then + printf "[ERR]\n" + panic "Missing $results_directory/con$conn.csv" + return 1 + fi + # Calculate Success Rate for csv (percent of requests resulting in 200) awk -F, ' $7 == 200 {ok++} @@ -116,17 +158,28 @@ process_results() { done # Transform csvs to dat files for gnuplot - for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" - done + csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv" # Generate gnuplots - generate_gnuplots latency success throughput + generate_gnuplots "$results_directory" "$__run_sh__base_path" || { + printf "[ERR]\n" + panic "failed to generate gnuplots" + } - # Cleanup, if requires - echo "[OK]" + printf "[OK]\n" + return 0 +} + +# Expected Symbol used by the framework +experiment_main() { + local -r target_hostname="$1" + local -r results_directory="$2" + run_samples "$target_hostname" || return 1 + run_experiments "$target_hostname" "$results_directory" || return 1 + process_results "$results_directory" || return 1 + + return 0 } main "$@" diff --git a/runtime/experiments/payload/body/generate.sh b/runtime/experiments/payload/body/generate.sh index 669e4dd..e8b8a19 100755 --- a/runtime/experiments/payload/body/generate.sh +++ b/runtime/experiments/payload/body/generate.sh @@ -1,12 +1,18 @@ #!/bin/bash # Generates payloads of 1KB, 10KB, 100KB, 1MB for size in 1024 $((1024 * 10)) $((1024 * 100)) $((1024 * 1024)); do - rm -rf $size.txt - i=0 - echo -n "Generating $size:" - while ((i < size)); do - printf 'a' >> $size.txt - ((i++)) - done - echo "[DONE]" + # If the file exists, but is not the right size, wipe it + if [[ -f "$size.txt" ]] && (("$(wc -c "$size.txt" | cut -d\ -f1)" != size)); then + rm -rf "$size.txt" + fi + + # Regenerate the file if missing + if [[ ! -f "$size.txt" ]]; then + echo -n "Generating $size: " + for ((i = 0; i < size; i++)); do + printf 'a' >> $size.txt + done + echo "[OK]" + fi + done diff --git a/runtime/experiments/payload/debug.sh b/runtime/experiments/payload/debug.sh deleted file mode 100755 index f40fa45..0000000 --- a/runtime/experiments/payload/debug.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Executes the runtime in GDB -# Substitutes the absolute path from the container with a path relatively derived from the location of this script -# This allows debugging outside of the Docker container -# Also disables pagination and stopping on SIGUSR1 - -experiment_directory=$(pwd) -project_directory=$(cd ../.. && pwd) -binary_directory=$(cd "$project_directory"/bin && pwd) - -export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" -export PATH="$binary_directory:$PATH" - -gdb --eval-command="handle SIGUSR1 nostop" \ - --eval-command="handle SIGPIPE nostop" \ - --eval-command="set pagination off" \ - --eval-command="set substitute-path /sledge/runtime $project_directory" \ - --eval-command="run $experiment_directory/spec.json" \ - sledgert diff --git a/runtime/experiments/payload/run.sh b/runtime/experiments/payload/run.sh index 7731a81..3e050dc 100755 --- a/runtime/experiments/payload/run.sh +++ b/runtime/experiments/payload/run.sh @@ -1,97 +1,149 @@ #!/bin/bash -source ../common.sh +# This experiment is intended to document how the level of concurrent requests influences +# - latency +# - throughput +# - success/failure rate + +# Add bash_libraries directory to path +__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")" +__run_sh__bash_libraries_relative_path="../bash_libraries" +__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd) +export PATH="$__run_sh__bash_libraries_absolute_path:$PATH" + +# Source libraries from bash_libraries directory +source "path_join.sh" || exit 1 +source "framework.sh" || exit 1 +source "get_result_count.sh" || exit 1 +source "generate_gnuplots.sh" || exit 1 + +# Experiment Globals and Setups +declare -ar payloads=(1024 10240 102400 1048576) + +declare -Ar ports=( + [1024]=10000 + [10240]=10001 + [102400]=10002 + [1048576]=10003 +) + +declare -ri iterations=10000 -# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate -# Use -d flag if running under gdb +# If the one of the expected body files doesn't exist, trigger the generation script. +cd "$__run_sh__base_path/body" && ./generate.sh && cd "$OLDPWD" || exit + +run_samples() { + local hostname="$1" + + # Scrape the perf window size from the source if possible + local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window.h)" + local -i perf_window_buffer_size + if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then + printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n" + printf "Defaulting to 16\n" + perf_window_buffer_size=16 + fi + local -ir perf_window_buffer_size + + # Execute workloads long enough for runtime to learn excepted execution time + printf "Running Samples: " + hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/1024.txt" "http://$hostname:10000" 1> /dev/null 2> /dev/null || { + printf "[ERR]\n" + panic "samples failed" + return 1 + } + hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/10240.txt" "http://$hostname:10001" 1> /dev/null 2> /dev/null || { + printf "[ERR]\n" + panic "samples failed" + return 1 + } + hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/102400.txt" "http://$hostname:10002" 1> /dev/null 2> /dev/null || { + printf "[ERR]\n" + panic "samples failed" + return 1 + } + hey -n "$iterations" --n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/1048576.txt" "http://$hostname:10003" 1> /dev/null 2> /dev/null || { + printf "[ERR]\n" + panic "samples failed" + return 1 + } + printf "[OK]\n" + return 0 +} + +run_experiments() { + if (($# != 2)); then + panic "invalid number of arguments \"$1\"" + return 1 + elif [[ ! -d "$2" ]]; then + panic "directory \"$2\" does not exist" + return 1 + fi -timestamp=$(date +%s) -experiment_directory=$(pwd) -binary_directory=$(cd ../../bin && pwd) -results_directory="$experiment_directory/res/$timestamp" -log=log.txt + local hostname="$1" + local results_directory="$2" + + # Execute the experiments + printf "Running Experiments:\n" + for payload in "${payloads[@]}"; do + printf "\t%d Payload: " "$payload" + hey -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" http://localhost:"${ports["$payload"]}" > "$results_directory/$payload.csv" 2> /dev/null || { + printf "[ERR]\n" + panic "experiment failed" + return 1 + } + get_result_count "$results_directory/$payload.csv" || { + printf "[ERR]\n" + panic "$payload.csv unexpectedly has zero requests" + return 1 + } + printf "[OK]\n" + done + + return 0 +} + +process_results() { + if (($# != 1)); then + panic "invalid number of arguments ($#, expected 1)" + return 1 + elif ! [[ -d "$1" ]]; then + panic "directory $1 does not exist" + return 1 + fi -mkdir -p "$results_directory" + local -r results_directory="$1" -log_environment >> "$results_directory/$log" + printf "Processing Results: " -# Start the runtime -if [ "$1" != "-d" ]; then - PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" & - sleep 1 -else - echo "Running under gdb" - echo "Running under gdb" >> "$results_directory/$log" -fi + printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" + printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" + printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" -payloads=(1024 10240 102400 1048576) -ports=(10000 10001 10002 10003) -iterations=10000 + for payload in ${payloads[*]}; do + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 {ok++} + END{printf "'"$payload"',%3.5f\n", (ok / '"$iterations"' * 100)} + ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" -# If the one of the expected body files doesn't exist, trigger the generation script. -for payload in ${payloads[*]}; do - if test -f "$experiment_directory/body/$payload.txt"; then - continue - else - echo "Generating Payloads: " - { - cd "$experiment_directory/body" && ./generate.sh - } - break - fi -done - -# Execute workloads long enough for runtime to learn excepted execution time -echo -n "Running Samples: " -hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1024.txt" http://localhost:10000 -hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/10240.txt" http://localhost:10001 -hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/102400.txt" http://localhost:10002 -hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1048576.txt" http://localhost:10003 -sleep 5 -echo "[DONE]" - -# Execute the experiments -echo "Running Experiments" -for i in {0..3}; do - printf "\t%d Payload: " "${payloads[$i]}" - hey -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$experiment_directory/body/${payloads[$i]}.txt" http://localhost:"${ports[$i]}" > "$results_directory/${payloads[$i]}.csv" - echo "[DONE]" -done - -# Stop the runtime -if [ "$1" != "-d" ]; then - sleep 5 - kill_runtime -fi - -# Generate *.csv and *.dat results -echo -n "Parsing Results: " - -printf "Payload,Success_Rate\n" >> "$results_directory/success.csv" -printf "Payload,Throughput\n" >> "$results_directory/throughput.csv" -printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv" - -for payload in ${payloads[*]}; do - # Calculate Success Rate for csv - awk -F, ' - $7 == 200 {ok++} - END{printf "'"$payload"',%3.5f\n", (ok / '"$iterations"' * 100)} - ' < "$results_directory/$payload.csv" >> "$results_directory/success.csv" - - # Filter on 200s, convery from s to ms, and sort - awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ - | sort -g > "$results_directory/$payload-response.csv" - - # Get Number of 200s - oks=$(wc -l < "$results_directory/$payload-response.csv") - ((oks == 0)) && continue # If all errors, skip line - - # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) - throughput=$(echo "$oks/$duration" | bc) - printf "%d,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" - - # Generate Latency Data for csv - awk ' + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \ + | sort -g > "$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l < "$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # We determine duration by looking at the timestamp of the last complete request + # TODO: Should this instead just use the client-side synthetic duration_sec value? + duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + + # Throughput is calculated as the mean number of successful requests per second + throughput=$(echo "$oks/$duration" | bc) + printf "%d,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' BEGIN { sum = 0 p50 = int('"$oks"' * 0.5) @@ -106,18 +158,37 @@ for payload in ${payloads[*]}; do NR==p100 {printf "%1.4f\n", $0} ' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv" - # Delete scratch file used for sorting/counting - rm -rf "$results_directory/$payload-response.csv" -done + # Delete scratch file used for sorting/counting + rm -rf "$results_directory/$payload-response.csv" + done + + # Transform csvs to dat files for gnuplot + for file in success latency throughput; do + printf "#" > "$results_directory/$file.dat" + tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" + done + + # Generate gnuplots + generate_gnuplots "$results_directory" "$__run_sh__base_path" || { + printf "[ERR]\n" + panic "failed to generate gnuplots" + } + + printf "[OK]\n" + return 0 +} + +# Expected Symbol used by the framework +experiment_main() { + local -r target_hostname="$1" + local -r results_directory="$2" -# Transform csvs to dat files for gnuplot -for file in success latency throughput; do - echo -n "#" > "$results_directory/$file.dat" - tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat" -done + run_samples "$target_hostname" || return 1 + run_experiments "$target_hostname" "$results_directory" || return 1 + process_results "$results_directory" || return 1 -# Generate gnuplots -generate_gnuplots + return 0 +} -# Cleanup, if requires -echo "[DONE]" +# Delegating to main provided by framework +main "$@" From 4955d3bab774ad0a1dfd5de2a8acf58c07ee0f2e Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Thu, 15 Apr 2021 20:27:51 +0000 Subject: [PATCH 12/13] fix: Correct hardcoded localhost and DRY up sample code --- runtime/experiments/payload/run.sh | 37 +++++++++++------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/runtime/experiments/payload/run.sh b/runtime/experiments/payload/run.sh index 3e050dc..6d758c7 100755 --- a/runtime/experiments/payload/run.sh +++ b/runtime/experiments/payload/run.sh @@ -45,28 +45,17 @@ run_samples() { local -ir perf_window_buffer_size # Execute workloads long enough for runtime to learn excepted execution time - printf "Running Samples: " - hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/1024.txt" "http://$hostname:10000" 1> /dev/null 2> /dev/null || { - printf "[ERR]\n" - panic "samples failed" - return 1 - } - hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/10240.txt" "http://$hostname:10001" 1> /dev/null 2> /dev/null || { - printf "[ERR]\n" - panic "samples failed" - return 1 - } - hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/102400.txt" "http://$hostname:10002" 1> /dev/null 2> /dev/null || { - printf "[ERR]\n" - panic "samples failed" - return 1 - } - hey -n "$iterations" --n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/1048576.txt" "http://$hostname:10003" 1> /dev/null 2> /dev/null || { - printf "[ERR]\n" - panic "samples failed" - return 1 - } - printf "[OK]\n" + printf "Running Samples:\n" + for payload in "${payloads[@]}"; do + printf "\t%d Payload: " "$payload" + hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" "http://$hostname:${ports["$payload"]}" 1> /dev/null 2> /dev/null || { + printf "[ERR]\n" + panic "samples failed" + return 1 + } + printf "[OK]\n" + done + return 0 } @@ -86,9 +75,9 @@ run_experiments() { printf "Running Experiments:\n" for payload in "${payloads[@]}"; do printf "\t%d Payload: " "$payload" - hey -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" http://localhost:"${ports["$payload"]}" > "$results_directory/$payload.csv" 2> /dev/null || { + hey -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" "http://$hostname:${ports["$payload"]}" > "$results_directory/$payload.csv" 2> /dev/null || { printf "[ERR]\n" - panic "experiment failed" + panic "$payload experiment failed" return 1 } get_result_count "$results_directory/$payload.csv" || { From 67af09ca4800d9375f66f39791f9c0e9e4552f5e Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Thu, 15 Apr 2021 20:28:07 +0000 Subject: [PATCH 13/13] chore: Delete useless script --- runtime/experiments/payload/test.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 runtime/experiments/payload/test.sh diff --git a/runtime/experiments/payload/test.sh b/runtime/experiments/payload/test.sh deleted file mode 100755 index 7115085..0000000 --- a/runtime/experiments/payload/test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -hey -n 100 -c 3 -q 100 -m GET -D "./body/1024.txt" http://localhost:10000