refactor: concurrency and payload use framework

sledge_graph
Sean McBride 4 years ago
parent d678e34ce3
commit 7e62eceb0e

@ -1,19 +0,0 @@
#!/bin/bash
# Executes the runtime in GDB
# Substitutes the absolute path from the container with a path relatively derived from the location of this script
# This allows debugging outside of the Docker container
# Also disables pagination and stopping on SIGUSR1
experiment_directory=$(pwd)
project_directory=$(cd ../.. && pwd)
binary_directory=$(cd "$project_directory"/bin && pwd)
export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
export PATH="$binary_directory:$PATH"
gdb --eval-command="handle SIGUSR1 nostop" \
--eval-command="handle SIGPIPE nostop" \
--eval-command="set pagination off" \
--eval-command="set substitute-path /sledge/runtime $project_directory" \
--eval-command="run $experiment_directory/spec.json" \
sledgert

@ -1,20 +0,0 @@
reset
set term jpeg
set output "latency.jpg"
set xlabel "Payload (bytes)"
set xrange [-5:1050000]
set ylabel "Latency (ms)"
set yrange [0:]
set key left top
set style histogram columnstacked
plot 'latency.dat' using 1:2 title 'p50', \
'latency.dat' using 1:3 title 'p90', \
'latency.dat' using 1:4 title 'p99', \
'latency.dat' using 1:5 title 'p100', \

@ -1,111 +0,0 @@
#!/bin/bash
source ../common.sh
# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
# Use -d flag if running under gdb
timestamp=$(date +%s)
experiment_directory=$(pwd)
binary_directory=$(cd ../../bin && pwd)
results_directory="$experiment_directory/res/$timestamp"
log=log.txt
mkdir -p "$results_directory"
log_environment >> "$results_directory/$log"
# Start the runtime
if [ "$1" != "-d" ]; then
PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" &
sleep 1
else
echo "Running under gdb"
echo "Running under gdb" >> "$results_directory/$log"
fi
payloads=(fivebyeight/5x8 handwriting/handwrt1 hyde/hyde)
ports=(10000 10001 10002)
iterations=1000
# Execute workloads long enough for runtime to learn excepted execution time
echo -n "Running Samples: "
for i in {0..2}; do
hey -n 200 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/${payloads[$i]}.pnm" "http://localhost:${ports[$i]}"
done
sleep 1
echo "[DONE]"
# Execute the experiments
echo "Running Experiments"
for i in {0..2}; do
printf "\t%s Payload: " "${payloads[$i]}"
file=$(echo "${payloads[$i]}" | awk -F/ '{print $2}').csv
hey -n "$iterations" -c 3 -cpus 2 -o csv -m GET -D "$experiment_directory/${payloads[$i]}.pnm" "http://localhost:${ports[$i]}" > "$results_directory/$file"
echo "[DONE]"
done
# Stop the runtime
if [ "$1" != "-d" ]; then
sleep 5
kill_runtime
fi
# Generate *.csv and *.dat results
echo -n "Parsing Results: "
printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv"
printf "Concurrency,Throughput\n" >> "$results_directory/throughput.csv"
printf "Con,p50,p90,p99,p100\n" >> "$results_directory/latency.csv"
for payload in ${payloads[*]}; do
# Calculate Success Rate for csv
file=$(echo "$payload" | awk -F/ '{print $2}')
awk -F, '
$7 == 200 {ok++}
END{printf "'"$file"',%3.5f\n", (ok / '"$iterations"' * 100)}
' < "$results_directory/$file.csv" >> "$results_directory/success.csv"
# Filter on 200s, convery from s to ms, and sort
awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$file.csv" \
| sort -g > "$results_directory/$file-response.csv"
# Get Number of 200s
oks=$(wc -l < "$results_directory/$file-response.csv")
((oks == 0)) && continue # If all errors, skip line
# Get Latest Timestamp
duration=$(tail -n1 "$results_directory/$file.csv" | cut -d, -f8)
throughput=$(echo "$oks/$duration" | bc)
printf "%s,%f\n" "$file" "$throughput" >> "$results_directory/throughput.csv"
# Generate Latency Data for csv
awk '
BEGIN {
sum = 0
p50 = int('"$oks"' * 0.5)
p90 = int('"$oks"' * 0.9)
p99 = int('"$oks"' * 0.99)
p100 = '"$oks"'
printf "'"$file"',"
}
NR==p50 {printf "%1.4f,", $0}
NR==p90 {printf "%1.4f,", $0}
NR==p99 {printf "%1.4f,", $0}
NR==p100 {printf "%1.4f\n", $0}
' < "$results_directory/$file-response.csv" >> "$results_directory/latency.csv"
# Delete scratch file used for sorting/counting
rm -rf "$results_directory/$file-response.csv"
done
# Transform csvs to dat files for gnuplot
for file in success latency throughput; do
echo -n "#" > "$results_directory/$file.dat"
tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat"
done
# Generate gnuplots
generate_gnuplots
# Cleanup, if requires
echo "[DONE]"

@ -1,42 +0,0 @@
{
"active": true,
"name": "gocr",
"path": "gocr.aso",
"port": 10000,
"relative-deadline-us": 50000000000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "text/plain",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "text/plain"
},
{
"active": true,
"name": "gocr",
"path": "gocr.aso",
"port": 10001,
"relative-deadline-us": 50000000000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "text/plain",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "text/plain"
},
{
"active": true,
"name": "gocr",
"path": "gocr.aso",
"port": 10002,
"relative-deadline-us": 50000000000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "text/plain",
"http-req-size": 5335057,
"http-resp-headers": [],
"http-resp-size": 5335057,
"http-resp-content-type": "text/plain"
}

@ -1,12 +0,0 @@
reset
set term jpeg
set output "success.jpg"
set xlabel "Connections"
set xrange [-5:105]
set ylabel "% 2XX"
set yrange [0:110]
plot 'success.dat' using 1:2 title '2XX'

@ -1,10 +0,0 @@
#!/bin/bash
cd handwriting || exit
./run.sh
cd .. || exit
cd hyde || exit
./run.sh
cd .. || exit
cd fivebyeight || exit
./run.sh
cd ..

@ -1,13 +0,0 @@
reset
set term jpeg
set output "throughput.jpg"
# TODO: Axis shouldn't be linear
set xlabel "Connections"
set xrange [-5:105]
set ylabel "Requests/sec"
set yrange [0:]
plot 'throughput.dat' using 1:2 title 'Reqs/sec'

@ -0,0 +1,29 @@
# shellcheck shell=bash
if [ -n "$__csv_to_dat_sh__" ]; then return; fi
__csv_to_dat_sh__=$(date)
source "panic.sh" || exit 1
# Takes a variadic number of paths to *.csv files and converts to *.dat files in the same directory
csv_to_dat() {
if (($# == 0)); then
panic "insufficient parameters"
return 1
fi
for arg in "$@"; do
if ! [[ "$arg" =~ ".csv"$ ]]; then
panic "$arg is not a *.csv file"
return 1
fi
if [[ ! -f "$arg" ]]; then
panic "$arg does not exit"
return 1
fi
done
for file in "$@"; do
echo -n "#" > "${file/.csv/.dat}"
tr ',' ' ' < "$file" | column -t >> "${file/.csv/.dat}"
done
}

@ -0,0 +1,7 @@
# shellcheck shell=bash
if [ -n "$__error_msg_sh__" ]; then return; fi
__error_msg_sh__=$(date)
error_msg() {
printf "%.23s %s() at %s:%s - %s\n" "$(date +%F.%T.%N)" "${FUNCNAME[0]}" "$(realpath "${BASH_SOURCE[0]##*/}")" "${BASH_LINENO[0]}" "${@}"
}

@ -0,0 +1,410 @@
# shellcheck shell=bash
if [ -n "$__framework_sh__" ]; then return; fi
__framework_sh__=$(date)
#
# This framework simplifies the scripting of experiments
#
# To use, import the framework source file and pass all arguments to the provided main function
# source "framework.sh"
#
# main "$@"
#
# In your script, implement the following functions above main:
# - experiment_main
#
source "path_join.sh" || exit 1
source "panic.sh" || exit 1
__framework_sh__usage() {
echo "$0 [options...]"
echo ""
echo "Options:"
echo " -t,--target=<target url> Execute as client against remote URL"
echo " -s,--serve=<EDF|FIFO> Serve with scheduling policy, but do not run client"
echo " -d,--debug=<EDF|FIFO> Debug under GDB with scheduling policy, but do not run client"
echo " -p,--perf=<EDF|FIFO> Run under perf with scheduling policy. Run on baremetal Linux host!"
}
# Declares application level global state
__framework_sh__initialize_globals() {
# timestamp is used to name the results directory for a particular test run
# shellcheck disable=SC2155
# shellcheck disable=SC2034
declare -gir __framework_sh__timestamp=$(date +%s)
# Globals used by parse_arguments
declare -g __framework_sh__target=""
declare -g __framework_sh__policy=""
declare -g __framework_sh__role=""
# Configure environment variables
# shellcheck disable=SC2155
declare -gr __framework_sh__application_directory="$(dirname "$(realpath "$0"))")"
local -r binary_directory="$(cd "$__framework_sh__application_directory" && cd ../../bin && pwd)"
export PATH=$binary_directory:$PATH
export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH
export SLEDGE_NWORKERS=5
}
# Parses arguments from the user and sets associates global state
__framework_sh__parse_arguments() {
for i in "$@"; do
case $i in
-t=* | --target=*)
if [[ "$__framework_sh__role" == "server" ]]; then
echo "Cannot set target when server"
__framework_sh__usage
return 1
fi
__framework_sh__role=client
__framework_sh__target="${i#*=}"
shift
;;
-s=* | --serve=*)
if [[ "$__framework_sh__role" == "client" ]]; then
echo "Cannot use -s,--serve with -t,--target"
__framework_sh__usage
return 1
fi
__framework_sh__role=server
__framework_sh__policy="${i#*=}"
if [[ ! $__framework_sh__policy =~ ^(EDF|FIFO)$ ]]; then
echo "\"$__framework_sh__policy\" is not a valid policy. EDF or FIFO allowed"
__framework_sh__usage
return 1
fi
shift
;;
-d=* | --debug=*)
if [[ "$__framework_sh__role" == "client" ]]; then
echo "Cannot use -d,--debug with -t,--target"
__framework_sh__usage
return 1
fi
__framework_sh__role=debug
__framework_sh__policy="${i#*=}"
if [[ ! $__framework_sh__policy =~ ^(EDF|FIFO)$ ]]; then
echo "\"$__framework_sh__policy\" is not a valid policy. EDF or FIFO allowed"
__framework_sh__usage
return 1
fi
shift
;;
-p=* | --perf=*)
if [[ "$__framework_sh__role" == "perf" ]]; then
echo "Cannot use -p,--perf with -t,--target"
__framework_sh__usage
return 1
fi
__framework_sh__role=perf
__framework_sh__policy="${i#*=}"
if [[ ! $__framework_sh__policy =~ ^(EDF|FIFO)$ ]]; then
echo "\"$__framework_sh__policy\" is not a valid policy. EDF or FIFO allowed"
__framework_sh__usage
return 1
fi
shift
;;
-h | --help)
__framework_sh__usage
exit 0
;;
*)
echo "$1 is a not a valid option"
__framework_sh__usage
return 1
;;
esac
done
# default to both if no arguments were passed
if [[ -z "$__framework_sh__role" ]]; then
__framework_sh__role="both"
__framework_sh__target="localhost"
fi
# Set globals as read only
declare -r __framework_sh__target
declare -r __framework_sh__policy
declare -r __framework_sh__role
}
# Log hardware and software info for the execution
__framework_sh__log_environment() {
if ! command -v git &> /dev/null; then
echo "git could not be found"
exit
fi
echo "*******"
echo "* Git *"
echo "*******"
git log | head -n 1 | cut -d' ' -f2
git status
echo ""
echo "************"
echo "* Makefile *"
echo "************"
cat "$(path_join "$__framework_sh__application_directory" ../../Makefile)"
echo ""
echo "**********"
echo "* Run.sh *"
echo "**********"
cat "$(path_join "$__framework_sh__application_directory" ./run.sh)"
echo ""
echo "************"
echo "* Hardware *"
echo "************"
lscpu
echo ""
echo "*************"
echo "* Execution *"
echo "*************"
}
# $1 - Scheduler Variant (EDF|FIFO)
# $2 - Results Directory
# $3 - How to run (foreground|background)
# $4 - JSON specification
__framework_sh__start_runtime() {
printf "Starting Runtime: "
if (($# != 4)); then
printf "[ERR]\n"
panic "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
printf "[ERR]\n"
panic "expected EDF or FIFO was \"$1\""
return 1
elif ! [[ -d "$2" ]]; then
printf "[ERR]\n"
panic "directory \"$2\" does not exist"
return 1
elif ! [[ $3 =~ ^(foreground|background)$ ]]; then
printf "[ERR]\n"
panic "expected foreground or background was \"$3\""
return 1
elif [[ ! -f "$4" || "$4" != *.json ]]; then
printf "[ERR]\n"
panic "\"$4\" does not exist or is not a JSON"
return 1
fi
local -r scheduler="$1"
local -r results_directory="$2"
local -r how_to_run="$3"
local -r specification="$4"
local -r log_name=log.txt
local log="$results_directory/${log_name}"
__framework_sh__log_environment >> "$log"
case "$how_to_run" in
"background")
SLEDGE_SCHEDULER="$scheduler" \
sledgert "$specification" >> "$log" 2>> "$log" &
;;
"foreground")
SLEDGE_SCHEDULER="$scheduler" \
sledgert "$specification"
;;
esac
printf "[OK]\n"
return 0
}
__framework_sh__run_server() {
if (($# != 2)); then
panic "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
panic "expected EDF or FIFO was \"$1\""
return 1
elif ! [[ $2 =~ ^(foreground|background)$ ]]; then
printf "[ERR]\n"
panic "expected foreground or background was \"$3\""
return 1
fi
local -r scheduler="$1"
local -r how_to_run="$2"
__framework_sh__start_runtime "$scheduler" "$RESULTS_DIRECTORY" "$how_to_run" "$__framework_sh__application_directory/spec.json" || {
echo "__framework_sh__start_runtime RC: $?"
panic "Error calling __framework_sh__start_runtime $scheduler $RESULTS_DIRECTORY $how_to_run $__framework_sh__application_directory/spec.json"
return 1
}
return 0
}
__framework_sh__run_perf() {
if (($# != 1)); then
printf "[ERR]\n"
panic "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
printf "[ERR]\n"
panic "expected EDF or FIFO was \"$1\""
return 1
fi
if ! command -v perf; then
echo "perf is not present."
exit 1
fi
local -r scheduler="$1"
SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$__framework_sh__application_directory/spec.json"
}
# Starts the Sledge Runtime under GDB
__framework_sh__run_debug() {
# shellcheck disable=SC2155
local project_directory=$(cd ../.. && pwd)
if (($# != 1)); then
printf "[ERR]\n"
panic "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
printf "[ERR]\n"
panic "expected EDF or FIFO was \"$1\""
return 1
fi
local -r scheduler="$1"
if [[ "$project_directory" != "/sledge/runtime" ]]; then
printf "It appears that you are not running in the container. Substituting path to match host environment\n"
SLEDGE_SCHEDULER="$scheduler" gdb \
--eval-command="handle SIGUSR1 nostop" \
--eval-command="handle SIGPIPE nostop" \
--eval-command="set pagination off" \
--eval-command="set substitute-path /sledge/runtime $project_directory" \
--eval-command="run $__framework_sh__application_directory/spec.json" \
sledgert
else
SLEDGE_SCHEDULER="$scheduler" gdb \
--eval-command="handle SIGUSR1 nostop" \
--eval-command="handle SIGPIPE nostop" \
--eval-command="set pagination off" \
--eval-command="run $__framework_sh__application_directory/spec.json" \
sledgert
fi
return 0
}
__framework_sh__run_client() {
experiment_main "$__framework_sh__target" "$RESULTS_DIRECTORY" || {
panic "Error calling process_results $RESULTS_DIRECTORY"
return 1
}
return 0
}
__framework_sh__run_both() {
local -ar schedulers=(EDF FIFO)
for scheduler in "${schedulers[@]}"; do
printf "Running %s\n" "$scheduler"
__framework_sh__run_server "$scheduler" background || {
panic "Error calling __framework_sh__run_server"
return 1
}
__framework_sh__run_client || {
panic "Error calling __framework_sh__run_client"
__framework_sh__stop_runtime
return 1
}
__framework_sh__stop_runtime || {
panic "Error calling __framework_sh__stop_runtime"
return 1
}
done
return 0
}
__framework_sh__create_and_export_results_directory() {
local dir=""
# If we are running both client, and server, we need to namespace by scheduler since we run both variants
case "$__framework_sh__role" in
"both")
dir="$__framework_sh__application_directory/res/$__framework_sh__timestamp/$scheduler"
;;
"client" | "server" | "debug" | "perf")
dir="$__framework_sh__application_directory/res/$__framework_sh__timestamp"
;;
*)
panic "${FUNCNAME[0]} Unexpected $__framework_sh__role"
return 1
;;
esac
mkdir -p "$dir" || {
panic "mkdir -p $dir"
return 1
}
export RESULTS_DIRECTORY="$dir"
}
# Responsible for ensuring that the experiment file meets framework assumptions
__framework_sh__validate_experiment() {
if [[ $(type -t experiment_main) != "function" ]]; then
panic "function experiment_main was not defined"
return 1
fi
}
main() {
__framework_sh__validate_experiment || exit 1
__framework_sh__initialize_globals || exit 1
__framework_sh__parse_arguments "$@" || exit 1
__framework_sh__create_and_export_results_directory || exit 1
case $__framework_sh__role in
both)
__framework_sh__run_both
;;
server)
__framework_sh__run_server "$__framework_sh__policy" foreground
;;
debug)
__framework_sh__run_debug "$__framework_sh__policy"
;;
perf)
__framework_sh__run_perf "$__framework_sh__policy"
;;
client)
__framework_sh__run_client
;;
*)
echo "Invalid state"
false
;;
esac
exit "$?"
}
__framework_sh__stop_runtime() {
printf "Stopping Runtime: "
pkill sledgert > /dev/null 2> /dev/null
pkill hey > /dev/null 2> /dev/null
printf "[OK]\n"
}

@ -0,0 +1,36 @@
# shellcheck shell=bash
if [ -n "$__generate_gnuplots_sh__" ]; then return; fi
__generate_gnuplots_sh__=$(date)
source "panic.sh" || exit 1
# Runs all *.gnuplot files found gnuplot_directory from results_directory
# Outputting resulting diagrams in results_directory
# $1 - results_directory containing the data file referenced in the gnuplot file
# $2 - gnuplot_directory containing the *.gnuplot specification files
generate_gnuplots() {
local -r results_directory="$1"
local -r experiment_directory="$2"
if ! command -v gnuplot &> /dev/null; then
panic "gnuplot could not be found in path"
return 1
fi
# shellcheck disable=SC2154
if [ -z "$results_directory" ]; then
panic "results_directory was unset or empty"
return 1
fi
# shellcheck disable=SC2154
if [ -z "$experiment_directory" ]; then
panic "error: EXPERIMENT_DIRECTORY was unset or empty"
return 1
fi
cd "$results_directory" || exit
shopt -s nullglob
for gnuplot_file in "$experiment_directory"/*.gnuplot; do
gnuplot "$gnuplot_file"
done
cd "$experiment_directory" || exit
}

@ -0,0 +1,38 @@
# shellcheck shell=bash
if [ -n "$__get_result_count_sh__" ]; then return; fi
__get_result_count_sh__=$(date)
source "panic.sh" || exit 1
# Given a file, returns the number of results
# This assumes a *.csv file with a header
# $1 the file we want to check for results
# $2 an optional return nameref
get_result_count() {
if (($# != 1)); then
panic "insufficient parameters. $#/1"
return 1
elif [[ ! -f $1 ]]; then
panic "the file $1 does not exist"
return 1
elif [[ ! -s $1 ]]; then
panic "the file $1 is size 0"
return 1
fi
local -r file=$1
# Subtract one line for the header
local -i count=$(($(wc -l < "$file") - 1))
if (($# == 2)); then
# shellcheck disable=2034
local -n __result=$2
fi
if ((count > 0)); then
return 0
else
return 1
fi
}

@ -0,0 +1,24 @@
# shellcheck shell=bash
if [ -n "$__panic_sh__" ]; then return; fi
__panic_sh__=$(date)
source "error_msg.sh" || exit 1
declare __common_did_dump_callstack=false
__common_dump_callstack() {
echo "Call Stack:"
# Skip the dump_bash_stack and error_msg_frames
for ((i = 2; i < ${#FUNCNAME[@]}; i++)); do
printf "\t%d - %s\n" "$((i - 2))" "${FUNCNAME[i]} (${BASH_SOURCE[i + 1]}:${BASH_LINENO[i]})"
done
}
# Public API
panic() {
error_msg "${@}"
[[ "$__common_did_dump_callstack" == false ]] && {
__common_dump_callstack
__common_did_dump_callstack=true
}
}

@ -0,0 +1,14 @@
# shellcheck shell=bash
if [ -n "$__path_join_sh__" ]; then return; fi
__path_join_sh__=$(date)
path_join() {
local base=$1
local relative=$2
relative_path="$(dirname "$relative")"
file_name="$(basename "$relative")"
absolute_path="$(cd "$base" && cd "$relative_path" && pwd)"
echo "$absolute_path/$file_name"
}

@ -1,5 +1,4 @@
#!/bin/bash
source ../common.sh
# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
# Success - The percentage of requests that complete by their deadlines
@ -7,50 +6,77 @@ source ../common.sh
# Throughput - The mean number of successful requests per second
# Latency - the rount-trip resonse time (unit?) of successful requests at the p50, p90, p99, and p100 percetiles
# Add bash_libraries directory to path
__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
__run_sh__bash_libraries_relative_path="../bash_libraries"
__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
source csv_to_dat.sh || exit 1
source framework.sh || exit 1
# source generate_gnuplots.sh || exit 1
source get_result_count.sh || exit 1
source panic.sh || exit 1
source path_join.sh || exit 1
# Sends requests until the per-module perf window buffers are full
# This ensures that Sledge has accurate estimates of execution time
run_samples() {
local hostname="${1:-localhost}"
if (($# != 1)); then
panic "invalid number of arguments \"$1\""
return 1
elif [[ -z "$1" ]]; then
panic "hostname \"$1\" was empty"
return 1
fi
local hostname="${1}"
# Scrape the perf window size from the source if possible
local -r perf_window_path="../../include/perf_window.h"
# TODO: Make a util function
local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window.h)"
local -i perf_window_buffer_size
if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then
echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h"
echo "Defaulting to 16"
printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n"
printf "Defaulting to 16\n"
perf_window_buffer_size=16
fi
local -ir perf_window_buffer_size
echo -n "Running Samples: "
printf "Running Samples: "
hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "40\n" "http://${hostname}:10040" 1> /dev/null 2> /dev/null || {
error_msg "fib40 samples failed"
printf "[ERR]\n"
panic "fib40 samples failed"
return 1
}
hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -cpus 3 -t 0 -o csv -m GET -d "10\n" "http://${hostname}:100010" 1> /dev/null 2> /dev/null || {
error_msg "fib10 samples failed"
printf "[ERR]\n"
panic "fib10 samples failed"
return 1
}
echo "[OK]"
printf "[OK]\n"
return 0
}
# Execute the fib10 and fib40 experiments sequentially and concurrently
# $1 (results_directory) - a directory where we will store our results
# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost
# $1 (hostname)
# $2 (results_directory) - a directory where we will store our results
run_experiments() {
if (($# < 1 || $# > 2)); then
error_msg "invalid number of arguments \"$1\""
if (($# != 2)); then
panic "invalid number of arguments \"$1\""
return 1
elif ! [[ -d "$1" ]]; then
error_msg "directory \"$1\" does not exist"
elif [[ -z "$1" ]]; then
panic "hostname \"$1\" was empty"
return 1
elif [[ ! -d "$2" ]]; then
panic "directory \"$2\" does not exist"
return 1
fi
local results_directory="$1"
local hostname="${2:-localhost}"
local hostname="$1"
local results_directory="$2"
# The duration in seconds that we want the client to send requests
local -ir duration_sec=15
@ -64,12 +90,12 @@ run_experiments() {
printf "\tfib40: "
hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$hostname:10040" > "$results_directory/fib40.csv" 2> /dev/null || {
printf "[ERR]\n"
error_msg "fib40 failed"
panic "fib40 failed"
return 1
}
get_result_count "$results_directory/fib40.csv" || {
printf "[ERR]\n"
error_msg "fib40 unexpectedly has zero requests"
panic "fib40 unexpectedly has zero requests"
return 1
}
printf "[OK]\n"
@ -77,12 +103,12 @@ run_experiments() {
printf "\tfib10: "
hey -z ${duration_sec}s -cpus 4 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$hostname:10010" > "$results_directory/fib10.csv" 2> /dev/null || {
printf "[ERR]\n"
error_msg "fib10 failed"
panic "fib10 failed"
return 1
}
get_result_count "$results_directory/fib10.csv" || {
printf "[ERR]\n"
error_msg "fib10 unexpectedly has zero requests"
panic "fib10 unexpectedly has zero requests"
return 1
}
printf "[OK]\n"
@ -103,24 +129,24 @@ run_experiments() {
wait -f "$fib10_con_PID" || {
printf "\tfib10_con: [ERR]\n"
error_msg "failed to wait -f ${fib10_con_PID}"
panic "failed to wait -f ${fib10_con_PID}"
return 1
}
get_result_count "$results_directory/fib10_con.csv" || {
printf "\tfib10_con: [ERR]\n"
error_msg "fib10_con has zero requests. This might be because fib40_con saturated the runtime"
panic "fib10_con has zero requests. This might be because fib40_con saturated the runtime"
return 1
}
printf "\tfib10_con: [OK]\n"
wait -f "$fib40_con_PID" || {
printf "\tfib40_con: [ERR]\n"
error_msg "failed to wait -f ${fib40_con_PID}"
panic "failed to wait -f ${fib40_con_PID}"
return 1
}
get_result_count "$results_directory/fib40_con.csv" || {
printf "\tfib40_con: [ERR]\n"
error_msg "fib40_con has zero requests."
panic "fib40_con has zero requests."
return 1
}
printf "\tfib40_con: [OK]\n"
@ -140,7 +166,7 @@ process_results() {
local -r results_directory="$1"
echo -n "Processing Results: "
printf "Processing Results: "
# Write headers to CSVs
printf "Payload,Success_Rate\n" >> "$results_directory/success.csv"
@ -163,14 +189,7 @@ process_results() {
# Strip the _con suffix when getting the deadline
local -i deadline=${deadlines_ms[${payload/_con/}]}
# Get Number of Requests, subtracting the header
local -i requests=$(($(wc -l < "$results_directory/$payload.csv") - 1))
((requests == 0)) && {
echo "$payload unexpectedly has zero requests"
continue
}
# Calculate Success Rate for csv
# Calculate Success Rate for csv (percent of requests that return 200 within deadline)
awk -F, '
$7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)}
@ -216,7 +235,25 @@ process_results() {
csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv"
# Generate gnuplots. Commented out because we don't have *.gnuplots defined
# generate_gnuplots
# generate_gnuplots "$results_directory" "$__run_sh__base_path" || {
# printf "[ERR]\n"
# panic "failed to generate gnuplots"
# }
printf "[OK]\n"
return 0
}
# Expected Symbol used by the framework
experiment_main() {
local -r target_hostname="$1"
local -r results_directory="$2"
run_samples "$target_hostname" || return 1
run_experiments "$target_hostname" "$results_directory" || return 1
process_results "$results_directory" || return 1
return 0
}
main "$@"

@ -1,487 +0,0 @@
#!/bin/bash
declare __common_did_dump_callstack=false
error_msg() {
[[ "$__common_did_dump_callstack" == false ]] && {
printf "%.23s %s() in %s, line %s: %s\n" "$(date +%F.%T.%N)" "${FUNCNAME[1]}" "${BASH_SOURCE[1]##*/}" "${BASH_LINENO[0]}" "${@}"
__common_dump_callstack
__common_did_dump_callstack=true
}
}
__common_dump_callstack() {
echo "Call Stack:"
# Skip the dump_bash_stack and error_msg_frames
for ((i = 2; i < ${#FUNCNAME[@]}; i++)); do
printf "\t%d - %s\n" "$((i - 2))" "${FUNCNAME[i]}"
done
}
log_environment() {
if ! command -v git &> /dev/null; then
echo "git could not be found"
exit
fi
echo "*******"
echo "* Git *"
echo "*******"
git log | head -n 1 | cut -d' ' -f2
git status
echo ""
echo "************"
echo "* Makefile *"
echo "************"
cat ../../Makefile
echo ""
echo "**********"
echo "* Run.sh *"
echo "**********"
cat run.sh
echo ""
echo "************"
echo "* Hardware *"
echo "************"
lscpu
echo ""
echo "*************"
echo "* Execution *"
echo "*************"
}
# Given a file, returns the number of results
# This assumes a *.csv file with a header
# $1 the file we want to check for results
# $2 an optional return nameref
get_result_count() {
if (($# != 1)); then
error_msg "insufficient parameters. $#/1"
return 1
elif [[ ! -f $1 ]]; then
error_msg "the file $1 does not exist"
return 1
elif [[ ! -s $1 ]]; then
error_msg "the file $1 is size 0"
return 1
fi
local -r file=$1
# Subtract one line for the header
local -i count=$(($(wc -l < "$file") - 1))
if (($# == 2)); then
# shellcheck disable=2034
local -n __result=$2
fi
if ((count > 0)); then
return 0
else
return 1
fi
}
usage() {
echo "$0 [options...]"
echo ""
echo "Options:"
echo " -t,--target=<target url> Execute as client against remote URL"
echo " -s,--serve=<EDF|FIFO> Serve with scheduling policy, but do not run client"
echo " -d,--debug=<EDF|FIFO> Debug under GDB with scheduling policy, but do not run client"
echo " -p,--perf=<EDF|FIFO> Run under perf with scheduling policy. Run on baremetal Linux host!"
}
# Parses arguments from the user and sets associates global state
parse_arguments() {
for i in "$@"; do
case $i in
-t=* | --target=*)
if [[ "$role" == "server" ]]; then
echo "Cannot set target when server"
usage
return 1
fi
role=client
target="${i#*=}"
shift
;;
-s=* | --serve=*)
if [[ "$role" == "client" ]]; then
echo "Cannot use -s,--serve with -t,--target"
usage
return 1
fi
role=server
policy="${i#*=}"
if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then
echo "\"$policy\" is not a valid policy. EDF or FIFO allowed"
usage
return 1
fi
shift
;;
-d=* | --debug=*)
if [[ "$role" == "client" ]]; then
echo "Cannot use -d,--debug with -t,--target"
usage
return 1
fi
role=debug
policy="${i#*=}"
if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then
echo "\"$policy\" is not a valid policy. EDF or FIFO allowed"
usage
return 1
fi
shift
;;
-p=* | --perf=*)
if [[ "$role" == "perf" ]]; then
echo "Cannot use -p,--perf with -t,--target"
usage
return 1
fi
role=perf
policy="${i#*=}"
if [[ ! $policy =~ ^(EDF|FIFO)$ ]]; then
echo "\"$policy\" is not a valid policy. EDF or FIFO allowed"
usage
return 1
fi
shift
;;
-h | --help)
usage
exit 0
;;
*)
echo "$1 is a not a valid option"
usage
return 1
;;
esac
done
# default to both if no arguments were passed
if [[ -z "$role" ]]; then
role="both"
fi
# Set globals as read only
declare -r target
declare -r policy
declare -r role
}
# Declares application level global state
initialize_globals() {
# timestamp is used to name the results directory for a particular test run
# shellcheck disable=SC2155
# shellcheck disable=SC2034
declare -gir timestamp=$(date +%s)
# shellcheck disable=SC2155
declare -gr experiment_directory=$(pwd)
# shellcheck disable=SC2155
declare -gr binary_directory=$(cd ../../bin && pwd)
# Globals used by parse_arguments
declare -g target=""
declare -g policy=""
declare -g role=""
# Configure environment variables
export PATH=$binary_directory:$PATH
export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH
export SLEDGE_NWORKERS=5
}
# $1 - Scheduler Variant (EDF|FIFO)
# $2 - Results Directory
# $3 - How to run (foreground|background)
# $4 - JSON specification
start_runtime() {
printf "Starting Runtime: "
if (($# != 4)); then
printf "[ERR]\n"
error_msg "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
printf "[ERR]\n"
error_msg "expected EDF or FIFO was \"$1\""
return 1
elif ! [[ -d "$2" ]]; then
printf "[ERR]\n"
error_msg "directory \"$2\" does not exist"
return 1
elif ! [[ $3 =~ ^(foreground|background)$ ]]; then
printf "[ERR]\n"
error_msg "expected foreground or background was \"$3\""
return 1
elif [[ ! -f "$4" || "$4" != *.json ]]; then
printf "[ERR]\n"
error_msg "\"$4\" does not exist or is not a JSON"
return 1
fi
local -r scheduler="$1"
local -r results_directory="$2"
local -r how_to_run="$3"
local -r specification="$4"
local -r log_name=log.txt
local log="$results_directory/${log_name}"
log_environment >> "$log"
case "$how_to_run" in
"background")
SLEDGE_SCHEDULER="$scheduler" \
sledgert "$specification" >> "$log" 2>> "$log" &
;;
"foreground")
SLEDGE_SCHEDULER="$scheduler" \
sledgert "$specification"
;;
esac
printf "[OK]\n"
return 0
}
run_server() {
if (($# != 1)); then
error_msg "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
error_msg "expected EDF or FIFO was \"$1\""
return 1
fi
local -r scheduler="$1"
if [[ "$role" == "both" ]]; then
local -r results_directory="$experiment_directory/res/$timestamp/$scheduler"
local -r how_to_run="background"
elif [[ "$role" == "server" ]]; then
local -r results_directory="$experiment_directory/res/$timestamp"
local -r how_to_run="foreground"
else
error_msg "Unexpected $role"
return 1
fi
mkdir -p "$results_directory"
start_runtime "$scheduler" "$results_directory" "$how_to_run" "$experiment_directory/spec.json" || {
echo "start_runtime RC: $?"
error_msg "Error calling start_runtime $scheduler $results_directory"
return 1
}
return 0
}
run_perf() {
if (($# != 1)); then
printf "[ERR]\n"
error_msg "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
printf "[ERR]\n"
error_msg "expected EDF or FIFO was \"$1\""
return 1
fi
if ! command -v perf; then
echo "perf is not present."
exit 1
fi
local -r scheduler="$1"
SLEDGE_SCHEDULER="$scheduler" perf record -g -s sledgert "$experiment_directory/spec.json"
}
# Starts the Sledge Runtime under GDB
run_debug() {
# shellcheck disable=SC2155
local project_directory=$(cd ../.. && pwd)
if (($# != 1)); then
printf "[ERR]\n"
error_msg "invalid number of arguments \"$1\""
return 1
elif ! [[ $1 =~ ^(EDF|FIFO)$ ]]; then
printf "[ERR]\n"
error_msg "expected EDF or FIFO was \"$1\""
return 1
fi
local -r scheduler="$1"
if [[ "$project_directory" != "/sledge/runtime" ]]; then
printf "It appears that you are not running in the container. Substituting path to match host environment\n"
SLEDGE_SCHEDULER="$scheduler" gdb \
--eval-command="handle SIGUSR1 nostop" \
--eval-command="handle SIGPIPE nostop" \
--eval-command="set pagination off" \
--eval-command="set substitute-path /sledge/runtime $project_directory" \
--eval-command="run $experiment_directory/spec.json" \
sledgert
else
SLEDGE_SCHEDULER="$scheduler" gdb \
--eval-command="handle SIGUSR1 nostop" \
--eval-command="handle SIGPIPE nostop" \
--eval-command="set pagination off" \
--eval-command="run $experiment_directory/spec.json" \
sledgert
fi
return 0
}
run_client() {
if [[ "$role" == "both" ]]; then
local results_directory="$experiment_directory/res/$timestamp/$scheduler"
elif [[ "$role" == "client" ]]; then
local results_directory="$experiment_directory/res/$timestamp"
else
error_msg "${FUNCNAME[0]} Unexpected $role"
return 1
fi
mkdir -p "$results_directory"
run_samples "$target" || {
error_msg "Error calling run_samples $target"
return 1
}
run_experiments "$results_directory" || {
error_msg "Error calling run_experiments $results_directory"
return 1
}
process_results "$results_directory" || {
error_msg "Error calling process_results $results_directory"
return 1
}
return 0
}
run_both() {
local -ar schedulers=(EDF FIFO)
for scheduler in "${schedulers[@]}"; do
printf "Running %s\n" "$scheduler"
run_server "$scheduler" || {
error_msg "Error calling run_server"
return 1
}
run_client || {
error_msg "Error calling run_client"
kill_runtime
return 1
}
kill_runtime || {
error_msg "Error calling kill_runtime"
return 1
}
done
return 0
}
main() {
initialize_globals
parse_arguments "$@" || {
exit 1
}
case $role in
both)
run_both
;;
server)
run_server "$policy"
;;
debug)
run_debug "$policy"
;;
perf)
run_perf "$policy"
;;
client)
run_client
;;
*)
echo "Invalid state"
false
;;
esac
exit "$?"
}
kill_runtime() {
printf "Stopping Runtime: "
pkill sledgert > /dev/null 2> /dev/null
pkill hey > /dev/null 2> /dev/null
printf "[OK]\n"
}
# Takes a variadic number of *.gnuplot filenames and generates the resulting images
# Assumes that the gnuplot definitions are in the experiment directory
generate_gnuplots() {
if ! command -v gnuplot &> /dev/null; then
echo "${FUNCNAME[0]} error: gnuplot could not be found in path"
exit
fi
# shellcheck disable=SC2154
if [ -z "$results_directory" ]; then
echo "${FUNCNAME[0]} error: results_directory was unset or empty"
dump_bash_stack
exit 1
fi
# shellcheck disable=SC2154
if [ -z "$experiment_directory" ]; then
echo "${FUNCNAME[0]} error: experiment_directory was unset or empty"
dump_bash_stack
exit 1
fi
cd "$results_directory" || exit
for gnuplot_file in "${@}"; do
gnuplot "$experiment_directory/$gnuplot_file.gnuplot"
done
cd "$experiment_directory" || exit
}
# Takes a variadic number of paths to *.csv files and converts to *.dat files in the same directory
csv_to_dat() {
if (($# == 0)); then
echo "${FUNCNAME[0]} error: insufficient parameters"
dump_bash_stack
fi
for arg in "$@"; do
if ! [[ "$arg" =~ ".csv"$ ]]; then
echo "${FUNCNAME[0]} error: $arg is not a *.csv file"
dump_bash_stack
exit 1
fi
done
for file in "$@"; do
echo -n "#" > "${file/.csv/.dat}"
tr ',' ' ' < "$file" | column -t >> "${file/.csv/.dat}"
done
}

@ -1,20 +1,41 @@
#!/bin/bash
source ../common.sh
# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
# Add bash_libraries directory to path
__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
__run_sh__bash_libraries_relative_path="../bash_libraries"
__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
source csv_to_dat.sh || exit 1
source framework.sh || exit 1
source generate_gnuplots.sh || exit 1
source get_result_count.sh || exit 1
source panic.sh || exit 1
source path_join.sh || exit 1
declare -gi iterations=10000
declare -ga concurrency=(1 20 40 60 80 100)
run_samples() {
local hostname="${1:-localhost}"
if (($# != 1)); then
panic "invalid number of arguments \"$1\""
return 1
elif [[ -z "$1" ]]; then
panic "hostname \"$1\" was empty"
return 1
fi
local hostname="$1"
# Scrape the perf window size from the source if possible
local -r perf_window_path="../../include/perf_window.h"
# TODO: Make a util function
local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window.h)"
local -i perf_window_buffer_size
if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then
echo "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h"
echo "Defaulting to 16"
printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n"
printf "Defaulting to 16\n"
perf_window_buffer_size=16
fi
local -ir perf_window_buffer_size
@ -22,50 +43,64 @@ run_samples() {
printf "Running Samples: "
hey -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -cpus 3 -o csv -m GET "http://${hostname}:10000" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
error_msg "samples failed"
panic "samples failed"
return 1
}
echo "[OK]"
printf "[OK]\n"
return 0
}
# Execute the experiments
# $1 (results_directory) - a directory where we will store our results
# $2 (hostname="localhost") - an optional parameter that sets the hostname. Defaults to localhost
# $1 (hostname)
# $2 (results_directory) - a directory where we will store our results
run_experiments() {
if (($# < 1 || $# > 2)); then
error_msg "invalid number of arguments \"$1\""
if (($# != 2)); then
panic "invalid number of arguments \"$1\""
return 1
elif ! [[ -d "$1" ]]; then
error_msg "directory \"$1\" does not exist"
elif [[ -z "$1" ]]; then
panic "hostname \"$1\" was empty"
return 1
elif [[ ! -d "$2" ]]; then
panic "directory \"$2\" does not exist"
return 1
fi
local results_directory="$1"
local hostname="${2:-localhost}"
local hostname="$1"
local results_directory="$2"
# Execute the experiments
echo "Running Experiments"
printf "Running Experiments:\n"
for conn in ${concurrency[*]}; do
printf "\t%d Concurrency: " "$conn"
hey -n "$iterations" -c "$conn" -cpus 2 -o csv -m GET "http://$hostname:10000" > "$results_directory/con$conn.csv" 2> /dev/null
echo "[OK]"
hey -n "$iterations" -c "$conn" -cpus 2 -o csv -m GET "http://$hostname:10000" > "$results_directory/con$conn.csv" 2> /dev/null || {
printf "[ERR]\n"
panic "experiment failed"
return 1
}
get_result_count "$results_directory/con$conn.csv" || {
printf "[ERR]\n"
panic "con$conn.csv unexpectedly has zero requests"
return 1
}
printf "[OK]\n"
done
return 0
}
process_results() {
if (($# != 1)); then
error_msg "invalid number of arguments ($#, expected 1)"
panic "invalid number of arguments ($#, expected 1)"
return 1
elif ! [[ -d "$1" ]]; then
error_msg "directory $1 does not exist"
panic "directory $1 does not exist"
return 1
fi
local -r results_directory="$1"
echo -n "Processing Results: "
printf "Processing Results: "
# Write headers to CSVs
printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv"
@ -73,6 +108,13 @@ process_results() {
printf "Con,p50,p90,p99,p100\n" >> "$results_directory/latency.csv"
for conn in ${concurrency[*]}; do
if [[ ! -f "$results_directory/con$conn.csv" ]]; then
printf "[ERR]\n"
panic "Missing $results_directory/con$conn.csv"
return 1
fi
# Calculate Success Rate for csv (percent of requests resulting in 200)
awk -F, '
$7 == 200 {ok++}
@ -116,17 +158,28 @@ process_results() {
done
# Transform csvs to dat files for gnuplot
for file in success latency throughput; do
echo -n "#" > "$results_directory/$file.dat"
tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat"
done
csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv"
# Generate gnuplots
generate_gnuplots latency success throughput
generate_gnuplots "$results_directory" "$__run_sh__base_path" || {
printf "[ERR]\n"
panic "failed to generate gnuplots"
}
# Cleanup, if requires
echo "[OK]"
printf "[OK]\n"
return 0
}
# Expected Symbol used by the framework
experiment_main() {
local -r target_hostname="$1"
local -r results_directory="$2"
run_samples "$target_hostname" || return 1
run_experiments "$target_hostname" "$results_directory" || return 1
process_results "$results_directory" || return 1
return 0
}
main "$@"

@ -1,12 +1,18 @@
#!/bin/bash
# Generates payloads of 1KB, 10KB, 100KB, 1MB
for size in 1024 $((1024 * 10)) $((1024 * 100)) $((1024 * 1024)); do
rm -rf $size.txt
i=0
echo -n "Generating $size:"
while ((i < size)); do
printf 'a' >> $size.txt
((i++))
done
echo "[DONE]"
# If the file exists, but is not the right size, wipe it
if [[ -f "$size.txt" ]] && (("$(wc -c "$size.txt" | cut -d\ -f1)" != size)); then
rm -rf "$size.txt"
fi
# Regenerate the file if missing
if [[ ! -f "$size.txt" ]]; then
echo -n "Generating $size: "
for ((i = 0; i < size; i++)); do
printf 'a' >> $size.txt
done
echo "[OK]"
fi
done

@ -1,19 +0,0 @@
#!/bin/bash
# Executes the runtime in GDB
# Substitutes the absolute path from the container with a path relatively derived from the location of this script
# This allows debugging outside of the Docker container
# Also disables pagination and stopping on SIGUSR1
experiment_directory=$(pwd)
project_directory=$(cd ../.. && pwd)
binary_directory=$(cd "$project_directory"/bin && pwd)
export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
export PATH="$binary_directory:$PATH"
gdb --eval-command="handle SIGUSR1 nostop" \
--eval-command="handle SIGPIPE nostop" \
--eval-command="set pagination off" \
--eval-command="set substitute-path /sledge/runtime $project_directory" \
--eval-command="run $experiment_directory/spec.json" \
sledgert

@ -1,97 +1,149 @@
#!/bin/bash
source ../common.sh
# This experiment is intended to document how the level of concurrent requests influences
# - latency
# - throughput
# - success/failure rate
# Add bash_libraries directory to path
__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
__run_sh__bash_libraries_relative_path="../bash_libraries"
__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
# Source libraries from bash_libraries directory
source "path_join.sh" || exit 1
source "framework.sh" || exit 1
source "get_result_count.sh" || exit 1
source "generate_gnuplots.sh" || exit 1
# Experiment Globals and Setups
declare -ar payloads=(1024 10240 102400 1048576)
declare -Ar ports=(
[1024]=10000
[10240]=10001
[102400]=10002
[1048576]=10003
)
declare -ri iterations=10000
# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
# Use -d flag if running under gdb
# If the one of the expected body files doesn't exist, trigger the generation script.
cd "$__run_sh__base_path/body" && ./generate.sh && cd "$OLDPWD" || exit
run_samples() {
local hostname="$1"
# Scrape the perf window size from the source if possible
local -r perf_window_path="$(path_join "$__run_sh__base_path" ../../include/perf_window.h)"
local -i perf_window_buffer_size
if ! perf_window_buffer_size=$(grep "#define PERF_WINDOW_BUFFER_SIZE" < "$perf_window_path" | cut -d\ -f3); then
printf "Failed to scrape PERF_WINDOW_BUFFER_SIZE from ../../include/perf_window.h\n"
printf "Defaulting to 16\n"
perf_window_buffer_size=16
fi
local -ir perf_window_buffer_size
# Execute workloads long enough for runtime to learn excepted execution time
printf "Running Samples: "
hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/1024.txt" "http://$hostname:10000" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "samples failed"
return 1
}
hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/10240.txt" "http://$hostname:10001" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "samples failed"
return 1
}
hey -n "$iterations" -n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/102400.txt" "http://$hostname:10002" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "samples failed"
return 1
}
hey -n "$iterations" --n "$perf_window_buffer_size" -c "$perf_window_buffer_size" -q 200 -o csv -m GET -D "$__run_sh__base_path/body/1048576.txt" "http://$hostname:10003" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "samples failed"
return 1
}
printf "[OK]\n"
return 0
}
run_experiments() {
if (($# != 2)); then
panic "invalid number of arguments \"$1\""
return 1
elif [[ ! -d "$2" ]]; then
panic "directory \"$2\" does not exist"
return 1
fi
timestamp=$(date +%s)
experiment_directory=$(pwd)
binary_directory=$(cd ../../bin && pwd)
results_directory="$experiment_directory/res/$timestamp"
log=log.txt
local hostname="$1"
local results_directory="$2"
# Execute the experiments
printf "Running Experiments:\n"
for payload in "${payloads[@]}"; do
printf "\t%d Payload: " "$payload"
hey -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$__run_sh__base_path/body/$payload.txt" http://localhost:"${ports["$payload"]}" > "$results_directory/$payload.csv" 2> /dev/null || {
printf "[ERR]\n"
panic "experiment failed"
return 1
}
get_result_count "$results_directory/$payload.csv" || {
printf "[ERR]\n"
panic "$payload.csv unexpectedly has zero requests"
return 1
}
printf "[OK]\n"
done
return 0
}
process_results() {
if (($# != 1)); then
panic "invalid number of arguments ($#, expected 1)"
return 1
elif ! [[ -d "$1" ]]; then
panic "directory $1 does not exist"
return 1
fi
mkdir -p "$results_directory"
local -r results_directory="$1"
log_environment >> "$results_directory/$log"
printf "Processing Results: "
# Start the runtime
if [ "$1" != "-d" ]; then
PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >> "$results_directory/$log" 2>> "$results_directory/$log" &
sleep 1
else
echo "Running under gdb"
echo "Running under gdb" >> "$results_directory/$log"
fi
printf "Payload,Success_Rate\n" >> "$results_directory/success.csv"
printf "Payload,Throughput\n" >> "$results_directory/throughput.csv"
printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv"
payloads=(1024 10240 102400 1048576)
ports=(10000 10001 10002 10003)
iterations=10000
for payload in ${payloads[*]}; do
# Calculate Success Rate for csv
awk -F, '
$7 == 200 {ok++}
END{printf "'"$payload"',%3.5f\n", (ok / '"$iterations"' * 100)}
' < "$results_directory/$payload.csv" >> "$results_directory/success.csv"
# If the one of the expected body files doesn't exist, trigger the generation script.
for payload in ${payloads[*]}; do
if test -f "$experiment_directory/body/$payload.txt"; then
continue
else
echo "Generating Payloads: "
{
cd "$experiment_directory/body" && ./generate.sh
}
break
fi
done
# Execute workloads long enough for runtime to learn excepted execution time
echo -n "Running Samples: "
hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1024.txt" http://localhost:10000
hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/10240.txt" http://localhost:10001
hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/102400.txt" http://localhost:10002
hey -n "$iterations" -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1048576.txt" http://localhost:10003
sleep 5
echo "[DONE]"
# Execute the experiments
echo "Running Experiments"
for i in {0..3}; do
printf "\t%d Payload: " "${payloads[$i]}"
hey -n "$iterations" -c 1 -cpus 2 -o csv -m GET -D "$experiment_directory/body/${payloads[$i]}.txt" http://localhost:"${ports[$i]}" > "$results_directory/${payloads[$i]}.csv"
echo "[DONE]"
done
# Stop the runtime
if [ "$1" != "-d" ]; then
sleep 5
kill_runtime
fi
# Generate *.csv and *.dat results
echo -n "Parsing Results: "
printf "Payload,Success_Rate\n" >> "$results_directory/success.csv"
printf "Payload,Throughput\n" >> "$results_directory/throughput.csv"
printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv"
for payload in ${payloads[*]}; do
# Calculate Success Rate for csv
awk -F, '
$7 == 200 {ok++}
END{printf "'"$payload"',%3.5f\n", (ok / '"$iterations"' * 100)}
' < "$results_directory/$payload.csv" >> "$results_directory/success.csv"
# Filter on 200s, convery from s to ms, and sort
awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \
| sort -g > "$results_directory/$payload-response.csv"
# Get Number of 200s
oks=$(wc -l < "$results_directory/$payload-response.csv")
((oks == 0)) && continue # If all errors, skip line
# Get Latest Timestamp
duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
throughput=$(echo "$oks/$duration" | bc)
printf "%d,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv"
# Generate Latency Data for csv
awk '
# Filter on 200s, convery from s to ms, and sort
awk -F, '$7 == 200 {print ($1 * 1000)}' < "$results_directory/$payload.csv" \
| sort -g > "$results_directory/$payload-response.csv"
# Get Number of 200s
oks=$(wc -l < "$results_directory/$payload-response.csv")
((oks == 0)) && continue # If all errors, skip line
# We determine duration by looking at the timestamp of the last complete request
# TODO: Should this instead just use the client-side synthetic duration_sec value?
duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
# Throughput is calculated as the mean number of successful requests per second
throughput=$(echo "$oks/$duration" | bc)
printf "%d,%f\n" "$payload" "$throughput" >> "$results_directory/throughput.csv"
# Generate Latency Data for csv
awk '
BEGIN {
sum = 0
p50 = int('"$oks"' * 0.5)
@ -106,18 +158,37 @@ for payload in ${payloads[*]}; do
NR==p100 {printf "%1.4f\n", $0}
' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv"
# Delete scratch file used for sorting/counting
rm -rf "$results_directory/$payload-response.csv"
done
# Delete scratch file used for sorting/counting
rm -rf "$results_directory/$payload-response.csv"
done
# Transform csvs to dat files for gnuplot
for file in success latency throughput; do
printf "#" > "$results_directory/$file.dat"
tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat"
done
# Generate gnuplots
generate_gnuplots "$results_directory" "$__run_sh__base_path" || {
printf "[ERR]\n"
panic "failed to generate gnuplots"
}
printf "[OK]\n"
return 0
}
# Expected Symbol used by the framework
experiment_main() {
local -r target_hostname="$1"
local -r results_directory="$2"
# Transform csvs to dat files for gnuplot
for file in success latency throughput; do
echo -n "#" > "$results_directory/$file.dat"
tr ',' ' ' < "$results_directory/$file.csv" | column -t >> "$results_directory/$file.dat"
done
run_samples "$target_hostname" || return 1
run_experiments "$target_hostname" "$results_directory" || return 1
process_results "$results_directory" || return 1
# Generate gnuplots
generate_gnuplots
return 0
}
# Cleanup, if requires
echo "[DONE]"
# Delegating to main provided by framework
main "$@"

Loading…
Cancel
Save