diff --git a/runtime/experiments/payload/.gitignore b/runtime/experiments/payload/.gitignore new file mode 100644 index 0000000..64f722e --- /dev/null +++ b/runtime/experiments/payload/.gitignore @@ -0,0 +1 @@ +res diff --git a/runtime/experiments/payload/body/.gitignore b/runtime/experiments/payload/body/.gitignore new file mode 100644 index 0000000..2211df6 --- /dev/null +++ b/runtime/experiments/payload/body/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/runtime/experiments/payload/body/generate.sh b/runtime/experiments/payload/body/generate.sh new file mode 100755 index 0000000..70959c4 --- /dev/null +++ b/runtime/experiments/payload/body/generate.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Generates payloads of 1KB, 10KB, 100KB, 1MB +for size in 1024 $((1024 * 10)) $((1024 * 100)) $((1024 * 1024)); do + rm -rf $size.txt + i=0 + echo -n "Generating $size:" + while ((i < size)); do + printf 'a' >>$size.txt + ((i++)) + done + echo "[DONE]" +done diff --git a/runtime/experiments/payload/debug.sh b/runtime/experiments/payload/debug.sh new file mode 100755 index 0000000..62c64e3 --- /dev/null +++ b/runtime/experiments/payload/debug.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Executes the runtime in GDB +# Substitutes the absolute path from the container with a path relatively derived from the location of this script +# This allows debugging outside of the Docker container +# Also disables pagination and stopping on SIGUSR1 + +experiment_directory=$(pwd) +project_directory=$(cd ../.. && pwd) +binary_directory=$(cd "$project_directory"/bin && pwd) + +export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" +export PATH="$binary_directory:$PATH" + +gdb --eval-command="handle SIGUSR1 nostop" \ + --eval-command="handle SIGPIPE nostop" \ + --eval-command="set pagination off" \ + --eval-command="set substitute-path /sledge/runtime $project_directory" \ + --eval-command="run $experiment_directory/spec.json" \ + sledgert diff --git a/runtime/experiments/payload/latency.gnuplot b/runtime/experiments/payload/latency.gnuplot new file mode 100644 index 0000000..095aa47 --- /dev/null +++ b/runtime/experiments/payload/latency.gnuplot @@ -0,0 +1,20 @@ +reset + +set term jpeg +set output "latency.jpg" + +set xlabel "Payload (bytes)" +set xrange [-5:1050000] + +set ylabel "Latency (ms)" +set yrange [0:] + +set key left top + + +set style histogram columnstacked + +plot 'latency.dat' using 1:2 title 'p50', \ + 'latency.dat' using 1:3 title 'p90', \ + 'latency.dat' using 1:4 title 'p99', \ + 'latency.dat' using 1:5 title 'p100', \ diff --git a/runtime/experiments/payload/run.sh b/runtime/experiments/payload/run.sh new file mode 100755 index 0000000..2e5158f --- /dev/null +++ b/runtime/experiments/payload/run.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +timestamp=$(date +%s) +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) +results_directory="$experiment_directory/res/$timestamp" +log=log.txt + +mkdir -p "$results_directory" + +{ + echo "*******" + echo "* Git *" + echo "*******" + git log | head -n 1 | cut -d' ' -f2 + git status + echo "" + + echo "************" + echo "* Makefile *" + echo "************" + cat ../../Makefile + echo "" + + echo "**********" + echo "* Run.sh *" + echo "**********" + cat run.sh + echo "" + + echo "************" + echo "* Hardware *" + echo "************" + lscpu + echo "" + + echo "*************" + echo "* Execution *" + echo "*************" +} >>"$results_directory/$log" + +# Start the runtime +if [ "$1" != "-d" ]; then + PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >>"$results_directory/$log" 2>>"$results_directory/$log" & + sleep 1 +else + echo "Running under gdb" + echo "Running under gdb" >>"$results_directory/$log" +fi + +payloads=(1024 10240 102400 1048576) +ports=(10000 10001 10002 10003) +# If the one of the expected body files doesn't exist, trigger the generation script. +for payload in ${payloads[*]}; do + if test -f "$experiment_directory/body/$payload.txt"; then + continue + else + echo "Generating Payloads: " + { + cd "$experiment_directory/body" && ./generate.sh + } + break + fi +done + +# Execute workloads long enough for runtime to learn excepted execution time +echo -n "Running Samples: " +hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1024.txt" http://localhost:10000 +hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/10240.txt" http://localhost:10001 +hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/102400.txt" http://localhost:10002 +hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1048576.txt" http://localhost:10003 +sleep 5 +echo "[DONE]" + +# Execute the experiments +echo "Running Experiments" +for i in {0..3}; do + printf "\t%d Payload: " "${payloads[$i]}" + hey -n 10000 -c 1 -cpus 2 -o csv -m GET -D "$experiment_directory/body/${payloads[$i]}.txt" http://localhost:"${ports[$i]}" >"$results_directory/${payloads[$i]}.csv" + echo "[DONE]" +done +# Stop the runtime + +if [ "$1" != "-d" ]; then + sleep 5 + echo -n "Running Cleanup: " + pkill sledgert >/dev/null 2>/dev/null + pkill wrk >/dev/null 2>/dev/null + echo "[DONE]" +fi + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" + +for payload in ${payloads[*]}; do + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 {ok++} + END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)} + ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + + # Get Latest Timestamp + duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + throughput=$(echo "$oks/$duration" | bc) + printf "%d,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency throughput; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots +cd "$results_directory" || exit +gnuplot ../../latency.gnuplot +gnuplot ../../success.gnuplot +gnuplot ../../throughput.gnuplot +cd "$experiment_directory" || exit + +# Cleanup, if requires +echo "[DONE]" diff --git a/runtime/experiments/payload/spec.json b/runtime/experiments/payload/spec.json new file mode 100644 index 0000000..34b9040 --- /dev/null +++ b/runtime/experiments/payload/spec.json @@ -0,0 +1,56 @@ +{ + "active": "yes", + "name": "work1k", + "path": "work1k_wasm.so", + "port": 10000, + "relative-deadline-us": 5000000, + "argsize": 1, + "http-req-headers": [], + "http-req-content-type": "text/plain", + "http-req-size": 1048, + "http-resp-headers": [], + "http-resp-size": 1048, + "http-resp-content-type": "text/plain" +}, +{ + "active": "yes", + "name": "work10k", + "path": "work10k_wasm.so", + "port": 10001, + "relative-deadline-us": 5000000, + "argsize": 1, + "http-req-headers": [], + "http-req-content-type": "text/plain", + "http-req-size": 10480, + "http-resp-headers": [], + "http-resp-size": 10480, + "http-resp-content-type": "text/plain" +}, +{ + "active": "yes", + "name": "work100k", + "path": "work100k_wasm.so", + "port": 10002, + "relative-deadline-us": 5000000, + "argsize": 1, + "http-req-headers": [], + "http-req-content-type": "text/plain", + "http-req-size": 104800, + "http-resp-headers": [], + "http-resp-size": 104800, + "http-resp-content-type": "text/plain" +}, +{ + "active": "yes", + "name": "work1m", + "path": "work1m_wasm.so", + "port": 10003, + "relative-deadline-us": 5000000, + "argsize": 1, + "http-req-headers": [], + "http-req-content-type": "text/plain", + "http-req-size": 1048776, + "http-resp-headers": [], + "http-resp-size": 1048776, + "http-resp-content-type": "text/plain" +} diff --git a/runtime/experiments/payload/success.gnuplot b/runtime/experiments/payload/success.gnuplot new file mode 100644 index 0000000..3a28692 --- /dev/null +++ b/runtime/experiments/payload/success.gnuplot @@ -0,0 +1,12 @@ +reset + +set term jpeg +set output "success.jpg" + +set xlabel "Payload (bytes)" +set xrange [-5:1050000] + +set ylabel "% 2XX" +set yrange [0:110] + +plot 'success.dat' using 1:2 title '2XX' diff --git a/runtime/experiments/payload/throughput.gnuplot b/runtime/experiments/payload/throughput.gnuplot new file mode 100644 index 0000000..23b7a68 --- /dev/null +++ b/runtime/experiments/payload/throughput.gnuplot @@ -0,0 +1,13 @@ +reset + +set term jpeg +set output "throughput.jpg" + +# TODO: Axis shouldn't be linear +set xlabel "Payload (bytes)" +set xrange [-5:1050000] + +set ylabel "Requests/sec" +set yrange [0:] + +plot 'throughput.dat' using 1:2 title 'Reqs/sec'