From 887c2de3120438cbd2357fadd6bc29986a5a09fe Mon Sep 17 00:00:00 2001
From: Sean McBride <spmcbride1201@gmail.com>
Date: Sat, 10 Oct 2020 12:51:16 -0400
Subject: [PATCH] test: Add payload experiment

---
 runtime/experiments/payload/.gitignore        |   1 +
 runtime/experiments/payload/body/.gitignore   |   1 +
 runtime/experiments/payload/body/generate.sh  |  12 ++
 runtime/experiments/payload/debug.sh          |  19 +++
 runtime/experiments/payload/latency.gnuplot   |  20 +++
 runtime/experiments/payload/run.sh            | 155 ++++++++++++++++++
 runtime/experiments/payload/spec.json         |  56 +++++++
 runtime/experiments/payload/success.gnuplot   |  12 ++
 .../experiments/payload/throughput.gnuplot    |  13 ++
 9 files changed, 289 insertions(+)
 create mode 100644 runtime/experiments/payload/.gitignore
 create mode 100644 runtime/experiments/payload/body/.gitignore
 create mode 100755 runtime/experiments/payload/body/generate.sh
 create mode 100755 runtime/experiments/payload/debug.sh
 create mode 100644 runtime/experiments/payload/latency.gnuplot
 create mode 100755 runtime/experiments/payload/run.sh
 create mode 100644 runtime/experiments/payload/spec.json
 create mode 100644 runtime/experiments/payload/success.gnuplot
 create mode 100644 runtime/experiments/payload/throughput.gnuplot

diff --git a/runtime/experiments/payload/.gitignore b/runtime/experiments/payload/.gitignore
new file mode 100644
index 0000000..64f722e
--- /dev/null
+++ b/runtime/experiments/payload/.gitignore
@@ -0,0 +1 @@
+res
diff --git a/runtime/experiments/payload/body/.gitignore b/runtime/experiments/payload/body/.gitignore
new file mode 100644
index 0000000..2211df6
--- /dev/null
+++ b/runtime/experiments/payload/body/.gitignore
@@ -0,0 +1 @@
+*.txt
diff --git a/runtime/experiments/payload/body/generate.sh b/runtime/experiments/payload/body/generate.sh
new file mode 100755
index 0000000..70959c4
--- /dev/null
+++ b/runtime/experiments/payload/body/generate.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+# Generates payloads of 1KB, 10KB, 100KB, 1MB
+for size in 1024 $((1024 * 10)) $((1024 * 100)) $((1024 * 1024)); do
+  rm -rf $size.txt
+  i=0
+  echo -n "Generating $size:"
+  while ((i < size)); do
+    printf 'a' >>$size.txt
+    ((i++))
+  done
+  echo "[DONE]"
+done
diff --git a/runtime/experiments/payload/debug.sh b/runtime/experiments/payload/debug.sh
new file mode 100755
index 0000000..62c64e3
--- /dev/null
+++ b/runtime/experiments/payload/debug.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Executes the runtime in GDB
+# Substitutes the absolute path from the container with a path relatively derived from the location of this script
+# This allows debugging outside of the Docker container
+# Also disables pagination and stopping on SIGUSR1
+
+experiment_directory=$(pwd)
+project_directory=$(cd ../.. && pwd)
+binary_directory=$(cd "$project_directory"/bin && pwd)
+
+export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
+export PATH="$binary_directory:$PATH"
+
+gdb --eval-command="handle SIGUSR1 nostop" \
+  --eval-command="handle SIGPIPE nostop" \
+  --eval-command="set pagination off" \
+  --eval-command="set substitute-path /sledge/runtime $project_directory" \
+  --eval-command="run $experiment_directory/spec.json" \
+  sledgert
diff --git a/runtime/experiments/payload/latency.gnuplot b/runtime/experiments/payload/latency.gnuplot
new file mode 100644
index 0000000..095aa47
--- /dev/null
+++ b/runtime/experiments/payload/latency.gnuplot
@@ -0,0 +1,20 @@
+reset
+
+set term jpeg 
+set output "latency.jpg"
+
+set xlabel "Payload (bytes)"
+set xrange [-5:1050000]
+
+set ylabel "Latency (ms)"
+set yrange [0:]
+
+set key left top
+
+
+set style histogram columnstacked
+
+plot 'latency.dat' using 1:2 title 'p50', \
+     'latency.dat' using 1:3 title 'p90', \
+     'latency.dat' using 1:4 title 'p99', \
+     'latency.dat' using 1:5 title 'p100', \
diff --git a/runtime/experiments/payload/run.sh b/runtime/experiments/payload/run.sh
new file mode 100755
index 0000000..2e5158f
--- /dev/null
+++ b/runtime/experiments/payload/run.sh
@@ -0,0 +1,155 @@
+#!/bin/bash
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+results_directory="$experiment_directory/res/$timestamp"
+log=log.txt
+
+mkdir -p "$results_directory"
+
+{
+  echo "*******"
+  echo "* Git *"
+  echo "*******"
+  git log | head -n 1 | cut -d' ' -f2
+  git status
+  echo ""
+
+  echo "************"
+  echo "* Makefile *"
+  echo "************"
+  cat ../../Makefile
+  echo ""
+
+  echo "**********"
+  echo "* Run.sh *"
+  echo "**********"
+  cat run.sh
+  echo ""
+
+  echo "************"
+  echo "* Hardware *"
+  echo "************"
+  lscpu
+  echo ""
+
+  echo "*************"
+  echo "* Execution *"
+  echo "*************"
+} >>"$results_directory/$log"
+
+# Start the runtime
+if [ "$1" != "-d" ]; then
+  PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" >>"$results_directory/$log" 2>>"$results_directory/$log" &
+  sleep 1
+else
+  echo "Running under gdb"
+  echo "Running under gdb" >>"$results_directory/$log"
+fi
+
+payloads=(1024 10240 102400 1048576)
+ports=(10000 10001 10002 10003)
+# If the one of the expected body files doesn't exist, trigger the generation script.
+for payload in ${payloads[*]}; do
+  if test -f "$experiment_directory/body/$payload.txt"; then
+    continue
+  else
+    echo "Generating Payloads: "
+    {
+      cd "$experiment_directory/body" && ./generate.sh
+    }
+    break
+  fi
+done
+
+# Execute workloads long enough for runtime to learn excepted execution time
+echo -n "Running Samples: "
+hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1024.txt" http://localhost:10000
+hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/10240.txt" http://localhost:10001
+hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/102400.txt" http://localhost:10002
+hey -n 10000 -c 3 -q 200 -o csv -m GET -D "$experiment_directory/body/1048576.txt" http://localhost:10003
+sleep 5
+echo "[DONE]"
+
+# Execute the experiments
+echo "Running Experiments"
+for i in {0..3}; do
+  printf "\t%d Payload: " "${payloads[$i]}"
+  hey -n 10000 -c 1 -cpus 2 -o csv -m GET -D "$experiment_directory/body/${payloads[$i]}.txt" http://localhost:"${ports[$i]}" >"$results_directory/${payloads[$i]}.csv"
+  echo "[DONE]"
+done
+# Stop the runtime
+
+if [ "$1" != "-d" ]; then
+  sleep 5
+  echo -n "Running Cleanup: "
+  pkill sledgert >/dev/null 2>/dev/null
+  pkill wrk >/dev/null 2>/dev/null
+  echo "[DONE]"
+fi
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+for payload in ${payloads[*]}; do
+  # Calculate Success Rate for csv
+  awk -F, '
+    $7 == 200 {ok++}
+    END{printf "'"$payload"',%3.5f\n", (ok / (NR - 1) * 100)}
+  ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+
+  # Get Latest Timestamp
+  duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%d,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+    BEGIN {
+      sum = 0
+      p50 = int('"$oks"' * 0.5)
+      p90 = int('"$oks"' * 0.9)
+      p99 = int('"$oks"' * 0.99)
+      p100 = '"$oks"'
+      printf "'"$payload"',"
+    }   
+    NR==p50 {printf "%1.4f,", $0}
+    NR==p90 {printf "%1.4f,", $0}
+    NR==p99 {printf "%1.4f,", $0}
+    NR==p100 {printf "%1.4f\n", $0}
+  ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots
+cd "$results_directory" || exit
+gnuplot ../../latency.gnuplot
+gnuplot ../../success.gnuplot
+gnuplot ../../throughput.gnuplot
+cd "$experiment_directory" || exit
+
+# Cleanup, if requires
+echo "[DONE]"
diff --git a/runtime/experiments/payload/spec.json b/runtime/experiments/payload/spec.json
new file mode 100644
index 0000000..34b9040
--- /dev/null
+++ b/runtime/experiments/payload/spec.json
@@ -0,0 +1,56 @@
+{
+	"active": "yes",
+	"name": "work1k",
+	"path": "work1k_wasm.so",
+	"port": 10000,
+	"relative-deadline-us": 5000000,
+	"argsize": 1,
+	"http-req-headers": [],
+	"http-req-content-type": "text/plain",
+	"http-req-size": 1048,
+	"http-resp-headers": [],
+	"http-resp-size": 1048,
+	"http-resp-content-type": "text/plain"
+},
+{
+	"active": "yes",
+	"name": "work10k",
+	"path": "work10k_wasm.so",
+	"port": 10001,
+	"relative-deadline-us": 5000000,
+	"argsize": 1,
+	"http-req-headers": [],
+	"http-req-content-type": "text/plain",
+	"http-req-size": 10480,
+	"http-resp-headers": [],
+	"http-resp-size": 10480,
+	"http-resp-content-type": "text/plain"
+},
+{
+	"active": "yes",
+	"name": "work100k",
+	"path": "work100k_wasm.so",
+	"port": 10002,  
+	"relative-deadline-us": 5000000,
+	"argsize": 1,
+	"http-req-headers": [],
+	"http-req-content-type": "text/plain",
+	"http-req-size": 104800,
+	"http-resp-headers": [],
+	"http-resp-size": 104800,
+	"http-resp-content-type": "text/plain"
+},
+{
+	"active": "yes",
+	"name": "work1m",
+	"path": "work1m_wasm.so",
+	"port": 10003,
+	"relative-deadline-us": 5000000,
+	"argsize": 1,
+	"http-req-headers": [],
+	"http-req-content-type": "text/plain",
+	"http-req-size": 1048776,
+	"http-resp-headers": [],
+	"http-resp-size": 1048776,
+	"http-resp-content-type": "text/plain"
+}
diff --git a/runtime/experiments/payload/success.gnuplot b/runtime/experiments/payload/success.gnuplot
new file mode 100644
index 0000000..3a28692
--- /dev/null
+++ b/runtime/experiments/payload/success.gnuplot
@@ -0,0 +1,12 @@
+reset
+
+set term jpeg 
+set output "success.jpg"
+
+set xlabel "Payload (bytes)"
+set xrange [-5:1050000]
+
+set ylabel "% 2XX"
+set yrange [0:110]
+
+plot 'success.dat' using 1:2 title '2XX'
diff --git a/runtime/experiments/payload/throughput.gnuplot b/runtime/experiments/payload/throughput.gnuplot
new file mode 100644
index 0000000..23b7a68
--- /dev/null
+++ b/runtime/experiments/payload/throughput.gnuplot
@@ -0,0 +1,13 @@
+reset
+
+set term jpeg 
+set output "throughput.jpg"
+
+# TODO: Axis shouldn't be linear
+set xlabel "Payload (bytes)"
+set xrange [-5:1050000]
+
+set ylabel "Requests/sec"
+set yrange [0:]
+
+plot 'throughput.dat' using 1:2 title 'Reqs/sec'