From 28ccac11b79fbba5f0ca54e47c0a2e56cbf93ce8 Mon Sep 17 00:00:00 2001
From: Sean McBride <spmcbride1201@gmail.com>
Date: Fri, 27 Nov 2020 19:20:05 -0500
Subject: [PATCH] chore: improve experimental scripts

---
 runtime/experiments/deadline/client.sh    | 106 ++++++++++++++++++++++
 runtime/experiments/deadline/demo.sh      |  34 -------
 runtime/experiments/deadline/server.sh    |  10 ++
 runtime/experiments/deadline/spec.json    |  66 +-------------
 runtime/experiments/preemption/backend.sh |  18 ++++
 runtime/experiments/preemption/debug.sh   |   1 +
 6 files changed, 139 insertions(+), 96 deletions(-)
 create mode 100755 runtime/experiments/deadline/client.sh
 delete mode 100755 runtime/experiments/deadline/demo.sh
 create mode 100755 runtime/experiments/deadline/server.sh
 create mode 100755 runtime/experiments/preemption/backend.sh

diff --git a/runtime/experiments/deadline/client.sh b/runtime/experiments/deadline/client.sh
new file mode 100755
index 0000000..37e267a
--- /dev/null
+++ b/runtime/experiments/deadline/client.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+# host=192.168.1.13
+host=localhost
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+inputs=(40 10)
+duration_sec=15
+offset=5
+
+# Execute workloads long enough for runtime to learn excepted execution time
+echo -n "Running Samples: "
+for input in ${inputs[*]}; do
+  hey -z ${duration_sec}s -cpus 3 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input))
+done
+echo "[DONE]"
+sleep 5
+
+echo "Running Experiments"
+
+# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run
+hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" &
+sleep $offset
+hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" &
+sleep $((duration_sec + offset + 15))
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+deadlines_ms=(20 20000)
+payloads=(fib10-con fib40-con)
+
+for ((i = 0; i < 2; i++)); do
+  payload=${payloads[$i]}
+  deadline=${deadlines_ms[$i]}
+
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  # Calculate Success Rate for csv
+  awk -F, '
+      $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
+      END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)}
+    ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+      BEGIN {
+        sum = 0
+        p50 = int('"$oks"' * 0.5)
+        p90 = int('"$oks"' * 0.9)
+        p99 = int('"$oks"' * 0.99)  
+        p100 = '"$oks"'
+        printf "'"$payload"',"
+      }
+      NR==p50  {printf "%1.4f,",  $0}
+      NR==p90  {printf "%1.4f,",  $0}
+      NR==p99  {printf "%1.4f,",  $0}
+      NR==p100 {printf "%1.4f\n", $0}
+    ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if requires
+echo "[DONE]"
diff --git a/runtime/experiments/deadline/demo.sh b/runtime/experiments/deadline/demo.sh
deleted file mode 100755
index 3fdb1e9..0000000
--- a/runtime/experiments/deadline/demo.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-cd ../../bin
-LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./sledgert ../tests/mixed_preemption/test_mixed_preemption.json &
-cd ../tests/mixed_preemption/
-
-# Run small samples on each port to let the runtime figure out the execution time
-sleep 10
-echo "Running Samples"
-wrk -d 20s -t1 -s post.lua http://localhost:10010 -- --delay 500 10\n
-wrk -d 20s -t1 -s post.lua http://localhost:10020 -- --delay 500 20\n
-wrk -d 20s -t1 -s post.lua http://localhost:10030 -- --delay 500 25\n
-
-# Run in Parallel
-sleep 10
-echo "Running Experiments"
-wrk -d 1m -t1 -s post.lua http://localhost:10010 -- --delay 125 10\n >./res/fib10.txt &
-wrk -d 2m -t1 -s post.lua http://localhost:10020 -- --delay 250 20\n >./res/fib20.txt &
-wrk -d 3m -t1 -s post.lua http://localhost:10025 -- --delay 500 25\n >./res/fib25.txt
-
-# Kill the Background Sledge processes
-sleep 10
-echo "Running Cleanup"
-pkill sledgert
-pkill wrk
-
-# Extract the Latency CSV Data from the Log
-
-echo 'Fib10, Fib10' >./res/fib10.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib10.txt >>./res/fib10.csv
-echo 'Fib20, Fib20' >./res/fib20.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib20.txt >>./res/fib20.csv
-echo 'Fib25, Fib25' >./res/fib25.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib25.txt >>./res/fib25.csv
-paste -d, ./res/fib10.csv ./res/fib20.csv ./res/fib25.csv >./res/merged.csv
diff --git a/runtime/experiments/deadline/server.sh b/runtime/experiments/deadline/server.sh
new file mode 100755
index 0000000..e1e2e5b
--- /dev/null
+++ b/runtime/experiments/deadline/server.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+source ../common.sh
+
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+# Start the runtime
+
+PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json"
diff --git a/runtime/experiments/deadline/spec.json b/runtime/experiments/deadline/spec.json
index 1d64c1b..91f9c41 100644
--- a/runtime/experiments/deadline/spec.json
+++ b/runtime/experiments/deadline/spec.json
@@ -4,67 +4,8 @@
   "path": "fibonacci_wasm.so",
   "port": 10010,
   "expected-execution-us": 600,
-  "relative-deadline-us": 2000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_20",
-  "path": "fibonacci_wasm.so",
-  "port": 10020,
-  "expected-execution-us": 900,
-  "relative-deadline-us": 5000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_25",
-  "path": "fibonacci_wasm.so",
-  "port": 10025,
-  "expected-execution-us": 90000,
-  "relative-deadline-us": 200000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_30",
-  "path": "fibonacci_wasm.so",
-  "port": 10030,
-  "expected-execution-us": 9000,
-  "relative-deadline-us": 80000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_35",
-  "path": "fibonacci_wasm.so",
-  "port": 10035,
-  "expected-execution-us": 9000,
-  "relative-deadline-us": 53000,
+  "admissions-percentile": 70,
+  "relative-deadline-us": 12000,
   "argsize": 1,
   "http-req-headers": [],
   "http-req-content-type": "text/plain",
@@ -79,7 +20,8 @@
   "path": "fibonacci_wasm.so",
   "port": 10040,
   "expected-execution-us": 550000,
-  "relative-deadline-us": 300000000,
+  "admissions-percentile": 70,
+  "relative-deadline-us": 60000000,
   "argsize": 1,
   "http-req-headers": [],
   "http-req-content-type": "text/plain",
diff --git a/runtime/experiments/preemption/backend.sh b/runtime/experiments/preemption/backend.sh
new file mode 100755
index 0000000..5bc869b
--- /dev/null
+++ b/runtime/experiments/preemption/backend.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp/$scheduler"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+# Start the runtime
+PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" | tee -a "$results_directory/$log"
diff --git a/runtime/experiments/preemption/debug.sh b/runtime/experiments/preemption/debug.sh
index 62c64e3..079f6a0 100755
--- a/runtime/experiments/preemption/debug.sh
+++ b/runtime/experiments/preemption/debug.sh
@@ -10,6 +10,7 @@ binary_directory=$(cd "$project_directory"/bin && pwd)
 
 export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
 export PATH="$binary_directory:$PATH"
+export SLEDGE_SCHEDULER="EDF"
 
 gdb --eval-command="handle SIGUSR1 nostop" \
   --eval-command="handle SIGPIPE nostop" \