Merge pull request #2 from gwsystems/mixed-preemption

Mixed preemption UPDATE

runtime/Makefile

@@ -56,7 +56,10 @@ USE_MEM = USE_MEM_VM
 CFLAGS += -D${ARCH}
 CFLAGS += -DNCORES=${TOTAL_CORES}
 CFLAGS += -DPAGE_SIZE=$(PAGE_SIZE)
-#CFLAGS += -DPREEMPT_DISABLE
+
+# Optionally Disable preemption
+# CFLAGS += -DPREEMPT_DISABLE
+
 CFLAGS += -D${USE_MEM}
 
 # Preprocessor

runtime/experiments/applications/fivebyeight/run.sh

@@ -42,6 +42,5 @@ if [ "$1" != "-d" ]; then
   sleep 5
   echo -n "Running Cleanup: "
   pkill sledgert >/dev/null 2>/dev/null
-  pkill wrk >/dev/null 2>/dev/null
   echo "[DONE]"
 fi

runtime/experiments/applications/handwriting/run.sh

@@ -42,6 +42,5 @@ if [ "$1" != "-d" ]; then
   sleep 5
   echo -n "Running Cleanup: "
   pkill sledgert >/dev/null 2>/dev/null
-  pkill wrk >/dev/null 2>/dev/null
   echo "[DONE]"
 fi

runtime/experiments/applications/hyde/run.sh

@@ -41,6 +41,5 @@ if [ "$1" != "-d" ]; then
   sleep 5
   echo -n "Running Cleanup: "
   pkill sledgert >/dev/null 2>/dev/null
-  pkill wrk >/dev/null 2>/dev/null
   echo "[DONE]"
 fi

runtime/experiments/deadline/client.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+host=localhost
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+inputs=(40 10)
+duration_sec=60
+offset=5
+
+# Execute workloads long enough for runtime to learn excepted execution time
+echo -n "Running Samples: "
+for input in ${inputs[*]}; do
+  hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input))
+done
+echo "[DONE]"
+sleep 5
+
+echo "Running Experiments"
+
+# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run
+hey -n 1000 -c 1000 -cpus 6 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv"
+# sleep $offset
+# hey -n 25000 -c 1000000 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" &
+# sleep $((duration_sec + offset + 45))
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+deadlines_ms=(20 20000)
+# durations_s=(60 70)
+payloads=(fib10-con fib40-con)
+
+for ((i = 1; i < 2; i++)); do
+  payload=${payloads[$i]}
+  deadline=${deadlines_ms[$i]}
+  # duration=${durations_s[$i]}
+
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  # Calculate Success Rate for csv
+  awk -F, '
+      $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
+      END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)}
+    ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  # throughput=$(echo "$oks/$duration" | bc)
+  # printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+      BEGIN {
+        sum = 0
+        p50 = int('"$oks"' * 0.5)
+        p90 = int('"$oks"' * 0.9)
+        p99 = int('"$oks"' * 0.99)  
+        p100 = '"$oks"'
+        printf "'"$payload"',"
+      }
+      NR==p50  {printf "%1.4f,",  $0}
+      NR==p90  {printf "%1.4f,",  $0}
+      NR==p99  {printf "%1.4f,",  $0}
+      NR==p100 {printf "%1.4f\n", $0}
+    ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if requires
+echo "[DONE]"

runtime/experiments/deadline/client2.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+host=192.168.1.13
+# host=localhost
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+inputs=(40 10)
+duration_sec=30
+offset=5
+
+# Execute workloads long enough for runtime to learn excepted execution time
+echo -n "Running Samples: "
+for input in ${inputs[*]}; do
+  hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input))
+done
+echo "[DONE]"
+sleep 5
+
+echo "Running Experiments"
+
+# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run
+hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" &
+sleep $offset
+hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" &
+sleep $((duration_sec + offset + 15))
+sleep 30
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+deadlines_ms=(20 20000)
+payloads=(fib10-con fib40-con)
+durations_s=(30 40)
+
+for ((i = 0; i < 2; i++)); do
+  payload=${payloads[$i]}
+  deadline=${deadlines_ms[$i]}
+  duration=${durations_s[$i]}
+
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  # Calculate Success Rate for csv
+  awk -F, '
+      $7 == 200 {denom++}
+      $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
+      END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)}
+    ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+      BEGIN {
+        sum = 0
+        p50 = int('"$oks"' * 0.5)
+        p90 = int('"$oks"' * 0.9)
+        p99 = int('"$oks"' * 0.99)  
+        p100 = '"$oks"'
+        printf "'"$payload"',"
+      }
+      NR==p50  {printf "%1.4f,",  $0}
+      NR==p90  {printf "%1.4f,",  $0}
+      NR==p99  {printf "%1.4f,",  $0}
+      NR==p100 {printf "%1.4f\n", $0}
+    ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if requires
+echo "[DONE]"

runtime/experiments/deadline/client3.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+host=192.168.1.13
+# host=localhost
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+inputs=(10)
+duration_sec=30
+offset=5
+
+# Execute workloads long enough for runtime to learn excepted execution time
+echo -n "Running Samples: "
+hey -n 16 -c 4 -t 0 -o csv -m GET -d "10\n" http://${host}:10010
+echo "[DONE]"
+sleep 5
+
+echo "Running Experiments"
+
+# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run
+# hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" &
+# sleep $offset
+hey -z ${duration_sec}s -cpus 6 -c 400 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv"
+# sleep $((duration_sec + offset + 15))
+# sleep 30
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+deadlines_ms=(20 20000)
+payloads=(fib10-con fib40-con)
+durations_s=(30 40)
+
+for ((i = 0; i < 1; i++)); do
+  payload=${payloads[$i]}
+  deadline=${deadlines_ms[$i]}
+  duration=${durations_s[$i]}
+
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  # Calculate Success Rate for csv
+  awk -F, '
+      $7 == 200 {denom++}
+      $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
+      END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)}
+    ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+      BEGIN {
+        sum = 0
+        p50 = int('"$oks"' * 0.5)
+        p90 = int('"$oks"' * 0.9)
+        p99 = int('"$oks"' * 0.99)  
+        p100 = '"$oks"'
+        printf "'"$payload"',"
+      }
+      NR==p50  {printf "%1.4f,",  $0}
+      NR==p90  {printf "%1.4f,",  $0}
+      NR==p99  {printf "%1.4f,",  $0}
+      NR==p100 {printf "%1.4f\n", $0}
+    ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if requires
+echo "[DONE]"

runtime/experiments/deadline/demo.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-cd ../../bin
-LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./sledgert ../tests/mixed_preemption/test_mixed_preemption.json &
-cd ../tests/mixed_preemption/
-
-# Run small samples on each port to let the runtime figure out the execution time
-sleep 10
-echo "Running Samples"
-wrk -d 20s -t1 -s post.lua http://localhost:10010 -- --delay 500 10\n
-wrk -d 20s -t1 -s post.lua http://localhost:10020 -- --delay 500 20\n
-wrk -d 20s -t1 -s post.lua http://localhost:10030 -- --delay 500 25\n
-
-# Run in Parallel
-sleep 10
-echo "Running Experiments"
-wrk -d 1m -t1 -s post.lua http://localhost:10010 -- --delay 125 10\n >./res/fib10.txt &
-wrk -d 2m -t1 -s post.lua http://localhost:10020 -- --delay 250 20\n >./res/fib20.txt &
-wrk -d 3m -t1 -s post.lua http://localhost:10025 -- --delay 500 25\n >./res/fib25.txt
-
-# Kill the Background Sledge processes
-sleep 10
-echo "Running Cleanup"
-pkill sledgert
-pkill wrk
-
-# Extract the Latency CSV Data from the Log
-
-echo 'Fib10, Fib10' >./res/fib10.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib10.txt >>./res/fib10.csv
-echo 'Fib20, Fib20' >./res/fib20.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib20.txt >>./res/fib20.csv
-echo 'Fib25, Fib25' >./res/fib25.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib25.txt >>./res/fib25.csv
-paste -d, ./res/fib10.csv ./res/fib20.csv ./res/fib25.csv >./res/merged.csv

runtime/experiments/deadline/fix_calcs.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+experiment_directory=$(pwd)
+results_directory="$experiment_directory/res/1606615320-fifo-adm"
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+deadlines_ms=(20 20000)
+payloads=(fib10-con fib40-con)
+
+for ((i = 0; i < 2; i++)); do
+  payload=${payloads[$i]}
+  deadline=${deadlines_ms[$i]}
+
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  # Calculate Success Rate for csv
+  awk -F, '
+      $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
+      END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)}
+    ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+      BEGIN {
+        sum = 0
+        p50 = int('"$oks"' * 0.5)
+        p90 = int('"$oks"' * 0.9)
+        p99 = int('"$oks"' * 0.99)  
+        p100 = '"$oks"'
+        printf "'"$payload"',"
+      }
+      NR==p50  {printf "%1.4f,",  $0}
+      NR==p90  {printf "%1.4f,",  $0}
+      NR==p99  {printf "%1.4f,",  $0}
+      NR==p100 {printf "%1.4f\n", $0}
+    ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if requires
+echo "[DONE]"

runtime/experiments/deadline/fix_calcs2.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+host=192.168.1.13
+# host=localhost
+# timestamp=$(date +%s)
+timestamp=1606697099
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+inputs=(40 10)
+duration_sec=60
+offset=5
+
+# Execute workloads long enough for runtime to learn excepted execution time
+# echo -n "Running Samples: "
+# for input in ${inputs[*]}; do
+#   hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input))
+# done
+# echo "[DONE]"
+# sleep 5
+
+# echo "Running Experiments"
+
+# # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run
+# hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" &
+# sleep $offset
+# hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" &
+# sleep $((duration_sec + offset + 15))
+# sleep 30
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
+
+deadlines_ms=(20 20000)
+payloads=(fib10-con fib40-con)
+durations_s=(60 70)
+
+for ((i = 0; i < 2; i++)); do
+  payload=${payloads[$i]}
+  deadline=${deadlines_ms[$i]}
+  duration=${durations_s[$i]}
+
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  # Calculate Success Rate for csv
+  awk -F, '
+      $7 == 200 {denom++}
+      $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++}
+      END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)}
+    ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  #   duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+      BEGIN {
+        sum = 0
+        p50 = int('"$oks"' * 0.5)
+        p90 = int('"$oks"' * 0.9)
+        p99 = int('"$oks"' * 0.99)  
+        p100 = '"$oks"'
+        printf "'"$payload"',"
+      }
+      NR==p50  {printf "%1.4f,",  $0}
+      NR==p90  {printf "%1.4f,",  $0}
+      NR==p99  {printf "%1.4f,",  $0}
+      NR==p100 {printf "%1.4f\n", $0}
+    ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if requires
+echo "[DONE]"

runtime/experiments/deadline/server.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+source ../common.sh
+
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+# Start the runtime
+
+PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json"

runtime/experiments/deadline/spec.json

@@ -3,68 +3,9 @@
   "name": "fibonacci_10",
   "path": "fibonacci_wasm.so",
   "port": 10010,
-  "expected-execution-us": 600,
-  "relative-deadline-us": 2000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_20",
-  "path": "fibonacci_wasm.so",
-  "port": 10020,
-  "expected-execution-us": 900,
-  "relative-deadline-us": 5000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_25",
-  "path": "fibonacci_wasm.so",
-  "port": 10025,
-  "expected-execution-us": 90000,
-  "relative-deadline-us": 200000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_30",
-  "path": "fibonacci_wasm.so",
-  "port": 10030,
-  "expected-execution-us": 9000,
-  "relative-deadline-us": 80000,
-  "argsize": 1,
-  "http-req-headers": [],
-  "http-req-content-type": "text/plain",
-  "http-req-size": 1024,
-  "http-resp-headers": [],
-  "http-resp-size": 1024,
-  "http-resp-content-type": "text/plain"
-},
-{
-  "active": "yes",
-  "name": "fibonacci_35",
-  "path": "fibonacci_wasm.so",
-  "port": 10035,
-  "expected-execution-us": 9000,
-  "relative-deadline-us": 53000,
+  "expected-execution-us": 6000,
+  "admissions-percentile": 70,
+  "relative-deadline-us": 20000,
   "argsize": 1,
   "http-req-headers": [],
   "http-req-content-type": "text/plain",
@@ -78,8 +19,9 @@
   "name": "fibonacci_40",
   "path": "fibonacci_wasm.so",
   "port": 10040,
-  "expected-execution-us": 550000,
-  "relative-deadline-us": 300000000,
+  "expected-execution-us": 10000000,
+  "admissions-percentile": 70,
+  "relative-deadline-us": 20000000,
   "argsize": 1,
   "http-req-headers": [],
   "http-req-content-type": "text/plain",

runtime/experiments/preemption/backend.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Use -d flag if running under gdb
+
+timestamp=$(date +%s)
+experiment_directory=$(pwd)
+binary_directory=$(cd ../../bin && pwd)
+
+results_directory="$experiment_directory/res/$timestamp/$scheduler"
+log=log.txt
+
+mkdir -p "$results_directory"
+log_environment >>"$results_directory/$log"
+
+# Start the runtime
+PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" | tee -a "$results_directory/$log"

runtime/experiments/preemption/client.sh

@@ -9,34 +9,38 @@ experiment_directory=$(pwd)
 host=192.168.1.13
 
 results_directory="$experiment_directory/res/$timestamp"
-log=log.txt
 
 mkdir -p "$results_directory"
-log_environment >>"$results_directory/$log"
 
 # Start the runtime
 inputs=(40 10)
-duration_sec=15
+duration_sec=30
 offset=5
 
 # Execute workloads long enough for runtime to learn excepted execution time
 echo -n "Running Samples: "
 for input in ${inputs[*]}; do
-  hey -z ${duration_sec}s -cpus 6 -t 0 -o csv -m GET -d "$input\n" http://"$host":$((10000 + input))
+  hey -n 45 -c 4 -t 0 -o csv -m GET -d "$input\n" http://"$host":$((10000 + input))
 done
 echo "[DONE]"
-sleep 5
+sleep 30
 
 echo "Running Experiments"
 # Run each separately
-hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" >"$results_directory/fib40.csv"
 hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$host:10010" >"$results_directory/fib10.csv"
+echo "fib(10) Complete"
+sleep 60
+
+hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" >"$results_directory/fib40.csv"
+echo "fib(40) Complete"
+sleep 120
 
 # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run
 hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" >"$results_directory/fib40-con.csv" &
 sleep $offset
 hey -z ${duration_sec}s -cpus 3 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$host:10010" >"$results_directory/fib10-con.csv" &
 sleep $((duration_sec + offset + 15))
+echo "fib(10) & fib(40) Complete"
 
 # Generate *.csv and *.dat results
 echo -n "Parsing Results: "
@@ -45,6 +49,7 @@ printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
 printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
 printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv"
 
+durations_s=(15 15 15 25)
 payloads=(fib10 fib10-con fib40 fib40-con)
 
 for payload in ${payloads[*]}; do
@@ -52,6 +57,8 @@ for payload in ${payloads[*]}; do
   requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
   ((requests == 0)) && continue
 
+  duration=${durations_s[$i]}
+
   # Calculate Success Rate for csv
   awk -F, '
     $7 == 200 {ok++}
@@ -67,7 +74,7 @@ for payload in ${payloads[*]}; do
   ((oks == 0)) && continue # If all errors, skip line
 
   # Get Latest Timestamp
-  duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
   throughput=$(echo "$oks/$duration" | bc)
   printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
 

runtime/experiments/preemption/debug.sh

@@ -10,6 +10,7 @@ binary_directory=$(cd "$project_directory"/bin && pwd)
 
 export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
 export PATH="$binary_directory:$PATH"
+export SLEDGE_SCHEDULER="EDF"
 
 gdb --eval-command="handle SIGUSR1 nostop" \
   --eval-command="handle SIGPIPE nostop" \

runtime/experiments/preemption/demo.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-cd ../../bin
-LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./sledgert ../tests/mixed_preemption/test_mixed_preemption.json &
-cd ../tests/mixed_preemption/
-
-# Run small samples on each port to let the runtime figure out the execution time
-sleep 10
-echo "Running Samples"
-wrk -d 20s -t1 -s post.lua http://localhost:10010 -- --delay 500 10\n
-wrk -d 20s -t1 -s post.lua http://localhost:10020 -- --delay 500 20\n
-wrk -d 20s -t1 -s post.lua http://localhost:10030 -- --delay 500 25\n
-
-# Run in Parallel
-sleep 10
-echo "Running Experiments"
-wrk -d 1m -t1 -s post.lua http://localhost:10010 -- --delay 125 10\n >./res/fib10.txt &
-wrk -d 2m -t1 -s post.lua http://localhost:10020 -- --delay 250 20\n >./res/fib20.txt &
-wrk -d 3m -t1 -s post.lua http://localhost:10025 -- --delay 500 25\n >./res/fib25.txt
-
-# Kill the Background Sledge processes
-sleep 10
-echo "Running Cleanup"
-pkill sledgert
-pkill wrk
-
-# Extract the Latency CSV Data from the Log
-
-echo 'Fib10, Fib10' >./res/fib10.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib10.txt >>./res/fib10.csv
-echo 'Fib20, Fib20' >./res/fib20.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib20.txt >>./res/fib20.csv
-echo 'Fib25, Fib25' >./res/fib25.csv
-grep -A200 -m1 -e 'Percentile, Latency' ./res/fib25.txt >>./res/fib25.csv
-paste -d, ./res/fib10.csv ./res/fib20.csv ./res/fib25.csv >./res/merged.csv

runtime/experiments/preemption/fix_results.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+source ../common.sh
+
+# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
+# Modified to target a remote host
+
+timestamp=1606608313-FIFO
+experiment_directory=$(pwd)
+results_directory="$experiment_directory/res/$timestamp"
+
+# Generate *.csv and *.dat results
+echo -n "Parsing Results: "
+
+printf "Payload,Success_Rate\n" >>"$results_directory/success.csv"
+printf "Payload,Throughput\n" >>"$results_directory/throughput.csv"
+printf "Payload,p50,p90,p99,p998,p999,p100\n" >>"$results_directory/latency.csv"
+
+durations_s=(15 15 15 25)
+payloads=(fib10 fib10-con fib40 fib40-con)
+
+for payload in ${payloads[*]}; do
+  # Get Number of Requests
+  requests=$(($(wc -l <"$results_directory/$payload.csv") - 1))
+  ((requests == 0)) && continue
+
+  duration=${durations_s[$i]}
+
+  # Calculate Success Rate for csv
+  awk -F, '
+    $7 == 200 {ok++}
+    END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)}
+' <"$results_directory/$payload.csv" >>"$results_directory/success.csv"
+
+  # Filter on 200s, convery from s to ms, and sort
+  awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" |
+    sort -g >"$results_directory/$payload-response.csv"
+
+  # Get Number of 200s
+  oks=$(wc -l <"$results_directory/$payload-response.csv")
+  ((oks == 0)) && continue # If all errors, skip line
+
+  # Get Latest Timestamp
+  # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8)
+  throughput=$(echo "$oks/$duration" | bc)
+  printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv"
+
+  # Generate Latency Data for csv
+  awk '
+    BEGIN {
+    sum = 0
+    p50 = int('"$oks"' * 0.5)
+    p90 = int('"$oks"' * 0.9)
+    p99 = int('"$oks"' * 0.99)
+    p998 = int('"$oks"' * 0.998)
+    p999 = int('"$oks"' * 0.999)
+    p100 = '"$oks"'
+    printf "'"$payload"',"
+    }
+    NR==p50  {printf "%1.4f,",  $0}
+    NR==p90  {printf "%1.4f,",  $0}
+    NR==p99  {printf "%1.4f,",  $0}
+    NR==p998 {printf "%1.4f,",  $0}
+    NR==p999 {printf "%1.4f,",  $0}
+    NR==p100 {printf "%1.4f\n", $0}
+' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv"
+
+  # Delete scratch file used for sorting/counting
+  # rm -rf "$results_directory/$payload-response.csv"
+done
+
+# Transform csvs to dat files for gnuplot
+for file in success latency throughput; do
+  echo -n "#" >"$results_directory/$file.dat"
+  tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat"
+done
+
+# Generate gnuplots. Commented out because we don't have *.gnuplots defined
+# generate_gnuplots
+
+# Cleanup, if required
+echo "[DONE]"

runtime/include/admissions_control.h

@@ -22,13 +22,15 @@
  */
 extern _Atomic uint64_t admissions_control_admitted;
 extern uint64_t         admissions_control_capacity;
+extern const double     admissions_control_overhead;
 
 static inline void
 admissions_control_initialize()
 {
 #ifdef ADMISSIONS_CONTROL
 	atomic_init(&admissions_control_admitted, 0);
-	admissions_control_capacity = runtime_worker_threads_count * ADMISSIONS_CONTROL_GRANULARITY;
+	admissions_control_capacity = runtime_worker_threads_count * ADMISSIONS_CONTROL_GRANULARITY
+	                              * ((double)1.0 - admissions_control_overhead);
 #endif
 }
 

runtime/include/admissions_info.h

@@ -32,6 +32,10 @@ admissions_info_initialize(struct admissions_info *self, int percentile, uint64_
 	self->percentile = percentile;
 
 	self->control_index = PERF_WINDOW_BUFFER_SIZE * percentile / 100;
+#ifdef LOG_ADMISSIONS_CONTROL
+	debuglog("Percentile: %d\n", self->percentile);
+	debuglog("Control Index: %d\n", self->control_index);
+#endif
 #endif
 }
 

runtime/include/arch/aarch64/context.h

@@ -1,6 +1,8 @@
 #pragma once
 
+#include <assert.h>
 #include "arch/common.h"
+#include "current_sandbox.h"
 
 #define ARCH_SIG_JMP_OFF 0x100 /* Based on code generated! */
 
@@ -16,6 +18,7 @@ static inline void
 arch_context_init(struct arch_context *actx, reg_t ip, reg_t sp)
 {
 	assert(actx != NULL);
+	assert(!software_interrupt_is_enabled());
 
 	if (ip == 0 && sp == 0) {
 		actx->variant = ARCH_CONTEXT_VARIANT_UNUSED;
@@ -68,6 +71,17 @@ arch_context_switch(struct arch_context *a, struct arch_context *b)
 	/* Assumption: Software Interrupts are disabled by caller */
 	assert(!software_interrupt_is_enabled());
 
+#ifndef NDEBUG
+	/*
+	 * Assumption: In the case of a slow context switch, the caller
+	 * set current_sandbox to the sandbox containing the target context
+	 */
+	if (b->variant == ARCH_CONTEXT_VARIANT_SLOW) {
+		struct sandbox *current = current_sandbox_get();
+		assert(current != NULL && b == &current->ctxt);
+	}
+#endif
+
 	/* if both a and b are NULL, there is no state change */
 	assert(a != NULL || b != NULL);
 

runtime/include/arch/common.h

@@ -49,7 +49,7 @@ arch_context_variant_print(arch_context_variant_t context)
 	case ARCH_CONTEXT_VARIANT_RUNNING:
 		return "Running";
 	default:
-		panic("Encountered unexpected arch_context variant\n");
+		panic("Encountered unexpected arch_context variant: %u\n", context);
 	}
 }
 

runtime/include/arch/x86_64/context.h

@@ -32,13 +32,15 @@ static void __attribute__((noinline)) arch_context_init(struct arch_context *act
 		 */
 		asm volatile("movq %%rsp, %%rbx\n\t" /* Temporarily save pointer of active stack to B */
 		             "movq %%rax, %%rsp\n\t" /* Set active stack to stack pointer in A(C variable sp) */
-		             "pushq %%rax\n\t"       /* Push A(C variable sp) onto the stack at sp */
+		             "pushq %%rax\n\t"       /* Push A register (C variable sp) onto the stack at sp */
 		             "movq %%rsp, %%rax\n\t" /* Write the incremented stack pointer to A(C variable sp) */
 		             "movq %%rbx, %%rsp\n\t" /* Restore original stack saved in B */
 		             : "=a"(sp)
 		             : "a"(sp)
 		             : "memory", "cc", "rbx");
 	}
+	// FIXME: Is the klobber list correct? Issue #129
+	//   : "memory", "cc", "rbx", "rsi", "rdi");
 
 	actx->regs[UREG_SP] = sp;
 	actx->regs[UREG_IP] = ip;
@@ -71,6 +73,33 @@ arch_context_restore(mcontext_t *active_context, struct arch_context *sandbox_co
 	active_context->gregs[REG_RIP] = sandbox_context->regs[UREG_IP] + ARCH_SIG_JMP_OFF;
 }
 
+/**
+ * Load a new sandbox that preempted an existing sandbox, restoring only the
+ * instruction pointer and stack pointer registers.
+ * I am unclear about setting the BP. Issue #131
+ * @param active_context - the context of the current worker thread
+ * @param sandbox_context - the context that we want to restore
+ */
+static inline void
+arch_context_restore_new(mcontext_t *active_context, struct arch_context *sandbox_context)
+{
+	assert(active_context != NULL);
+	assert(sandbox_context != NULL);
+
+	/* Assumption: Base Context is only ever used by arch_context_switch */
+	assert(sandbox_context != &worker_thread_base_context);
+
+	assert(sandbox_context->regs[UREG_SP]);
+	assert(sandbox_context->regs[UREG_IP]);
+
+	/* Transitioning from Fast -> Running */
+	assert(sandbox_context->variant == ARCH_CONTEXT_VARIANT_FAST);
+	sandbox_context->variant = ARCH_CONTEXT_VARIANT_RUNNING;
+
+	active_context->gregs[REG_RSP] = sandbox_context->regs[UREG_SP];
+	active_context->gregs[REG_RIP] = sandbox_context->regs[UREG_IP];
+}
+
 /**
  * @param a - the registers and context of the thing running
  * @param b - the registers and context of what we're switching to

runtime/include/panic.h

@@ -5,6 +5,7 @@
 #include <sched.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 #define panic(fmt, ...)                                                                                           \
 	{                                                                                                         \

runtime/src/admissions_control.c

@@ -2,3 +2,5 @@
 
 _Atomic uint64_t admissions_control_admitted;
 uint64_t         admissions_control_capacity;
+
+const double admissions_control_overhead = 0.2;

runtime/src/global_request_scheduler_minheap.c

@@ -76,7 +76,7 @@ sandbox_request_get_priority_fn(void *element)
 void
 global_request_scheduler_minheap_initialize()
 {
-	global_request_scheduler_minheap = priority_queue_initialize(1000, true, sandbox_request_get_priority_fn);
+	global_request_scheduler_minheap = priority_queue_initialize(4096, true, sandbox_request_get_priority_fn);
 
 	struct global_request_scheduler_config config = {
 		.add_fn               = global_request_scheduler_minheap_add,

runtime/src/local_runqueue_minheap.c

@@ -185,7 +185,7 @@ local_runqueue_minheap_preempt(ucontext_t *user_context)
 		 * user-level context switch state, so do not enable software interrupts.
 		 * TODO: Review the interrupt logic here. Issue #63
 		 */
-		arch_context_restore(&user_context->uc_mcontext, &next_sandbox->ctxt);
+		arch_context_restore_new(&user_context->uc_mcontext, &next_sandbox->ctxt);
 		should_enable_software_interrupt = false;
 	}
 done:

runtime/src/sandbox.c

@@ -373,7 +373,6 @@ sandbox_allocate_memory(struct module *module)
 		error_message = "sandbox_allocate_memory - memory allocation failed";
 		goto alloc_failed;
 	}
-	sandbox = (struct sandbox *)addr;
 
 	/* Set the struct sandbox, HTTP Req/Resp buffer, and the initial Wasm Pages as read/write */
 	errno         = 0;
@@ -384,6 +383,8 @@ sandbox_allocate_memory(struct module *module)
 		goto set_rw_failed;
 	}
 
+	sandbox = (struct sandbox *)addr_rw;
+
 	/* Populate Sandbox members */
 	sandbox->state                  = SANDBOX_UNINITIALIZED;
 	sandbox->linear_memory_start    = (char *)addr + sandbox_size;
@@ -411,12 +412,23 @@ sandbox_allocate_stack(struct sandbox *sandbox)
 {
 	assert(sandbox);
 	assert(sandbox->module);
+	assert(!software_interrupt_is_enabled());
 
-	errno                = 0;
+	errno      = 0;
+	char *addr = mmap(NULL, sandbox->module->stack_size + /* guard page */ PAGE_SIZE, PROT_NONE,
+	                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (addr == MAP_FAILED) goto err_stack_allocation_failed;
+
+	/* Set the struct sandbox, HTTP Req/Resp buffer, and the initial Wasm Pages as read/write */
+	errno         = 0;
+	char *addr_rw = mmap(addr + /* guard page */ PAGE_SIZE, sandbox->module->stack_size, PROT_READ | PROT_WRITE,
+	                     MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+
+	/* TODO: Fix leak here. Issue #132 */
+	if (addr_rw == MAP_FAILED) goto err_stack_allocation_failed;
+
+	sandbox->stack_start = addr_rw;
 	sandbox->stack_size  = sandbox->module->stack_size;
-	sandbox->stack_start = mmap(NULL, sandbox->stack_size, PROT_READ | PROT_WRITE,
-	                            MAP_PRIVATE | MAP_ANONYMOUS | MAP_GROWSDOWN, -1, 0);
-	if (sandbox->stack_start == MAP_FAILED) goto err_stack_allocation_failed;
 
 done:
 	return 0;
@@ -450,8 +462,9 @@ sandbox_set_as_initialized(struct sandbox *sandbox, struct sandbox_request *sand
 	sandbox->state                     = SANDBOX_SET_AS_INITIALIZED;
 
 	/* Initialize the sandbox's context, stack, and instruction pointer */
+	/* stack_start points to the bottom of the usable stack, so add stack_size to get to top */
 	arch_context_init(&sandbox->ctxt, (reg_t)current_sandbox_main,
-	                  (reg_t)(sandbox->stack_start + sandbox->stack_size));
+	                  (reg_t)sandbox->stack_start + sandbox->stack_size);
 
 	/* Initialize file descriptors to -1 */
 	for (int i = 0; i < SANDBOX_MAX_IO_HANDLE_COUNT; i++) sandbox->io_handles[i].file_descriptor = -1;
@@ -875,13 +888,11 @@ sandbox_free(struct sandbox *sandbox)
 
 	module_release(sandbox->module);
 
-	void * stkaddr = sandbox->stack_start;
-	size_t stksz   = sandbox->stack_size;
-
-
 	/* Free Sandbox Stack */
 	errno = 0;
-	rc    = munmap(stkaddr, stksz);
+
+	/* The stack start is the bottom of the usable stack, but we allocated a guard page below this */
+	rc = munmap((char *)sandbox->stack_start - PAGE_SIZE, sandbox->stack_size + PAGE_SIZE);
 	if (rc == -1) {
 		debuglog("Failed to unmap stack of Sandbox %lu\n", sandbox->id);
 		goto err_free_stack_failed;

runtime/src/software_interrupt.c

@@ -129,6 +129,8 @@ sigusr1_handler(siginfo_t *signal_info, ucontext_t *user_context, struct sandbox
 
 #ifdef LOG_PREEMPTION
 	debuglog("Total SIGUSR1 Received: %d\n", software_interrupt_SIGUSR_count);
+	debuglog("Restoring sandbox: %lu, Stack %llu\n", current_sandbox->id,
+	         current_sandbox->ctxt.mctx.gregs[REG_RSP]);
 #endif
 
 	arch_mcontext_restore(&user_context->uc_mcontext, &current_sandbox->ctxt);
@@ -176,7 +178,7 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void
 	case SIGUSR1: {
 		return sigusr1_handler(signal_info, user_context, current_sandbox);
 	}
-	default:
+	default: {
 		if (signal_info->si_code == SI_TKILL) {
 			panic("Unexpectedly received signal %d from a thread kill, but we have no handler\n",
 			      signal_type);
@@ -184,6 +186,7 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void
 			panic("Unexpectedly received signal %d from the kernel, but we have no handler\n", signal_type);
 		}
 	}
+	}
 #endif
 }
 

runtime/src/worker_thread.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
@@ -102,24 +103,34 @@ worker_thread_switch_to_sandbox(struct sandbox *next_sandbox)
 		sandbox_set_as_running(next_sandbox, next_sandbox->state);
 
 #ifdef LOG_CONTEXT_SWITCHES
-		debuglog("Base Context (%s) > Sandbox %lu (%s)\n",
-		         arch_context_variant_print(worker_thread_base_context.variant), next_sandbox->id,
+		debuglog("Base Context (@%p) (%s) > Sandbox %lu (@%p) (%s)\n", &worker_thread_base_context,
+		         arch_context_variant_print(worker_thread_base_context.variant), next_sandbox->id, next_context,
 		         arch_context_variant_print(next_context->variant));
 #endif
+		/* Assumption: If a slow context switch, current sandbox should be set to the target */
+		assert(next_context->variant != ARCH_CONTEXT_VARIANT_SLOW
+		       || &current_sandbox_get()->ctxt == next_context);
 
 		arch_context_switch(NULL, next_context);
 	} else {
 		/* Set the current sandbox to the next */
 		assert(next_sandbox != current_sandbox);
 
+		struct arch_context *current_context = &current_sandbox->ctxt;
+
+#ifdef LOG_CONTEXT_SWITCHES
+		debuglog("Sandbox %lu (@%p) (%s) > Sandbox %lu (@%p) (%s)\n", current_sandbox->id,
+		         &current_sandbox->ctxt, arch_context_variant_print(current_sandbox->ctxt.variant),
+		         next_sandbox->id, &next_sandbox->ctxt, arch_context_variant_print(next_context->variant));
+#endif
+
 		worker_thread_transition_exiting_sandbox(current_sandbox);
 
 		sandbox_set_as_running(next_sandbox, next_sandbox->state);
 
-		struct arch_context *current_context = &current_sandbox->ctxt;
-
-#ifdef LOG_CONTEXT_SWITCHES
-		debuglog("Sandbox %lu > Sandbox %lu\n", current_sandbox->id, next_sandbox->id);
+#ifndef NDEBUG
+		assert(next_context->variant != ARCH_CONTEXT_VARIANT_SLOW
+		       || &current_sandbox_get()->ctxt == next_context);
 #endif
 
 		/* Switch to the associated context. */
@@ -146,22 +157,21 @@ worker_thread_switch_to_base_context()
 	}
 #endif
 
-	worker_thread_transition_exiting_sandbox(current_sandbox);
-
 	/* Assumption: Base Context should never switch to Base Context */
 	assert(current_sandbox != NULL);
-	assert(&current_sandbox->ctxt != &worker_thread_base_context);
-
-	current_sandbox_set(NULL);
+	struct arch_context *current_context = &current_sandbox->ctxt;
+	assert(current_context != &worker_thread_base_context);
 
 #ifdef LOG_CONTEXT_SWITCHES
-	debuglog("Sandbox %lu (%s) > Base Context (%s)\n", current_sandbox->id,
-	         arch_context_variant_print(current_sandbox->ctxt.variant),
+	debuglog("Sandbox %lu (@%p) (%s)> Base Context (@%p) (%s)\n", current_sandbox->id, current_context,
+	         arch_context_variant_print(current_sandbox->ctxt.variant), &worker_thread_base_context,
 	         arch_context_variant_print(worker_thread_base_context.variant));
 #endif
 
-	arch_context_switch(&current_sandbox->ctxt, &worker_thread_base_context);
-
+	worker_thread_transition_exiting_sandbox(current_sandbox);
+	current_sandbox_set(NULL);
+	assert(worker_thread_base_context.variant == ARCH_CONTEXT_VARIANT_FAST);
+	arch_context_switch(current_context, &worker_thread_base_context);
 	software_interrupt_enable();
 }
 
@@ -234,7 +244,7 @@ worker_thread_execute_epoll_loop(void)
 
 				if (sandbox->state == SANDBOX_BLOCKED) worker_thread_wakeup_sandbox(sandbox);
 			} else if (epoll_events[i].events & (EPOLLERR | EPOLLHUP)) {
-				/* Mystery: This seems to never fire. Why? */
+				/* Mystery: This seems to never fire. Why? Issue #130 */
 
 				/* Close socket and set as error on socket error or unexpected client hangup */
 				struct sandbox *sandbox = (struct sandbox *)epoll_events[i].data.ptr;
@@ -282,7 +292,9 @@ worker_thread_main(void *return_code)
 	worker_thread_start_timestamp = __getcycles();
 	worker_thread_lock_duration   = 0;
 
-	/* Initialize Base Context */
+	/* Initialize Base Context as unused
+	 * The SP and IP are populated during the first FAST switch away
+	 */
 	arch_context_init(&worker_thread_base_context, 0, 0);
 
 	/* Initialize Runqueue Variant */