diff --git a/runtime/Makefile b/runtime/Makefile index 0e23dfb..f9244c4 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -56,7 +56,10 @@ USE_MEM = USE_MEM_VM CFLAGS += -D${ARCH} CFLAGS += -DNCORES=${TOTAL_CORES} CFLAGS += -DPAGE_SIZE=$(PAGE_SIZE) -#CFLAGS += -DPREEMPT_DISABLE + +# Optionally Disable preemption +# CFLAGS += -DPREEMPT_DISABLE + CFLAGS += -D${USE_MEM} # Preprocessor diff --git a/runtime/experiments/applications/fivebyeight/run.sh b/runtime/experiments/applications/fivebyeight/run.sh index b702c23..3271539 100755 --- a/runtime/experiments/applications/fivebyeight/run.sh +++ b/runtime/experiments/applications/fivebyeight/run.sh @@ -42,6 +42,5 @@ if [ "$1" != "-d" ]; then sleep 5 echo -n "Running Cleanup: " pkill sledgert >/dev/null 2>/dev/null - pkill wrk >/dev/null 2>/dev/null echo "[DONE]" fi diff --git a/runtime/experiments/applications/handwriting/run.sh b/runtime/experiments/applications/handwriting/run.sh index 82fedd8..f6d2287 100755 --- a/runtime/experiments/applications/handwriting/run.sh +++ b/runtime/experiments/applications/handwriting/run.sh @@ -42,6 +42,5 @@ if [ "$1" != "-d" ]; then sleep 5 echo -n "Running Cleanup: " pkill sledgert >/dev/null 2>/dev/null - pkill wrk >/dev/null 2>/dev/null echo "[DONE]" fi diff --git a/runtime/experiments/applications/hyde/run.sh b/runtime/experiments/applications/hyde/run.sh index e014e9c..32a6d6d 100755 --- a/runtime/experiments/applications/hyde/run.sh +++ b/runtime/experiments/applications/hyde/run.sh @@ -41,6 +41,5 @@ if [ "$1" != "-d" ]; then sleep 5 echo -n "Running Cleanup: " pkill sledgert >/dev/null 2>/dev/null - pkill wrk >/dev/null 2>/dev/null echo "[DONE]" fi diff --git a/runtime/experiments/deadline/client.sh b/runtime/experiments/deadline/client.sh new file mode 100755 index 0000000..2449ce3 --- /dev/null +++ b/runtime/experiments/deadline/client.sh @@ -0,0 +1,106 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +host=localhost +timestamp=$(date +%s) +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) + +results_directory="$experiment_directory/res/$timestamp" +log=log.txt + +mkdir -p "$results_directory" +log_environment >>"$results_directory/$log" + +inputs=(40 10) +duration_sec=60 +offset=5 + +# Execute workloads long enough for runtime to learn excepted execution time +echo -n "Running Samples: " +for input in ${inputs[*]}; do + hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input)) +done +echo "[DONE]" +sleep 5 + +echo "Running Experiments" + +# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run +hey -n 1000 -c 1000 -cpus 6 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" +# sleep $offset +# hey -n 25000 -c 1000000 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" & +# sleep $((duration_sec + offset + 45)) + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" + +deadlines_ms=(20 20000) +# durations_s=(60 70) +payloads=(fib10-con fib40-con) + +for ((i = 1; i < 2; i++)); do + payload=${payloads[$i]} + deadline=${deadlines_ms[$i]} + # duration=${durations_s[$i]} + + # Get Number of Requests + requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) + ((requests == 0)) && continue + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} + END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Get Latest Timestamp + # throughput=$(echo "$oks/$duration" | bc) + # printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots. Commented out because we don't have *.gnuplots defined +# generate_gnuplots + +# Cleanup, if requires +echo "[DONE]" diff --git a/runtime/experiments/deadline/client2.sh b/runtime/experiments/deadline/client2.sh new file mode 100755 index 0000000..e5909ea --- /dev/null +++ b/runtime/experiments/deadline/client2.sh @@ -0,0 +1,110 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +host=192.168.1.13 +# host=localhost +timestamp=$(date +%s) +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) + +results_directory="$experiment_directory/res/$timestamp" +log=log.txt + +mkdir -p "$results_directory" +log_environment >>"$results_directory/$log" + +inputs=(40 10) +duration_sec=30 +offset=5 + +# Execute workloads long enough for runtime to learn excepted execution time +echo -n "Running Samples: " +for input in ${inputs[*]}; do + hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input)) +done +echo "[DONE]" +sleep 5 + +echo "Running Experiments" + +# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run +hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" & +sleep $offset +hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" & +sleep $((duration_sec + offset + 15)) +sleep 30 + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" + +deadlines_ms=(20 20000) +payloads=(fib10-con fib40-con) +durations_s=(30 40) + +for ((i = 0; i < 2; i++)); do + payload=${payloads[$i]} + deadline=${deadlines_ms[$i]} + duration=${durations_s[$i]} + + # Get Number of Requests + requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) + ((requests == 0)) && continue + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 {denom++} + $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} + END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)} + ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Get Latest Timestamp + duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + throughput=$(echo "$oks/$duration" | bc) + printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency throughput; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots. Commented out because we don't have *.gnuplots defined +# generate_gnuplots + +# Cleanup, if requires +echo "[DONE]" diff --git a/runtime/experiments/deadline/client3.sh b/runtime/experiments/deadline/client3.sh new file mode 100755 index 0000000..391c53f --- /dev/null +++ b/runtime/experiments/deadline/client3.sh @@ -0,0 +1,108 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +host=192.168.1.13 +# host=localhost +timestamp=$(date +%s) +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) + +results_directory="$experiment_directory/res/$timestamp" +log=log.txt + +mkdir -p "$results_directory" +log_environment >>"$results_directory/$log" + +inputs=(10) +duration_sec=30 +offset=5 + +# Execute workloads long enough for runtime to learn excepted execution time +echo -n "Running Samples: " +hey -n 16 -c 4 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 +echo "[DONE]" +sleep 5 + +echo "Running Experiments" + +# Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run +# hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" & +# sleep $offset +hey -z ${duration_sec}s -cpus 6 -c 400 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" +# sleep $((duration_sec + offset + 15)) +# sleep 30 + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" + +deadlines_ms=(20 20000) +payloads=(fib10-con fib40-con) +durations_s=(30 40) + +for ((i = 0; i < 1; i++)); do + payload=${payloads[$i]} + deadline=${deadlines_ms[$i]} + duration=${durations_s[$i]} + + # Get Number of Requests + requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) + ((requests == 0)) && continue + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 {denom++} + $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} + END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)} + ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Get Latest Timestamp + duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + throughput=$(echo "$oks/$duration" | bc) + printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency throughput; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots. Commented out because we don't have *.gnuplots defined +# generate_gnuplots + +# Cleanup, if requires +echo "[DONE]" diff --git a/runtime/experiments/deadline/demo.sh b/runtime/experiments/deadline/demo.sh deleted file mode 100755 index 3fdb1e9..0000000 --- a/runtime/experiments/deadline/demo.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -cd ../../bin -LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./sledgert ../tests/mixed_preemption/test_mixed_preemption.json & -cd ../tests/mixed_preemption/ - -# Run small samples on each port to let the runtime figure out the execution time -sleep 10 -echo "Running Samples" -wrk -d 20s -t1 -s post.lua http://localhost:10010 -- --delay 500 10\n -wrk -d 20s -t1 -s post.lua http://localhost:10020 -- --delay 500 20\n -wrk -d 20s -t1 -s post.lua http://localhost:10030 -- --delay 500 25\n - -# Run in Parallel -sleep 10 -echo "Running Experiments" -wrk -d 1m -t1 -s post.lua http://localhost:10010 -- --delay 125 10\n >./res/fib10.txt & -wrk -d 2m -t1 -s post.lua http://localhost:10020 -- --delay 250 20\n >./res/fib20.txt & -wrk -d 3m -t1 -s post.lua http://localhost:10025 -- --delay 500 25\n >./res/fib25.txt - -# Kill the Background Sledge processes -sleep 10 -echo "Running Cleanup" -pkill sledgert -pkill wrk - -# Extract the Latency CSV Data from the Log - -echo 'Fib10, Fib10' >./res/fib10.csv -grep -A200 -m1 -e 'Percentile, Latency' ./res/fib10.txt >>./res/fib10.csv -echo 'Fib20, Fib20' >./res/fib20.csv -grep -A200 -m1 -e 'Percentile, Latency' ./res/fib20.txt >>./res/fib20.csv -echo 'Fib25, Fib25' >./res/fib25.csv -grep -A200 -m1 -e 'Percentile, Latency' ./res/fib25.txt >>./res/fib25.csv -paste -d, ./res/fib10.csv ./res/fib20.csv ./res/fib25.csv >./res/merged.csv diff --git a/runtime/experiments/deadline/fix_calcs.sh b/runtime/experiments/deadline/fix_calcs.sh new file mode 100755 index 0000000..cf13246 --- /dev/null +++ b/runtime/experiments/deadline/fix_calcs.sh @@ -0,0 +1,77 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +experiment_directory=$(pwd) +results_directory="$experiment_directory/res/1606615320-fifo-adm" + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" + +deadlines_ms=(20 20000) +payloads=(fib10-con fib40-con) + +for ((i = 0; i < 2; i++)); do + payload=${payloads[$i]} + deadline=${deadlines_ms[$i]} + + # Get Number of Requests + requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) + ((requests == 0)) && continue + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} + END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} + ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Get Latest Timestamp + duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + throughput=$(echo "$oks/$duration" | bc) + printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency throughput; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots. Commented out because we don't have *.gnuplots defined +# generate_gnuplots + +# Cleanup, if requires +echo "[DONE]" diff --git a/runtime/experiments/deadline/fix_calcs2.sh b/runtime/experiments/deadline/fix_calcs2.sh new file mode 100755 index 0000000..86f0f13 --- /dev/null +++ b/runtime/experiments/deadline/fix_calcs2.sh @@ -0,0 +1,111 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +host=192.168.1.13 +# host=localhost +# timestamp=$(date +%s) +timestamp=1606697099 +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) + +results_directory="$experiment_directory/res/$timestamp" +log=log.txt + +mkdir -p "$results_directory" +log_environment >>"$results_directory/$log" + +inputs=(40 10) +duration_sec=60 +offset=5 + +# Execute workloads long enough for runtime to learn excepted execution time +# echo -n "Running Samples: " +# for input in ${inputs[*]}; do +# hey -n 16 -c 4 -t 0 -o csv -m GET -d "$input\n" http://${host}:$((10000 + input)) +# done +# echo "[DONE]" +# sleep 5 + +# echo "Running Experiments" + +# # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run +# hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 200 -t 0 -o csv -m GET -d "40\n" http://${host}:10040 >"$results_directory/fib40-con.csv" & +# sleep $offset +# hey -z ${duration_sec}s -cpus 3 -c 200 -t 0 -o csv -m GET -d "10\n" http://${host}:10010 >"$results_directory/fib10-con.csv" & +# sleep $((duration_sec + offset + 15)) +# sleep 30 + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" + +deadlines_ms=(20 20000) +payloads=(fib10-con fib40-con) +durations_s=(60 70) + +for ((i = 0; i < 2; i++)); do + payload=${payloads[$i]} + deadline=${deadlines_ms[$i]} + duration=${durations_s[$i]} + + # Get Number of Requests + requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) + ((requests == 0)) && continue + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 {denom++} + $7 == 200 && ($1 * 1000) <= '"$deadline"' {ok++} + END{printf "'"$payload"',%3.5f%\n", (ok / denom * 100)} + ' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Get Latest Timestamp + # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + throughput=$(echo "$oks/$duration" | bc) + printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} + ' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency throughput; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots. Commented out because we don't have *.gnuplots defined +# generate_gnuplots + +# Cleanup, if requires +echo "[DONE]" diff --git a/runtime/experiments/deadline/server.sh b/runtime/experiments/deadline/server.sh new file mode 100755 index 0000000..e1e2e5b --- /dev/null +++ b/runtime/experiments/deadline/server.sh @@ -0,0 +1,10 @@ +#!/bin/bash +source ../common.sh + +timestamp=$(date +%s) +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) + +# Start the runtime + +PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" diff --git a/runtime/experiments/deadline/spec.json b/runtime/experiments/deadline/spec.json index 1d64c1b..c1c36e2 100644 --- a/runtime/experiments/deadline/spec.json +++ b/runtime/experiments/deadline/spec.json @@ -3,68 +3,9 @@ "name": "fibonacci_10", "path": "fibonacci_wasm.so", "port": 10010, - "expected-execution-us": 600, - "relative-deadline-us": 2000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" -}, -{ - "active": "yes", - "name": "fibonacci_20", - "path": "fibonacci_wasm.so", - "port": 10020, - "expected-execution-us": 900, - "relative-deadline-us": 5000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" -}, -{ - "active": "yes", - "name": "fibonacci_25", - "path": "fibonacci_wasm.so", - "port": 10025, - "expected-execution-us": 90000, - "relative-deadline-us": 200000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" -}, -{ - "active": "yes", - "name": "fibonacci_30", - "path": "fibonacci_wasm.so", - "port": 10030, - "expected-execution-us": 9000, - "relative-deadline-us": 80000, - "argsize": 1, - "http-req-headers": [], - "http-req-content-type": "text/plain", - "http-req-size": 1024, - "http-resp-headers": [], - "http-resp-size": 1024, - "http-resp-content-type": "text/plain" -}, -{ - "active": "yes", - "name": "fibonacci_35", - "path": "fibonacci_wasm.so", - "port": 10035, - "expected-execution-us": 9000, - "relative-deadline-us": 53000, + "expected-execution-us": 6000, + "admissions-percentile": 70, + "relative-deadline-us": 20000, "argsize": 1, "http-req-headers": [], "http-req-content-type": "text/plain", @@ -78,8 +19,9 @@ "name": "fibonacci_40", "path": "fibonacci_wasm.so", "port": 10040, - "expected-execution-us": 550000, - "relative-deadline-us": 300000000, + "expected-execution-us": 10000000, + "admissions-percentile": 70, + "relative-deadline-us": 20000000, "argsize": 1, "http-req-headers": [], "http-req-content-type": "text/plain", diff --git a/runtime/experiments/preemption/backend.sh b/runtime/experiments/preemption/backend.sh new file mode 100755 index 0000000..5bc869b --- /dev/null +++ b/runtime/experiments/preemption/backend.sh @@ -0,0 +1,18 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Use -d flag if running under gdb + +timestamp=$(date +%s) +experiment_directory=$(pwd) +binary_directory=$(cd ../../bin && pwd) + +results_directory="$experiment_directory/res/$timestamp/$scheduler" +log=log.txt + +mkdir -p "$results_directory" +log_environment >>"$results_directory/$log" + +# Start the runtime +PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" | tee -a "$results_directory/$log" diff --git a/runtime/experiments/preemption/client.sh b/runtime/experiments/preemption/client.sh index d8cf193..d6ffff6 100755 --- a/runtime/experiments/preemption/client.sh +++ b/runtime/experiments/preemption/client.sh @@ -9,34 +9,38 @@ experiment_directory=$(pwd) host=192.168.1.13 results_directory="$experiment_directory/res/$timestamp" -log=log.txt mkdir -p "$results_directory" -log_environment >>"$results_directory/$log" # Start the runtime inputs=(40 10) -duration_sec=15 +duration_sec=30 offset=5 # Execute workloads long enough for runtime to learn excepted execution time echo -n "Running Samples: " for input in ${inputs[*]}; do - hey -z ${duration_sec}s -cpus 6 -t 0 -o csv -m GET -d "$input\n" http://"$host":$((10000 + input)) + hey -n 45 -c 4 -t 0 -o csv -m GET -d "$input\n" http://"$host":$((10000 + input)) done echo "[DONE]" -sleep 5 +sleep 30 echo "Running Experiments" # Run each separately -hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" >"$results_directory/fib40.csv" hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$host:10010" >"$results_directory/fib10.csv" +echo "fib(10) Complete" +sleep 60 + +hey -z ${duration_sec}s -cpus 6 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" >"$results_directory/fib40.csv" +echo "fib(40) Complete" +sleep 120 # Run lower priority first, then higher priority. The lower priority has offsets to ensure it runs the entire time the high priority is trying to run hey -z $((duration_sec + 2 * offset))s -cpus 3 -c 100 -t 0 -o csv -m GET -d "40\n" "http://$host:10040" >"$results_directory/fib40-con.csv" & sleep $offset hey -z ${duration_sec}s -cpus 3 -c 100 -t 0 -o csv -m GET -d "10\n" "http://$host:10010" >"$results_directory/fib10-con.csv" & sleep $((duration_sec + offset + 15)) +echo "fib(10) & fib(40) Complete" # Generate *.csv and *.dat results echo -n "Parsing Results: " @@ -45,6 +49,7 @@ printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" printf "Payload,p50,p90,p99,p100\n" >>"$results_directory/latency.csv" +durations_s=(15 15 15 25) payloads=(fib10 fib10-con fib40 fib40-con) for payload in ${payloads[*]}; do @@ -52,6 +57,8 @@ for payload in ${payloads[*]}; do requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) ((requests == 0)) && continue + duration=${durations_s[$i]} + # Calculate Success Rate for csv awk -F, ' $7 == 200 {ok++} @@ -67,7 +74,7 @@ for payload in ${payloads[*]}; do ((oks == 0)) && continue # If all errors, skip line # Get Latest Timestamp - duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) throughput=$(echo "$oks/$duration" | bc) printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" diff --git a/runtime/experiments/preemption/debug.sh b/runtime/experiments/preemption/debug.sh index 62c64e3..079f6a0 100755 --- a/runtime/experiments/preemption/debug.sh +++ b/runtime/experiments/preemption/debug.sh @@ -10,6 +10,7 @@ binary_directory=$(cd "$project_directory"/bin && pwd) export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" export PATH="$binary_directory:$PATH" +export SLEDGE_SCHEDULER="EDF" gdb --eval-command="handle SIGUSR1 nostop" \ --eval-command="handle SIGPIPE nostop" \ diff --git a/runtime/experiments/preemption/demo.sh b/runtime/experiments/preemption/demo.sh deleted file mode 100755 index 3fdb1e9..0000000 --- a/runtime/experiments/preemption/demo.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -cd ../../bin -LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./sledgert ../tests/mixed_preemption/test_mixed_preemption.json & -cd ../tests/mixed_preemption/ - -# Run small samples on each port to let the runtime figure out the execution time -sleep 10 -echo "Running Samples" -wrk -d 20s -t1 -s post.lua http://localhost:10010 -- --delay 500 10\n -wrk -d 20s -t1 -s post.lua http://localhost:10020 -- --delay 500 20\n -wrk -d 20s -t1 -s post.lua http://localhost:10030 -- --delay 500 25\n - -# Run in Parallel -sleep 10 -echo "Running Experiments" -wrk -d 1m -t1 -s post.lua http://localhost:10010 -- --delay 125 10\n >./res/fib10.txt & -wrk -d 2m -t1 -s post.lua http://localhost:10020 -- --delay 250 20\n >./res/fib20.txt & -wrk -d 3m -t1 -s post.lua http://localhost:10025 -- --delay 500 25\n >./res/fib25.txt - -# Kill the Background Sledge processes -sleep 10 -echo "Running Cleanup" -pkill sledgert -pkill wrk - -# Extract the Latency CSV Data from the Log - -echo 'Fib10, Fib10' >./res/fib10.csv -grep -A200 -m1 -e 'Percentile, Latency' ./res/fib10.txt >>./res/fib10.csv -echo 'Fib20, Fib20' >./res/fib20.csv -grep -A200 -m1 -e 'Percentile, Latency' ./res/fib20.txt >>./res/fib20.csv -echo 'Fib25, Fib25' >./res/fib25.csv -grep -A200 -m1 -e 'Percentile, Latency' ./res/fib25.txt >>./res/fib25.csv -paste -d, ./res/fib10.csv ./res/fib20.csv ./res/fib25.csv >./res/merged.csv diff --git a/runtime/experiments/preemption/fix_results.sh b/runtime/experiments/preemption/fix_results.sh new file mode 100755 index 0000000..83ef32d --- /dev/null +++ b/runtime/experiments/preemption/fix_results.sh @@ -0,0 +1,81 @@ +#!/bin/bash +source ../common.sh + +# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate +# Modified to target a remote host + +timestamp=1606608313-FIFO +experiment_directory=$(pwd) +results_directory="$experiment_directory/res/$timestamp" + +# Generate *.csv and *.dat results +echo -n "Parsing Results: " + +printf "Payload,Success_Rate\n" >>"$results_directory/success.csv" +printf "Payload,Throughput\n" >>"$results_directory/throughput.csv" +printf "Payload,p50,p90,p99,p998,p999,p100\n" >>"$results_directory/latency.csv" + +durations_s=(15 15 15 25) +payloads=(fib10 fib10-con fib40 fib40-con) + +for payload in ${payloads[*]}; do + # Get Number of Requests + requests=$(($(wc -l <"$results_directory/$payload.csv") - 1)) + ((requests == 0)) && continue + + duration=${durations_s[$i]} + + # Calculate Success Rate for csv + awk -F, ' + $7 == 200 {ok++} + END{printf "'"$payload"',%3.5f%\n", (ok / (NR - 1) * 100)} +' <"$results_directory/$payload.csv" >>"$results_directory/success.csv" + + # Filter on 200s, convery from s to ms, and sort + awk -F, '$7 == 200 {print ($1 * 1000)}' <"$results_directory/$payload.csv" | + sort -g >"$results_directory/$payload-response.csv" + + # Get Number of 200s + oks=$(wc -l <"$results_directory/$payload-response.csv") + ((oks == 0)) && continue # If all errors, skip line + + # Get Latest Timestamp + # duration=$(tail -n1 "$results_directory/$payload.csv" | cut -d, -f8) + throughput=$(echo "$oks/$duration" | bc) + printf "%s,%f\n" "$payload" "$throughput" >>"$results_directory/throughput.csv" + + # Generate Latency Data for csv + awk ' + BEGIN { + sum = 0 + p50 = int('"$oks"' * 0.5) + p90 = int('"$oks"' * 0.9) + p99 = int('"$oks"' * 0.99) + p998 = int('"$oks"' * 0.998) + p999 = int('"$oks"' * 0.999) + p100 = '"$oks"' + printf "'"$payload"'," + } + NR==p50 {printf "%1.4f,", $0} + NR==p90 {printf "%1.4f,", $0} + NR==p99 {printf "%1.4f,", $0} + NR==p998 {printf "%1.4f,", $0} + NR==p999 {printf "%1.4f,", $0} + NR==p100 {printf "%1.4f\n", $0} +' <"$results_directory/$payload-response.csv" >>"$results_directory/latency.csv" + + # Delete scratch file used for sorting/counting + # rm -rf "$results_directory/$payload-response.csv" +done + +# Transform csvs to dat files for gnuplot +for file in success latency throughput; do + echo -n "#" >"$results_directory/$file.dat" + tr ',' ' ' <"$results_directory/$file.csv" | column -t >>"$results_directory/$file.dat" +done + +# Generate gnuplots. Commented out because we don't have *.gnuplots defined +# generate_gnuplots + +# Cleanup, if required +echo "[DONE]" diff --git a/runtime/include/admissions_control.h b/runtime/include/admissions_control.h index 8e03440..d88c258 100644 --- a/runtime/include/admissions_control.h +++ b/runtime/include/admissions_control.h @@ -22,13 +22,15 @@ */ extern _Atomic uint64_t admissions_control_admitted; extern uint64_t admissions_control_capacity; +extern const double admissions_control_overhead; static inline void admissions_control_initialize() { #ifdef ADMISSIONS_CONTROL atomic_init(&admissions_control_admitted, 0); - admissions_control_capacity = runtime_worker_threads_count * ADMISSIONS_CONTROL_GRANULARITY; + admissions_control_capacity = runtime_worker_threads_count * ADMISSIONS_CONTROL_GRANULARITY + * ((double)1.0 - admissions_control_overhead); #endif } diff --git a/runtime/include/admissions_info.h b/runtime/include/admissions_info.h index d61ae8c..55211e3 100644 --- a/runtime/include/admissions_info.h +++ b/runtime/include/admissions_info.h @@ -32,6 +32,10 @@ admissions_info_initialize(struct admissions_info *self, int percentile, uint64_ self->percentile = percentile; self->control_index = PERF_WINDOW_BUFFER_SIZE * percentile / 100; +#ifdef LOG_ADMISSIONS_CONTROL + debuglog("Percentile: %d\n", self->percentile); + debuglog("Control Index: %d\n", self->control_index); +#endif #endif } diff --git a/runtime/include/arch/aarch64/context.h b/runtime/include/arch/aarch64/context.h index e83fec6..95f7125 100644 --- a/runtime/include/arch/aarch64/context.h +++ b/runtime/include/arch/aarch64/context.h @@ -1,6 +1,8 @@ #pragma once +#include #include "arch/common.h" +#include "current_sandbox.h" #define ARCH_SIG_JMP_OFF 0x100 /* Based on code generated! */ @@ -16,6 +18,7 @@ static inline void arch_context_init(struct arch_context *actx, reg_t ip, reg_t sp) { assert(actx != NULL); + assert(!software_interrupt_is_enabled()); if (ip == 0 && sp == 0) { actx->variant = ARCH_CONTEXT_VARIANT_UNUSED; @@ -68,6 +71,17 @@ arch_context_switch(struct arch_context *a, struct arch_context *b) /* Assumption: Software Interrupts are disabled by caller */ assert(!software_interrupt_is_enabled()); +#ifndef NDEBUG + /* + * Assumption: In the case of a slow context switch, the caller + * set current_sandbox to the sandbox containing the target context + */ + if (b->variant == ARCH_CONTEXT_VARIANT_SLOW) { + struct sandbox *current = current_sandbox_get(); + assert(current != NULL && b == ¤t->ctxt); + } +#endif + /* if both a and b are NULL, there is no state change */ assert(a != NULL || b != NULL); diff --git a/runtime/include/arch/common.h b/runtime/include/arch/common.h index 507c22f..46ac0d5 100644 --- a/runtime/include/arch/common.h +++ b/runtime/include/arch/common.h @@ -49,7 +49,7 @@ arch_context_variant_print(arch_context_variant_t context) case ARCH_CONTEXT_VARIANT_RUNNING: return "Running"; default: - panic("Encountered unexpected arch_context variant\n"); + panic("Encountered unexpected arch_context variant: %u\n", context); } } diff --git a/runtime/include/arch/x86_64/context.h b/runtime/include/arch/x86_64/context.h index 0b5e7f8..b033079 100644 --- a/runtime/include/arch/x86_64/context.h +++ b/runtime/include/arch/x86_64/context.h @@ -32,13 +32,15 @@ static void __attribute__((noinline)) arch_context_init(struct arch_context *act */ asm volatile("movq %%rsp, %%rbx\n\t" /* Temporarily save pointer of active stack to B */ "movq %%rax, %%rsp\n\t" /* Set active stack to stack pointer in A(C variable sp) */ - "pushq %%rax\n\t" /* Push A(C variable sp) onto the stack at sp */ + "pushq %%rax\n\t" /* Push A register (C variable sp) onto the stack at sp */ "movq %%rsp, %%rax\n\t" /* Write the incremented stack pointer to A(C variable sp) */ "movq %%rbx, %%rsp\n\t" /* Restore original stack saved in B */ : "=a"(sp) : "a"(sp) : "memory", "cc", "rbx"); } + // FIXME: Is the klobber list correct? Issue #129 + // : "memory", "cc", "rbx", "rsi", "rdi"); actx->regs[UREG_SP] = sp; actx->regs[UREG_IP] = ip; @@ -71,6 +73,33 @@ arch_context_restore(mcontext_t *active_context, struct arch_context *sandbox_co active_context->gregs[REG_RIP] = sandbox_context->regs[UREG_IP] + ARCH_SIG_JMP_OFF; } +/** + * Load a new sandbox that preempted an existing sandbox, restoring only the + * instruction pointer and stack pointer registers. + * I am unclear about setting the BP. Issue #131 + * @param active_context - the context of the current worker thread + * @param sandbox_context - the context that we want to restore + */ +static inline void +arch_context_restore_new(mcontext_t *active_context, struct arch_context *sandbox_context) +{ + assert(active_context != NULL); + assert(sandbox_context != NULL); + + /* Assumption: Base Context is only ever used by arch_context_switch */ + assert(sandbox_context != &worker_thread_base_context); + + assert(sandbox_context->regs[UREG_SP]); + assert(sandbox_context->regs[UREG_IP]); + + /* Transitioning from Fast -> Running */ + assert(sandbox_context->variant == ARCH_CONTEXT_VARIANT_FAST); + sandbox_context->variant = ARCH_CONTEXT_VARIANT_RUNNING; + + active_context->gregs[REG_RSP] = sandbox_context->regs[UREG_SP]; + active_context->gregs[REG_RIP] = sandbox_context->regs[UREG_IP]; +} + /** * @param a - the registers and context of the thing running * @param b - the registers and context of what we're switching to diff --git a/runtime/include/panic.h b/runtime/include/panic.h index 962956c..4547088 100644 --- a/runtime/include/panic.h +++ b/runtime/include/panic.h @@ -5,6 +5,7 @@ #include #include #include +#include #define panic(fmt, ...) \ { \ diff --git a/runtime/src/admissions_control.c b/runtime/src/admissions_control.c index d5aba8b..275bee3 100644 --- a/runtime/src/admissions_control.c +++ b/runtime/src/admissions_control.c @@ -2,3 +2,5 @@ _Atomic uint64_t admissions_control_admitted; uint64_t admissions_control_capacity; + +const double admissions_control_overhead = 0.2; diff --git a/runtime/src/global_request_scheduler_minheap.c b/runtime/src/global_request_scheduler_minheap.c index f3721ae..c6ddd2d 100644 --- a/runtime/src/global_request_scheduler_minheap.c +++ b/runtime/src/global_request_scheduler_minheap.c @@ -76,7 +76,7 @@ sandbox_request_get_priority_fn(void *element) void global_request_scheduler_minheap_initialize() { - global_request_scheduler_minheap = priority_queue_initialize(1000, true, sandbox_request_get_priority_fn); + global_request_scheduler_minheap = priority_queue_initialize(4096, true, sandbox_request_get_priority_fn); struct global_request_scheduler_config config = { .add_fn = global_request_scheduler_minheap_add, diff --git a/runtime/src/local_runqueue_minheap.c b/runtime/src/local_runqueue_minheap.c index 05fc340..d93bfc3 100644 --- a/runtime/src/local_runqueue_minheap.c +++ b/runtime/src/local_runqueue_minheap.c @@ -185,7 +185,7 @@ local_runqueue_minheap_preempt(ucontext_t *user_context) * user-level context switch state, so do not enable software interrupts. * TODO: Review the interrupt logic here. Issue #63 */ - arch_context_restore(&user_context->uc_mcontext, &next_sandbox->ctxt); + arch_context_restore_new(&user_context->uc_mcontext, &next_sandbox->ctxt); should_enable_software_interrupt = false; } done: diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index 21bfab7..3eff92d 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -373,7 +373,6 @@ sandbox_allocate_memory(struct module *module) error_message = "sandbox_allocate_memory - memory allocation failed"; goto alloc_failed; } - sandbox = (struct sandbox *)addr; /* Set the struct sandbox, HTTP Req/Resp buffer, and the initial Wasm Pages as read/write */ errno = 0; @@ -384,6 +383,8 @@ sandbox_allocate_memory(struct module *module) goto set_rw_failed; } + sandbox = (struct sandbox *)addr_rw; + /* Populate Sandbox members */ sandbox->state = SANDBOX_UNINITIALIZED; sandbox->linear_memory_start = (char *)addr + sandbox_size; @@ -411,12 +412,23 @@ sandbox_allocate_stack(struct sandbox *sandbox) { assert(sandbox); assert(sandbox->module); + assert(!software_interrupt_is_enabled()); - errno = 0; + errno = 0; + char *addr = mmap(NULL, sandbox->module->stack_size + /* guard page */ PAGE_SIZE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) goto err_stack_allocation_failed; + + /* Set the struct sandbox, HTTP Req/Resp buffer, and the initial Wasm Pages as read/write */ + errno = 0; + char *addr_rw = mmap(addr + /* guard page */ PAGE_SIZE, sandbox->module->stack_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + + /* TODO: Fix leak here. Issue #132 */ + if (addr_rw == MAP_FAILED) goto err_stack_allocation_failed; + + sandbox->stack_start = addr_rw; sandbox->stack_size = sandbox->module->stack_size; - sandbox->stack_start = mmap(NULL, sandbox->stack_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_GROWSDOWN, -1, 0); - if (sandbox->stack_start == MAP_FAILED) goto err_stack_allocation_failed; done: return 0; @@ -450,8 +462,9 @@ sandbox_set_as_initialized(struct sandbox *sandbox, struct sandbox_request *sand sandbox->state = SANDBOX_SET_AS_INITIALIZED; /* Initialize the sandbox's context, stack, and instruction pointer */ + /* stack_start points to the bottom of the usable stack, so add stack_size to get to top */ arch_context_init(&sandbox->ctxt, (reg_t)current_sandbox_main, - (reg_t)(sandbox->stack_start + sandbox->stack_size)); + (reg_t)sandbox->stack_start + sandbox->stack_size); /* Initialize file descriptors to -1 */ for (int i = 0; i < SANDBOX_MAX_IO_HANDLE_COUNT; i++) sandbox->io_handles[i].file_descriptor = -1; @@ -875,13 +888,11 @@ sandbox_free(struct sandbox *sandbox) module_release(sandbox->module); - void * stkaddr = sandbox->stack_start; - size_t stksz = sandbox->stack_size; - - /* Free Sandbox Stack */ errno = 0; - rc = munmap(stkaddr, stksz); + + /* The stack start is the bottom of the usable stack, but we allocated a guard page below this */ + rc = munmap((char *)sandbox->stack_start - PAGE_SIZE, sandbox->stack_size + PAGE_SIZE); if (rc == -1) { debuglog("Failed to unmap stack of Sandbox %lu\n", sandbox->id); goto err_free_stack_failed; diff --git a/runtime/src/software_interrupt.c b/runtime/src/software_interrupt.c index 90df9fd..a0dce8a 100644 --- a/runtime/src/software_interrupt.c +++ b/runtime/src/software_interrupt.c @@ -129,6 +129,8 @@ sigusr1_handler(siginfo_t *signal_info, ucontext_t *user_context, struct sandbox #ifdef LOG_PREEMPTION debuglog("Total SIGUSR1 Received: %d\n", software_interrupt_SIGUSR_count); + debuglog("Restoring sandbox: %lu, Stack %llu\n", current_sandbox->id, + current_sandbox->ctxt.mctx.gregs[REG_RSP]); #endif arch_mcontext_restore(&user_context->uc_mcontext, ¤t_sandbox->ctxt); @@ -176,7 +178,7 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void case SIGUSR1: { return sigusr1_handler(signal_info, user_context, current_sandbox); } - default: + default: { if (signal_info->si_code == SI_TKILL) { panic("Unexpectedly received signal %d from a thread kill, but we have no handler\n", signal_type); @@ -184,6 +186,7 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void panic("Unexpectedly received signal %d from the kernel, but we have no handler\n", signal_type); } } + } #endif } diff --git a/runtime/src/worker_thread.c b/runtime/src/worker_thread.c index 24d22a7..0250401 100644 --- a/runtime/src/worker_thread.c +++ b/runtime/src/worker_thread.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -102,24 +103,34 @@ worker_thread_switch_to_sandbox(struct sandbox *next_sandbox) sandbox_set_as_running(next_sandbox, next_sandbox->state); #ifdef LOG_CONTEXT_SWITCHES - debuglog("Base Context (%s) > Sandbox %lu (%s)\n", - arch_context_variant_print(worker_thread_base_context.variant), next_sandbox->id, + debuglog("Base Context (@%p) (%s) > Sandbox %lu (@%p) (%s)\n", &worker_thread_base_context, + arch_context_variant_print(worker_thread_base_context.variant), next_sandbox->id, next_context, arch_context_variant_print(next_context->variant)); #endif + /* Assumption: If a slow context switch, current sandbox should be set to the target */ + assert(next_context->variant != ARCH_CONTEXT_VARIANT_SLOW + || ¤t_sandbox_get()->ctxt == next_context); arch_context_switch(NULL, next_context); } else { /* Set the current sandbox to the next */ assert(next_sandbox != current_sandbox); + struct arch_context *current_context = ¤t_sandbox->ctxt; + +#ifdef LOG_CONTEXT_SWITCHES + debuglog("Sandbox %lu (@%p) (%s) > Sandbox %lu (@%p) (%s)\n", current_sandbox->id, + ¤t_sandbox->ctxt, arch_context_variant_print(current_sandbox->ctxt.variant), + next_sandbox->id, &next_sandbox->ctxt, arch_context_variant_print(next_context->variant)); +#endif + worker_thread_transition_exiting_sandbox(current_sandbox); sandbox_set_as_running(next_sandbox, next_sandbox->state); - struct arch_context *current_context = ¤t_sandbox->ctxt; - -#ifdef LOG_CONTEXT_SWITCHES - debuglog("Sandbox %lu > Sandbox %lu\n", current_sandbox->id, next_sandbox->id); +#ifndef NDEBUG + assert(next_context->variant != ARCH_CONTEXT_VARIANT_SLOW + || ¤t_sandbox_get()->ctxt == next_context); #endif /* Switch to the associated context. */ @@ -146,22 +157,21 @@ worker_thread_switch_to_base_context() } #endif - worker_thread_transition_exiting_sandbox(current_sandbox); - /* Assumption: Base Context should never switch to Base Context */ assert(current_sandbox != NULL); - assert(¤t_sandbox->ctxt != &worker_thread_base_context); - - current_sandbox_set(NULL); + struct arch_context *current_context = ¤t_sandbox->ctxt; + assert(current_context != &worker_thread_base_context); #ifdef LOG_CONTEXT_SWITCHES - debuglog("Sandbox %lu (%s) > Base Context (%s)\n", current_sandbox->id, - arch_context_variant_print(current_sandbox->ctxt.variant), + debuglog("Sandbox %lu (@%p) (%s)> Base Context (@%p) (%s)\n", current_sandbox->id, current_context, + arch_context_variant_print(current_sandbox->ctxt.variant), &worker_thread_base_context, arch_context_variant_print(worker_thread_base_context.variant)); #endif - arch_context_switch(¤t_sandbox->ctxt, &worker_thread_base_context); - + worker_thread_transition_exiting_sandbox(current_sandbox); + current_sandbox_set(NULL); + assert(worker_thread_base_context.variant == ARCH_CONTEXT_VARIANT_FAST); + arch_context_switch(current_context, &worker_thread_base_context); software_interrupt_enable(); } @@ -234,7 +244,7 @@ worker_thread_execute_epoll_loop(void) if (sandbox->state == SANDBOX_BLOCKED) worker_thread_wakeup_sandbox(sandbox); } else if (epoll_events[i].events & (EPOLLERR | EPOLLHUP)) { - /* Mystery: This seems to never fire. Why? */ + /* Mystery: This seems to never fire. Why? Issue #130 */ /* Close socket and set as error on socket error or unexpected client hangup */ struct sandbox *sandbox = (struct sandbox *)epoll_events[i].data.ptr; @@ -282,7 +292,9 @@ worker_thread_main(void *return_code) worker_thread_start_timestamp = __getcycles(); worker_thread_lock_duration = 0; - /* Initialize Base Context */ + /* Initialize Base Context as unused + * The SP and IP are populated during the first FAST switch away + */ arch_context_init(&worker_thread_base_context, 0, 0); /* Initialize Runqueue Variant */