fix: working with client data

master
Sean McBride 4 years ago
parent 2447c031a6
commit 8b0358c336

@ -0,0 +1,15 @@
{
"active": true,
"name": "ekf",
"path": "ekf_wasm.so",
"port": 10000,
"expected-execution-us": 5000,
"relative-deadline-us": 50000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream"
}

@ -0,0 +1,15 @@
{
"active": true,
"name": "gocr",
"path": "gocr_wasm.so",
"port": 10000,
"expected-execution-us": 5000,
"relative-deadline-us": 360000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "text/plain",
"http-req-size": 5335057,
"http-resp-headers": [],
"http-resp-size": 5335057,
"http-resp-content-type": "text/plain"
}

@ -0,0 +1,15 @@
{
"active": true,
"name": "lpd",
"path": "lpd_wasm.so",
"port": 10000,
"expected-execution-us": 5000,
"relative-deadline-us": 50000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain"
}

@ -0,0 +1,15 @@
{
"active": true,
"name": "resize",
"path": "resize_wasm.so",
"port": 10000,
"expected-execution-us": 5000,
"relative-deadline-us": 50000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png"
}

@ -12,8 +12,11 @@ source get_result_count.sh || exit 1
source panic.sh || exit 1
source path_join.sh || exit 1
# TODO: Excluding gocr because of difficulty used gocr with hey
# Please keep the element ordered alphabetically!
# declare -a workloads=(ekf resize lpd gocr)
declare -a workloads=(ekf resize lpd)
declare -a workloads=(ekf lpd resize)
declare -a multiples=(1.5 1.6 1.7 1.8 1.9 2.0)
profile() {
local hostname="$1"
@ -61,12 +64,6 @@ get_baseline_execution() {
' < "$response_times_file"
}
get_random_from_interval() {
local -r lower="$1"
local -r upper="$2"
awk "BEGIN { \"date +%N\" | getline seed; srand(seed); print rand() * ($upper - $lower) + $lower}"
}
calculate_relative_deadline() {
local -r baseline="$1"
local -r multiplier="$2"
@ -77,62 +74,45 @@ generate_spec() {
local results_directory="$1"
# Multiplier Interval and Expected Execution Percentile is currently the same for all workloads
local -r multiplier_interval_lower_bound="1.5"
local -r multiplier_interval_upper_bound="2.0"
local -ri percentile=90
((percentile < 50 || percentile > 99)) && panic "Percentile should be between 50 and 99 inclusive, was $percentile"
local -A multiplier=()
local -A baseline_execution=()
local -A relative_deadline=()
local -i port=10000
local relative_deadline
for workload in "${workloads[@]}"; do
multiplier["$workload"]="$(get_random_from_interval $multiplier_interval_lower_bound $multiplier_interval_upper_bound)"
baseline_execution["$workload"]="$(get_baseline_execution "$results_directory" "$workload" $percentile)"
[[ -z "${baseline_execution[$workload]}" ]] && {
panic "Failed to get baseline execution for $workload"
exit 1
}
[[ -z "${multiplier[$workload]}" ]] && {
panic "Failed to generate multiplier for $workload"
exit 1
}
relative_deadline["$workload"]="$(calculate_relative_deadline "${baseline_execution[$workload]}" "${multiplier[$workload]}")"
{
echo "$workload"
printf "\tbaseline: %s\n" "${baseline_execution[$workload]}"
printf "\tmultiplier: %s\n" "${multiplier[$workload]}"
printf "\tdeadline: %s\n" "${relative_deadline[$workload]}"
} >> "$results_directory/log.txt"
done
# TODO: Excluding gocr because of difficulty used gocr with hey
# Generates unique module specs on different ports using the different multiples
for multiple in "${multiples[@]}"; do
relative_deadline=$(calculate_relative_deadline "${baseline_execution[$workload]}" "${multiple}")
jq ". + { \
\"admissions-percentile\": $percentile,\
\"expected-execution-us\": ${baseline_execution[${workload}]},\
\"name\": \"${workload}_${multiple}\",\
\"port\": $port,\
\"relative-deadline-us\": $relative_deadline}" \
< "./${workload}/template.json" \
> "./${workload}/result_${multiple}.json"
((port++))
done
# Merges all of the multiple specs for a single module
jq -s '.' ./"${workload}"/result_*.json > "./${workload}/workload_result.json"
rm ./"${workload}"/result_*.json
done
# Merges all of the specs for all modules
# Our JSON format is not spec complaint. I have to hack in a wrapping array before jq and delete it afterwards
# expected-execution-us and admissions-percentile is only used by admissions control
{
echo "["
cat ./spec.json
echo "]"
} | jq "\
[ \
.[] | \
if (.name == \"ekf\") then . + { \
\"relative-deadline-us\": ${relative_deadline[ekf]},\
\"expected-execution-us\": ${baseline_execution[ekf]},\
\"admissions-percentile\": $percentile
} else . end | \
if (.name == \"resize\") then . + { \
\"relative-deadline-us\": ${relative_deadline[resize]},\
\"expected-execution-us\": ${baseline_execution[resize]},\
\"admissions-percentile\": $percentile
} else . end | \
if (.name == \"lpd\") then . + { \
\"relative-deadline-us\": ${relative_deadline[lpd]},\
\"expected-execution-us\": ${baseline_execution[lpd]},\
\"admissions-percentile\": $percentile
} else . end \
]" | tail -n +2 | head -n-1 > "$results_directory/spec.json"
jq -s '. | flatten' ./*/workload_result.json | tail -n +2 | head -n-1 > "$results_directory/spec.json"
rm ./*/workload_result*.json
}
# Process the experimental results and generate human-friendly results for success rate, throughput, and latency

@ -0,0 +1,19 @@
# EKF errors out. Use this when fixed.
500,ekf_1.5
500,ekf_1.6
500,ekf_1.7
500,ekf_1.8
500,ekf_1.9
500,ekf_2.0
5,lpd_1.5
5,lpd_1.6
5,lpd_1.7
5,lpd_1.8
5,lpd_1.9
5,lpd_2.0
1,resize_1.5
1,resize_1.6
1,resize_1.7
1,resize_1.8
1,resize_1.9
1,resize_2.0
1 # EKF errors out. Use this when fixed.
2 500,ekf_1.5
3 500,ekf_1.6
4 500,ekf_1.7
5 500,ekf_1.8
6 500,ekf_1.9
7 500,ekf_2.0
8 5,lpd_1.5
9 5,lpd_1.6
10 5,lpd_1.7
11 5,lpd_1.8
12 5,lpd_1.9
13 5,lpd_2.0
14 1,resize_1.5
15 1,resize_1.6
16 1,resize_1.7
17 1,resize_1.8
18 1,resize_1.9
19 1,resize_2.0

@ -1,3 +1,6 @@
33,ekf
33,resize
33,lpd
5,lpd_1.5
5,lpd_1.6
5,lpd_1.7
5,lpd_1.8
5,lpd_1.9
5,lpd_2.0

1 33 5 ekf lpd_1.5
2 33 5 resize lpd_1.6
3 33 5 lpd lpd_1.7
4 5 lpd_1.8
5 5 lpd_1.9
6 5 lpd_2.0

@ -24,6 +24,29 @@ if ! command -v hey > /dev/null; then
exit 1
fi
declare -a workloads=()
declare -A port=()
# Example of stripping off multiple suffix from variable in format string_multiple
# test="ekf_12223.23343"
# ${test%%_+([[:digit:]]).+([[:digit:]])}
declare -Ar body=(
[ekf]="-D ./ekf/ekf_raw.dat"
[resize]="-D ./resize/shrinking_man_large.jpg"
[lpd]="-D ./lpd/Cars0.png"
)
initialize_globals() {
while read -r line; do
# Read into buffer array, splitting on commas
readarray -t -d, buffer < <(echo -n "$line")
workload="${buffer[1]}"
# Update workload mix structures
workloads+=("$workload")
port+=(["$workload"]=$(get_port "$workload"))
done < mix.csv
}
# Sends requests until the per-module perf window buffers are full
# This ensures that Sledge has accurate estimates of execution time
run_samples() {
@ -50,33 +73,29 @@ run_samples() {
printf "Running Samples: "
# EKF
hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c 1 -cpus 1 -t 0 -o csv -m GET -D "./ekf/initial_state.dat" "http://${hostname}:10000" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "ekf samples failed with $?"
return 1
}
# Resize
hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c 1 -cpus 1 -t 0 -o csv -m GET -D "./resize/shrinking_man_large.jpg" "http://${hostname}:10001" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "resize samples failed with $?"
return 1
}
# lpd
hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c 1 -cpus 1 -t 0 -o csv -m GET -D "./lpd/Cars0.png" "http://${hostname}:10002" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "resize samples failed with $?"
return 1
}
# TODO: gocr
local workload_base
for workload in "${workloads[@]}"; do
workload_base="${workload%%_+([[:digit:]]).+([[:digit:]])}"
hey -disable-compression -disable-keepalive -disable-redirects -n "$perf_window_buffer_size" -c 1 -cpus 1 -t 0 -o csv -m GET -D "${body[$workload_base]}" "http://${hostname}:${port[$workload]}" 1> /dev/null 2> /dev/null || {
printf "[ERR]\n"
panic "$workload samples failed with $?"
return 1
}
done
printf "[OK]\n"
return 0
}
get_port() {
local name="$1"
{
echo "["
cat ./spec.json
echo "]"
} | jq ".[] | select(.name == \"$name\") | .port"
}
# Execute the fib10 and fib40 experiments sequentially and concurrently
# $1 (hostname)
# $2 (results_directory) - a directory where we will store our results
@ -95,20 +114,6 @@ run_experiments() {
local hostname="$1"
local results_directory="$2"
local -a workloads=()
local -Ar port=(
[ekf]=10000
[resize]=10001
[lpd]=10002
)
local -Ar body=(
[ekf]="-D ./ekf/initial_state.dat"
[resize]="-D ./resize/shrinking_man_large.jpg"
[lpd]="-D ./lpd/Cars0.png"
)
local -A floor=()
local -A length=()
local -i total=0
@ -116,14 +121,14 @@ run_experiments() {
local -a buffer=()
local workload=""
local -i odds=0
# Update workload mix structures
while read -r line; do
# Read into buffer array, splitting on commas
readarray -t -d, buffer < <(echo -n "$line")
# Use human friendly names
odds="${buffer[0]}"
workload="${buffer[1]}"
# Update workload mix structures
workloads+=("$workload")
floor+=(["$workload"]=$total)
length+=(["$workload"]=$odds)
((total += odds))
@ -137,10 +142,10 @@ run_experiments() {
fi
# TODO: Check that workload is in spec.json
local -ir batch_size=1
local -ir batch_size=100
local -i batch_id=0
local -i roll=0
local -ir total_iterations=10000
local -ir total_iterations=100000
local -ir worker_max=50
local pids
@ -166,8 +171,8 @@ run_experiments() {
printf "[OK]\n"
for workload in "${workloads[@]}"; do
tail --quiet -n +2 "$results_directory/${workload}"_*.csv >> "$results_directory/${workload}.csv"
rm "$results_directory/${workload}"_*.csv
tail --quiet -n +2 "$results_directory"/"${workload}"_*.csv >> "$results_directory"/"${workload}".csv
rm "$results_directory"/"${workload}"_*.csv
done
return 0
@ -190,14 +195,14 @@ process_results() {
# Write headers to CSVs
printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv"
local -ar payloads=(ekf resize lpd)
for payload in "${payloads[@]}"; do
# local -ar payloads=(ekf resize lpd)
for workloads in "${workloads[@]}"; do
# Filter on 200s, subtract DNS time, convert from s to ms, and sort
awk -F, '$7 == 200 {print (($1 - $2) * 1000)}' < "$results_directory/$payload.csv" \
| sort -g > "$results_directory/$payload-response.csv"
awk -F, '$7 == 200 {print (($1 - $2) * 1000)}' < "$results_directory/$workloads.csv" \
| sort -g > "$results_directory/$workloads-response.csv"
oks=$(wc -l < "$results_directory/$payload-response.csv")
oks=$(wc -l < "$results_directory/$workloads-response.csv")
((oks == 0)) && continue # If all errors, skip line
# Generate Latency Data for csv
@ -208,16 +213,16 @@ process_results() {
p90 = int('"$oks"' * 0.9)
p99 = int('"$oks"' * 0.99)
p100 = '"$oks"'
printf "'"$payload"',"
printf "'"$workloads"',"
}
NR==p50 {printf "%1.4f,", $0}
NR==p90 {printf "%1.4f,", $0}
NR==p99 {printf "%1.4f,", $0}
NR==p100 {printf "%1.4f\n", $0}
' < "$results_directory/$payload-response.csv" >> "$results_directory/latency.csv"
' < "$results_directory/$workloads-response.csv" >> "$results_directory/latency.csv"
# Delete scratch file used for sorting/counting
rm -rf "$results_directory/$payload-response.csv"
rm -rf "$results_directory/$workloads-response.csv"
done
# Transform csvs to dat files for gnuplot
@ -232,6 +237,7 @@ experiment_main() {
local -r target_hostname="$1"
local -r results_directory="$2"
initialize_globals
run_samples "$target_hostname" || return 1
run_experiments "$target_hostname" "$results_directory" || return 1
process_results "$results_directory" || return 1

@ -1,60 +1,288 @@
{
"active": true,
"name": "ekf",
"path": "ekf_wasm.so",
"port": 10000,
"expected-execution-us": 5000,
"relative-deadline-us": 354,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream"
},
{
"active": true,
"name": "resize",
"path": "resize_wasm.so",
"port": 10001,
"expected-execution-us": 5000,
"relative-deadline-us": 242058,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "image/png"
},
{
"active": true,
"name": "lpd",
"path": "lpd_wasm.so",
"port": 10002,
"expected-execution-us": 5000,
"relative-deadline-us": 52425,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain"
},
{
"active": true,
"name": "gocr",
"path": "gocr_wasm.so",
"port": 10003,
"expected-execution-us": 5000,
"relative-deadline-us": 360000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "text/plain",
"http-req-size": 5335057,
"http-resp-headers": [],
"http-resp-size": 5335057,
"http-resp-content-type": "text/plain"
}
{
"active": true,
"name": "ekf_1.5",
"path": "ekf_wasm.so",
"port": 10000,
"expected-execution-us": 500,
"relative-deadline-us": 750,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream",
"admissions-percentile": 90
},
{
"active": true,
"name": "ekf_1.6",
"path": "ekf_wasm.so",
"port": 10001,
"expected-execution-us": 500,
"relative-deadline-us": 800,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream",
"admissions-percentile": 90
},
{
"active": true,
"name": "ekf_1.7",
"path": "ekf_wasm.so",
"port": 10002,
"expected-execution-us": 500,
"relative-deadline-us": 850,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream",
"admissions-percentile": 90
},
{
"active": true,
"name": "ekf_1.8",
"path": "ekf_wasm.so",
"port": 10003,
"expected-execution-us": 500,
"relative-deadline-us": 900,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream",
"admissions-percentile": 90
},
{
"active": true,
"name": "ekf_1.9",
"path": "ekf_wasm.so",
"port": 10004,
"expected-execution-us": 500,
"relative-deadline-us": 950,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream",
"admissions-percentile": 90
},
{
"active": true,
"name": "ekf_2.0",
"path": "ekf_wasm.so",
"port": 10005,
"expected-execution-us": 500,
"relative-deadline-us": 1000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "application/octet-stream",
"http-req-size": 1024000,
"http-resp-headers": [],
"http-resp-size": 1024000,
"http-resp-content-type": "application/octet-stream",
"admissions-percentile": 90
},
{
"active": true,
"name": "lpd_1.5",
"path": "lpd_wasm.so",
"port": 10006,
"expected-execution-us": 62400,
"relative-deadline-us": 93600,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain",
"admissions-percentile": 90
},
{
"active": true,
"name": "lpd_1.6",
"path": "lpd_wasm.so",
"port": 10007,
"expected-execution-us": 62400,
"relative-deadline-us": 99840,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain",
"admissions-percentile": 90
},
{
"active": true,
"name": "lpd_1.7",
"path": "lpd_wasm.so",
"port": 10008,
"expected-execution-us": 62400,
"relative-deadline-us": 106080,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain",
"admissions-percentile": 90
},
{
"active": true,
"name": "lpd_1.8",
"path": "lpd_wasm.so",
"port": 10009,
"expected-execution-us": 62400,
"relative-deadline-us": 112320,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain",
"admissions-percentile": 90
},
{
"active": true,
"name": "lpd_1.9",
"path": "lpd_wasm.so",
"port": 10010,
"expected-execution-us": 62400,
"relative-deadline-us": 118560,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain",
"admissions-percentile": 90
},
{
"active": true,
"name": "lpd_2.0",
"path": "lpd_wasm.so",
"port": 10011,
"expected-execution-us": 62400,
"relative-deadline-us": 124800,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1002400,
"http-resp-headers": [],
"http-resp-size": 1048576,
"http-resp-content-type": "text/plain",
"admissions-percentile": 90
},
{
"active": true,
"name": "resize_1.5",
"path": "resize_wasm.so",
"port": 10012,
"expected-execution-us": 295500,
"relative-deadline-us": 443250,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png",
"admissions-percentile": 90
},
{
"active": true,
"name": "resize_1.6",
"path": "resize_wasm.so",
"port": 10013,
"expected-execution-us": 295500,
"relative-deadline-us": 472800,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png",
"admissions-percentile": 90
},
{
"active": true,
"name": "resize_1.7",
"path": "resize_wasm.so",
"port": 10014,
"expected-execution-us": 295500,
"relative-deadline-us": 502350,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png",
"admissions-percentile": 90
},
{
"active": true,
"name": "resize_1.8",
"path": "resize_wasm.so",
"port": 10015,
"expected-execution-us": 295500,
"relative-deadline-us": 531900,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png",
"admissions-percentile": 90
},
{
"active": true,
"name": "resize_1.9",
"path": "resize_wasm.so",
"port": 10016,
"expected-execution-us": 295500,
"relative-deadline-us": 561450,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png",
"admissions-percentile": 90
},
{
"active": true,
"name": "resize_2.0",
"path": "resize_wasm.so",
"port": 10017,
"expected-execution-us": 295500,
"relative-deadline-us": 591000,
"argsize": 1,
"http-req-headers": [],
"http-req-content-type": "image/jpeg",
"http-req-size": 1524000,
"http-resp-headers": [],
"http-resp-size": 1524000,
"http-resp-content-type": "image/png",
"admissions-percentile": 90
}

Loading…
Cancel
Save