ImageClassification framework

4 years ago · 7e0d281541
parent 4be2bb9589
commit 7e0d281541
7 changed files with 197 additions and 67 deletions
--- a/runtime/experiments/applications/imageclassification/.gitignore
+++ b/runtime/experiments/applications/imageclassification/.gitignore
@ -1 +1,2 @@
-flower.jpg
+log.csv
+res/*
--- a/runtime/experiments/applications/imageclassification/edf_nopreemption.env
+++ b/runtime/experiments/applications/imageclassification/edf_nopreemption.env
@ -0,0 +1,2 @@
+SLEDGE_SCHEDULER=EDF
+SLEDGE_DISABLE_PREEMPTION=true
--- a/runtime/experiments/applications/imageclassification/edf_preemption.env
+++ b/runtime/experiments/applications/imageclassification/edf_preemption.env
@ -0,0 +1,3 @@
+SLEDGE_SCHEDULER=EDF
+SLEDGE_DISABLE_PREEMPTION=false
+SLEDGE_SIGALRM_HANDLER=TRIAGED
--- a/runtime/experiments/applications/imageclassification/fifo_nopreemption.env
+++ b/runtime/experiments/applications/imageclassification/fifo_nopreemption.env
@ -0,0 +1,2 @@
+SLEDGE_SCHEDULER=FIFO
+SLEDGE_DISABLE_PREEMPTION=true
--- a/runtime/experiments/applications/imageclassification/fifo_preemption.env
+++ b/runtime/experiments/applications/imageclassification/fifo_preemption.env
@ -0,0 +1,2 @@
+SLEDGE_SCHEDULER=FIFO
+SLEDGE_DISABLE_PREEMPTION=false
--- a/runtime/experiments/applications/imageclassification/run.sh
+++ b/runtime/experiments/applications/imageclassification/run.sh
@ -1,69 +1,173 @@
 #!/bin/bash
-# Executes the runtime in GDB
-# Substitutes the absolute path from the container with a path relatively derived from the location of this script
-# This allows debugging outside of the Docker container
-# Also disables pagination and stopping on SIGUSR1
-
-experiment_directory=$(pwd)
-echo "$experiment_directory"
-project_directory=$(cd ../../.. && pwd)
-binary_directory=$(cd "$project_directory"/bin && pwd)
-
-# Copy License Plate Image if not here
-# if [[ ! -f "./plate.jpg" ]]; then
-#   cp ../../../tests/sod/bin/plate.jpg ./plate.jpg
-# fi
-
-if [ "$1" != "-d" ]; then
-	PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" &
-	sleep 1
-else
-	echo "Running under gdb"
-fi
-
-expected_size="$(find expected_result.jpg -printf "%s")"
-success_count=0
-total_count=50
-
-# We are currently unclear why the image classifier isn't working properly
-# Both bmp and png formats are added to the repo while debugging
-file_type=bmp
-# file_type=png
-
-for class in airplane automobile bird cat deer dog frog horse ship truck; do
-	for instance in 1 2 3 4 5 6 7 8 9 10; do
-		echo "Classifying $class$instance.$file_type"
-		curl -H 'Expect:' -H "Content-Type: Image/$file_type" --data-binary "@images/$file_type/$class$instance.$file_type" localhost:10000 2> /dev/null
+
+__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
+__run_sh__bash_libraries_relative_path="../../bash_libraries"
+__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
+export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
+
+source csv_to_dat.sh || exit 1
+source framework.sh || exit 1
+source get_result_count.sh || exit 1
+source panic.sh || exit 1
+source path_join.sh || exit 1
+source validate_dependencies.sh || exit 1
+
+get_random_image() {
+	# local workload="$1"
+	local -n __random_image="$1"
+
+	__random_image="${cifar10_images[$((RANDOM % ${#cifar10_images[@]}))]}"
+}
+
+run_functional_tests() {
+	local hostname="$1"
+	local results_directory="$2"
+	# same_image="./images/bmp/airplane5.bmp"
+
+	printf "Func Tests: \n"
+
+	# Functional Testing on each image
+	for image in "${cifar10_images[@]}"; do
+		echo "${image}" >> "${results_directory}/cifar10_rand.txt"
+		curl --data-binary "@${image}" -s "${hostname}:10000" >> "${results_directory}/cifar10_rand.txt"
 	done
-done
-
-exit
-
-for ((i = 0; i < total_count; i++)); do
-	echo "$i"
-	ext="$RANDOM"
-	curl -H 'Expect:' -H "Content-Type: image/jpg" --data-binary "@plate.jpg" --output "result_$ext.jpg" localhost:10000 2> /dev/null
-	actual_size="$(find result_"$ext".jpg -printf "%s")"
-
-	# echo "$result"
-	if [[ "$expected_size" == "$actual_size" ]]; then
-		echo "SUCCESS $success_count"
-	else
-		echo "FAIL"
-		echo "Expected Size:"
-		echo "$expected_size"
-		echo "==============================================="
-		echo "Actual Size:"
-		echo "$actual_size"
+	
+	echo "$same_image" >> "${results_directory}/cifar10_same.txt"
+	curl --data-binary "@$same_image" -s "${hostname}:10001" >> "${results_directory}/cifar10_same.txt"
+
+	# file_type=bmp
+	# # file_type=png
+
+	# for class in airplane automobile bird cat deer dog frog horse ship truck; do
+	# 	for instance in 1 2 3 4 5 6 7 8 9 10; do
+	# 		echo "Classifying $class$instance.$file_type" >> "${results_directory}/cifar10_rand.txt"
+	# 		curl -H 'Expect:' -H "Content-Type: Image/$file_type" --data-binary "@images/$file_type/$class$instance.$file_type" localhost:10000 >> "${results_directory}/cifar10_rand.txt" 
+	# 	done
+	# 	{
+	# 		echo "==== ERROR RATE: x/10 ===="  # the 'x' is to be coded
+	# 		echo ""
+	# 		echo ""
+	# 	} >> "${results_directory}/cifar10_rand.txt"
+	# done
+
+	printf "[OK]\n"
+}
+
+run_perf_tests() {
+	local hostname="$1"
+	local results_directory="$2"
+
+	local -ir total_iterations=100
+	local -ir worker_max=10
+	local -ir batch_size=10
+	local -i batch_id=0
+	local image
+	local pids
+
+	printf "Perf Tests: \n"
+	for workload in "${workloads[@]}"; do
+		batch_id=0
+		for ((i = 0; i < total_iterations; i += batch_size)); do
+			# Block waiting for a worker to finish if we are at our max
+			while (($(pgrep --count hey) >= worker_max)); do
+				wait -n "$(pgrep hey | tr '\n' ' ')"
+			done
+			((batch_id++))
+			
+			if [ "$workload" = "cifar10_rand" ]; then
+				get_random_image image
+			else
+				image=$same_image
+			fi
+			hey -disable-compression -disable-keepalive -disable-redirects -n $batch_size -c 1 -cpus 1 -t 0 -o csv -m GET -D "${image}" "http://${hostname}:${port[$workload]}" > "$results_directory/${workload}_${batch_id}.csv" 2> /dev/null &
+			# curl --data-binary "@$image" --output - "${hostname}:${port[$workload]}" >> "${results_directory}/${workload}-2.txt"
+		done
+		pids=$(pgrep hey | tr '\n' ' ')
+		[[ -n $pids ]] && wait -f $pids
+	done
+	printf "[OK]\n"
+
+	for workload in "${workloads[@]}"; do
+		tail --quiet -n +2 "$results_directory/${workload}"_*.csv >> "$results_directory/${workload}.csv"
+		rm "$results_directory/${workload}"_*.csv
+	done
+}
+
+
+# Process the experimental results and generate human-friendly results for success rate, throughput, and latency
+process_results() {
+	if (($# != 1)); then
+		error_msg "invalid number of arguments ($#, expected 1)"
+		return 1
+	elif ! [[ -d "$1" ]]; then
+		error_msg "directory $1 does not exist"
+		return 1
 	fi
-done

-echo "$success_count / $total_count"
+	local -r results_directory="$1"
+
+	printf "Processing Results: \n"
+
+	# Write headers to CSVs
+	printf "Payload,p50,p90,p99,p100\n" >> "$results_directory/latency.csv"
+
+	for workload in "${workloads[@]}"; do
+
+		# Filter on 200s, subtract DNS time, convert from s to ms, and sort
+		awk -F, '$7 == 200 {print (($1 - $2) * 1000)}' < "$results_directory/$workload.csv" \
+			| sort -g > "$results_directory/$workload-response.csv"
+
+		oks=$(wc -l < "$results_directory/$workload-response.csv")
+		((oks == 0)) && continue # If all errors, skip line
+
+		# Generate Latency Data for csv
+		awk '
+			BEGIN {
+				sum = 0
+				p50 = int('"$oks"' * 0.5)
+				p90 = int('"$oks"' * 0.9)
+				p99 = int('"$oks"' * 0.99)
+				p100 = '"$oks"'
+				printf "'"$workload"',"
+			}
+			NR==p50  {printf "%1.4f,",  $0}
+			NR==p90  {printf "%1.4f,",  $0}
+			NR==p99  {printf "%1.4f,",  $0}
+			NR==p100 {printf "%1.4f\n", $0}
+		' < "$results_directory/$workload-response.csv" >> "$results_directory/latency.csv"
+
+		# Delete scratch file used for sorting/counting
+		rm -rf "$results_directory/$workload-response.csv"
+	done
+
+	# Transform csvs to dat files for gnuplot
+	csv_to_dat "$results_directory/latency.csv"
+
+	printf "[OK]\n"
+	return 0
+}
+
+experiment_main() {
+	local -r hostname="$1"
+	local -r results_directory="$2"
+	local same_image # for the cifar10_same workload
+
+	get_random_image same_image
+
+	run_functional_tests "$hostname" "$results_directory" || return 1
+	run_perf_tests "$hostname" "$results_directory" || return 1
+	process_results "$results_directory" || return 1
+}
+
+validate_dependencies curl
+
+declare -ar workloads=(cifar10_rand cifar10_same)
+declare -Ar port=(
+	[cifar10_rand]=10000
+	[cifar10_same]=10001
+)
+
+# Sort the images by the number of labeled plates
+declare -a cifar10_images=(./images/bmp/*)

-if [ "$1" != "-d" ]; then
-	sleep 5
-	echo -n "Running Cleanup: "
-	rm result_*.jpg
-	pkill sledgert > /dev/null 2> /dev/null
-	echo "[DONE]"
-fi
+main "$@"
--- a/runtime/experiments/applications/imageclassification/spec.json
+++ b/runtime/experiments/applications/imageclassification/spec.json
@ -1,15 +1,31 @@
 {
  "active": true,
-  "name": "cifar10",
+  "name": "cifar10_rand",
  "path": "cifar10_wasm.so",
  "port": 10000,
  "expected-execution-us": 5000,
  "relative-deadline-us": 50000,
  "argsize": 1,
  "http-req-headers": [],
-  "http-req-content-type": "image/png",
+  "http-req-content-type": "image/bmp",
  "http-req-size": 4096,
  "http-resp-headers": [],
  "http-resp-size": 128,
  "http-resp-content-type": "text/plain"
+},
+{
+ "active": true,
+ "name": "cifar10_same",
+ "path": "cifar10_wasm.so",
+ "port": 10001,
+ "expected-execution-us": 5000,
+ "relative-deadline-us": 50000,
+ "argsize": 1,
+ "http-req-headers": [],
+ "http-req-content-type": "image/bmp",
+ "http-req-size": 4096,
+ "http-resp-headers": [],
+ "http-resp-size": 128,
+ "http-resp-content-type": "text/plain"
 }
+