Merge pull request #134 from gwsystems/fix-tests

Automate real world tests

.gitmodules

@@ -13,4 +13,17 @@ path = runtime/thirdparty/jsmn
 url = https://github.com/gwsystems/jsmn.git
 [submodule "runtime/tests/gocr"]
 	path = runtime/tests/gocr
-	url = https://github.com/gwsystems/gocr
+	url = https://github.com/gwsystems/gocr.git
+	branch = sledge
+[submodule "runtime/tests/TinyEKF"]
+	path = runtime/tests/TinyEKF
+	url = https://github.com/gwsystems/TinyEKF.git
+	branch = sledge
+[submodule "runtime/tests/CMSIS_5_NN"]
+	path = runtime/tests/CMSIS_5_NN
+	url = https://github.com/gwsystems/CMSIS_5_NN.git
+	branch = sledge
+[submodule "runtime/tests/sod"]
+	path = runtime/tests/sod
+	url = https://github.com/gwsystems/sod.git
+	branch = sledge

README.md

@@ -116,6 +116,10 @@ Content-type: text/plain
 
 When done, terminal the SLEdge runtime with `Ctrl+c`
 
+## Running Test Workloads
+
+Various synthetic and real-world tests can be found in `runtime/experiments`. Generally, each experiment can be run be executing the `run.sh` script.
+
 ## Removing the SLEdge Runtime
 
 If you are finished working with the SLEdge runtime and wish to remove it, run the following command to delete our Docker build and runtime images.

runtime/experiments/applications/ekf/.gitignore

@@ -0,0 +1 @@
+result.dat

runtime/experiments/applications/ekf/README.md

@@ -0,0 +1,9 @@
+# EKF
+
+Executes TinyEKF as shows by [You Chong's GPS example](http://www.mathworks.com/matlabcentral/fileexchange/31487-extended-kalman-filter-ekf--for-gps)
+
+In order to be compatible with the stdin/stdout model of serverless, the input and output files are binary concatenations of various C structs.
+
+See `main()` in `runtime/tests/TinyEKF/extras/c/gps_ekf_fn.c` for specifics.
+
+This test executes multiple iterations, comparing the binary result against a known memoized result stored at `expected_result.dat`.

runtime/experiments/applications/ekf/debug.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Executes the runtime in GDB
+# Substitutes the absolute path from the container with a path relatively derived from the location of this script
+# This allows debugging outside of the Docker container
+# Also disables pagination and stopping on SIGUSR1
+
+experiment_directory=$(pwd)
+project_directory=$(cd ../../.. && pwd)
+binary_directory=$(cd "$project_directory"/bin && pwd)
+
+export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
+export PATH="$binary_directory:$PATH"
+
+gdb --eval-command="handle SIGUSR1 nostop" \
+  --eval-command="handle SIGPIPE nostop" \
+  --eval-command="set pagination off" \
+  --eval-command="set substitute-path /sledge/runtime $project_directory" \
+  --eval-command="run $experiment_directory/spec.json" \
+  sledgert

runtime/experiments/applications/ekf/run.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Executes the runtime in GDB
+# Substitutes the absolute path from the container with a path relatively derived from the location of this script
+# This allows debugging outside of the Docker container
+# Also disables pagination and stopping on SIGUSR1
+
+experiment_directory=$(pwd)
+project_directory=$(cd ../../.. && pwd)
+binary_directory=$(cd "$project_directory"/bin && pwd)
+
+# Copy data if not here
+if [[ ! -f "./ekf_raw.dat" ]]; then
+  cp ../../../tests/TinyEKF/extras/c/ekf_raw.dat ./ekf_raw.dat
+fi
+
+if [ "$1" != "-d" ]; then
+  PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" &
+  sleep 1
+else
+  echo "Running under gdb"
+fi
+
+expected_result="$(tr -d '\0' <./expected_result.dat)"
+
+success_count=0
+total_count=50
+
+for ((i = 0; i < total_count; i++)); do
+  echo "$i"
+  result="$(curl -H 'Expect:' -H "Content-Type: application/octet-stream" --data-binary "@ekf_raw.dat" localhost:10000 2>/dev/null | tr -d '\0')"
+  if [[ "$expected_result" == "$result" ]]; then
+    success_count=$((success_count + 1))
+  else
+    echo "FAIL"
+  fi
+done
+
+echo "$success_count / $total_count"
+
+if [ "$1" != "-d" ]; then
+  sleep 5
+  echo -n "Running Cleanup: "
+  pkill sledgert >/dev/null 2>/dev/null
+  echo "[DONE]"
+fi

runtime/experiments/applications/ekf/spec.json

@@ -0,0 +1,14 @@
+{
+  "active": "yes",
+  "name": "resize",
+  "path": "ekf_wasm.so",
+  "port": 10000,
+  "relative-deadline-us": 50000,
+  "argsize": 1,
+  "http-req-headers": [],
+  "http-req-content-type": "application/octet-stream",
+  "http-req-size": 1024000,
+  "http-resp-headers": [],
+  "http-resp-size": 1024000,
+  "http-resp-content-type": "application/octet-stream"
+}

runtime/experiments/applications/fivebyeight/spec.json

@@ -1,7 +1,7 @@
 {
   "active": "yes",
   "name": "gocr",
-  "path": "gocr.aso",
+  "path": "gocr_wasm.so",
   "port": 10000,
   "relative-deadline-us": 500000000,
   "expected-execution-us": 5000000,

runtime/experiments/applications/handwriting/spec.json

@@ -1,7 +1,7 @@
 {
   "active": "yes",
   "name": "gocr",
-  "path": "gocr.aso",
+  "path": "gocr_wasm.so",
   "port": 10000,
   "relative-deadline-us": 50000000000,
   "argsize": 1,

runtime/experiments/applications/hyde/spec.json

@@ -1,7 +1,7 @@
 {
   "active": "yes",
   "name": "gocr",
-  "path": "gocr.aso",
+  "path": "gocr_wasm.so",
   "port": 10000,
   "relative-deadline-us": 50000000000,
   "argsize": 1,

runtime/experiments/applications/imageclassification/.gitignore

@@ -0,0 +1 @@
+flower.jpg

runtime/experiments/applications/imageclassification/README.md

@@ -0,0 +1,15 @@
+# Image Classification
+
+Classifies a tiny 32x32 image from the CIFAR-10 dataset into one of ten different classes:
+airplane
+automobile
+bird
+cat
+deer
+dog
+frog
+horse
+ship
+truck
+
+An initial run incorrectly classified all images as class 8. It seems that this was because the image data was not being read correctly. @emil916 created a fix, but the performance on the images pulled from the classifier's associated website still seems poor and is captured in `erroneous_output.txt`.

runtime/experiments/applications/imageclassification/debug.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Executes the runtime in GDB
+# Substitutes the absolute path from the container with a path relatively derived from the location of this script
+# This allows debugging outside of the Docker container
+# Also disables pagination and stopping on SIGUSR1
+
+experiment_directory=$(pwd)
+project_directory=$(cd ../../.. && pwd)
+binary_directory=$(cd "$project_directory"/bin && pwd)
+
+export LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH"
+export PATH="$binary_directory:$PATH"
+
+gdb --eval-command="handle SIGUSR1 nostop" \
+  --eval-command="handle SIGPIPE nostop" \
+  --eval-command="set pagination off" \
+  --eval-command="set substitute-path /sledge/runtime $project_directory" \
+  --eval-command="run $experiment_directory/spec.json" \
+  sledgert

runtime/experiments/applications/imageclassification/erroneous_output.txt

@@ -0,0 +1,200 @@
+Classifying airplane1.png
+1
+Classifying airplane2.png
+1
+Classifying airplane3.png
+9
+Classifying airplane4.png
+1
+Classifying airplane5.png
+1
+Classifying airplane6.png
+1
+Classifying airplane7.png
+8
+Classifying airplane8.png
+9
+Classifying airplane9.png
+1
+Classifying airplane10.png
+1
+Classifying automobile1.png
+1
+Classifying automobile2.png
+9
+Classifying automobile3.png
+9
+Classifying automobile4.png
+1
+Classifying automobile5.png
+1
+Classifying automobile6.png
+9
+Classifying automobile7.png
+9
+Classifying automobile8.png
+1
+Classifying automobile9.png
+1
+Classifying automobile10.png
+9
+Classifying bird1.png
+1
+Classifying bird2.png
+1
+Classifying bird3.png
+8
+Classifying bird4.png
+1
+Classifying bird5.png
+1
+Classifying bird6.png
+8
+Classifying bird7.png
+1
+Classifying bird8.png
+9
+Classifying bird9.png
+1
+Classifying bird10.png
+9
+Classifying cat1.png
+1
+Classifying cat2.png
+9
+Classifying cat3.png
+9
+Classifying cat4.png
+9
+Classifying cat5.png
+9
+Classifying cat6.png
+1
+Classifying cat7.png
+9
+Classifying cat8.png
+1
+Classifying cat9.png
+1
+Classifying cat10.png
+9
+Classifying deer1.png
+9
+Classifying deer2.png
+9
+Classifying deer3.png
+1
+Classifying deer4.png
+1
+Classifying deer5.png
+9
+Classifying deer6.png
+9
+Classifying deer7.png
+9
+Classifying deer8.png
+9
+Classifying deer9.png
+9
+Classifying deer10.png
+6
+Classifying dog1.png
+9
+Classifying dog2.png
+1
+Classifying dog3.png
+9
+Classifying dog4.png
+9
+Classifying dog5.png
+9
+Classifying dog6.png
+9
+Classifying dog7.png
+1
+Classifying dog8.png
+9
+Classifying dog9.png
+9
+Classifying dog10.png
+9
+Classifying frog1.png
+1
+Classifying frog2.png
+1
+Classifying frog3.png
+9
+Classifying frog4.png
+9
+Classifying frog5.png
+9
+Classifying frog6.png
+9
+Classifying frog7.png
+9
+Classifying frog8.png
+9
+Classifying frog9.png
+9
+Classifying frog10.png
+9
+Classifying horse1.png
+1
+Classifying horse2.png
+9
+Classifying horse3.png
+9
+Classifying horse4.png
+9
+Classifying horse5.png
+9
+Classifying horse6.png
+9
+Classifying horse7.png
+1
+Classifying horse8.png
+9
+Classifying horse9.png
+9
+Classifying horse10.png
+1
+Classifying ship1.png
+9
+Classifying ship2.png
+9
+Classifying ship3.png
+1
+Classifying ship4.png
+8
+Classifying ship5.png
+1
+Classifying ship6.png
+8
+Classifying ship7.png
+9
+Classifying ship8.png
+1
+Classifying ship9.png
+9
+Classifying ship10.png
+1
+Classifying truck1.png
+9
+Classifying truck2.png
+9
+Classifying truck3.png
+9
+Classifying truck4.png
+9
+Classifying truck5.png
+9
+Classifying truck6.png
+1
+Classifying truck7.png
+9
+Classifying truck8.png
+9
+Classifying truck9.png
+1
+Classifying truck10.png
+9