haiwan
/
sledge
0
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: better test driver and more resilient tests

Browse Source
main
Sean McBride 4 years ago
parent 72ed672b16
commit cc4cc96cc3
6 changed files with 241 additions and 53 deletions
Show Stats Download Patch File Download Diff File

2
runtime/Makefile
Unescape View File

@ -108,7 +108,7 @@ CFILES += thirdparty/dist/lib/http_parser.o
JSMNCFLAGS += -DJSMN_STATIC
JSMNCFLAGS += -DJSMN_STRICT
all: clean runtime
all: runtime
runtime:
@echo "Compiling runtime"

2
runtime/experiments/applications/imageresize/test/run.sh
Unescape View File

@ -16,7 +16,7 @@ fi
if [ "$1" != "-d" ]; then
PATH="$binary_directory:$PATH" LD_LIBRARY_PATH="$binary_directory:$LD_LIBRARY_PATH" sledgert "$experiment_directory/spec.json" &
sleep 1
sleep 2
else
echo "Running under gdb"
fi

16
runtime/experiments/applications/ocr/fivebyeight/run.sh
Unescape View File

@ -20,6 +20,22 @@ fi
expected_result="$(cat ./expected_result.txt)"
# Retry 5 times in case the runtime startup is slow
retries=5
for ((i = 0; i < retries; i++)); do
result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@5x8.pnm" localhost:10000 2> /dev/null)
if [[ "$result" == "$expected_result" ]]; then
break
fi
if ((i == 4)); then
echo "Retries exhaused"
exit 1
fi
sleep 1
done
success_count=0
total_count=50

17
runtime/experiments/applications/ocr/handwriting/run.sh
Unescape View File

@ -19,6 +19,23 @@ else
fi
expected_result="$(cat ./expected_result.txt)"
# Retry 5 times in case the runtime startup is slow
retries=5
for ((i = 0; i < retries; i++)); do
result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@handwrt1.pnm" localhost:10000 2> /dev/null)
if [[ "$result" == "$expected_result" ]]; then
break
fi
if ((i == 4)); then
echo "Retries exhaused"
exit 1
fi
sleep 1
done
success_count=0
total_count=50

17
runtime/experiments/applications/ocr/hyde/run.sh
Unescape View File

@ -18,6 +18,23 @@ else
fi
expected_result="$(cat ./expected_result.txt)"
# Retry 5 times in case the runtime startup is slow
retries=5
for ((i = 0; i < retries; i++)); do
result=$(curl -H 'Expect:' -H "Content-Type: text/plain" --data-binary "@hyde.pnm" localhost:10000 2> /dev/null)
if [[ "$result" == "$expected_result" ]]; then
break
fi
if ((i == 4)); then
echo "Retries exhaused"
exit 1
fi
sleep 1
done
success_count=0
total_count=50

240
test.sh
Unescape View File

@ -1,5 +1,6 @@
#!/bin/bash
# Test Driver Script
# TODO: Consider refactoring into Make infrastructure
if [[ $0 != "./test.sh" ]]; then
echo "Must run in same directory as ./test.sh"
@ -8,62 +9,199 @@ fi
base_dir=$(pwd)
# Awsm
cd "$base_dir/awsm" && cargo build --release || exit 1
declare -ra tests=(
ocr_hyde
ocr_handwriting
ocr_fivebyeight
ocr_by_word
ocr_by_font
ocr_by_dpi
ekf_by_iteration
ekf_one_iteration
image_classification
image_resize
lpd_by_resolution
lpd_by_plate_count
bimodal
concurrency
payload
)
# Sledge
cd "$base_dir/runtime" && make clean all || exit 1
# OCR Build
# FIXME: gocr incorectly reports up to date
make clean gocr -B -C "$base_dir/runtime/tests" || exit 1
declare -a failed_tests=()
# OCR Tests
# FIXME: OCR tests seem to sporadically fail and then work on rerun. Don't bail on fail for now
# cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh || exit 1
cd "$base_dir/runtime/experiments/applications/ocr/hyde" && ./run.sh
cd "$base_dir/runtime/experiments/applications/ocr/handwriting" && ./run.sh
cd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" && ./run.sh
cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./install.sh
cd "$base_dir/runtime/experiments/applications/ocr/by_word" && ./run.sh
cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./install.sh
cd "$base_dir/runtime/experiments/applications/ocr/by_font" && ./run.sh
cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./install.sh
cd "$base_dir/runtime/experiments/applications/ocr/by_dpi" && ./run.sh
# EKF Build
make clean tinyekf -B -C "$base_dir/runtime/tests" || exit 1
# FIXME: OCR tests seem to sporadically fail and then work on rerun.
ocr_hyde() {
make gocr -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ocr/hyde" || exit 1
./run.sh || failed_tests+=("ocr_hyde")
popd || exit 1
return 0
}
ocr_handwriting() {
make gocr -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ocr/handwriting" || exit 1
./run.sh || failed_tests+=("ocr_handwriting")
popd || exit 1
return 0
}
ocr_fivebyeight() {
make gocr -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ocr/fivebyeight" || exit 1
./run.sh || failed_tests+=("ocr_fivebyeight")
popd || exit 1
return 0
}
ocr_by_word() {
make gocr -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ocr/by_word" || exit 1
./install.sh || exit 1
./run.sh || failed_tests+=("ocr_by_word")
popd || exit 1
return 0
}
ocr_by_font() {
make gocr -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ocr/by_font" || exit 1
./install.sh || exit 1
./run.sh || failed_tests+=("ocr_by_font")
popd || exit 1
return 0
}
ocr_by_dpi() {
make gocr -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ocr/by_dpi" || exit 1
./install.sh || exit 1
./run.sh || failed_tests+=("ocr_by_dpi")
popd || exit 1
return 0
}
# EKF Tests
cd "$base_dir/runtime/experiments/applications/ekf/by_iteration" && ./run.sh || exit 1
cd "$base_dir/runtime/experiments/applications/ekf/one_iteration" && ./run.sh || exit 1
ekf_by_iteration() {
make tinyekf -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ekf/by_iteration" || exit 1
./run.sh || failed_tests+=("ocr_by_dpi")
popd || exit 1
return 0
}
# cifar10 Build
make clean cifar10 -B -C "$base_dir/runtime/tests" || exit 1
ekf_one_iteration() {
make tinyekf -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/ekf/one_iteration" || exit 1
./run.sh || failed_tests+=("ekf_one_iteration")
popd || exit 1
return 0
}
# cifar10 Tests
cd "$base_dir/runtime/experiments/applications/imageclassification" && ./run.sh || exit 1
# sod Build
make clean sod -B -C "$base_dir/runtime/tests" || exit 1
# sod Tests
# FIXME: sod sporadically fails. May be related to a file descriptor bug
# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh || exit 1
# cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh || exit 1
cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./install.sh
cd "$base_dir/runtime/experiments/applications/imageresize/test" && ./run.sh
cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./install.sh
cd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" && ./run.sh
cd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" && ./run.sh
image_classification() {
make cifar10 -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/imageclassification" || exit 1
./run.sh || failed_tests+=("image_classification")
popd || exit 1
return 0
}
image_resize() {
make sod -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/imageresize/test" || exit 1
./install.sh || exit 1
./run.sh || failed_tests+=("image_resize")
popd || exit 1
return 0
}
lpd_by_resolution() {
# Disabled... Seems to be dumping raw image data
return 0
make sod -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/imageresize/by_resolution" || exit 1
./install.sh || exit 1
./run.sh || failed_tests+=("lpd_by_resolution")
popd || exit 1
return 0
}
lpd_by_plate_count() {
# Disabled... Seems to be dumping raw image data
return 0
make sod -C "$base_dir/runtime/tests" || exit 1
pushd "$base_dir/runtime/experiments/applications/licenseplate/by_plate_count" || exit 1
./run.sh || failed_tests+=("lpd_by_plate_count")
popd || exit 1
return 0
}
bimodal() {
echo "Bimodal"
# TODO: Make Dependency "fibonacci_wasm.so"
pushd "$base_dir/runtime/experiments/bimodal/" || exit 1
./run.sh || failed_tests+=("bimodal")
popd || exit 1
return 0
}
concurrency() {
echo "Concurrency"
# TODO: Make Dependency "empty_wasm.so"
pushd "$base_dir/runtime/experiments/concurrency/" || exit 1
./run.sh || failed_tests+=("concurrency")
popd || exit 1
return 0
}
payload() {
echo "Payload"
# TODO: Make Dependency "work1k_wasm.so" "work10k_wasm.so" "work100k_wasm.so" "work1m_wasm.so"
pushd "$base_dir/runtime/experiments/payload/" || exit 1
./run.sh || failed_tests+=("concurrency")
popd || exit 1
return 0
}
main() {
cd "$base_dir/awsm" && cargo build --release || exit 1
make all -C "$base_dir/runtime" || exit 1
# If no arguments are provided, run all tests
if (($# == 0)); then
for test in "${tests[@]}"; do
"$test"
done
local -i failure_count=${#failed_tests[@]}
if ((failure_count > 0)); then
printf "Failed Tests\n"
for test in "${failed_tests[@]}"; do
printf "\t%s\n" "$test"
done
exit 1
fi
exit 0
fi
# Otherwise, only run the tests passed as arguments
for test in "$@"; do
if [[ ! " ${tests[*]} " =~ " ${test} " ]]; then
printf "Error: %s is not a known test\n" "$test"
else
if "$test"; then
printf "%s: Pass\n" "$test"
else
printf "%s: Fail\n" "$test"
fi
fi
done
}
main "$@"

Write Preview
Loading…
Cancel
Save