You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
550 lines
18 KiB
550 lines
18 KiB
# shellcheck shell=bash
|
|
# shellcheck disable=SC1091,SC2034,SC2155
|
|
if [ -n "$__framework_sh__" ]; then return; fi
|
|
__framework_sh__=$(date)
|
|
|
|
#
|
|
# This framework simplifies the scripting of experiments.
|
|
#
|
|
# It is designed around the idea of static experiments composed of one or more variants expressed by
|
|
# environment variables written to .env files. The default behavior is localhost mode, which runs a background
|
|
# server daemon and then execute a client driver script. If multiple .env files are defined, the framework
|
|
# automatically sets environment variables, starts the runtime as a background process, executes the client driver,
|
|
# stops the runtime, and clears the environment variables. The framework allows you to run the same logic on separate
|
|
# client and server hosts. It also provides various options to run under perf, gdb, valgrind, etc.
|
|
#
|
|
# To keep experiments relatively uniform, I suggest adding a single run.sh file inside your experiment.
|
|
#
|
|
# Your run.sh file should be started with the following snippet, which sources the framework.sh file
|
|
# and delegates all external arguments to the framework via the framework_init function. The first few lines are
|
|
# used to temporary add the directory containing BASH library scripts to your PATH environment variable. You may
|
|
# need to modify __run_sh__bash_libraries_relative_path to adjust the relative path depending on the location
|
|
# of your experimental directory.
|
|
#
|
|
###############################################################################################################################
|
|
# #!/bin/bash
|
|
# __run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
|
|
# __run_sh__bash_libraries_relative_path="../bash_libraries"
|
|
# __run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
|
|
# export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
|
|
#
|
|
# source framework.sh || exit 1
|
|
#
|
|
# framework_init "$@"
|
|
###############################################################################################################################
|
|
#
|
|
# Use chmod +x run.sh to make your script executable, and then run ./run.sh --help to test it.
|
|
# You should see help information if successful.
|
|
#
|
|
# At this point, your script can run the server with defaults usings the --debug, --perf, --serve, and --valgrind
|
|
#
|
|
# To run a client or the default localhost mode that runs a client and a server on the same machine, you have to
|
|
# implement a function called experiment_client in run.sh above your call to framework_init.
|
|
#
|
|
# This function receives two arguments:
|
|
# - a results directory where you should intermediate files and reports / charts
|
|
# - a target hostname where you should target requests
|
|
#
|
|
# This function is called once per experimental variant, and the results directory is adjusted accordingly to
|
|
# keep your data organized.
|
|
#
|
|
# If your server logs data that you need to process, you can execute post-processing logic by implementing
|
|
# experiment_server_post, which gets called once per variant after the server background task terminates
|
|
#
|
|
# If you want to do one time setup before executing any variants, perform this logic in run.sh before calling
|
|
# framework_init.
|
|
#
|
|
# In addition to framework.sh, the bash_libraries directory contains a number of utility functions that are useful
|
|
# for cleaning and refactoring your data and error handling. Feel free to contribute utility functions to this
|
|
# directory if you write bash functions that you believe are reusable!
|
|
#
|
|
# It is also a good idea to look at the other experiments to get ideas for your script.
|
|
#
|
|
# If you are using VSCode, you have a number of extensions that help with bash development. This includes:
|
|
# - Bash IDE - an IntelliSense langauge server for bash
|
|
# - shell-format - Auto-format on save for bash
|
|
# - ShellCheck - an excellent linter for shell scripts.
|
|
#
|
|
# If you are not using VSCode, you may need to manually run shfmt against your script to keep formatting consistent
|
|
#
|
|
# Happy Scripting!
|
|
|
|
source "fn_exists.sh" || exit 1
|
|
source "path_join.sh" || exit 1
|
|
source "panic.sh" || exit 1
|
|
source "type_checks.sh" || exit 1
|
|
source "validate_dependencies.sh" || exit 1
|
|
|
|
__framework_sh__usage() {
|
|
echo "$0 [options...]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " -d,--debug Debug under GDB but do not run client"
|
|
echo " -e,--envfile=<file name> Load an Env File. No path and pass filename with *.env extension"
|
|
echo " -h,--help Display usage information"
|
|
echo " -n,--name=<experiment> Provide a unique name for this experimental run. Defaults to timestamp"
|
|
echo " -p,--perf Run under perf. Limited to running on a baremetal Linux host!"
|
|
echo " -s,--serve Serve but do not run client"
|
|
echo " -t,--target=<target url> Execute as client against remote URL"
|
|
echo " -l,--loadgen=<hey|lt> Pick load generator to use for client. Default is hey."
|
|
echo " -v,--valgrind Debug under Valgrind but do not run client"
|
|
}
|
|
|
|
# Declares application level global state
|
|
__framework_sh__initialize_globals() {
|
|
# timestamp is used to name the results directory for a particular test run
|
|
# This can be manually overridden via the name argument
|
|
declare -gir __framework_sh__timestamp=$(date +%s)
|
|
declare -g __framework_sh__experiment_name="$__framework_sh__timestamp"
|
|
|
|
# Globals used by parse_arguments
|
|
declare -g __framework_sh__target=""
|
|
declare -g __framework_sh__role=""
|
|
declare -g __framework_sh__envfile=""
|
|
declare -g __framework_sh__loadgen=""
|
|
|
|
# Configure environment variables
|
|
declare -gr __framework_sh__application_directory="$(dirname "$(realpath "$0"))")"
|
|
declare -gr __framework_sh__path=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
|
|
declare -gr common_directory="$(cd "$__framework_sh__path" && cd ../common && pwd)"
|
|
local -r binary_directory="$(cd "$__framework_sh__path" && cd ../../runtime/bin && pwd)"
|
|
export PATH=$binary_directory:$PATH
|
|
export LD_LIBRARY_PATH=$binary_directory:$LD_LIBRARY_PATH
|
|
}
|
|
|
|
# Parses arguments from the user and sets associates global state
|
|
__framework_sh__parse_arguments() {
|
|
for i in "$@"; do
|
|
case $i in
|
|
-t=* | --target=*)
|
|
if [[ "$__framework_sh__role" == "server" ]]; then
|
|
echo "Cannot set target when server"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__role=client
|
|
__framework_sh__target="${i#*=}"
|
|
shift
|
|
;;
|
|
-s | --serve)
|
|
if [[ "$__framework_sh__role" == "client" ]]; then
|
|
echo "Cannot use -s,--serve with -t,--target"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__role=server
|
|
shift
|
|
;;
|
|
-d | --debug)
|
|
if [[ "$__framework_sh__role" == "client" ]]; then
|
|
echo "Cannot use -d,--debug with -t,--target"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__role=debug
|
|
shift
|
|
;;
|
|
-p | --perf)
|
|
if [[ "$__framework_sh__role" == "client" ]]; then
|
|
echo "Cannot use -p,--perf with -t,--target"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__role=perf
|
|
shift
|
|
;;
|
|
-v | --valgrind)
|
|
if [[ "$__framework_sh__role" == "client" ]]; then
|
|
echo "Cannot use -v,--valgrind with -t,--target"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__role=valgrind
|
|
shift
|
|
;;
|
|
-n=* | --name=*)
|
|
echo "Set experiment name to ${i#*=}"
|
|
__framework_sh__experiment_name+=" ${i#*=}"
|
|
shift
|
|
;;
|
|
-e=* | --envfile=*)
|
|
if [[ "$__framework_sh__role" == "client" ]]; then
|
|
echo "Expected to be used with run by the server"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__envfile="${i#*=}"
|
|
echo "Set envfile to $__framework_sh__envfile"
|
|
;;
|
|
-l=* | --loadgen=*)
|
|
if [[ "$__framework_sh__role" == "server" ]]; then
|
|
echo "Expected to be used with run by the client"
|
|
__framework_sh__usage
|
|
return 1
|
|
fi
|
|
__framework_sh__loadgen="${i#*=}"
|
|
echo "Set load generator to $__framework_sh__loadgen"
|
|
shift
|
|
;;
|
|
-h | --help)
|
|
__framework_sh__usage
|
|
exit 0
|
|
;;
|
|
nuclio | Nuclio)
|
|
echo "Running for Nuclio"
|
|
;;
|
|
*)
|
|
echo "$1 is a not a valid option"
|
|
__framework_sh__usage
|
|
return 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$__framework_sh__envfile" ]]; then
|
|
if [[ -d "$__framework_sh__application_directory/res-$__framework_sh__role/$__framework_sh__experiment_name/" ]]; then
|
|
echo "Experiment $__framework_sh__experiment_name already exists. Pick a unique name!"
|
|
exit 1
|
|
fi
|
|
else
|
|
if [[ ! -f "$__framework_sh__envfile" ]]; then
|
|
echo "$__framework_sh__envfile not found!!!"
|
|
exit 1
|
|
fi
|
|
short_name="$(basename "${__framework_sh__envfile/.env/}")"
|
|
echo "$__framework_sh__application_directory/res-$__framework_sh__role/$__framework_sh__experiment_name/$short_name/"
|
|
if [[ -d "$__framework_sh__application_directory/res-$__framework_sh__role/$__framework_sh__experiment_name/$short_name/" ]]; then
|
|
echo "Variant $short_name was already run in experiment ${__framework_sh__experiment_name}."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# default to both if no arguments were passed
|
|
if [[ -z "$__framework_sh__role" ]]; then
|
|
__framework_sh__role="both"
|
|
__framework_sh__target="localhost"
|
|
fi
|
|
|
|
# Set globals as read only
|
|
declare -r __framework_sh__target
|
|
declare -r __framework_sh__role
|
|
}
|
|
|
|
# Log hardware and software info for the execution
|
|
__framework_sh__log_environment() {
|
|
validate_dependencies git
|
|
echo "*******"
|
|
echo "* Git *"
|
|
echo "*******"
|
|
git log | head -n 1 | cut -d' ' -f2
|
|
git status
|
|
echo ""
|
|
|
|
echo "************"
|
|
echo "* Spec.json *"
|
|
echo "************"
|
|
cat "$(path_join "$__framework_sh__application_directory" ./spec.json)"
|
|
echo ""
|
|
|
|
echo "************"
|
|
echo "* Makefile *"
|
|
echo "************"
|
|
cat "$(path_join "$__framework_sh__path" ../../Makefile)"
|
|
echo ""
|
|
|
|
echo "**************"
|
|
echo "* " "$0" " *"
|
|
echo "**************"
|
|
cat "$(path_join "$__framework_sh__application_directory" "$0")"
|
|
echo ""
|
|
|
|
echo "************"
|
|
echo "* Hardware *"
|
|
echo "************"
|
|
lscpu
|
|
echo ""
|
|
|
|
echo "*************"
|
|
echo "* Execution *"
|
|
echo "*************"
|
|
}
|
|
|
|
# $1 - Results Directory
|
|
# $2 - How to run (foreground|background)
|
|
# $3 - JSON specification
|
|
__framework_sh__start_runtime() {
|
|
printf "Starting Runtime: "
|
|
if (($# != 2)); then
|
|
printf "[ERR]\n"
|
|
panic "invalid number of arguments \"$1\""
|
|
return 1
|
|
elif ! [[ $1 =~ ^(foreground|background)$ ]]; then
|
|
printf "[ERR]\n"
|
|
panic "expected foreground or background was \"$1\""
|
|
return 1
|
|
elif [[ ! -f "$2" || "$2" != *.json ]]; then
|
|
printf "[ERR]\n"
|
|
panic "\"$2\" does not exist or is not a JSON"
|
|
return 1
|
|
fi
|
|
|
|
local -r how_to_run="$1"
|
|
local -r specification="$2"
|
|
|
|
local -r log_name=log_server.txt
|
|
local log="$RESULTS_DIRECTORY/${log_name}"
|
|
|
|
local -i sledgert_pid=0
|
|
|
|
__framework_sh__log_environment >> "$log"
|
|
|
|
case "$how_to_run" in
|
|
"background")
|
|
sledgert "$specification" >> "$log" 2>> "$log" &
|
|
sledgert_pid=$!
|
|
# Pad with a sleep to allow runtime to initialize before startup tasks run
|
|
# This should be improved adding some sort of ping/ping heartbeat to the runtime
|
|
# so the script can spin until initializaiton is complete
|
|
sleep 1
|
|
if ! kill -0 $sledgert_pid; then
|
|
printf "[ERR]\n"
|
|
return 1
|
|
fi
|
|
;;
|
|
"foreground")
|
|
sledgert "$specification"
|
|
fn_exists experiment_server_post && experiment_server_post "$RESULTS_DIRECTORY"
|
|
;;
|
|
esac
|
|
|
|
printf "[OK]\n"
|
|
return 0
|
|
}
|
|
|
|
__framework_sh__run_server() {
|
|
if (($# != 1)); then
|
|
printf "[ERR]\n"
|
|
panic "Invalid number of arguments. Saw $#. Expected 1."
|
|
return 1
|
|
elif ! [[ $1 =~ ^(foreground|background)$ ]]; then
|
|
printf "[ERR]\n"
|
|
panic "expected foreground or background was \"$3\""
|
|
return 1
|
|
fi
|
|
|
|
local -r how_to_run="$1"
|
|
|
|
__framework_sh__start_runtime "$how_to_run" "$__framework_sh__application_directory/spec.json" || {
|
|
echo "__framework_sh__start_runtime RC: $?"
|
|
panic "Error calling __framework_sh__start_runtime $how_to_run $__framework_sh__application_directory/spec.json"
|
|
return 1
|
|
}
|
|
|
|
return 0
|
|
}
|
|
|
|
__framework_sh__run_perf() {
|
|
validate_dependencies perf
|
|
perf record -g -s sledgert "$__framework_sh__application_directory/spec.json"
|
|
fn_exists experiment_server_post && experiment_server_post "$RESULTS_DIRECTORY"
|
|
}
|
|
|
|
__framework_sh__run_valgrind() {
|
|
validate_dependencies valgrind
|
|
valgrind --leak-check=full sledgert "$__framework_sh__application_directory/spec.json"
|
|
fn_exists experiment_server_post && experiment_server_post "$RESULTS_DIRECTORY"
|
|
}
|
|
|
|
# Starts the Sledge Runtime under GDB
|
|
__framework_sh__run_debug() {
|
|
validate_dependencies gdb
|
|
|
|
local project_directory=$(cd ../.. && pwd)
|
|
|
|
if [[ "$project_directory" != "/sledge/runtime" ]]; then
|
|
printf "It appears that you are not running in the container. Substituting path to match host environment\n"
|
|
gdb \
|
|
--eval-command="handle SIGUSR1 noprint nostop" \
|
|
--eval-command="handle SIGPIPE noprint nostop" \
|
|
--eval-command="set pagination off" \
|
|
--eval-command="set print pretty" \
|
|
--eval-command="set substitute-path /sledge/runtime $project_directory" \
|
|
--eval-command="run $__framework_sh__application_directory/spec.json" \
|
|
sledgert
|
|
else
|
|
gdb \
|
|
--eval-command="handle SIGUSR1 noprint nostop" \
|
|
--eval-command="handle SIGPIPE noprint nostop" \
|
|
--eval-command="set pagination off" \
|
|
--eval-command="set print pretty" \
|
|
--eval-command="run $__framework_sh__application_directory/spec.json" \
|
|
sledgert
|
|
fi
|
|
|
|
fn_exists experiment_server_post && experiment_server_post "$RESULTS_DIRECTORY"
|
|
return 0
|
|
}
|
|
|
|
__framework_sh__run_client() {
|
|
local -r log_name=log_client.txt
|
|
local log="$RESULTS_DIRECTORY/${log_name}"
|
|
|
|
__framework_sh__log_environment >> "$log"
|
|
|
|
experiment_client "$__framework_sh__target" "$RESULTS_DIRECTORY" "$__framework_sh__loadgen" || return 1
|
|
|
|
return 0
|
|
}
|
|
|
|
__framework_sh__load_env_file() {
|
|
local envfile="${1:?envfile not defined}"
|
|
[[ ! -f "$envfile" ]] && echo "envfile not found" && return 1
|
|
|
|
local short_name
|
|
short_name="$(basename "${envfile/.env/}")"
|
|
printf "Running %s\n" "$short_name"
|
|
|
|
while read -r line; do
|
|
echo export "${line?}"
|
|
export "${line?}"
|
|
done < "$envfile"
|
|
|
|
__framework_sh__create_and_export_results_directory "$short_name"
|
|
}
|
|
|
|
__framework_sh__unset_env_file() {
|
|
local envfile="${1:?envfile not defined}"
|
|
[[ ! -f "$envfile" ]] && echo "envfile not found" && return 1
|
|
|
|
while read -r line; do
|
|
echo unset "${line//=*/}"
|
|
unset "${line//=*/}"
|
|
done < "$envfile"
|
|
}
|
|
|
|
__framework_sh__run_both_env() {
|
|
local envfile="${1:?envfile not defined}"
|
|
__framework_sh__load_env_file "$envfile"
|
|
|
|
__framework_sh__run_server background || {
|
|
panic "Error calling __framework_sh__run_server"
|
|
return 1
|
|
}
|
|
|
|
__framework_sh__run_client || {
|
|
__framework_sh__unset_env_file "$envfile"
|
|
__framework_sh__stop_runtime
|
|
return 1
|
|
}
|
|
|
|
__framework_sh__stop_runtime || {
|
|
panic "Error calling __framework_sh__stop_runtime"
|
|
__framework_sh__unset_env_file "$envfile"
|
|
return 1
|
|
}
|
|
|
|
__framework_sh__unset_env_file "$envfile"
|
|
}
|
|
|
|
# If envfile explicitly passed, just run that. Otherwise, run all
|
|
__framework_sh__run_both() {
|
|
shopt -s nullglob
|
|
|
|
if [[ -n "$__framework_sh__envfile" ]]; then
|
|
__framework_sh__run_both_env "$__framework_sh__envfile"
|
|
else
|
|
local -i envfiles_found=0
|
|
for envfile in "$common_directory"/*.env; do
|
|
((envfiles_found++))
|
|
__framework_sh__run_both_env "$envfile" || exit 1
|
|
done
|
|
((envfiles_found == 0)) && {
|
|
echo "No *.env files found. Nothing to run!"
|
|
exit 1
|
|
}
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Optionally accepts a subdirectory
|
|
# This is intended to namespace distinct runtime configs under a single namespace
|
|
__framework_sh__create_and_export_results_directory() {
|
|
local -r subdirectory=${1:-""}
|
|
|
|
local dir="$__framework_sh__application_directory/res-$__framework_sh__role/$__framework_sh__experiment_name"
|
|
# local dir="$__framework_sh__application_directory/res-$__framework_sh__role/$__framework_sh__experiment_name/$subdirectory"
|
|
|
|
mkdir -p "$dir" || {
|
|
panic "mkdir -p $dir"
|
|
return 1
|
|
}
|
|
|
|
export RESULTS_DIRECTORY="$dir"
|
|
}
|
|
|
|
# Responsible for ensuring that the experiment file meets framework assumptions
|
|
__framework_sh__validate_client() {
|
|
if [[ $(type -t experiment_client) != "function" ]]; then
|
|
panic "function experiment_client was not defined"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
framework_init() {
|
|
__framework_sh__initialize_globals || exit 1
|
|
__framework_sh__parse_arguments "$@" || exit 1
|
|
__framework_sh__create_and_export_results_directory || exit 1
|
|
|
|
case $__framework_sh__role in
|
|
both)
|
|
__framework_sh__validate_client || exit 1
|
|
__framework_sh__run_both
|
|
;;
|
|
server)
|
|
[[ -n "$__framework_sh__envfile" ]] && __framework_sh__load_env_file "$__framework_sh__application_directory/$__framework_sh__envfile"
|
|
__framework_sh__run_server foreground
|
|
;;
|
|
debug)
|
|
[[ -n "$__framework_sh__envfile" ]] && __framework_sh__load_env_file "$__framework_sh__application_directory/$__framework_sh__envfile"
|
|
__framework_sh__run_debug
|
|
;;
|
|
perf)
|
|
[[ -n "$__framework_sh__envfile" ]] && __framework_sh__load_env_file "$__framework_sh__application_directory/$__framework_sh__envfile"
|
|
__framework_sh__run_perf
|
|
;;
|
|
valgrind)
|
|
[[ -n "$__framework_sh__envfile" ]] && __framework_sh__load_env_file "$__framework_sh__application_directory/$__framework_sh__envfile"
|
|
__framework_sh__run_valgrind
|
|
;;
|
|
client)
|
|
__framework_sh__validate_client || exit 1
|
|
__framework_sh__run_client
|
|
;;
|
|
*)
|
|
echo "Invalid state"
|
|
false
|
|
;;
|
|
esac
|
|
|
|
return "$?"
|
|
}
|
|
|
|
__framework_sh__stop_runtime() {
|
|
printf "Stopping Runtime: "
|
|
# Ignoring RC of 1, as it indicates no matching process
|
|
pkill sledgert > /dev/null 2> /dev/null
|
|
(($? > 1)) && {
|
|
printf "[ERR]\npkill sledgrt: %d\n" $?
|
|
exit 1
|
|
}
|
|
pkill hey > /dev/null 2> /dev/null
|
|
(($? > 1)) && {
|
|
printf "[ERR]\npkill hey: %d\n" $?
|
|
exit 1
|
|
}
|
|
|
|
fn_exists experiment_server_post && experiment_server_post "$RESULTS_DIRECTORY"
|
|
printf "[OK]\n"
|
|
}
|