Merge branch 'master' of github.com:gwsystems/sledge-serverless-framework into http-parsing-listener

Dockerfile.x86_64

@@ -5,7 +5,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ARG HEY_URL=https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64
 ARG WASI_SDK_URL=https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk_12.0_amd64.deb
 ARG SHFMT_URL=https://github.com/mvdan/sh/releases/download/v3.2.4/shfmt_v3.2.4_linux_amd64
-ARG SHELLCHECK_URL=https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
+ARG SHELLCHECK_URL=https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz
 
 # Use bash, not sh
 SHELL ["/bin/bash", "-c"]

devenv.sh

@@ -136,6 +136,7 @@ envrun() {
 		echo "Starting ${SYS_DOC_NAME}"
 		docker run \
 			--privileged \
+			--network="host" \
 			--security-opt seccomp:unconfined \
 			--name=${SYS_DOC_NAME} \
 			--detach \

runtime/include/runtime.h

@@ -30,6 +30,7 @@
 #define RUNTIME_MAX_TENANT_COUNT          32 /* Static buffer size for per-worker globals */
 #define RUNTIME_READ_WRITE_VECTOR_LENGTH  16
 #define RUNTIME_RELATIVE_DEADLINE_US_MAX  3600000000 /* One Hour. Fits in uint32_t */
+#define RUNTIME_RUNQUEUE_SIZE             256        /* Minimum guaranteed size. Might grow! */
 
 enum RUNTIME_SIGALRM_HANDLER
 {

runtime/include/sandbox_set_as_allocated.h

@@ -8,6 +8,7 @@
 #include "current_sandbox.h"
 #include "ps_list.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 /**
@@ -28,4 +29,8 @@ sandbox_set_as_allocated(struct sandbox *sandbox)
 	sandbox_state_history_init(&sandbox->state_history);
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_ALLOCATED);
 	sandbox_state_totals_increment(SANDBOX_ALLOCATED);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, SANDBOX_UNINITIALIZED);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_ALLOCATED);
 }

runtime/include/sandbox_set_as_asleep.h

@@ -8,6 +8,7 @@
 #include "sandbox_types.h"
 #include "sandbox_state.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 
 /**
  * Transitions a sandbox to the SANDBOX_ASLEEP state.
@@ -37,11 +38,16 @@ sandbox_set_as_asleep(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_ASLEEP);
 	sandbox_state_totals_increment(SANDBOX_ASLEEP);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_ASLEEP);
 }
 
 static inline void

runtime/include/sandbox_set_as_complete.h

@@ -10,6 +10,7 @@
 #include "sandbox_perf_log.h"
 #include "sandbox_state.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_summarize_page_allocations.h"
 #include "sandbox_types.h"
 
@@ -40,7 +41,8 @@ sandbox_set_as_complete(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_COMPLETE);
 	sandbox_state_totals_increment(SANDBOX_COMPLETE);
@@ -55,6 +57,10 @@ sandbox_set_as_complete(struct sandbox *sandbox, sandbox_state_t last_state)
 	sandbox_perf_log_print_entry(sandbox);
 	sandbox_summarize_page_allocations(sandbox);
 
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_COMPLETE);
+
 	/* Does not add to completion queue until in cooperative scheduler */
 }
 

runtime/include/sandbox_set_as_error.h

@@ -10,6 +10,7 @@
 #include "sandbox_functions.h"
 #include "sandbox_perf_log.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_summarize_page_allocations.h"
 #include "panic.h"
 
@@ -48,8 +49,9 @@ sandbox_set_as_error(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
-	sandbox->duration_of_state[last_state] += duration_of_last_state;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
+	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_ERROR);
 	sandbox_state_totals_increment(SANDBOX_ERROR);
 	sandbox_state_totals_decrement(last_state);
@@ -61,6 +63,10 @@ sandbox_set_as_error(struct sandbox *sandbox, sandbox_state_t last_state)
 	sandbox_perf_log_print_entry(sandbox);
 	sandbox_summarize_page_allocations(sandbox);
 
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_ERROR);
+
 	/* Does not add to completion queue until in cooperative scheduler */
 }
 

runtime/include/sandbox_set_as_initialized.h

@@ -8,6 +8,7 @@
 #include "current_sandbox.h"
 #include "ps_list.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 /**
@@ -35,9 +36,14 @@ sandbox_set_as_initialized(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_INITIALIZED);
 	sandbox_state_totals_increment(SANDBOX_INITIALIZED);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_INITIALIZED);
 }

runtime/include/sandbox_set_as_interrupted.h

@@ -8,6 +8,7 @@
 #include "panic.h"
 #include "sandbox_functions.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 static inline void
@@ -23,11 +24,16 @@ sandbox_set_as_interrupted(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	/* We do not append SANDBOX_INTERRUPTED to the sandbox_state_history because it would quickly fill the buffer */
 	sandbox_state_totals_increment(SANDBOX_INTERRUPTED);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_INTERRUPTED);
 }
 
 static inline void

runtime/include/sandbox_set_as_preempted.h

@@ -7,6 +7,7 @@
 #include "local_runqueue.h"
 #include "panic.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 /**
@@ -37,11 +38,16 @@ sandbox_set_as_preempted(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_PREEMPTED);
 	sandbox_state_totals_increment(SANDBOX_PREEMPTED);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_PREEMPTED);
 }
 
 static inline void

runtime/include/sandbox_set_as_returned.h

@@ -9,6 +9,7 @@
 #include "sandbox_functions.h"
 #include "sandbox_state.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 /**
@@ -45,9 +46,14 @@ sandbox_set_as_returned(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_RETURNED);
 	sandbox_state_totals_increment(SANDBOX_RETURNED);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_RETURNED);
 }

runtime/include/sandbox_set_as_runnable.h

@@ -7,6 +7,7 @@
 #include "local_runqueue.h"
 #include "panic.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 /**
@@ -44,11 +45,16 @@ sandbox_set_as_runnable(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_RUNNABLE);
 	sandbox_state_totals_increment(SANDBOX_RUNNABLE);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_RUNNABLE);
 }
 
 

runtime/include/sandbox_set_as_running_sys.h

@@ -8,6 +8,7 @@
 #include "panic.h"
 #include "sandbox_functions.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 
 static inline void
@@ -39,11 +40,15 @@ sandbox_set_as_running_sys(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
-	sandbox->timestamp_of.last_state_change = now;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_RUNNING_SYS);
 	sandbox_state_totals_increment(SANDBOX_RUNNING_SYS);
 	sandbox_state_totals_decrement(last_state);
+
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_RUNNING_SYS);
 }
 
 static inline void

runtime/include/sandbox_set_as_running_user.h

@@ -7,6 +7,7 @@
 #include "current_sandbox.h"
 #include "panic.h"
 #include "sandbox_state_history.h"
+#include "sandbox_state_transition.h"
 #include "sandbox_types.h"
 #include "sandbox_functions.h"
 
@@ -35,12 +36,17 @@ sandbox_set_as_running_user(struct sandbox *sandbox, sandbox_state_t last_state)
 
 	/* State Change Bookkeeping */
 	assert(now > sandbox->timestamp_of.last_state_change);
-	sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change);
+	sandbox->last_state_duration = now - sandbox->timestamp_of.last_state_change;
+	sandbox->duration_of_state[last_state] += sandbox->last_state_duration;
 	sandbox->timestamp_of.last_state_change = now;
 	sandbox_state_history_append(&sandbox->state_history, SANDBOX_RUNNING_USER);
 	sandbox_state_totals_increment(SANDBOX_RUNNING_USER);
 	sandbox_state_totals_decrement(last_state);
 
+	/* State Change Hooks */
+	sandbox_state_transition_from_hook(sandbox, last_state);
+	sandbox_state_transition_to_hook(sandbox, SANDBOX_RUNNING_USER);
+
 	barrier();
 	sandbox->state = SANDBOX_RUNNING_USER;
 	/* WARNING: All code after this assignment is preemptable */

runtime/include/sandbox_state_transition.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <assert.h>
+
+#include "sandbox_state.h"
+#include "sandbox_types.h"
+
+
+typedef void (*sandbox_state_transition_hook_t)(struct sandbox *);
+extern sandbox_state_transition_hook_t sandbox_state_transition_from_hooks[SANDBOX_STATE_COUNT];
+extern sandbox_state_transition_hook_t sandbox_state_transition_to_hooks[SANDBOX_STATE_COUNT];
+
+static inline void
+sandbox_state_transition_from_hook(struct sandbox *sandbox, sandbox_state_t state)
+{
+	assert(sandbox != NULL);
+	assert(state < SANDBOX_STATE_COUNT);
+
+	sandbox_state_transition_from_hooks[state](sandbox);
+}
+
+static inline void
+sandbox_state_transition_to_hook(struct sandbox *sandbox, sandbox_state_t state)
+{
+	assert(sandbox != NULL);
+	assert(state < SANDBOX_STATE_COUNT);
+
+	sandbox_state_transition_to_hooks[state](sandbox);
+}
+
+static inline void
+sandbox_state_transition_from_hook_register(sandbox_state_t state, sandbox_state_transition_hook_t cb)
+{
+	assert(state < SANDBOX_STATE_COUNT);
+	assert(cb != NULL);
+
+	sandbox_state_transition_from_hooks[state] = cb;
+}
+
+static inline void
+sandbox_state_transition_to_hook_register(sandbox_state_t state, sandbox_state_transition_hook_t cb)
+{
+	assert(state < SANDBOX_STATE_COUNT);
+	assert(cb != NULL);
+
+	sandbox_state_transition_to_hooks[state] = cb;
+}

runtime/include/sandbox_types.h

@@ -57,6 +57,7 @@ struct sandbox {
 	/* Scheduling and Temporal State */
 	struct sandbox_timestamps timestamp_of;
 	uint64_t                  duration_of_state[SANDBOX_STATE_COUNT];
+	uint64_t                  last_state_duration;
 
 	uint64_t absolute_deadline;
 	uint64_t admissions_estimate; /* estimated execution time (cycles) * runtime_admissions_granularity / relative

runtime/src/local_runqueue_minheap.c

@@ -12,8 +12,6 @@
 #include "sandbox_functions.h"
 #include "runtime.h"
 
-#define INITIAL_LOCAL_RUNQUEUE_MINHEAP_CAPACITY 256
-
 thread_local static struct priority_queue *local_runqueue_minheap;
 
 /**
@@ -84,8 +82,7 @@ void
 local_runqueue_minheap_initialize()
 {
 	/* Initialize local state */
-	local_runqueue_minheap = priority_queue_initialize(INITIAL_LOCAL_RUNQUEUE_MINHEAP_CAPACITY, false,
+	local_runqueue_minheap = priority_queue_initialize(RUNTIME_RUNQUEUE_SIZE, false, sandbox_get_priority);
-	                                                   sandbox_get_priority);
 
 	/* Register Function Pointers for Abstract Scheduling API */
 	struct local_runqueue_config config = { .add_fn      = local_runqueue_minheap_add,

runtime/src/sandbox_state_transition.c

@@ -0,0 +1,41 @@
+#include "sandbox_state_transition.h"
+
+void
+sandbox_state_transition_hook_default(struct sandbox *sandbox)
+{
+	return;
+}
+
+/* Called while nonpreemptable */
+sandbox_state_transition_hook_t sandbox_state_transition_to_hooks[SANDBOX_STATE_COUNT] = {
+
+	[SANDBOX_UNINITIALIZED] = sandbox_state_transition_hook_default,
+	[SANDBOX_ALLOCATED]     = sandbox_state_transition_hook_default,
+	[SANDBOX_INITIALIZED]   = sandbox_state_transition_hook_default,
+	[SANDBOX_RUNNABLE]      = sandbox_state_transition_hook_default,
+	[SANDBOX_INTERRUPTED]   = sandbox_state_transition_hook_default,
+	[SANDBOX_PREEMPTED]     = sandbox_state_transition_hook_default,
+	[SANDBOX_RUNNING_SYS]   = sandbox_state_transition_hook_default,
+	[SANDBOX_RUNNING_USER]  = sandbox_state_transition_hook_default,
+	[SANDBOX_ASLEEP]        = sandbox_state_transition_hook_default,
+	[SANDBOX_RETURNED]      = sandbox_state_transition_hook_default,
+	[SANDBOX_COMPLETE]      = sandbox_state_transition_hook_default,
+	[SANDBOX_ERROR]         = sandbox_state_transition_hook_default
+};
+
+/* Called while nonpreemptable */
+sandbox_state_transition_hook_t sandbox_state_transition_from_hooks[SANDBOX_STATE_COUNT] = {
+
+	[SANDBOX_UNINITIALIZED] = sandbox_state_transition_hook_default,
+	[SANDBOX_ALLOCATED]     = sandbox_state_transition_hook_default,
+	[SANDBOX_INITIALIZED]   = sandbox_state_transition_hook_default,
+	[SANDBOX_RUNNABLE]      = sandbox_state_transition_hook_default,
+	[SANDBOX_INTERRUPTED]   = sandbox_state_transition_hook_default,
+	[SANDBOX_PREEMPTED]     = sandbox_state_transition_hook_default,
+	[SANDBOX_RUNNING_SYS]   = sandbox_state_transition_hook_default,
+	[SANDBOX_RUNNING_USER]  = sandbox_state_transition_hook_default,
+	[SANDBOX_ASLEEP]        = sandbox_state_transition_hook_default,
+	[SANDBOX_RETURNED]      = sandbox_state_transition_hook_default,
+	[SANDBOX_COMPLETE]      = sandbox_state_transition_hook_default,
+	[SANDBOX_ERROR]         = sandbox_state_transition_hook_default
+};

tests/bash_libraries/framework.sh

@@ -227,16 +227,22 @@ __framework_sh__log_environment() {
 	git status
 	echo ""
 
+	echo "************"
+	echo "* Spec.json *"
+	echo "************"
+	cat "$(path_join "$__framework_sh__application_directory" ./spec.json)"
+	echo ""
+
 	echo "************"
 	echo "* Makefile *"
 	echo "************"
 	cat "$(path_join "$__framework_sh__path" ../../Makefile)"
 	echo ""
 
-	echo "**********"
+	echo "**************"
-	echo "* Run.sh *"
+	echo "* " "$0" " *"
-	echo "**********"
+	echo "**************"
-	cat "$(path_join "$__framework_sh__application_directory" ./run.sh)"
+	cat "$(path_join "$__framework_sh__application_directory" "$0")"
 	echo ""
 
 	echo "************"

tests/bash_libraries/percentiles_table.sh

@@ -36,7 +36,7 @@ percentiles_table_row() {
 	local -r table_file="${2:?table_file not set}"
 	check_file table_file
 	local -r row_label="${3:?row_label not set}"
-	local -r format_string="${4:-%1.4f}"
+	local -r format_string="${4:-%1.2f}"
 
 	# Count the number of results
 	local -i sample_size

tests/deadline_description/run.sh

@@ -17,7 +17,6 @@ source validate_dependencies.sh || exit 1
 validate_dependencies awk hey jq
 
 # Please keep the element ordered alphabetically!
-# declare -a workloads=(cifar10 ekf gocr lpd resize)
 declare -a workloads=(cifar10 ekf gocr lpd resize)
 declare -a multiples=(1.5 2.0 3.0 4.0)
 declare -ri percentile=50

tests/stack_overflow/edf_preemption.env

@@ -1,3 +1,2 @@
 SLEDGE_SCHEDULER=EDF
 SLEDGE_DISABLE_PREEMPTION=false
-SLEDGE_SIGALRM_HANDLER=TRIAGED