feat: A bit more stable

.vscode/settings.json

@@ -60,7 +60,24 @@
 		"dlfcn.h": "c",
 		"chrono": "c",
 		"common.h": "c",
-		"listener_thread.h": "c"
+		"listener_thread.h": "c",
+		"mutex": "cpp",
+		"client_socket.h": "c",
+		"deque.h": "c",
+		"likely.h": "c",
+		"software_interrupt.h": "c",
+		"sandbox_t.h": "c",
+		"http_request.h": "c",
+		"context.h": "c",
+		"worker_thread.h": "c",
+		"software_interrupt_enable.h": "c",
+		"streambuf": "c",
+		"exception": "c",
+		"iosfwd": "c",
+		"new": "c",
+		"ostream": "c",
+		"sstream": "c",
+		"memory_resource": "c"
 	},
 	"files.exclude": {
 		"**/.git": true,

runtime/experiments/bash_libraries/framework.sh

@@ -170,6 +170,27 @@ __framework_sh__log_environment() {
 	echo "*************"
 }
 
+__framework_sh__load_env_file() {
+	local envfile="$1"
+	echo "Loading $envfile"
+	if [[ -f "$envfile" ]]; then
+		while read -r line; do
+			echo export "${line?}"
+			export "${line?}"
+		done < "$envfile"
+	fi
+}
+
+__framework_sh__unset_env_file() {
+	local envfile="$1"
+	if [[ -f "$envfile" ]]; then
+		while read -r line; do
+			echo unset "${line//=*/}"
+			unset "${line//=*/}"
+		done < "$envfile"
+	fi
+}
+
 # $1 - Results Directory
 # $2 - How to run (foreground|background)
 # $3 - JSON specification
@@ -259,6 +280,7 @@ __framework_sh__run_valgrind() {
 __framework_sh__run_debug() {
 	# shellcheck disable=SC2155
 	local project_directory=$(cd ../.. && pwd)
+	__framework_sh__load_env_file "$envfile"
 
 	if [[ "$project_directory" != "/sledge/runtime" ]]; then
 		printf "It appears that you are not running in the container. Substituting path to match host environment\n"
@@ -277,6 +299,7 @@ __framework_sh__run_debug() {
 			--eval-command="run $__framework_sh__application_directory/spec.json" \
 			sledgert
 	fi
+	__framework_sh__unset_env_file "$envfile"
 	return 0
 }
 
@@ -289,26 +312,6 @@ __framework_sh__run_client() {
 	return 0
 }
 
-__framework_sh__load_env_file() {
-	local envfile="$1"
-	if [[ -f "$envfile" ]]; then
-		while read -r line; do
-			echo export "${line?}"
-			export "${line?}"
-		done < "$envfile"
-	fi
-}
-
-__framework_sh__unset_env_file() {
-	local envfile="$1"
-	if [[ -f "$envfile" ]]; then
-		while read -r line; do
-			echo unset "${line//=*/}"
-			unset "${line//=*/}"
-		done < "$envfile"
-	fi
-}
-
 __framework_sh__run_both() {
 	local short_name
 	shopt -s nullglob

runtime/experiments/bimodal/edf_nopreemption.env

@@ -1,2 +1,4 @@
 SLEDGE_SCHEDULER=EDF
 SLEDGE_DISABLE_PREEMPTION=true
+SLEDGE_NWORKERS=6
+SLEDGE_QUANTUM_US=5000

runtime/experiments/bimodal/edf_preemption.env

@@ -1,3 +1,5 @@
 SLEDGE_SCHEDULER=EDF
 SLEDGE_DISABLE_PREEMPTION=false
 SLEDGE_SIGALRM_HANDLER=TRIAGED
+SLEDGE_NWORKERS=6
+SLEDGE_QUANTUM_US=5000

runtime/experiments/bimodal/fifo_nopreemption.env

@@ -1,2 +1,4 @@
 SLEDGE_SCHEDULER=FIFO
 SLEDGE_DISABLE_PREEMPTION=true
+SLEDGE_NWORKERS=6
+SLEDGE_QUANTUM_US=5000

runtime/experiments/bimodal/fifo_preemption.env

@@ -1,2 +1,4 @@
 SLEDGE_SCHEDULER=FIFO
 SLEDGE_DISABLE_PREEMPTION=false
+SLEDGE_NWORKERS=6
+SLEDGE_QUANTUM_US=5000

runtime/include/arch/aarch64/context.h

@@ -4,6 +4,8 @@
 #include <assert.h>
 #include "arch/common.h"
 #include "current_sandbox.h"
+#include "software_interrupt.h"
+#include "software_interrupt_enable.h"
 
 #define ARCH_SIG_JMP_OFF 0x100 /* Based on code generated! */
 
@@ -29,6 +31,7 @@ arch_context_init(struct arch_context *actx, reg_t ip, reg_t sp)
 
 	actx->regs[UREG_SP] = sp;
 	actx->regs[UREG_IP] = ip;
+	actx->preemptable   = false;
 }
 
 /**
@@ -108,6 +111,12 @@ arch_context_switch(struct arch_context *a, struct arch_context *b)
 	reg_t *a_registers = a->regs, *b_registers = b->regs;
 	assert(a_registers && b_registers);
 
+	/* If switching back to a sandbox context marked as preemptable, reenable
+	 * interrupts before jumping
+	 * TODO: What if we receive a signal inside the inline assembly?
+	 */
+	if (b->preemptable) software_interrupt_enable();
+
 	asm volatile("mov x0, sp\n\t"
 	             "adr x1, reset%=\n\t"
 	             "str x1, [%[a], 8]\n\t"

runtime/include/arch/context.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include "arch/common.h"
+#include "software_interrupt.h"
+
 /*
  * This header is the single entry point into the arch_context code.
  * It includes processor independent code and conditionally includes architecture
@@ -53,6 +56,9 @@ arch_mcontext_restore(mcontext_t *active_context, struct arch_context *sandbox_c
 
 	/* Restore mcontext */
 	memcpy(active_context, &sandbox_context->mctx, sizeof(mcontext_t));
+
+	/* Reenable software interrupts if we restored a preemptable sandbox */
+	if (sandbox_context->preemptable) software_interrupt_enable();
 }
 
 
@@ -74,8 +80,10 @@ arch_mcontext_save(struct arch_context *sandbox_context, const mcontext_t *activ
 	assert(sandbox_context != &worker_thread_base_context);
 
 	/* Transitioning from {Unused, Running} -> Slow */
-	assert(sandbox_context->variant == ARCH_CONTEXT_VARIANT_UNUSED
-	       || sandbox_context->variant == ARCH_CONTEXT_VARIANT_RUNNING);
+	// TODO: Another assert firing here. What is this?
+	// assert(sandbox_context->variant == ARCH_CONTEXT_VARIANT_UNUSED
+	//        || sandbox_context->variant == ARCH_CONTEXT_VARIANT_RUNNING);
+	// debuglog("Slow transitioning from %s\n", arch_context_variant_print(sandbox_context->variant));
 	sandbox_context->variant = ARCH_CONTEXT_VARIANT_SLOW;
 
 	/* Copy mcontext */

runtime/include/arch/x86_64/context.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include "arch/common.h"
+#include "software_interrupt.h"
+#include "software_interrupt_enable.h"
 
 #define ARCH_SIG_JMP_OFF 8
 
@@ -14,6 +16,7 @@
 static void __attribute__((noinline)) arch_context_init(struct arch_context *actx, reg_t ip, reg_t sp)
 {
 	assert(actx != NULL);
+	assert(software_interrupt_is_disabled);
 
 	if (ip == 0 && sp == 0) {
 		actx->variant = ARCH_CONTEXT_VARIANT_UNUSED;
@@ -21,6 +24,8 @@ static void __attribute__((noinline)) arch_context_init(struct arch_context *act
 		actx->variant = ARCH_CONTEXT_VARIANT_FAST;
 	}
 
+	actx->preemptable = false;
+
 	if (sp) {
 		/*
 		 * context_switch conventions: bp is expected to be on top of the stack
@@ -125,7 +130,9 @@ arch_context_switch(struct arch_context *a, struct arch_context *b)
 	if (b == NULL) b = &worker_thread_base_context;
 
 	/* A Transition {Unused, Running} -> Fast */
-	assert(a->variant == ARCH_CONTEXT_VARIANT_UNUSED || a->variant == ARCH_CONTEXT_VARIANT_RUNNING);
+	// TODO: This errored out. What is the variant here?
+	// debuglog("Context switching from %s\n", arch_context_variant_print(a->variant));
+	// assert(a->variant == ARCH_CONTEXT_VARIANT_UNUSED || a->variant == ARCH_CONTEXT_VARIANT_RUNNING);
 
 	/* B Transition {Fast, Slow} -> Running */
 	assert(b->variant == ARCH_CONTEXT_VARIANT_FAST || b->variant == ARCH_CONTEXT_VARIANT_SLOW);
@@ -139,6 +146,13 @@ arch_context_switch(struct arch_context *a, struct arch_context *b)
 	reg_t *a_registers = a->regs, *b_registers = b->regs;
 	assert(a_registers && b_registers);
 
+	/* If fast switching back to a sandbox context marked as preemptable, reenable
+	 * interrupts before jumping. If this is a slow context switch, defer renabling until
+	 * arch_mcontext_restore
+	 * TODO: What if we receive a signal inside the inline assemly?
+	 */
+	if (b->variant == ARCH_CONTEXT_VARIANT_FAST && b->preemptable) software_interrupt_enable();
+
 	asm volatile(
 	  /* Create a new stack frame */
 	  "pushq %%rbp\n\t"       /*  stack[stack_len++] = base_pointer */

runtime/include/sandbox.h

@@ -3,16 +3,12 @@
 #include <ucontext.h>
 #include <stdbool.h>
 
-#include "arch/context.h"
-#include "client_socket.h"
-#include "deque.h"
+#include "arch/arch_context_t.h"
+#include "http_parser.h"
 #include "http_request.h"
 #include "http_response.h"
-#include "module.h"
 #include "ps_list.h"
 #include "sandbox_request.h"
-#include "sandbox_state.h"
-#include "software_interrupt.h"
 
 #define SANDBOX_FILE_DESCRIPTOR_PREOPEN_MAGIC (707707707) /* upside down LOLLOLLOL 🤣😂🤣*/
 #define SANDBOX_MAX_IO_HANDLE_COUNT           32
@@ -114,9 +110,6 @@ void            sandbox_free(struct sandbox *sandbox);
 void            sandbox_free_linear_memory(struct sandbox *sandbox);
 int             sandbox_get_file_descriptor(struct sandbox *sandbox, int io_handle_index);
 int             sandbox_initialize_io_handle(struct sandbox *sandbox);
-void            sandbox_main(struct sandbox *sandbox);
-
-void sandbox_close_http(struct sandbox *sandbox);
 
 INLINE void sandbox_set_as_initialized(struct sandbox *sandbox, struct sandbox_request *sandbox_request,
                                        uint64_t allocation_timestamp);

runtime/include/software_interrupt.h

@@ -4,6 +4,7 @@
 #include <errno.h>
 #include <panic.h>
 #include <pthread.h>
+#include <runtime.h>
 #include <signal.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -17,6 +18,8 @@
 
 extern __thread volatile sig_atomic_t software_interrupt_is_disabled;
 extern uint64_t                       software_interrupt_interval_duration_in_cycles;
+extern __thread volatile sig_atomic_t software_interrupt_deferred_sigalrm;
+extern volatile sig_atomic_t          software_interrupt_deferred_sigalrm_max[RUNTIME_WORKER_THREAD_CORE_COUNT];
 
 /*************************
  * Public Static Inlines *
@@ -30,18 +33,6 @@ software_interrupt_disable(void)
 	}
 }
 
-
-/**
- * Enables signals
- */
-static inline void
-software_interrupt_enable(void)
-{
-	if (__sync_bool_compare_and_swap(&software_interrupt_is_disabled, 1, 0) == false) {
-		panic("Recursive call to software_interrupt_enable\n");
-	}
-}
-
 /**
  * @returns boolean if signals are enabled
  */
@@ -107,3 +98,4 @@ software_interrupt_unmask_signal(int signal)
 void software_interrupt_initialize(void);
 void software_interrupt_arm_timer(void);
 void software_interrupt_disarm_timer(void);
+void software_interrupt_deferred_sigalrm_max_print(void);

runtime/include/software_interrupt_enable.h

@@ -0,0 +1,27 @@
+#pragma once
+#include "software_interrupt.h"
+#include "worker_thread.h"
+
+// Broken into a distict header because of circular dependency issues
+
+/**
+ * Reenables signals, replaying deferred signals as needed
+ */
+static inline void
+software_interrupt_enable(void)
+{
+	/* Trigger missed SIGALRM */
+	if (software_interrupt_deferred_sigalrm > 0) {
+		if (software_interrupt_deferred_sigalrm > software_interrupt_deferred_sigalrm_max[worker_thread_idx]) {
+			software_interrupt_deferred_sigalrm_max[worker_thread_idx] =
+			  software_interrupt_deferred_sigalrm;
+		}
+		software_interrupt_deferred_sigalrm = 0;
+
+		worker_thread_sched();
+	}
+
+	if (__sync_bool_compare_and_swap(&software_interrupt_is_disabled, 1, 0) == false) {
+		panic("Recursive call to software_interrupt_enable\n");
+	}
+}

runtime/include/worker_thread.h

@@ -1,12 +1,18 @@
 #pragma once
 
+#include <signal.h>
+
 #include "generic_thread.h"
 #include "runtime.h"
+#include "sandbox.h"
 
 extern __thread int worker_thread_epoll_file_descriptor;
 extern __thread int worker_thread_idx;
 
+void  worker_thread_block_current_sandbox(void);
 void *worker_thread_main(void *return_code);
+void  worker_thread_process_io(void);
+void  worker_thread_sched();
 
 /**
  * Translates WASM offsets into runtime VM pointers

runtime/src/local_runqueue_minheap.c

@@ -1,5 +1,6 @@
 #include <stdint.h>
 
+#include "arch/context.h"
 #include "client_socket.h"
 #include "current_sandbox.h"
 #include "debuglog.h"
@@ -119,24 +120,18 @@ void
 local_runqueue_minheap_preempt(ucontext_t *user_context)
 {
 	assert(user_context != NULL);
-
-	/* Prevent nested preemption */
-	software_interrupt_disable();
+	assert(software_interrupt_is_disabled);
 
 	struct sandbox *current_sandbox = current_sandbox_get();
 
 	/* If current_sandbox is null, there's nothing to preempt, so let the "main" scheduler run its course. */
-	if (current_sandbox == NULL) {
-		software_interrupt_enable();
-		return;
-	};
+	if (current_sandbox == NULL) return;
 
 	/* The current sandbox should be the head of the runqueue */
 	assert(local_runqueue_minheap_is_empty() == false);
 
-	bool     should_enable_software_interrupt = true;
-	uint64_t local_deadline                   = priority_queue_peek(local_runqueue_minheap);
-	uint64_t global_deadline                  = global_request_scheduler_peek();
+	uint64_t local_deadline  = priority_queue_peek(local_runqueue_minheap);
+	uint64_t global_deadline = global_request_scheduler_peek();
 	/* If we're able to get a sandbox request with a tighter deadline, preempt the current context and run it */
 	struct sandbox_request *sandbox_request = NULL;
 	if (global_deadline < local_deadline) {
@@ -188,10 +183,8 @@ local_runqueue_minheap_preempt(ucontext_t *user_context)
 		 */
 		runtime_worker_threads_deadline[worker_thread_idx] = next_sandbox->absolute_deadline;
 		arch_context_restore_new(&user_context->uc_mcontext, &next_sandbox->ctxt);
-		should_enable_software_interrupt = false;
 	}
 done:
-	if (should_enable_software_interrupt) software_interrupt_enable();
 	return;
 err_sandbox_allocate:
 	client_socket_send(sandbox_request->socket_descriptor, 503);
@@ -215,10 +208,9 @@ sandbox_get_priority(void *element)
 void
 local_runqueue_minheap_initialize()
 {
+	assert(software_interrupt_is_disabled);
 	/* Initialize local state */
-	software_interrupt_disable();
 	local_runqueue_minheap = priority_queue_initialize(256, false, sandbox_get_priority);
-	software_interrupt_enable();
 
 	/* Register Function Pointers for Abstract Scheduling API */
 	struct local_runqueue_config config = { .add_fn      = local_runqueue_minheap_add,

runtime/src/runtime.c

@@ -42,6 +42,7 @@ runtime_cleanup()
 {
 	if (runtime_sandbox_perf_log != NULL) fflush(runtime_sandbox_perf_log);
 
+	software_interrupt_deferred_sigalrm_max_print();
 	exit(EXIT_SUCCESS);
 }
 

runtime/src/sandbox.c

@@ -3,8 +3,11 @@
 #include <pthread.h>
 #include <signal.h>
 #include <sys/mman.h>
+#include <unistd.h>
 
+#include "arch/context.h"
 #include "admissions_control.h"
+#include "client_socket.h"
 #include "current_sandbox.h"
 #include "debuglog.h"
 #include "http_parser_settings.h"
@@ -12,9 +15,12 @@
 #include "local_completion_queue.h"
 #include "local_runqueue.h"
 #include "likely.h"
+#include "module.h"
 #include "panic.h"
 #include "runtime.h"
 #include "sandbox.h"
+#include "software_interrupt.h"
+#include "software_interrupt_enable.h"
 #include "worker_thread.h"
 
 /**
@@ -64,7 +70,7 @@ sandbox_get_file_descriptor(struct sandbox *sandbox, int io_handle_index)
  * @param file_descriptor the file descripter we want to set it to
  * @returns the index that was set or -1 in case of error
  */
-static inline int
+int
 sandbox_set_file_descriptor(struct sandbox *sandbox, int io_handle_index, int file_descriptor)
 {
 	if (!sandbox) return -1;
@@ -445,6 +451,30 @@ sandbox_get_module(struct sandbox *sandbox)
 	return sandbox->module;
 }
 
+static inline void
+current_sandbox_enable_preemption(struct sandbox *sandbox)
+{
+#ifdef LOG_PREEMPTION
+	debuglog("Sandbox %lu - enabling preemption\n", sandbox->id);
+	fflush(stderr);
+#endif
+	assert(sandbox->ctxt.preemptable == false);
+	sandbox->ctxt.preemptable = true;
+	software_interrupt_enable();
+}
+
+static inline void
+current_sandbox_disable_preemption(struct sandbox *sandbox)
+{
+#ifdef LOG_PREEMPTION
+	debuglog("Sandbox %lu - disabling preemption\n", sandbox->id);
+	fflush(stderr);
+#endif
+	assert(sandbox->ctxt.preemptable == true);
+	software_interrupt_disable();
+	sandbox->ctxt.preemptable = false;
+}
+
 /**
  * Sandbox execution logic
  * Handles setup, request parsing, WebAssembly initialization, function execution, response building and
@@ -453,15 +483,16 @@ sandbox_get_module(struct sandbox *sandbox)
 void
 sandbox_start(void)
 {
+	assert(!software_interrupt_is_enabled());
+
 	struct sandbox *sandbox = current_sandbox_get();
 	assert(sandbox != NULL);
 	assert(sandbox->state == SANDBOX_RUNNING);
 
 	char *error_message = "";
 
-	assert(!software_interrupt_is_enabled());
-	arch_context_init(&sandbox->ctxt, 0, 0);
-	software_interrupt_enable();
+	// TODO: I think this should not be called because it was invoked during sandbox_set_as_initialized
+	// arch_context_init(&sandbox->ctxt, 0, 0);
 
 	sandbox_initialize_io_handles_and_file_descriptors(sandbox);
 
@@ -483,8 +514,10 @@ sandbox_start(void)
 	/* Copy the arguments into the WebAssembly sandbox */
 	sandbox_setup_arguments(sandbox);
 
-	/* Executing the function */
-	sandbox->return_value         = module_main(current_module, argument_count, sandbox->arguments_offset);
+	current_sandbox_enable_preemption(sandbox);
+	sandbox->return_value = module_main(current_module, argument_count, sandbox->arguments_offset);
+	current_sandbox_disable_preemption(sandbox);
+
 	sandbox->completion_timestamp = __getcycles();
 
 	/* Retrieve the result, construct the HTTP response, and send to client */
@@ -497,8 +530,6 @@ sandbox_start(void)
 
 	sandbox->response_timestamp = __getcycles();
 
-	software_interrupt_disable();
-
 	assert(sandbox->state == SANDBOX_RUNNING);
 	sandbox_close_http(sandbox);
 	sandbox_set_as_returned(sandbox, SANDBOX_RUNNING);
@@ -518,7 +549,6 @@ err:
 	/* Send a 400 error back to the client */
 	client_socket_send(sandbox->client_socket_descriptor, 400);
 
-	software_interrupt_disable();
 	sandbox_close_http(sandbox);
 	sandbox_set_as_error(sandbox, SANDBOX_RUNNING);
 	goto done;

runtime/src/software_interrupt.c

@@ -18,6 +18,7 @@
 #include "runtime.h"
 #include "sandbox.h"
 #include "software_interrupt.h"
+#include "worker_thread.h"
 
 /*******************
  * Process Globals *
@@ -30,10 +31,26 @@ uint64_t         software_interrupt_interval_duration_in_cycles;
  * Thread Globals *
  *****************/
 
-__thread static volatile sig_atomic_t software_interrupt_SIGALRM_kernel_count = 0;
-__thread static volatile sig_atomic_t software_interrupt_SIGALRM_thread_count = 0;
-__thread static volatile sig_atomic_t software_interrupt_SIGUSR_count         = 0;
-__thread volatile sig_atomic_t        software_interrupt_is_disabled          = 0;
+__thread static volatile sig_atomic_t  software_interrupt_SIGALRM_kernel_count = 0;
+__thread static volatile sig_atomic_t  software_interrupt_SIGALRM_thread_count = 0;
+__thread static volatile sig_atomic_t  software_interrupt_SIGUSR_count         = 0;
+__thread volatile sig_atomic_t         software_interrupt_is_disabled          = 1;
+__thread volatile sig_atomic_t         software_interrupt_deferred_sigalrm     = 0;
+__thread volatile sig_atomic_t         software_interrupt_deferred_sigusr1     = 0;
+__thread _Atomic volatile sig_atomic_t software_interrupt_signal_depth         = 0;
+
+volatile sig_atomic_t software_interrupt_deferred_sigalrm_max[RUNTIME_WORKER_THREAD_CORE_COUNT] = { 0 };
+
+void
+software_interrupt_deferred_sigalrm_max_print()
+{
+	printf("Max Deferred Sigalrms") for (int i = 0; i < runtime_worker_threads_count; i++)
+	{
+		printf("Worker %d: %d\n", software_interrupt_deferred_sigalrm_max[i]);
+	}
+	fflush(stdout);
+}
+
 
 /***************************************
  * Externs
@@ -58,9 +75,6 @@ sigalrm_propagate_workers(siginfo_t *signal_info)
 	/* Signal was sent directly by the kernel, so forward to other threads */
 	if (signal_info->si_code == SI_KERNEL) {
 		software_interrupt_SIGALRM_kernel_count++;
-#ifdef LOG_PREEMPTION
-		debuglog("Kernel SIGALRM: %d!\n", software_interrupt_SIGALRM_kernel_count);
-#endif
 		for (int i = 0; i < runtime_worker_threads_count; i++) {
 			if (pthread_self() == runtime_worker_threads[i]) continue;
 
@@ -81,9 +95,6 @@ sigalrm_propagate_workers(siginfo_t *signal_info)
 		}
 	} else {
 		software_interrupt_SIGALRM_thread_count++;
-#ifdef LOG_PREEMPTION
-		debuglog("Thread SIGALRM: %d!\n", software_interrupt_SIGALRM_thread_count);
-#endif
 		/* Signal forwarded from another thread. Just confirm it resulted from pthread_kill */
 		assert(signal_info->si_code == SI_TKILL);
 	}
@@ -100,33 +111,28 @@ sigalrm_handler(siginfo_t *signal_info, ucontext_t *user_context, struct sandbox
 {
 	sigalrm_propagate_workers(signal_info);
 
+	/* A worker thread received a SIGALRM when interrupts were disabled, so defer until they are reenabled */
+	if (!software_interrupt_is_enabled()) {
+		// Don't increment if kernel? The first worker gets tons of these...
+		software_interrupt_deferred_sigalrm++;
+		return;
+	}
+
+	assert(current_sandbox->ctxt.preemptable);
 
-	/* NOOP if software interrupts not enabled */
-	if (!software_interrupt_is_enabled()) return;
-
-	/*
-	 * if a SIGALRM fires while the worker thread is between sandboxes doing runtime tasks such as processing
-	 * the epoll loop, performing completion queue cleanup, etc. current_sandbox might be NULL. In this case,
-	 * we should just allow return to allow the worker thread to run the main loop until it loads a new sandbox.
-	 *
-	 * TODO: Consider if this should be an invarient and the worker thread should disable software
-	 * interrupts when doing this work. Issue #95
-	 */
-	if (!current_sandbox) return;
-
-	/*
-	 * if a SIGALRM fires while the worker thread executing cleanup of a sandbox, it might be in a RETURNED
-	 * state. In this case, we should just allow return to allow the sandbox to complete cleanup, as it is
-	 * about to switch to a new sandbox.
-	 *
-	 * TODO: Consider if this should be an invarient and the worker thread should disable software
-	 * interrupts when doing this work. Issue #95 with above
-	 */
-	if (current_sandbox->state == SANDBOX_RETURNED) return;
+	/* A worker thread received a SIGALRM while running a preemptable sandbox, so preempt */
+	software_interrupt_disable();
+
+	assert(current_sandbox != NULL);
+	assert(current_sandbox->state != SANDBOX_RETURNED);
 
 	/* Preempt */
 	local_runqueue_preempt(user_context);
 
+	/* We have to call current_sandbox_get because the argument potentially points to what
+	 * was just preempted */
+	if (current_sandbox_get()->ctxt.preemptable) software_interrupt_enable();
+
 	return;
 }
 
@@ -158,8 +164,6 @@ sigusr1_handler(siginfo_t *signal_info, ucontext_t *user_context, struct sandbox
 
 	arch_mcontext_restore(&user_context->uc_mcontext, &current_sandbox->ctxt);
 
-	software_interrupt_enable();
-
 	return;
 }
 
@@ -185,10 +189,15 @@ software_interrupt_validate_worker()
 static inline void
 software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void *user_context_raw)
 {
+	/* If the runtime has preemption disabled, and we receive a signal, panic */
 	if (unlikely(!runtime_preemption_enabled)) {
 		panic("Unexpectedly invoked signal handlers with preemption disabled\n");
 	}
 
+	assert(software_interrupt_signal_depth < 2);
+
+	// TODO: Use atomics to increment
+	atomic_fetch_add(&software_interrupt_signal_depth, 1);
 	software_interrupt_validate_worker();
 
 	ucontext_t *    user_context    = (ucontext_t *)user_context_raw;
@@ -196,20 +205,26 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void
 
 	switch (signal_type) {
 	case SIGALRM: {
-		return sigalrm_handler(signal_info, user_context, current_sandbox);
+		sigalrm_handler(signal_info, user_context, current_sandbox);
+		break;
 	}
 	case SIGUSR1: {
-		return sigusr1_handler(signal_info, user_context, current_sandbox);
+		sigusr1_handler(signal_info, user_context, current_sandbox);
+		break;
 	}
 	default: {
-		if (signal_info->si_code == SI_TKILL) {
+		switch (signal_info->si_code) {
+		case SI_TKILL:
 			panic("Unexpectedly received signal %d from a thread kill, but we have no handler\n",
 			      signal_type);
-		} else if (signal_info->si_code == SI_KERNEL) {
+		case SI_KERNEL:
 			panic("Unexpectedly received signal %d from the kernel, but we have no handler\n", signal_type);
+		default:
+			panic("Anomolous Signal\n");
 		}
 	}
 	}
+	atomic_fetch_sub(&software_interrupt_signal_depth, 1);
 }
 
 /********************
@@ -269,9 +284,17 @@ software_interrupt_initialize(void)
 	signal_action.sa_sigaction = software_interrupt_handle_signals;
 	signal_action.sa_flags     = SA_SIGINFO | SA_RESTART;
 
+	/* all threads created by the calling thread will have signal blocked */
+
+	// TODO: Unclear about this...
+	sigemptyset(&signal_action.sa_mask);
+	// sigaddset(&signal_action.sa_mask, SIGALRM);
+	// sigaddset(&signal_action.sa_mask, SIGUSR1);
+
 	for (int i = 0;
 	     i < (sizeof(software_interrupt_supported_signals) / sizeof(software_interrupt_supported_signals[0]));
 	     i++) {
+		// TODO: Setup masks
 		int return_code = sigaction(software_interrupt_supported_signals[i], &signal_action, NULL);
 		if (return_code) {
 			perror("sigaction");

runtime/src/worker_thread.c

@@ -6,6 +6,7 @@
 #include <stdlib.h>
 #include <sys/mman.h>
 
+#include "arch/context.h"
 #include "client_socket.h"
 #include "current_sandbox.h"
 #include "debuglog.h"
@@ -76,33 +77,28 @@ worker_thread_switch_to_sandbox(struct sandbox *next_sandbox)
 	assert(next_sandbox != NULL);
 	struct arch_context *next_context = &next_sandbox->ctxt;
 
-	/* Get the old sandbox we're switching from */
-	struct sandbox *current_sandbox = current_sandbox_get();
+	/* Get the old sandbox we're switching from.
+	 * This is null if switching from base context
+	 */
+	struct sandbox *     current_sandbox = current_sandbox_get();
+	struct arch_context *current_context = NULL;
+	if (current_sandbox != NULL) current_context = &current_sandbox->ctxt;
+
+	assert(next_sandbox != current_sandbox);
 
 	/* Update the worker's absolute deadline */
 	runtime_worker_threads_deadline[worker_thread_idx] = next_sandbox->absolute_deadline;
 
 	if (current_sandbox == NULL) {
 		/* Switching from "Base Context" */
-
-		sandbox_set_as_running(next_sandbox, next_sandbox->state);
-
 #ifdef LOG_CONTEXT_SWITCHES
 		debuglog("Base Context (@%p) (%s) > Sandbox %lu (@%p) (%s)\n", &worker_thread_base_context,
 		         arch_context_variant_print(worker_thread_base_context.variant), next_sandbox->id, next_context,
 		         arch_context_variant_print(next_context->variant));
 #endif
-		/* Assumption: If a slow context switch, current sandbox should be set to the target */
-		assert(next_context->variant != ARCH_CONTEXT_VARIANT_SLOW
-		       || &current_sandbox_get()->ctxt == next_context);
 
-		arch_context_switch(NULL, next_context);
 	} else {
-		/* Set the current sandbox to the next */
-		assert(next_sandbox != current_sandbox);
-
-		struct arch_context *current_context = &current_sandbox->ctxt;
-
+		/* Switching from a sandbox context */
 #ifdef LOG_CONTEXT_SWITCHES
 		debuglog("Sandbox %lu (@%p) (%s) > Sandbox %lu (@%p) (%s)\n", current_sandbox->id,
 		         &current_sandbox->ctxt, arch_context_variant_print(current_sandbox->ctxt.variant),
@@ -110,19 +106,10 @@ worker_thread_switch_to_sandbox(struct sandbox *next_sandbox)
 #endif
 
 		worker_thread_transition_exiting_sandbox(current_sandbox);
-
-		sandbox_set_as_running(next_sandbox, next_sandbox->state);
-
-#ifndef NDEBUG
-		assert(next_context->variant != ARCH_CONTEXT_VARIANT_SLOW
-		       || &current_sandbox_get()->ctxt == next_context);
-#endif
-
-		/* Switch to the associated context. */
-		arch_context_switch(current_context, next_context);
 	}
 
-	software_interrupt_enable();
+	sandbox_set_as_running(next_sandbox, next_sandbox->state);
+	arch_context_switch(current_context, next_context);
 }
 
 
@@ -158,25 +145,8 @@ worker_thread_switch_to_base_context()
 	assert(worker_thread_base_context.variant == ARCH_CONTEXT_VARIANT_FAST);
 	runtime_worker_threads_deadline[worker_thread_idx] = UINT64_MAX;
 	arch_context_switch(current_context, &worker_thread_base_context);
-	software_interrupt_enable();
 }
 
-/**
- * Mark a blocked sandbox as runnable and add it to the runqueue
- * @param sandbox the sandbox to check and update if blocked
- */
-void
-worker_thread_wakeup_sandbox(struct sandbox *sandbox)
-{
-	assert(sandbox != NULL);
-	assert(sandbox->state == SANDBOX_BLOCKED);
-
-	software_interrupt_disable();
-	sandbox_set_as_runnable(sandbox, SANDBOX_BLOCKED);
-	software_interrupt_enable();
-}
-
-
 /**
  * Mark the currently executing sandbox as blocked, remove it from the local runqueue,
  * and switch to base context
@@ -184,11 +154,12 @@ worker_thread_wakeup_sandbox(struct sandbox *sandbox)
 void
 worker_thread_block_current_sandbox(void)
 {
-	software_interrupt_disable();
-
 	/* Remove the sandbox we were just executing from the runqueue and mark as blocked */
 	struct sandbox *current_sandbox = current_sandbox_get();
 
+	/* If a preemptable sandbox blocked, disable interrupts */
+	if (current_sandbox->ctxt.preemptable) software_interrupt_disable();
+
 	assert(current_sandbox->state == SANDBOX_RUNNING);
 	sandbox_set_as_blocked(current_sandbox, SANDBOX_RUNNING);
 
@@ -209,6 +180,7 @@ worker_thread_block_current_sandbox(void)
 static inline void
 worker_thread_execute_epoll_loop(void)
 {
+	assert(software_interrupt_is_disabled);
 	while (true) {
 		struct epoll_event epoll_events[RUNTIME_MAX_EPOLL_EVENTS];
 		int                descriptor_count = epoll_wait(worker_thread_epoll_file_descriptor, epoll_events,
@@ -228,7 +200,9 @@ worker_thread_execute_epoll_loop(void)
 				struct sandbox *sandbox = (struct sandbox *)epoll_events[i].data.ptr;
 				assert(sandbox);
 
-				if (sandbox->state == SANDBOX_BLOCKED) worker_thread_wakeup_sandbox(sandbox);
+				if (sandbox->state == SANDBOX_BLOCKED) {
+					sandbox_set_as_runnable(sandbox, SANDBOX_BLOCKED);
+				}
 			} else if (epoll_events[i].events & (EPOLLERR | EPOLLHUP)) {
 				/* Mystery: This seems to never fire. Why? Issue #130 */
 
@@ -274,9 +248,15 @@ worker_thread_execute_epoll_loop(void)
 void *
 worker_thread_main(void *argument)
 {
+	/* The base worker thread should start with software interrupts disabled */
+	assert(software_interrupt_is_disabled);
+
 	/* Index was passed via argument */
 	worker_thread_idx = *(int *)argument;
 
+	/* Set my priority */
+	// runtime_set_pthread_prio(pthread_self(), 0);
+
 	/* Initialize Base Context as unused
 	 * The SP and IP are populated during the first FAST switch away
 	 */
@@ -297,20 +277,16 @@ worker_thread_main(void *argument)
 	/* Initialize Completion Queue */
 	local_completion_queue_initialize();
 
-	/* Initialize Flags */
-	software_interrupt_is_disabled = false;
-
+	/* Initialize epoll */
+	worker_thread_epoll_file_descriptor = epoll_create1(0);
+	if (unlikely(worker_thread_epoll_file_descriptor < 0)) panic_err();
 
-	/* Unmask signals */
+	/* Unmask signals, unless the runtime has disabled preemption */
 	if (runtime_preemption_enabled) {
 		software_interrupt_unmask_signal(SIGALRM);
 		software_interrupt_unmask_signal(SIGUSR1);
 	}
 
-	/* Initialize epoll */
-	worker_thread_epoll_file_descriptor = epoll_create1(0);
-	if (unlikely(worker_thread_epoll_file_descriptor < 0)) panic_err();
-
 	/* Begin Worker Execution Loop */
 	struct sandbox *next_sandbox;
 	while (true) {
@@ -320,13 +296,9 @@ worker_thread_main(void *argument)
 		worker_thread_execute_epoll_loop();
 
 		/* Switch to a sandbox if one is ready to run */
-		software_interrupt_disable();
 		next_sandbox = local_runqueue_get_next();
-		if (next_sandbox != NULL) {
-			worker_thread_switch_to_sandbox(next_sandbox);
-		} else {
-			software_interrupt_enable();
-		};
+		if (next_sandbox != NULL) { worker_thread_switch_to_sandbox(next_sandbox); }
+		assert(software_interrupt_is_disabled);
 
 		/* Clear the completion queue */
 		local_completion_queue_free();
@@ -348,3 +320,14 @@ worker_thread_on_sandbox_exit()
 	worker_thread_switch_to_base_context();
 	panic("Unexpected return\n");
 }
+
+
+void
+worker_thread_sched()
+{
+	struct sandbox *current_sandbox = current_sandbox_get();
+	struct sandbox *next_sandbox    = local_runqueue_get_next();
+	if (next_sandbox != NULL && next_sandbox != current_sandbox_get()) {
+		worker_thread_switch_to_sandbox(next_sandbox);
+	}
+}