feat: Non-crashing demo state

demo.txt

@@ -0,0 +1,10 @@
+LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./awsmrt ../tests/test_fibonacci_multiple.json
+
+
+# fib(40)
+
+ab -n 10000 -c 1000 -p post_body.txt -v 4 localhost:10000/
+
+# fib(10)
+
+ab -n 1000 -c 100 -p post_body2.txt -v 4 localhost:10001/

post_body.txt

@@ -1 +1 @@
-40
+43

runtime/include/priority_queue.h

@@ -25,10 +25,10 @@ struct priority_queue {
 };
 
 void     priority_queue_initialize(struct priority_queue *self, priority_queue_get_priority_t get_priority);
-int      priority_queue_enqueue(struct priority_queue *self, void *value);
-void *   priority_queue_dequeue(struct priority_queue *self);
+int      priority_queue_enqueue(struct priority_queue *self, void *value, char *name);
+void *   priority_queue_dequeue(struct priority_queue *self, char *name);
 int      priority_queue_length(struct priority_queue *self);
 uint64_t priority_queue_peek(struct priority_queue *self);
-int      priority_queue_delete(struct priority_queue *self, void *value);
+int      priority_queue_delete(struct priority_queue *self, void *value, char *name);
 
 #endif /* PRIORITY_QUEUE_H */

runtime/include/sandbox_run_queue.h

@@ -6,27 +6,30 @@
 
 // Returns pointer back if successful, null otherwise
 typedef struct sandbox *(*sandbox_run_queue_add_t)(struct sandbox *);
-typedef struct sandbox *(*sandbox_run_queue_remove_t)(void);
 typedef bool (*sandbox_run_queue_is_empty_t)(void);
 typedef void (*sandbox_run_queue_delete_t)(struct sandbox *sandbox);
 typedef struct sandbox *(*sandbox_run_queue_get_next_t)();
 
+typedef void (*sandbox_run_queue_preempt_t)(ucontext_t *);
+
 typedef struct sandbox_run_queue_config_t {
 	sandbox_run_queue_add_t      add;
 	sandbox_run_queue_is_empty_t is_empty;
-	sandbox_run_queue_remove_t   remove;
 	sandbox_run_queue_delete_t delete;
 	sandbox_run_queue_get_next_t get_next;
+	sandbox_run_queue_preempt_t  preempt;
 } sandbox_run_queue_config_t;
 
 
 void sandbox_run_queue_initialize(sandbox_run_queue_config_t *config);
 
+
+// This is currently only used by worker_thread_wakeup_sandbox
 struct sandbox *sandbox_run_queue_add(struct sandbox *);
 void            sandbox_run_queue_delete(struct sandbox *);
-struct sandbox *sandbox_run_queue_remove();
 bool            sandbox_run_queue_is_empty();
 struct sandbox *sandbox_run_queue_get_next();
+void            sandbox_run_queue_preempt(ucontext_t *);
 
 
 #endif /* SFRT_SANDBOX_RUN_QUEUE_H */

runtime/src/priority_queue.c

@@ -2,6 +2,7 @@
 #include <limits.h>
 #include <stdio.h>
 #include <string.h>
+#include <stdlib.h>
 
 #include <priority_queue.h>
 
@@ -19,11 +20,12 @@ static inline int
 priority_queue_append(struct priority_queue *self, void *new_item)
 {
 	assert(self != NULL);
+	assert(ck_spinlock_fas_locked(&self->lock));
 
 	if (self->first_free >= MAX) return -1;
 
-	self->items[self->first_free] = new_item;
-	self->first_free++;
+	self->items[self->first_free++] = new_item;
+	// self->first_free++;
 	return 0;
 }
 
@@ -36,9 +38,11 @@ priority_queue_percolate_up(struct priority_queue *self)
 {
 	assert(self != NULL);
 	assert(self->get_priority != NULL);
+	assert(ck_spinlock_fas_locked(&self->lock));
 
 	for (int i = self->first_free - 1;
 	     i / 2 != 0 && self->get_priority(self->items[i]) < self->get_priority(self->items[i / 2]); i /= 2) {
+		assert(self->get_priority(self->items[i]) != ULONG_MAX);
 		void *temp         = self->items[i / 2];
 		self->items[i / 2] = self->items[i];
 		self->items[i]     = temp;
@@ -59,6 +63,7 @@ priority_queue_find_smallest_child(struct priority_queue *self, int parent_index
 	assert(self != NULL);
 	assert(parent_index >= 1 && parent_index < self->first_free);
 	assert(self->get_priority != NULL);
+	assert(ck_spinlock_fas_locked(&self->lock));
 
 	int left_child_index  = 2 * parent_index;
 	int right_child_index = 2 * parent_index + 1;
@@ -86,6 +91,7 @@ priority_queue_percolate_down(struct priority_queue *self, int parent_index)
 {
 	assert(self != NULL);
 	assert(self->get_priority != NULL);
+	assert(ck_spinlock_fas_locked(&self->lock));
 
 	int left_child_index = 2 * parent_index;
 	while (left_child_index >= 2 && left_child_index < self->first_free) {
@@ -123,7 +129,8 @@ priority_queue_initialize(struct priority_queue *self, priority_queue_get_priori
 	memset(self->items, 0, sizeof(void *) * MAX);
 
 	ck_spinlock_fas_init(&self->lock);
-	self->first_free   = 1;
+	self->first_free = 1;
+	printf("[Init] First Free: %d\n", self->first_free);
 	self->get_priority = get_priority;
 
 	// We're assuming a min-heap implementation, so set to larget possible value
@@ -137,9 +144,13 @@ priority_queue_initialize(struct priority_queue *self, priority_queue_get_priori
 int
 priority_queue_length(struct priority_queue *self)
 {
+	// printf("[Length] First Free: %d\n", self->first_free);
 	assert(self != NULL);
-
-	return self->first_free - 1;
+	ck_spinlock_fas_lock(&self->lock);
+	assert(ck_spinlock_fas_locked(&self->lock));
+	int length = self->first_free - 1;
+	ck_spinlock_fas_unlock(&self->lock);
+	return length;
 }
 
 /**
@@ -148,29 +159,37 @@ priority_queue_length(struct priority_queue *self)
  * @returns 0 on success. -1 on full. -2 on unable to take lock
  **/
 int
-priority_queue_enqueue(struct priority_queue *self, void *value)
+priority_queue_enqueue(struct priority_queue *self, void *value, char *name)
 {
 	assert(self != NULL);
-	int rc;
+	ck_spinlock_fas_lock(&self->lock);
 
-	if (ck_spinlock_fas_trylock(&self->lock) == false) return -2;
+	int pre_length = self->first_free - 1;
 
 	// Start of Critical Section
-	if (priority_queue_append(self, value) == 0) {
-		if (self->first_free > 2) {
-			priority_queue_percolate_up(self);
-		} else {
-			// If this is the first element we add, update the highest priority
-			self->highest_priority = self->get_priority(value);
-		}
-		rc = 0;
+	if (priority_queue_append(self, value) == -1) {
+		printf("Priority Queue is full");
+		fflush(stdout);
+		exit(EXIT_FAILURE);
+		ck_spinlock_fas_unlock(&self->lock);
+		return -1;
+	}
+
+	int post_length = self->first_free - 1;
+	printf("[%s Enqueue] First Free: %d\n", name, self->first_free);
+
+	// We should have appended here
+	assert(post_length == pre_length + 1);
+
+	// If this is the first element we add, update the highest priority
+	if (self->first_free == 2) {
+		self->highest_priority = self->get_priority(value);
 	} else {
-		// Priority Queue is full
-		rc = -1;
+		priority_queue_percolate_up(self);
 	}
 	// End of Critical Section
 	ck_spinlock_fas_unlock(&self->lock);
-	return rc;
+	return 0;
 }
 /**
  * @param self - the priority queue we want to delete from
@@ -178,23 +197,27 @@ priority_queue_enqueue(struct priority_queue *self, void *value)
  * @returns 0 on success. -1 on not found. -2 on unable to take lock
  **/
 int
-priority_queue_delete(struct priority_queue *self, void *value)
+priority_queue_delete(struct priority_queue *self, void *value, char *name)
 {
 	assert(self != NULL);
-	if (ck_spinlock_fas_trylock(&self->lock) == false) return -2;
+	ck_spinlock_fas_lock(&self->lock);
 
 	bool did_delete = false;
 	for (int i = 1; i < self->first_free; i++) {
 		if (self->items[i] == value) {
-			self->items[i]                    = self->items[self->first_free - 1];
-			self->items[self->first_free - 1] = NULL;
-			self->first_free--;
+			self->items[i]                = self->items[--self->first_free];
+			self->items[self->first_free] = NULL;
 			priority_queue_percolate_down(self, i);
 			did_delete = true;
 		}
 	}
 	ck_spinlock_fas_unlock(&self->lock);
-	if (!did_delete) return -1;
+	assert(did_delete);
+	if (!did_delete) {
+		printf("[priority_queue_delete] Not found!\n");
+		return -1;
+	};
+	printf("[%s Delete] First Free: %d\n", name, self->first_free);
 	return 0;
 }
 
@@ -202,8 +225,12 @@ static bool
 priority_queue_is_empty(struct priority_queue *self)
 {
 	assert(self != NULL);
+	bool caller_locked = ck_spinlock_fas_locked(&self->lock);
+	if (!caller_locked) ck_spinlock_fas_lock(&self->lock);
 	assert(self->first_free != 0);
-	return self->first_free == 1;
+	bool is_empty = self->first_free == 1;
+	if (!caller_locked) ck_spinlock_fas_unlock(&self->lock);
+	return is_empty;
 }
 
 /**
@@ -211,19 +238,20 @@ priority_queue_is_empty(struct priority_queue *self)
  * @returns The head of the priority queue or NULL when empty
  **/
 void *
-priority_queue_dequeue(struct priority_queue *self)
+priority_queue_dequeue(struct priority_queue *self, char *name)
 {
 	assert(self != NULL);
 	assert(self->get_priority != NULL);
 	if (priority_queue_is_empty(self)) return NULL;
-	if (ck_spinlock_fas_trylock(&self->lock) == false) return NULL;
+
+	ck_spinlock_fas_lock(&self->lock);
+	assert(ck_spinlock_fas_locked(&self->lock));
 	// Start of Critical Section
 	void *min = NULL;
 	if (!priority_queue_is_empty(self)) {
-		min                               = self->items[1];
-		self->items[1]                    = self->items[self->first_free - 1];
-		self->items[self->first_free - 1] = NULL;
-		self->first_free--;
+		min                           = self->items[1];
+		self->items[1]                = self->items[--self->first_free];
+		self->items[self->first_free] = NULL;
 		// Because of 1-based indices, first_free is 2 when there is only one element
 		if (self->first_free > 2) priority_queue_percolate_down(self, 1);
 
@@ -231,13 +259,26 @@ priority_queue_dequeue(struct priority_queue *self)
 		self->highest_priority = !priority_queue_is_empty(self) ? self->get_priority(self->items[1])
 		                                                        : ULONG_MAX;
 	}
+	printf("[%s Dequeue] First Free: %d\n", name, self->first_free);
 	ck_spinlock_fas_unlock(&self->lock);
 	// End of Critical Section
 	return min;
 }
 
+// /**
+//  * Returns the head of the priority queue without removing it
+//  **/
+// void *
+// priority_queue_get_head(struct priority_queue *self)
+// {
+// 	ck_spinlock_fas_lock(&self->lock);
+
+// 	ck_spinlock_fas_unlock(&self->lock);
+// }
+
 uint64_t
 priority_queue_peek(struct priority_queue *self)
 {
-	return self->highest_priority;
+	uint64_t highest_priority = self->highest_priority;
+	return highest_priority;
 }

runtime/src/runtime.c

@@ -44,8 +44,8 @@ runtime_initialize(void)
 	assert(runtime_epoll_file_descriptor >= 0);
 
 	// Allocate and Initialize the global deque
-	sandbox_request_scheduler_fifo_initialize();
-	// sandbox_request_scheduler_ps_initialize();
+	// sandbox_request_scheduler_fifo_initialize();
+	sandbox_request_scheduler_ps_initialize();
 
 	// Mask Signals
 	software_interrupt_mask_signal(SIGUSR1);

runtime/src/sandbox.c

@@ -139,12 +139,12 @@ current_sandbox_build_and_send_client_response(void)
 
 done:
 	assert(sndsz == curr->request_response_data_length);
-	// Get End Timestamp
-	u64 end_time     = __getcycles();
-	curr->total_time = end_time - curr->start_time;
+	curr->total_time       = __getcycles() - curr->start_time;
+	uint64_t total_time_us = curr->total_time / runtime_processor_speed_MHz;
+
 	// TODO: Refactor to log file
 	printf("%s():%d, %d, %lu\n", curr->module->name, curr->module->port, curr->module->relative_deadline_us,
-	       (uint64_t)(curr->total_time / runtime_processor_speed_MHz));
+	       total_time_us);
 	// if (end_time < curr->absolute_deadline) {
 	// 	printf("meadDeadline Met with %f us to spare\n",
 	// 	       (curr->absolute_deadline - end_time) / runtime_processor_speed_MHz);

runtime/src/sandbox_request_scheduler_ps.c

@@ -10,11 +10,16 @@ static struct priority_queue sandbox_request_scheduler_ps;
  * @returns pointer to request if added. NULL otherwise
  **/
 static sandbox_request_t *
-sandbox_request_scheduler_ps_add(void *sandbox_request_raw)
+sandbox_request_scheduler_ps_add(void *sandbox_request)
 {
-	int return_code = priority_queue_enqueue(&sandbox_request_scheduler_ps, sandbox_request_raw);
+	int return_code = priority_queue_enqueue(&sandbox_request_scheduler_ps, sandbox_request, "Request");
 
-	return return_code == 0 ? sandbox_request_raw : NULL;
+	if (return_code == -1) {
+		printf("Request Queue is full\n");
+		exit(EXIT_FAILURE);
+	}
+
+	return return_code == 0 ? sandbox_request : NULL;
 }
 
 /**
@@ -24,7 +29,7 @@ sandbox_request_scheduler_ps_add(void *sandbox_request_raw)
 static sandbox_request_t *
 sandbox_request_scheduler_ps_remove(void)
 {
-	return (sandbox_request_t *)priority_queue_dequeue(&sandbox_request_scheduler_ps);
+	return (sandbox_request_t *)priority_queue_dequeue(&sandbox_request_scheduler_ps, "Request");
 }
 
 /**

runtime/src/sandbox_run_queue.c

@@ -25,18 +25,12 @@ sandbox_run_queue_delete(struct sandbox *sandbox)
 	sandbox_run_queue.delete(sandbox);
 }
 
-// Removes a sandbox request
-struct sandbox *
-sandbox_run_queue_remove()
-{
-	assert(sandbox_run_queue.remove != NULL);
-	return sandbox_run_queue.remove();
-}
 
 bool
 sandbox_run_queue_is_empty()
 {
-	return sandbox_run_queue_is_empty();
+	assert(sandbox_run_queue.is_empty != NULL);
+	return sandbox_run_queue.is_empty();
 }
 
 struct sandbox *
@@ -45,3 +39,10 @@ sandbox_run_queue_get_next()
 	assert(sandbox_run_queue.get_next != NULL);
 	return sandbox_run_queue.get_next();
 };
+
+void
+sandbox_run_queue_preempt(ucontext_t *context)
+{
+	assert(sandbox_run_queue.preempt != NULL);
+	return sandbox_run_queue.preempt(context);
+};

runtime/src/sandbox_run_queue_fifo.c

@@ -27,6 +27,14 @@ sandbox_run_queue_fifo_remove(struct sandbox *sandbox_to_remove)
 	ps_list_rem_d(sandbox_to_remove);
 }
 
+struct sandbox *
+sandbox_run_queue_fifo_remove_and_return()
+{
+	struct sandbox *sandbox_to_remove = ps_list_head_first_d(&sandbox_run_queue_fifo, struct sandbox);
+	ps_list_rem_d(sandbox_to_remove);
+	return sandbox_to_remove;
+}
+
 /**
  * Execute the sandbox at the head of the thread local runqueue
  * If the runqueue is empty, pull a fresh batch of sandbox requests, instantiate them, and then execute the new head
@@ -47,7 +55,7 @@ sandbox_run_queue_fifo_get_next()
 	}
 
 	// Execute Round Robin Scheduling Logic
-	struct sandbox *next_sandbox = sandbox_run_queue_remove();
+	struct sandbox *next_sandbox = sandbox_run_queue_fifo_remove_and_return();
 	assert(next_sandbox->state != RETURNED);
 	sandbox_run_queue_add(next_sandbox);
 
@@ -67,14 +75,6 @@ sandbox_run_queue_fifo_append(struct sandbox *sandbox_to_append)
 	return sandbox_to_append;
 }
 
-struct sandbox *
-sandbox_run_queue_fifo_remove_and_return()
-{
-	struct sandbox *sandbox_to_remove = ps_list_head_first_d(&sandbox_run_queue_fifo, struct sandbox);
-	ps_list_rem_d(sandbox_to_remove);
-	return sandbox_to_remove;
-}
-
 
 void
 sandbox_run_queue_fifo_initialize()
@@ -84,7 +84,6 @@ sandbox_run_queue_fifo_initialize()
 	// Register Function Pointers for Abstract Scheduling API
 	sandbox_run_queue_config_t config = { .add      = sandbox_run_queue_fifo_append,
 		                              .is_empty = sandbox_run_queue_fifo_is_empty,
-		                              .remove   = sandbox_run_queue_fifo_remove_and_return,
 		                              .delete   = sandbox_run_queue_fifo_remove,
 		                              .get_next = sandbox_run_queue_fifo_get_next };
 

runtime/src/sandbox_run_queue_ps.c

@@ -2,6 +2,9 @@
 #include "sandbox_run_queue.h"
 #include "priority_queue.h"
 #include "sandbox_request_scheduler.h"
+#include "current_sandbox.h"
+
+#include <stdint.h>
 
 // Local State
 __thread static struct priority_queue sandbox_run_queue_ps;
@@ -9,6 +12,9 @@ __thread static struct priority_queue sandbox_run_queue_ps;
 bool
 sandbox_run_queue_ps_is_empty()
 {
+	int length = priority_queue_length(&sandbox_run_queue_ps);
+	if (length > 1) printf("[is_empty] Runqueue length: %d\n", length);
+	assert(length < 5);
 	return priority_queue_length(&sandbox_run_queue_ps) == 0;
 }
 
@@ -20,9 +26,23 @@ sandbox_run_queue_ps_is_empty()
 static struct sandbox *
 sandbox_run_queue_ps_add(struct sandbox *sandbox)
 {
-	int return_code = priority_queue_enqueue(&sandbox_run_queue_ps, sandbox);
+	int original_length = priority_queue_length(&sandbox_run_queue_ps);
+	if (original_length > 1) printf("[Add] Original Runqueue length: %d\n", original_length);
+
+	int return_code = priority_queue_enqueue(&sandbox_run_queue_ps, sandbox, "Run Queue");
+	if (return_code == -1) {
+		printf("Thread Runqueue is full!\n");
+		exit(EXIT_FAILURE);
+	}
 
-	return return_code == 0 ? sandbox : NULL;
+	int final_length = priority_queue_length(&sandbox_run_queue_ps);
+	if (final_length > 1) printf("[Add] Final Runqueue length: %d\n", final_length);
+
+	assert(final_length == original_length + 1);
+
+	// printf("Added Sandbox to runqueue\n");
+	assert(return_code == 0);
+	return sandbox;
 }
 
 /**
@@ -32,19 +52,18 @@ sandbox_run_queue_ps_add(struct sandbox *sandbox)
 static struct sandbox *
 sandbox_run_queue_ps_remove(void)
 {
-	return (struct sandbox *)priority_queue_dequeue(&sandbox_run_queue_ps);
+	return (struct sandbox *)priority_queue_dequeue(&sandbox_run_queue_ps, "Runqueue");
 }
 
 /**
- *
  * @returns A Sandbox Request or NULL
  **/
 static void
 sandbox_run_queue_ps_delete(struct sandbox *sandbox)
 {
 	assert(sandbox != NULL);
-	int rc = priority_queue_delete(&sandbox_run_queue_ps, sandbox);
-	assert(rc != -2);
+	int rc = priority_queue_delete(&sandbox_run_queue_ps, sandbox, "Runqueue");
+	assert(rc == 0);
 }
 
 /**
@@ -57,48 +76,84 @@ sandbox_run_queue_ps_delete(struct sandbox *sandbox)
 struct sandbox *
 sandbox_run_queue_ps_get_next()
 {
-	// At any point, we may need to run the head of the request scheduler, the head of the local runqueue, or we
-	// might want to continue executing the current sandbox. If we want to keep executing the current sandbox, we
-	// should have a fast path to be able to resume without context switches.
+	if (sandbox_run_queue_ps.first_free != 1) {
+		printf("Runqueue First Free: %d\n", sandbox_run_queue_ps.first_free);
+	}
 
-	// If the run queue is empty, we've run the current sandbox to completion
+	// Try to pull a sandbox request and return NULL if we're unable to get one
+	sandbox_request_t *sandbox_request;
+	if ((sandbox_request = sandbox_request_scheduler_remove()) == NULL) {
+		// printf("Global Request Queue was empty!\n");
+		return NULL;
+	};
+
+	// Otherwise, allocate the sandbox request as a runnable sandbox and place on the runqueue
+	printf("Sandbox Runqueue was empty, so adding sandbox from global queue\n");
+	struct sandbox *sandbox = sandbox_allocate(sandbox_request);
+	assert(sandbox);
+	free(sandbox_request);
+	sandbox->state = RUNNABLE;
+	sandbox_run_queue_ps_add(sandbox);
+	return sandbox;
+}
 
-	// We assume that the current sandbox is always on the runqueue when in a runnable state, we know it's the
-	// highest priority thing on the runqueue.
 
-	// Case 1: Current runqueue is empty, so pull from global queue and add to runqueue
-	if (sandbox_run_queue_is_empty()) {
-		sandbox_request_t *sandbox_request = sandbox_request_scheduler_remove();
-		if (sandbox_request == NULL) return NULL;
-		struct sandbox *sandbox = sandbox_allocate(sandbox_request);
-		assert(sandbox);
-		free(sandbox_request);
-		sandbox->state = RUNNABLE;
-		sandbox_run_queue_add(sandbox);
-		return sandbox;
-	}
+// Conditionally checks to see if current sandbox should be preempted
+void
+sandbox_run_queue_ps_preempt(ucontext_t *user_context)
+{
+	software_interrupt_disable(); // no nesting!
+
+	struct sandbox *current_sandbox = current_sandbox_get();
+	// If current_sandbox is null, there's nothing to preempt, so let the "main" scheduler run its course.
+	if (current_sandbox == NULL) {
+		software_interrupt_enable();
+		return;
+	};
+
+	// The current sandbox should be the head of the runqueue
+	assert(sandbox_run_queue_ps_is_empty() == false);
+
+	// TODO: Factor quantum and/or sandbox allocation time into decision
+	// uint64_t global_deadline = sandbox_request_scheduler_peek() -
+	// SOFTWARE_INTERRUPT_INTERVAL_DURATION_IN_CYCLES;
+
+	bool     should_enable_software_interrupt = true;
+	uint64_t local_deadline                   = priority_queue_peek(&sandbox_run_queue_ps);
+	uint64_t global_deadline                  = sandbox_request_scheduler_peek();
+
+	if (local_deadline == ULONG_MAX) { assert(sandbox_run_queue_ps.first_free == 1); };
 
-	// Case 2: Current runqueue is not empty, so compare head of runqueue to head of global request queue and return
-	// highest priority
-
-	uint64_t global_deadline = sandbox_request_scheduler_peek() - SOFTWARE_INTERRUPT_INTERVAL_DURATION_IN_CYCLES;
-	// This should be refactored to peek at the top of the runqueue
-	struct sandbox *head_of_runqueue = sandbox_run_queue_remove();
-	uint64_t        local_deadline   = head_of_runqueue->absolute_deadline;
-	sandbox_run_queue_add(head_of_runqueue);
-
-	if (local_deadline <= global_deadline) {
-		return head_of_runqueue;
-	} else {
-		sandbox_request_t *sandbox_request = sandbox_request_scheduler_remove();
-		struct sandbox *   sandbox         = sandbox_allocate(sandbox_request);
-		assert(sandbox);
+	// If we're able to get a sandbox request with a tighter deadline, preempt the current context and run it
+	sandbox_request_t *sandbox_request;
+	if (global_deadline < local_deadline && (sandbox_request = sandbox_request_scheduler_remove()) != NULL) {
+		printf("Thread %lu Preempted %lu for %lu\n", pthread_self(), local_deadline,
+		       sandbox_request->absolute_deadline);
+
+		// Allocate the request
+		struct sandbox *next_sandbox = sandbox_allocate(sandbox_request);
+		assert(next_sandbox);
 		free(sandbox_request);
-		sandbox->state = RUNNABLE;
-		sandbox_run_queue_add(sandbox);
+		next_sandbox->state = RUNNABLE;
+
+		// Add it to the runqueue
+		printf("adding new sandbox to runqueue\n");
+		sandbox_run_queue_add(next_sandbox);
 		debuglog("[%p: %s]\n", sandbox, sandbox->module->name);
-		return sandbox;
+
+		// Save the context of the currently executing sandbox before switching from it
+		arch_mcontext_save(&current_sandbox->ctxt, &user_context->uc_mcontext);
+
+		// Update current_sandbox to the next sandbox
+		current_sandbox_set(next_sandbox);
+
+		// And load the context of this new sandbox
+		// RC of 1 indicates that sandbox was last in a user-level context switch state,
+		// so do not enable software interrupts.
+		if (arch_mcontext_restore(&user_context->uc_mcontext, &next_sandbox->ctxt) == 1)
+			should_enable_software_interrupt = false;
 	}
+	if (should_enable_software_interrupt) software_interrupt_enable();
 }
 
 
@@ -118,9 +173,9 @@ sandbox_run_queue_ps_initialize()
 	// Register Function Pointers for Abstract Scheduling API
 	sandbox_run_queue_config_t config = { .add      = sandbox_run_queue_ps_add,
 		                              .is_empty = sandbox_run_queue_ps_is_empty,
-		                              .remove   = sandbox_run_queue_ps_remove,
 		                              .delete   = sandbox_run_queue_ps_delete,
-		                              .get_next = sandbox_run_queue_ps_get_next };
+		                              .get_next = sandbox_run_queue_ps_get_next,
+		                              .preempt  = sandbox_run_queue_ps_preempt };
 
 	sandbox_run_queue_initialize(&config);
 }

runtime/src/software_interrupt.c

@@ -117,32 +117,8 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void
 static inline void
 software_interrupt_schedule_alarm(void *user_context_raw)
 {
-	software_interrupt_disable(); // no nesting!
-	struct sandbox *current_sandbox                  = current_sandbox_get();
-	bool            should_enable_software_interrupt = true;
-
-	// If current_sandbox is null, there's nothing to preempt, so let the "main" scheduler run its course.
-	if (current_sandbox != NULL) {
-		struct sandbox *next_sandbox = sandbox_run_queue_get_next();
-
-		if (next_sandbox != NULL && next_sandbox != current_sandbox) {
-			ucontext_t *user_context = (ucontext_t *)user_context_raw;
-
-			// Save the context of the currently executing sandbox before switching from it
-			arch_mcontext_save(&current_sandbox->ctxt, &user_context->uc_mcontext);
-
-			// Update current_sandbox to the next sandbox
-			current_sandbox_set(next_sandbox);
-
-			// And load the context of this new sandbox
-			// RC of 1 indicates that sandbox was last in a user-level context switch state,
-			// so do not enable software interrupts.
-			if (arch_mcontext_restore(&user_context->uc_mcontext, &next_sandbox->ctxt) == 1)
-				should_enable_software_interrupt = false;
-		}
-	}
-
-	if (should_enable_software_interrupt) software_interrupt_enable();
+	ucontext_t *user_context = (ucontext_t *)user_context_raw;
+	sandbox_run_queue_preempt(user_context);
 }
 
 /***************************************

runtime/src/worker_thread.c

@@ -84,6 +84,7 @@ worker_thread_wakeup_sandbox(sandbox_t *sandbox)
 	// debuglog("[%p: %s]\n", sandbox, sandbox->module->name);
 	if (sandbox->state == BLOCKED) {
 		sandbox->state = RUNNABLE;
+		printf("Marking blocked sandbox as runnable\n");
 		sandbox_run_queue_add(sandbox);
 	}
 	software_interrupt_enable();
@@ -173,8 +174,8 @@ worker_thread_main(void *return_code)
 	// Initialize Worker State
 	arch_context_init(&worker_thread_base_context, 0, 0);
 
-	sandbox_run_queue_fifo_initialize();
-	// sandbox_run_queue_ps_initialize();
+	// sandbox_run_queue_fifo_initialize();
+	sandbox_run_queue_ps_initialize();
 
 	sandbox_completion_queue_initialize();
 	software_interrupt_is_disabled = false;