feat: Modified scheduler state machine

master
Sean McBride 4 years ago
parent 2c1a33970e
commit 815546852c

@ -1,6 +1,7 @@
{
"files.associations": {
"*.ld": "llvm",
"*.wasm": "wasm",
"*.inc": "cpp",
"arm_nnexamples_cifar10_parameter.h": "c",
"arm_nnexamples_cifar10_weights.h": "c",
@ -90,6 +91,8 @@
"sandbox_set_as_running.h": "c",
"sandbox_summarize_page_allocations.h": "c",
"wasm_types.h": "c",
"atomic": "c",
"sandbox_set_as_running_kernel.h": "c"
},
"files.exclude": {
"**/.git": true,

@ -47,7 +47,6 @@ BINARY_NAME=sledgert
# Various Informational Logs for Debugging
# CFLAGS += -DLOG_HTTP_PARSER
# CFLAGS += -DLOG_STATE_CHANGES
# CFLAGS += -DLOG_LOCK_OVERHEAD
# CFLAGS += -DLOG_CONTEXT_SWITCHES
# CFLAGS += -DLOG_ADMISSIONS_CONTROL
@ -55,6 +54,11 @@ BINARY_NAME=sledgert
# CFLAGS += -DLOG_PREEMPTION
# CFLAGS += -DLOG_MODULE_LOADING
# This adds an array of sandbox states to all sandbox structs and appends states at each transition
# The history trucates when the number of elements equal SANDBOX_STATE_HISTORY_CAPACITY
CFLAGS += -DLOG_STATE_CHANGES
# This dumps per module *.csv files containing the cycle a sandbox has been in RUNNING when each
# page is allocated. This helps understand the relationship to memory allocation and execution time.
# CFLAGS += -DLOG_SANDBOX_MEMORY_PROFILE

@ -96,10 +96,58 @@ sandbox_print_perf(struct sandbox *sandbox)
* becomes more intelligent, then peak linear memory size needs to be tracked
* seperately from current linear memory size.
*/
fprintf(runtime_sandbox_perf_log, "%lu,%s,%d,%s,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%u,%u\n", sandbox->id,
fprintf(runtime_sandbox_perf_log, "%lu,%s,%d,%s,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%u,%u\n", sandbox->id,
sandbox->module->name, sandbox->module->port, sandbox_state_stringify(sandbox->state),
sandbox->module->relative_deadline, sandbox->total_time, queued_duration,
sandbox->duration_of_state.initializing, sandbox->duration_of_state.runnable,
sandbox->duration_of_state.running, sandbox->duration_of_state.blocked,
sandbox->duration_of_state.preempted, sandbox->duration_of_state.running_kernel,
sandbox->duration_of_state.running_user, sandbox->duration_of_state.blocked,
sandbox->duration_of_state.returned, runtime_processor_speed_MHz, sandbox->memory.size);
}
static inline void
sandbox_enable_preemption(struct sandbox *sandbox)
{
#ifdef LOG_PREEMPTION
debuglog("Sandbox %lu - enabling preemption - Missed %d SIGALRM\n", sandbox->id,
software_interrupt_deferred_sigalrm);
fflush(stderr);
#endif
if (__sync_bool_compare_and_swap(&sandbox->ctxt.preemptable, 0, 1) == false) {
panic("Recursive call to current_sandbox_enable_preemption\n");
}
if (software_interrupt_deferred_sigalrm > 0) {
/* Update Max */
if (software_interrupt_deferred_sigalrm > software_interrupt_deferred_sigalrm_max[worker_thread_idx]) {
software_interrupt_deferred_sigalrm_max[worker_thread_idx] =
software_interrupt_deferred_sigalrm;
}
software_interrupt_deferred_sigalrm = 0;
// TODO: Replay. Does the replay need to be before or after enabling preemption?
}
}
static inline void
sandbox_disable_preemption(struct sandbox *sandbox)
{
#ifdef LOG_PREEMPTION
debuglog("Sandbox %lu - disabling preemption\n", sandbox->id);
fflush(stderr);
#endif
if (__sync_bool_compare_and_swap(&sandbox->ctxt.preemptable, 1, 0) == false) {
panic("Recursive call to current_sandbox_disable_preemption\n");
}
}
static inline void
sandbox_state_history_append(struct sandbox *sandbox, sandbox_state_t state)
{
#ifdef LOG_STATE_CHANGES
if (likely(sandbox->state_history_count < SANDBOX_STATE_HISTORY_CAPACITY)) {
sandbox->state_history[sandbox->state_history_count++] = state;
}
#endif
}

@ -25,10 +25,11 @@ sandbox_set_as_blocked(struct sandbox *sandbox, sandbox_state_t last_state)
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_BLOCKED;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_BLOCKED);
switch (last_state) {
case SANDBOX_RUNNING: {
sandbox->duration_of_state.running += duration_of_last_state;
case SANDBOX_RUNNING_KERNEL: {
sandbox->duration_of_state.running_kernel += duration_of_last_state;
local_runqueue_delete(sandbox);
break;
}
@ -42,7 +43,7 @@ sandbox_set_as_blocked(struct sandbox *sandbox, sandbox_state_t last_state)
sandbox->state = SANDBOX_BLOCKED;
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox->id, last_state, SANDBOX_BLOCKED);
sandbox_state_history_append(sandbox, SANDBOX_BLOCKED);
runtime_sandbox_total_increment(SANDBOX_BLOCKED);
runtime_sandbox_total_decrement(last_state);
}

@ -27,6 +27,7 @@ sandbox_set_as_complete(struct sandbox *sandbox, sandbox_state_t last_state)
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_COMPLETE;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_COMPLETE);
switch (last_state) {
case SANDBOX_RETURNED: {
@ -44,12 +45,13 @@ sandbox_set_as_complete(struct sandbox *sandbox, sandbox_state_t last_state)
sandbox->state = SANDBOX_COMPLETE;
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox->id, last_state, SANDBOX_COMPLETE);
sandbox_state_history_append(sandbox, SANDBOX_COMPLETE);
runtime_sandbox_total_increment(SANDBOX_COMPLETE);
runtime_sandbox_total_decrement(last_state);
/* Admissions Control Post Processing */
admissions_info_update(&sandbox->module->admissions_info, sandbox->duration_of_state.running);
admissions_info_update(&sandbox->module->admissions_info,
sandbox->duration_of_state.running_user + sandbox->duration_of_state.running_kernel);
admissions_control_subtract(sandbox->admissions_estimate);
/* Terminal State Logging */

@ -32,14 +32,15 @@ sandbox_set_as_error(struct sandbox *sandbox, sandbox_state_t last_state)
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_ERROR;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_ERROR);
switch (last_state) {
case SANDBOX_SET_AS_INITIALIZED:
/* Technically, this is a degenerate sandbox that we generate by hand */
sandbox->duration_of_state.initializing += duration_of_last_state;
break;
case SANDBOX_RUNNING: {
sandbox->duration_of_state.running += duration_of_last_state;
case SANDBOX_RUNNING_KERNEL: {
sandbox->duration_of_state.running_kernel += duration_of_last_state;
local_runqueue_delete(sandbox);
break;
}
@ -59,7 +60,7 @@ sandbox_set_as_error(struct sandbox *sandbox, sandbox_state_t last_state)
local_completion_queue_add(sandbox);
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox_id, last_state, SANDBOX_ERROR);
sandbox_state_history_append(sandbox, SANDBOX_ERROR);
runtime_sandbox_total_increment(SANDBOX_ERROR);
runtime_sandbox_total_decrement(last_state);
}

@ -32,6 +32,7 @@ sandbox_set_as_initialized(struct sandbox *sandbox, struct sandbox_request *sand
sandbox->timestamp_of.request_arrival = sandbox_request->request_arrival_timestamp;
sandbox->timestamp_of.allocation = allocation_timestamp;
sandbox->state = SANDBOX_SET_AS_INITIALIZED;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_INITIALIZED);
/* Initialize the sandbox's context, stack, and instruction pointer */
/* stack.start points to the bottom of the usable stack, so add stack_size to get to top */
@ -50,6 +51,6 @@ sandbox_set_as_initialized(struct sandbox *sandbox, struct sandbox_request *sand
sandbox->state = SANDBOX_INITIALIZED;
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox->id, SANDBOX_UNINITIALIZED, SANDBOX_INITIALIZED);
sandbox_state_history_append(sandbox, SANDBOX_INITIALIZED);
runtime_sandbox_total_increment(SANDBOX_INITIALIZED);
}

@ -0,0 +1,58 @@
#pragma once
#include <assert.h>
#include <stdint.h>
#include "arch/getcycles.h"
#include "local_runqueue.h"
#include "panic.h"
#include "sandbox_types.h"
/**
* Transitions a sandbox to the SANDBOX_PREEMPTED state.
*
* This occurs in the following scenarios:
* - A sandbox in the SANDBOX_INITIALIZED state completes initialization and is ready to be run
* - A sandbox in the SANDBOX_BLOCKED state completes what was blocking it and is ready to be run
*
* @param sandbox
* @param last_state the state the sandbox is transitioning from. This is expressed as a constant to
* enable the compiler to perform constant propagation optimizations.
*/
static inline void
sandbox_set_as_preempted(struct sandbox *sandbox, sandbox_state_t last_state)
{
assert(sandbox);
/* Preemption occurs indirectly via the SANDBOX_RUNNING_KERNEL state, so preemptable is set
* to false during the process of preemption.
*/
assert(sandbox->ctxt.preemptable == false);
uint64_t now = __getcycles();
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_PREEMPTED;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_PREEMPTED);
switch (last_state) {
case SANDBOX_RUNNING_KERNEL: {
sandbox->duration_of_state.preempted += duration_of_last_state;
current_sandbox_set(NULL);
runtime_worker_threads_deadline[worker_thread_idx] = UINT64_MAX;
break;
}
default: {
panic("Sandbox %lu | Illegal transition from %s to Preempted\n", sandbox->id,
sandbox_state_stringify(last_state));
}
}
sandbox->timestamp_of.last_state_change = now;
sandbox->state = SANDBOX_PREEMPTED;
/* State Change Bookkeeping */
sandbox_state_history_append(sandbox, SANDBOX_PREEMPTED);
runtime_sandbox_total_increment(SANDBOX_PREEMPTED);
runtime_sandbox_total_decrement(last_state);
}

@ -28,12 +28,13 @@ sandbox_set_as_returned(struct sandbox *sandbox, sandbox_state_t last_state)
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_RETURNED;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_RETURNED);
switch (last_state) {
case SANDBOX_RUNNING: {
case SANDBOX_RUNNING_KERNEL: {
sandbox->timestamp_of.response = now;
sandbox->total_time = now - sandbox->timestamp_of.request_arrival;
sandbox->duration_of_state.running += duration_of_last_state;
sandbox->duration_of_state.running_kernel += duration_of_last_state;
local_runqueue_delete(sandbox);
sandbox_free_linear_memory(sandbox);
break;
@ -48,7 +49,7 @@ sandbox_set_as_returned(struct sandbox *sandbox, sandbox_state_t last_state)
sandbox->state = SANDBOX_RETURNED;
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox->id, last_state, SANDBOX_RETURNED);
sandbox_state_history_append(sandbox, SANDBOX_RETURNED);
runtime_sandbox_total_increment(SANDBOX_RETURNED);
runtime_sandbox_total_decrement(last_state);
}

@ -28,6 +28,7 @@ sandbox_set_as_runnable(struct sandbox *sandbox, sandbox_state_t last_state)
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_RUNNABLE;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_RUNNABLE);
switch (last_state) {
case SANDBOX_INITIALIZED: {
@ -40,11 +41,6 @@ sandbox_set_as_runnable(struct sandbox *sandbox, sandbox_state_t last_state)
local_runqueue_add(sandbox);
break;
}
case SANDBOX_RUNNING: {
sandbox->duration_of_state.running += duration_of_last_state;
/* No need to add to runqueue, as already on it */
break;
}
default: {
panic("Sandbox %lu | Illegal transition from %s to Runnable\n", sandbox->id,
sandbox_state_stringify(last_state));
@ -55,7 +51,7 @@ sandbox_set_as_runnable(struct sandbox *sandbox, sandbox_state_t last_state)
sandbox->state = SANDBOX_RUNNABLE;
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox->id, last_state, SANDBOX_RUNNABLE);
sandbox_state_history_append(sandbox, SANDBOX_RUNNABLE);
runtime_sandbox_total_increment(SANDBOX_RUNNABLE);
runtime_sandbox_total_decrement(last_state);
}

@ -1,41 +0,0 @@
#pragma once
#include <assert.h>
#include <stdint.h>
#include "arch/getcycles.h"
#include "panic.h"
#include "sandbox_types.h"
static inline void
sandbox_set_as_running(struct sandbox *sandbox, sandbox_state_t last_state)
{
assert(sandbox);
uint64_t now = __getcycles();
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_RUNNING;
switch (last_state) {
case SANDBOX_RUNNABLE: {
sandbox->duration_of_state.runnable += duration_of_last_state;
current_sandbox_set(sandbox);
runtime_worker_threads_deadline[worker_thread_idx] = sandbox->absolute_deadline;
/* Does not handle context switch because the caller knows if we need to use fast or slow switched */
break;
}
default: {
panic("Sandbox %lu | Illegal transition from %s to Running\n", sandbox->id,
sandbox_state_stringify(last_state));
}
}
sandbox->timestamp_of.last_state_change = now;
sandbox->state = SANDBOX_RUNNING;
/* State Change Bookkeeping */
sandbox_state_log_transition(sandbox->id, last_state, SANDBOX_RUNNING);
runtime_sandbox_total_increment(SANDBOX_RUNNING);
runtime_sandbox_total_decrement(last_state);
}

@ -0,0 +1,71 @@
#pragma once
#include <assert.h>
#include <stdint.h>
#include "arch/getcycles.h"
#include "current_sandbox.h"
#include "panic.h"
#include "sandbox_types.h"
#include "sandbox_functions.h"
static inline void
sandbox_set_as_running_kernel(struct sandbox *sandbox, sandbox_state_t last_state)
{
assert(sandbox);
/* Disable preemption at start of state transition */
if (last_state == SANDBOX_RUNNING_USER) {
assert(sandbox->ctxt.preemptable == true);
sandbox_disable_preemption(sandbox);
} else {
assert(sandbox->ctxt.preemptable == false);
}
uint64_t now = __getcycles();
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_RUNNING_KERNEL;
sandbox_state_history_append(sandbox, SANDBOX_SET_AS_RUNNING_KERNEL);
switch (last_state) {
case SANDBOX_RUNNING_USER: {
assert(sandbox == current_sandbox_get());
sandbox->duration_of_state.running_user += duration_of_last_state;
assert(worker_thread_current_sandbox == sandbox);
assert(runtime_worker_threads_deadline[worker_thread_idx] == sandbox->absolute_deadline);
break;
}
case SANDBOX_RUNNABLE: {
assert(sandbox);
sandbox->duration_of_state.runnable += duration_of_last_state;
current_sandbox_set(sandbox);
runtime_worker_threads_deadline[worker_thread_idx] = sandbox->absolute_deadline;
/* Does not handle context switch because the caller knows if we need to use fast or slow switched. We
* can fix this by breakout out SANDBOX_RUNNABLE and SANDBOX_PREEMPTED */
break;
}
case SANDBOX_PREEMPTED: {
assert(sandbox);
assert(sandbox->interrupted_state == SANDBOX_RUNNING_USER);
sandbox->duration_of_state.preempted += duration_of_last_state;
current_sandbox_set(sandbox);
runtime_worker_threads_deadline[worker_thread_idx] = sandbox->absolute_deadline;
/* Does not handle context switch because the caller knows if we need to use fast or slow switched. We
* can fix this by breakout out SANDBOX_RUNNABLE and SANDBOX_PREEMPTED */
break;
}
default: {
panic("Sandbox %lu | Illegal transition from %s to Running Kernel\n", sandbox->id,
sandbox_state_stringify(last_state));
}
}
sandbox->timestamp_of.last_state_change = now;
sandbox->state = SANDBOX_RUNNING_KERNEL;
/* State Change Bookkeeping */
sandbox_state_history_append(sandbox, SANDBOX_RUNNING_KERNEL);
runtime_sandbox_total_increment(SANDBOX_RUNNING_KERNEL);
runtime_sandbox_total_decrement(last_state);
}

@ -0,0 +1,48 @@
#pragma once
#include <assert.h>
#include <stdint.h>
#include "arch/getcycles.h"
#include "current_sandbox.h"
#include "panic.h"
#include "sandbox_types.h"
#include "sandbox_functions.h"
static inline void
sandbox_set_as_running_user(struct sandbox *sandbox, sandbox_state_t last_state)
{
assert(sandbox);
uint64_t now = __getcycles();
uint64_t duration_of_last_state = now - sandbox->timestamp_of.last_state_change;
sandbox->state = SANDBOX_SET_AS_RUNNING_USER;
switch (last_state) {
case SANDBOX_RUNNING_KERNEL: {
sandbox->duration_of_state.running_user += duration_of_last_state;
assert(sandbox == current_sandbox_get());
assert(runtime_worker_threads_deadline[worker_thread_idx] == sandbox->absolute_deadline);
break;
}
default: {
panic("Sandbox %lu | Illegal transition from %s to Running\n", sandbox->id,
sandbox_state_stringify(last_state));
}
}
sandbox->timestamp_of.last_state_change = now;
sandbox->state = SANDBOX_RUNNING_USER;
/* State Change Bookkeeping */
sandbox_state_history_append(sandbox, SANDBOX_RUNNING_USER);
runtime_sandbox_total_increment(SANDBOX_RUNNING_USER);
runtime_sandbox_total_decrement(last_state);
/* Enable preemption at the end of state transition*/
assert(last_state == SANDBOX_RUNNING_KERNEL);
sandbox_enable_preemption(sandbox);
}

@ -14,8 +14,12 @@ typedef enum
SANDBOX_INITIALIZED,
SANDBOX_SET_AS_RUNNABLE,
SANDBOX_RUNNABLE,
SANDBOX_SET_AS_RUNNING,
SANDBOX_RUNNING,
SANDBOX_SET_AS_PREEMPTED,
SANDBOX_PREEMPTED,
SANDBOX_SET_AS_RUNNING_KERNEL,
SANDBOX_RUNNING_KERNEL,
SANDBOX_SET_AS_RUNNING_USER,
SANDBOX_RUNNING_USER,
SANDBOX_SET_AS_BLOCKED,
SANDBOX_BLOCKED,
SANDBOX_SET_AS_RETURNED,
@ -31,7 +35,9 @@ typedef enum
struct sandbox_state_durations {
uint64_t initializing;
uint64_t runnable;
uint64_t running;
uint64_t running_kernel;
uint64_t running_user;
uint64_t preempted;
uint64_t blocked;
uint64_t returned;
};
@ -47,16 +53,6 @@ sandbox_state_stringify(sandbox_state_t state)
return sandbox_state_labels[state];
}
static inline void
sandbox_state_log_transition(uint64_t sandbox_id, sandbox_state_t last_state, sandbox_state_t current_state)
{
#ifdef LOG_STATE_CHANGES
debuglog("Sandbox %lu | %s => %s\n", sandbox_id, sandbox_state_stringify(last_state),
sandbox_state_stringify(current_state));
#endif
}
#ifdef LOG_SANDBOX_COUNT
extern _Atomic uint32_t sandbox_state_count[SANDBOX_STATE_COUNT];
#endif

@ -18,6 +18,10 @@
#define SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT 1024
#endif
#ifdef LOG_STATE_CHANGES
#define SANDBOX_STATE_HISTORY_CAPACITY 100
#endif
/*********************
* Structs and Types *
********************/
@ -70,7 +74,14 @@ struct sandbox_buffer {
struct sandbox {
uint64_t id;
sandbox_state_t state;
struct ps_list list; /* used by ps_list's default name-based MACROS for the scheduling runqueue */
sandbox_state_t interrupted_state;
#ifdef LOG_STATE_CHANGES
sandbox_state_t state_history[SANDBOX_STATE_HISTORY_CAPACITY];
uint16_t state_history_count;
#endif
struct ps_list list; /* used by ps_list's default name-based MACROS for the scheduling runqueue */
/* HTTP State */
struct sockaddr client_address; /* client requesting connection! */

@ -18,8 +18,9 @@
#include "sandbox_functions.h"
#include "sandbox_types.h"
#include "sandbox_set_as_blocked.h"
#include "sandbox_set_as_preempted.h"
#include "sandbox_set_as_runnable.h"
#include "sandbox_set_as_running.h"
#include "sandbox_set_as_running_kernel.h"
#include "worker_thread_execute_epoll_loop.h"
enum SCHEDULER
@ -155,9 +156,10 @@ scheduler_preempt(ucontext_t *user_context)
struct sandbox *current = current_sandbox_get();
assert(current != NULL);
assert(current->state == SANDBOX_RUNNING);
assert(current->state == SANDBOX_RUNNING_KERNEL);
struct sandbox *next = scheduler_get_next();
/* Assumption: the current sandbox is on the runqueue, so the scheduler should always return something */
assert(next != NULL);
/* If current equals next, no switch is necessary, so resume execution */
@ -168,15 +170,18 @@ scheduler_preempt(ucontext_t *user_context)
#endif
/* Save the context of the currently executing sandbox before switching from it */
sandbox_set_as_runnable(current, SANDBOX_RUNNING);
/* How do I switch back to "user running" when this is resumed? */
sandbox_set_as_preempted(current, SANDBOX_RUNNING_KERNEL);
arch_mcontext_save(&current->ctxt, &user_context->uc_mcontext);
/* Update current_sandbox to the next sandbox */
assert(next->state == SANDBOX_RUNNABLE);
sandbox_set_as_running(next, SANDBOX_RUNNABLE);
// assert(next->state == SANDBOX_RUNNABLE);
switch (next->ctxt.variant) {
case ARCH_CONTEXT_VARIANT_FAST: {
assert(next->state == SANDBOX_RUNNABLE);
sandbox_set_as_running_kernel(next, SANDBOX_RUNNABLE);
arch_context_restore_new(&user_context->uc_mcontext, &next->ctxt);
break;
}
@ -197,8 +202,9 @@ scheduler_preempt(ucontext_t *user_context)
* either a fast or a slow context to be restored during "round robin" execution.
*/
assert(scheduler != SCHEDULER_EDF);
assert(next->state == SANDBOX_PREEMPTED);
arch_mcontext_restore(&user_context->uc_mcontext, &next->ctxt);
sandbox_set_as_running_kernel(next, SANDBOX_PREEMPTED);
break;
}
default: {
@ -245,7 +251,7 @@ static inline void
scheduler_switch_to(struct sandbox *next_sandbox)
{
assert(next_sandbox != NULL);
assert(next_sandbox->state == SANDBOX_RUNNABLE);
assert(next_sandbox->state == SANDBOX_RUNNABLE || next_sandbox->state == SANDBOX_PREEMPTED);
struct arch_context *next_context = &next_sandbox->ctxt;
/* Get the old sandbox we're switching from.
@ -261,7 +267,7 @@ scheduler_switch_to(struct sandbox *next_sandbox)
}
scheduler_log_sandbox_switch(current_sandbox, next_sandbox);
sandbox_set_as_running(next_sandbox, next_sandbox->state);
sandbox_set_as_running_kernel(next_sandbox, next_sandbox->state);
arch_context_switch(current_context, next_context);
}
@ -305,8 +311,8 @@ scheduler_block(void)
/* Remove the sandbox we were just executing from the runqueue and mark as blocked */
struct sandbox *current_sandbox = current_sandbox_get();
assert(current_sandbox->state == SANDBOX_RUNNING);
sandbox_set_as_blocked(current_sandbox, SANDBOX_RUNNING);
assert(current_sandbox->state == SANDBOX_RUNNING_KERNEL);
sandbox_set_as_blocked(current_sandbox, SANDBOX_RUNNING_KERNEL);
generic_thread_dump_lock_overhead();
scheduler_yield();

@ -6,6 +6,8 @@
#include "sandbox_send_response.h"
#include "sandbox_set_as_error.h"
#include "sandbox_set_as_returned.h"
#include "sandbox_set_as_running_user.h"
#include "sandbox_set_as_running_kernel.h"
#include "sandbox_setup_arguments.h"
#include "scheduler.h"
#include "software_interrupt.h"
@ -21,42 +23,6 @@ thread_local struct sandbox_context_cache local_sandbox_context_cache = {
.module_indirect_table = NULL,
};
static inline void
current_sandbox_enable_preemption(struct sandbox *sandbox)
{
#ifdef LOG_PREEMPTION
debuglog("Sandbox %lu - enabling preemption - Missed %d SIGALRM\n", sandbox->id,
software_interrupt_deferred_sigalrm);
fflush(stderr);
#endif
if (__sync_bool_compare_and_swap(&sandbox->ctxt.preemptable, 0, 1) == false) {
panic("Recursive call to current_sandbox_enable_preemption\n");
}
if (software_interrupt_deferred_sigalrm > 0) {
/* Update Max */
if (software_interrupt_deferred_sigalrm > software_interrupt_deferred_sigalrm_max[worker_thread_idx]) {
software_interrupt_deferred_sigalrm_max[worker_thread_idx] =
software_interrupt_deferred_sigalrm;
}
software_interrupt_deferred_sigalrm = 0;
// TODO: Replay. Does the replay need to be before or after enabling preemption?
}
}
static inline void
current_sandbox_disable_preemption(struct sandbox *sandbox)
{
#ifdef LOG_PREEMPTION
debuglog("Sandbox %lu - disabling preemption\n", sandbox->id);
fflush(stderr);
#endif
if (__sync_bool_compare_and_swap(&sandbox->ctxt.preemptable, 1, 0) == false) {
panic("Recursive call to current_sandbox_disable_preemption\n");
}
}
/**
* Sandbox execution logic
* Handles setup, request parsing, WebAssembly initialization, function execution, response building and
@ -67,7 +33,7 @@ current_sandbox_start(void)
{
struct sandbox *sandbox = current_sandbox_get();
assert(sandbox != NULL);
assert(sandbox->state == SANDBOX_RUNNING);
assert(sandbox->state == SANDBOX_RUNNING_KERNEL);
char *error_message = "";
int rc = 0;
@ -92,9 +58,13 @@ current_sandbox_start(void)
/* Executing the function */
int32_t argument_count = 0;
current_sandbox_enable_preemption(sandbox);
assert(sandbox->state == SANDBOX_RUNNING_KERNEL);
assert(sandbox->ctxt.preemptable == false);
sandbox_set_as_running_user(sandbox, SANDBOX_RUNNING_KERNEL);
assert(sandbox->ctxt.preemptable == true);
sandbox->return_value = module_entrypoint(current_module, argument_count, sandbox->arguments_offset);
current_sandbox_disable_preemption(sandbox);
assert(sandbox->state == SANDBOX_RUNNING_USER);
sandbox_set_as_running_kernel(sandbox, SANDBOX_RUNNING_USER);
sandbox->timestamp_of.completion = __getcycles();
/* Retrieve the result, construct the HTTP response, and send to client */
@ -107,9 +77,9 @@ current_sandbox_start(void)
sandbox->timestamp_of.response = __getcycles();
assert(sandbox->state == SANDBOX_RUNNING);
assert(sandbox->state == SANDBOX_RUNNING_KERNEL);
sandbox_close_http(sandbox);
sandbox_set_as_returned(sandbox, SANDBOX_RUNNING);
sandbox_set_as_returned(sandbox, SANDBOX_RUNNING_KERNEL);
done:
/* Cleanup connection and exit sandbox */
@ -122,9 +92,9 @@ done:
assert(0);
err:
debuglog("%s", error_message);
assert(sandbox->state == SANDBOX_RUNNING);
assert(sandbox->state == SANDBOX_RUNNING_KERNEL);
sandbox_close_http(sandbox);
sandbox_set_as_error(sandbox, SANDBOX_RUNNING);
sandbox_set_as_error(sandbox, SANDBOX_RUNNING_KERNEL);
goto done;
}

@ -60,7 +60,8 @@ local_runqueue_list_rotate()
if (ps_list_head_one_node(&local_runqueue_list)) return;
struct sandbox *sandbox_at_head = local_runqueue_list_remove_and_return();
assert(sandbox_at_head->state == SANDBOX_RUNNING || sandbox_at_head->state == SANDBOX_RUNNABLE);
assert(sandbox_at_head->state == SANDBOX_RUNNING_KERNEL || sandbox_at_head->state == SANDBOX_RUNNABLE
|| sandbox_at_head->state == SANDBOX_PREEMPTED);
local_runqueue_list_append(sandbox_at_head);
}

@ -16,7 +16,7 @@ expand_memory(void)
{
struct sandbox *sandbox = current_sandbox_get();
assert(sandbox->state == SANDBOX_RUNNING);
assert(sandbox->state == SANDBOX_RUNNING_USER || sandbox->state == SANDBOX_RUNNING_KERNEL);
assert(local_sandbox_context_cache.memory.size % WASM_PAGE_SIZE == 0);
/* Return -1 if we've hit the linear memory max */

@ -1,4 +1,5 @@
#include <assert.h>
#include <string.h>
#include <sys/mman.h>
#include "current_sandbox.h"
@ -153,6 +154,12 @@ sandbox_allocate(struct sandbox_request *sandbox_request)
}
sandbox->state = SANDBOX_ALLOCATED;
#ifdef LOG_STATE_CHANGES
sandbox->state_history_count = 0;
sandbox->state_history[sandbox->state_history_count++] = SANDBOX_ALLOCATED;
memset(&sandbox->state_history, 0, 100);
#endif
/* Set state to initializing */
sandbox_set_as_initialized(sandbox, sandbox_request, now);

@ -8,31 +8,40 @@
#include "sandbox_state.h"
const bool sandbox_state_is_terminal[SANDBOX_STATE_COUNT] = {
[SANDBOX_UNINITIALIZED] = false, [SANDBOX_ALLOCATED] = false, [SANDBOX_INITIALIZED] = true,
[SANDBOX_SET_AS_RUNNABLE] = false, [SANDBOX_RUNNABLE] = true, [SANDBOX_SET_AS_RUNNING] = false,
[SANDBOX_RUNNING] = true, [SANDBOX_SET_AS_BLOCKED] = false, [SANDBOX_BLOCKED] = true,
[SANDBOX_SET_AS_RETURNED] = false, [SANDBOX_RETURNED] = true, [SANDBOX_SET_AS_COMPLETE] = false,
[SANDBOX_COMPLETE] = true, [SANDBOX_SET_AS_ERROR] = false, [SANDBOX_ERROR] = true
[SANDBOX_UNINITIALIZED] = false, [SANDBOX_ALLOCATED] = false,
[SANDBOX_INITIALIZED] = true, [SANDBOX_SET_AS_RUNNABLE] = false,
[SANDBOX_RUNNABLE] = true, [SANDBOX_SET_AS_PREEMPTED] = false,
[SANDBOX_PREEMPTED] = true, [SANDBOX_SET_AS_RUNNING_KERNEL] = false,
[SANDBOX_RUNNING_KERNEL] = true, [SANDBOX_SET_AS_RUNNING_USER] = false,
[SANDBOX_RUNNING_USER] = true, [SANDBOX_SET_AS_BLOCKED] = false,
[SANDBOX_BLOCKED] = true, [SANDBOX_SET_AS_RETURNED] = false,
[SANDBOX_RETURNED] = true, [SANDBOX_SET_AS_COMPLETE] = false,
[SANDBOX_COMPLETE] = true, [SANDBOX_SET_AS_ERROR] = false,
[SANDBOX_ERROR] = true
};
const char *sandbox_state_labels[SANDBOX_STATE_COUNT] = {
[SANDBOX_UNINITIALIZED] = "Uninitialized",
[SANDBOX_ALLOCATED] = "Allocated",
[SANDBOX_SET_AS_INITIALIZED] = "Transitioning to Initialized",
[SANDBOX_INITIALIZED] = "Initialized",
[SANDBOX_SET_AS_RUNNABLE] = "Transitioning to Runnable",
[SANDBOX_RUNNABLE] = "Runnable",
[SANDBOX_SET_AS_RUNNING] = "Transitioning to Running",
[SANDBOX_RUNNING] = "Running",
[SANDBOX_SET_AS_BLOCKED] = "Transitioning to Blocked",
[SANDBOX_BLOCKED] = "Blocked",
[SANDBOX_SET_AS_RETURNED] = "Transitioning to Returned",
[SANDBOX_RETURNED] = "Returned",
[SANDBOX_SET_AS_COMPLETE] = "Transitioning to Complete",
[SANDBOX_COMPLETE] = "Complete",
[SANDBOX_SET_AS_ERROR] = "Transitioning to Error",
[SANDBOX_ERROR] = "Error"
[SANDBOX_UNINITIALIZED] = "Uninitialized",
[SANDBOX_ALLOCATED] = "Allocated",
[SANDBOX_SET_AS_INITIALIZED] = "Transitioning to Initialized",
[SANDBOX_INITIALIZED] = "Initialized",
[SANDBOX_SET_AS_RUNNABLE] = "Transitioning to Runnable",
[SANDBOX_RUNNABLE] = "Runnable",
[SANDBOX_SET_AS_PREEMPTED] = "Transitioning to Preempted",
[SANDBOX_PREEMPTED] = "Preempted",
[SANDBOX_SET_AS_RUNNING_KERNEL] = "Transitioning to Running Kernel",
[SANDBOX_RUNNING_KERNEL] = "Running Kernel",
[SANDBOX_SET_AS_RUNNING_USER] = "Transitioning to Running User",
[SANDBOX_RUNNING_USER] = "Running User",
[SANDBOX_SET_AS_BLOCKED] = "Transitioning to Blocked",
[SANDBOX_BLOCKED] = "Blocked",
[SANDBOX_SET_AS_RETURNED] = "Transitioning to Returned",
[SANDBOX_RETURNED] = "Returned",
[SANDBOX_SET_AS_COMPLETE] = "Transitioning to Complete",
[SANDBOX_COMPLETE] = "Complete",
[SANDBOX_SET_AS_ERROR] = "Transitioning to Error",
[SANDBOX_ERROR] = "Error"
};
#ifdef LOG_SANDBOX_COUNT

@ -18,6 +18,8 @@
#include "module.h"
#include "panic.h"
#include "runtime.h"
#include "sandbox_set_as_running_kernel.h"
#include "sandbox_set_as_running_user.h"
#include "sandbox_types.h"
#include "scheduler.h"
#include "software_interrupt.h"
@ -146,18 +148,23 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void
switch (signal_type) {
case SIGALRM: {
bool preemptable = false;
if (current_sandbox) {
preemptable = current_sandbox->ctxt.preemptable;
current_sandbox->interrupted_state = current_sandbox->state;
if (preemptable && current_sandbox->state == SANDBOX_RUNNING_USER) {
sandbox_set_as_running_kernel(current_sandbox, SANDBOX_RUNNING_USER);
}
}
sigalrm_propagate_workers(signal_info);
if (current_sandbox == NULL || current_sandbox->ctxt.preemptable == false) {
/* Cannot preempt, so defer signal
* TODO: First worker gets tons of kernel sigalrms, should these be treated the same?
* When current_sandbox is NULL, we are looping through the scheduler, so sigalrm is redundant
* Maybe track time of last scheduling decision? i.e. when scheduler_get_next was last called.
*/
atomic_fetch_add(&software_interrupt_deferred_sigalrm, 1);
} else {
/* A worker thread received a SIGALRM while running a preemptable sandbox, so preempt */
assert(current_sandbox->state == SANDBOX_RUNNING);
if (preemptable) {
atomic_store(&software_interrupt_deferred_sigalrm, 0);
scheduler_preempt(user_context);
current_sandbox = current_sandbox_get();
} else {
atomic_fetch_add(&software_interrupt_deferred_sigalrm, 1);
}
goto done;
}
@ -188,6 +195,10 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void
}
done:
atomic_fetch_sub(&software_interrupt_signal_depth, 1);
if (current_sandbox && current_sandbox->interrupted_state == SANDBOX_RUNNING_USER) {
sandbox_set_as_running_user(current_sandbox, SANDBOX_RUNNING_KERNEL);
}
return;
}
/********************

Loading…
Cancel
Save