From 06910736d1d535ddd0ff0737ee2d64a214733085 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Mon, 22 Nov 2021 15:38:07 -0500 Subject: [PATCH] feat: Improved sandbox timekeeping --- .vscode/settings.json | 3 +- docs/sledge-states/README.md | 2 + docs/sledge-states/states.dot | 12 +- docs/sledge-states/states.svg | 171 ++++++++++-------- .../experiments/bimodal/edf_preemption.env | 1 + runtime/include/sandbox_functions.h | 6 - runtime/include/sandbox_perf_log.h | 11 +- runtime/include/sandbox_set_as_interrupted.h | 62 +++++++ runtime/include/sandbox_set_as_preempted.h | 6 +- runtime/include/sandbox_set_as_running_sys.h | 6 +- runtime/include/sandbox_set_as_running_user.h | 1 - runtime/include/sandbox_state.h | 1 + runtime/include/scheduler.h | 7 +- runtime/src/current_sandbox.c | 4 +- runtime/src/sandbox_state.c | 1 + runtime/src/software_interrupt.c | 22 ++- 16 files changed, 202 insertions(+), 114 deletions(-) create mode 100644 runtime/include/sandbox_set_as_interrupted.h diff --git a/.vscode/settings.json b/.vscode/settings.json index 444dcd4..ddc6b45 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -100,7 +100,8 @@ "sandbox_set_as_running_user.h": "c", "scheduler.h": "c", "sandbox_set_as_returned.h": "c", - "software_interrupt_counts.h": "c" + "software_interrupt_counts.h": "c", + "sandbox_set_as_running_sys.h": "c" }, "files.exclude": { "**/.git": true, diff --git a/docs/sledge-states/README.md b/docs/sledge-states/README.md index 0d0adbb..366221d 100644 --- a/docs/sledge-states/README.md +++ b/docs/sledge-states/README.md @@ -1 +1,3 @@ This is a state transition diagram of a sandbox. This maps to the state transition functions defined in `runtime/include/sandbox_set_as_*.h` + +Technically, this does not capture all state transitions to or from SANDBOX_INTERRUPTED, as any state can be interrupted by a SIGALRM and this would clutter the diagram. The only transitions shown to or from SANDBOX_INTERRUPTED are those leading to SANDBOX_PREEMPTED, as this reflects actual changes to a sandbox within the scheduler. All other transitions to/from SANDBOX_INTERRUPTED are mostly concerned with preventing scheduler execution time from being counted against sandbox execution times. diff --git a/docs/sledge-states/states.dot b/docs/sledge-states/states.dot index 6e85804..f64f276 100644 --- a/docs/sledge-states/states.dot +++ b/docs/sledge-states/states.dot @@ -2,11 +2,19 @@ digraph { Uninitialized -> Initialized Initialized -> {Runnable Error} Runnable -> Running_Sys - Running_User -> Running_Sys [label="interrupt"] + + Running_User -> Interrupted [label="interrupt"] + Running_User -> Running_Sys [label="syscall"] + + + Interrupted -> Preempted [label="preempt"] + Interrupted -> Running_User + + + Running_Sys -> Asleep [label="sleep"] Running_Sys -> {Error Returned} Running_Sys -> Running_User [label="return"] - Running_Sys -> Preempted [label="preempt"] Preempted -> Running_User Returned -> Complete [label="exit_success"] Asleep -> Runnable [label="wakeup"] diff --git a/docs/sledge-states/states.svg b/docs/sledge-states/states.svg index 7b036f6..2471742 100644 --- a/docs/sledge-states/states.svg +++ b/docs/sledge-states/states.svg @@ -4,154 +4,173 @@ - - + + %3 - + Uninitialized - -Uninitialized + +Uninitialized Initialized - -Initialized + +Initialized Uninitialized->Initialized - - + + Runnable - -Runnable + +Runnable Initialized->Runnable - - + + Error - -Error + +Error Initialized->Error - - + + Running_Sys - -Running_Sys + +Running_Sys Runnable->Running_Sys - - + + - + Running_Sys->Error - - + + Running_User - -Running_User + +Running_User - + Running_Sys->Running_User - - -return + + +return - + Asleep - -Asleep + +Asleep - + Running_Sys->Asleep - - -sleep + + +sleep - + Returned - -Returned + +Returned - + Running_Sys->Returned - - + + + + + +Running_User->Running_Sys + + +syscall + + + +Interrupted + +Interrupted + + + +Running_User->Interrupted + + +interrupt + + + +Interrupted->Running_User + + - + Preempted - -Preempted + +Preempted - - -Running_Sys->Preempted - - -preempt + + +Interrupted->Preempted + + +preempt - - -Running_User->Running_Sys - - -interrupt + + +Preempted->Running_User + + - + Asleep->Runnable - - -wakeup + + +wakeup - + Complete - -Complete + +Complete - + Returned->Complete - - -exit_success - - - -Preempted->Running_User - - + + +exit_success diff --git a/runtime/experiments/bimodal/edf_preemption.env b/runtime/experiments/bimodal/edf_preemption.env index 302a324..8305be6 100644 --- a/runtime/experiments/bimodal/edf_preemption.env +++ b/runtime/experiments/bimodal/edf_preemption.env @@ -1,3 +1,4 @@ SLEDGE_SCHEDULER=EDF SLEDGE_DISABLE_PREEMPTION=false SLEDGE_SIGALRM_HANDLER=TRIAGED +SLEDGE_NWORKERS=4 diff --git a/runtime/include/sandbox_functions.h b/runtime/include/sandbox_functions.h index 6bc0798..002893c 100644 --- a/runtime/include/sandbox_functions.h +++ b/runtime/include/sandbox_functions.h @@ -59,12 +59,6 @@ sandbox_get_priority(void *element) return sandbox->absolute_deadline; }; -static inline bool -sandbox_is_preemptable(struct sandbox *sandbox) -{ - return sandbox && sandbox->state == SANDBOX_RUNNING_USER; -}; - static inline void sandbox_open_http(struct sandbox *sandbox) { diff --git a/runtime/include/sandbox_perf_log.h b/runtime/include/sandbox_perf_log.h index 2463ba9..2bf3b51 100644 --- a/runtime/include/sandbox_perf_log.h +++ b/runtime/include/sandbox_perf_log.h @@ -36,15 +36,16 @@ sandbox_perf_log_print_entry(struct sandbox *sandbox) * becomes more intelligent, then peak linear memory size needs to be tracked * seperately from current linear memory size. */ - fprintf(sandbox_perf_log, "%lu,%s,%d,%s,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%u,%u\n", + fprintf(sandbox_perf_log, "%lu,%s,%d,%s,%lu,%lu,%lu,,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%u,%u\n", sandbox->id, sandbox->module->name, sandbox->module->port, sandbox_state_stringify(sandbox->state), sandbox->module->relative_deadline, sandbox->total_time, queued_duration, sandbox->duration_of_state[SANDBOX_UNINITIALIZED], sandbox->duration_of_state[SANDBOX_ALLOCATED], sandbox->duration_of_state[SANDBOX_INITIALIZED], sandbox->duration_of_state[SANDBOX_RUNNABLE], - sandbox->duration_of_state[SANDBOX_PREEMPTED], sandbox->duration_of_state[SANDBOX_RUNNING_SYS], - sandbox->duration_of_state[SANDBOX_RUNNING_USER], sandbox->duration_of_state[SANDBOX_ASLEEP], - sandbox->duration_of_state[SANDBOX_RETURNED], sandbox->duration_of_state[SANDBOX_COMPLETE], - sandbox->duration_of_state[SANDBOX_ERROR], runtime_processor_speed_MHz, sandbox->memory.size); + sandbox->duration_of_state[SANDBOX_INTERRUPTED], sandbox->duration_of_state[SANDBOX_PREEMPTED], + sandbox->duration_of_state[SANDBOX_RUNNING_SYS], sandbox->duration_of_state[SANDBOX_RUNNING_USER], + sandbox->duration_of_state[SANDBOX_ASLEEP], sandbox->duration_of_state[SANDBOX_RETURNED], + sandbox->duration_of_state[SANDBOX_COMPLETE], sandbox->duration_of_state[SANDBOX_ERROR], + runtime_processor_speed_MHz, sandbox->memory.size); } static inline void diff --git a/runtime/include/sandbox_set_as_interrupted.h b/runtime/include/sandbox_set_as_interrupted.h new file mode 100644 index 0000000..3ee1216 --- /dev/null +++ b/runtime/include/sandbox_set_as_interrupted.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +#include "arch/getcycles.h" +#include "current_sandbox.h" +#include "panic.h" +#include "sandbox_functions.h" +#include "sandbox_state_history.h" +#include "sandbox_types.h" + +static inline void +sandbox_set_as_interrupted(struct sandbox *sandbox, sandbox_state_t last_state) +{ + assert(sandbox); + + /* WARNING: All code before this assignment is preemptable */ + sandbox->state = SANDBOX_INTERRUPTED; + barrier(); + + uint64_t now = __getcycles(); + + /* State Change Bookkeeping */ + sandbox->duration_of_state[last_state] += (now - sandbox->timestamp_of.last_state_change); + sandbox->timestamp_of.last_state_change = now; + /* We do not append SANDBOX_INTERRUPTED to the sandbox_state_history because it would quickly fill the buffer */ + runtime_sandbox_total_increment(SANDBOX_INTERRUPTED); + runtime_sandbox_total_decrement(last_state); +} + +static inline void +sandbox_interrupt(struct sandbox *sandbox) +{ + sandbox_set_as_interrupted(sandbox, sandbox->state); +} + + +/** + * @brief Transition sandbox back to interrupted state + * @param sandbox + * @param interrupted_state - state to return to + */ +static inline void +sandbox_interrupt_return(struct sandbox *sandbox, sandbox_state_t interrupted_state) +{ + assert(sandbox); + assert(interrupted_state != SANDBOX_INTERRUPTED); + + uint64_t now = __getcycles(); + + /* State Change Bookkeeping */ + sandbox->duration_of_state[SANDBOX_INTERRUPTED] += (now - sandbox->timestamp_of.last_state_change); + sandbox->timestamp_of.last_state_change = now; + /* We do not append SANDBOX_INTERRUPTED to the sandbox_state_history because it would quickly fill the buffer */ + runtime_sandbox_total_increment(interrupted_state); + runtime_sandbox_total_decrement(SANDBOX_INTERRUPTED); + + barrier(); + /* WARNING: Code after this assignment may be preemptable */ + sandbox->state = interrupted_state; +} diff --git a/runtime/include/sandbox_set_as_preempted.h b/runtime/include/sandbox_set_as_preempted.h index 4d999c8..8dec5cb 100644 --- a/runtime/include/sandbox_set_as_preempted.h +++ b/runtime/include/sandbox_set_as_preempted.h @@ -26,7 +26,7 @@ sandbox_set_as_preempted(struct sandbox *sandbox, sandbox_state_t last_state) uint64_t now = __getcycles(); switch (last_state) { - case SANDBOX_RUNNING_SYS: { + case SANDBOX_INTERRUPTED: { break; } default: { @@ -46,6 +46,6 @@ sandbox_set_as_preempted(struct sandbox *sandbox, sandbox_state_t last_state) static inline void sandbox_preempt(struct sandbox *sandbox) { - assert(sandbox->state == SANDBOX_RUNNING_SYS); - sandbox_set_as_preempted(sandbox, SANDBOX_RUNNING_SYS); + assert(sandbox->state == SANDBOX_INTERRUPTED); + sandbox_set_as_preempted(sandbox, SANDBOX_INTERRUPTED); } diff --git a/runtime/include/sandbox_set_as_running_sys.h b/runtime/include/sandbox_set_as_running_sys.h index fd87035..67d6159 100644 --- a/runtime/include/sandbox_set_as_running_sys.h +++ b/runtime/include/sandbox_set_as_running_sys.h @@ -29,12 +29,10 @@ sandbox_set_as_running_sys(struct sandbox *sandbox, sandbox_state_t last_state) } case SANDBOX_RUNNABLE: { assert(sandbox); - /* Does not handle context switch because the caller knows if we need to use fast or slow switched. We - * can fix this by breakout out SANDBOX_RUNNABLE and SANDBOX_PREEMPTED */ break; } default: { - panic("Sandbox %lu | Illegal transition from %s to Running Kernel\n", sandbox->id, + panic("Sandbox %lu | Illegal transition from %s to Running Sys\n", sandbox->id, sandbox_state_stringify(last_state)); } } @@ -48,7 +46,7 @@ sandbox_set_as_running_sys(struct sandbox *sandbox, sandbox_state_t last_state) } static inline void -sandbox_interrupt(struct sandbox *sandbox) +sandbox_syscall(struct sandbox *sandbox) { assert(sandbox->state == SANDBOX_RUNNING_USER); sandbox_set_as_running_sys(sandbox, SANDBOX_RUNNING_USER); diff --git a/runtime/include/sandbox_set_as_running_user.h b/runtime/include/sandbox_set_as_running_user.h index 4ee3372..d25ca42 100644 --- a/runtime/include/sandbox_set_as_running_user.h +++ b/runtime/include/sandbox_set_as_running_user.h @@ -24,7 +24,6 @@ sandbox_set_as_running_user(struct sandbox *sandbox, sandbox_state_t last_state) break; } case SANDBOX_PREEMPTED: { - assert(sandbox); break; } default: { diff --git a/runtime/include/sandbox_state.h b/runtime/include/sandbox_state.h index 2dcd693..f107cf7 100644 --- a/runtime/include/sandbox_state.h +++ b/runtime/include/sandbox_state.h @@ -15,6 +15,7 @@ typedef enum SANDBOX_PREEMPTED, SANDBOX_RUNNING_SYS, SANDBOX_RUNNING_USER, + SANDBOX_INTERRUPTED, SANDBOX_ASLEEP, SANDBOX_RETURNED, SANDBOX_COMPLETE, diff --git a/runtime/include/scheduler.h b/runtime/include/scheduler.h index da46f69..19f05f9 100644 --- a/runtime/include/scheduler.h +++ b/runtime/include/scheduler.h @@ -19,6 +19,7 @@ #include "sandbox_set_as_preempted.h" #include "sandbox_set_as_runnable.h" #include "sandbox_set_as_running_sys.h" +#include "sandbox_set_as_interrupted.h" #include "sandbox_set_as_running_user.h" #include "scheduler_execute_epoll_loop.h" @@ -214,9 +215,7 @@ scheduler_preemptive_sched(ucontext_t *interrupted_context) struct sandbox *current = current_sandbox_get(); assert(current != NULL); - assert(current->state == SANDBOX_RUNNING_USER); - - sandbox_interrupt(current); + assert(current->state == SANDBOX_INTERRUPTED); struct sandbox *next = scheduler_get_next(); /* Assumption: the current sandbox is on the runqueue, so the scheduler should always return something */ @@ -224,7 +223,7 @@ scheduler_preemptive_sched(ucontext_t *interrupted_context) /* If current equals next, no switch is necessary, so resume execution */ if (current == next) { - sandbox_return(current); + sandbox_interrupt_return(current, SANDBOX_RUNNING_USER); return; } diff --git a/runtime/src/current_sandbox.c b/runtime/src/current_sandbox.c index aa4ca29..1f4d695 100644 --- a/runtime/src/current_sandbox.c +++ b/runtime/src/current_sandbox.c @@ -35,7 +35,7 @@ current_sandbox_sleep() { struct sandbox *sandbox = current_sandbox_get(); current_sandbox_set(NULL); - + assert(sandbox != NULL); struct arch_context *current_context = &sandbox->ctxt; @@ -139,7 +139,7 @@ current_sandbox_fini() assert(sandbox != NULL); char *error_message = ""; - sandbox_interrupt(sandbox); + sandbox_syscall(sandbox); sandbox->timestamp_of.completion = __getcycles(); diff --git a/runtime/src/sandbox_state.c b/runtime/src/sandbox_state.c index 3af06cf..2c9d160 100644 --- a/runtime/src/sandbox_state.c +++ b/runtime/src/sandbox_state.c @@ -13,6 +13,7 @@ const char *sandbox_state_labels[SANDBOX_STATE_COUNT] = { [SANDBOX_ALLOCATED] = "Allocated", [SANDBOX_INITIALIZED] = "Initialized", [SANDBOX_RUNNABLE] = "Runnable", + [SANDBOX_INTERRUPTED] = "Interrupted", [SANDBOX_PREEMPTED] = "Preempted", [SANDBOX_RUNNING_SYS] = "Running Sys", [SANDBOX_RUNNING_USER] = "Running User", diff --git a/runtime/src/software_interrupt.c b/runtime/src/software_interrupt.c index a04428a..27d47a9 100644 --- a/runtime/src/software_interrupt.c +++ b/runtime/src/software_interrupt.c @@ -19,6 +19,7 @@ #include "panic.h" #include "runtime.h" #include "sandbox_set_as_running_user.h" +#include "sandbox_set_as_interrupted.h" #include "sandbox_types.h" #include "scheduler.h" #include "software_interrupt.h" @@ -43,12 +44,6 @@ worker_thread_is_running_cooperative_scheduler(void) return current_sandbox_get() == NULL; } -static inline void -defer_sigalrm() -{ - atomic_fetch_add(&deferred_sigalrm, 1); -} - /** * A POSIX signal is delivered to only one thread. * This function broadcasts the sigalarm signal to all other worker threads @@ -112,14 +107,21 @@ software_interrupt_handle_signals(int signal_type, siginfo_t *signal_info, void switch (signal_type) { case SIGALRM: { - propagate_sigalrm(signal_info); + if (worker_thread_is_running_cooperative_scheduler()) { + propagate_sigalrm(signal_info); + break; + } - if (worker_thread_is_running_cooperative_scheduler()) break; + sandbox_state_t interrupted_state = current_sandbox->state; + sandbox_interrupt(current_sandbox); + propagate_sigalrm(signal_info); - if (sandbox_is_preemptable(current_sandbox)) { + if (interrupted_state == SANDBOX_RUNNING_USER) { + /* Preemptable, so run scheduler. The scheduler handles outgoing state changes */ scheduler_preemptive_sched(interrupted_context); } else { - defer_sigalrm(); + atomic_fetch_add(&deferred_sigalrm, 1); + sandbox_interrupt_return(current_sandbox, interrupted_state); } break;