diff --git a/runtime/include/global_request_scheduler.h b/runtime/include/global_request_scheduler.h index f4c1304..e699366 100644 --- a/runtime/include/global_request_scheduler.h +++ b/runtime/include/global_request_scheduler.h @@ -16,6 +16,10 @@ struct global_request_scheduler_config { void global_request_scheduler_initialize(struct global_request_scheduler_config *config); +#define GLOBAL_REQUEST_SCHEDULER_REMOVE_OK 0 +#define GLOBAL_REQUEST_SCHEDULER_REMOVE_EMPTY -1 +#define GLOBAL_REQUEST_SCHEDULER_REMOVE_NOLOCK -2 + struct sandbox_request *global_request_scheduler_add(struct sandbox_request *); int global_request_scheduler_remove(struct sandbox_request **); uint64_t global_request_scheduler_peek(); diff --git a/runtime/include/sandbox.h b/runtime/include/sandbox.h index e263155..7c7a949 100644 --- a/runtime/include/sandbox.h +++ b/runtime/include/sandbox.h @@ -92,11 +92,13 @@ extern void worker_thread_wakeup_sandbox(struct sandbox *sandbox); * Public API * **************************/ -struct sandbox *sandbox_allocate(struct sandbox_request *sandbox_request); -void sandbox_free(struct sandbox *sandbox); -void sandbox_main(struct sandbox *sandbox); -int sandbox_parse_http_request(struct sandbox *sandbox, size_t length); +int sandbox_allocate(struct sandbox **sandbox, struct sandbox_request *sandbox_request); +#define SANDBOX_ALLOCATE_OK 0 +#define SANDBOX_ALLOCATE_ERR -1 +void sandbox_free(struct sandbox *sandbox); +void sandbox_main(struct sandbox *sandbox); +int sandbox_parse_http_request(struct sandbox *sandbox, size_t length); /** * Given a sandbox, returns the module that sandbox is executing diff --git a/runtime/src/local_runqueue_list.c b/runtime/src/local_runqueue_list.c index 84342c6..da2cd89 100644 --- a/runtime/src/local_runqueue_list.c +++ b/runtime/src/local_runqueue_list.c @@ -43,22 +43,24 @@ local_runqueue_list_remove_and_return() struct sandbox * local_runqueue_list_get_next() { + struct sandbox_request *sandbox_request; + // If our local runqueue is empty, try to pull and allocate a sandbox request from the global request scheduler if (local_runqueue_is_empty()) { - struct sandbox_request *sandbox_request; - - int return_code = global_request_scheduler_remove(&sandbox_request); - if (return_code != 0) return NULL; + if (global_request_scheduler_remove(&sandbox_request) != GLOBAL_REQUEST_SCHEDULER_REMOVE_OK) goto err; - /* TODO: sandbox_allocate should free sandbox_request on success */ - /* TODO: sandbox_allocate should return RC so we can readd sandbox_request to global_request_scheduler - * if needed */ - struct sandbox *sandbox = sandbox_allocate(sandbox_request); - assert(sandbox); - free(sandbox_request); + struct sandbox *sandbox; + if (sandbox_allocate(&sandbox, sandbox_request) == SANDBOX_ALLOCATE_ERR) goto sandbox_allocate_err; sandbox->state = SANDBOX_RUNNABLE; local_runqueue_add(sandbox); + + done: return sandbox; + sandbox_allocate_err: + global_request_scheduler_add(sandbox_request); + err: + sandbox = NULL; + goto done; } /* Execute Round Robin Scheduling Logic */ diff --git a/runtime/src/local_runqueue_minheap.c b/runtime/src/local_runqueue_minheap.c index d614e74..db3277f 100644 --- a/runtime/src/local_runqueue_minheap.c +++ b/runtime/src/local_runqueue_minheap.c @@ -70,8 +70,9 @@ local_runqueue_minheap_delete(struct sandbox *sandbox) struct sandbox * local_runqueue_minheap_get_next() { - struct sandbox *sandbox = NULL; - int sandbox_rc = local_runqueue_minheap_remove(&sandbox); + struct sandbox * sandbox = NULL; + struct sandbox_request *sandbox_request; + int sandbox_rc = local_runqueue_minheap_remove(&sandbox); if (sandbox_rc == 0) { /* Sandbox ready pulled from local runqueue */ @@ -82,20 +83,24 @@ local_runqueue_minheap_get_next() local_runqueue_minheap_add(sandbox); } else if (sandbox_rc == -1) { /* local runqueue was empty, try to pull a sandbox request and return NULL if we're unable to get one */ - struct sandbox_request *sandbox_request; - int sandbox_request_rc = global_request_scheduler_remove(&sandbox_request); - if (sandbox_request_rc != 0) return NULL; + if (global_request_scheduler_remove(&sandbox_request) != GLOBAL_REQUEST_SCHEDULER_REMOVE_OK) goto err; + + /* Try to allocate a sandbox, returning the request on failure */ + if (sandbox_allocate(&sandbox, sandbox_request) == SANDBOX_ALLOCATE_ERR) goto sandbox_allocate_err; - sandbox = sandbox_allocate(sandbox_request); - assert(sandbox); - free(sandbox_request); sandbox->state = SANDBOX_RUNNABLE; local_runqueue_minheap_add(sandbox); } else if (sandbox_rc == -2) { /* Unable to take lock, so just return NULL and try later */ assert(sandbox == NULL); } +done: return sandbox; +sandbox_allocate_err: + global_request_scheduler_add(sandbox_request); +err: + sandbox = NULL; + goto done; } @@ -126,37 +131,29 @@ local_runqueue_minheap_preempt(ucontext_t *user_context) uint64_t local_deadline = priority_queue_peek(&local_runqueue_minheap); uint64_t global_deadline = global_request_scheduler_peek(); - /* Our local deadline should only be ULONG_MAX if our local runqueue is empty */ - if (local_deadline == ULONG_MAX) { assert(local_runqueue_minheap.first_free == 1); }; - - /* - * If we're able to get a sandbox request with a tighter deadline, preempt the current context and run it - * - * TODO: Factor quantum and/or sandbox allocation time into decision - * Something like global_request_scheduler_peek() - software_interrupt_interval_duration_in_cycles; - */ - + /* If we're able to get a sandbox request with a tighter deadline, preempt the current context and run it */ + struct sandbox_request *sandbox_request; if (global_deadline < local_deadline) { debuglog("Thread %lu | Sandbox %lu | Had deadline of %lu. Trying to preempt for request with %lu\n", pthread_self(), current_sandbox->allocation_timestamp, local_deadline, global_deadline); - struct sandbox_request *sandbox_request; - int return_code = global_request_scheduler_remove(&sandbox_request); + int return_code = global_request_scheduler_remove(&sandbox_request); - // If we were unable to get a sandbox_request, exit + /* If we were unable to get a sandbox_request, exit */ if (return_code != 0) goto done; debuglog("Thread %lu Preempted %lu for %lu\n", pthread_self(), local_deadline, sandbox_request->absolute_deadline); /* Allocate the request */ - struct sandbox *next_sandbox = sandbox_allocate(sandbox_request); - assert(next_sandbox); - free(sandbox_request); - next_sandbox->state = SANDBOX_RUNNABLE; + struct sandbox *next_sandbox; + /* If sandbox allocation fails, add the request back and exit*/ + assert(software_interrupt_is_disabled); + if (sandbox_allocate(&next_sandbox, sandbox_request) == -1) goto err_sandbox_allocate; - /* Add it to the runqueue */ + /* Set as runnable and add it to the runqueue */ + next_sandbox->state = SANDBOX_RUNNABLE; local_runqueue_add(next_sandbox); - debuglog("[%p: %s]\n", sandbox, sandbox->module->name); + assert(software_interrupt_is_disabled); /* Save the context of the currently executing sandbox before switching from it */ arch_mcontext_save(¤t_sandbox->ctxt, &user_context->uc_mcontext); @@ -174,6 +171,12 @@ local_runqueue_minheap_preempt(ucontext_t *user_context) } done: if (should_enable_software_interrupt) software_interrupt_enable(); + return; +err_sandbox_allocate: + assert(sandbox_request != NULL); + global_request_scheduler_add(sandbox_request); +err: + goto done; } diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index 5ec4149..4dffbea 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -380,63 +380,70 @@ err_stack_allocation_failed: return -1; } -struct sandbox * -sandbox_allocate(struct sandbox_request *sandbox_request) +/** + * Allocates a new sandbox from a sandbox request, freeing the sandbox request on success + * @param sandbox pointer to destination pointer to set to newly allocated sandbox + * @param sandbox_request request being allocated + * @returns 0 on success, -1 on error + */ +int +sandbox_allocate(struct sandbox **sandbox, struct sandbox_request *sandbox_request) { + assert(sandbox != NULL); assert(sandbox_request != NULL); assert(sandbox_request->module != NULL); assert(module_is_valid(sandbox_request->module)); - char * error_message = ""; - int rc; - struct sandbox *sandbox = NULL; + char *error_message = ""; + int rc; /* Allocate Sandbox control structures, buffers, and linear memory in a 4GB address space */ - sandbox = (struct sandbox *)sandbox_allocate_memory(sandbox_request->module); + *sandbox = (struct sandbox *)sandbox_allocate_memory(sandbox_request->module); if (!sandbox) { error_message = "failed to allocate sandbox heap and linear memory"; goto err_memory_allocation_failed; } /* Set state to initializing */ - sandbox->state = SANDBOX_INITIALIZING; + (*sandbox)->state = SANDBOX_INITIALIZING; /* Allocate the Stack */ - rc = sandbox_allocate_stack(sandbox); - if (rc != 0) { + if (sandbox_allocate_stack(*sandbox) == -1) { error_message = "failed to allocate sandbox heap and linear memory"; goto err_stack_allocation_failed; } /* Copy the socket descriptor, address, and arguments of the client invocation */ - sandbox->absolute_deadline = sandbox_request->absolute_deadline; - sandbox->arguments = (void *)sandbox_request->arguments; - sandbox->client_socket_descriptor = sandbox_request->socket_descriptor; - sandbox->request_arrival_timestamp = sandbox_request->request_arrival_timestamp; + (*sandbox)->absolute_deadline = sandbox_request->absolute_deadline; + (*sandbox)->arguments = (void *)sandbox_request->arguments; + (*sandbox)->client_socket_descriptor = sandbox_request->socket_descriptor; + (*sandbox)->request_arrival_timestamp = sandbox_request->request_arrival_timestamp; /* Initialize the sandbox's context, stack, and instruction pointer */ - arch_context_init(&sandbox->ctxt, (reg_t)current_sandbox_main, - (reg_t)(sandbox->stack_start + sandbox->stack_size)); + arch_context_init(&(*sandbox)->ctxt, (reg_t)current_sandbox_main, + (reg_t)((*sandbox)->stack_start + (*sandbox)->stack_size)); /* TODO: What does it mean if there isn't a socket_address? Shouldn't this be a hard requirement? It seems that only the socket descriptor is used to send response */ const struct sockaddr *socket_address = sandbox_request->socket_address; - if (socket_address) memcpy(&sandbox->client_address, socket_address, sizeof(struct sockaddr)); + if (socket_address) memcpy(&(*sandbox)->client_address, socket_address, sizeof(struct sockaddr)); /* Initialize file descriptors to -1 */ - for (int i = 0; i < SANDBOX_MAX_IO_HANDLE_COUNT; i++) sandbox->io_handles[i].file_descriptor = -1; + for (int i = 0; i < SANDBOX_MAX_IO_HANDLE_COUNT; i++) (*sandbox)->io_handles[i].file_descriptor = -1; /* Initialize Parsec control structures (used by Completion Queue) */ - ps_list_init_d(sandbox); + ps_list_init_d(*sandbox); + free(sandbox_request); + rc = 0; done: - return sandbox; + return rc; err_stack_allocation_failed: - sandbox_free(sandbox); + sandbox_free(*sandbox); err_memory_allocation_failed: err: perror(error_message); - sandbox = NULL; + rc = -1; goto done; }