nijinlong1
/
sledge-serverles-framwork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: sandbox memory and module cleanup

Browse Source
master
Sean McBride 4 years ago
parent a5739f338d
commit ae533694e9
10 changed files with 111 additions and 110 deletions
Show Stats Download Patch File Download Diff File

63
runtime/include/module.h
Unescape View File

@ -41,43 +41,42 @@
#endif
struct module {
char name[MODULE_MAX_NAME_LENGTH];
char path[MODULE_MAX_PATH_LENGTH];
void * dynamic_library_handle; /* Handle to the *.so of the serverless function */
int32_t argument_count;
uint32_t stack_size; /* a specification? */
uint64_t max_memory; /* perhaps a specification of the module. (max 4GB) */
uint32_t relative_deadline_us;
uint64_t relative_deadline; /* cycles */
_Atomic uint32_t reference_count; /* ref count how many instances exist here. */
struct indirect_table_entry indirect_table[INDIRECT_TABLE_SIZE];
struct sockaddr_in socket_address;
int socket_descriptor;
struct admissions_info admissions_info;
int port;
unsigned long max_request_size;
char request_headers[HTTP_MAX_HEADER_COUNT][HTTP_MAX_HEADER_LENGTH];
int request_header_count;
char request_content_type[HTTP_MAX_HEADER_VALUE_LENGTH];
/* resp size including headers! */
unsigned long max_response_size;
int response_header_count;
char response_content_type[HTTP_MAX_HEADER_VALUE_LENGTH];
char response_headers[HTTP_MAX_HEADER_COUNT][HTTP_MAX_HEADER_LENGTH];
/* Equals the largest of either max_request_size or max_response_size */
unsigned long max_request_or_response_size;
/* Functions to initialize aspects of sandbox */
/* Metadata from JSON Config */
char name[MODULE_MAX_NAME_LENGTH];
char path[MODULE_MAX_PATH_LENGTH];
int32_t argument_count;
uint32_t stack_size; /* a specification? */
uint64_t max_memory; /* perhaps a specification of the module. (max 4GB) */
uint32_t relative_deadline_us;
int port;
unsigned long max_request_size;
unsigned long max_response_size; /* resp size including headers! */
struct admissions_info admissions_info;
uint64_t relative_deadline; /* cycles */
/* HTTP State */
unsigned long max_request_or_response_size; /* largest of max_request_size or max_response_size */
char request_headers[HTTP_MAX_HEADER_COUNT][HTTP_MAX_HEADER_LENGTH];
int request_header_count;
char request_content_type[HTTP_MAX_HEADER_VALUE_LENGTH];
char response_content_type[HTTP_MAX_HEADER_VALUE_LENGTH];
char response_headers[HTTP_MAX_HEADER_COUNT][HTTP_MAX_HEADER_LENGTH];
int response_header_count;
struct sockaddr_in socket_address;
int socket_descriptor;
/* Dynamic Library Handle and Symbols */
void *dynamic_library_handle; /* Handle to the *.so of the serverless function */
/* Functions to initialize aspects of sandbox */
mod_glb_fn_t initialize_globals;
mod_mem_fn_t initialize_memory;
mod_tbl_fn_t initialize_tables;
mod_libc_fn_t initialize_libc;
mod_main_fn_t main; /* Entry Function */
/* Entry Function to invoke serverless function */
mod_main_fn_t main;
_Atomic uint32_t reference_count; /* ref count how many instances exist here. */
struct indirect_table_entry indirect_table[INDIRECT_TABLE_SIZE];
};
/*************************

2
runtime/include/sandbox_functions.h
Unescape View File

@ -35,7 +35,7 @@ sandbox_close_http(struct sandbox *sandbox)
static inline void
sandbox_free_linear_memory(struct sandbox *sandbox)
{
int rc = munmap(sandbox->memory.start, SANDBOX_MAX_MEMORY + PAGE_SIZE);
int rc = munmap(sandbox->memory.start, sandbox->memory.max + PAGE_SIZE);
if (rc == -1) panic("sandbox_free_linear_memory - munmap failed\n");
sandbox->memory.start = NULL;
}

10
runtime/include/sandbox_receive_request.h
Unescape View File

@ -25,7 +25,7 @@ sandbox_receive_request(struct sandbox *sandbox)
{
assert(sandbox != NULL);
assert(sandbox->module->max_request_size > 0);
assert(sandbox->request_response_data_length == 0);
assert(sandbox->buffer.length == 0);
int rc = 0;
@ -37,8 +37,8 @@ sandbox_receive_request(struct sandbox *sandbox)
const http_parser_settings *settings = http_parser_settings_get();
int fd = sandbox->client_socket_descriptor;
char * buf = &sandbox->request_response_data[sandbox->request_response_data_length];
size_t len = sandbox->module->max_request_size - sandbox->request_response_data_length;
char * buf = &sandbox->buffer.start[sandbox->buffer.length];
size_t len = sandbox->module->max_request_size - sandbox->buffer.length;
ssize_t recved = recv(fd, buf, len, 0);
@ -85,11 +85,11 @@ sandbox_receive_request(struct sandbox *sandbox)
}
sandbox->request_response_data_length += nparsed;
sandbox->buffer.length += nparsed;
}
sandbox->request_length = sandbox->request_response_data_length;
sandbox->http_request_length = sandbox->buffer.length;
rc = 0;
done:

31
runtime/include/sandbox_send_response.h
Unescape View File

@ -31,7 +31,7 @@ sandbox_send_response(struct sandbox *sandbox)
* smaller than the HTTP Request header, which allows us to use memmove once without copying
* to an intermediate buffer.
*/
memset(sandbox->request_response_data, 0, sandbox->request_length);
memset(sandbox->buffer.start, 0, sandbox->http_request_length);
/*
* We use this cursor to keep track of our position in the buffer and later assert that we
@ -40,42 +40,42 @@ sandbox_send_response(struct sandbox *sandbox)
size_t response_cursor = 0;
/* Append 200 OK */
strncpy(sandbox->request_response_data, HTTP_RESPONSE_200_OK, strlen(HTTP_RESPONSE_200_OK));
strncpy(sandbox->buffer.start, HTTP_RESPONSE_200_OK, strlen(HTTP_RESPONSE_200_OK));
response_cursor += strlen(HTTP_RESPONSE_200_OK);
/* Content Type */
strncpy(sandbox->request_response_data + response_cursor, HTTP_RESPONSE_CONTENT_TYPE,
strncpy(sandbox->buffer.start + response_cursor, HTTP_RESPONSE_CONTENT_TYPE,
strlen(HTTP_RESPONSE_CONTENT_TYPE));
response_cursor += strlen(HTTP_RESPONSE_CONTENT_TYPE);
/* Custom content type if provided, text/plain by default */
if (strlen(sandbox->module->response_content_type) <= 0) {
strncpy(sandbox->request_response_data + response_cursor, HTTP_RESPONSE_CONTENT_TYPE_PLAIN,
strncpy(sandbox->buffer.start + response_cursor, HTTP_RESPONSE_CONTENT_TYPE_PLAIN,
strlen(HTTP_RESPONSE_CONTENT_TYPE_PLAIN));
response_cursor += strlen(HTTP_RESPONSE_CONTENT_TYPE_PLAIN);
} else {
strncpy(sandbox->request_response_data + response_cursor, sandbox->module->response_content_type,
strncpy(sandbox->buffer.start + response_cursor, sandbox->module->response_content_type,
strlen(sandbox->module->response_content_type));
response_cursor += strlen(sandbox->module->response_content_type);
}
strncpy(sandbox->request_response_data + response_cursor, HTTP_RESPONSE_CONTENT_TYPE_TERMINATOR,
strncpy(sandbox->buffer.start + response_cursor, HTTP_RESPONSE_CONTENT_TYPE_TERMINATOR,
strlen(HTTP_RESPONSE_CONTENT_TYPE_TERMINATOR));
response_cursor += strlen(HTTP_RESPONSE_CONTENT_TYPE_TERMINATOR);
/* Content Length */
strncpy(sandbox->request_response_data + response_cursor, HTTP_RESPONSE_CONTENT_LENGTH,
strncpy(sandbox->buffer.start + response_cursor, HTTP_RESPONSE_CONTENT_LENGTH,
strlen(HTTP_RESPONSE_CONTENT_LENGTH));
response_cursor += strlen(HTTP_RESPONSE_CONTENT_LENGTH);
size_t body_size = sandbox->request_response_data_length - sandbox->request_length;
size_t body_size = sandbox->buffer.length - sandbox->http_request_length;
char len[10] = { 0 };
sprintf(len, "%zu", body_size);
strncpy(sandbox->request_response_data + response_cursor, len, strlen(len));
strncpy(sandbox->buffer.start + response_cursor, len, strlen(len));
response_cursor += strlen(len);
strncpy(sandbox->request_response_data + response_cursor, HTTP_RESPONSE_CONTENT_LENGTH_TERMINATOR,
strncpy(sandbox->buffer.start + response_cursor, HTTP_RESPONSE_CONTENT_LENGTH_TERMINATOR,
strlen(HTTP_RESPONSE_CONTENT_LENGTH_TERMINATOR));
response_cursor += strlen(HTTP_RESPONSE_CONTENT_LENGTH_TERMINATOR);
@ -84,14 +84,14 @@ sandbox_send_response(struct sandbox *sandbox)
* actual data that the program appended to the HTTP Request. If proves to be a bad assumption,
* we have to copy the STDOUT string to a temporary buffer before writing the header
*/
if (unlikely(response_cursor >= sandbox->request_length)) {
if (unlikely(response_cursor >= sandbox->http_request_length)) {
panic("Response Cursor: %zd is less that Request Length: %zd\n", response_cursor,
sandbox->request_length);
sandbox->http_request_length);
}
/* Move the Sandbox's Data after the HTTP Response Data */
memmove(sandbox->request_response_data + response_cursor,
sandbox->request_response_data + sandbox->request_length, body_size);
memmove(sandbox->buffer.start + response_cursor, sandbox->buffer.start + sandbox->http_request_length,
body_size);
response_cursor += body_size;
/* Capture Timekeeping data for end-to-end latency */
@ -101,8 +101,7 @@ sandbox_send_response(struct sandbox *sandbox)
int rc;
int sent = 0;
while (sent < response_cursor) {
rc = write(sandbox->client_socket_descriptor, &sandbox->request_response_data[sent],
response_cursor - sent);
rc = write(sandbox->client_socket_descriptor, &sandbox->buffer.start[sent], response_cursor - sent);
if (rc < 0) {
if (errno == EAGAIN)
scheduler_block();

66
runtime/include/sandbox_types.h
Unescape View File

@ -14,10 +14,6 @@
#include "sandbox_state.h"
#include "wasm_types.h"
#define SANDBOX_FILE_DESCRIPTOR_PREOPEN_MAGIC (707707707) /* upside down LOLLOLLOL 🤣😂🤣*/
#define SANDBOX_MAX_FD_COUNT 32
#define SANDBOX_MAX_MEMORY (1L << 32) /* 4GB */
#ifdef LOG_SANDBOX_MEMORY_PROFILE
#define SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT 1024
#endif
@ -26,10 +22,6 @@
* Structs and Types *
********************/
struct sandbox_io_handle {
int file_descriptor;
};
struct sandbox_stack {
void * start; /* points to the bottom of the usable stack */
uint32_t size;
@ -47,10 +39,42 @@ struct sandbox_timestamps {
#endif
};
/*
* In-memory buffer used to read requests, buffer write to STDOUT, and write HTTP responses
* The HTTP request is read in, updating buffer.length and http_request_length
* --------------------------------------------------
* | Request | Empty |
* --------------------------------------------------
* Writes to STDOUT are written starting at http_request_length, updating buffer.length
* --------------------------------------------------
* | Request | STDOUT | Empty |
* --------------------------------------------------
* The HTTP Response is written over the Request (assumes the response is smaller)
* --------------------------------------------------
* | Response | Gap | STDOUT | Empty |
* --------------------------------------------------
* And the STDOUT buffer is compacted to immediately follow the response
* --------------------------------------------------
* | Response | STDOUT | Empty |
* --------------------------------------------------
*/
struct sandbox_buffer {
ssize_t length; /* Should be <= module->max_request_or_response_size */
char start[1];
};
struct sandbox {
uint64_t id;
sandbox_state_t state;
uint32_t sandbox_size; /* The struct plus enough buffer to hold the request or response (sized off largest) */
struct ps_list list; /* used by ps_list's default name-based MACROS for the scheduling runqueue */
/* HTTP State */
struct sockaddr client_address; /* client requesting connection! */
int client_socket_descriptor;
http_parser http_parser;
struct http_request http_request;
ssize_t http_request_length;
/* WebAssembly Module State */
struct module *module; /* the module this is an instance of */
@ -67,33 +91,13 @@ struct sandbox {
uint64_t absolute_deadline;
uint64_t admissions_estimate; /* estimated execution time (cycles) * runtime_admissions_granularity / relative
deadline (cycles) */
uint64_t total_time; /* From Request to Response */
uint64_t total_time; /* Total time from Request to Response */
/* System Interface State */
int32_t arguments_offset; /* actual placement of arguments in the sandbox. */
void * arguments; /* arguments from request, must be of module->argument_count size. */
int32_t return_value;
struct sockaddr client_address; /* client requesting connection! */
int client_socket_descriptor;
bool is_repeat_header;
http_parser http_parser;
struct http_request http_request;
char * read_buffer;
ssize_t read_length, read_size;
/* Used for the scheduling runqueue as an in-place linked list data structure. */
/* The variable name "list" is used for ps_list's default name-based MACROS. */
struct ps_list list;
/*
* The length of the HTTP Request.
* This acts as an offset to the STDOUT of the Sandbox
*/
ssize_t request_length;
ssize_t request_response_data_length; /* Should be <= module->max_request_or_response_size */
char request_response_data[1]; /* of request_response_data_length, following sandbox mem.. */
/* This contains a Variable Length Array and thus MUST be the final member of this struct */
struct sandbox_buffer buffer;
} PAGE_ALIGNED;

15
runtime/include/wasm_types.h
Unescape View File

@ -3,14 +3,15 @@
#include <stdint.h>
/* FIXME: per-module configuration? Issue #101 */
#define WASM_PAGE_SIZE (1024 * 64) /* 64KB */
#define WASM_START_PAGES (1 << 8) /* 16MB */
#define WASM_MAX_PAGES (1 << 15) /* 4GB */
#define WASM_STACK_SIZE (1 << 19) /* 512KB */
#define WASM_PAGE_SIZE (1024 * 64) /* 64KB */
#define WASM_MEMORY_PAGES_INITIAL (1 << 8) /* 256 Pages ~16MB */
#define WASM_MEMORY_PAGES_MAX (1 << 15) /* 32,768 Pages ~4GB */
#define WASM_STACK_SIZE (1 << 19) /* 512KB */
/* bytes, not wasm pages */
struct wasm_memory {
void * start; /* after sandbox struct */
uint32_t size; /* from after sandbox struct */
uint64_t max; /* 4GB */
void * start;
uint32_t size;
uint64_t max;
};

10
runtime/src/libc/syscall.c
Unescape View File

@ -135,11 +135,11 @@ wasm_write(int32_t fd, int32_t buf_offset, int32_t buf_size)
if (fd == 1 || fd == 2) {
char *buffer = worker_thread_get_memory_ptr_void(buf_offset, buf_size);
int l = s->module->max_response_size - s->request_response_data_length;
int l = s->module->max_response_size - s->buffer.length;
if (l > buf_size) l = buf_size;
if (l == 0) return 0;
memcpy(s->request_response_data + s->request_response_data_length, buffer, l);
s->request_response_data_length += l;
memcpy(s->buffer.start + s->buffer.length, buffer, l);
s->buffer.length += l;
return l;
}
@ -266,8 +266,8 @@ wasm_writev(int32_t fd, int32_t iov_offset, int32_t iovcnt)
iovcnt * sizeof(struct wasm_iovec));
for (int i = 0; i < iovcnt; i++) {
char *b = worker_thread_get_memory_ptr_void(iov[i].base_offset, iov[i].len);
memcpy(c->request_response_data + c->request_response_data_length, b, iov[i].len);
c->request_response_data_length += iov[i].len;
memcpy(c->buffer.start + c->buffer.length, b, iov[i].len);
c->buffer.length += iov[i].len;
len += iov[i].len;
}

18
runtime/src/memory/64bit_nix.c
Unescape View File

@ -13,22 +13,20 @@ expand_memory(void)
{
struct sandbox *sandbox = current_sandbox_get();
// FIXME: max_pages = 0 => no limit. Issue #103.
assert((sandbox->sandbox_size + local_sandbox_context_cache.memory.size) / WASM_PAGE_SIZE < WASM_MAX_PAGES);
assert(sandbox->state == SANDBOX_RUNNING);
// Remap the relevant wasm page to readable
char *mem_as_chars = local_sandbox_context_cache.memory.start;
char *page_address = &mem_as_chars[local_sandbox_context_cache.memory.size];
void *map_result = mmap(page_address, WASM_PAGE_SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
// TODO: Refactor to return RC signifying out-of-mem to caller. Issue #96.
if (map_result == MAP_FAILED) panic("Mapping of new memory failed");
if (local_sandbox_context_cache.memory.size > local_sandbox_context_cache.memory.max)
if (local_sandbox_context_cache.memory.size + WASM_PAGE_SIZE >= local_sandbox_context_cache.memory.max)
panic("expand_memory - Out of Memory!. %u out of %lu\n", local_sandbox_context_cache.memory.size,
local_sandbox_context_cache.memory.max);
// Remap the relevant wasm page to readable
char *mem_as_chars = local_sandbox_context_cache.memory.start;
char *page_address = &mem_as_chars[local_sandbox_context_cache.memory.size];
void *map_result = mmap(page_address, WASM_PAGE_SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
if (map_result == MAP_FAILED) panic("Mapping of new memory failed");
local_sandbox_context_cache.memory.size += WASM_PAGE_SIZE;
#ifdef LOG_SANDBOX_MEMORY_PROFILE

2
runtime/src/module.c
Unescape View File

@ -225,7 +225,7 @@ module_new(char *name, char *path, int32_t argument_count, uint32_t stack_size,
module->argument_count = argument_count;
module->stack_size = ((uint32_t)(round_up_to_page(stack_size == 0 ? WASM_STACK_SIZE : stack_size)));
debuglog("Stack Size: %u", module->stack_size);
module->max_memory = max_memory == 0 ? ((uint64_t)WASM_PAGE_SIZE * WASM_MAX_PAGES) : max_memory;
module->max_memory = max_memory == 0 ? ((uint64_t)WASM_PAGE_SIZE * WASM_MEMORY_PAGES_MAX) : max_memory;
module->socket_descriptor = -1;
module->port = port;

4
runtime/src/sandbox.c
Unescape View File

@ -20,8 +20,8 @@ sandbox_allocate_memory(struct module *module)
assert(module != NULL);
char * error_message = NULL;
unsigned long memory_size = WASM_PAGE_SIZE * WASM_START_PAGES; /* The initial pages */
uint64_t memory_max = (uint64_t)SANDBOX_MAX_MEMORY;
unsigned long memory_size = WASM_PAGE_SIZE * WASM_MEMORY_PAGES_INITIAL; /* The initial pages */
uint64_t memory_max = (uint64_t)WASM_PAGE_SIZE * WASM_MEMORY_PAGES_MAX;
struct sandbox *sandbox = NULL;
unsigned long sandbox_size = sizeof(struct sandbox) + module->max_request_or_response_size;

Write Preview
Loading…
Cancel
Save