From ef09fcc3f13819853ef0d44400ca13c563cfe938 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 1 Dec 2021 17:57:33 -0500 Subject: [PATCH] refactor: cleanup linear memory code --- .vscode/settings.json | 5 +- runtime/compiletime/memory/64bit_nix.c | 411 +++++++++++++++++----- runtime/include/current_sandbox.h | 3 +- runtime/include/debuglog.h | 1 + runtime/include/module.h | 1 + runtime/include/runtime.h | 19 - runtime/include/sandbox_context_cache.h | 11 + runtime/include/sandbox_functions.h | 3 + runtime/include/sandbox_setup_arguments.h | 13 +- runtime/include/wasm_indirect_table.h | 11 + runtime/include/wasm_linear_memory.h | 254 ++++++++++--- runtime/src/libc/syscall.c | 24 +- runtime/src/memory/64bit_nix.c | 103 ------ runtime/src/memory/common.c | 37 -- runtime/src/sandbox.c | 17 +- 15 files changed, 576 insertions(+), 337 deletions(-) create mode 100644 runtime/include/sandbox_context_cache.h create mode 100644 runtime/include/wasm_indirect_table.h delete mode 100644 runtime/src/memory/64bit_nix.c delete mode 100644 runtime/src/memory/common.c diff --git a/.vscode/settings.json b/.vscode/settings.json index bcc4efed..411724c6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -102,7 +102,10 @@ "sandbox_set_as_returned.h": "c", "wasm_linear_memory.h": "c", "software_interrupt_counts.h": "c", - "sandbox_set_as_running_sys.h": "c" + "sandbox_set_as_running_sys.h": "c", + "compiletime.h": "c", + "sandbox_context_cache.h": "c", + "wasm_indirect_table.h": "c" }, "files.exclude": { "**/.git": true, diff --git a/runtime/compiletime/memory/64bit_nix.c b/runtime/compiletime/memory/64bit_nix.c index c52f8208..db369e39 100644 --- a/runtime/compiletime/memory/64bit_nix.c +++ b/runtime/compiletime/memory/64bit_nix.c @@ -1,182 +1,405 @@ #include +#include +#include + #include "types.h" +#include "sandbox_context_cache.h" -uint32_t +/* This file contains the stub functions that the aWsm compiler expects + * This corresponds to awsm/src/codegen/runtime_stubs.rs + * This should be linked with the *.bc file generated by aWsm in order to compile a module as a *.so + */ + +extern thread_local struct sandbox_context_cache local_sandbox_context_cache; + +EXPORT void +initialize_region(uint32_t offset, uint32_t region_size, uint8_t region[region_size]) +{ + wasm_linear_memory_initialize_region(local_sandbox_context_cache.memory, offset, region_size, region); +} + +EXPORT uint32_t instruction_memory_size() { - return local_sandbox_context_cache.memory.size / WASM_PAGE_SIZE; + return wasm_linear_memory_get_page_count(local_sandbox_context_cache.memory); } -// All of these are pretty generic -INLINE float -get_f32(uint32_t offset) +/** + * @brief Stub that implements the WebAssembly memory.grow instruction + * + * @param count number of pages to grow the WebAssembly linear memory by + * @return The previous size of the linear memory in pages or -1 if enough memory cannot be allocated + */ +EXPORT int32_t +instruction_memory_grow(uint32_t count) { - assert(offset + sizeof(float) <= local_sandbox_context_cache.memory.size); + int rc = local_sandbox_context_cache.memory->size / WASM_PAGE_SIZE; + + /* Return -1 if we've hit the linear memory max */ + if (unlikely(wasm_linear_memory_expand(local_sandbox_context_cache.memory, WASM_PAGE_SIZE * count) == -1)) + return -1; + +#ifdef LOG_SANDBOX_MEMORY_PROFILE + // Cache the runtime of the first N page allocations + for (int i = 0; i < count; i++) { + if (likely(sandbox->timestamp_of.page_allocations_size < SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT)) { + sandbox->timestamp_of.page_allocations[sandbox->timestamp_of.page_allocations_size++] = + sandbox->duration_of_state.running + + (uint32_t)(__getcycles() - sandbox->timestamp_of.last_state_change); + } + } +#endif + + return rc; +} - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; +EXPORT float +get_f32(uint32_t offset) +{ + return wasm_linear_memory_get_float(local_sandbox_context_cache.memory, offset); +} - return *(float *)address; +EXPORT void +set_f32(uint32_t offset, float v) +{ + wasm_linear_memory_set_float(local_sandbox_context_cache.memory, offset, v); } -INLINE double +EXPORT double get_f64(uint32_t offset) { - assert(offset + sizeof(double) <= local_sandbox_context_cache.memory.size); - - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; + return wasm_linear_memory_get_double(local_sandbox_context_cache.memory, offset); +} - return *(double *)address; +EXPORT void +set_f64(uint32_t offset, double v) +{ + wasm_linear_memory_set_double(local_sandbox_context_cache.memory, offset, v); } -INLINE int8_t +EXPORT int8_t get_i8(uint32_t offset) { - assert(offset + sizeof(int8_t) <= local_sandbox_context_cache.memory.size); - - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; + return wasm_linear_memory_get_int8(local_sandbox_context_cache.memory, offset); +} - return *(int8_t *)address; +EXPORT void +set_i8(uint32_t offset, int8_t v) +{ + wasm_linear_memory_set_int8(local_sandbox_context_cache.memory, offset, v); } -INLINE int16_t +EXPORT int16_t get_i16(uint32_t offset) { - assert(offset + sizeof(int16_t) <= local_sandbox_context_cache.memory.size); - - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; + return wasm_linear_memory_get_int16(local_sandbox_context_cache.memory, offset); +} - return *(int16_t *)address; +EXPORT void +set_i16(uint32_t offset, int16_t v) +{ + wasm_linear_memory_set_int16(local_sandbox_context_cache.memory, offset, v); } -INLINE int32_t +EXPORT int32_t get_i32(uint32_t offset) { - assert(offset + sizeof(int32_t) <= local_sandbox_context_cache.memory.size); - - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; + return wasm_linear_memory_get_int32(local_sandbox_context_cache.memory, offset); +} - return *(int32_t *)address; +EXPORT void +set_i32(uint32_t offset, int32_t v) +{ + wasm_linear_memory_set_int32(local_sandbox_context_cache.memory, offset, v); } -INLINE int64_t +EXPORT int64_t get_i64(uint32_t offset) { - assert(offset + sizeof(int64_t) <= local_sandbox_context_cache.memory.size); + return wasm_linear_memory_get_int64(local_sandbox_context_cache.memory, offset); +} + +EXPORT void +set_i64(uint32_t offset, int64_t v) +{ + wasm_linear_memory_set_int64(local_sandbox_context_cache.memory, offset, v); +} + +EXPORT void +add_function_to_table(uint32_t idx, uint32_t type_id, char *pointer) +{ + assert(idx < INDIRECT_TABLE_SIZE); + assert(local_sandbox_context_cache.module_indirect_table != NULL); - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; + /* TODO: atomic for multiple concurrent invocations? Issue #97 */ + if (local_sandbox_context_cache.module_indirect_table[idx].type_id == type_id + && local_sandbox_context_cache.module_indirect_table[idx].func_pointer == pointer) + return; - return *(int64_t *)address; + local_sandbox_context_cache.module_indirect_table[idx] = (struct indirect_table_entry){ + .type_id = type_id, .func_pointer = pointer + }; } -INLINE int32_t +/* + * Table handling functionality + * This was moved from compiletime in order to place the + * function in the callstack in GDB. It can be moved back + * to runtime/compiletime/memory/64bit_nix.c to remove the + * additional function call + */ +char * +get_function_from_table(uint32_t idx, uint32_t type_id) +{ +#ifdef LOG_FUNCTION_TABLE + fprintf(stderr, "get_function_from_table(idx: %u, type_id: %u)\n", idx, type_id); + fprintf(stderr, "indirect_table_size: %u\n", INDIRECT_TABLE_SIZE); +#endif + assert(idx < INDIRECT_TABLE_SIZE); + + struct indirect_table_entry f = local_sandbox_context_cache.module_indirect_table[idx]; +#ifdef LOG_FUNCTION_TABLE + fprintf(stderr, "assumed type: %u, type in table: %u\n", type_id, f.type_id); +#endif + // FIXME: Commented out function type check because of gocr + // assert(f.type_id == type_id); + + assert(f.func_pointer != NULL); + + return f.func_pointer; +} + + +EXPORT int32_t get_global_i32(uint32_t offset) { return get_i32(offset); } -INLINE int64_t +EXPORT void +set_global_i32(uint32_t offset, int32_t v) +{ + set_i32(offset, v); +} + +EXPORT int64_t get_global_i64(uint32_t offset) { return get_i64(offset); } -// Now setting routines -INLINE void -set_f32(uint32_t offset, float v) +EXPORT void +set_global_i64(uint32_t offset, int64_t v) { - assert(offset + sizeof(float) <= local_sandbox_context_cache.memory.size); + set_i64(offset, v); +} + +#define CHAR_BIT 8 - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; +// TODO: Throughout here we use `assert` for error conditions, which isn't optimal +// Instead we should use `unlikely` branches to a single trapping function (which should optimize better) +// The below functions are for implementing WASM instructions + +// ROTL and ROTR helper functions +INLINE uint32_t +rotl_u32(uint32_t n, uint32_t c_u32) +{ + // WASM requires a modulus here (usually a single bitwise op, but it means we need no assert) + unsigned int c = c_u32 % (CHAR_BIT * sizeof(n)); + const unsigned int mask = (CHAR_BIT * sizeof(n) - 1); // assumes width is a power of 2. - *(float *)address = v; + c &= mask; + return (n << c) | (n >> ((-c) & mask)); } -INLINE void -set_f64(uint32_t offset, double v) +INLINE uint32_t +rotr_u32(uint32_t n, uint32_t c_u32) { - assert(offset + sizeof(double) <= local_sandbox_context_cache.memory.size); + // WASM requires a modulus here (usually a single bitwise op, but it means we need no assert) + unsigned int c = c_u32 % (CHAR_BIT * sizeof(n)); + const unsigned int mask = (CHAR_BIT * sizeof(n) - 1); + + c &= mask; + return (n >> c) | (n << ((-c) & mask)); +} - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; +INLINE uint64_t +rotl_u64(uint64_t n, uint64_t c_u64) +{ + // WASM requires a modulus here (usually a single bitwise op, but it means we need no assert) + unsigned int c = c_u64 % (CHAR_BIT * sizeof(n)); + const unsigned int mask = (CHAR_BIT * sizeof(n) - 1); // assumes width is a power of 2. - *(double *)address = v; + c &= mask; + return (n << c) | (n >> ((-c) & mask)); } -INLINE void -set_i8(uint32_t offset, int8_t v) +INLINE uint64_t +rotr_u64(uint64_t n, uint64_t c_u64) { - assert(offset + sizeof(int8_t) <= local_sandbox_context_cache.memory.size); + // WASM requires a modulus here (usually a single bitwise op, but it means we need no assert) + unsigned int c = c_u64 % (CHAR_BIT * sizeof(n)); + const unsigned int mask = (CHAR_BIT * sizeof(n) - 1); - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; + c &= mask; + return (n >> c) | (n << ((-c) & mask)); +} - *(int8_t *)address = v; +// Now safe division and remainder +INLINE uint32_t +u32_div(uint32_t a, uint32_t b) +{ + assert(b); + return a / b; } -INLINE void -set_i16(uint32_t offset, int16_t v) +INLINE uint32_t +u32_rem(uint32_t a, uint32_t b) { - assert(offset + sizeof(int16_t) <= local_sandbox_context_cache.memory.size); + assert(b); + return a % b; +} - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; +INLINE int32_t +i32_div(int32_t a, int32_t b) +{ + assert(b && (a != INT32_MIN || b != -1)); + return a / b; +} - *(int16_t *)address = v; +INLINE int32_t +i32_rem(int32_t a, int32_t b) +{ + assert(b && (a != INT32_MIN || b != -1)); + return a % b; } -INLINE void -set_i32(uint32_t offset, int32_t v) +INLINE uint64_t +u64_div(uint64_t a, uint64_t b) { - assert(offset + sizeof(int32_t) <= local_sandbox_context_cache.memory.size); + assert(b); + return a / b; +} - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; +INLINE uint64_t +u64_rem(uint64_t a, uint64_t b) +{ + assert(b); + return a % b; +} - *(int32_t *)address = v; +INLINE int64_t +i64_div(int64_t a, int64_t b) +{ + assert(b && (a != INT64_MIN || b != -1)); + return a / b; } -INLINE void -set_i64(uint32_t offset, int64_t v) +INLINE int64_t +i64_rem(int64_t a, int64_t b) { - assert(offset + sizeof(int64_t) <= local_sandbox_context_cache.memory.size); + assert(b && (a != INT64_MIN || b != -1)); + return a % b; +} - char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start; - void *address = &mem_as_chars[offset]; +// float to integer conversion methods +// In C, float => int conversions always truncate +// If a int2float(int::min_value) <= float <= int2float(int::max_value), it must always be safe to truncate it +uint32_t +u32_trunc_f32(float f) +{ + assert(0 <= f && f <= (float)UINT32_MAX); + return (uint32_t)f; +} - *(int64_t *)address = v; +int32_t +i32_trunc_f32(float f) +{ + assert(INT32_MIN <= f && f <= (float)INT32_MAX); + return (int32_t)f; } -INLINE void -set_global_i32(uint32_t offset, int32_t v) +uint32_t +u32_trunc_f64(double f) { - set_i32(offset, v); + assert(0 <= f && f <= (double)UINT32_MAX); + return (uint32_t)f; } -INLINE void -set_global_i64(uint32_t offset, int64_t v) +int32_t +i32_trunc_f64(double f) { - set_i64(offset, v); + assert(INT32_MIN <= f && f <= (double)INT32_MAX); + return (int32_t)f; } -// Table handling functionality -// INLINE char * -// get_function_from_table(uint32_t idx, uint32_t type_id) -// { -// assert(idx < INDIRECT_TABLE_SIZE); +uint64_t +u64_trunc_f32(float f) +{ + assert(0 <= f && f <= (float)UINT64_MAX); + return (uint64_t)f; +} -// struct indirect_table_entry f = local_sandbox_context_cache.module_indirect_table[idx]; +int64_t +i64_trunc_f32(float f) +{ + assert(INT64_MIN <= f && f <= (float)INT64_MAX); + return (int64_t)f; +} -// // FIXME: Commented out function type check because of gocr -// // assert(f.type_id == type_id); +uint64_t +u64_trunc_f64(double f) +{ + assert(0 <= f && f <= (double)UINT64_MAX); + return (uint64_t)f; +} + +int64_t +i64_trunc_f64(double f) +{ + assert(INT64_MIN <= f && f <= (double)INT64_MAX); + return (int64_t)f; +} + +// Float => Float truncation functions +INLINE float +f32_trunc_f32(float f) +{ + return trunc(f); +} + +INLINE float +f32_min(float a, float b) +{ + return a < b ? a : b; +} -// assert(f.func_pointer); +INLINE float +f32_max(float a, float b) +{ + return a > b ? a : b; +} -// return f.func_pointer; -// } +INLINE float +f32_floor(float a) +{ + return floor(a); +} + +INLINE double +f64_min(double a, double b) +{ + return a < b ? a : b; +} + +INLINE double +f64_max(double a, double b) +{ + return a > b ? a : b; +} + +INLINE double +f64_floor(double a) +{ + return floor(a); +} diff --git a/runtime/include/current_sandbox.h b/runtime/include/current_sandbox.h index 29a99cce..67d7a49f 100644 --- a/runtime/include/current_sandbox.h +++ b/runtime/include/current_sandbox.h @@ -3,12 +3,11 @@ #include #include "sandbox_types.h" +#include "sandbox_context_cache.h" /* current sandbox that is active.. */ extern thread_local struct sandbox *worker_thread_current_sandbox; -extern thread_local struct sandbox_context_cache local_sandbox_context_cache; - void current_sandbox_start(void); /** diff --git a/runtime/include/debuglog.h b/runtime/include/debuglog.h index 0b877d0e..05c3207f 100644 --- a/runtime/include/debuglog.h +++ b/runtime/include/debuglog.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/runtime/include/module.h b/runtime/include/module.h index 5d774f05..d7a34ce0 100644 --- a/runtime/include/module.h +++ b/runtime/include/module.h @@ -12,6 +12,7 @@ #include "panic.h" #include "pool.h" #include "types.h" +#include "wasm_indirect_table.h" #define MODULE_DEFAULT_REQUEST_RESPONSE_SIZE (PAGE_SIZE) diff --git a/runtime/include/runtime.h b/runtime/include/runtime.h index 06b08dc4..ea15a5e3 100644 --- a/runtime/include/runtime.h +++ b/runtime/include/runtime.h @@ -47,32 +47,13 @@ extern uint32_t runtime_worker_threads_count; extern int * runtime_worker_threads_argument; extern uint64_t * runtime_worker_threads_deadline; - -/* memory also provides the table access functions */ -#define INDIRECT_TABLE_SIZE (1 << 10) - -struct indirect_table_entry { - uint32_t type_id; - void * func_pointer; -}; - -/* Cache of Frequently Accessed Members used to avoid pointer chasing */ -struct sandbox_context_cache { - struct wasm_linear_memory * memory; - struct indirect_table_entry *module_indirect_table; -}; - -extern thread_local struct sandbox_context_cache local_sandbox_context_cache; - extern void runtime_initialize(void); extern void runtime_set_pthread_prio(pthread_t thread, unsigned int nice); extern void runtime_set_resource_limits_to_max(void); /* External Symbols */ extern void alloc_linear_memory(void); -extern int expand_memory(void); INLINE char *get_function_from_table(uint32_t idx, uint32_t type_id); -INLINE char *get_memory_ptr_for_runtime(uint32_t offset, uint32_t bounds_check); extern void stub_init(int32_t offset); static inline char * diff --git a/runtime/include/sandbox_context_cache.h b/runtime/include/sandbox_context_cache.h new file mode 100644 index 00000000..9dadeb9a --- /dev/null +++ b/runtime/include/sandbox_context_cache.h @@ -0,0 +1,11 @@ +#pragma once +#include "wasm_linear_memory.h" +#include "wasm_indirect_table.h" + +/* Cache of Frequently Accessed Members used to avoid pointer chasing */ +struct sandbox_context_cache { + struct wasm_linear_memory * memory; + struct indirect_table_entry *module_indirect_table; +}; + +extern thread_local struct sandbox_context_cache local_sandbox_context_cache; diff --git a/runtime/include/sandbox_functions.h b/runtime/include/sandbox_functions.h index 0df43dde..f1218aa5 100644 --- a/runtime/include/sandbox_functions.h +++ b/runtime/include/sandbox_functions.h @@ -36,9 +36,12 @@ sandbox_close_http(struct sandbox *sandbox) static inline void sandbox_free_linear_memory(struct sandbox *sandbox) { + /* TODO Replace pool with parsec linked list */ + wasm_linear_memory_wipe(sandbox->memory); if (pool_free_object(sandbox->module->linear_memory_pool[worker_thread_idx], sandbox) < 0) { wasm_linear_memory_free(sandbox->memory); } + sandbox->memory = NULL; } /** diff --git a/runtime/include/sandbox_setup_arguments.h b/runtime/include/sandbox_setup_arguments.h index 5002f7ab..6778d217 100644 --- a/runtime/include/sandbox_setup_arguments.h +++ b/runtime/include/sandbox_setup_arguments.h @@ -15,12 +15,13 @@ sandbox_setup_arguments(struct sandbox *sandbox) { assert(sandbox != NULL); int32_t argument_count = 0; - /* whatever gregor has, to be able to pass arguments to a module! */ - sandbox->arguments_offset = local_sandbox_context_cache.memory->size; - assert(local_sandbox_context_cache.memory->data == sandbox->memory->data); - expand_memory(); - int32_t string_off = sandbox->arguments_offset; + /* Copy arguments into linear memory. It seems like malloc would clobber this, but I think this goes away in + * WASI, so not worth fixing*/ - stub_init(string_off); + sandbox->arguments_offset = wasm_linear_memory_get_size(sandbox->memory); + int rc = wasm_linear_memory_expand(sandbox->memory, WASM_PAGE_SIZE); + assert(rc == 0); + + stub_init(sandbox->arguments_offset); } diff --git a/runtime/include/wasm_indirect_table.h b/runtime/include/wasm_indirect_table.h new file mode 100644 index 00000000..82694ba4 --- /dev/null +++ b/runtime/include/wasm_indirect_table.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +/* memory also provides the table access functions */ +#define INDIRECT_TABLE_SIZE (1 << 10) + +struct indirect_table_entry { + uint32_t type_id; + void * func_pointer; +}; diff --git a/runtime/include/wasm_linear_memory.h b/runtime/include/wasm_linear_memory.h index 15b9e423..b8abc3e4 100644 --- a/runtime/include/wasm_linear_memory.h +++ b/runtime/include/wasm_linear_memory.h @@ -14,8 +14,9 @@ #define WASM_LINEAR_MEMORY_MAX (size_t) UINT32_MAX + 1 struct wasm_linear_memory { - size_t size; /* Initial Size in bytes */ - size_t max; /* Soft cap in bytes. Defaults to 4GB */ + size_t size; /* Initial Size in bytes */ + size_t capacity; /* Size backed by actual pages */ + size_t max; /* Soft cap in bytes. Defaults to 4GB */ uint8_t data[]; }; @@ -27,35 +28,30 @@ wasm_linear_memory_allocate(size_t initial, size_t max) assert(max > 0); assert(max <= (size_t)UINT32_MAX + 1); - char * error_message = NULL; - int rc = 0; - struct wasm_linear_memory *self = NULL; - - /* Allocate contiguous virtual addresses and map to fault */ + /* Allocate contiguous virtual addresses for struct, full linear memory, and guard page */ size_t size_to_alloc = sizeof(struct wasm_linear_memory) + WASM_LINEAR_MEMORY_MAX + /* guard page */ PAGE_SIZE; - - void *addr = mmap(NULL, size_to_alloc, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) { + void * temp = mmap(NULL, size_to_alloc, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (temp == MAP_FAILED) { debuglog("wasm_linear_memory_allocate - allocation failed, (size: %lu) %s\n", size_to_alloc, strerror(errno)); - return self; + return NULL; } + struct wasm_linear_memory *self = (struct wasm_linear_memory *)temp; /* Set the struct and initial pages to read / write */ size_t size_to_read_write = sizeof(struct wasm_linear_memory) + initial; - void *addr_rw = mmap(addr, size_to_read_write, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, - -1, 0); - if (addr_rw == MAP_FAILED) { + int rc = mprotect(self, size_to_read_write, PROT_READ | PROT_WRITE); + if (rc != 0) { perror("wasm_linear_memory_allocate - prot r/w failed"); - munmap(addr, size_to_alloc); - return self; + munmap(self, size_to_alloc); + assert(0); + return NULL; } - - self = (struct wasm_linear_memory *)addr_rw; - self->max = max; - self->size = initial; + self->size = initial; + self->capacity = initial; + self->max = max; return self; } @@ -66,51 +62,31 @@ wasm_linear_memory_free(struct wasm_linear_memory *self) munmap(self, size_to_free); } - -static inline int -wasm_linear_memory_expand(struct wasm_linear_memory *self, size_t size_to_expand) +static inline void +wasm_linear_memory_wipe(struct wasm_linear_memory *self) { - if (unlikely(self->size + size_to_expand >= self->max)) { - debuglog("wasm_linear_memory_expand - Out of Memory!.\n"); - return -1; - } - - void *temp = mmap(self, sizeof(struct wasm_linear_memory) + self->size + size_to_expand, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); - - if (temp == NULL) { - perror("wasm_linear_memory_expand mmap"); - return -1; - } - - /* Assumption: We are not actually resizing our wasm_linear_memory capacity. We are just adjusting the R/W rules - * within a preallocated wasm_linear_memory of size max */ - assert(self == temp); - - self->size += size_to_expand; - return 0; + memset(self->data, 0, self->size); } static inline int -wasm_linear_memory_resize(struct wasm_linear_memory *self, size_t target_size) +wasm_linear_memory_expand(struct wasm_linear_memory *self, size_t size_to_expand) { - if (unlikely(target_size >= self->max)) { - debuglog("wasm_linear_memory_expand - Out of Memory!. %lu out of %lu\n", self->size, self->max); + size_t target_size = self->size + size_to_expand; + if (unlikely(target_size > self->max)) { + fprintf(stderr, "wasm_linear_memory_expand - Out of Memory!. %lu out of %lu\n", self->size, self->max); return -1; } - void *temp = mmap(self, sizeof(struct wasm_linear_memory) + target_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (target_size > self->capacity) { + int rc = mprotect(self, sizeof(struct wasm_linear_memory) + target_size, PROT_READ | PROT_WRITE); + if (rc != 0) { + perror("wasm_linear_memory_expand mprotect"); + return -1; + } - if (temp == NULL) { - perror("wasm_linear_memory_resize mmap"); - return -1; + self->capacity = target_size; } - assert(self == temp); - - /* Assumption: We are not actually resizing our wasm_linear_memory capacity. We are just adjusting the R/W rules - * within a preallocated wasm_linear_memory of size max */ self->size = target_size; return 0; } @@ -140,6 +116,84 @@ wasm_linear_memory_get_char(struct wasm_linear_memory *self, uint32_t offset) return (char)self->data[offset]; } +/** + * Get an float from WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return float at the offset + */ +static inline float +wasm_linear_memory_get_float(struct wasm_linear_memory *self, uint32_t offset) +{ + assert(offset + sizeof(float) <= self->size); + return *(float *)&self->data[offset]; +} + +/** + * Get a double from WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return double at the offset + */ +static inline double +wasm_linear_memory_get_double(struct wasm_linear_memory *self, uint32_t offset) +{ + assert(offset + sizeof(double) <= self->size); + return *(double *)&self->data[offset]; +} + +/** + * Get a int8_t from WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int8_t at the offset + */ +static inline int8_t +wasm_linear_memory_get_int8(struct wasm_linear_memory *self, uint32_t offset) +{ + assert(offset + sizeof(int8_t) <= self->size); + return (int8_t)self->data[offset]; +} + +/** + * Get a int16_t from WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int16_t at the offset + */ +static inline int16_t +wasm_linear_memory_get_int16(struct wasm_linear_memory *self, uint32_t offset) +{ + assert(offset + sizeof(int16_t) <= self->size); + return *(int16_t *)&self->data[offset]; +} + +/** + * Get a int32_t from WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int32_t at the offset + */ +static inline int32_t +wasm_linear_memory_get_int32(struct wasm_linear_memory *self, uint32_t offset) +{ + assert(offset + sizeof(int32_t) <= self->size); + return *(int32_t *)&self->data[offset]; +} + +/** + * Get a int32_t from WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int32_t at the offset + */ +static inline int64_t +wasm_linear_memory_get_int64(struct wasm_linear_memory *self, uint32_t offset) +{ + assert(offset + sizeof(int64_t) <= self->size); + return *(int64_t *)&self->data[offset]; +} + +static inline uint32_t +wasm_linear_memory_get_page_count(struct wasm_linear_memory *self) +{ + return (uint32_t)(self->size / WASM_PAGE_SIZE); +} + /** * Get a null-terminated String from WebAssembly linear memory * @param offset an offset into the WebAssembly linear memory @@ -156,3 +210,95 @@ wasm_linear_memory_get_string(struct wasm_linear_memory *self, uint32_t offset, } return NULL; } + +/** + * Write a double to WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return double at the offset + */ +static inline void +wasm_linear_memory_set_double(struct wasm_linear_memory *self, uint32_t offset, double value) +{ + assert(offset + sizeof(double) <= self->size); + *(double *)&self->data[offset] = value; +} + +/** + * Write a float to WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return float at the offset + */ +static inline void +wasm_linear_memory_set_float(struct wasm_linear_memory *self, uint32_t offset, float value) +{ + assert(offset + sizeof(float) <= self->size); + *(float *)&self->data[offset] = value; +} + +/** + * Write a int8_t to WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int8_t at the offset + */ +static inline void +wasm_linear_memory_set_int8(struct wasm_linear_memory *self, uint32_t offset, int8_t value) +{ + assert(offset + sizeof(int8_t) <= self->size); + self->data[offset] = value; +} + +/** + * Write a int16_t to WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int16_t at the offset + */ +static inline void +wasm_linear_memory_set_int16(struct wasm_linear_memory *self, uint32_t offset, int16_t value) +{ + assert(offset + sizeof(int16_t) <= self->size); + *(int16_t *)&self->data[offset] = value; +} + +/** + * Write a int32_t to WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int32_t at the offset + */ +static inline void +wasm_linear_memory_set_int32(struct wasm_linear_memory *self, uint32_t offset, int32_t value) +{ + assert(offset + sizeof(int32_t) <= self->size); + *(int32_t *)&self->data[offset] = value; +} + +/** + * Write a int64_t to WebAssembly linear memory + * @param offset an offset into the WebAssembly linear memory + * @return int64_t at the offset + */ +static inline void +wasm_linear_memory_set_int64(struct wasm_linear_memory *self, uint64_t offset, int64_t value) +{ + assert(offset + sizeof(int64_t) <= self->size); + *(int32_t *)&self->data[offset] = value; +} + +static inline void +wasm_linear_memory_set_size(struct wasm_linear_memory *self, size_t size) +{ + self->size = size; +} + +static inline size_t +wasm_linear_memory_get_size(struct wasm_linear_memory *self) +{ + return self->size; +} + +static inline void +wasm_linear_memory_initialize_region(struct wasm_linear_memory *self, uint32_t offset, uint32_t region_size, + uint8_t region[region_size]) +{ + assert((size_t)offset + region_size <= self->size); + memcpy(&self->data[offset], region, region_size); +} diff --git a/runtime/src/libc/syscall.c b/runtime/src/libc/syscall.c index ea46bf53..04d00e6f 100644 --- a/runtime/src/libc/syscall.c +++ b/runtime/src/libc/syscall.c @@ -14,6 +14,7 @@ #include "scheduler.h" #include "sandbox_functions.h" #include "worker_thread.h" +#include "sandbox_context_cache.h" // What should we tell the child program its UID and GID are? #define UID 0xFF @@ -39,11 +40,12 @@ void stub_init(int32_t offset) { + struct sandbox *current_sandbox = current_sandbox_get(); // What program name will we put in the auxiliary vectors - char *program_name = current_sandbox_get()->module->name; + char *program_name = current_sandbox->module->name; // Copy the program name into WASM accessible memory int32_t program_name_offset = offset; - strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name); + strcpy(wasm_linear_memory_get_ptr_void(current_sandbox->memory, offset, sizeof(program_name)), program_name); offset += sizeof(program_name); // The construction of this is: @@ -69,7 +71,8 @@ stub_init(int32_t offset) 0, }; int32_t env_vec_offset = offset; - memcpy(get_memory_ptr_for_runtime(env_vec_offset, sizeof(env_vec)), env_vec, sizeof(env_vec)); + memcpy(wasm_linear_memory_get_ptr_void(current_sandbox->memory, env_vec_offset, sizeof(env_vec)), env_vec, + sizeof(env_vec)); module_initialize_libc(current_sandbox_get()->module, env_vec_offset, program_name_offset); } @@ -219,8 +222,8 @@ wasm_mmap(int32_t addr, int32_t len, int32_t prot, int32_t flags, int32_t fd, in assert(len % WASM_PAGE_SIZE == 0); - int32_t result = local_sandbox_context_cache.memory->size; - for (int i = 0; i < len / WASM_PAGE_SIZE; i++) { expand_memory(); } + int32_t result = wasm_linear_memory_get_size(local_sandbox_context_cache.memory); + if (wasm_linear_memory_expand(local_sandbox_context_cache.memory, len) == -1) { result = (uint32_t)-1; } return result; } @@ -318,19 +321,14 @@ wasm_mremap(int32_t offset, int32_t old_size, int32_t new_size, int32_t flags) // If at end of linear memory, just expand and return same address if (offset + old_size == local_sandbox_context_cache.memory->size) { - int32_t amount_to_expand = new_size - old_size; - int32_t pages_to_allocate = amount_to_expand / WASM_PAGE_SIZE; - if (amount_to_expand % WASM_PAGE_SIZE > 0) pages_to_allocate++; - for (int i = 0; i < pages_to_allocate; i++) expand_memory(); - + int32_t amount_to_expand = new_size - old_size; + wasm_linear_memory_expand(local_sandbox_context_cache.memory, amount_to_expand); return offset; } // Otherwise allocate at end of address space and copy - int32_t pages_to_allocate = new_size / WASM_PAGE_SIZE; - if (new_size % WASM_PAGE_SIZE > 0) pages_to_allocate++; int32_t new_offset = local_sandbox_context_cache.memory->size; - for (int i = 0; i < pages_to_allocate; i++) expand_memory(); + wasm_linear_memory_expand(local_sandbox_context_cache.memory, new_size); // Get pointer of old offset and pointer of new offset uint8_t *linear_mem = local_sandbox_context_cache.memory->data; diff --git a/runtime/src/memory/64bit_nix.c b/runtime/src/memory/64bit_nix.c deleted file mode 100644 index 5db231de..00000000 --- a/runtime/src/memory/64bit_nix.c +++ /dev/null @@ -1,103 +0,0 @@ -#include "current_sandbox.h" -#include "panic.h" -#include "runtime.h" -#include "sandbox_types.h" -#include "types.h" - -#include - -/** - * @brief Expand the linear memory of the active WebAssembly sandbox by a single page - * - * @return int - */ -int -expand_memory(void) -{ - struct sandbox *sandbox = current_sandbox_get(); - - assert(sandbox->state == SANDBOX_RUNNING_USER || sandbox->state == SANDBOX_RUNNING_SYS); - assert(local_sandbox_context_cache.memory->size % WASM_PAGE_SIZE == 0); - - /* Return -1 if we've hit the linear memory max */ - if (unlikely(wasm_linear_memory_expand(local_sandbox_context_cache.memory, WASM_PAGE_SIZE) == -1)) return -1; - -#ifdef LOG_SANDBOX_MEMORY_PROFILE - // Cache the runtime of the first N page allocations - if (likely(sandbox->timestamp_of.page_allocations_size < SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT)) { - sandbox->timestamp_of.page_allocations[sandbox->timestamp_of.page_allocations_size++] = - sandbox->duration_of_state.running - + (uint32_t)(__getcycles() - sandbox->timestamp_of.last_state_change); - } -#endif - - return 0; -} - -INLINE char * -get_memory_ptr_for_runtime(uint32_t offset, uint32_t size) -{ - return (char *)wasm_linear_memory_get_ptr_void(local_sandbox_context_cache.memory, offset, size); -} - -/** - * @brief Stub that implements the WebAssembly memory.grow instruction - * - * @param count number of pages to grow the WebAssembly linear memory by - * @return The previous size of the linear memory in pages or -1 if enough memory cannot be allocated - */ -int32_t -instruction_memory_grow(uint32_t count) -{ - int rc = local_sandbox_context_cache.memory->size / WASM_PAGE_SIZE; - - struct sandbox *sandbox = current_sandbox_get(); - - assert(sandbox->state == SANDBOX_RUNNING_USER || sandbox->state == SANDBOX_RUNNING_SYS); - assert(local_sandbox_context_cache.memory->size % WASM_PAGE_SIZE == 0); - - /* Return -1 if we've hit the linear memory max */ - if (unlikely(wasm_linear_memory_expand(local_sandbox_context_cache.memory, WASM_PAGE_SIZE * count) == -1)) - return -1; - -#ifdef LOG_SANDBOX_MEMORY_PROFILE - // Cache the runtime of the first N page allocations - for (int i = 0; i < count; i++) { - if (likely(sandbox->timestamp_of.page_allocations_size < SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT)) { - sandbox->timestamp_of.page_allocations[sandbox->timestamp_of.page_allocations_size++] = - sandbox->duration_of_state.running - + (uint32_t)(__getcycles() - sandbox->timestamp_of.last_state_change); - } - } -#endif - - return local_sandbox_context_cache.memory->size / WASM_PAGE_SIZE; -} - -/* - * Table handling functionality - * This was moved from compiletime in order to place the - * function in the callstack in GDB. It can be moved back - * to runtime/compiletime/memory/64bit_nix.c to remove the - * additional function call - */ -char * -get_function_from_table(uint32_t idx, uint32_t type_id) -{ -#ifdef LOG_FUNCTION_TABLE - fprintf(stderr, "get_function_from_table(idx: %u, type_id: %u)\n", idx, type_id); - fprintf(stderr, "indirect_table_size: %u\n", INDIRECT_TABLE_SIZE); -#endif - assert(idx < INDIRECT_TABLE_SIZE); - - struct indirect_table_entry f = local_sandbox_context_cache.module_indirect_table[idx]; -#ifdef LOG_FUNCTION_TABLE - fprintf(stderr, "assumed type: %u, type in table: %u\n", type_id, f.type_id); -#endif - // FIXME: Commented out function type check because of gocr - // assert(f.type_id == type_id); - - assert(f.func_pointer != NULL); - - return f.func_pointer; -} diff --git a/runtime/src/memory/common.c b/runtime/src/memory/common.c deleted file mode 100644 index 56253416..00000000 --- a/runtime/src/memory/common.c +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include - -#include "runtime.h" -#include "types.h" - -/* Region initialization helper function */ -EXPORT void -initialize_region(uint32_t offset, uint32_t region_size, uint8_t region[region_size]) -{ - assert((size_t)offset + region_size < local_sandbox_context_cache.memory->size); - - memcpy(get_memory_ptr_for_runtime(offset, region_size), region, region_size); -} - -void -add_function_to_table(uint32_t idx, uint32_t type_id, char *pointer) -{ - assert(idx < INDIRECT_TABLE_SIZE); - assert(local_sandbox_context_cache.module_indirect_table != NULL); - - /* TODO: atomic for multiple concurrent invocations? Issue #97 */ - if (local_sandbox_context_cache.module_indirect_table[idx].type_id == type_id - && local_sandbox_context_cache.module_indirect_table[idx].func_pointer == pointer) - return; - - local_sandbox_context_cache.module_indirect_table[idx] = (struct indirect_table_entry){ - .type_id = type_id, .func_pointer = pointer - }; -} - -/* If we are using runtime globals, we need to populate them */ -WEAK void -populate_globals() -{ - assert(0); /* FIXME: is this used in WASM as dynamic modules? Issue #105. */ -} diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index d062d1a8..68b3c3f7 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -24,21 +24,22 @@ sandbox_allocate_linear_memory(struct sandbox *self) char *error_message = NULL; - struct wasm_linear_memory *linear_memory = (struct wasm_linear_memory *)pool_allocate_object( - self->module->linear_memory_pool[worker_thread_idx]); - size_t initial = (size_t)self->module->abi.starting_pages * WASM_PAGE_SIZE; size_t max = (size_t)self->module->abi.max_pages * WASM_PAGE_SIZE; assert(initial <= (size_t)UINT32_MAX + 1); assert(max <= (size_t)UINT32_MAX + 1); - if (linear_memory == NULL) { - linear_memory = wasm_linear_memory_allocate(initial, max); - if (unlikely(linear_memory == NULL)) return -1; + self->memory = (struct wasm_linear_memory *)pool_allocate_object( + self->module->linear_memory_pool[worker_thread_idx]); + + if (self->memory == NULL) { + self->memory = wasm_linear_memory_allocate(initial, max); + if (unlikely(self->memory == NULL)) return -1; + } else { + wasm_linear_memory_set_size(self->memory, self->module->abi.starting_pages * WASM_PAGE_SIZE); } - self->memory = linear_memory; return 0; } @@ -196,7 +197,7 @@ sandbox_free(struct sandbox *sandbox) */ /* Linear Memory and Guard Page should already have been munmaped and set to NULL */ - assert(sandbox->memory->data == NULL); + assert(sandbox->memory == NULL); free(sandbox->request.base); free(sandbox->response.base);