diff --git a/runtime/include/local_completion_queue.h b/runtime/include/local_completion_queue.h index 74e3bfb8..ed869712 100644 --- a/runtime/include/local_completion_queue.h +++ b/runtime/include/local_completion_queue.h @@ -1,5 +1,6 @@ #pragma once +#include "ps_list.h" #include "sandbox_types.h" void local_completion_queue_add(struct sandbox *sandbox); diff --git a/runtime/include/module.h b/runtime/include/module.h index 94f3f5a0..ade54f34 100644 --- a/runtime/include/module.h +++ b/runtime/include/module.h @@ -26,7 +26,7 @@ INIT_POOL(wasm_stack, wasm_stack_free) struct module_pool { struct wasm_memory_pool memory; struct wasm_stack_pool stack; -} __attribute__((aligned(CACHE_PAD))); +} CACHE_PAD_ALIGNED; struct module { char *path; @@ -39,7 +39,7 @@ struct module { struct sledge_abi__wasm_table *indirect_table; struct module_pool *pools; -} __attribute__((aligned(CACHE_PAD))); +} CACHE_PAD_ALIGNED; /******************************** * Public Methods from module.c * diff --git a/runtime/include/sandbox_set_as_allocated.h b/runtime/include/sandbox_set_as_allocated.h index ae69734b..f77b2155 100644 --- a/runtime/include/sandbox_set_as_allocated.h +++ b/runtime/include/sandbox_set_as_allocated.h @@ -6,7 +6,6 @@ #include "arch/context.h" #include "current_sandbox.h" -#include "ps_list.h" #include "sandbox_state_history.h" #include "sandbox_state_transition.h" #include "sandbox_types.h" diff --git a/runtime/include/sandbox_set_as_initialized.h b/runtime/include/sandbox_set_as_initialized.h index 131df076..9ff7771a 100644 --- a/runtime/include/sandbox_set_as_initialized.h +++ b/runtime/include/sandbox_set_as_initialized.h @@ -6,7 +6,6 @@ #include "arch/context.h" #include "current_sandbox.h" -#include "ps_list.h" #include "sandbox_state_history.h" #include "sandbox_state_transition.h" #include "sandbox_types.h" diff --git a/runtime/include/sandbox_types.h b/runtime/include/sandbox_types.h index d2b20647..f9f2ca1a 100644 --- a/runtime/include/sandbox_types.h +++ b/runtime/include/sandbox_types.h @@ -32,11 +32,14 @@ struct sandbox_timestamps { }; struct sandbox { + /* used by ps_list's default name-based MACROS for the scheduling runqueue */ + /* Keep as first member of sandbox struct to ensure ps_list maintains alignment */ + struct ps_list list; + uint64_t id; sandbox_state_t state; struct sandbox_state_history state_history; - struct ps_list list; /* used by ps_list's default name-based MACROS for the scheduling runqueue */ /* Accounting Info */ struct route *route; diff --git a/runtime/include/types.h b/runtime/include/types.h index e61800e7..80709291 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -21,11 +21,13 @@ #define round_to_cache_pad(x) round_to_pow2((unsigned long)(x), CACHE_PAD) #define round_up_to_cache_pad(x) round_up_to_pow2((unsigned long)(x), CACHE_PAD) -#define EXPORT __attribute__((visibility("default"))) -#define IMPORT __attribute__((visibility("default"))) -#define INLINE __attribute__((always_inline)) -#define PAGE_ALIGNED __attribute__((aligned(PAGE_SIZE))) -#define WEAK __attribute__((weak)) +#define PAGE_ALIGNED __attribute__((aligned(PAGE_SIZE))) +#define CACHE_PAD_ALIGNED __attribute__((aligned(CACHE_PAD))) + +#define EXPORT __attribute__((visibility("default"))) +#define IMPORT __attribute__((visibility("default"))) +#define INLINE __attribute__((always_inline)) +#define WEAK __attribute__((weak)) #ifndef unlikely diff --git a/runtime/include/wasm_memory.h b/runtime/include/wasm_memory.h index 6d926052..8be98b98 100644 --- a/runtime/include/wasm_memory.h +++ b/runtime/include/wasm_memory.h @@ -56,7 +56,12 @@ static INLINE void wasm_memory_set_f64(struct wasm_memory *wasm_memory, uint3 static INLINE struct wasm_memory * wasm_memory_alloc(uint64_t initial, uint64_t max) { - struct wasm_memory *wasm_memory = aligned_alloc(4096, round_up_to_page(sizeof(struct wasm_memory))); + size_t alignment = (size_t)PAGE_SIZE; + size_t size_to_alloc = (size_t)round_up_to_page(sizeof(struct wasm_memory)); + + assert(size_to_alloc % alignment == 0); + + struct wasm_memory *wasm_memory = aligned_alloc(alignment, size_to_alloc); if (wasm_memory == NULL) return wasm_memory; int rc = wasm_memory_init(wasm_memory, initial, max); diff --git a/runtime/include/wasm_stack.h b/runtime/include/wasm_stack.h index db9d1dbb..18211725 100644 --- a/runtime/include/wasm_stack.h +++ b/runtime/include/wasm_stack.h @@ -5,6 +5,7 @@ #include #include +#include "ps_list.h" #include "types.h" /** diff --git a/runtime/src/local_completion_queue.c b/runtime/src/local_completion_queue.c index 588e2e8d..1c6df189 100644 --- a/runtime/src/local_completion_queue.c +++ b/runtime/src/local_completion_queue.c @@ -3,7 +3,8 @@ #include "local_completion_queue.h" #include "sandbox_functions.h" -thread_local static struct ps_list_head local_completion_queue __attribute__((aligned(16))); +/* Must be the same alignment as sandbox structs because of how the ps_list macros work */ +thread_local static struct ps_list_head local_completion_queue PAGE_ALIGNED; void @@ -39,8 +40,6 @@ local_completion_queue_add(struct sandbox *sandbox) void local_completion_queue_free() { - if (local_completion_queue_is_empty()) return; - struct sandbox *sandbox_iterator = NULL; struct sandbox *buffer = NULL; diff --git a/runtime/src/module.c b/runtime/src/module.c index 92f9dabe..093106f6 100644 --- a/runtime/src/module.c +++ b/runtime/src/module.c @@ -111,6 +111,8 @@ module_alloc(char *path) size_t alignment = (size_t)CACHE_PAD; size_t size_to_alloc = (size_t)round_to_cache_pad(sizeof(struct module)); + assert(size_to_alloc % alignment == 0); + struct module *module = aligned_alloc(alignment, size_to_alloc); if (!module) { fprintf(stderr, "Failed to allocate module: %s\n", strerror(errno)); diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index b488d087..c9028aaa 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -180,11 +180,16 @@ struct sandbox * sandbox_alloc(struct module *module, struct http_session *session, struct route *route, struct tenant *tenant, uint64_t admissions_estimate) { - struct sandbox *sandbox = NULL; - size_t page_aligned_sandbox_size = round_up_to_page(sizeof(struct sandbox)); - sandbox = aligned_alloc(PAGE_SIZE, page_aligned_sandbox_size); - memset(sandbox, 0, page_aligned_sandbox_size); + size_t alignment = (size_t)PAGE_SIZE; + size_t size_to_alloc = (size_t)round_up_to_page(sizeof(struct module)); + + assert(size_to_alloc % alignment == 0); + + struct sandbox *sandbox = NULL; + sandbox = aligned_alloc(alignment, size_to_alloc); + if (unlikely(sandbox == NULL)) return NULL; + memset(sandbox, 0, size_to_alloc); sandbox_set_as_allocated(sandbox); sandbox_init(sandbox, module, session, route, tenant, admissions_estimate);