chore: update tinyekf

.vscode/c_cpp_properties.json

@@ -6,6 +6,7 @@
 			"includePath": [
 				"/usr/include/",
 				"${workspaceFolder}/runtime/include/",
+				"${workspaceFolder}/runtime/include/common",
 				"${workspaceFolder}/runtime/thirdparty/ck/include/",
 				"${workspaceFolder}/runtime/thirdparty/http-parser/",
 				"${workspaceFolder}/runtime/thirdparty/jsmn/"

.vscode/settings.json

@@ -100,8 +100,12 @@
 		"sandbox_set_as_running_user.h": "c",
 		"scheduler.h": "c",
 		"sandbox_set_as_returned.h": "c",
+		"wasm_memory.h": "c",
 		"software_interrupt_counts.h": "c",
-		"sandbox_set_as_running_sys.h": "c"
+		"sandbox_set_as_running_sys.h": "c",
+		"compiletime.h": "c",
+		"wasm_store.h": "c",
+		"wasm_table.h": "c"
 	},
 	"files.exclude": {
 		"**/.git": true,

runtime/Makefile

@@ -105,8 +105,6 @@ INCLUDES += -Iinclude/ -Ithirdparty/dist/include/
 CFILES += src/*.c
 CFILES += src/arch/${ARCH}/*.c
 CFILES += src/libc/*.c
-CFILES += src/memory/common.c
-CFILES += src/memory/64bit_nix.c
 CFILES += thirdparty/dist/lib/http_parser.o
 
 # Configuring Jasmine

runtime/compiletime/README.md

@@ -0,0 +1,6 @@
+A WebAssembly module instance is statically linked with the backing functions implementing the wasm32 ABI, yielding a *.so file that SLEdge can execute. This ensures that the instance is able to aggressively inline and optimize this code.
+
+They are broken into instruction types as on https://webassembly.github.io/spec/core/exec/instructions.html. They depend on common headers for the WebAssembly types located in the WebAssembly instance struct. These are located in runtime/include/common.
+
+The stubs correspond to awsm/src/codegen/runtime_stubs.rs
+

runtime/compiletime/memory/64bit_nix.c

@@ -1,182 +0,0 @@
-#include <assert.h>
-#include "types.h"
-
-uint32_t
-instruction_memory_size()
-{
-	return local_sandbox_context_cache.memory.size / WASM_PAGE_SIZE;
-}
-
-// All of these are pretty generic
-INLINE float
-get_f32(uint32_t offset)
-{
-	assert(offset + sizeof(float) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	return *(float *)address;
-}
-
-INLINE double
-get_f64(uint32_t offset)
-{
-	assert(offset + sizeof(double) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	return *(double *)address;
-}
-
-INLINE int8_t
-get_i8(uint32_t offset)
-{
-	assert(offset + sizeof(int8_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	return *(int8_t *)address;
-}
-
-INLINE int16_t
-get_i16(uint32_t offset)
-{
-	assert(offset + sizeof(int16_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	return *(int16_t *)address;
-}
-
-INLINE int32_t
-get_i32(uint32_t offset)
-{
-	assert(offset + sizeof(int32_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	return *(int32_t *)address;
-}
-
-INLINE int64_t
-get_i64(uint32_t offset)
-{
-	assert(offset + sizeof(int64_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	return *(int64_t *)address;
-}
-
-INLINE int32_t
-get_global_i32(uint32_t offset)
-{
-	return get_i32(offset);
-}
-
-INLINE int64_t
-get_global_i64(uint32_t offset)
-{
-	return get_i64(offset);
-}
-
-// Now setting routines
-INLINE void
-set_f32(uint32_t offset, float v)
-{
-	assert(offset + sizeof(float) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	*(float *)address = v;
-}
-
-INLINE void
-set_f64(uint32_t offset, double v)
-{
-	assert(offset + sizeof(double) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	*(double *)address = v;
-}
-
-INLINE void
-set_i8(uint32_t offset, int8_t v)
-{
-	assert(offset + sizeof(int8_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	*(int8_t *)address = v;
-}
-
-INLINE void
-set_i16(uint32_t offset, int16_t v)
-{
-	assert(offset + sizeof(int16_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	*(int16_t *)address = v;
-}
-
-INLINE void
-set_i32(uint32_t offset, int32_t v)
-{
-	assert(offset + sizeof(int32_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	*(int32_t *)address = v;
-}
-
-INLINE void
-set_i64(uint32_t offset, int64_t v)
-{
-	assert(offset + sizeof(int64_t) <= local_sandbox_context_cache.memory.size);
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	void *address      = &mem_as_chars[offset];
-
-	*(int64_t *)address = v;
-}
-
-INLINE void
-set_global_i32(uint32_t offset, int32_t v)
-{
-	set_i32(offset, v);
-}
-
-INLINE void
-set_global_i64(uint32_t offset, int64_t v)
-{
-	set_i64(offset, v);
-}
-
-// Table handling functionality
-// INLINE char *
-// get_function_from_table(uint32_t idx, uint32_t type_id)
-// {
-// 	assert(idx < INDIRECT_TABLE_SIZE);
-
-// 	struct indirect_table_entry f = local_sandbox_context_cache.module_indirect_table[idx];
-
-// 	// FIXME: Commented out function type check because of gocr
-// 	// assert(f.type_id == type_id);
-
-// 	assert(f.func_pointer);
-
-// 	return f.func_pointer;
-// }

runtime/compiletime/memory_instructions.c

@@ -0,0 +1,144 @@
+#include <assert.h>
+#include <assert.h>
+#include <math.h>
+
+#include "wasm_store.h"
+
+extern thread_local struct wasm_module_instance current_wasm_module_instance;
+
+INLINE void
+initialize_region(uint32_t offset, uint32_t region_size, uint8_t region[region_size])
+{
+	wasm_memory_initialize_region(current_wasm_module_instance.memory, offset, region_size, region);
+}
+
+INLINE uint32_t
+instruction_memory_size()
+{
+	return wasm_memory_get_page_count(current_wasm_module_instance.memory);
+}
+
+/**
+ * @brief Stub that implements the WebAssembly memory.grow instruction
+ *
+ * @param count number of pages to grow the WebAssembly linear memory by
+ * @return The previous size of the linear memory in pages or -1 if enough memory cannot be allocated
+ */
+INLINE int32_t
+instruction_memory_grow(uint32_t count)
+{
+	int old_page_count = current_wasm_module_instance.memory->size / WASM_PAGE_SIZE;
+
+	/* Return -1 if we've hit the linear memory max */
+	int rc = wasm_memory_expand(current_wasm_module_instance.memory, WASM_PAGE_SIZE * count);
+	if (unlikely(rc == -1)) return -1;
+
+#ifdef LOG_SANDBOX_MEMORY_PROFILE
+	// Cache the runtime of the first N page allocations
+	for (int i = 0; i < count; i++) {
+		if (likely(sandbox->timestamp_of.page_allocations_size < SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT)) {
+			sandbox->timestamp_of.page_allocations[sandbox->timestamp_of.page_allocations_size++] =
+			  sandbox->duration_of_state.running
+			  + (uint32_t)(__getcycles() - sandbox->timestamp_of.last_state_change);
+		}
+	}
+#endif
+
+	return rc;
+}
+
+INLINE float
+get_f32(uint32_t offset)
+{
+	return wasm_memory_get_float(current_wasm_module_instance.memory, offset);
+}
+
+INLINE void
+set_f32(uint32_t offset, float v)
+{
+	wasm_memory_set_float(current_wasm_module_instance.memory, offset, v);
+}
+
+INLINE double
+get_f64(uint32_t offset)
+{
+	return wasm_memory_get_double(current_wasm_module_instance.memory, offset);
+}
+
+INLINE void
+set_f64(uint32_t offset, double v)
+{
+	wasm_memory_set_double(current_wasm_module_instance.memory, offset, v);
+}
+
+INLINE int8_t
+get_i8(uint32_t offset)
+{
+	return wasm_memory_get_int8(current_wasm_module_instance.memory, offset);
+}
+
+INLINE void
+set_i8(uint32_t offset, int8_t v)
+{
+	wasm_memory_set_int8(current_wasm_module_instance.memory, offset, v);
+}
+
+INLINE int16_t
+get_i16(uint32_t offset)
+{
+	return wasm_memory_get_int16(current_wasm_module_instance.memory, offset);
+}
+
+INLINE void
+set_i16(uint32_t offset, int16_t v)
+{
+	wasm_memory_set_int16(current_wasm_module_instance.memory, offset, v);
+}
+
+INLINE int32_t
+get_i32(uint32_t offset)
+{
+	return wasm_memory_get_int32(current_wasm_module_instance.memory, offset);
+}
+
+INLINE void
+set_i32(uint32_t offset, int32_t v)
+{
+	wasm_memory_set_int32(current_wasm_module_instance.memory, offset, v);
+}
+
+INLINE int64_t
+get_i64(uint32_t offset)
+{
+	return wasm_memory_get_int64(current_wasm_module_instance.memory, offset);
+}
+
+INLINE void
+set_i64(uint32_t offset, int64_t v)
+{
+	wasm_memory_set_int64(current_wasm_module_instance.memory, offset, v);
+}
+
+INLINE int32_t
+get_global_i32(uint32_t offset)
+{
+	return get_i32(offset);
+}
+
+INLINE void
+set_global_i32(uint32_t offset, int32_t v)
+{
+	set_i32(offset, v);
+}
+
+INLINE int64_t
+get_global_i64(uint32_t offset)
+{
+	return get_i64(offset);
+}
+
+INLINE void
+set_global_i64(uint32_t offset, int64_t v)
+{
+	set_i64(offset, v);
+}

runtime/compiletime/numeric_instructions.c

@@ -1,6 +1,7 @@
 #include <assert.h>
 #include <math.h>
-#include <types.h>
+
+#include "types.h"
 
 #define CHAR_BIT 8
 

runtime/compiletime/table_instructions.c

@@ -0,0 +1,23 @@
+
+#include "types.h"
+#include "wasm_store.h"
+
+/* This file contains the stub functions that the aWsm compiler expects
+ * This corresponds to awsm/src/codegen/runtime_stubs.rs
+ * This should be linked with the *.bc file generated by aWsm in order to compile a module as a *.so
+ */
+
+extern thread_local struct wasm_module_instance current_wasm_module_instance;
+
+INLINE void
+add_function_to_table(uint32_t idx, uint32_t type_id, char *pointer)
+{
+	wasm_table_set(current_wasm_module_instance.table, idx, type_id, pointer);
+}
+
+/* char * is used as a generic pointer to a function pointer */
+INLINE char *
+get_function_from_table(uint32_t idx, uint32_t type_id)
+{
+	return wasm_table_get(current_wasm_module_instance.table, idx, type_id);
+}

runtime/include/awsm_abi.h

@@ -3,19 +3,8 @@
 #include <assert.h>
 #include <dlfcn.h>
 
-/* Wasm initialization functions generated by the compiler */
+#include "common/wasm_types.h"
-#define AWSM_ABI_INITIALIZE_GLOBALS "populate_globals"
+#include "debuglog.h"
-#define AWSM_ABI_INITIALIZE_MEMORY  "populate_memory"
-#define AWSM_ABI_INITIALIZE_TABLE   "populate_table"
-#define AWSM_ABI_INITIALIZE_LIBC    "wasmf___init_libc"
-#define AWSM_ABI_ENTRYPOINT         "wasmf_main"
-
-/* functions in the module to lookup and call per sandbox. */
-typedef int32_t (*awsm_abi_entrypoint_fn_t)(int32_t a, int32_t b);
-typedef void (*awsm_abi_init_globals_fn_t)(void);
-typedef void (*awsm_abi_init_mem_fn_t)(void);
-typedef void (*awsm_abi_init_tbl_fn_t)(void);
-typedef void (*awsm_abi_init_libc_fn_t)(int32_t, int32_t);
 
 struct awsm_abi {
 	void *                     handle;
@@ -24,6 +13,8 @@ struct awsm_abi {
 	awsm_abi_init_tbl_fn_t     initialize_tables;
 	awsm_abi_init_libc_fn_t    initialize_libc;
 	awsm_abi_entrypoint_fn_t   entrypoint;
+	uint32_t                   starting_pages;
+	uint32_t                   max_pages;
 };
 
 /* Initializes the ABI object using the *.so file at path */
@@ -76,6 +67,26 @@ awsm_abi_init(struct awsm_abi *abi, char *path)
 		goto dl_error;
 	}
 
+	abi->starting_pages = *(uint32_t *)dlsym(abi->handle, AWSM_ABI_STARTING_PAGES);
+	if (abi->starting_pages == 0) {
+		fprintf(stderr, "Failed to resolve symbol %s in %s with error: %s\n", AWSM_ABI_STARTING_PAGES, path,
+		        dlerror());
+		goto dl_error;
+	}
+
+	abi->max_pages = *(uint32_t *)dlsym(abi->handle, AWSM_ABI_MAX_PAGES);
+	if (abi->max_pages == 0) {
+		/* This seems to not always be present. I assume this is only there if the source module explicitly
+		 * specified this */
+		abi->max_pages = WASM_MEMORY_PAGES_MAX;
+		debuglog("max_pages symbols not defined. Defaulting to MAX defined by spec.\n")
+
+		// TODO: We need to prove that this actually can get generated by awsm
+		// fprintf(stderr, "Failed to resolve symbol %s in %s with error: %s\n", AWSM_ABI_MAX_PAGES, path,
+		//         dlerror());
+		//   goto dl_error;
+	}
+
 done:
 	return rc;
 dl_error:

runtime/include/client_socket.h

@@ -3,6 +3,7 @@
 #include <assert.h>
 #include <arpa/inet.h>
 #include <errno.h>
+#include <stdbool.h>
 #include <string.h>
 #include <sys/socket.h>
 #include <unistd.h>
@@ -32,53 +33,54 @@ client_socket_close(int client_socket, struct sockaddr *client_address)
 	}
 }
 
+typedef void (*void_cb)(void);
+
 /**
  * Rejects request due to admission control or error
  * @param client_socket - the client we are rejecting
- * @param status_code - either 503 or 400
+ * @param buffer - buffer to write to socket
+ * @param on_eagain - cb to execute when client socket returns EAGAIN. If NULL, error out
+ * @returns 0
  */
 static inline int
-client_socket_send(int client_socket, int status_code)
+client_socket_send(int client_socket, const char *buffer, size_t buffer_len, void_cb on_eagain)
 {
-	const char *response;
+	int rc;
-	int         rc;
-	switch (status_code) {
-	case 503:
-		response = HTTP_RESPONSE_503_SERVICE_UNAVAILABLE;
-		http_total_increment_5XX();
-		break;
-	case 413:
-		response = HTTP_RESPONSE_413_PAYLOAD_TOO_LARGE;
-		http_total_increment_4XX();
-		break;
-	case 400:
-		response = HTTP_RESPONSE_400_BAD_REQUEST;
-		http_total_increment_4XX();
-		break;
-	default:
-		panic("%d is not a valid status code\n", status_code);
-	}
 
-	size_t total_sent = 0;
+	size_t cursor = 0;
-	size_t to_send    = strlen(response);
 
-	while (total_sent < to_send) {
+	while (cursor < buffer_len) {
-		ssize_t sent = write(client_socket, &response[total_sent], to_send - total_sent);
+		ssize_t sent = write(client_socket, &buffer[cursor], buffer_len - cursor);
 		if (sent < 0) {
-			if (errno == EAGAIN) { debuglog("Unexpectedly blocking on write of %s\n", response); }
+			if (errno == EAGAIN) {
-
+				if (on_eagain) {
-			debuglog("Error with %s\n", strerror(errno));
+					on_eagain();
-
+				} else {
-			goto send_err;
+					rc = -1;
+					goto done;
+				}
+			} else {
+				debuglog("Error sending to client: %s", strerror(errno));
+				rc = -1;
+				goto done;
+			}
 		}
-		total_sent += sent;
+		cursor += sent;
 	};
 
 	rc = 0;
 done:
 	return rc;
-send_err:
+}
-	debuglog("Error sending to client: %s", strerror(errno));
+
-	rc = -1;
+/**
-	goto done;
+ * Rejects request due to admission control or error
+ * @param client_socket - the client we are rejecting
+ * @param buffer - buffer to write to socket
+ * @returns 0
+ */
+static inline int
+client_socket_send_oneshot(int client_socket, const char *buffer, size_t buffer_len)
+{
+	return client_socket_send(client_socket, buffer, buffer_len, NULL);
 }

runtime/include/common/README.md

@@ -0,0 +1,3 @@
+This folder includes headers used by the compiletime and runtime components.
+
+Changes in here break our ABI and require *.so module to be recompiled.

runtime/include/common/types.h

@@ -4,8 +4,6 @@
 #include <stdio.h>
 #include <threads.h>
 
-#include "wasm_types.h"
-
 /* For this family of macros, do NOT pass zero as the pow2 */
 #define round_to_pow2(x, pow2)    (((unsigned long)(x)) & (~((pow2)-1)))
 #define round_up_to_pow2(x, pow2) (round_to_pow2(((unsigned long)(x)) + (pow2)-1, (pow2)))
@@ -20,18 +18,10 @@
 #define PAGE_SIZE    (unsigned long)(1 << 12)
 #define WEAK         __attribute__((weak))
 
-/* memory also provides the table access functions */
+#ifndef unlikely
-#define INDIRECT_TABLE_SIZE (1 << 10)
+#define unlikely(x) __builtin_expect(!!(x), 0)
-
+#endif
-struct indirect_table_entry {
-	uint32_t type_id;
-	void *   func_pointer;
-};
-
-/* Cache of Frequently Accessed Members used to avoid pointer chasing */
-struct sandbox_context_cache {
-	struct wasm_memory           memory;
-	struct indirect_table_entry *module_indirect_table;
-};
 
-extern thread_local struct sandbox_context_cache local_sandbox_context_cache;
+#ifndef likely
+#define likely(x) __builtin_expect(!!(x), 1)
+#endif

runtime/include/common/wasm_memory.h

@@ -0,0 +1,303 @@
+#pragma once
+
+#include <assert.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+
+#include "types.h" /* PAGE_SIZE */
+#include "wasm_types.h"
+
+#define wasm_memory_MAX (size_t) UINT32_MAX + 1
+
+struct wasm_memory {
+	size_t  size;     /* Initial Size in bytes */
+	size_t  capacity; /* Size backed by actual pages */
+	size_t  max;      /* Soft cap in bytes. Defaults to 4GB */
+	uint8_t data[];
+};
+
+static inline struct wasm_memory *
+wasm_memory_allocate(size_t initial, size_t max)
+{
+	assert(initial > 0);
+	assert(initial <= (size_t)UINT32_MAX + 1);
+	assert(max > 0);
+	assert(max <= (size_t)UINT32_MAX + 1);
+
+	/* Allocate contiguous virtual addresses for struct, full linear memory, and guard page */
+	size_t size_to_alloc = sizeof(struct wasm_memory) + wasm_memory_MAX + /* guard page */ PAGE_SIZE;
+	void * temp          = mmap(NULL, size_to_alloc, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (temp == MAP_FAILED) {
+		fprintf(stderr, "wasm_memory_allocate - allocation failed, (size: %lu) %s\n", size_to_alloc,
+		        strerror(errno));
+		return NULL;
+	}
+	struct wasm_memory *self = (struct wasm_memory *)temp;
+
+	/* Set the struct and initial pages to read / write */
+	size_t size_to_read_write = sizeof(struct wasm_memory) + initial;
+
+	int rc = mprotect(self, size_to_read_write, PROT_READ | PROT_WRITE);
+	if (rc != 0) {
+		perror("wasm_memory_allocate - prot r/w failed");
+		munmap(self, size_to_alloc);
+		assert(0);
+		return NULL;
+	}
+
+	self->size     = initial;
+	self->capacity = initial;
+	self->max      = max;
+	return self;
+}
+
+static inline void
+wasm_memory_free(struct wasm_memory *self)
+{
+	size_t size_to_free = sizeof(struct wasm_memory) + self->max + /* guard page */ PAGE_SIZE;
+	munmap(self, size_to_free);
+}
+
+static inline void
+wasm_memory_wipe(struct wasm_memory *self)
+{
+	memset(self->data, 0, self->size);
+}
+
+static inline int
+wasm_memory_expand(struct wasm_memory *self, size_t size_to_expand)
+{
+	size_t target_size = self->size + size_to_expand;
+	if (unlikely(target_size > self->max)) {
+		fprintf(stderr, "wasm_memory_expand - Out of Memory!. %lu out of %lu\n", self->size, self->max);
+		return -1;
+	}
+
+	if (target_size > self->capacity) {
+		int rc = mprotect(self, sizeof(struct wasm_memory) + target_size, PROT_READ | PROT_WRITE);
+		if (rc != 0) {
+			perror("wasm_memory_expand mprotect");
+			return -1;
+		}
+
+		self->capacity = target_size;
+	}
+
+	self->size = target_size;
+	return 0;
+}
+
+/**
+ * Translates WASM offsets into runtime VM pointers
+ * @param offset an offset into the WebAssembly linear memory
+ * @param bounds_check the size of the thing we are pointing to
+ * @return void pointer to something in WebAssembly linear memory
+ */
+static inline void *
+wasm_memory_get_ptr_void(struct wasm_memory *self, uint32_t offset, uint32_t size)
+{
+	assert(offset + size <= self->size);
+	return (void *)&self->data[offset];
+}
+
+/**
+ * Get an ASCII character from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return char at the offset
+ */
+static inline char
+wasm_memory_get_char(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(char) <= self->size);
+	return (char)self->data[offset];
+}
+
+/**
+ * Get an float from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return float at the offset
+ */
+static inline float
+wasm_memory_get_float(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(float) <= self->size);
+	return *(float *)&self->data[offset];
+}
+
+/**
+ * Get a double from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return double at the offset
+ */
+static inline double
+wasm_memory_get_double(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(double) <= self->size);
+	return *(double *)&self->data[offset];
+}
+
+/**
+ * Get a int8_t from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int8_t at the offset
+ */
+static inline int8_t
+wasm_memory_get_int8(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(int8_t) <= self->size);
+	return (int8_t)self->data[offset];
+}
+
+/**
+ * Get a int16_t from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int16_t at the offset
+ */
+static inline int16_t
+wasm_memory_get_int16(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(int16_t) <= self->size);
+	return *(int16_t *)&self->data[offset];
+}
+
+/**
+ * Get a int32_t from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int32_t at the offset
+ */
+static inline int32_t
+wasm_memory_get_int32(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(int32_t) <= self->size);
+	return *(int32_t *)&self->data[offset];
+}
+
+/**
+ * Get a int32_t from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int32_t at the offset
+ */
+static inline int64_t
+wasm_memory_get_int64(struct wasm_memory *self, uint32_t offset)
+{
+	assert(offset + sizeof(int64_t) <= self->size);
+	return *(int64_t *)&self->data[offset];
+}
+
+static inline uint32_t
+wasm_memory_get_page_count(struct wasm_memory *self)
+{
+	return (uint32_t)(self->size / WASM_PAGE_SIZE);
+}
+
+/**
+ * Get a null-terminated String from WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @param size the maximum expected length in characters
+ * @return pointer to the string or NULL if max_length is reached without finding null-terminator
+ */
+static inline char *
+wasm_memory_get_string(struct wasm_memory *self, uint32_t offset, uint32_t size)
+{
+	assert(offset + (sizeof(char) * size) <= self->size);
+
+	for (uint32_t i = 0; i < size; i++) {
+		if (self->data[offset + i] == '\0') return (char *)&self->data[offset];
+	}
+	return NULL;
+}
+
+/**
+ * Write a double to WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return double at the offset
+ */
+static inline void
+wasm_memory_set_double(struct wasm_memory *self, uint32_t offset, double value)
+{
+	assert(offset + sizeof(double) <= self->size);
+	*(double *)&self->data[offset] = value;
+}
+
+/**
+ * Write a float to WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return float at the offset
+ */
+static inline void
+wasm_memory_set_float(struct wasm_memory *self, uint32_t offset, float value)
+{
+	assert(offset + sizeof(float) <= self->size);
+	*(float *)&self->data[offset] = value;
+}
+
+/**
+ * Write a int8_t to WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int8_t at the offset
+ */
+static inline void
+wasm_memory_set_int8(struct wasm_memory *self, uint32_t offset, int8_t value)
+{
+	assert(offset + sizeof(int8_t) <= self->size);
+	self->data[offset] = value;
+}
+
+/**
+ * Write a int16_t to WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int16_t at the offset
+ */
+static inline void
+wasm_memory_set_int16(struct wasm_memory *self, uint32_t offset, int16_t value)
+{
+	assert(offset + sizeof(int16_t) <= self->size);
+	*(int16_t *)&self->data[offset] = value;
+}
+
+/**
+ * Write a int32_t to WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int32_t at the offset
+ */
+static inline void
+wasm_memory_set_int32(struct wasm_memory *self, uint32_t offset, int32_t value)
+{
+	assert(offset + sizeof(int32_t) <= self->size);
+	*(int32_t *)&self->data[offset] = value;
+}
+
+/**
+ * Write a int64_t to WebAssembly linear memory
+ * @param offset an offset into the WebAssembly linear memory
+ * @return int64_t at the offset
+ */
+static inline void
+wasm_memory_set_int64(struct wasm_memory *self, uint64_t offset, int64_t value)
+{
+	assert(offset + sizeof(int64_t) <= self->size);
+	*(int32_t *)&self->data[offset] = value;
+}
+
+static inline void
+wasm_memory_set_size(struct wasm_memory *self, size_t size)
+{
+	self->size = size;
+}
+
+static inline size_t
+wasm_memory_get_size(struct wasm_memory *self)
+{
+	return self->size;
+}
+
+static inline void
+wasm_memory_initialize_region(struct wasm_memory *self, uint32_t offset, uint32_t region_size,
+                              uint8_t region[region_size])
+{
+	assert((size_t)offset + region_size <= self->size);
+	memcpy(&self->data[offset], region, region_size);
+}

runtime/include/common/wasm_store.h

@@ -0,0 +1,14 @@
+#pragma once
+#include "wasm_memory.h"
+#include "wasm_table.h"
+
+/* This structure is the runtime representation of the unique state of a module instance
+ * Currently this is not spec-compliant, as it only supports a single table and a single memory and it excludes many
+ * entities https://webassembly.github.io/spec/core/exec/runtime.html#module-instances
+ */
+struct wasm_module_instance {
+	struct wasm_memory *memory;
+	struct wasm_table * table;
+};
+
+extern thread_local struct wasm_module_instance current_wasm_module_instance;

runtime/include/common/wasm_table.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/* memory also provides the table access functions */
+#define INDIRECT_TABLE_SIZE (1 << 10)
+
+struct wasm_table_entry {
+	uint32_t type_id;
+	void *   func_pointer;
+};
+
+struct wasm_table {
+	uint32_t                length;
+	uint32_t                capacity;
+	struct wasm_table_entry data[];
+};
+
+static inline struct wasm_table *
+wasm_table_allocate(size_t capacity)
+{
+	struct wasm_table *self = (struct wasm_table *)malloc(sizeof(struct wasm_table)
+	                                                      + capacity * sizeof(struct wasm_table_entry));
+
+	self->capacity = capacity;
+	self->length   = 0;
+
+	return self;
+}
+
+static inline void
+wasm_table_free(struct wasm_table *self)
+{
+	assert(self != NULL);
+	free(self);
+}
+
+static inline char *
+wasm_table_get(struct wasm_table *self, uint32_t idx, uint32_t type_id)
+{
+	assert(self != NULL);
+	assert(idx < self->capacity);
+
+	struct wasm_table_entry f = self->data[idx];
+	// FIXME: Commented out function type check because of gocr
+	// assert(f.type_id == type_id);
+
+	assert(f.func_pointer != NULL);
+
+	return f.func_pointer;
+}
+
+static inline void
+wasm_table_set(struct wasm_table *self, uint32_t idx, uint32_t type_id, char *pointer)
+{
+	assert(self != NULL);
+	assert(idx < self->capacity);
+
+	/* TODO: atomic for multiple concurrent invocations? Issue #97 */
+	if (self->data[idx].type_id == type_id && self->data[idx].func_pointer == pointer) return;
+
+	self->data[idx] = (struct wasm_table_entry){ .type_id = type_id, .func_pointer = pointer };
+}

runtime/include/common/wasm_types.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#define WASM_PAGE_SIZE        (1024 * 64) /* 64KB */
+#define WASM_MEMORY_PAGES_MAX (1 << 16)   /* 32,768 Pages ~4GB */
+#define WASM_STACK_SIZE       (1 << 19)   /* 512KB */
+
+/* Wasm initialization functions generated by the compiler */
+#define AWSM_ABI_INITIALIZE_GLOBALS "populate_globals"
+#define AWSM_ABI_INITIALIZE_MEMORY  "populate_memory"
+#define AWSM_ABI_INITIALIZE_TABLE   "populate_table"
+#define AWSM_ABI_INITIALIZE_LIBC    "wasmf___init_libc"
+#define AWSM_ABI_ENTRYPOINT         "wasmf_main"
+
+#define AWSM_ABI_STARTING_PAGES "starting_pages"
+#define AWSM_ABI_MAX_PAGES      "max_pages"
+
+/* functions in the module to lookup and call per sandbox. */
+typedef int32_t (*awsm_abi_entrypoint_fn_t)(int32_t a, int32_t b);
+typedef void (*awsm_abi_init_globals_fn_t)(void);
+typedef void (*awsm_abi_init_mem_fn_t)(void);
+typedef void (*awsm_abi_init_tbl_fn_t)(void);
+typedef void (*awsm_abi_init_libc_fn_t)(int32_t, int32_t);

runtime/include/current_sandbox.h

@@ -3,12 +3,11 @@
 #include <threads.h>
 
 #include "sandbox_types.h"
+#include "common/wasm_store.h"
 
 /* current sandbox that is active.. */
 extern thread_local struct sandbox *worker_thread_current_sandbox;
 
-extern thread_local struct sandbox_context_cache local_sandbox_context_cache;
-
 void current_sandbox_start(void);
 
 /**
@@ -30,25 +29,40 @@ current_sandbox_set(struct sandbox *sandbox)
 {
 	/* Unpack hierarchy to avoid pointer chasing */
 	if (sandbox == NULL) {
-		local_sandbox_context_cache = (struct sandbox_context_cache){
+		current_wasm_module_instance = (struct wasm_module_instance){
-			.memory = {
+			.memory = NULL,
-				.start          = NULL,
+			.table  = NULL,
-				.size           = 0,
-				.max           = 0,
-			},
-			.module_indirect_table = NULL,
 		};
 		worker_thread_current_sandbox                      = NULL;
 		runtime_worker_threads_deadline[worker_thread_idx] = UINT64_MAX;
 	} else {
-		local_sandbox_context_cache = (struct sandbox_context_cache){
+		current_wasm_module_instance = (struct wasm_module_instance){
-			.memory                = sandbox->memory,
+			.memory = sandbox->memory,
-			.module_indirect_table = sandbox->module->indirect_table,
+			.table  = sandbox->module->indirect_table,
 		};
 		worker_thread_current_sandbox                      = sandbox;
 		runtime_worker_threads_deadline[worker_thread_idx] = sandbox->absolute_deadline;
 	}
 }
 
-
 extern void current_sandbox_sleep();
+
+static inline void *
+current_sandbox_get_ptr_void(uint32_t offset, uint32_t bounds_check)
+{
+	assert(current_wasm_module_instance.memory != NULL);
+	return wasm_memory_get_ptr_void(current_wasm_module_instance.memory, offset, bounds_check);
+}
+
+static inline char
+current_sandbox_get_char(uint32_t offset)
+{
+	assert(current_wasm_module_instance.memory != NULL);
+	return wasm_memory_get_char(current_wasm_module_instance.memory, offset);
+}
+
+static inline char *
+current_sandbox_get_string(uint32_t offset, uint32_t size)
+{
+	return wasm_memory_get_string(current_wasm_module_instance.memory, offset, size);
+}

runtime/include/current_sandbox_send_response.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <assert.h>
+#include <errno.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "current_sandbox.h"
+#include "http.h"
+#include "http_total.h"
+#include "likely.h"
+#include "sandbox_types.h"
+#include "scheduler.h"
+#include "panic.h"
+
+/**
+ * Sends Response Back to Client
+ * @return RC. -1 on Failure
+ */
+static inline int
+current_sandbox_send_response()
+{
+	struct sandbox *sandbox = current_sandbox_get();
+	assert(sandbox != NULL);
+
+	int     rc;
+	ssize_t sent = 0;
+
+	/* Determine values to template into our HTTP response */
+	ssize_t     response_body_size  = sandbox->response.length;
+	char *      module_content_type = sandbox->module->response_content_type;
+	const char *content_type        = strlen(module_content_type) > 0 ? module_content_type : "text/plain";
+
+	/* Capture Timekeeping data for end-to-end latency */
+	uint64_t end_time   = __getcycles();
+	sandbox->total_time = end_time - sandbox->timestamp_of.request_arrival;
+
+	/* Generate and send HTTP Response Headers */
+	ssize_t response_header_size = http_response_200_size(content_type, response_body_size);
+	char    response_header_buffer[response_header_size + 1];
+	rc = http_header_200_build(response_header_buffer, content_type, response_body_size);
+	if (rc <= 0) {
+		perror("sprintf");
+		goto err;
+	}
+	client_socket_send(sandbox->client_socket_descriptor, response_header_buffer, response_header_size,
+	                   current_sandbox_sleep);
+
+	/* Send HTTP Response Body */
+	client_socket_send(sandbox->client_socket_descriptor, sandbox->response.base, response_body_size,
+	                   current_sandbox_sleep);
+
+	http_total_increment_2xx();
+	rc = 0;
+
+done:
+	return rc;
+err:
+	debuglog("Error sending to client: %s", strerror(errno));
+	rc = -1;
+	goto done;
+}

runtime/include/debuglog.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <pthread.h>
+#include <sched.h>
 #include <stdint.h>
 #include <stdio.h>
 

runtime/include/http.h

@@ -2,6 +2,9 @@
 
 #include <string.h>
 
+#include "http_total.h"
+#include "panic.h"
+
 #define HTTP_MAX_HEADER_COUNT        16
 #define HTTP_MAX_HEADER_LENGTH       32
 #define HTTP_MAX_HEADER_VALUE_LENGTH 64
@@ -30,46 +33,73 @@
 	"Server: SLEdge\r\n"       \
 	"Connection: close\r\n"    \
 	"Content-Type: %s\r\n"     \
-	"Content-Length: %s\r\n"   \
+	"Content-Length: %lu\r\n"  \
 	"\r\n"
 
 /* The sum of format specifier characters in the template above */
-#define HTTP_RESPONSE_200_TEMPLATE_FORMAT_SPECIFIER_LENGTH 4
+#define HTTP_RESPONSE_200_TEMPLATE_FORMAT_SPECIFIER_LENGTH 5
 
 /**
  * Calculates the number of bytes of the HTTP response containing the passed header values
  * @return total size in bytes
  */
 static inline size_t
-http_response_200_size(char *content_type, char *content_length)
+http_response_200_size(const char *content_type, ssize_t content_length)
 {
 	size_t size = 0;
 	size += strlen(HTTP_RESPONSE_200_TEMPLATE) - HTTP_RESPONSE_200_TEMPLATE_FORMAT_SPECIFIER_LENGTH;
 	size += strlen(content_type);
-	size += strlen(content_length);
+
+	while (content_length > 0) {
+		content_length /= 10;
+		size++;
+	}
+
 	return size;
 }
 
-/**
- * Writes the HTTP response header to the destination. This is assumed to have been sized
- * using the value returned by http_response_200_size. We have to use an intermediate buffer
- * in order to truncate off the null terminator
- * @return 0 on success, -1 otherwise
- */
 static inline int
-http_response_200(char *destination, char *content_type, char *content_length)
+http_header_200_build(char *buffer, const char *content_type, ssize_t content_length)
+{
+	return sprintf(buffer, HTTP_RESPONSE_200_TEMPLATE, content_type, content_length);
+}
+
+static inline const char *
+http_header_build(int status_code)
+{
+	const char *response;
+	int         rc;
+	switch (status_code) {
+	case 503:
+		response = HTTP_RESPONSE_503_SERVICE_UNAVAILABLE;
+		http_total_increment_5XX();
+		break;
+	case 413:
+		response = HTTP_RESPONSE_413_PAYLOAD_TOO_LARGE;
+		http_total_increment_4XX();
+		break;
+	case 400:
+		response = HTTP_RESPONSE_400_BAD_REQUEST;
+		http_total_increment_4XX();
+		break;
+	default:
+		panic("%d is not a valid status code\n", status_code);
+	}
+
+	return response;
+}
+
+static inline int
+http_header_len(int status_code)
 {
-	size_t response_size = http_response_200_size(content_type, content_length);
+	switch (status_code) {
-	char   buffer[response_size + 1];
+	case 503:
-	int    rc = 0;
+		return strlen(HTTP_RESPONSE_503_SERVICE_UNAVAILABLE);
-	rc        = sprintf(buffer, HTTP_RESPONSE_200_TEMPLATE, content_type, content_length);
+	case 413:
-	if (rc <= 0) goto err;
+		return strlen(HTTP_RESPONSE_413_PAYLOAD_TOO_LARGE);
-	memmove(destination, buffer, response_size);
+	case 400:
-	rc = 0;
+		return strlen(HTTP_RESPONSE_400_BAD_REQUEST);
-
+	default:
-done:
+		panic("%d is not a valid status code\n", status_code);
-	return rc;
+	}
-err:
-	rc = -1;
-	goto done;
 }

runtime/include/module.h

@@ -10,7 +10,10 @@
 #include "awsm_abi.h"
 #include "http.h"
 #include "panic.h"
-#include "types.h"
+#include "pool.h"
+
+#include "common/types.h"
+#include "common/wasm_table.h"
 
 #define MODULE_DEFAULT_REQUEST_RESPONSE_SIZE (PAGE_SIZE)
 
@@ -53,8 +56,9 @@ struct module {
 	/* Handle and ABI Symbols for *.so file */
 	struct awsm_abi abi;
 
-	_Atomic uint32_t            reference_count; /* ref count how many instances exist here. */
+	_Atomic uint32_t   reference_count; /* ref count how many instances exist here. */
-	struct indirect_table_entry indirect_table[INDIRECT_TABLE_SIZE];
+	struct wasm_table *indirect_table;
+	struct pool **     linear_memory_pool;
 };
 
 /*************************

runtime/include/pool.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+#include "generic_thread.h"
+#include "lock.h"
+
+struct pool {
+	bool    use_lock;
+	lock_t  lock;
+	ssize_t top;
+	size_t  capacity;
+	void *  buffer[];
+};
+
+static inline bool
+pool_is_empty(struct pool *self)
+{
+	return self->top == -1;
+}
+
+static inline bool
+pool_is_full(struct pool *self)
+{
+	return self->top + 1 == self->capacity;
+}
+
+static inline void *
+pool_allocate_object_nolock(struct pool *self)
+{
+	assert(self != NULL);
+	assert(!self->use_lock || LOCK_IS_LOCKED(&self->lock));
+
+	void *result = NULL;
+
+	if (pool_is_empty(self)) return result;
+
+	result = self->buffer[self->top--];
+	return result;
+}
+
+static inline void *
+pool_allocate_object(struct pool *self)
+{
+	assert(self != NULL);
+	assert(self->use_lock);
+
+	void *result = NULL;
+
+	if (pool_is_empty(self)) return result;
+
+	LOCK_LOCK(&self->lock);
+	if (pool_is_empty(self)) {
+		LOCK_UNLOCK(&self->lock);
+		return result;
+	}
+
+	result = self->buffer[self->top--];
+	assert(result != NULL);
+	LOCK_UNLOCK(&self->lock);
+	return result;
+}
+
+static inline int
+pool_free_object_nolock(struct pool *self, void *obj)
+{
+	assert(self != NULL);
+	assert(obj != NULL);
+	assert(!self->use_lock || LOCK_IS_LOCKED(&self->lock));
+
+	if (pool_is_full(self)) return -1;
+
+	self->buffer[++self->top] = obj;
+	return 0;
+}
+
+static inline int
+pool_free_object(struct pool *self, void *obj)
+{
+	assert(self != NULL);
+	assert(obj != NULL);
+	assert(self->use_lock);
+
+	if (pool_is_full(self)) return -1;
+
+	LOCK_LOCK(&self->lock);
+	if (pool_is_full(self)) {
+		LOCK_UNLOCK(&self->lock);
+		return -1;
+	}
+
+	self->buffer[++self->top] = obj;
+	LOCK_UNLOCK(&self->lock);
+	return 0;
+}
+
+static inline struct pool *
+pool_init(size_t capacity, bool use_lock)
+{
+	struct pool *self = (struct pool *)calloc(1, sizeof(struct pool) + capacity * sizeof(void *));
+	self->top         = -1;
+	self->capacity    = capacity;
+	self->use_lock    = use_lock;
+
+	if (use_lock) LOCK_INIT(&self->lock);
+
+	return self;
+}
+
+static inline void
+pool_free(struct pool *self)
+{
+	while (!pool_is_empty(self)) free(pool_allocate_object(self));
+
+	free(self);
+}

runtime/include/runtime.h

@@ -6,7 +6,7 @@
 #include <stdbool.h>
 
 #include "likely.h"
-#include "types.h"
+#include "common/types.h"
 
 /**
  * Optimizing compilers and modern CPUs reorder instructions however it sees fit. This means that the resulting
@@ -50,13 +50,6 @@ extern void runtime_initialize(void);
 extern void runtime_set_pthread_prio(pthread_t thread, unsigned int nice);
 extern void runtime_set_resource_limits_to_max(void);
 
-/* External Symbols */
-extern void  alloc_linear_memory(void);
-extern int   expand_memory(void);
-INLINE char *get_function_from_table(uint32_t idx, uint32_t type_id);
-INLINE char *get_memory_ptr_for_runtime(uint32_t offset, uint32_t bounds_check);
-extern void  stub_init(int32_t offset);
-
 static inline char *
 runtime_print_sigalrm_handler(enum RUNTIME_SIGALRM_HANDLER variant)
 {

runtime/include/sandbox_functions.h

@@ -6,7 +6,9 @@
 
 #include "client_socket.h"
 #include "panic.h"
+#include "pool.h"
 #include "sandbox_request.h"
+#include "common/wasm_memory.h"
 
 /***************************
  * Public API              *
@@ -35,9 +37,12 @@ sandbox_close_http(struct sandbox *sandbox)
 static inline void
 sandbox_free_linear_memory(struct sandbox *sandbox)
 {
-	int rc = munmap(sandbox->memory.start, sandbox->memory.max + PAGE_SIZE);
+	/* TODO Replace pool with parsec linked list */
-	if (rc == -1) panic("sandbox_free_linear_memory - munmap failed\n");
+	wasm_memory_wipe(sandbox->memory);
-	sandbox->memory.start = NULL;
+	if (pool_free_object(sandbox->module->linear_memory_pool[worker_thread_idx], sandbox) < 0) {
+		wasm_memory_free(sandbox->memory);
+	}
+	sandbox->memory = NULL;
 }
 
 /**

runtime/include/sandbox_perf_log.h

@@ -36,16 +36,15 @@ sandbox_perf_log_print_entry(struct sandbox *sandbox)
 	 * becomes more intelligent, then peak linear memory size needs to be tracked
 	 * seperately from current linear memory size.
 	 */
-	fprintf(sandbox_perf_log, "%lu,%s,%d,%s,%lu,%lu,%lu,,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%u,%u\n",
+	fprintf(sandbox_perf_log, "%lu,%s,%d,%s,%lu,%lu,%lu,,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu,%u,%lu\n",
 	        sandbox->id, sandbox->module->name, sandbox->module->port, sandbox_state_stringify(sandbox->state),
 	        sandbox->module->relative_deadline, sandbox->total_time, queued_duration,
-	        sandbox->duration_of_state[SANDBOX_UNINITIALIZED], sandbox->duration_of_state[SANDBOX_ALLOCATED],
+	        sandbox->duration_of_state[SANDBOX_UNINITIALIZED], sandbox->duration_of_state[SANDBOX_INITIALIZED],
-	        sandbox->duration_of_state[SANDBOX_INITIALIZED], sandbox->duration_of_state[SANDBOX_RUNNABLE],
+	        sandbox->duration_of_state[SANDBOX_RUNNABLE], sandbox->duration_of_state[SANDBOX_INTERRUPTED],
-	        sandbox->duration_of_state[SANDBOX_INTERRUPTED], sandbox->duration_of_state[SANDBOX_PREEMPTED],
+	        sandbox->duration_of_state[SANDBOX_PREEMPTED], sandbox->duration_of_state[SANDBOX_RUNNING_SYS],
-	        sandbox->duration_of_state[SANDBOX_RUNNING_SYS], sandbox->duration_of_state[SANDBOX_RUNNING_USER],
+	        sandbox->duration_of_state[SANDBOX_RUNNING_USER], sandbox->duration_of_state[SANDBOX_ASLEEP],
-	        sandbox->duration_of_state[SANDBOX_ASLEEP], sandbox->duration_of_state[SANDBOX_RETURNED],
+	        sandbox->duration_of_state[SANDBOX_RETURNED], sandbox->duration_of_state[SANDBOX_COMPLETE],
-	        sandbox->duration_of_state[SANDBOX_COMPLETE], sandbox->duration_of_state[SANDBOX_ERROR],
+	        sandbox->duration_of_state[SANDBOX_ERROR], runtime_processor_speed_MHz, sandbox->memory->size);
-	        runtime_processor_speed_MHz, sandbox->memory.size);
 }
 
 static inline void

runtime/include/sandbox_send_response.h

@@ -1,74 +0,0 @@
-#pragma once
-
-#include <assert.h>
-#include <errno.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "current_sandbox.h"
-#include "http.h"
-#include "http_total.h"
-#include "likely.h"
-#include "sandbox_types.h"
-#include "scheduler.h"
-#include "panic.h"
-
-/**
- * Sends Response Back to Client
- * @return RC. -1 on Failure
- */
-static inline int
-sandbox_send_response(struct sandbox *sandbox)
-{
-	assert(sandbox != NULL);
-	/* Assumption: The HTTP Request Buffer immediately precedes the HTTP Response Buffer,
-	 * meaning that when we prepend, we are overwritting the tail of the HTTP request buffer */
-	assert(sandbox->request.base + sandbox->module->max_request_size == sandbox->response.base);
-
-	int rc;
-
-	/* Determine values to template into our HTTP response */
-	ssize_t response_body_size = sandbox->response.length;
-	char    content_length[20] = { 0 };
-	sprintf(content_length, "%zu", response_body_size);
-	char *module_content_type = sandbox->module->response_content_type;
-	char *content_type        = strlen(module_content_type) > 0 ? module_content_type : "text/plain";
-
-	/* Prepend HTTP Response Headers */
-	size_t response_header_size = http_response_200_size(content_type, content_length);
-	char * response_header      = sandbox->response.base - response_header_size;
-	rc                          = http_response_200(response_header, content_type, content_length);
-	if (rc < 0) goto err;
-
-	/* Capture Timekeeping data for end-to-end latency */
-	uint64_t end_time   = __getcycles();
-	sandbox->total_time = end_time - sandbox->timestamp_of.request_arrival;
-
-	/* Send HTTP Response */
-	int    sent          = 0;
-	size_t response_size = response_header_size + response_body_size;
-	while (sent < response_size) {
-		rc = write(sandbox->client_socket_descriptor, response_header, response_size - sent);
-		if (rc < 0) {
-			if (errno == EAGAIN)
-				current_sandbox_sleep();
-			else {
-				perror("write");
-				goto err;
-			}
-		}
-		sent += rc;
-	}
-
-	http_total_increment_2xx();
-	rc = 0;
-
-done:
-	return rc;
-err:
-	rc = -1;
-	goto done;
-}

runtime/include/sandbox_set_as_initialized.h

@@ -19,39 +19,47 @@
  * @param allocation_timestamp timestamp of allocation
  */
 static inline void
-sandbox_set_as_initialized(struct sandbox *sandbox, struct sandbox_request *sandbox_request,
+sandbox_set_as_initialized(struct sandbox *self, struct sandbox_request *sandbox_request, uint64_t allocation_timestamp)
-                           uint64_t allocation_timestamp)
 {
-	assert(sandbox);
+	assert(self);
-	assert(sandbox->state == SANDBOX_ALLOCATED);
+	assert(self->state == SANDBOX_UNINITIALIZED);
-	assert(sandbox_request != NULL);
+	assert(self != NULL);
 	assert(allocation_timestamp > 0);
-	sandbox->state = SANDBOX_INITIALIZED;
+	uint64_t now = __getcycles();
-	uint64_t now   = __getcycles();
+
-
+	self->id    = sandbox_request->id;
-	/* Copy State from Sandbox Request */
+	self->state = SANDBOX_INITIALIZED;
-	sandbox->id                           = sandbox_request->id;
+
-	sandbox->absolute_deadline            = sandbox_request->absolute_deadline;
+#ifdef LOG_STATE_CHANGES
-	sandbox->admissions_estimate          = sandbox_request->admissions_estimate;
+	sandbox_state_history_append(self, SANDBOX_UNINITIALIZED);
-	sandbox->client_socket_descriptor     = sandbox_request->socket_descriptor;
+	memset(&sandbox->state_history, 0, SANDBOX_STATE_HISTORY_CAPACITY * sizeof(sandbox_state_t));
-	sandbox->timestamp_of.request_arrival = sandbox_request->request_arrival_timestamp;
+	sandbox->state_history_count                           = 0;
-	/* Copy the socket descriptor and address of the client invocation */
+	sandbox->state_history[sandbox->state_history_count++] = SANDBOX_UNINITIALIZED;
-	memcpy(&sandbox->client_address, &sandbox_request->socket_address, sizeof(struct sockaddr));
+#endif
 
+#ifdef LOG_SANDBOX_MEMORY_PROFILE
+	self->timestamp_of.page_allocations_size = 0;
+#endif
 
-	/* Initialize the sandbox's context, stack, and instruction pointer */
+	ps_list_init_d(self);
-	/* stack.start points to the bottom of the usable stack, so add stack_size to get to top */
+
-	arch_context_init(&sandbox->ctxt, (reg_t)current_sandbox_start,
+	self->absolute_deadline            = sandbox_request->absolute_deadline;
-	                  (reg_t)sandbox->stack.start + sandbox->stack.size);
+	self->admissions_estimate          = sandbox_request->admissions_estimate;
+	self->client_socket_descriptor     = sandbox_request->socket_descriptor;
+	self->timestamp_of.request_arrival = sandbox_request->request_arrival_timestamp;
+	/* Copy the socket descriptor and address of the client invocation */
+	memcpy(&self->client_address, &sandbox_request->socket_address, sizeof(struct sockaddr));
 
-	/* Initialize Parsec control structures */
+	/* Allocations require the module to be set */
-	ps_list_init_d(sandbox);
+	self->module = sandbox_request->module;
+	module_acquire(self->module);
 
+	memset(&self->duration_of_state, 0, SANDBOX_STATE_COUNT * sizeof(uint64_t));
 
 	/* State Change Bookkeeping */
-	sandbox->duration_of_state[SANDBOX_ALLOCATED] = now - allocation_timestamp;
+	self->duration_of_state[SANDBOX_UNINITIALIZED] = now - allocation_timestamp;
-	sandbox->timestamp_of.allocation              = allocation_timestamp;
+	self->timestamp_of.allocation                  = allocation_timestamp;
-	sandbox->timestamp_of.last_state_change       = allocation_timestamp;
+	self->timestamp_of.last_state_change           = allocation_timestamp;
-	sandbox_state_history_append(sandbox, SANDBOX_INITIALIZED);
+	sandbox_state_history_append(self, SANDBOX_INITIALIZED);
 	runtime_sandbox_total_increment(SANDBOX_INITIALIZED);
 }

runtime/include/sandbox_setup_arguments.h

@@ -7,6 +7,8 @@
 
 #include "sandbox_types.h"
 
+extern void stub_init(int32_t offset);
+
 /**
  * Takes the arguments from the sandbox struct and writes them into the WebAssembly linear memory
  */
@@ -15,12 +17,13 @@ sandbox_setup_arguments(struct sandbox *sandbox)
 {
 	assert(sandbox != NULL);
 	int32_t argument_count = 0;
-	/* whatever gregor has, to be able to pass arguments to a module! */
-	sandbox->arguments_offset = local_sandbox_context_cache.memory.size;
-	assert(local_sandbox_context_cache.memory.start == sandbox->memory.start);
-	expand_memory();
 
-	int32_t string_off = sandbox->arguments_offset;
+	/* Copy arguments into linear memory. It seems like malloc would clobber this, but I think this goes away in
+	 * WASI, so not worth fixing*/
+
+	sandbox->arguments_offset = wasm_memory_get_size(sandbox->memory);
+	int rc                    = wasm_memory_expand(sandbox->memory, WASM_PAGE_SIZE);
+	assert(rc == 0);
 
-	stub_init(string_off);
+	stub_init(sandbox->arguments_offset);
 }

runtime/include/sandbox_state.h

@@ -9,7 +9,6 @@
 typedef enum
 {
 	SANDBOX_UNINITIALIZED = 0, /* Assumption: mmap zeros out structure */
-	SANDBOX_ALLOCATED,
 	SANDBOX_INITIALIZED,
 	SANDBOX_RUNNABLE,
 	SANDBOX_PREEMPTED,

runtime/include/sandbox_state_history.h

@@ -3,12 +3,25 @@
 #include "sandbox_state.h"
 #include "sandbox_types.h"
 
+/* TODO: Define a struct and make the first argument a struct sandbox_state_history */
+
 static inline void
-sandbox_state_history_append(struct sandbox *sandbox, sandbox_state_t state)
+sandbox_state_history_append(struct sandbox *self, sandbox_state_t state)
 {
 #ifdef LOG_STATE_CHANGES
-	if (likely(sandbox->state_history_count < SANDBOX_STATE_HISTORY_CAPACITY)) {
+	if (likely(self->state_history_count < SANDBOX_STATE_HISTORY_CAPACITY)) {
-		sandbox->state_history[sandbox->state_history_count++] = state;
+		sandbox->state_history[self->state_history_count++] = state;
 	}
 #endif
 }
+
+static inline void
+sandbox_state_history_init(struct sandbox *self)
+{
+#ifdef LOG_STATE_CHANGES
+	sandbox_state_history_append(self, SANDBOX_UNINITIALIZED);
+	memset(&sandbox->state_history, 0, SANDBOX_STATE_HISTORY_CAPACITY * sizeof(sandbox_state_t));
+	sandbox->state_history_count                           = 0;
+	sandbox->state_history[sandbox->state_history_count++] = SANDBOX_UNINITIALIZED;
+#endif
+}

runtime/include/sandbox_types.h

@@ -12,7 +12,8 @@
 #include "module.h"
 #include "ps_list.h"
 #include "sandbox_state.h"
-#include "wasm_types.h"
+#include "common/wasm_types.h"
+#include "common/wasm_memory.h"
 
 #ifdef LOG_SANDBOX_MEMORY_PROFILE
 #define SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT 1024
@@ -87,7 +88,6 @@ struct sandbox {
 	int                   client_socket_descriptor;
 	http_parser           http_parser;
 	struct http_request   http_request;
-	ssize_t               http_request_length; /* TODO: Get rid of me */
 	struct sandbox_buffer request;
 	struct sandbox_buffer response;
 
@@ -97,7 +97,7 @@ struct sandbox {
 	/* WebAssembly Instance State  */
 	struct arch_context  ctxt;
 	struct sandbox_stack stack;
-	struct wasm_memory   memory;
+	struct wasm_memory * memory;
 
 	/* Scheduling and Temporal State */
 	struct sandbox_timestamps timestamp_of;

runtime/include/scheduler.h

@@ -98,7 +98,7 @@ scheduler_edf_get_next()
 done:
 	return local_runqueue_get_next();
 err_allocate:
-	client_socket_send(request->socket_descriptor, 503);
+	client_socket_send_oneshot(request->socket_descriptor, http_header_build(503), http_header_len(503));
 	client_socket_close(request->socket_descriptor, &request->socket_address);
 	free(request);
 	goto done;
@@ -129,7 +129,7 @@ scheduler_fifo_get_next()
 done:
 	return sandbox;
 err_allocate:
-	client_socket_send(sandbox_request->socket_descriptor, 503);
+	client_socket_send_oneshot(sandbox_request->socket_descriptor, http_header_build(503), http_header_len(503));
 	client_socket_close(sandbox_request->socket_descriptor, &sandbox->client_address);
 	free(sandbox_request);
 err:

runtime/include/scheduler_execute_epoll_loop.h

@@ -63,7 +63,8 @@ scheduler_execute_epoll_loop(void)
 				case SANDBOX_ERROR:
 					panic("Expected to have closed socket");
 				default:
-					client_socket_send(sandbox->client_socket_descriptor, 503);
+					client_socket_send_oneshot(sandbox->client_socket_descriptor,
+					                           http_header_build(503), http_header_len(503));
 					sandbox_close_http(sandbox);
 					sandbox_set_as_error(sandbox, sandbox->state);
 				}

runtime/include/wasm_types.h

@@ -1,17 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-
-/* FIXME: per-module configuration? Issue #101 */
-#define WASM_PAGE_SIZE            (1024 * 64) /* 64KB */
-#define WASM_MEMORY_PAGES_INITIAL (1 << 8)    /* 256 Pages ~16MB */
-#define WASM_MEMORY_PAGES_MAX     (1 << 15)   /* 32,768 Pages ~4GB */
-
-#define WASM_STACK_SIZE (1 << 19) /* 512KB */
-
-/* bytes, not wasm pages */
-struct wasm_memory {
-	void *   start;
-	uint32_t size;
-	uint64_t max;
-};

runtime/include/worker_thread.h

@@ -10,43 +10,3 @@ extern thread_local int                 worker_thread_epoll_file_descriptor;
 extern thread_local int                 worker_thread_idx;
 
 void *worker_thread_main(void *return_code);
-
-/**
- * Translates WASM offsets into runtime VM pointers
- * @param offset an offset into the WebAssembly linear memory
- * @param bounds_check the size of the thing we are pointing to
- * @return void pointer to something in WebAssembly linear memory
- */
-static inline void *
-worker_thread_get_memory_ptr_void(uint32_t offset, uint32_t bounds_check)
-{
-	return (void *)get_memory_ptr_for_runtime(offset, bounds_check);
-}
-
-/**
- * Get a single-byte extended ASCII character from WebAssembly linear memory
- * @param offset an offset into the WebAssembly linear memory
- * @return char at the offset
- */
-static inline char
-worker_thread_get_memory_character(uint32_t offset)
-{
-	return get_memory_ptr_for_runtime(offset, 1)[0];
-}
-
-/**
- * Get a null-terminated String from WebAssembly linear memory
- * @param offset an offset into the WebAssembly linear memory
- * @param max_length the maximum expected length in characters
- * @return pointer to the string or NULL if max_length is reached without finding null-terminator
- */
-static inline char *
-worker_thread_get_memory_string(uint32_t offset, uint32_t max_length)
-{
-	for (uint32_t i = 0; i < max_length; i++) {
-		if (worker_thread_get_memory_character(offset + i) == '\0') {
-			return (char *)worker_thread_get_memory_ptr_void(offset, 1);
-		}
-	}
-	return NULL;
-}

runtime/src/current_sandbox.c

@@ -1,9 +1,9 @@
 #include <threads.h>
 
 #include "current_sandbox.h"
+#include "current_sandbox_send_response.h"
 #include "sandbox_functions.h"
 #include "sandbox_receive_request.h"
-#include "sandbox_send_response.h"
 #include "sandbox_set_as_asleep.h"
 #include "sandbox_set_as_error.h"
 #include "sandbox_set_as_returned.h"
@@ -16,13 +16,9 @@
 
 thread_local struct sandbox *worker_thread_current_sandbox = NULL;
 
-thread_local struct sandbox_context_cache local_sandbox_context_cache = {
+thread_local struct wasm_module_instance current_wasm_module_instance = {
-	.memory = {
+	.memory = NULL,
-		.start = NULL,
+	.table  = NULL,
-		.size = 0,
-		.max = 0,
-	},
-	.module_indirect_table = NULL,
 };
 
 /**
@@ -96,10 +92,12 @@ current_sandbox_init()
 	rc = sandbox_receive_request(sandbox);
 	if (rc == -2) {
 		/* Request size exceeded Buffer, send 413 Payload Too Large */
-		client_socket_send(sandbox->client_socket_descriptor, 413);
+		client_socket_send(sandbox->client_socket_descriptor, http_header_build(413), http_header_len(413),
+		                   current_sandbox_sleep);
 		goto err;
 	} else if (rc == -1) {
-		client_socket_send(sandbox->client_socket_descriptor, 400);
+		client_socket_send(sandbox->client_socket_descriptor, http_header_build(400), http_header_len(400),
+		                   current_sandbox_sleep);
 		goto err;
 	}
 
@@ -133,7 +131,7 @@ current_sandbox_fini()
 	sandbox->timestamp_of.completion = __getcycles();
 
 	/* Retrieve the result, construct the HTTP response, and send to client */
-	if (sandbox_send_response(sandbox) < 0) {
+	if (current_sandbox_send_response() < 0) {
 		error_message = "Unable to build and send client response\n";
 		goto err;
 	};

runtime/src/env.c

@@ -4,6 +4,7 @@
 
 #include "arch/getcycles.h"
 #include "worker_thread.h"
+#include "current_sandbox.h"
 
 extern int32_t inner_syscall_handler(int32_t n, int32_t a, int32_t b, int32_t c, int32_t d, int32_t e, int32_t f);
 
@@ -36,7 +37,7 @@ env_a_ctz_64(uint64_t x)
 INLINE void
 env_a_and_64(int32_t p_off, uint64_t v)
 {
-	uint64_t *p = worker_thread_get_memory_ptr_void(p_off, sizeof(uint64_t));
+	uint64_t *p = current_sandbox_get_ptr_void(p_off, sizeof(uint64_t));
 	ck_pr_and_64(p, v);
 }
 
@@ -44,7 +45,7 @@ INLINE void
 env_a_or_64(int32_t p_off, int64_t v)
 {
 	assert(sizeof(int64_t) == sizeof(uint64_t));
-	uint64_t *p = worker_thread_get_memory_ptr_void(p_off, sizeof(int64_t));
+	uint64_t *p = current_sandbox_get_ptr_void(p_off, sizeof(int64_t));
 	ck_pr_or_64(p, v);
 }
 
@@ -52,7 +53,7 @@ int32_t
 env_a_cas(int32_t p_off, int32_t t, int32_t s)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *p = worker_thread_get_memory_ptr_void(p_off, sizeof(int32_t));
+	int *p = current_sandbox_get_ptr_void(p_off, sizeof(int32_t));
 
 	return ck_pr_cas_int(p, t, s);
 }
@@ -61,7 +62,7 @@ void
 env_a_or(int32_t p_off, int32_t v)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *p = worker_thread_get_memory_ptr_void(p_off, sizeof(int32_t));
+	int *p = current_sandbox_get_ptr_void(p_off, sizeof(int32_t));
 	ck_pr_or_int(p, v);
 }
 
@@ -69,7 +70,7 @@ int32_t
 env_a_swap(int32_t x_off, int32_t v)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *x = worker_thread_get_memory_ptr_void(x_off, sizeof(int32_t));
+	int *x = current_sandbox_get_ptr_void(x_off, sizeof(int32_t));
 
 	int p;
 	do {
@@ -84,7 +85,7 @@ int32_t
 env_a_fetch_add(int32_t x_off, int32_t v)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *x = worker_thread_get_memory_ptr_void(x_off, sizeof(int32_t));
+	int *x = current_sandbox_get_ptr_void(x_off, sizeof(int32_t));
 	return ck_pr_faa_int(x, v);
 }
 
@@ -92,7 +93,7 @@ void
 env_a_inc(int32_t x_off)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *x = worker_thread_get_memory_ptr_void(x_off, sizeof(int32_t));
+	int *x = current_sandbox_get_ptr_void(x_off, sizeof(int32_t));
 	ck_pr_inc_int(x);
 }
 
@@ -100,7 +101,7 @@ void
 env_a_dec(int32_t x_off)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *x = worker_thread_get_memory_ptr_void(x_off, sizeof(int32_t));
+	int *x = (int *)current_sandbox_get_ptr_void(x_off, sizeof(int32_t));
 	ck_pr_dec_int(x);
 }
 
@@ -108,7 +109,7 @@ void
 env_a_store(int32_t p_off, int32_t x)
 {
 	assert(sizeof(int32_t) == sizeof(volatile int));
-	int *p = worker_thread_get_memory_ptr_void(p_off, sizeof(int32_t));
+	int *p = (int *)current_sandbox_get_ptr_void(p_off, sizeof(int32_t));
 	ck_pr_store_int(p, x);
 }
 

runtime/src/libc/syscall.c

@@ -14,6 +14,7 @@
 #include "scheduler.h"
 #include "sandbox_functions.h"
 #include "worker_thread.h"
+#include "common/wasm_store.h"
 
 // What should we tell the child program its UID and GID are?
 #define UID 0xFF
@@ -39,11 +40,12 @@
 void
 stub_init(int32_t offset)
 {
+	struct sandbox *current_sandbox = current_sandbox_get();
 	// What program name will we put in the auxiliary vectors
-	char *program_name = current_sandbox_get()->module->name;
+	char *program_name = current_sandbox->module->name;
 	// Copy the program name into WASM accessible memory
 	int32_t program_name_offset = offset;
-	strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name);
+	strcpy(wasm_memory_get_ptr_void(current_sandbox->memory, offset, sizeof(program_name)), program_name);
 	offset += sizeof(program_name);
 
 	// The construction of this is:
@@ -69,7 +71,8 @@ stub_init(int32_t offset)
 		0,
 	};
 	int32_t env_vec_offset = offset;
-	memcpy(get_memory_ptr_for_runtime(env_vec_offset, sizeof(env_vec)), env_vec, sizeof(env_vec));
+	memcpy(wasm_memory_get_ptr_void(current_sandbox->memory, env_vec_offset, sizeof(env_vec)), env_vec,
+	       sizeof(env_vec));
 
 	module_initialize_libc(current_sandbox_get()->module, env_vec_offset, program_name_offset);
 }
@@ -92,7 +95,7 @@ wasm_read(int32_t filedes, int32_t buf_offset, int32_t nbyte)
 
 	/* Non-blocking copy on stdin */
 	if (filedes == 0) {
-		char *               buffer          = worker_thread_get_memory_ptr_void(buf_offset, nbyte);
+		char *               buffer          = current_sandbox_get_ptr_void(buf_offset, nbyte);
 		struct http_request *current_request = &current_sandbox->http_request;
 		if (current_request->body_length <= 0) return 0;
 		int bytes_to_read = nbyte > current_request->body_length ? current_request->body_length : nbyte;
@@ -102,7 +105,7 @@ wasm_read(int32_t filedes, int32_t buf_offset, int32_t nbyte)
 		return bytes_to_read;
 	}
 
-	char *buf = worker_thread_get_memory_ptr_void(buf_offset, nbyte);
+	char *buf = current_sandbox_get_ptr_void(buf_offset, nbyte);
 
 	int32_t res = 0;
 	while (res < nbyte) {
@@ -133,7 +136,7 @@ int32_t
 wasm_write(int32_t fd, int32_t buf_offset, int32_t buf_size)
 {
 	struct sandbox *s      = current_sandbox_get();
-	char *          buffer = worker_thread_get_memory_ptr_void(buf_offset, buf_size);
+	char *          buffer = current_sandbox_get_ptr_void(buf_offset, buf_size);
 
 	if (fd == STDERR_FILENO) { write(STDERR_FILENO, buffer, buf_size); }
 
@@ -173,7 +176,7 @@ err:
 int32_t
 wasm_open(int32_t path_off, int32_t flags, int32_t mode)
 {
-	char *path = worker_thread_get_memory_string(path_off, MODULE_MAX_PATH_LENGTH);
+	char *path = current_sandbox_get_string(path_off, MODULE_MAX_PATH_LENGTH);
 
 	int res = ENOTSUP;
 
@@ -219,8 +222,8 @@ wasm_mmap(int32_t addr, int32_t len, int32_t prot, int32_t flags, int32_t fd, in
 
 	assert(len % WASM_PAGE_SIZE == 0);
 
-	int32_t result = local_sandbox_context_cache.memory.size;
+	int32_t result = wasm_memory_get_size(current_wasm_module_instance.memory);
-	for (int i = 0; i < len / WASM_PAGE_SIZE; i++) { expand_memory(); }
+	if (wasm_memory_expand(current_wasm_module_instance.memory, len) == -1) { result = (uint32_t)-1; }
 
 	return result;
 }
@@ -252,7 +255,7 @@ int32_t
 wasm_readv(int32_t fd, int32_t iov_offset, int32_t iovcnt)
 {
 	int32_t            read = 0;
-	struct wasm_iovec *iov  = worker_thread_get_memory_ptr_void(iov_offset, iovcnt * sizeof(struct wasm_iovec));
+	struct wasm_iovec *iov  = current_sandbox_get_ptr_void(iov_offset, iovcnt * sizeof(struct wasm_iovec));
 	for (int i = 0; i < iovcnt; i++) { read += wasm_read(fd, iov[i].base_offset, iov[i].len); }
 
 	return read;
@@ -266,8 +269,7 @@ wasm_writev(int32_t fd, int32_t iov_offset, int32_t iovcnt)
 	if (fd == STDOUT_FILENO || fd == STDERR_FILENO) {
 		// both 1 and 2 go to client.
 		int                len = 0;
-		struct wasm_iovec *iov = worker_thread_get_memory_ptr_void(iov_offset,
+		struct wasm_iovec *iov = current_sandbox_get_ptr_void(iov_offset, iovcnt * sizeof(struct wasm_iovec));
-		                                                           iovcnt * sizeof(struct wasm_iovec));
 		for (int i = 0; i < iovcnt; i++) { len += wasm_write(fd, iov[i].base_offset, iov[i].len); }
 
 		return len;
@@ -277,7 +279,7 @@ wasm_writev(int32_t fd, int32_t iov_offset, int32_t iovcnt)
 	assert(0);
 
 
-	struct wasm_iovec *iov = worker_thread_get_memory_ptr_void(iov_offset, iovcnt * sizeof(struct wasm_iovec));
+	struct wasm_iovec *iov = current_sandbox_get_ptr_void(iov_offset, iovcnt * sizeof(struct wasm_iovec));
 
 	// If we aren't on MUSL, pass writev to printf if possible
 #if defined(__GLIBC__)
@@ -285,7 +287,7 @@ wasm_writev(int32_t fd, int32_t iov_offset, int32_t iovcnt)
 		int sum = 0;
 		for (int i = 0; i < iovcnt; i++) {
 			int32_t len = iov[i].len;
-			void *  ptr = worker_thread_get_memory_ptr_void(iov[i].base_offset, len);
+			void *  ptr = current_sandbox_get_ptr_void(iov[i].base_offset, len);
 
 			printf("%.*s", len, (char *)ptr);
 			sum += len;
@@ -297,7 +299,7 @@ wasm_writev(int32_t fd, int32_t iov_offset, int32_t iovcnt)
 	struct iovec vecs[iovcnt];
 	for (int i = 0; i < iovcnt; i++) {
 		int32_t len = iov[i].len;
-		void *  ptr = worker_thread_get_memory_ptr_void(iov[i].base_offset, len);
+		void *  ptr = current_sandbox_get_ptr_void(iov[i].base_offset, len);
 		vecs[i]     = (struct iovec){ ptr, len };
 	}
 
@@ -318,25 +320,20 @@ wasm_mremap(int32_t offset, int32_t old_size, int32_t new_size, int32_t flags)
 	if (new_size <= old_size) return offset;
 
 	// If at end of linear memory, just expand and return same address
-	if (offset + old_size == local_sandbox_context_cache.memory.size) {
+	if (offset + old_size == current_wasm_module_instance.memory->size) {
-		int32_t amount_to_expand  = new_size - old_size;
+		int32_t amount_to_expand = new_size - old_size;
-		int32_t pages_to_allocate = amount_to_expand / WASM_PAGE_SIZE;
+		wasm_memory_expand(current_wasm_module_instance.memory, amount_to_expand);
-		if (amount_to_expand % WASM_PAGE_SIZE > 0) pages_to_allocate++;
-		for (int i = 0; i < pages_to_allocate; i++) expand_memory();
-
 		return offset;
 	}
 
 	// Otherwise allocate at end of address space and copy
-	int32_t pages_to_allocate = new_size / WASM_PAGE_SIZE;
+	int32_t new_offset = current_wasm_module_instance.memory->size;
-	if (new_size % WASM_PAGE_SIZE > 0) pages_to_allocate++;
+	wasm_memory_expand(current_wasm_module_instance.memory, new_size);
-	int32_t new_offset = local_sandbox_context_cache.memory.size;
-	for (int i = 0; i < pages_to_allocate; i++) expand_memory();
 
 	// Get pointer of old offset and pointer of new offset
-	char *linear_mem = local_sandbox_context_cache.memory.start;
+	uint8_t *linear_mem = current_wasm_module_instance.memory->data;
-	char *src        = &linear_mem[offset];
+	uint8_t *src        = &linear_mem[offset];
-	char *dest       = &linear_mem[new_offset];
+	uint8_t *dest       = &linear_mem[new_offset];
 
 	// Copy Values. We can use memcpy because we don't overlap
 	memcpy((void *)dest, (void *)src, old_size);
@@ -384,8 +381,7 @@ wasm_get_time(int32_t clock_id, int32_t timespec_off)
 		assert(0);
 	}
 
-	struct wasm_time_spec *timespec = worker_thread_get_memory_ptr_void(timespec_off,
+	struct wasm_time_spec *timespec = current_sandbox_get_ptr_void(timespec_off, sizeof(struct wasm_time_spec));
-	                                                                    sizeof(struct wasm_time_spec));
 
 	struct timespec native_timespec = { 0, 0 };
 	int             res             = clock_gettime(real_clock, &native_timespec);

runtime/src/linear_memory_pool.c

@@ -0,0 +1,3 @@
+#include "pool.h"
+
+struct pool **runtime_linear_memory_pools;

runtime/src/listener_thread.c

@@ -169,7 +169,8 @@ listener_thread_main(void *dummy)
 				 */
 				uint64_t work_admitted = admissions_control_decide(module->admissions_info.estimate);
 				if (work_admitted == 0) {
-					client_socket_send(client_socket, 503);
+					client_socket_send_oneshot(client_socket, http_header_build(503),
+					                           http_header_len(503));
 					if (unlikely(close(client_socket) < 0))
 						debuglog("Error closing client socket - %s", strerror(errno));
 

runtime/src/memory/64bit_nix.c

@@ -1,118 +0,0 @@
-#include "current_sandbox.h"
-#include "panic.h"
-#include "runtime.h"
-#include "sandbox_types.h"
-#include "types.h"
-
-#include <sys/mman.h>
-
-/**
- * @brief Expand the linear memory of the active WebAssembly sandbox by a single page
- *
- * @return int
- */
-int
-expand_memory(void)
-{
-	struct sandbox *sandbox = current_sandbox_get();
-
-	assert(sandbox->state == SANDBOX_RUNNING_USER || sandbox->state == SANDBOX_RUNNING_SYS);
-	assert(local_sandbox_context_cache.memory.size % WASM_PAGE_SIZE == 0);
-
-	/* Return -1 if we've hit the linear memory max */
-	if (unlikely(local_sandbox_context_cache.memory.size + WASM_PAGE_SIZE
-	             >= local_sandbox_context_cache.memory.max)) {
-		debuglog("expand_memory - Out of Memory!. %u out of %lu\n", local_sandbox_context_cache.memory.size,
-		         local_sandbox_context_cache.memory.max);
-		return -1;
-	}
-
-	// Remap the relevant wasm page to readable
-	char *mem_as_chars = local_sandbox_context_cache.memory.start;
-	char *page_address = &mem_as_chars[local_sandbox_context_cache.memory.size];
-	void *map_result   = mmap(page_address, WASM_PAGE_SIZE, PROT_READ | PROT_WRITE,
-                                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
-	if (map_result == MAP_FAILED) {
-		debuglog("Mapping of new memory failed");
-		return -1;
-	}
-
-	local_sandbox_context_cache.memory.size += WASM_PAGE_SIZE;
-
-#ifdef LOG_SANDBOX_MEMORY_PROFILE
-	// Cache the runtime of the first N page allocations
-	if (likely(sandbox->timestamp_of.page_allocations_size < SANDBOX_PAGE_ALLOCATION_TIMESTAMP_COUNT)) {
-		sandbox->timestamp_of.page_allocations[sandbox->timestamp_of.page_allocations_size++] =
-		  sandbox->duration_of_state.running
-		  + (uint32_t)(__getcycles() - sandbox->timestamp_of.last_state_change);
-	}
-#endif
-
-	// local_sandbox_context_cache is "forked state", so update authoritative member
-	sandbox->memory.size = local_sandbox_context_cache.memory.size;
-	return 0;
-}
-
-INLINE char *
-get_memory_ptr_for_runtime(uint32_t offset, uint32_t bounds_check)
-{
-	// Due to how we setup memory for x86, the virtual memory mechanism will catch the error, if bounds <
-	// WASM_PAGE_SIZE
-	assert(bounds_check < WASM_PAGE_SIZE
-	       || (local_sandbox_context_cache.memory.size > bounds_check
-	           && offset <= local_sandbox_context_cache.memory.size - bounds_check));
-
-	char *mem_as_chars = (char *)local_sandbox_context_cache.memory.start;
-	char *address      = &mem_as_chars[offset];
-
-	return address;
-}
-
-/**
- * @brief Stub that implements the WebAssembly memory.grow instruction
- *
- * @param count number of pages to grow the WebAssembly linear memory by
- * @return The previous size of the linear memory in pages or -1 if enough memory cannot be allocated
- */
-int32_t
-instruction_memory_grow(uint32_t count)
-{
-	int rc = local_sandbox_context_cache.memory.size / WASM_PAGE_SIZE;
-
-	for (int i = 0; i < count; i++) {
-		if (unlikely(expand_memory() != 0)) {
-			rc = -1;
-			break;
-		}
-	}
-
-	return rc;
-}
-
-/*
- * Table handling functionality
- * This was moved from compiletime in order to place the
- * function in the callstack in GDB. It can be moved back
- * to runtime/compiletime/memory/64bit_nix.c to remove the
- * additional function call
- */
-char *
-get_function_from_table(uint32_t idx, uint32_t type_id)
-{
-#ifdef LOG_FUNCTION_TABLE
-	fprintf(stderr, "get_function_from_table(idx: %u, type_id: %u)\n", idx, type_id);
-	fprintf(stderr, "indirect_table_size: %u\n", INDIRECT_TABLE_SIZE);
-#endif
-	assert(idx < INDIRECT_TABLE_SIZE);
-
-	struct indirect_table_entry f = local_sandbox_context_cache.module_indirect_table[idx];
-#ifdef LOG_FUNCTION_TABLE
-	fprintf(stderr, "assumed type: %u, type in table: %u\n", type_id, f.type_id);
-#endif
-	// FIXME: Commented out function type check because of gocr
-	// assert(f.type_id == type_id);
-
-	assert(f.func_pointer != NULL);
-
-	return f.func_pointer;
-}

runtime/src/memory/common.c

@@ -1,38 +0,0 @@
-#include <assert.h>
-#include <string.h>
-
-#include "runtime.h"
-#include "types.h"
-
-/* Region initialization helper function */
-EXPORT void
-initialize_region(uint32_t offset, uint32_t data_count, char *data)
-{
-	assert(local_sandbox_context_cache.memory.size >= data_count);
-	assert(offset < local_sandbox_context_cache.memory.size - data_count);
-
-	memcpy(get_memory_ptr_for_runtime(offset, data_count), data, data_count);
-}
-
-void
-add_function_to_table(uint32_t idx, uint32_t type_id, char *pointer)
-{
-	assert(idx < INDIRECT_TABLE_SIZE);
-	assert(local_sandbox_context_cache.module_indirect_table != NULL);
-
-	/* TODO: atomic for multiple concurrent invocations? Issue #97 */
-	if (local_sandbox_context_cache.module_indirect_table[idx].type_id == type_id
-	    && local_sandbox_context_cache.module_indirect_table[idx].func_pointer == pointer)
-		return;
-
-	local_sandbox_context_cache.module_indirect_table[idx] = (struct indirect_table_entry){
-		.type_id = type_id, .func_pointer = pointer
-	};
-}
-
-/* If we are using runtime globals, we need to populate them */
-WEAK void
-populate_globals()
-{
-	assert(0); /* FIXME: is this used in WASM as dynamic modules? Issue #105. */
-}

runtime/src/module.c

@@ -16,6 +16,7 @@
 #include "panic.h"
 #include "runtime.h"
 #include "scheduler.h"
+#include "common/wasm_table.h"
 
 const int JSON_MAX_ELEMENT_COUNT = 16;
 const int JSON_MAX_ELEMENT_SIZE  = 1024;
@@ -122,6 +123,12 @@ module_free(struct module *module)
 
 	close(module->socket_descriptor);
 	awsm_abi_deinit(&module->abi);
+
+	for (int i = 0; i < runtime_worker_threads_count; i++) pool_free(module->linear_memory_pool[i]);
+	free(module->linear_memory_pool);
+
+	/* Initialize per worker linear memory pools */
+	module->linear_memory_pool = calloc(runtime_worker_threads_count, sizeof(struct pool *));
 	free(module);
 }
 
@@ -182,6 +189,11 @@ module_new(char *name, char *path, uint32_t stack_size, uint32_t max_memory, uin
 	admissions_info_initialize(&module->admissions_info, admissions_percentile, expected_execution,
 	                           module->relative_deadline);
 
+	/* WebAssembly Indirect Table */
+	/* TODO: Should this be part of the module or per-sandbox? */
+	/* TODO: How should this table be sized? */
+	module->indirect_table = wasm_table_allocate(INDIRECT_TABLE_SIZE);
+
 	/* Request Response Buffer */
 	if (request_size == 0) request_size = MODULE_DEFAULT_REQUEST_RESPONSE_SIZE;
 	if (response_size == 0) response_size = MODULE_DEFAULT_REQUEST_RESPONSE_SIZE;
@@ -189,22 +201,26 @@ module_new(char *name, char *path, uint32_t stack_size, uint32_t max_memory, uin
 	module->max_response_size = round_up_to_page(response_size);
 
 	/* Table initialization calls a function that runs within the sandbox. Rather than setting the current sandbox,
-	 * we partially fake this out by only setting the module_indirect_table and then clearing after table
+	 * we partially fake this out by only setting the table and then clearing after table
 	 * initialization is complete.
 	 *
 	 * assumption: This approach depends on module_new only being invoked at program start before preemption is
-	 * enabled. We are check that local_sandbox_context_cache.module_indirect_table is NULL to gain confidence that
+	 * enabled. We are check that current_wasm_module_instance.table is NULL to gain confidence that
 	 * we are not invoking this in a way that clobbers a current module.
 	 *
 	 * If we want to be able to do this later, we can possibly defer module_initialize_table until the first
-	 * invocation. Alternatively, we can maintain the module_indirect_table per sandbox and call initialize
+	 * invocation. Alternatively, we can maintain the table per sandbox and call initialize
 	 * on each sandbox if this "assumption" is too restrictive and we're ready to pay a per-sandbox performance hit.
 	 */
 
-	assert(local_sandbox_context_cache.module_indirect_table == NULL);
+	assert(current_wasm_module_instance.table == NULL);
-	local_sandbox_context_cache.module_indirect_table = module->indirect_table;
+	current_wasm_module_instance.table = module->indirect_table;
 	module_initialize_table(module);
-	local_sandbox_context_cache.module_indirect_table = NULL;
+	current_wasm_module_instance.table = NULL;
+
+	/* Initialize per worker linear memory pools */
+	module->linear_memory_pool = calloc(runtime_worker_threads_count, sizeof(struct pool *));
+	for (int i = 0; i < runtime_worker_threads_count; i++) { module->linear_memory_pool[i] = pool_init(10, true); }
 
 	/* Start listening for requests */
 	rc = module_listen(module);

runtime/src/sandbox.c

@@ -5,9 +5,11 @@
 #include "current_sandbox.h"
 #include "debuglog.h"
 #include "panic.h"
+#include "pool.h"
 #include "sandbox_functions.h"
 #include "sandbox_set_as_error.h"
 #include "sandbox_set_as_initialized.h"
+#include "common/wasm_memory.h"
 
 /**
  * Allocates a WebAssembly sandbox represented by the following layout
@@ -15,77 +17,29 @@
  * @param module the module that we want to run
  * @returns the resulting sandbox or NULL if mmap failed
  */
-static inline struct sandbox *
+static inline int
-sandbox_allocate_memory(struct module *module)
+sandbox_allocate_linear_memory(struct sandbox *self)
 {
-	assert(module != NULL);
+	assert(self != NULL);
 
-	char *          error_message             = NULL;
+	char *error_message = NULL;
-	unsigned long   memory_size               = WASM_PAGE_SIZE * WASM_MEMORY_PAGES_INITIAL; /* The initial pages */
-	uint64_t        memory_max                = (uint64_t)WASM_PAGE_SIZE * WASM_MEMORY_PAGES_MAX;
-	struct sandbox *sandbox                   = NULL;
-	unsigned long   page_aligned_sandbox_size = round_up_to_page(sizeof(struct sandbox));
 
-	unsigned long size_to_alloc = page_aligned_sandbox_size + module->max_request_size + module->max_request_size
+	size_t initial = (size_t)self->module->abi.starting_pages * WASM_PAGE_SIZE;
-	                              + memory_max + /* guard page */ PAGE_SIZE;
+	size_t max     = (size_t)self->module->abi.max_pages * WASM_PAGE_SIZE;
-	unsigned long size_to_read_write = page_aligned_sandbox_size + module->max_request_size
-	                                   + module->max_request_size + memory_size;
 
-	/*
+	assert(initial <= (size_t)UINT32_MAX + 1);
-	 * Control information should be page-aligned
+	assert(max <= (size_t)UINT32_MAX + 1);
-	 */
-	assert(round_up_to_page(size_to_alloc) == size_to_alloc);
-
-	/* At an address of the system's choosing, allocate the memory, marking it as inaccessible */
-	errno      = 0;
-	void *addr = mmap(NULL, size_to_alloc, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-	if (addr == MAP_FAILED) {
-		error_message = "sandbox_allocate_memory - memory allocation failed";
-		goto alloc_failed;
-	}
 
-	assert(addr != NULL);
+	self->memory = (struct wasm_memory *)pool_allocate_object(self->module->linear_memory_pool[worker_thread_idx]);
 
-	/* Set the struct sandbox, HTTP Req/Resp buffer, and the initial Wasm Pages as read/write */
+	if (self->memory == NULL) {
-	errno         = 0;
+		self->memory = wasm_memory_allocate(initial, max);
-	void *addr_rw = mmap(addr, size_to_read_write, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
+		if (unlikely(self->memory == NULL)) return -1;
-	                     -1, 0);
+	} else {
-	if (addr_rw == MAP_FAILED) {
+		wasm_memory_set_size(self->memory, self->module->abi.starting_pages * WASM_PAGE_SIZE);
-		error_message = "set to r/w";
-		goto set_rw_failed;
 	}
 
-	sandbox = (struct sandbox *)addr_rw;
+	return 0;
-
-	/* Populate Sandbox members */
-	sandbox->state  = SANDBOX_UNINITIALIZED;
-	sandbox->module = module;
-	module_acquire(module);
-
-	sandbox->request.base   = (char *)addr + page_aligned_sandbox_size;
-	sandbox->request.length = 0;
-
-	sandbox->response.base  = (char *)addr + page_aligned_sandbox_size + module->max_request_size;
-	sandbox->request.length = 0;
-
-	sandbox->memory.start = (char *)addr + page_aligned_sandbox_size + module->max_request_size
-	                        + module->max_request_size;
-	sandbox->memory.size = memory_size;
-	sandbox->memory.max  = memory_max;
-
-	memset(&sandbox->duration_of_state, 0, SANDBOX_STATE_COUNT * sizeof(uint64_t));
-
-done:
-	return sandbox;
-set_rw_failed:
-	sandbox = NULL;
-	errno   = 0;
-	int rc  = munmap(addr, size_to_alloc);
-	if (rc == -1) perror("Failed to munmap after fail to set r/w");
-alloc_failed:
-err:
-	perror(error_message);
-	goto done;
 }
 
 static inline int
@@ -96,7 +50,7 @@ sandbox_allocate_stack(struct sandbox *sandbox)
 
 	int rc = 0;
 
-	char *addr = mmap(NULL, sandbox->module->stack_size + /* guard page */ PAGE_SIZE, PROT_NONE,
+	char *addr = mmap(NULL, /* guard page */ PAGE_SIZE + sandbox->module->stack_size, PROT_NONE,
 	                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 	if (unlikely(addr == MAP_FAILED)) {
 		perror("sandbox allocate stack");
@@ -126,6 +80,20 @@ err_stack_allocation_failed:
 	goto done;
 }
 
+static inline int
+sandbox_allocate_http_buffers(struct sandbox *self)
+{
+	self->request.base = calloc(1, self->module->max_request_size);
+	if (self->request.base == NULL) return -1;
+	self->request.length = 0;
+
+	self->response.base = calloc(1, self->module->max_response_size);
+	if (self->response.base == NULL) return -1;
+	self->response.length = 0;
+
+	return 0;
+}
+
 /**
  * Allocates a new sandbox from a sandbox request
  * Frees the sandbox request on success
@@ -138,51 +106,58 @@ sandbox_allocate(struct sandbox_request *sandbox_request)
 	/* Validate Arguments */
 	assert(sandbox_request != NULL);
 
-	struct sandbox *sandbox;
+	char *   error_message = "";
-	char *          error_message = "";
+	uint64_t now           = __getcycles();
-	uint64_t        now           = __getcycles();
 
-	/* Allocate Sandbox control structures, buffers, and linear memory in a 4GB address space */
+	int rc;
-	sandbox = sandbox_allocate_memory(sandbox_request->module);
+
-	if (!sandbox) {
+	struct sandbox *self                      = NULL;
-		error_message = "failed to allocate sandbox heap and linear memory";
+	size_t          page_aligned_sandbox_size = round_up_to_page(sizeof(struct sandbox));
+	self                                      = calloc(1, page_aligned_sandbox_size);
+	if (self == NULL) goto err_struct_allocation_failed;
+
+	/* Set state to initializing */
+	sandbox_set_as_initialized(self, sandbox_request, now);
+
+	if (sandbox_allocate_http_buffers(self)) {
+		error_message = "failed to allocate http buffers";
+		goto err_http_allocation_failed;
+	}
+
+	/* Allocate linear memory in a 4GB address space */
+	if (sandbox_allocate_linear_memory(self)) {
+		error_message = "failed to allocate sandbox linear memory";
 		goto err_memory_allocation_failed;
 	}
 
 	/* Allocate the Stack */
-	if (sandbox_allocate_stack(sandbox) < 0) {
+	if (sandbox_allocate_stack(self) < 0) {
 		error_message = "failed to allocate sandbox stack";
 		goto err_stack_allocation_failed;
 	}
-	sandbox->state = SANDBOX_ALLOCATED;
 
-#ifdef LOG_STATE_CHANGES
+	/* Initialize the sandbox's context, stack, and instruction pointer */
-	sandbox->state_history_count                           = 0;
+	/* stack.start points to the bottom of the usable stack, so add stack_size to get to top */
-	sandbox->state_history[sandbox->state_history_count++] = SANDBOX_ALLOCATED;
+	arch_context_init(&self->ctxt, (reg_t)current_sandbox_start, (reg_t)self->stack.start + self->stack.size);
-	memset(&sandbox->state_history, 0, SANDBOX_STATE_HISTORY_CAPACITY * sizeof(sandbox_state_t));
-#endif
-
-	/* Set state to initializing */
-	sandbox_set_as_initialized(sandbox, sandbox_request, now);
 
 	free(sandbox_request);
 done:
-	return sandbox;
+	return self;
 err_stack_allocation_failed:
 	/*
 	 * This is a degenerate sandbox that never successfully completed initialization, so we need to
 	 * hand jam some things to be able to cleanly transition to ERROR state
 	 */
-	sandbox->state                          = SANDBOX_UNINITIALIZED;
+	self->state                          = SANDBOX_UNINITIALIZED;
-	sandbox->timestamp_of.last_state_change = now;
+	self->timestamp_of.last_state_change = now;
-#ifdef LOG_SANDBOX_MEMORY_PROFILE
+
-	sandbox->timestamp_of.page_allocations_size = 0;
+	ps_list_init_d(self);
-#endif
-	ps_list_init_d(sandbox);
 err_memory_allocation_failed:
-	sandbox_set_as_error(sandbox, SANDBOX_UNINITIALIZED);
+err_http_allocation_failed:
+	sandbox_set_as_error(self, SANDBOX_UNINITIALIZED);
 	perror(error_message);
-	sandbox = NULL;
+err_struct_allocation_failed:
+	self = NULL;
 	goto done;
 }
 
@@ -221,17 +196,11 @@ sandbox_free(struct sandbox *sandbox)
 	 */
 
 	/* Linear Memory and Guard Page should already have been munmaped and set to NULL */
-	assert(sandbox->memory.start == NULL);
+	assert(sandbox->memory == NULL);
-	errno = 0;
-
-	unsigned long size_to_unmap = round_up_to_page(sizeof(struct sandbox)) + sandbox->module->max_request_size
-	                              + sandbox->module->max_response_size;
-	munmap(sandbox, size_to_unmap);
-	if (rc == -1) {
-		debuglog("Failed to unmap Sandbox %lu\n", sandbox->id);
-		goto err_free_sandbox_failed;
-	};
 
+	free(sandbox->request.base);
+	free(sandbox->response.base);
+	free(sandbox);
 done:
 	return;
 err_free_sandbox_failed:

runtime/src/sandbox_state.c

@@ -10,7 +10,6 @@
 const char *sandbox_state_labels[SANDBOX_STATE_COUNT] = {
 
 	[SANDBOX_UNINITIALIZED] = "Uninitialized",
-	[SANDBOX_ALLOCATED]     = "Allocated",
 	[SANDBOX_INITIALIZED]   = "Initialized",
 	[SANDBOX_RUNNABLE]      = "Runnable",
 	[SANDBOX_INTERRUPTED]   = "Interrupted",

runtime/tests/.gitignore

@@ -0,0 +1,3 @@
+*.wasm
+*.bc
+*.so

runtime/tests/Makefile

@@ -1,60 +1,67 @@
-include Makefile.inc
+CC=clang # Source -> Native
+WASMCC=wasm32-unknown-unknown-wasm-clang # Source -> WebAssembly
 
-TESTS=fibonacci empty empty
+OPTFLAGS=-O0 -g
+TMP_DIR=tmp/
 
-TESTSRT=$(TESTS:%=%_rt)
+# same stack-size for all
+WASMLINKFLAGS=-Wl,-z,stack-size=524288,--allow-undefined,--no-threads,--stack-first,--no-entry,--export-all,--export=main,--export=dummy
+WASMCFLAGS=${WASMLINKFLAGS} -nostartfiles
 
-.PHONY: all
+# aWsm Compiler Runtime (WebAssembly -> LLVM Bitcode)
-all: rttests tinyekf cifar10 gocr sod
+AWSM_NAME=awsm
-	@echo "Test Compilation done!"
+AWSM_BASE_DIR=../../${AWSM_NAME}
+DUMMY=${AWSM_BASE_DIR}/code_benches/dummy.c
+
+# for SLEdge
+SLEDGE_BASE_DIR=../../
+SLEDGE_RT_DIR=${SLEDGE_BASE_DIR}/runtime/
+SLEDGE_COMPILETIME_INC=${SLEDGE_RT_DIR}/include/common
+SLEDGE_BIN_DIR=${SLEDGE_RT_DIR}/bin/
+SLEDGE_COMPILETIME_SRC=${SLEDGE_RT_DIR}/compiletime/*.c
+
+ALL=fibonacci empty license_plate_detection resize_image gocr cifar10
+ALL_COPY_DEST=$(ALL:%=../../runtime/bin/%_wasm.so)
 
-.PHONY: rttests
+.PHONY: all
-rttests: $(TESTSRT)
+all: $(ALL_COPY_DEST)
 
 .PHONY: clean
 clean:
-	@echo "Cleaning Test Applications"
+	@make clean -C ./CMSIS_5_NN/ -f Makefile
-	@rm -rf ${TMP_DIR}
 	@make clean -C ./gocr/src/ -f wasm.mk
 	@make clean -C ./sod/
+	@make clean -C ./TinyEKF/extras/c/ -f wasm.mk
+	@rm -f ../../runtime/bin/*.so
+
+cifar10.wasm:
+	@make cifar10.wasm -C ./CMSIS_5_NN/ -f Makefile
+	@cp ./CMSIS_5_NN/cifar10.wasm cifar10.wasm
+
+gocr.wasm:
+	@make gocr.wasm -C ./gocr/src/ -f wasm.mk
+	@cp ./gocr/src/gocr.wasm gocr.wasm
+
+resize_image.wasm:
+	@make dir resize_image.wasm -C ./sod/
+	@cp ./sod/bin/resize_image.wasm resize_image.wasm
+
+license_plate_detection.wasm:
+	@make dir license_plate_detection.wasm -C ./sod/
+	@cp ./sod/bin/license_plate_detection.wasm license_plate_detection.wasm
+
+gps_ekf_fn.wasm:
+	@make gps_ekf_fn.wasm -C ./TinyEKF/extras/c/ -f wasm.mk
+	@cp ./TinyEKF/extras/c/gps_ekf_fn.wasm gps_ekf_fn.wasm
+
+%.wasm: %.c $(DUMMY)
+	${WASMCC} ${WASMCFLAGS} ${OPTFLAGS} $^ -o $@
+
+%.bc: %.wasm
+	${AWSM_NAME} --inline-constant-globals --runtime-globals $^ -o $@
 
-.PHONY: tinyekf
+%_wasm.so: %.bc ${SLEDGE_COMPILETIME_SRC}
-tinyekf:
+	${CC} --shared -fPIC ${OPTFLAGS} -I${SLEDGE_COMPILETIME_INC} $^ -o $@
-	@echo "Making and Installing tinyekf"
-	@make gps_ekf_fn.so -C ./TinyEKF/extras/c/ -f wasm.mk
-	@cp ./TinyEKF/extras/c/gps_ekf_fn.so ${SLEDGE_BIN_DIR}/ekf_wasm.so
-
-.PHONY: cifar10
-cifar10:
-	@echo "Making and Installing cifar10"
-	@make cifar10.so -C ./CMSIS_5_NN/ -f Makefile
-	@cp ./CMSIS_5_NN/cifar10.so ${SLEDGE_BIN_DIR}/cifar10_wasm.so
-
-.PHONY: gocr
-gocr:
-	@echo "Making and Installing gocr"
-	@make gocr.so -C ./gocr/src/ -f wasm.mk
-	@cp ./gocr/src/gocr.so ${SLEDGE_BIN_DIR}/gocr_wasm.so
-
-.PHONY: sod
-sod:
-	@echo "Making and Installing license_plate_detection and image_resize"
-	@make dir samples.so -C ./sod/
-	@cp ./sod/bin/license_plate_detection.so ${SLEDGE_BIN_DIR}/lpd_wasm.so
-	@cp ./sod/bin/resize_image.so ${SLEDGE_BIN_DIR}/resize_wasm.so
-
-%_rt:
-	@mkdir -p ${TMP_DIR}
-	# Compile the wasm file
-	@echo "Compiling $(@:%_rt=%)"
-	${WASMCC} ${$(@:%_rt=%)_CFLAGS} ${WASMCFLAGS} ${OPTFLAGS} $(@:%_rt=%)/*.c $(AWSM_DUMMY) -o ${TMP_DIR}/$(@:%_rt=%).wasm
-
-	# Compile the *.bc file
-
-	${AWSM_NAME} --inline-constant-globals --runtime-globals ${TMP_DIR}/$(@:%_rt=%).wasm -o ${TMP_DIR}/$(@:%_rt=%).bc
-	# Compile the *.so file
-	${CC} --shared -fPIC ${OPTFLAGS} -I${SLEDGE_RT_INC} -D${USE_MEM} ${TMP_DIR}/$(@:%_rt=%).bc ${SLEDGE_MEMC} ${SLEDGE_WASMISA} -o ${TMP_DIR}/$(@:%_rt=%)_wasm.so
-	# Copy the *.so file to the binary directory
-	@cp ${TMP_DIR}/$(@:%_rt=%)_wasm.so ${SLEDGE_BIN_DIR} 
-#	@rm -rf ${TMP_DIR}
 
+../../runtime/bin/%.so: %.so
+	cp $^ $@

runtime/tests/Makefile.inc

@@ -1,42 +0,0 @@
-ARCH := $(shell uname -m)
-
-CC=clang # Source -> Native
-WASMCC=wasm32-unknown-unknown-wasm-clang # Source -> WebAssembly
-
-OPTFLAGS=-O3 -flto
-MEMC_64=64bit_nix.c
-# MEMC_NO=no_protection.c
-# MEMC_GEN=generic.c
-# MEMC_MPX=mpx.c
-# MEMC_SEG=segmented.c
-TMP_DIR=tmp/
-
-# same stack-size for all
-WASMLINKFLAGS=-Wl,-z,stack-size=524288,--allow-undefined,--no-threads,--stack-first,--no-entry,--export-all,--export=main,--export=dummy
-WASMCFLAGS=${WASMLINKFLAGS} -nostartfiles
-
-# aWsm Compiler Runtime (WebAssembly -> LLVM Bitcode)
-AWSM_NAME=awsm
-AWSM_BASE_DIR=../../${AWSM_NAME}
-AWSM_RT_DIR=${AWSM_BASE_DIR}/runtime/
-AWSM_RT_MEM=${AWSM_RT_DIR}/memory/
-AWSM_RT_LIBC=${AWSM_RT_DIR}/libc/libc_backing.c
-AWSM_RT_RT=${AWSM_RT_DIR}/runtime.c
-AWSM_RT_ENV=${AWSM_RT_DIR}/libc/env.c
-# Is seems like the arch specific code no longer exists
-# AWSM_RT_ENV=${AWSM_RT_DIR}/libc/env.c ${AWSM_RT_DIR}/libc/env_${ARCH}.c
-AWSM_MEMC=${AWSM_RT_MEM}/${MEMC_64}
-AWSM_DUMMY=${AWSM_BASE_DIR}/code_benches/dummy.c
-
-# for SLEdge
-SLEDGE_BASE_DIR=../../
-SLEDGE_RT_DIR=${SLEDGE_BASE_DIR}/runtime/
-SLEDGE_RT_INC=${SLEDGE_RT_DIR}/include/
-SLEDGE_BIN_DIR=${SLEDGE_RT_DIR}/bin/
-SLEDGE_WASMISA=${SLEDGE_RT_DIR}/compiletime/instr.c
-
-USE_MEM=USE_MEM_VM
-
-ifeq ($(USE_MEM),USE_MEM_VM)
-SLEDGE_MEMC=${SLEDGE_RT_DIR}/compiletime/memory/${MEMC_64}
-endif

runtime/tests/TinyEKF

@@ -1 +1 @@
-Subproject commit fe4d4600a3b02f313b74892990595354dfc98913
+Subproject commit 92a8aa9a5e0aed719836558825fa839b1b1efb0b

runtime/tests/fibonacci.c

@@ -1,5 +1,4 @@
 #include <stdio.h>
-// #include "get_time.h"
 unsigned long int
 fib(unsigned long int n)
 {
@@ -12,11 +11,7 @@ main(int argc, char **argv)
 {
 	unsigned long n = 0, r;
 	scanf("%lu", &n);
-	//	unsigned long long st = get_time(), en;
 	r = fib(n);
-	//	en = get_time();
 	printf("%lu\n", r);
-
-	//	print_time(st, en);
 	return 0;
 }

runtime/tests/fibonacci/get_time.h

@@ -1,44 +0,0 @@
-#ifndef GET_TIME_H
-#define GET_TIME_H
-
-#include <time.h>
-#include <sys/time.h>
-#include <stdio.h>
-
-#ifndef WASM
-#ifndef CPU_FREQ
-#define CPU_FREQ 1000
-#endif
-#endif
-
-static unsigned long long
-get_time()
-{
-#if 0
-  unsigned long long int ret = 0;
-  unsigned int cycles_lo;
-  unsigned int cycles_hi;
-  __asm__ volatile ("rdtsc" : "=a" (cycles_lo), "=d" (cycles_hi));
-  ret = (unsigned long long int)cycles_hi << 32 | cycles_lo;
-
-  return ret;
-#else
-	struct timeval Tp;
-	int            stat;
-	stat = gettimeofday(&Tp, NULL);
-	if (stat != 0) printf("Error return from gettimeofday: %d", stat);
-	return (Tp.tv_sec * 1000000 + Tp.tv_usec);
-#endif
-}
-
-static inline void
-print_time(unsigned long long s, unsigned long long e)
-{
-#if 0
-	printf("%llu cycs, %llu us\n", e - s, (e - s) / CPU_FREQ);
-#else
-	fprintf(stdout, "%llu us\n", e - s);
-#endif
-}
-
-#endif /* GET_TIME_H */

runtime/tests/fibonacci/run_fib.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-ITERS=$3
-
-# before running this benchmark,
-# copy fibonacci to fibonacci_native.out
-
-testeach() {
-	tmp_cnt=${ITERS}
-	exe_relpath=$1
-
-	echo "${exe_relpath} ($2) for ${tmp_cnt}"
-
-	while [ ${tmp_cnt} -gt 0 ]; do
-		bench=$(echo $2 | $exe_relpath 2> /dev/null)
-		tmp_cnt=$((tmp_cnt - 1))
-		echo "$bench"
-	done
-
-	echo "Done!"
-}
-
-MAXNUM=$2
-
-tmp1_cnt=${MAXNUM}
-
-while [ ${tmp1_cnt} -gt 28 ]; do
-	testeach ./fibonacci_$1.out ${tmp1_cnt}
-	tmp1_cnt=$((tmp1_cnt - 1))
-done
-
-echo "All done!"