From 576bc4b086fd5c5314ea43ee35b1a835da44f7ac Mon Sep 17 00:00:00 2001
From: phani <phanikishoreg@gwu.edu>
Date: Fri, 3 Jan 2020 11:21:16 -0500
Subject: [PATCH] sandbox alloc on each core, for scalability

---
 runtime/include/sandbox.h | 42 +++++++++++++++++++++++++++++++++------
 runtime/src/runtime.c     | 17 ++++++++++++----
 runtime/src/sandbox.c     |  2 ++
 3 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/runtime/include/sandbox.h b/runtime/include/sandbox.h
index 82ea254..aeaaacd 100644
--- a/runtime/include/sandbox.h
+++ b/runtime/include/sandbox.h
@@ -76,7 +76,19 @@ struct sandbox {
 	char req_resp_data[1]; //of rr_data_sz, following sandbox mem..
 } PAGE_ALIGNED;
 
-DEQUE_PROTOTYPE(sandbox, struct sandbox *);
+#ifndef STANDALONE
+struct sandbox_request {
+	struct module *mod;
+	char *args;
+	int sock;
+	struct sockaddr *addr;
+};
+typedef struct sandbox_request sbox_request_t;
+#else
+typedef struct sandbox sbox_request_t;
+#endif
+
+DEQUE_PROTOTYPE(sandbox, sbox_request_t *);
 
 // a runtime resource, malloc on this!
 struct sandbox *sandbox_alloc(struct module *mod, char *args, int sock, const struct sockaddr *addr);
@@ -88,6 +100,24 @@ extern __thread struct sandbox *current_sandbox;
 extern __thread arch_context_t *next_context;
 
 typedef struct sandbox sandbox_t;
+void sandbox_run(sbox_request_t *s);
+
+static inline sbox_request_t *
+sbox_request_alloc(struct module *mod, char *args, int sock, const struct sockaddr *addr)
+{
+#ifndef STANDALONE
+	sbox_request_t *s = malloc(sizeof(sbox_request_t));
+	assert(s);
+	s->mod = mod;
+	s->args = args;
+	s->sock = sock;
+	s->addr = (struct sockaddr *)addr;
+	sandbox_run(s);
+	return s;
+#else
+	return sandbox_alloc(mod, args, sock, addr);
+#endif
+}
 
 static inline struct sandbox *
 sandbox_current(void)
@@ -144,7 +174,7 @@ sandbox_args(void)
 	return (char *)c->args;
 }
 
-void sandbox_run(struct sandbox *s);
+//void sandbox_run(struct sandbox *s);
 void *sandbox_run_func(void *data);
 struct sandbox *sandbox_schedule(void);
 void sandbox_block(void);
@@ -159,7 +189,7 @@ extern struct deque_sandbox *glb_dq;
 extern pthread_mutex_t glbq_mtx; 
 
 static inline int
-sandbox_deque_push(struct sandbox *s)
+sandbox_deque_push(sbox_request_t *s)
 {
 	int ret;
 
@@ -175,7 +205,7 @@ sandbox_deque_push(struct sandbox *s)
 }
 
 static inline int
-sandbox_deque_pop(struct sandbox **s)
+sandbox_deque_pop(sbox_request_t **s)
 {
 	int ret;
 
@@ -190,10 +220,10 @@ sandbox_deque_pop(struct sandbox **s)
 	return ret;
 }
 
-static inline struct sandbox *
+static inline sbox_request_t *
 sandbox_deque_steal(void)
 {
-	struct sandbox *s = NULL;
+	sbox_request_t *s = NULL;
 
 #if NCORES == 1
 	sandbox_deque_pop(&s);
diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c
index 20c9a6b..acda7ca 100644
--- a/runtime/src/runtime.c
+++ b/runtime/src/runtime.c
@@ -44,11 +44,19 @@ sandbox_pull(void)
 	int n = 0;
 
 	while (n < SBOX_PULL_MAX) {
-		struct sandbox *s = sandbox_deque_steal();
+		sbox_request_t *s = sandbox_deque_steal();
 
 		if (!s) break;
+#ifndef STANDALONE
+		struct sandbox *sb = sandbox_alloc(s->mod, s->args, s->sock, s->addr);
+		assert(sb);
+		free(s);
+		sb->state = SANDBOX_RUNNABLE;
+		sandbox_local_run(sb);
+#else
 		assert(s->state == SANDBOX_RUNNABLE);
 		sandbox_local_run(s);
+#endif
 		n++;
 	}
 
@@ -208,14 +216,14 @@ sandbox_run_func(void *data)
 }
 
 void
-sandbox_run(struct sandbox *s)
+sandbox_run(sbox_request_t *s)
 {
 #ifndef STANDALONE
 	// for now, a pull model... 
 	// sandbox_run adds to the global ready queue..
 	// each sandboxing thread pulls off of that global ready queue..
 	debuglog("[%p: %s]\n", s, s->mod->name);
-	s->state = SANDBOX_RUNNABLE;
+//	s->state = SANDBOX_RUNNABLE;
 	sandbox_deque_push(s);
 #else
 	sandbox_switch(s);
@@ -269,7 +277,8 @@ runtime_accept_thdfn(void *d)
 			}
 			nreqs++;
 
-			struct sandbox *sb = sandbox_alloc(m, m->name, s, (const struct sockaddr *)&client);
+			//struct sandbox *sb = sandbox_alloc(m, m->name, s, (const struct sockaddr *)&client);
+			sbox_request_t *sb = sbox_request_alloc(m, m->name, s, (const struct sockaddr *)&client);
 			assert(sb);
 		}
 	}
diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c
index 89c9751..078a27b 100644
--- a/runtime/src/sandbox.c
+++ b/runtime/src/sandbox.c
@@ -312,7 +312,9 @@ sandbox_alloc(struct module *mod, char *args, int sock, const struct sockaddr *a
 	ps_list_init_d(sb);
 
 	arch_context_init(&sb->ctxt, (reg_t)sandbox_entry, (reg_t)(sb->stack_start + sb->stack_size));
+#ifdef STANDALONE
 	sandbox_run(sb);
+#endif
 
 	return sb;
 }