From 645d138787c3d6133950be24a1f7610e9d03000c Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 4 Jan 2020 13:18:19 -0500 Subject: [PATCH 1/8] for runtime with multi-sandboxing, globals are to be populated per sandbox, that was what causing random crashes! --- runtime/compiletime/memory/64bit_nix.c | 36 ++++++++++++++++++++++++ runtime/include/module.h | 17 ++++++++++- runtime/include/runtime.h | 2 +- runtime/include/sandbox.h | 9 ++++++ runtime/include/types.h | 6 ++-- runtime/src/libc/syscall.c | 37 ++++++++++++++++++++++-- runtime/src/libc/uvio.c | 39 +++++++++++++++++++++++--- runtime/src/module.c | 8 ++++-- runtime/src/runtime.c | 2 +- runtime/src/sandbox.c | 7 +++-- runtime/tests/Makefile | 4 ++- 11 files changed, 148 insertions(+), 19 deletions(-) diff --git a/runtime/compiletime/memory/64bit_nix.c b/runtime/compiletime/memory/64bit_nix.c index b63ebe1..9e1ab28 100644 --- a/runtime/compiletime/memory/64bit_nix.c +++ b/runtime/compiletime/memory/64bit_nix.c @@ -58,6 +58,24 @@ get_i64(i32 offset) return *(i64 *)address; } +INLINE i32 +get_global_i32(i32 offset) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + return *(i32 *)address; +} + +INLINE i64 +get_global_i64(i32 offset) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + return *(i64 *)address; +} + // Now setting routines INLINE void set_f32(i32 offset, float v) @@ -113,6 +131,24 @@ set_i64(i32 offset, i64 v) *(i64 *)address = v; } +INLINE void +set_global_i32(i32 offset, i32 v) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + *(i32 *)address = v; +} + +INLINE void +set_global_i64(i32 offset, i64 v) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + *(i64 *)address = v; +} + // Table handling functionality INLINE char * get_function_from_table(u32 idx, u32 type_id) diff --git a/runtime/include/module.h b/runtime/include/module.h index 6764967..751e732 100644 --- a/runtime/include/module.h +++ b/runtime/include/module.h @@ -13,6 +13,7 @@ struct module { mod_glb_fn_t glb_init_fn; mod_mem_fn_t mem_init_fn; mod_tbl_fn_t tbl_init_fn; + mod_libc_fn_t libc_init_fn; struct indirect_table_entry indirect_table[INDIRECT_TABLE_SIZE]; @@ -71,12 +72,26 @@ module_is_valid(struct module *mod) } static inline void -module_table_init(struct module *mod) +module_globals_init(struct module *mod) { // called in a sandbox. + mod->glb_init_fn(); +} + +static inline void +module_table_init(struct module *mod) +{ + // called at module creation time (once only per module). mod->tbl_init_fn(); } +static inline void +module_libc_init(struct module *mod, i32 env, i32 args) +{ + // called in a sandbox. + mod->libc_init_fn(env, args); +} + static inline void module_memory_init(struct module *mod) { diff --git a/runtime/include/runtime.h b/runtime/include/runtime.h index 9161de8..bd15555 100644 --- a/runtime/include/runtime.h +++ b/runtime/include/runtime.h @@ -42,7 +42,7 @@ get_memory_string(u32 offset) INLINE char *get_function_from_table(u32 idx, u32 type_id); // libc/* might need to do some setup for the libc setup -void stub_init(char *modulename, i32 offset, mod_init_libc_fn_t fn); +void stub_init(i32 offset); void runtime_init(void); void runtime_thd_init(void); diff --git a/runtime/include/sandbox.h b/runtime/include/sandbox.h index dcf4383..e4b99c0 100644 --- a/runtime/include/sandbox.h +++ b/runtime/include/sandbox.h @@ -144,6 +144,15 @@ sandbox_current_set(struct sandbox *sbox) module_indirect_table = sbox->mod->indirect_table; } +static inline void +sandbox_current_check(void) +{ + struct sandbox *c = sandbox_current(); + + assert(c && c->linear_start == sandbox_lmbase && c->linear_size == sandbox_lmbound); + assert(c->mod->indirect_table == module_indirect_table); +} + static inline struct module * sandbox_module(struct sandbox *s) { diff --git a/runtime/include/types.h b/runtime/include/types.h index b71f4d5..b2e34a6 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -71,7 +71,7 @@ void populate_table(void); // memory/* also provides the table access functions // TODO: Change this to use a compiled in size -#define INDIRECT_TABLE_SIZE 1024 +#define INDIRECT_TABLE_SIZE (1<<10) struct indirect_table_entry { u32 type_id; @@ -90,7 +90,7 @@ typedef i32 (*mod_main_fn_t)(i32 a, i32 b); typedef void (*mod_glb_fn_t)(void); typedef void (*mod_mem_fn_t)(void); typedef void (*mod_tbl_fn_t)(void); -typedef void (*mod_init_libc_fn_t)(i32, i32); +typedef void (*mod_libc_fn_t)(i32, i32); typedef enum { MOD_ARG_MODPATH = 0, @@ -104,7 +104,7 @@ typedef enum { #define MOD_GLB_FN "populate_globals" #define MOD_MEM_FN "populate_memory" #define MOD_TBL_FN "populate_table" -#define MOD_INIT_LIBC_FN "wasmf___init_libc" +#define MOD_LIBC_FN "wasmf___init_libc" #define MOD_MAX_ARGS 16 #define MOD_ARG_MAX_SZ 64 diff --git a/runtime/src/libc/syscall.c b/runtime/src/libc/syscall.c index b337091..0f39af0 100644 --- a/runtime/src/libc/syscall.c +++ b/runtime/src/libc/syscall.c @@ -2,6 +2,7 @@ /* code from https://github.com/gwsystems/silverfish/blob/master/runtime/libc/libc_backing.c */ #include +#include #include #include @@ -32,9 +33,41 @@ // offset = a WASM ptr to memory the runtime can use void -stub_init(char *program_name, i32 offset, mod_init_libc_fn_t libcfn) +stub_init(i32 offset) { - printf("Don't think we should reinit libc! so ignore for now!\n"); + // What program name will we put in the auxiliary vectors + char *program_name = sandbox_current()->mod->name; + // Copy the program name into WASM accessible memory + i32 program_name_offset = offset; + strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name); + offset += sizeof(program_name); + + // The construction of this is: + // evn1, env2, ..., NULL, auxv_n1, auxv_1, auxv_n2, auxv_2 ..., NULL + i32 env_vec[] = { + // Env variables would live here, but we don't supply any + 0, + // We supply only the bare minimum AUX vectors + AT_PAGESZ, + WASM_PAGE_SIZE, + AT_UID, + UID, + AT_EUID, + UID, + AT_GID, + GID, + AT_EGID, + GID, + AT_SECURE, + 0, + AT_RANDOM, + (i32) rand(), // It's pretty stupid to use rand here, but w/e + 0, + }; + i32 env_vec_offset = offset; + memcpy(get_memory_ptr_for_runtime(env_vec_offset, sizeof(env_vec)), env_vec, sizeof(env_vec)); + + module_libc_init(sandbox_current()->mod, env_vec_offset, program_name_offset); } // Emulated syscall implementations diff --git a/runtime/src/libc/uvio.c b/runtime/src/libc/uvio.c index e615722..c8e6132 100644 --- a/runtime/src/libc/uvio.c +++ b/runtime/src/libc/uvio.c @@ -1,5 +1,6 @@ #ifdef USE_UVIO #include +#include #include #include @@ -30,9 +31,41 @@ // offset = a WASM ptr to memory the runtime can use void -stub_init(char *program_name, i32 offset, mod_init_libc_fn_t libcfn) +stub_init(i32 offset) { - printf("Don't think we should reinit libc! so ignore for now!\n"); + // What program name will we put in the auxiliary vectors + char *program_name = sandbox_current()->mod->name; + // Copy the program name into WASM accessible memory + i32 program_name_offset = offset; + strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name); + offset += sizeof(program_name); + + // The construction of this is: + // evn1, env2, ..., NULL, auxv_n1, auxv_1, auxv_n2, auxv_2 ..., NULL + i32 env_vec[] = { + // Env variables would live here, but we don't supply any + 0, + // We supply only the bare minimum AUX vectors + AT_PAGESZ, + WASM_PAGE_SIZE, + AT_UID, + UID, + AT_EUID, + UID, + AT_GID, + GID, + AT_EGID, + GID, + AT_SECURE, + 0, + AT_RANDOM, + (i32) rand(), // It's pretty stupid to use rand here, but w/e + 0, + }; + i32 env_vec_offset = offset; + memcpy(get_memory_ptr_for_runtime(env_vec_offset, sizeof(env_vec)), env_vec, sizeof(env_vec)); + + module_libc_init(sandbox_current()->mod, env_vec_offset, program_name_offset); } // Emulated syscall implementations @@ -120,13 +153,11 @@ wasm_write(i32 fd, i32 buf_offset, i32 buf_size) uv_fs_t req = UV_FS_REQ_INIT(); char* buf = get_memory_ptr_void(buf_offset, buf_size); - printf("[%p] start[%d:%d, n%d]\n", uv_fs_get_data(&req), fd, f, buf_size); uv_buf_t bufv = uv_buf_init(buf, buf_size); uv_fs_write(runtime_uvio(), &req, f, &bufv, 1, -1, wasm_fs_callback); sandbox_block(); int ret = uv_fs_get_result(&req); - printf("[%p] end[%d]\n", uv_fs_get_data(&req), ret); uv_fs_req_cleanup(&req); return ret; diff --git a/runtime/src/module.c b/runtime/src/module.c index 264f744..f0d5f4c 100644 --- a/runtime/src/module.c +++ b/runtime/src/module.c @@ -91,9 +91,8 @@ module_alloc(char *modname, char *modpath, i32 nargs, u32 stacksz, u32 maxheap, mod->entry_fn = (mod_main_fn_t)dlsym(mod->dl_handle, MOD_MAIN_FN); if (mod->entry_fn == NULL) goto dl_error; - // TODO: don't think this is necessary or implemented. - //mod->glb_init_fn = (mod_glb_fn_t)dlsym(mod->dl_handle, MOD_GLB_FN); - //if (mod->glb_init_fn == NULL) goto dl_error; + mod->glb_init_fn = (mod_glb_fn_t)dlsym(mod->dl_handle, MOD_GLB_FN); + if (mod->glb_init_fn == NULL) goto dl_error; mod->mem_init_fn = (mod_mem_fn_t)dlsym(mod->dl_handle, MOD_MEM_FN); if (mod->mem_init_fn == NULL) goto dl_error; @@ -101,6 +100,9 @@ module_alloc(char *modname, char *modpath, i32 nargs, u32 stacksz, u32 maxheap, mod->tbl_init_fn = (mod_tbl_fn_t)dlsym(mod->dl_handle, MOD_TBL_FN); if (mod->tbl_init_fn == NULL) goto dl_error; + mod->libc_init_fn = (mod_libc_fn_t)dlsym(mod->dl_handle, MOD_LIBC_FN); + if (mod->libc_init_fn == NULL) goto dl_error; + strncpy(mod->name, modname, MOD_NAME_MAX); strncpy(mod->path, modpath, MOD_PATH_MAX); diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c index 4d7f7e1..7ff4347 100644 --- a/runtime/src/runtime.c +++ b/runtime/src/runtime.c @@ -113,7 +113,7 @@ sandbox_local_free(unsigned int n) { int i = 0; - while (i < n) { + while (i < n && !ps_list_head_empty(&endq)) { i++; struct sandbox *s = ps_list_head_first_d(&endq, struct sandbox); if (!s) break; diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index 3f500ec..4502ed6 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -46,7 +46,6 @@ sandbox_args_setup(i32 argc) { struct sandbox *curr = sandbox_current(); char *args = sandbox_args(); - if (!args) return; // whatever gregor has, to be able to pass args to a module! curr->args_offset = sandbox_lmbound; @@ -66,6 +65,7 @@ sandbox_args_setup(i32 argc) string_off += str_sz; } + stub_init(string_off); } static inline void @@ -249,7 +249,6 @@ sandbox_entry(void) assert(f == 1); f = io_handle_open(2); assert(f == 2); - sandbox_args_setup(argc); #ifndef STANDALONE http_parser_init(&curr->hp, HTTP_REQUEST); @@ -266,10 +265,12 @@ sandbox_entry(void) { curr->rr_data_len = 0; // TODO: do this on first write to body. alloc_linear_memory(); - // perhaps only initialized for the first instance? or TODO! //module_table_init(curr_mod); + module_globals_init(curr_mod); module_memory_init(curr_mod); + sandbox_args_setup(argc); + curr->retval = module_entry(curr_mod, argc, curr->args_offset); sandbox_client_response_set(); diff --git a/runtime/tests/Makefile b/runtime/tests/Makefile index de00ce6..36e2104 100644 --- a/runtime/tests/Makefile +++ b/runtime/tests/Makefile @@ -2,7 +2,7 @@ include Makefile.inc BENCH_DIR=../../silverfish/code_benches/ -TESTS=fibonacci #empty work +TESTS=fibonacci empty work #TESTS=forever filesys sockserver sockclient empty TESTSRT=$(TESTS:%=%_rt) BENCHES=adpcm basic_math binarytrees bitcount blowfish crc dijkstra fft function_pointers \ @@ -38,6 +38,7 @@ clean: ${WASMCC} ${$(@:%_sf=%)_CFLAGS} ${WASMCFLAGS} ${OPTFLAGS} ${BENCH_DIR}/$(@:%_sf=%)/*.c $(DUMMY) -o ${TMP_DIR}/$(@:%_sf=%).wasm ${SFCC} ${TMP_DIR}/$(@:%_sf=%).wasm -o ${TMP_DIR}/$(@:%_sf=%).bc ${CC} ${CFLAGS} ${OPTFLAGS} -D${USE_MEM} ${TMP_DIR}/$(@:%_sf=%).bc ${MEMC} ${RT_LIBC} ${RT_RT} -o ${TMP_DIR}/$(@:%_sf=%)_wasm.out + ${SFCC} --inline-constant-globals --runtime-globals ${TMP_DIR}/$(@:%_sf=%).wasm -o ${TMP_DIR}/$(@:%_sf=%).bc ${CC} --shared -fPIC ${OPTFLAGS} -I${ART_INC} -D${USE_MEM} ${TMP_DIR}/$(@:%_sf=%).bc ${AMEMC} ${WASMISA} -o ${TMP_DIR}/$(@:%_sf=%)_wasm.so @cp ${TMP_DIR}/$(@:%_sf=%)_wasm.so ${ABIN_DIR} # @rm -rf ${TMP_DIR} @@ -48,6 +49,7 @@ clean: ${WASMCC} ${$(@:%_rt=%)_CFLAGS} ${WASMCFLAGS} ${OPTFLAGS} $(@:%_rt=%)/*.c $(DUMMY) -o ${TMP_DIR}/$(@:%_rt=%).wasm ${SFCC} ${TMP_DIR}/$(@:%_rt=%).wasm -o ${TMP_DIR}/$(@:%_rt=%).bc ${CC} ${CFLAGS} ${OPTFLAGS} -D${USE_MEM} ${TMP_DIR}/$(@:%_rt=%).bc ${MEMC} ${RT_LIBC} ${RT_RT} -o ${TMP_DIR}/$(@:%_rt=%)_wasm.out + ${SFCC} --inline-constant-globals --runtime-globals ${TMP_DIR}/$(@:%_rt=%).wasm -o ${TMP_DIR}/$(@:%_rt=%).bc ${CC} --shared -fPIC ${OPTFLAGS} -I${ART_INC} -D${USE_MEM} ${TMP_DIR}/$(@:%_rt=%).bc ${AMEMC} ${WASMISA} -o ${TMP_DIR}/$(@:%_rt=%)_wasm.so @cp ${TMP_DIR}/$(@:%_rt=%)_wasm.so ${ABIN_DIR} # @rm -rf ${TMP_DIR} From f8cc5edb895725d2c085285908c6c2a62f18f7f0 Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 6 Jan 2020 11:18:03 -0500 Subject: [PATCH 2/8] fibonacci serverless fn in awsm --- runtime/tests/fibonacci/main.c | 8 ++++---- runtime/tests/fibonacci/run_fib.sh | 2 +- runtime/tests/test_fibonacci.json | 13 +++++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 runtime/tests/test_fibonacci.json diff --git a/runtime/tests/fibonacci/main.c b/runtime/tests/fibonacci/main.c index b271cb5..7c27956 100644 --- a/runtime/tests/fibonacci/main.c +++ b/runtime/tests/fibonacci/main.c @@ -13,11 +13,11 @@ main(int argc, char **argv) { unsigned long n = 0, r; scanf("%lu", &n); - unsigned long long st = get_time(), en; +// unsigned long long st = get_time(), en; r = fib(n); - en = get_time(); - fprintf(stderr, "%lu\n", r); +// en = get_time(); + printf("%lu\n", r); - print_time(st, en); +// print_time(st, en); return 0; } diff --git a/runtime/tests/fibonacci/run_fib.sh b/runtime/tests/fibonacci/run_fib.sh index 3e9439b..4463b8a 100755 --- a/runtime/tests/fibonacci/run_fib.sh +++ b/runtime/tests/fibonacci/run_fib.sh @@ -25,7 +25,7 @@ MAXNUM=$2 tmp1_cnt=${MAXNUM} -while [ ${tmp1_cnt} -gt 0 ]; do +while [ ${tmp1_cnt} -gt 28 ]; do testeach ./fibonacci_$1.out ${tmp1_cnt} tmp1_cnt=$((tmp1_cnt - 1)) done diff --git a/runtime/tests/test_fibonacci.json b/runtime/tests/test_fibonacci.json new file mode 100644 index 0000000..725feee --- /dev/null +++ b/runtime/tests/test_fibonacci.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "fibonacci", + "path" : "fibonacci_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1024, + "http-resp-headers" : [ ], + "http-resp-size" : 1024, + "http-resp-content-type" : "text/plain" +} From 9528f65b322ad566dad34c78dbb67c29927c4839 Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 11 Jan 2020 14:32:45 -0500 Subject: [PATCH 3/8] Synchronous I/O for http request/response fixes 99%tile latencies * Async I/O is being used for reading and writing http request and response respectively. This causes the worker core to steal new tasks as when one sandbox blocks on "read" or "write". This causes true round-robin on timer interrupts, leading to those high latencies. * Ideally, the request/response should be synchronous I belive because we are expected to have "low"/near-realtime latencies and the read/writes are only performed when a client connects, so ideally not block! Plus, we expect only small data transfers (in KBs) so that should also support synchronous request/response processing. --- runtime/Makefile | 1 + runtime/include/http.h | 5 +++++ runtime/include/http_api.h | 6 +++--- runtime/include/sandbox.h | 5 ++++- runtime/include/types.h | 2 +- runtime/src/http.c | 24 +++++++++++++++++++++++- runtime/src/runtime.c | 32 ++++++++++++++++++++++++++------ runtime/src/sandbox.c | 35 +++++++++++++++++------------------ runtime/src/softint.c | 2 +- 9 files changed, 81 insertions(+), 31 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index f0816a8..c8d4de4 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,6 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO +#CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE diff --git a/runtime/include/http.h b/runtime/include/http.h index 04b9e4d..ac0fd09 100644 --- a/runtime/include/http.h +++ b/runtime/include/http.h @@ -4,6 +4,7 @@ #include #include #include +#include /* all in-memory ptrs.. don't mess around with that! */ struct http_header { @@ -34,7 +35,11 @@ struct http_response { int bodylen; char *status; int stlen; +#ifdef USE_HTTP_UVIO uv_buf_t bufs[HTTP_HEADERS_MAX * 2 + 3]; //max headers, one line for status code, remaining for body! +#else + struct iovec bufs[HTTP_HEADERS_MAX * 2 + 3]; +#endif }; #endif /* SFRT_HTTP_H */ diff --git a/runtime/include/http_api.h b/runtime/include/http_api.h index ba7c493..2e99a0d 100644 --- a/runtime/include/http_api.h +++ b/runtime/include/http_api.h @@ -8,7 +8,7 @@ int http_request_parse_sb(struct sandbox *s, size_t l); int http_response_header_set_sb(struct sandbox *s, char *h, int len); int http_response_body_set_sb(struct sandbox *s, char *body, int len); int http_response_status_set_sb(struct sandbox *s, char *status, int len); -int http_response_uv_sb(struct sandbox *s); +int http_response_vector_sb(struct sandbox *s); void http_init(void); @@ -37,9 +37,9 @@ http_response_status_set(char *status, int len) } static inline int -http_response_uv(void) +http_response_vector(void) { - return http_response_uv_sb(sandbox_current()); + return http_response_vector_sb(sandbox_current()); } static inline int diff --git a/runtime/include/sandbox.h b/runtime/include/sandbox.h index e4b99c0..5e348d2 100644 --- a/runtime/include/sandbox.h +++ b/runtime/include/sandbox.h @@ -193,9 +193,12 @@ sandbox_args(void) //void sandbox_run(struct sandbox *s); void *sandbox_run_func(void *data); -struct sandbox *sandbox_schedule(void); +struct sandbox *sandbox_schedule(int interrupt); void sandbox_block(void); void sandbox_wakeup(sandbox_t *sb); +// called in sandbox_entry() before and after fn() execution +// for http request/response processing using uvio +void sandbox_block_http(void); void sandbox_response(void); // should be the entry-point for each sandbox so it can do per-sandbox mem/etc init. diff --git a/runtime/include/types.h b/runtime/include/types.h index b2e34a6..27d5839 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -116,7 +116,7 @@ typedef enum { #define JSON_ELE_MAX 16 // FIXME: some naive work-stealing here.. -#define SBOX_PULL_MAX 16 +#define SBOX_PULL_MAX 1 #define SBOX_MAX_OPEN 32 #define SBOX_PREOPEN_MAGIC (707707707) // reads lol lol lol upside down diff --git a/runtime/src/http.c b/runtime/src/http.c index 17a3fb4..806de88 100644 --- a/runtime/src/http.c +++ b/runtime/src/http.c @@ -163,12 +163,13 @@ http_response_status_set_sb(struct sandbox *c, char *status, int len) } int -http_response_uv_sb(struct sandbox *c) +http_response_vector_sb(struct sandbox *c) { int nb = 0; #ifndef STANDALONE struct http_response *r = &c->rsi; +#ifdef USE_HTTP_UVIO r->bufs[nb] = uv_buf_init(r->status, r->stlen); nb++; @@ -183,6 +184,27 @@ http_response_uv_sb(struct sandbox *c) r->bufs[nb] = uv_buf_init(r->status + r->stlen - 2, 2); //for crlf nb++; } +#else + r->bufs[nb].iov_base = r->status; + r->bufs[nb].iov_len = r->stlen; + nb++; + + for (int i = 0; i < r->nheaders; i++) { + r->bufs[nb].iov_base = r->headers[i].hdr; + r->bufs[nb].iov_len = r->headers[i].len; + nb++; + } + + if (r->body) { + r->bufs[nb].iov_base = r->body; + r->bufs[nb].iov_len = r->bodylen; + nb++; + + r->bufs[nb].iov_base = r->status + r->stlen - 2; + r->bufs[nb].iov_len = 2; + nb++; + } +#endif #endif return nb; diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c index 7ff4347..4683eda 100644 --- a/runtime/src/runtime.c +++ b/runtime/src/runtime.c @@ -87,10 +87,12 @@ sandbox_io_nowait(void) } struct sandbox * -sandbox_schedule(void) +sandbox_schedule(int interrupt) { struct sandbox *s = NULL; if (ps_list_head_empty(&runq)) { + // this is in an interrupt context, don't steal work here! + if (interrupt) return NULL; if (sandbox_pull() == 0) { //debuglog("[null: null]\n"); return NULL; @@ -130,7 +132,7 @@ sandbox_schedule_io(void) if (!in_callback) sandbox_io_nowait(); softint_disable(); - struct sandbox *s = sandbox_schedule(); + struct sandbox *s = sandbox_schedule(0); softint_enable(); assert(s == NULL || s->state == SANDBOX_RUNNABLE); @@ -143,11 +145,12 @@ sandbox_wakeup(sandbox_t *s) #ifndef STANDALONE softint_disable(); debuglog("[%p: %s]\n", s, s->mod->name); - // perhaps 2 lists in the sandbox to make sure sandbox is either in runlist or waitlist.. + if (s->state != SANDBOX_BLOCKED) goto done; assert(s->state == SANDBOX_BLOCKED); assert(ps_list_singleton_d(s)); s->state = SANDBOX_RUNNABLE; ps_list_head_append_d(&runq, s); +done: softint_enable(); #endif } @@ -156,13 +159,12 @@ void sandbox_block(void) { #ifndef STANDALONE - // perhaps 2 lists in the sandbox to make sure sandbox is either in runlist or waitlist.. assert(in_callback == 0); softint_disable(); struct sandbox *c = sandbox_current(); ps_list_rem_d(c); c->state = SANDBOX_BLOCKED; - struct sandbox *s = sandbox_schedule(); + struct sandbox *s = sandbox_schedule(0); debuglog("[%p: %s, %p: %s]\n", c, c->mod->name, s, s ? s->mod->name: ""); softint_enable(); sandbox_switch(s); @@ -171,6 +173,24 @@ sandbox_block(void) #endif } +void +sandbox_block_http(void) +{ +#ifdef USE_HTTP_UVIO +#ifdef USE_HTTP_SYNC + // realistically, we're processing all async I/O on this core when a sandbox blocks on http processing, not great! + // if there is a way (TODO), perhaps RUN_ONCE and check if your I/O is processed, if yes, return + // else do async block! + uv_run(runtime_uvio(), UV_RUN_DEFAULT); +#else + sandbox_block(); +#endif +#else + assert(0); + //it should not be called if not using uvio for http +#endif +} + void __attribute__((noinline)) __attribute__((noreturn)) sandbox_switch_preempt(void) { @@ -248,7 +268,7 @@ sandbox_exit(void) sandbox_local_stop(curr); curr->state = SANDBOX_RETURNED; // free resources from "main function execution", as stack still in use. - struct sandbox *n = sandbox_schedule(); + struct sandbox *n = sandbox_schedule(0); assert(n != curr); softint_enable(); //sandbox_local_end(curr); diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index 4502ed6..b854d04 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -133,11 +133,11 @@ sandbox_client_request_get(void) struct sandbox *curr = sandbox_current(); curr->rr_data_len = 0; -#ifndef USE_UVIO +#ifndef USE_HTTP_UVIO int r = 0; r = recv(curr->csock, (curr->req_resp_data), curr->mod->max_req_sz, 0); if (r <= 0) { - perror("recv"); + if (r < 0) perror("recv1"); return r; } while (r > 0) { @@ -148,13 +148,13 @@ sandbox_client_request_get(void) r = recv(curr->csock, (curr->req_resp_data + r), curr->mod->max_req_sz - r, 0); if (r < 0) { - perror("recv"); + perror("recv2"); return r; } } #else int r = uv_read_start((uv_stream_t *)&curr->cuv, sb_alloc_callback, sb_read_callback); - sandbox_block(); + sandbox_block_http(); if (curr->rr_data_len == 0) return 0; #endif @@ -170,15 +170,6 @@ sandbox_client_response_set(void) #ifndef STANDALONE struct sandbox *curr = sandbox_current(); -#ifndef USE_UVIO - strcpy(curr->req_resp_data + curr->rr_data_len, "HTTP/1.1 200 OK\r\n"); - - // TODO: response set in req_resp_data - curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); - - int r = send(curr->csock, curr->req_resp_data, curr->rr_data_len, 0); - if (r < 0) perror("send"); -#else int bodylen = curr->rr_data_len; if (bodylen > 0) { http_response_body_set(curr->req_resp_data, bodylen); @@ -217,10 +208,14 @@ sandbox_client_response_set(void) curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); http_response_status_set(st, strlen("HTTP/1.1 200 OK\r\n")); + int n = http_response_vector(); +#ifndef USE_HTTP_UVIO + int r = writev(curr->csock, curr->rsi.bufs, n); + if (r < 0) perror("writev"); +#else uv_write_t req = { .data = curr, }; - int n = http_response_uv(); int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); - sandbox_block(); + sandbox_block_http(); #endif return r; #else @@ -253,7 +248,11 @@ sandbox_entry(void) #ifndef STANDALONE http_parser_init(&curr->hp, HTTP_REQUEST); curr->hp.data = curr; -#ifdef USE_UVIO +#ifdef USE_HTTP_UVIO +#ifndef USE_UVIO + printf("UVIO not enabled!\n"); + assert(0); +#endif int r = uv_tcp_init(runtime_uvio(), (uv_tcp_t *)&curr->cuv); assert(r == 0); curr->cuv.data = curr; @@ -277,9 +276,9 @@ sandbox_entry(void) } #ifndef STANDALONE -#ifdef USE_UVIO +#ifdef USE_HTTP_UVIO uv_close((uv_handle_t *)&curr->cuv, sb_close_callback); - sandbox_block(); + sandbox_block_http(); #else close(curr->csock); #endif diff --git a/runtime/src/softint.c b/runtime/src/softint.c index dff8011..bf3b50c 100644 --- a/runtime/src/softint.c +++ b/runtime/src/softint.c @@ -67,7 +67,7 @@ softint_alarm_schedule(void *u) if (curr == NULL) goto done; // find a next sandbox to run.. - struct sandbox *next = sandbox_schedule(); + struct sandbox *next = sandbox_schedule(1); if (next == NULL) goto done; if (next == curr) goto done; // only this sandbox to schedule.. return to it! // save the current sandbox, state from uc! From da43b22407c605e6f53ae3cedf5bf1998c6da372 Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 11 Jan 2020 16:40:54 -0500 Subject: [PATCH 4/8] work 1k to 1m functions --- runtime/src/main.c | 16 ++++++++++++++++ runtime/tests/Makefile | 2 +- runtime/tests/test_work100k.json | 13 +++++++++++++ runtime/tests/test_work10k.json | 13 +++++++++++++ runtime/tests/test_work1k.json | 13 +++++++++++++ runtime/tests/test_work1m.json | 13 +++++++++++++ runtime/tests/work/main.c | 19 ++----------------- runtime/tests/work100k/main.c | 24 ++++++++++++++++++++++++ runtime/tests/work10k/main.c | 24 ++++++++++++++++++++++++ runtime/tests/work1k/main.c | 24 ++++++++++++++++++++++++ runtime/tests/work1m/main.c | 24 ++++++++++++++++++++++++ 11 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 runtime/tests/test_work100k.json create mode 100644 runtime/tests/test_work10k.json create mode 100644 runtime/tests/test_work1k.json create mode 100644 runtime/tests/test_work1m.json create mode 100644 runtime/tests/work100k/main.c create mode 100644 runtime/tests/work10k/main.c create mode 100644 runtime/tests/work1k/main.c create mode 100644 runtime/tests/work1m/main.c diff --git a/runtime/src/main.c b/runtime/src/main.c index 2883205..2241d5f 100644 --- a/runtime/src/main.c +++ b/runtime/src/main.c @@ -20,6 +20,18 @@ u32 ncores = 0, sbox_ncores = 0, sbox_core_st = 0; pthread_t rtthd[SBOX_NCORES]; +static unsigned long long +get_time() +{ + struct timeval Tp; + int stat; + stat = gettimeofday (&Tp, NULL); + if (stat != 0) + printf ("Error return from gettimeofday: %d", stat); + return (Tp.tv_sec * 1000000 + Tp.tv_usec); +} + + static void usage(char *cmd) { @@ -141,7 +153,11 @@ main(int argc, char** argv) /* in current dir! */ struct module *m = module_alloc(args, args, ac, 0, 0, 0, 0, 0, 0); assert(m); + + //unsigned long long st = get_time(), en; struct sandbox *s = sandbox_alloc(m, args, 0, NULL); + //en = get_time(); + //fprintf(stderr, "%llu\n", en - st); exit(0); #endif diff --git a/runtime/tests/Makefile b/runtime/tests/Makefile index 36e2104..bcfa715 100644 --- a/runtime/tests/Makefile +++ b/runtime/tests/Makefile @@ -2,7 +2,7 @@ include Makefile.inc BENCH_DIR=../../silverfish/code_benches/ -TESTS=fibonacci empty work +TESTS=fibonacci empty work work1k work10k work100k work1m #TESTS=forever filesys sockserver sockclient empty TESTSRT=$(TESTS:%=%_rt) BENCHES=adpcm basic_math binarytrees bitcount blowfish crc dijkstra fft function_pointers \ diff --git a/runtime/tests/test_work100k.json b/runtime/tests/test_work100k.json new file mode 100644 index 0000000..a1d9d29 --- /dev/null +++ b/runtime/tests/test_work100k.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work100k", + "path" : "work100k_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 102600, + "http-resp-headers" : [ ], + "http-resp-size" : 102600, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/test_work10k.json b/runtime/tests/test_work10k.json new file mode 100644 index 0000000..fc0a0fd --- /dev/null +++ b/runtime/tests/test_work10k.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work10k", + "path" : "work10k_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 10480, + "http-resp-headers" : [ ], + "http-resp-size" : 10480, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/test_work1k.json b/runtime/tests/test_work1k.json new file mode 100644 index 0000000..c332049 --- /dev/null +++ b/runtime/tests/test_work1k.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work1k", + "path" : "work1k_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1200, + "http-resp-headers" : [ ], + "http-resp-size" : 1200, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/test_work1m.json b/runtime/tests/test_work1m.json new file mode 100644 index 0000000..4cdffde --- /dev/null +++ b/runtime/tests/test_work1m.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work1m", + "path" : "work1m_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1048576, + "http-resp-headers" : [ ], + "http-resp-size" : 1048776, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/work/main.c b/runtime/tests/work/main.c index 2694657..3ad7416 100644 --- a/runtime/tests/work/main.c +++ b/runtime/tests/work/main.c @@ -2,24 +2,9 @@ #include #include -#define CPU_CYCS 2100 +#ifndef MAX_BUF #define MAX_BUF (1024*1024*1) //1m - -#define ITERS_15US 125500 -#define MULTIPLE 5 - -#define SPIN_ITERS (ITERS_15US*MULTIPLE) - -__attribute__((optnone)) static void -wrk(void) -{ - unsigned int spin = 0; - - while (spin < SPIN_ITERS) { - //__asm__ __volatile__("nop": : :"memory"); - spin++; - } -} +#endif //__attribute__((optnone)) int int diff --git a/runtime/tests/work100k/main.c b/runtime/tests/work100k/main.c new file mode 100644 index 0000000..57a483f --- /dev/null +++ b/runtime/tests/work100k/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF 102400 + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} diff --git a/runtime/tests/work10k/main.c b/runtime/tests/work10k/main.c new file mode 100644 index 0000000..d034c1e --- /dev/null +++ b/runtime/tests/work10k/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF 10240 + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} diff --git a/runtime/tests/work1k/main.c b/runtime/tests/work1k/main.c new file mode 100644 index 0000000..eb44340 --- /dev/null +++ b/runtime/tests/work1k/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF 1024 + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} diff --git a/runtime/tests/work1m/main.c b/runtime/tests/work1m/main.c new file mode 100644 index 0000000..b7f2273 --- /dev/null +++ b/runtime/tests/work1m/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF (1024*1024*1) //1m + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} From b9d7278c54309df7047d1f7b78addb794cc6e12d Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 11 Jan 2020 19:05:04 -0500 Subject: [PATCH 5/8] fixes 99% latency problem here (but a problem with 1m data transfer test, it is broken.) --- runtime/Makefile | 2 +- runtime/src/runtime.c | 3 +++ runtime/src/sandbox.c | 12 ++++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index c8d4de4..c3759ad 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,7 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO -#CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC +CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c index 4683eda..7e79be2 100644 --- a/runtime/src/runtime.c +++ b/runtime/src/runtime.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -271,6 +272,8 @@ sandbox_exit(void) struct sandbox *n = sandbox_schedule(0); assert(n != curr); softint_enable(); + //unmap linear memory only! + munmap(curr->linear_start, SBOX_MAX_MEM + PAGE_SIZE); //sandbox_local_end(curr); sandbox_switch(n); #else diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index b854d04..f677478 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -21,7 +21,7 @@ sandbox_memory_map(struct module *m) if (lm_sz + sb_sz > mem_sz) return NULL; assert(round_up_to_page(sb_sz) == sb_sz); unsigned long rw_sz = sb_sz + lm_sz; - void *addr = mmap(NULL, mem_sz + /* guard page */ PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void *addr = mmap(NULL, sb_sz + mem_sz + /* guard page */ PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) return NULL; void *addr_rw = mmap(addr, sb_sz + lm_sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); @@ -217,7 +217,7 @@ sandbox_client_response_set(void) int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); sandbox_block_http(); #endif - return r; + return 0; #else return 0; #endif @@ -335,6 +335,10 @@ sandbox_free(struct sandbox *sb) // TODO: this needs to be enhanced. you may be killing a sandbox when its in any other execution states. if (sb->state != SANDBOX_RETURNED) return; + int sz = sizeof(struct sandbox); +#ifndef STANDALONE + sz += sb->mod->max_rr_sz; +#endif module_release(sb->mod); // TODO free(sb->args); @@ -342,10 +346,10 @@ sandbox_free(struct sandbox *sb) size_t stksz = sb->stack_size; // depending on the memory type - free_linear_memory(sb->linear_start, sb->linear_size, sb->linear_max_size); + //free_linear_memory(sb->linear_start, sb->linear_size, sb->linear_max_size); // mmaped memory includes sandbox structure in there. - ret = munmap(sb, SBOX_MAX_MEM + PAGE_SIZE); + ret = munmap(sb, sz); if (ret) perror("munmap sandbox"); // remove stack! From 0c92419978e6a7818c7659800f6bca84ac9705e6 Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 13 Jan 2020 12:11:08 -0500 Subject: [PATCH 6/8] fixed syscall based sync http-req/resp --- runtime/Makefile | 2 +- runtime/include/types.h | 5 ++ runtime/src/sandbox.c | 151 ++++++++++++++++++++++++++++------------ 3 files changed, 113 insertions(+), 45 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index c3759ad..06a36ea 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,7 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO -CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC +#CFLAGS += -DUSE_HTTP_UVIO #-DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE diff --git a/runtime/include/types.h b/runtime/include/types.h index 27d5839..59f8361 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -157,4 +157,9 @@ typedef enum { #define HTTP_HEADER_MAXSZ 32 #define HTTP_HEADERVAL_MAXSZ 64 +#define HTTP_RESP_200OK "HTTP/1.1 200 OK\r\n" +#define HTTP_RESP_CONTTYPE "Content-type: \r\n" +#define HTTP_RESP_CONTLEN "Content-length: \r\n\r\n" //content body follows this +#define HTTP_RESP_CONTTYPE_PLAIN "text/plain" + #endif /* SFRT_TYPES_H */ diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index f677478..bd798e8 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -146,7 +146,7 @@ sandbox_client_request_get(void) struct http_request *rh = &curr->rqi; if (rh->message_end) break; - r = recv(curr->csock, (curr->req_resp_data + r), curr->mod->max_req_sz - r, 0); + r = recv(curr->csock, (curr->req_resp_data + curr->rr_data_len), curr->mod->max_req_sz - curr->rr_data_len, 0); if (r < 0) { perror("recv2"); return r; @@ -168,61 +168,118 @@ static inline int sandbox_client_response_set(void) { #ifndef STANDALONE + int sndsz = 0; struct sandbox *curr = sandbox_current(); - - int bodylen = curr->rr_data_len; - if (bodylen > 0) { - http_response_body_set(curr->req_resp_data, bodylen); - char len[16] = { 0 }; - sprintf(len, "%d", bodylen); - //content-length = body length - char *key = curr->req_resp_data + curr->rr_data_len; - int lenlen = strlen("content-length: "), dlen = strlen(len); - strcpy(key, "content-length: "); - strncat(key + lenlen, len, dlen); - strncat(key + lenlen + dlen, "\r\n", 2); - http_response_header_set(key, lenlen + dlen + 2); - curr->rr_data_len += lenlen + dlen + 2; - - //content-type as set in the headers. - key = curr->req_resp_data + curr->rr_data_len; - strcpy(key, "content-type: "); - lenlen = strlen("content-type: "); - dlen = strlen(curr->mod->rspctype); - if (dlen == 0) { - int l = strlen("text/plain\r\n\r\n"); - strncat(key + lenlen, "text/plain\r\n\r\n", l); - http_response_header_set(key, lenlen + l); - curr->rr_data_len += lenlen + l; - } else { - strncat(key + lenlen, curr->mod->rspctype, dlen); - strncat(key + lenlen + dlen, "\r\n\r\n", 4); - http_response_header_set(key, lenlen + dlen + 4); - curr->rr_data_len += lenlen + dlen + 4; - } - //TODO - other headers requested in module! + int rsp_hdr_len = strlen(HTTP_RESP_200OK) + strlen(HTTP_RESP_CONTTYPE) + strlen(HTTP_RESP_CONTLEN); + int bodylen = curr->rr_data_len - rsp_hdr_len; + + memset(curr->req_resp_data, 0, strlen(HTTP_RESP_200OK) + strlen(HTTP_RESP_CONTTYPE) + strlen(HTTP_RESP_CONTLEN)); + strncpy(curr->req_resp_data, HTTP_RESP_200OK, strlen(HTTP_RESP_200OK)); + sndsz += strlen(HTTP_RESP_200OK); + + if (bodylen == 0) goto done; + strncpy(curr->req_resp_data + sndsz, HTTP_RESP_CONTTYPE, strlen(HTTP_RESP_CONTTYPE)); + if (strlen(curr->mod->rspctype) <= 0) { + strncpy(curr->req_resp_data + sndsz + strlen("Content-type: "), HTTP_RESP_CONTTYPE_PLAIN, strlen(HTTP_RESP_CONTTYPE_PLAIN)); + } else { + strncpy(curr->req_resp_data + sndsz + strlen("Content-type: "), curr->mod->rspctype, strlen(curr->mod->rspctype)); } + sndsz += strlen(HTTP_RESP_CONTTYPE); + char len[10] = { 0 }; + sprintf(len, "%d", bodylen); + strncpy(curr->req_resp_data + sndsz, HTTP_RESP_CONTLEN, strlen(HTTP_RESP_CONTLEN)); + strncpy(curr->req_resp_data + sndsz + strlen("Content-length: "), len, strlen(len)); + sndsz += strlen(HTTP_RESP_CONTLEN); + sndsz += bodylen; - char *st = curr->req_resp_data + curr->rr_data_len; - strcpy(st, "HTTP/1.1 200 OK\r\n"); - curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); +done: + assert(sndsz == curr->rr_data_len); - http_response_status_set(st, strlen("HTTP/1.1 200 OK\r\n")); - int n = http_response_vector(); #ifndef USE_HTTP_UVIO - int r = writev(curr->csock, curr->rsi.bufs, n); - if (r < 0) perror("writev"); + int r = send(curr->csock, curr->req_resp_data, sndsz, 0); + if (r < 0) { + perror("send"); + return -1; + } + while (r < sndsz) { + int s = send(curr->csock, curr->req_resp_data + r, sndsz - r, 0); + if (s < 0) { + perror("send"); + return -1; + } + r += s; + } #else uv_write_t req = { .data = curr, }; - int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); + uv_buf_t bufv = uv_buf_init(curr->req_resp_data, sndsz); + int r = uv_write(&req, (uv_stream_t *)&curr->cuv, &bufv, 1, sb_write_callback); sandbox_block_http(); #endif - return 0; -#else + + return 0; #endif } +//static inline int +//sandbox_client_response_set(void) +//{ +//#ifndef STANDALONE +// struct sandbox *curr = sandbox_current(); +// +// int bodylen = curr->rr_data_len; +// if (bodylen > 0) { +// http_response_body_set(curr->req_resp_data, bodylen); +// char len[16] = { 0 }; +// sprintf(len, "%d", bodylen); +// //content-length = body length +// char *key = curr->req_resp_data + curr->rr_data_len; +// int lenlen = strlen("content-length: "), dlen = strlen(len); +// strcpy(key, "content-length: "); +// strncat(key + lenlen, len, dlen); +// strncat(key + lenlen + dlen, "\r\n", 2); +// http_response_header_set(key, lenlen + dlen + 2); +// curr->rr_data_len += lenlen + dlen + 2; +// +// //content-type as set in the headers. +// key = curr->req_resp_data + curr->rr_data_len; +// strcpy(key, "content-type: "); +// lenlen = strlen("content-type: "); +// dlen = strlen(curr->mod->rspctype); +// if (dlen == 0) { +// int l = strlen("text/plain\r\n\r\n"); +// strncat(key + lenlen, "text/plain\r\n\r\n", l); +// http_response_header_set(key, lenlen + l); +// curr->rr_data_len += lenlen + l; +// } else { +// strncat(key + lenlen, curr->mod->rspctype, dlen); +// strncat(key + lenlen + dlen, "\r\n\r\n", 4); +// http_response_header_set(key, lenlen + dlen + 4); +// curr->rr_data_len += lenlen + dlen + 4; +// } +// //TODO - other headers requested in module! +// } +// +// char *st = curr->req_resp_data + curr->rr_data_len; +// strcpy(st, "HTTP/1.1 200 OK\r\n"); +// curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); +// +// http_response_status_set(st, strlen("HTTP/1.1 200 OK\r\n")); +// int n = http_response_vector(); +//#ifndef USE_HTTP_UVIO +// int r = writev(curr->csock, curr->rsi.bufs, n); +// if (r < 0) perror("writev"); +//#else +// uv_write_t req = { .data = curr, }; +// int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); +// sandbox_block_http(); +//#endif +// return 0; +//#else +// return 0; +//#endif +//} + void sandbox_entry(void) { @@ -248,6 +305,8 @@ sandbox_entry(void) #ifndef STANDALONE http_parser_init(&curr->hp, HTTP_REQUEST); curr->hp.data = curr; + // NOTE: if more headers, do offset by that! + int rsp_hdr_len = strlen(HTTP_RESP_200OK) + strlen(HTTP_RESP_CONTTYPE) + strlen(HTTP_RESP_CONTLEN); #ifdef USE_HTTP_UVIO #ifndef USE_UVIO printf("UVIO not enabled!\n"); @@ -262,7 +321,11 @@ sandbox_entry(void) if (sandbox_client_request_get() > 0) #endif { - curr->rr_data_len = 0; // TODO: do this on first write to body. +#ifndef STANDALONE + curr->rr_data_len = rsp_hdr_len; // TODO: do this on first write to body. +#else + curr->rr_data_len = 0; +#endif alloc_linear_memory(); // perhaps only initialized for the first instance? or TODO! //module_table_init(curr_mod); From 77ae44337ccc70c6009167709b4a4f13b1d7abbc Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 13 Jan 2020 13:31:30 -0500 Subject: [PATCH 7/8] buffer problem in work1m tests --- runtime/tests/test_work.json | 2 +- runtime/tests/test_work1m.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/tests/test_work.json b/runtime/tests/test_work.json index ee7d258..8400485 100644 --- a/runtime/tests/test_work.json +++ b/runtime/tests/test_work.json @@ -6,7 +6,7 @@ "argsize" : 1, "http-req-headers" : [ ], "http-req-content-type" : "text/plain", - "http-req-size": 1048576, + "http-req-size": 1048776, "http-resp-headers" : [ ], "http-resp-size" : 1048776, "http-resp-content-type" : "text/plain" diff --git a/runtime/tests/test_work1m.json b/runtime/tests/test_work1m.json index 4cdffde..ef8804d 100644 --- a/runtime/tests/test_work1m.json +++ b/runtime/tests/test_work1m.json @@ -6,7 +6,7 @@ "argsize" : 1, "http-req-headers" : [ ], "http-req-content-type" : "text/plain", - "http-req-size": 1048576, + "http-req-size": 1048776, "http-resp-headers" : [ ], "http-resp-size" : 1048776, "http-resp-content-type" : "text/plain" From ed234f9ac5aeb1c9e7fb52393dabd55dfd9afb03 Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 13 Jan 2020 13:38:52 -0500 Subject: [PATCH 8/8] default: async io for http req/resp. sync based io doesn't seem to work with huge data like 500k-1m I tested. They just timeout with concurrency --- runtime/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Makefile b/runtime/Makefile index 06a36ea..1c5bf46 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,7 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO -#CFLAGS += -DUSE_HTTP_UVIO #-DUSE_HTTP_SYNC +CFLAGS += -DUSE_HTTP_UVIO #-DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE