From 645d138787c3d6133950be24a1f7610e9d03000c Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 4 Jan 2020 13:18:19 -0500 Subject: [PATCH 1/8] for runtime with multi-sandboxing, globals are to be populated per sandbox, that was what causing random crashes! --- runtime/compiletime/memory/64bit_nix.c | 36 ++++++++++++++++++++++++ runtime/include/module.h | 17 ++++++++++- runtime/include/runtime.h | 2 +- runtime/include/sandbox.h | 9 ++++++ runtime/include/types.h | 6 ++-- runtime/src/libc/syscall.c | 37 ++++++++++++++++++++++-- runtime/src/libc/uvio.c | 39 +++++++++++++++++++++++--- runtime/src/module.c | 8 ++++-- runtime/src/runtime.c | 2 +- runtime/src/sandbox.c | 7 +++-- runtime/tests/Makefile | 4 ++- 11 files changed, 148 insertions(+), 19 deletions(-) diff --git a/runtime/compiletime/memory/64bit_nix.c b/runtime/compiletime/memory/64bit_nix.c index b63ebe1a..9e1ab281 100644 --- a/runtime/compiletime/memory/64bit_nix.c +++ b/runtime/compiletime/memory/64bit_nix.c @@ -58,6 +58,24 @@ get_i64(i32 offset) return *(i64 *)address; } +INLINE i32 +get_global_i32(i32 offset) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + return *(i32 *)address; +} + +INLINE i64 +get_global_i64(i32 offset) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + return *(i64 *)address; +} + // Now setting routines INLINE void set_f32(i32 offset, float v) @@ -113,6 +131,24 @@ set_i64(i32 offset, i64 v) *(i64 *)address = v; } +INLINE void +set_global_i32(i32 offset, i32 v) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + *(i32 *)address = v; +} + +INLINE void +set_global_i64(i32 offset, i64 v) +{ + char *mem_as_chars = (char *)sandbox_lmbase; + void *address = &mem_as_chars[offset]; + + *(i64 *)address = v; +} + // Table handling functionality INLINE char * get_function_from_table(u32 idx, u32 type_id) diff --git a/runtime/include/module.h b/runtime/include/module.h index 67649671..751e732a 100644 --- a/runtime/include/module.h +++ b/runtime/include/module.h @@ -13,6 +13,7 @@ struct module { mod_glb_fn_t glb_init_fn; mod_mem_fn_t mem_init_fn; mod_tbl_fn_t tbl_init_fn; + mod_libc_fn_t libc_init_fn; struct indirect_table_entry indirect_table[INDIRECT_TABLE_SIZE]; @@ -71,12 +72,26 @@ module_is_valid(struct module *mod) } static inline void -module_table_init(struct module *mod) +module_globals_init(struct module *mod) { // called in a sandbox. + mod->glb_init_fn(); +} + +static inline void +module_table_init(struct module *mod) +{ + // called at module creation time (once only per module). mod->tbl_init_fn(); } +static inline void +module_libc_init(struct module *mod, i32 env, i32 args) +{ + // called in a sandbox. + mod->libc_init_fn(env, args); +} + static inline void module_memory_init(struct module *mod) { diff --git a/runtime/include/runtime.h b/runtime/include/runtime.h index 9161de8a..bd155550 100644 --- a/runtime/include/runtime.h +++ b/runtime/include/runtime.h @@ -42,7 +42,7 @@ get_memory_string(u32 offset) INLINE char *get_function_from_table(u32 idx, u32 type_id); // libc/* might need to do some setup for the libc setup -void stub_init(char *modulename, i32 offset, mod_init_libc_fn_t fn); +void stub_init(i32 offset); void runtime_init(void); void runtime_thd_init(void); diff --git a/runtime/include/sandbox.h b/runtime/include/sandbox.h index dcf43834..e4b99c0a 100644 --- a/runtime/include/sandbox.h +++ b/runtime/include/sandbox.h @@ -144,6 +144,15 @@ sandbox_current_set(struct sandbox *sbox) module_indirect_table = sbox->mod->indirect_table; } +static inline void +sandbox_current_check(void) +{ + struct sandbox *c = sandbox_current(); + + assert(c && c->linear_start == sandbox_lmbase && c->linear_size == sandbox_lmbound); + assert(c->mod->indirect_table == module_indirect_table); +} + static inline struct module * sandbox_module(struct sandbox *s) { diff --git a/runtime/include/types.h b/runtime/include/types.h index b71f4d59..b2e34a61 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -71,7 +71,7 @@ void populate_table(void); // memory/* also provides the table access functions // TODO: Change this to use a compiled in size -#define INDIRECT_TABLE_SIZE 1024 +#define INDIRECT_TABLE_SIZE (1<<10) struct indirect_table_entry { u32 type_id; @@ -90,7 +90,7 @@ typedef i32 (*mod_main_fn_t)(i32 a, i32 b); typedef void (*mod_glb_fn_t)(void); typedef void (*mod_mem_fn_t)(void); typedef void (*mod_tbl_fn_t)(void); -typedef void (*mod_init_libc_fn_t)(i32, i32); +typedef void (*mod_libc_fn_t)(i32, i32); typedef enum { MOD_ARG_MODPATH = 0, @@ -104,7 +104,7 @@ typedef enum { #define MOD_GLB_FN "populate_globals" #define MOD_MEM_FN "populate_memory" #define MOD_TBL_FN "populate_table" -#define MOD_INIT_LIBC_FN "wasmf___init_libc" +#define MOD_LIBC_FN "wasmf___init_libc" #define MOD_MAX_ARGS 16 #define MOD_ARG_MAX_SZ 64 diff --git a/runtime/src/libc/syscall.c b/runtime/src/libc/syscall.c index b337091a..0f39af03 100644 --- a/runtime/src/libc/syscall.c +++ b/runtime/src/libc/syscall.c @@ -2,6 +2,7 @@ /* code from https://github.com/gwsystems/silverfish/blob/master/runtime/libc/libc_backing.c */ #include +#include #include #include @@ -32,9 +33,41 @@ // offset = a WASM ptr to memory the runtime can use void -stub_init(char *program_name, i32 offset, mod_init_libc_fn_t libcfn) +stub_init(i32 offset) { - printf("Don't think we should reinit libc! so ignore for now!\n"); + // What program name will we put in the auxiliary vectors + char *program_name = sandbox_current()->mod->name; + // Copy the program name into WASM accessible memory + i32 program_name_offset = offset; + strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name); + offset += sizeof(program_name); + + // The construction of this is: + // evn1, env2, ..., NULL, auxv_n1, auxv_1, auxv_n2, auxv_2 ..., NULL + i32 env_vec[] = { + // Env variables would live here, but we don't supply any + 0, + // We supply only the bare minimum AUX vectors + AT_PAGESZ, + WASM_PAGE_SIZE, + AT_UID, + UID, + AT_EUID, + UID, + AT_GID, + GID, + AT_EGID, + GID, + AT_SECURE, + 0, + AT_RANDOM, + (i32) rand(), // It's pretty stupid to use rand here, but w/e + 0, + }; + i32 env_vec_offset = offset; + memcpy(get_memory_ptr_for_runtime(env_vec_offset, sizeof(env_vec)), env_vec, sizeof(env_vec)); + + module_libc_init(sandbox_current()->mod, env_vec_offset, program_name_offset); } // Emulated syscall implementations diff --git a/runtime/src/libc/uvio.c b/runtime/src/libc/uvio.c index e6157228..c8e61327 100644 --- a/runtime/src/libc/uvio.c +++ b/runtime/src/libc/uvio.c @@ -1,5 +1,6 @@ #ifdef USE_UVIO #include +#include #include #include @@ -30,9 +31,41 @@ // offset = a WASM ptr to memory the runtime can use void -stub_init(char *program_name, i32 offset, mod_init_libc_fn_t libcfn) +stub_init(i32 offset) { - printf("Don't think we should reinit libc! so ignore for now!\n"); + // What program name will we put in the auxiliary vectors + char *program_name = sandbox_current()->mod->name; + // Copy the program name into WASM accessible memory + i32 program_name_offset = offset; + strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name); + offset += sizeof(program_name); + + // The construction of this is: + // evn1, env2, ..., NULL, auxv_n1, auxv_1, auxv_n2, auxv_2 ..., NULL + i32 env_vec[] = { + // Env variables would live here, but we don't supply any + 0, + // We supply only the bare minimum AUX vectors + AT_PAGESZ, + WASM_PAGE_SIZE, + AT_UID, + UID, + AT_EUID, + UID, + AT_GID, + GID, + AT_EGID, + GID, + AT_SECURE, + 0, + AT_RANDOM, + (i32) rand(), // It's pretty stupid to use rand here, but w/e + 0, + }; + i32 env_vec_offset = offset; + memcpy(get_memory_ptr_for_runtime(env_vec_offset, sizeof(env_vec)), env_vec, sizeof(env_vec)); + + module_libc_init(sandbox_current()->mod, env_vec_offset, program_name_offset); } // Emulated syscall implementations @@ -120,13 +153,11 @@ wasm_write(i32 fd, i32 buf_offset, i32 buf_size) uv_fs_t req = UV_FS_REQ_INIT(); char* buf = get_memory_ptr_void(buf_offset, buf_size); - printf("[%p] start[%d:%d, n%d]\n", uv_fs_get_data(&req), fd, f, buf_size); uv_buf_t bufv = uv_buf_init(buf, buf_size); uv_fs_write(runtime_uvio(), &req, f, &bufv, 1, -1, wasm_fs_callback); sandbox_block(); int ret = uv_fs_get_result(&req); - printf("[%p] end[%d]\n", uv_fs_get_data(&req), ret); uv_fs_req_cleanup(&req); return ret; diff --git a/runtime/src/module.c b/runtime/src/module.c index 264f7440..f0d5f4cb 100644 --- a/runtime/src/module.c +++ b/runtime/src/module.c @@ -91,9 +91,8 @@ module_alloc(char *modname, char *modpath, i32 nargs, u32 stacksz, u32 maxheap, mod->entry_fn = (mod_main_fn_t)dlsym(mod->dl_handle, MOD_MAIN_FN); if (mod->entry_fn == NULL) goto dl_error; - // TODO: don't think this is necessary or implemented. - //mod->glb_init_fn = (mod_glb_fn_t)dlsym(mod->dl_handle, MOD_GLB_FN); - //if (mod->glb_init_fn == NULL) goto dl_error; + mod->glb_init_fn = (mod_glb_fn_t)dlsym(mod->dl_handle, MOD_GLB_FN); + if (mod->glb_init_fn == NULL) goto dl_error; mod->mem_init_fn = (mod_mem_fn_t)dlsym(mod->dl_handle, MOD_MEM_FN); if (mod->mem_init_fn == NULL) goto dl_error; @@ -101,6 +100,9 @@ module_alloc(char *modname, char *modpath, i32 nargs, u32 stacksz, u32 maxheap, mod->tbl_init_fn = (mod_tbl_fn_t)dlsym(mod->dl_handle, MOD_TBL_FN); if (mod->tbl_init_fn == NULL) goto dl_error; + mod->libc_init_fn = (mod_libc_fn_t)dlsym(mod->dl_handle, MOD_LIBC_FN); + if (mod->libc_init_fn == NULL) goto dl_error; + strncpy(mod->name, modname, MOD_NAME_MAX); strncpy(mod->path, modpath, MOD_PATH_MAX); diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c index 4d7f7e18..7ff43475 100644 --- a/runtime/src/runtime.c +++ b/runtime/src/runtime.c @@ -113,7 +113,7 @@ sandbox_local_free(unsigned int n) { int i = 0; - while (i < n) { + while (i < n && !ps_list_head_empty(&endq)) { i++; struct sandbox *s = ps_list_head_first_d(&endq, struct sandbox); if (!s) break; diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index 3f500ecc..4502ed6d 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -46,7 +46,6 @@ sandbox_args_setup(i32 argc) { struct sandbox *curr = sandbox_current(); char *args = sandbox_args(); - if (!args) return; // whatever gregor has, to be able to pass args to a module! curr->args_offset = sandbox_lmbound; @@ -66,6 +65,7 @@ sandbox_args_setup(i32 argc) string_off += str_sz; } + stub_init(string_off); } static inline void @@ -249,7 +249,6 @@ sandbox_entry(void) assert(f == 1); f = io_handle_open(2); assert(f == 2); - sandbox_args_setup(argc); #ifndef STANDALONE http_parser_init(&curr->hp, HTTP_REQUEST); @@ -266,10 +265,12 @@ sandbox_entry(void) { curr->rr_data_len = 0; // TODO: do this on first write to body. alloc_linear_memory(); - // perhaps only initialized for the first instance? or TODO! //module_table_init(curr_mod); + module_globals_init(curr_mod); module_memory_init(curr_mod); + sandbox_args_setup(argc); + curr->retval = module_entry(curr_mod, argc, curr->args_offset); sandbox_client_response_set(); diff --git a/runtime/tests/Makefile b/runtime/tests/Makefile index de00ce69..36e21040 100644 --- a/runtime/tests/Makefile +++ b/runtime/tests/Makefile @@ -2,7 +2,7 @@ include Makefile.inc BENCH_DIR=../../silverfish/code_benches/ -TESTS=fibonacci #empty work +TESTS=fibonacci empty work #TESTS=forever filesys sockserver sockclient empty TESTSRT=$(TESTS:%=%_rt) BENCHES=adpcm basic_math binarytrees bitcount blowfish crc dijkstra fft function_pointers \ @@ -38,6 +38,7 @@ clean: ${WASMCC} ${$(@:%_sf=%)_CFLAGS} ${WASMCFLAGS} ${OPTFLAGS} ${BENCH_DIR}/$(@:%_sf=%)/*.c $(DUMMY) -o ${TMP_DIR}/$(@:%_sf=%).wasm ${SFCC} ${TMP_DIR}/$(@:%_sf=%).wasm -o ${TMP_DIR}/$(@:%_sf=%).bc ${CC} ${CFLAGS} ${OPTFLAGS} -D${USE_MEM} ${TMP_DIR}/$(@:%_sf=%).bc ${MEMC} ${RT_LIBC} ${RT_RT} -o ${TMP_DIR}/$(@:%_sf=%)_wasm.out + ${SFCC} --inline-constant-globals --runtime-globals ${TMP_DIR}/$(@:%_sf=%).wasm -o ${TMP_DIR}/$(@:%_sf=%).bc ${CC} --shared -fPIC ${OPTFLAGS} -I${ART_INC} -D${USE_MEM} ${TMP_DIR}/$(@:%_sf=%).bc ${AMEMC} ${WASMISA} -o ${TMP_DIR}/$(@:%_sf=%)_wasm.so @cp ${TMP_DIR}/$(@:%_sf=%)_wasm.so ${ABIN_DIR} # @rm -rf ${TMP_DIR} @@ -48,6 +49,7 @@ clean: ${WASMCC} ${$(@:%_rt=%)_CFLAGS} ${WASMCFLAGS} ${OPTFLAGS} $(@:%_rt=%)/*.c $(DUMMY) -o ${TMP_DIR}/$(@:%_rt=%).wasm ${SFCC} ${TMP_DIR}/$(@:%_rt=%).wasm -o ${TMP_DIR}/$(@:%_rt=%).bc ${CC} ${CFLAGS} ${OPTFLAGS} -D${USE_MEM} ${TMP_DIR}/$(@:%_rt=%).bc ${MEMC} ${RT_LIBC} ${RT_RT} -o ${TMP_DIR}/$(@:%_rt=%)_wasm.out + ${SFCC} --inline-constant-globals --runtime-globals ${TMP_DIR}/$(@:%_rt=%).wasm -o ${TMP_DIR}/$(@:%_rt=%).bc ${CC} --shared -fPIC ${OPTFLAGS} -I${ART_INC} -D${USE_MEM} ${TMP_DIR}/$(@:%_rt=%).bc ${AMEMC} ${WASMISA} -o ${TMP_DIR}/$(@:%_rt=%)_wasm.so @cp ${TMP_DIR}/$(@:%_rt=%)_wasm.so ${ABIN_DIR} # @rm -rf ${TMP_DIR} From f8cc5edb895725d2c085285908c6c2a62f18f7f0 Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 6 Jan 2020 11:18:03 -0500 Subject: [PATCH 2/8] fibonacci serverless fn in awsm --- runtime/tests/fibonacci/main.c | 8 ++++---- runtime/tests/fibonacci/run_fib.sh | 2 +- runtime/tests/test_fibonacci.json | 13 +++++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 runtime/tests/test_fibonacci.json diff --git a/runtime/tests/fibonacci/main.c b/runtime/tests/fibonacci/main.c index b271cb58..7c27956d 100644 --- a/runtime/tests/fibonacci/main.c +++ b/runtime/tests/fibonacci/main.c @@ -13,11 +13,11 @@ main(int argc, char **argv) { unsigned long n = 0, r; scanf("%lu", &n); - unsigned long long st = get_time(), en; +// unsigned long long st = get_time(), en; r = fib(n); - en = get_time(); - fprintf(stderr, "%lu\n", r); +// en = get_time(); + printf("%lu\n", r); - print_time(st, en); +// print_time(st, en); return 0; } diff --git a/runtime/tests/fibonacci/run_fib.sh b/runtime/tests/fibonacci/run_fib.sh index 3e9439b4..4463b8ae 100755 --- a/runtime/tests/fibonacci/run_fib.sh +++ b/runtime/tests/fibonacci/run_fib.sh @@ -25,7 +25,7 @@ MAXNUM=$2 tmp1_cnt=${MAXNUM} -while [ ${tmp1_cnt} -gt 0 ]; do +while [ ${tmp1_cnt} -gt 28 ]; do testeach ./fibonacci_$1.out ${tmp1_cnt} tmp1_cnt=$((tmp1_cnt - 1)) done diff --git a/runtime/tests/test_fibonacci.json b/runtime/tests/test_fibonacci.json new file mode 100644 index 00000000..725feeec --- /dev/null +++ b/runtime/tests/test_fibonacci.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "fibonacci", + "path" : "fibonacci_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1024, + "http-resp-headers" : [ ], + "http-resp-size" : 1024, + "http-resp-content-type" : "text/plain" +} From 9528f65b322ad566dad34c78dbb67c29927c4839 Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 11 Jan 2020 14:32:45 -0500 Subject: [PATCH 3/8] Synchronous I/O for http request/response fixes 99%tile latencies * Async I/O is being used for reading and writing http request and response respectively. This causes the worker core to steal new tasks as when one sandbox blocks on "read" or "write". This causes true round-robin on timer interrupts, leading to those high latencies. * Ideally, the request/response should be synchronous I belive because we are expected to have "low"/near-realtime latencies and the read/writes are only performed when a client connects, so ideally not block! Plus, we expect only small data transfers (in KBs) so that should also support synchronous request/response processing. --- runtime/Makefile | 1 + runtime/include/http.h | 5 +++++ runtime/include/http_api.h | 6 +++--- runtime/include/sandbox.h | 5 ++++- runtime/include/types.h | 2 +- runtime/src/http.c | 24 +++++++++++++++++++++++- runtime/src/runtime.c | 32 ++++++++++++++++++++++++++------ runtime/src/sandbox.c | 35 +++++++++++++++++------------------ runtime/src/softint.c | 2 +- 9 files changed, 81 insertions(+), 31 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index f0816a86..c8d4de49 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,6 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO +#CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE diff --git a/runtime/include/http.h b/runtime/include/http.h index 04b9e4dc..ac0fd09a 100644 --- a/runtime/include/http.h +++ b/runtime/include/http.h @@ -4,6 +4,7 @@ #include #include #include +#include /* all in-memory ptrs.. don't mess around with that! */ struct http_header { @@ -34,7 +35,11 @@ struct http_response { int bodylen; char *status; int stlen; +#ifdef USE_HTTP_UVIO uv_buf_t bufs[HTTP_HEADERS_MAX * 2 + 3]; //max headers, one line for status code, remaining for body! +#else + struct iovec bufs[HTTP_HEADERS_MAX * 2 + 3]; +#endif }; #endif /* SFRT_HTTP_H */ diff --git a/runtime/include/http_api.h b/runtime/include/http_api.h index ba7c4939..2e99a0db 100644 --- a/runtime/include/http_api.h +++ b/runtime/include/http_api.h @@ -8,7 +8,7 @@ int http_request_parse_sb(struct sandbox *s, size_t l); int http_response_header_set_sb(struct sandbox *s, char *h, int len); int http_response_body_set_sb(struct sandbox *s, char *body, int len); int http_response_status_set_sb(struct sandbox *s, char *status, int len); -int http_response_uv_sb(struct sandbox *s); +int http_response_vector_sb(struct sandbox *s); void http_init(void); @@ -37,9 +37,9 @@ http_response_status_set(char *status, int len) } static inline int -http_response_uv(void) +http_response_vector(void) { - return http_response_uv_sb(sandbox_current()); + return http_response_vector_sb(sandbox_current()); } static inline int diff --git a/runtime/include/sandbox.h b/runtime/include/sandbox.h index e4b99c0a..5e348d28 100644 --- a/runtime/include/sandbox.h +++ b/runtime/include/sandbox.h @@ -193,9 +193,12 @@ sandbox_args(void) //void sandbox_run(struct sandbox *s); void *sandbox_run_func(void *data); -struct sandbox *sandbox_schedule(void); +struct sandbox *sandbox_schedule(int interrupt); void sandbox_block(void); void sandbox_wakeup(sandbox_t *sb); +// called in sandbox_entry() before and after fn() execution +// for http request/response processing using uvio +void sandbox_block_http(void); void sandbox_response(void); // should be the entry-point for each sandbox so it can do per-sandbox mem/etc init. diff --git a/runtime/include/types.h b/runtime/include/types.h index b2e34a61..27d58394 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -116,7 +116,7 @@ typedef enum { #define JSON_ELE_MAX 16 // FIXME: some naive work-stealing here.. -#define SBOX_PULL_MAX 16 +#define SBOX_PULL_MAX 1 #define SBOX_MAX_OPEN 32 #define SBOX_PREOPEN_MAGIC (707707707) // reads lol lol lol upside down diff --git a/runtime/src/http.c b/runtime/src/http.c index 17a3fb4c..806de881 100644 --- a/runtime/src/http.c +++ b/runtime/src/http.c @@ -163,12 +163,13 @@ http_response_status_set_sb(struct sandbox *c, char *status, int len) } int -http_response_uv_sb(struct sandbox *c) +http_response_vector_sb(struct sandbox *c) { int nb = 0; #ifndef STANDALONE struct http_response *r = &c->rsi; +#ifdef USE_HTTP_UVIO r->bufs[nb] = uv_buf_init(r->status, r->stlen); nb++; @@ -183,6 +184,27 @@ http_response_uv_sb(struct sandbox *c) r->bufs[nb] = uv_buf_init(r->status + r->stlen - 2, 2); //for crlf nb++; } +#else + r->bufs[nb].iov_base = r->status; + r->bufs[nb].iov_len = r->stlen; + nb++; + + for (int i = 0; i < r->nheaders; i++) { + r->bufs[nb].iov_base = r->headers[i].hdr; + r->bufs[nb].iov_len = r->headers[i].len; + nb++; + } + + if (r->body) { + r->bufs[nb].iov_base = r->body; + r->bufs[nb].iov_len = r->bodylen; + nb++; + + r->bufs[nb].iov_base = r->status + r->stlen - 2; + r->bufs[nb].iov_len = 2; + nb++; + } +#endif #endif return nb; diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c index 7ff43475..4683eda6 100644 --- a/runtime/src/runtime.c +++ b/runtime/src/runtime.c @@ -87,10 +87,12 @@ sandbox_io_nowait(void) } struct sandbox * -sandbox_schedule(void) +sandbox_schedule(int interrupt) { struct sandbox *s = NULL; if (ps_list_head_empty(&runq)) { + // this is in an interrupt context, don't steal work here! + if (interrupt) return NULL; if (sandbox_pull() == 0) { //debuglog("[null: null]\n"); return NULL; @@ -130,7 +132,7 @@ sandbox_schedule_io(void) if (!in_callback) sandbox_io_nowait(); softint_disable(); - struct sandbox *s = sandbox_schedule(); + struct sandbox *s = sandbox_schedule(0); softint_enable(); assert(s == NULL || s->state == SANDBOX_RUNNABLE); @@ -143,11 +145,12 @@ sandbox_wakeup(sandbox_t *s) #ifndef STANDALONE softint_disable(); debuglog("[%p: %s]\n", s, s->mod->name); - // perhaps 2 lists in the sandbox to make sure sandbox is either in runlist or waitlist.. + if (s->state != SANDBOX_BLOCKED) goto done; assert(s->state == SANDBOX_BLOCKED); assert(ps_list_singleton_d(s)); s->state = SANDBOX_RUNNABLE; ps_list_head_append_d(&runq, s); +done: softint_enable(); #endif } @@ -156,13 +159,12 @@ void sandbox_block(void) { #ifndef STANDALONE - // perhaps 2 lists in the sandbox to make sure sandbox is either in runlist or waitlist.. assert(in_callback == 0); softint_disable(); struct sandbox *c = sandbox_current(); ps_list_rem_d(c); c->state = SANDBOX_BLOCKED; - struct sandbox *s = sandbox_schedule(); + struct sandbox *s = sandbox_schedule(0); debuglog("[%p: %s, %p: %s]\n", c, c->mod->name, s, s ? s->mod->name: ""); softint_enable(); sandbox_switch(s); @@ -171,6 +173,24 @@ sandbox_block(void) #endif } +void +sandbox_block_http(void) +{ +#ifdef USE_HTTP_UVIO +#ifdef USE_HTTP_SYNC + // realistically, we're processing all async I/O on this core when a sandbox blocks on http processing, not great! + // if there is a way (TODO), perhaps RUN_ONCE and check if your I/O is processed, if yes, return + // else do async block! + uv_run(runtime_uvio(), UV_RUN_DEFAULT); +#else + sandbox_block(); +#endif +#else + assert(0); + //it should not be called if not using uvio for http +#endif +} + void __attribute__((noinline)) __attribute__((noreturn)) sandbox_switch_preempt(void) { @@ -248,7 +268,7 @@ sandbox_exit(void) sandbox_local_stop(curr); curr->state = SANDBOX_RETURNED; // free resources from "main function execution", as stack still in use. - struct sandbox *n = sandbox_schedule(); + struct sandbox *n = sandbox_schedule(0); assert(n != curr); softint_enable(); //sandbox_local_end(curr); diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index 4502ed6d..b854d047 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -133,11 +133,11 @@ sandbox_client_request_get(void) struct sandbox *curr = sandbox_current(); curr->rr_data_len = 0; -#ifndef USE_UVIO +#ifndef USE_HTTP_UVIO int r = 0; r = recv(curr->csock, (curr->req_resp_data), curr->mod->max_req_sz, 0); if (r <= 0) { - perror("recv"); + if (r < 0) perror("recv1"); return r; } while (r > 0) { @@ -148,13 +148,13 @@ sandbox_client_request_get(void) r = recv(curr->csock, (curr->req_resp_data + r), curr->mod->max_req_sz - r, 0); if (r < 0) { - perror("recv"); + perror("recv2"); return r; } } #else int r = uv_read_start((uv_stream_t *)&curr->cuv, sb_alloc_callback, sb_read_callback); - sandbox_block(); + sandbox_block_http(); if (curr->rr_data_len == 0) return 0; #endif @@ -170,15 +170,6 @@ sandbox_client_response_set(void) #ifndef STANDALONE struct sandbox *curr = sandbox_current(); -#ifndef USE_UVIO - strcpy(curr->req_resp_data + curr->rr_data_len, "HTTP/1.1 200 OK\r\n"); - - // TODO: response set in req_resp_data - curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); - - int r = send(curr->csock, curr->req_resp_data, curr->rr_data_len, 0); - if (r < 0) perror("send"); -#else int bodylen = curr->rr_data_len; if (bodylen > 0) { http_response_body_set(curr->req_resp_data, bodylen); @@ -217,10 +208,14 @@ sandbox_client_response_set(void) curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); http_response_status_set(st, strlen("HTTP/1.1 200 OK\r\n")); + int n = http_response_vector(); +#ifndef USE_HTTP_UVIO + int r = writev(curr->csock, curr->rsi.bufs, n); + if (r < 0) perror("writev"); +#else uv_write_t req = { .data = curr, }; - int n = http_response_uv(); int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); - sandbox_block(); + sandbox_block_http(); #endif return r; #else @@ -253,7 +248,11 @@ sandbox_entry(void) #ifndef STANDALONE http_parser_init(&curr->hp, HTTP_REQUEST); curr->hp.data = curr; -#ifdef USE_UVIO +#ifdef USE_HTTP_UVIO +#ifndef USE_UVIO + printf("UVIO not enabled!\n"); + assert(0); +#endif int r = uv_tcp_init(runtime_uvio(), (uv_tcp_t *)&curr->cuv); assert(r == 0); curr->cuv.data = curr; @@ -277,9 +276,9 @@ sandbox_entry(void) } #ifndef STANDALONE -#ifdef USE_UVIO +#ifdef USE_HTTP_UVIO uv_close((uv_handle_t *)&curr->cuv, sb_close_callback); - sandbox_block(); + sandbox_block_http(); #else close(curr->csock); #endif diff --git a/runtime/src/softint.c b/runtime/src/softint.c index dff80118..bf3b50ce 100644 --- a/runtime/src/softint.c +++ b/runtime/src/softint.c @@ -67,7 +67,7 @@ softint_alarm_schedule(void *u) if (curr == NULL) goto done; // find a next sandbox to run.. - struct sandbox *next = sandbox_schedule(); + struct sandbox *next = sandbox_schedule(1); if (next == NULL) goto done; if (next == curr) goto done; // only this sandbox to schedule.. return to it! // save the current sandbox, state from uc! From da43b22407c605e6f53ae3cedf5bf1998c6da372 Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 11 Jan 2020 16:40:54 -0500 Subject: [PATCH 4/8] work 1k to 1m functions --- runtime/src/main.c | 16 ++++++++++++++++ runtime/tests/Makefile | 2 +- runtime/tests/test_work100k.json | 13 +++++++++++++ runtime/tests/test_work10k.json | 13 +++++++++++++ runtime/tests/test_work1k.json | 13 +++++++++++++ runtime/tests/test_work1m.json | 13 +++++++++++++ runtime/tests/work/main.c | 19 ++----------------- runtime/tests/work100k/main.c | 24 ++++++++++++++++++++++++ runtime/tests/work10k/main.c | 24 ++++++++++++++++++++++++ runtime/tests/work1k/main.c | 24 ++++++++++++++++++++++++ runtime/tests/work1m/main.c | 24 ++++++++++++++++++++++++ 11 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 runtime/tests/test_work100k.json create mode 100644 runtime/tests/test_work10k.json create mode 100644 runtime/tests/test_work1k.json create mode 100644 runtime/tests/test_work1m.json create mode 100644 runtime/tests/work100k/main.c create mode 100644 runtime/tests/work10k/main.c create mode 100644 runtime/tests/work1k/main.c create mode 100644 runtime/tests/work1m/main.c diff --git a/runtime/src/main.c b/runtime/src/main.c index 28832050..2241d5ff 100644 --- a/runtime/src/main.c +++ b/runtime/src/main.c @@ -20,6 +20,18 @@ u32 ncores = 0, sbox_ncores = 0, sbox_core_st = 0; pthread_t rtthd[SBOX_NCORES]; +static unsigned long long +get_time() +{ + struct timeval Tp; + int stat; + stat = gettimeofday (&Tp, NULL); + if (stat != 0) + printf ("Error return from gettimeofday: %d", stat); + return (Tp.tv_sec * 1000000 + Tp.tv_usec); +} + + static void usage(char *cmd) { @@ -141,7 +153,11 @@ main(int argc, char** argv) /* in current dir! */ struct module *m = module_alloc(args, args, ac, 0, 0, 0, 0, 0, 0); assert(m); + + //unsigned long long st = get_time(), en; struct sandbox *s = sandbox_alloc(m, args, 0, NULL); + //en = get_time(); + //fprintf(stderr, "%llu\n", en - st); exit(0); #endif diff --git a/runtime/tests/Makefile b/runtime/tests/Makefile index 36e21040..bcfa715e 100644 --- a/runtime/tests/Makefile +++ b/runtime/tests/Makefile @@ -2,7 +2,7 @@ include Makefile.inc BENCH_DIR=../../silverfish/code_benches/ -TESTS=fibonacci empty work +TESTS=fibonacci empty work work1k work10k work100k work1m #TESTS=forever filesys sockserver sockclient empty TESTSRT=$(TESTS:%=%_rt) BENCHES=adpcm basic_math binarytrees bitcount blowfish crc dijkstra fft function_pointers \ diff --git a/runtime/tests/test_work100k.json b/runtime/tests/test_work100k.json new file mode 100644 index 00000000..a1d9d298 --- /dev/null +++ b/runtime/tests/test_work100k.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work100k", + "path" : "work100k_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 102600, + "http-resp-headers" : [ ], + "http-resp-size" : 102600, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/test_work10k.json b/runtime/tests/test_work10k.json new file mode 100644 index 00000000..fc0a0fd9 --- /dev/null +++ b/runtime/tests/test_work10k.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work10k", + "path" : "work10k_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 10480, + "http-resp-headers" : [ ], + "http-resp-size" : 10480, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/test_work1k.json b/runtime/tests/test_work1k.json new file mode 100644 index 00000000..c332049f --- /dev/null +++ b/runtime/tests/test_work1k.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work1k", + "path" : "work1k_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1200, + "http-resp-headers" : [ ], + "http-resp-size" : 1200, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/test_work1m.json b/runtime/tests/test_work1m.json new file mode 100644 index 00000000..4cdffde4 --- /dev/null +++ b/runtime/tests/test_work1m.json @@ -0,0 +1,13 @@ +{ + "active" : "yes", + "name" : "work1m", + "path" : "work1m_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1048576, + "http-resp-headers" : [ ], + "http-resp-size" : 1048776, + "http-resp-content-type" : "text/plain" +} diff --git a/runtime/tests/work/main.c b/runtime/tests/work/main.c index 2694657a..3ad7416b 100644 --- a/runtime/tests/work/main.c +++ b/runtime/tests/work/main.c @@ -2,24 +2,9 @@ #include #include -#define CPU_CYCS 2100 +#ifndef MAX_BUF #define MAX_BUF (1024*1024*1) //1m - -#define ITERS_15US 125500 -#define MULTIPLE 5 - -#define SPIN_ITERS (ITERS_15US*MULTIPLE) - -__attribute__((optnone)) static void -wrk(void) -{ - unsigned int spin = 0; - - while (spin < SPIN_ITERS) { - //__asm__ __volatile__("nop": : :"memory"); - spin++; - } -} +#endif //__attribute__((optnone)) int int diff --git a/runtime/tests/work100k/main.c b/runtime/tests/work100k/main.c new file mode 100644 index 00000000..57a483ff --- /dev/null +++ b/runtime/tests/work100k/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF 102400 + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} diff --git a/runtime/tests/work10k/main.c b/runtime/tests/work10k/main.c new file mode 100644 index 00000000..d034c1e8 --- /dev/null +++ b/runtime/tests/work10k/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF 10240 + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} diff --git a/runtime/tests/work1k/main.c b/runtime/tests/work1k/main.c new file mode 100644 index 00000000..eb443403 --- /dev/null +++ b/runtime/tests/work1k/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF 1024 + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} diff --git a/runtime/tests/work1m/main.c b/runtime/tests/work1m/main.c new file mode 100644 index 00000000..b7f22732 --- /dev/null +++ b/runtime/tests/work1m/main.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#define MAX_BUF (1024*1024*1) //1m + +//__attribute__((optnone)) int +int +main(void) +{ + char *d = malloc(MAX_BUF + 1); + int r = read(0, d, MAX_BUF); + +// unsigned long long st = rdtsc(), en = 0; +// wrk(); +// en = rdtsc(); + +// if (r <= 0) printf("%llu\n", en > st ? (en - st)/CPU_CYCS : -1); + if (r < 0) printf("E\n"); + else if (r <= 1) printf("D\n"); + else write(1, d, r); + + return 0; +} From b9d7278c54309df7047d1f7b78addb794cc6e12d Mon Sep 17 00:00:00 2001 From: phani Date: Sat, 11 Jan 2020 19:05:04 -0500 Subject: [PATCH 5/8] fixes 99% latency problem here (but a problem with 1m data transfer test, it is broken.) --- runtime/Makefile | 2 +- runtime/src/runtime.c | 3 +++ runtime/src/sandbox.c | 12 ++++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index c8d4de49..c3759add 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,7 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO -#CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC +CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE diff --git a/runtime/src/runtime.c b/runtime/src/runtime.c index 4683eda6..7e79be2b 100644 --- a/runtime/src/runtime.c +++ b/runtime/src/runtime.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -271,6 +272,8 @@ sandbox_exit(void) struct sandbox *n = sandbox_schedule(0); assert(n != curr); softint_enable(); + //unmap linear memory only! + munmap(curr->linear_start, SBOX_MAX_MEM + PAGE_SIZE); //sandbox_local_end(curr); sandbox_switch(n); #else diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index b854d047..f6774783 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -21,7 +21,7 @@ sandbox_memory_map(struct module *m) if (lm_sz + sb_sz > mem_sz) return NULL; assert(round_up_to_page(sb_sz) == sb_sz); unsigned long rw_sz = sb_sz + lm_sz; - void *addr = mmap(NULL, mem_sz + /* guard page */ PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void *addr = mmap(NULL, sb_sz + mem_sz + /* guard page */ PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) return NULL; void *addr_rw = mmap(addr, sb_sz + lm_sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); @@ -217,7 +217,7 @@ sandbox_client_response_set(void) int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); sandbox_block_http(); #endif - return r; + return 0; #else return 0; #endif @@ -335,6 +335,10 @@ sandbox_free(struct sandbox *sb) // TODO: this needs to be enhanced. you may be killing a sandbox when its in any other execution states. if (sb->state != SANDBOX_RETURNED) return; + int sz = sizeof(struct sandbox); +#ifndef STANDALONE + sz += sb->mod->max_rr_sz; +#endif module_release(sb->mod); // TODO free(sb->args); @@ -342,10 +346,10 @@ sandbox_free(struct sandbox *sb) size_t stksz = sb->stack_size; // depending on the memory type - free_linear_memory(sb->linear_start, sb->linear_size, sb->linear_max_size); + //free_linear_memory(sb->linear_start, sb->linear_size, sb->linear_max_size); // mmaped memory includes sandbox structure in there. - ret = munmap(sb, SBOX_MAX_MEM + PAGE_SIZE); + ret = munmap(sb, sz); if (ret) perror("munmap sandbox"); // remove stack! From 0c92419978e6a7818c7659800f6bca84ac9705e6 Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 13 Jan 2020 12:11:08 -0500 Subject: [PATCH 6/8] fixed syscall based sync http-req/resp --- runtime/Makefile | 2 +- runtime/include/types.h | 5 ++ runtime/src/sandbox.c | 151 ++++++++++++++++++++++++++++------------ 3 files changed, 113 insertions(+), 45 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index c3759add..06a36ead 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,7 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO -CFLAGS += -DUSE_HTTP_UVIO -DUSE_HTTP_SYNC +#CFLAGS += -DUSE_HTTP_UVIO #-DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE diff --git a/runtime/include/types.h b/runtime/include/types.h index 27d58394..59f83615 100644 --- a/runtime/include/types.h +++ b/runtime/include/types.h @@ -157,4 +157,9 @@ typedef enum { #define HTTP_HEADER_MAXSZ 32 #define HTTP_HEADERVAL_MAXSZ 64 +#define HTTP_RESP_200OK "HTTP/1.1 200 OK\r\n" +#define HTTP_RESP_CONTTYPE "Content-type: \r\n" +#define HTTP_RESP_CONTLEN "Content-length: \r\n\r\n" //content body follows this +#define HTTP_RESP_CONTTYPE_PLAIN "text/plain" + #endif /* SFRT_TYPES_H */ diff --git a/runtime/src/sandbox.c b/runtime/src/sandbox.c index f6774783..bd798e84 100644 --- a/runtime/src/sandbox.c +++ b/runtime/src/sandbox.c @@ -146,7 +146,7 @@ sandbox_client_request_get(void) struct http_request *rh = &curr->rqi; if (rh->message_end) break; - r = recv(curr->csock, (curr->req_resp_data + r), curr->mod->max_req_sz - r, 0); + r = recv(curr->csock, (curr->req_resp_data + curr->rr_data_len), curr->mod->max_req_sz - curr->rr_data_len, 0); if (r < 0) { perror("recv2"); return r; @@ -168,61 +168,118 @@ static inline int sandbox_client_response_set(void) { #ifndef STANDALONE + int sndsz = 0; struct sandbox *curr = sandbox_current(); - - int bodylen = curr->rr_data_len; - if (bodylen > 0) { - http_response_body_set(curr->req_resp_data, bodylen); - char len[16] = { 0 }; - sprintf(len, "%d", bodylen); - //content-length = body length - char *key = curr->req_resp_data + curr->rr_data_len; - int lenlen = strlen("content-length: "), dlen = strlen(len); - strcpy(key, "content-length: "); - strncat(key + lenlen, len, dlen); - strncat(key + lenlen + dlen, "\r\n", 2); - http_response_header_set(key, lenlen + dlen + 2); - curr->rr_data_len += lenlen + dlen + 2; - - //content-type as set in the headers. - key = curr->req_resp_data + curr->rr_data_len; - strcpy(key, "content-type: "); - lenlen = strlen("content-type: "); - dlen = strlen(curr->mod->rspctype); - if (dlen == 0) { - int l = strlen("text/plain\r\n\r\n"); - strncat(key + lenlen, "text/plain\r\n\r\n", l); - http_response_header_set(key, lenlen + l); - curr->rr_data_len += lenlen + l; - } else { - strncat(key + lenlen, curr->mod->rspctype, dlen); - strncat(key + lenlen + dlen, "\r\n\r\n", 4); - http_response_header_set(key, lenlen + dlen + 4); - curr->rr_data_len += lenlen + dlen + 4; - } - //TODO - other headers requested in module! + int rsp_hdr_len = strlen(HTTP_RESP_200OK) + strlen(HTTP_RESP_CONTTYPE) + strlen(HTTP_RESP_CONTLEN); + int bodylen = curr->rr_data_len - rsp_hdr_len; + + memset(curr->req_resp_data, 0, strlen(HTTP_RESP_200OK) + strlen(HTTP_RESP_CONTTYPE) + strlen(HTTP_RESP_CONTLEN)); + strncpy(curr->req_resp_data, HTTP_RESP_200OK, strlen(HTTP_RESP_200OK)); + sndsz += strlen(HTTP_RESP_200OK); + + if (bodylen == 0) goto done; + strncpy(curr->req_resp_data + sndsz, HTTP_RESP_CONTTYPE, strlen(HTTP_RESP_CONTTYPE)); + if (strlen(curr->mod->rspctype) <= 0) { + strncpy(curr->req_resp_data + sndsz + strlen("Content-type: "), HTTP_RESP_CONTTYPE_PLAIN, strlen(HTTP_RESP_CONTTYPE_PLAIN)); + } else { + strncpy(curr->req_resp_data + sndsz + strlen("Content-type: "), curr->mod->rspctype, strlen(curr->mod->rspctype)); } + sndsz += strlen(HTTP_RESP_CONTTYPE); + char len[10] = { 0 }; + sprintf(len, "%d", bodylen); + strncpy(curr->req_resp_data + sndsz, HTTP_RESP_CONTLEN, strlen(HTTP_RESP_CONTLEN)); + strncpy(curr->req_resp_data + sndsz + strlen("Content-length: "), len, strlen(len)); + sndsz += strlen(HTTP_RESP_CONTLEN); + sndsz += bodylen; - char *st = curr->req_resp_data + curr->rr_data_len; - strcpy(st, "HTTP/1.1 200 OK\r\n"); - curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); +done: + assert(sndsz == curr->rr_data_len); - http_response_status_set(st, strlen("HTTP/1.1 200 OK\r\n")); - int n = http_response_vector(); #ifndef USE_HTTP_UVIO - int r = writev(curr->csock, curr->rsi.bufs, n); - if (r < 0) perror("writev"); + int r = send(curr->csock, curr->req_resp_data, sndsz, 0); + if (r < 0) { + perror("send"); + return -1; + } + while (r < sndsz) { + int s = send(curr->csock, curr->req_resp_data + r, sndsz - r, 0); + if (s < 0) { + perror("send"); + return -1; + } + r += s; + } #else uv_write_t req = { .data = curr, }; - int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); + uv_buf_t bufv = uv_buf_init(curr->req_resp_data, sndsz); + int r = uv_write(&req, (uv_stream_t *)&curr->cuv, &bufv, 1, sb_write_callback); sandbox_block_http(); #endif - return 0; -#else + + return 0; #endif } +//static inline int +//sandbox_client_response_set(void) +//{ +//#ifndef STANDALONE +// struct sandbox *curr = sandbox_current(); +// +// int bodylen = curr->rr_data_len; +// if (bodylen > 0) { +// http_response_body_set(curr->req_resp_data, bodylen); +// char len[16] = { 0 }; +// sprintf(len, "%d", bodylen); +// //content-length = body length +// char *key = curr->req_resp_data + curr->rr_data_len; +// int lenlen = strlen("content-length: "), dlen = strlen(len); +// strcpy(key, "content-length: "); +// strncat(key + lenlen, len, dlen); +// strncat(key + lenlen + dlen, "\r\n", 2); +// http_response_header_set(key, lenlen + dlen + 2); +// curr->rr_data_len += lenlen + dlen + 2; +// +// //content-type as set in the headers. +// key = curr->req_resp_data + curr->rr_data_len; +// strcpy(key, "content-type: "); +// lenlen = strlen("content-type: "); +// dlen = strlen(curr->mod->rspctype); +// if (dlen == 0) { +// int l = strlen("text/plain\r\n\r\n"); +// strncat(key + lenlen, "text/plain\r\n\r\n", l); +// http_response_header_set(key, lenlen + l); +// curr->rr_data_len += lenlen + l; +// } else { +// strncat(key + lenlen, curr->mod->rspctype, dlen); +// strncat(key + lenlen + dlen, "\r\n\r\n", 4); +// http_response_header_set(key, lenlen + dlen + 4); +// curr->rr_data_len += lenlen + dlen + 4; +// } +// //TODO - other headers requested in module! +// } +// +// char *st = curr->req_resp_data + curr->rr_data_len; +// strcpy(st, "HTTP/1.1 200 OK\r\n"); +// curr->rr_data_len += strlen("HTTP/1.1 200 OK\r\n"); +// +// http_response_status_set(st, strlen("HTTP/1.1 200 OK\r\n")); +// int n = http_response_vector(); +//#ifndef USE_HTTP_UVIO +// int r = writev(curr->csock, curr->rsi.bufs, n); +// if (r < 0) perror("writev"); +//#else +// uv_write_t req = { .data = curr, }; +// int r = uv_write(&req, (uv_stream_t *)&curr->cuv, curr->rsi.bufs, n, sb_write_callback); +// sandbox_block_http(); +//#endif +// return 0; +//#else +// return 0; +//#endif +//} + void sandbox_entry(void) { @@ -248,6 +305,8 @@ sandbox_entry(void) #ifndef STANDALONE http_parser_init(&curr->hp, HTTP_REQUEST); curr->hp.data = curr; + // NOTE: if more headers, do offset by that! + int rsp_hdr_len = strlen(HTTP_RESP_200OK) + strlen(HTTP_RESP_CONTTYPE) + strlen(HTTP_RESP_CONTLEN); #ifdef USE_HTTP_UVIO #ifndef USE_UVIO printf("UVIO not enabled!\n"); @@ -262,7 +321,11 @@ sandbox_entry(void) if (sandbox_client_request_get() > 0) #endif { - curr->rr_data_len = 0; // TODO: do this on first write to body. +#ifndef STANDALONE + curr->rr_data_len = rsp_hdr_len; // TODO: do this on first write to body. +#else + curr->rr_data_len = 0; +#endif alloc_linear_memory(); // perhaps only initialized for the first instance? or TODO! //module_table_init(curr_mod); From 77ae44337ccc70c6009167709b4a4f13b1d7abbc Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 13 Jan 2020 13:31:30 -0500 Subject: [PATCH 7/8] buffer problem in work1m tests --- runtime/tests/test_work.json | 2 +- runtime/tests/test_work1m.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/tests/test_work.json b/runtime/tests/test_work.json index ee7d2583..84004856 100644 --- a/runtime/tests/test_work.json +++ b/runtime/tests/test_work.json @@ -6,7 +6,7 @@ "argsize" : 1, "http-req-headers" : [ ], "http-req-content-type" : "text/plain", - "http-req-size": 1048576, + "http-req-size": 1048776, "http-resp-headers" : [ ], "http-resp-size" : 1048776, "http-resp-content-type" : "text/plain" diff --git a/runtime/tests/test_work1m.json b/runtime/tests/test_work1m.json index 4cdffde4..ef8804de 100644 --- a/runtime/tests/test_work1m.json +++ b/runtime/tests/test_work1m.json @@ -6,7 +6,7 @@ "argsize" : 1, "http-req-headers" : [ ], "http-req-content-type" : "text/plain", - "http-req-size": 1048576, + "http-req-size": 1048776, "http-resp-headers" : [ ], "http-resp-size" : 1048776, "http-resp-content-type" : "text/plain" From ed234f9ac5aeb1c9e7fb52393dabd55dfd9afb03 Mon Sep 17 00:00:00 2001 From: phani Date: Mon, 13 Jan 2020 13:38:52 -0500 Subject: [PATCH 8/8] default: async io for http req/resp. sync based io doesn't seem to work with huge data like 500k-1m I tested. They just timeout with concurrency --- runtime/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Makefile b/runtime/Makefile index 06a36ead..1c5bf469 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -16,7 +16,7 @@ CFLAGS += -D_GNU_SOURCE #CFLAGS += -DNOSTDIO #CFLAGS += -DSTANDALONE CFLAGS += -DUSE_UVIO -#CFLAGS += -DUSE_HTTP_UVIO #-DUSE_HTTP_SYNC +CFLAGS += -DUSE_HTTP_UVIO #-DUSE_HTTP_SYNC CFLAGS += -DSBOX_SCALE_ALLOC #CFLAGS += -DUSE_SYSCALL #CFLAGS += -DPREEMPT_DISABLE