refactor: break JSON parsing out from module.c

master
Sean McBride 3 years ago
parent b79eb44958
commit e27ffa5620

@ -110,7 +110,8 @@
"current_wasm_module_instance.h": "c",
"wasm_memory.h": "c",
"sledge_abi.h": "c",
"vec.h": "c"
"vec.h": "c",
"module_database.h": "c"
},
"files.exclude": {
"**/.git": true,

@ -0,0 +1,230 @@
#pragma once
#include <assert.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <jsmn.h>
#include "runtime.h"
#include "http.h"
#include "module.h"
static const int JSON_MAX_ELEMENT_COUNT = 16;
static const int JSON_MAX_ELEMENT_SIZE = 1024;
struct module_config {
char *name;
char *path;
int port;
uint32_t expected_execution_us;
int admissions_percentile;
uint32_t relative_deadline_us;
int32_t http_req_size;
int32_t http_resp_size;
char *http_resp_content_type;
};
static void
print_module_config(struct module_config *config)
{
printf("Name: %s\n", config->name);
printf("Path: %s\n", config->path);
printf("Port: %d\n", config->port);
printf("expected_execution_us: %u\n", config->expected_execution_us);
printf("admissions_percentile: %d\n", config->admissions_percentile);
printf("relative_deadline_us: %u\n", config->relative_deadline_us);
printf("http_req_size: %u\n", config->http_req_size);
printf("http_resp_size: %u\n", config->http_resp_size);
printf("http_resp_content_type: %s\n", config->http_resp_content_type);
}
static inline char *
jsmn_type(jsmntype_t type)
{
switch (type) {
case JSMN_UNDEFINED:
return "Undefined";
case JSMN_OBJECT:
return "Object";
case JSMN_ARRAY:
return "Array";
case JSMN_STRING:
return "String";
case JSMN_PRIMITIVE:
return "Primitive";
default:
return "Invalid";
}
}
/**
* Parses a JSON file into an array of module configs
* @param file_name The path of the JSON file
* @return module_config_vec_len on success. -1 on Error
*/
static inline int
parse_json(const char *json_buf, ssize_t json_buf_size, struct module_config **module_config_vec)
{
assert(json_buf != NULL);
assert(json_buf_size > 0);
assert(module_config_vec != NULL);
jsmntok_t tokens[JSON_MAX_ELEMENT_SIZE * JSON_MAX_ELEMENT_COUNT];
int module_config_vec_len = 0;
/* Initialize the Jasmine Parser and an array to hold the tokens */
jsmn_parser module_parser;
jsmn_init(&module_parser);
/* Use Jasmine to parse the JSON */
int total_tokens = jsmn_parse(&module_parser, json_buf, json_buf_size, tokens,
sizeof(tokens) / sizeof(tokens[0]));
if (total_tokens < 0) {
if (total_tokens == JSMN_ERROR_INVAL) {
fprintf(stderr, "Error parsing %s: bad token, JSON string is corrupted\n", json_buf);
} else if (total_tokens == JSMN_ERROR_PART) {
fprintf(stderr, "Error parsing %s: JSON string is too short, expecting more JSON data\n",
json_buf);
} else if (total_tokens == JSMN_ERROR_NOMEM) {
/*
* According to the README at https://github.com/zserge/jsmn, this is a potentially recoverable
* error. More tokens can be allocated and jsmn_parse can be re-invoked.
*/
fprintf(stderr, "Error parsing %s: Not enough tokens, JSON string is too large\n", json_buf);
}
goto err;
}
if (tokens[0].type != JSMN_ARRAY) {
fprintf(stderr, "Outermost Config should be a JSON array\n");
goto err;
}
module_config_vec_len = tokens[0].size;
if (module_config_vec_len == 0) {
fprintf(stderr, "Config is an empty JSON array\n");
goto err;
}
*module_config_vec = (struct module_config *)calloc(module_config_vec_len, sizeof(struct module_config));
int module_idx = -1;
int module_fields_remaining = 0;
for (int i = 1; i < total_tokens; i++) {
char key[32] = { 0 };
char val[256] = { 0 };
if (tokens[i].type == JSMN_OBJECT) {
assert(module_fields_remaining == 0);
module_fields_remaining = tokens[i].size;
module_idx++;
} else {
/* Inside Object */
assert(tokens[i].type == JSMN_STRING);
sprintf(key, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
if (strcmp(key, "name") == 0) {
/* Should always have a value */
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_STRING);
assert(tokens[i].end - tokens[i].start > 0);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
(*module_config_vec)[module_idx].name = strndup(val, tokens[i].end - tokens[i].start);
} else if (strcmp(key, "path") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_STRING);
assert(tokens[i].end - tokens[i].start > 0);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
(*module_config_vec)[module_idx].path = strndup(val, tokens[i].end - tokens[i].start);
} else if (strcmp(key, "port") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_PRIMITIVE);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
// Validate sane port
// If already taken, will error on bind call in module_listen
int buffer = atoi(val);
if (buffer < 0 || buffer > 65535)
panic("Expected port between 0 and 65535, saw %d\n", buffer);
(*module_config_vec)[module_idx].port = buffer;
} else if (strcmp(key, "expected-execution-us") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_PRIMITIVE);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > (int64_t)RUNTIME_EXPECTED_EXECUTION_US_MAX)
panic("Relative-deadline-us must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_EXPECTED_EXECUTION_US_MAX, buffer);
(*module_config_vec)[module_idx].expected_execution_us = (uint32_t)buffer;
} else if (strcmp(key, "admissions-percentile") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_PRIMITIVE);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
int32_t buffer = strtol(val, NULL, 10);
if (buffer > 99 || buffer < 50)
panic("admissions-percentile must be > 50 and <= 99 but was %d\n", buffer);
(*module_config_vec)[module_idx].admissions_percentile = buffer;
} else if (strcmp(key, "relative-deadline-us") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_PRIMITIVE);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > (int64_t)RUNTIME_RELATIVE_DEADLINE_US_MAX)
panic("Relative-deadline-us must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_RELATIVE_DEADLINE_US_MAX, buffer);
(*module_config_vec)[module_idx].relative_deadline_us = (uint32_t)buffer;
} else if (strcmp(key, "http-req-size") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_PRIMITIVE);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > RUNTIME_HTTP_REQUEST_SIZE_MAX)
panic("http-req-size must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_HTTP_REQUEST_SIZE_MAX, buffer);
(*module_config_vec)[module_idx].http_req_size = (int32_t)buffer;
} else if (strcmp(key, "http-resp-size") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_PRIMITIVE);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > RUNTIME_HTTP_REQUEST_SIZE_MAX)
panic("http-req-size must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_HTTP_RESPONSE_SIZE_MAX, buffer);
(*module_config_vec)[module_idx].http_resp_size = (int32_t)buffer;
} else if (strcmp(key, "http-resp-content-type") == 0) {
assert(tokens[i].size == 1);
i++;
assert(tokens[i].type == JSMN_STRING);
sprintf(val, "%.*s", tokens[i].end - tokens[i].start, json_buf + tokens[i].start);
(*module_config_vec)[module_idx].http_resp_content_type = strndup(val,
tokens[i].end
- tokens[i].start);
} else {
fprintf(stderr, "%s is not a valid key\n", key);
goto json_parse_err;
}
module_fields_remaining--;
}
}
#ifdef LOG_MODULE_LOADING
debuglog("Loaded %d module%s!\n", module_count, module_count > 1 ? "s" : "");
#endif
done:
return module_config_vec_len;
json_parse_err:
free(*module_config_vec);
err:
module_config_vec_len = -1;
goto done;
}

@ -242,6 +242,6 @@ module_free_linear_memory(struct module *module, struct wasm_memory *memory)
*******************************/
void module_free(struct module *module);
struct module *module_alloc(char *mod_name, char *mod_path, uint32_t stack_sz, uint32_t relative_deadline_us, int port,
int req_sz, int resp_sz, int admissions_percentile, uint32_t expected_execution_us);
int module_alloc_from_json(const char *json_buf, ssize_t json_buf_size);
struct module *
module_alloc(char *mod_name, char *mod_path, uint32_t stack_sz, uint32_t relative_deadline_us, int port, int req_sz,
int resp_sz, int admissions_percentile, uint32_t expected_execution_us, char *response_content_type);

@ -15,6 +15,7 @@
#include <sys/fcntl.h>
#endif
#include "json.h"
#include "pretty_print.h"
#include "debuglog.h"
#include "listener_thread.h"
@ -456,8 +457,23 @@ main(int argc, char **argv)
char *json_buf = NULL;
ssize_t json_buf_len = load_file_into_buffer(json_path, &json_buf);
if (unlikely(json_buf_len <= 0)) panic("failed to initialize module(s) defined in %s\n", json_path);
int rc = module_alloc_from_json(json_buf, json_buf_len);
if (unlikely(rc != 0)) panic("failed to initialize module(s) defined in %s\n", json_path);
struct module_config *module_config_vec;
int module_config_vec_len = parse_json(json_buf, json_buf_len, &module_config_vec);
for (int module_idx = 0; module_idx < module_config_vec_len; module_idx++) {
/* Automatically calls listen */
struct module *module = module_alloc(module_config_vec[module_idx].name,
module_config_vec[module_idx].path, 0,
module_config_vec[module_idx].relative_deadline_us,
module_config_vec[module_idx].port,
module_config_vec[module_idx].http_req_size,
module_config_vec[module_idx].http_resp_size,
module_config_vec[module_idx].admissions_percentile,
module_config_vec[module_idx].expected_execution_us,
module_config_vec[module_idx].http_resp_content_type);
if (unlikely(module == NULL)) panic("failed to initialize module(s) defined in %s\n", json_path);
}
for (int i = 0; i < runtime_worker_threads_count; i++) {
int ret = pthread_join(runtime_worker_threads[i], NULL);

@ -1,6 +1,5 @@
#include <assert.h>
#include <dlfcn.h>
#include <jsmn.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -17,9 +16,6 @@
#include "scheduler.h"
#include "wasm_table.h"
const int JSON_MAX_ELEMENT_COUNT = 16;
const int JSON_MAX_ELEMENT_SIZE = 1024;
/*************************
* Private Static Inline *
************************/
@ -85,20 +81,6 @@ err:
goto done;
}
/**
* Sets the HTTP Response Content type on a module
* @param module
* @param response_content_type
*/
static inline void
module_set_http_info(struct module *module, char response_content_type[])
{
assert(module);
strcpy(module->response_content_type, response_content_type);
}
/***************************************
* Public Methods
***************************************/
@ -127,7 +109,8 @@ module_free(struct module *module)
static inline int
module_init(struct module *module, char *name, char *path, uint32_t stack_size, uint32_t relative_deadline_us, int port,
int request_size, int response_size, int admissions_percentile, uint32_t expected_execution_us)
int request_size, int response_size, int admissions_percentile, uint32_t expected_execution_us,
char *response_content_type)
{
assert(module != NULL);
@ -141,6 +124,7 @@ module_init(struct module *module, char *name, char *path, uint32_t stack_size,
/* Set fields in the module struct */
strncpy(module->name, name, MODULE_MAX_NAME_LENGTH);
strncpy(module->path, path, MODULE_MAX_PATH_LENGTH);
strncpy(module->response_content_type, response_content_type, HTTP_MAX_HEADER_VALUE_LENGTH);
module->stack_size = ((uint32_t)(round_up_to_page(stack_size == 0 ? WASM_STACK_SIZE : stack_size)));
module->socket_descriptor = -1;
@ -193,7 +177,7 @@ err:
struct module *
module_alloc(char *name, char *path, uint32_t stack_size, uint32_t relative_deadline_us, int port, int request_size,
int response_size, int admissions_percentile, uint32_t expected_execution_us)
int response_size, int admissions_percentile, uint32_t expected_execution_us, char *response_content_type)
{
/* Validate presence of required fields */
if (strlen(name) == 0) panic("name field is required\n");
@ -224,7 +208,7 @@ module_alloc(char *name, char *path, uint32_t stack_size, uint32_t relative_dead
};
int rc = module_init(module, name, path, stack_size, relative_deadline_us, port, request_size, response_size,
admissions_percentile, expected_execution_us);
admissions_percentile, expected_execution_us, response_content_type);
if (rc < 0) goto init_err;
done:
@ -236,152 +220,3 @@ err:
module = NULL;
goto done;
}
/**
* Parses a JSON file and allocates one or more new modules
* @param file_name The path of the JSON file
* @return RC 0 on Success. -1 on Error
*/
int
module_alloc_from_json(const char *json_buf, ssize_t json_buf_size)
{
assert(json_buf != NULL);
assert(json_buf_size > 0);
int return_code = -1;
jsmntok_t tokens[JSON_MAX_ELEMENT_SIZE * JSON_MAX_ELEMENT_COUNT];
/* Initialize the Jasmine Parser and an array to hold the tokens */
jsmn_parser module_parser;
jsmn_init(&module_parser);
/* Use Jasmine to parse the JSON */
int total_tokens = jsmn_parse(&module_parser, json_buf, json_buf_size, tokens,
sizeof(tokens) / sizeof(tokens[0]));
if (total_tokens < 0) {
if (total_tokens == JSMN_ERROR_INVAL) {
fprintf(stderr, "Error parsing %s: bad token, JSON string is corrupted\n", json_buf);
} else if (total_tokens == JSMN_ERROR_PART) {
fprintf(stderr, "Error parsing %s: JSON string is too short, expecting more JSON data\n",
json_buf);
} else if (total_tokens == JSMN_ERROR_NOMEM) {
/*
* According to the README at https://github.com/zserge/jsmn, this is a potentially recoverable
* error. More tokens can be allocated and jsmn_parse can be re-invoked.
*/
fprintf(stderr, "Error parsing %s: Not enough tokens, JSON string is too large\n", json_buf);
}
goto json_parse_err;
}
int module_count = 0;
for (int i = 0; i < total_tokens; i++) {
/* If we have multiple objects, they should be wrapped in a JSON array */
if (tokens[i].type == JSMN_ARRAY) continue;
assert(tokens[i].type == JSMN_OBJECT);
char module_name[MODULE_MAX_NAME_LENGTH] = { 0 };
char module_path[MODULE_MAX_PATH_LENGTH] = { 0 };
int32_t request_size = 0;
int32_t response_size = 0;
uint32_t port = 0;
uint32_t relative_deadline_us = 0;
uint32_t expected_execution_us = 0;
int admissions_percentile = 50;
int j = 1;
int ntoks = 2 * tokens[i].size;
char response_content_type[HTTP_MAX_HEADER_VALUE_LENGTH] = { 0 };
for (; j < ntoks;) {
int ntks = 1;
char key[32] = { 0 };
char val[256] = { 0 };
sprintf(val, "%.*s", tokens[j + i + 1].end - tokens[j + i + 1].start,
json_buf + tokens[j + i + 1].start);
sprintf(key, "%.*s", tokens[j + i].end - tokens[j + i].start, json_buf + tokens[j + i].start);
if (strlen(key) == 0) panic("Unexpected encountered empty key\n");
if (strlen(val) == 0) panic("%s field contained empty string\n", key);
if (strcmp(key, "name") == 0) {
// TODO: Currently, multiple modules can have identical names. Ports are the true unique
// identifiers. Consider enforcing unique names in future
strcpy(module_name, val);
} else if (strcmp(key, "path") == 0) {
// Invalid path will crash on dlopen
strcpy(module_path, val);
} else if (strcmp(key, "port") == 0) {
// Validate sane port
// If already taken, will error on bind call in module_listen
int buffer = atoi(val);
if (buffer < 0 || buffer > 65535)
panic("Expected port between 0 and 65535, saw %d\n", buffer);
port = buffer;
} else if (strcmp(key, "relative-deadline-us") == 0) {
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > (int64_t)RUNTIME_RELATIVE_DEADLINE_US_MAX)
panic("Relative-deadline-us must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_RELATIVE_DEADLINE_US_MAX, buffer);
relative_deadline_us = (uint32_t)buffer;
} else if (strcmp(key, "expected-execution-us") == 0) {
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > (int64_t)RUNTIME_EXPECTED_EXECUTION_US_MAX)
panic("Relative-deadline-us must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_EXPECTED_EXECUTION_US_MAX, buffer);
expected_execution_us = (uint32_t)buffer;
} else if (strcmp(key, "admissions-percentile") == 0) {
int32_t buffer = strtol(val, NULL, 10);
if (buffer > 99 || buffer < 50)
panic("admissions-percentile must be > 50 and <= 99 but was %d\n", buffer);
admissions_percentile = (int)buffer;
} else if (strcmp(key, "http-req-size") == 0) {
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > RUNTIME_HTTP_REQUEST_SIZE_MAX)
panic("http-req-size must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_HTTP_REQUEST_SIZE_MAX, buffer);
request_size = (int32_t)buffer;
} else if (strcmp(key, "http-resp-size") == 0) {
int64_t buffer = strtoll(val, NULL, 10);
if (buffer < 0 || buffer > RUNTIME_HTTP_REQUEST_SIZE_MAX)
panic("http-resp-size must be between 0 and %ld, was %ld\n",
(int64_t)RUNTIME_HTTP_REQUEST_SIZE_MAX, buffer);
response_size = (int32_t)buffer;
} else if (strcmp(key, "http-resp-content-type") == 0) {
if (strlen(val) == 0) panic("http-resp-content-type was unexpectedly an empty string");
strcpy(response_content_type, val);
} else {
#ifdef LOG_MODULE_LOADING
debuglog("Invalid (%s,%s)\n", key, val);
#endif
}
j += ntks;
}
i += ntoks;
/* Allocate a module based on the values from the JSON */
struct module *module = module_alloc(module_name, module_path, 0, relative_deadline_us, port,
request_size, response_size, admissions_percentile,
expected_execution_us);
if (module == NULL) goto module_alloc_err;
assert(module);
module_set_http_info(module, response_content_type);
module_count++;
}
if (module_count == 0) panic("%s contained no active modules\n", json_buf);
#ifdef LOG_MODULE_LOADING
debuglog("Loaded %d module%s!\n", module_count, module_count > 1 ? "s" : "");
#endif
return_code = 0;
done:
return return_code;
module_alloc_err:
json_parse_err:
err:
return_code = -1;
goto done;
}

Loading…
Cancel
Save