|
|
|
|
@ -22,10 +22,48 @@ int runtime_epoll_file_descriptor;
|
|
|
|
|
double runtime_admitted;
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_TOTAL_REQS_RESPS
|
|
|
|
|
_Atomic uint32_t runtime_total_requests;
|
|
|
|
|
_Atomic uint32_t runtime_total_2XX_responses;
|
|
|
|
|
_Atomic uint32_t runtime_total_4XX_responses;
|
|
|
|
|
_Atomic uint32_t runtime_total_5XX_responses;
|
|
|
|
|
_Atomic uint32_t runtime_total_requests = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_2XX_responses = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_4XX_responses = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_5XX_responses = 0;
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
runtime_log_requests_responses()
|
|
|
|
|
{
|
|
|
|
|
int64_t total_responses = runtime_total_2XX_responses + runtime_total_4XX_responses
|
|
|
|
|
+ runtime_total_5XX_responses;
|
|
|
|
|
int64_t outstanding_requests = (int64_t)runtime_total_requests - total_responses;
|
|
|
|
|
|
|
|
|
|
debuglog("Requests: %u (%ld outstanding)\n\tResponses: %ld\n\t\t2XX: %u\n\t\t4XX: %u\n\t\t5XX: %u\n",
|
|
|
|
|
runtime_total_requests, outstanding_requests, total_responses, runtime_total_2XX_responses,
|
|
|
|
|
runtime_total_4XX_responses, runtime_total_5XX_responses);
|
|
|
|
|
};
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_SANDBOX_TOTALS
|
|
|
|
|
_Atomic uint32_t runtime_total_freed_requests = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_initialized_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_runnable_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_blocked_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_running_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_preempted_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_returned_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_error_sandboxes = 0;
|
|
|
|
|
_Atomic uint32_t runtime_total_complete_sandboxes = 0;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Function intended to be interactively run in a debugger to look at sandbox totals
|
|
|
|
|
* via `call runtime_log_sandbox_states()`
|
|
|
|
|
*/
|
|
|
|
|
void
|
|
|
|
|
runtime_log_sandbox_states()
|
|
|
|
|
{
|
|
|
|
|
debuglog("Initialized: %u\n\tRunnable: %u\n\tBlocked: %u\n\tRunning: %u\n\tPreempted: %u\n\tReturned: "
|
|
|
|
|
"%u\n\tError: %u\n\tComplete: %u\n",
|
|
|
|
|
runtime_total_initialized_sandboxes, runtime_total_runnable_sandboxes, runtime_total_blocked_sandboxes,
|
|
|
|
|
runtime_total_running_sandboxes, runtime_total_preempted_sandboxes, runtime_total_returned_sandboxes,
|
|
|
|
|
runtime_total_error_sandboxes, runtime_total_complete_sandboxes);
|
|
|
|
|
};
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/******************************************
|
|
|
|
|
@ -42,15 +80,6 @@ runtime_initialize(void)
|
|
|
|
|
runtime_epoll_file_descriptor = epoll_create1(0);
|
|
|
|
|
assert(runtime_epoll_file_descriptor >= 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_TOTAL_REQS_RESPS
|
|
|
|
|
/* Setup Counts */
|
|
|
|
|
runtime_total_requests = 0;
|
|
|
|
|
runtime_total_2XX_responses = 0;
|
|
|
|
|
runtime_total_4XX_responses = 0;
|
|
|
|
|
runtime_total_5XX_responses = 0;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/* Allocate and Initialize the global deque
|
|
|
|
|
TODO: Improve to expose variant as a config #Issue 93
|
|
|
|
|
*/
|
|
|
|
|
@ -85,8 +114,7 @@ listener_thread_reject(int client_socket)
|
|
|
|
|
int to_send = strlen(HTTP_RESPONSE_504_SERVICE_UNAVAILABLE);
|
|
|
|
|
|
|
|
|
|
while (sent < to_send) {
|
|
|
|
|
rc = write(client_socket, HTTP_RESPONSE_504_SERVICE_UNAVAILABLE,
|
|
|
|
|
strlen(HTTP_RESPONSE_504_SERVICE_UNAVAILABLE));
|
|
|
|
|
rc = write(client_socket, &HTTP_RESPONSE_504_SERVICE_UNAVAILABLE[sent], to_send - sent);
|
|
|
|
|
if (rc < 0) {
|
|
|
|
|
if (errno == EAGAIN) continue;
|
|
|
|
|
|
|
|
|
|
@ -97,7 +125,6 @@ listener_thread_reject(int client_socket)
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_TOTAL_REQS_RESPS
|
|
|
|
|
runtime_total_5XX_responses++;
|
|
|
|
|
runtime_log_requests_responses();
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
close:
|
|
|
|
|
@ -128,15 +155,31 @@ listener_thread_main(void *dummy)
|
|
|
|
|
* Block indefinitely on the epoll file descriptor, waiting on up to a max number of events
|
|
|
|
|
* TODO: Is LISTENER_THREAD_MAX_EPOLL_EVENTS actually limited to the max number of modules?
|
|
|
|
|
*/
|
|
|
|
|
int request_count = epoll_wait(runtime_epoll_file_descriptor, (struct epoll_event *)&epoll_events,
|
|
|
|
|
LISTENER_THREAD_MAX_EPOLL_EVENTS, -1);
|
|
|
|
|
if (request_count < 0) panic("epoll_wait: %s", strerror(errno));
|
|
|
|
|
if (request_count == 0) panic("Unexpectedly returned with epoll_wait timeout not set\n");
|
|
|
|
|
int descriptor_count = epoll_wait(runtime_epoll_file_descriptor, (struct epoll_event *)&epoll_events,
|
|
|
|
|
LISTENER_THREAD_MAX_EPOLL_EVENTS, -1);
|
|
|
|
|
if (descriptor_count < 0) {
|
|
|
|
|
if (errno == EINTR) continue;
|
|
|
|
|
|
|
|
|
|
panic("epoll_wait: %s", strerror(errno));
|
|
|
|
|
}
|
|
|
|
|
if (descriptor_count == 0) panic("Unexpectedly returned with epoll_wait timeout not set\n");
|
|
|
|
|
|
|
|
|
|
/* Capture Start Time to calculate absolute deadline */
|
|
|
|
|
uint64_t request_arrival_timestamp = __getcycles();
|
|
|
|
|
for (int i = 0; i < request_count; i++) {
|
|
|
|
|
if (epoll_events[i].events & EPOLLERR) panic("epoll_wait: %s", strerror(errno));
|
|
|
|
|
for (int i = 0; i < descriptor_count; i++) {
|
|
|
|
|
/* Check Event to determine if epoll returned an error */
|
|
|
|
|
if ((epoll_events[i].events & EPOLLERR) == EPOLLERR) {
|
|
|
|
|
int error = 0;
|
|
|
|
|
socklen_t errlen = sizeof(error);
|
|
|
|
|
if (getsockopt(epoll_events[i].data.fd, SOL_SOCKET, SO_ERROR, (void *)&error, &errlen)
|
|
|
|
|
== 0) {
|
|
|
|
|
panic("epoll_wait: %s\n", strerror(error));
|
|
|
|
|
}
|
|
|
|
|
assert(0);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/* Assumption: We have only registered EPOLLIN events, so we should see no others here */
|
|
|
|
|
assert((epoll_events[i].events & EPOLLIN) == EPOLLIN);
|
|
|
|
|
|
|
|
|
|
/* Unpack module from epoll event */
|
|
|
|
|
struct module *module = (struct module *)epoll_events[i].data.ptr;
|
|
|
|
|
@ -145,80 +188,73 @@ listener_thread_main(void *dummy)
|
|
|
|
|
/* Accept Client Request as a nonblocking socket, saving address information */
|
|
|
|
|
struct sockaddr_in client_address;
|
|
|
|
|
socklen_t address_length = sizeof(client_address);
|
|
|
|
|
int client_socket = accept4(module->socket_descriptor, (struct sockaddr *)&client_address,
|
|
|
|
|
&address_length, SOCK_NONBLOCK);
|
|
|
|
|
if (client_socket < 0) {
|
|
|
|
|
switch (errno) {
|
|
|
|
|
/* Note: Assumes EAGAIN and EWOULDBLOCK are identical, as on Linux */
|
|
|
|
|
case EWOULDBLOCK: {
|
|
|
|
|
/*
|
|
|
|
|
* According to accept(2), it is possible that a connection might
|
|
|
|
|
* have been removed between us receiving an event via epoll_wait
|
|
|
|
|
* and us calling accept. Thus we just want to gracefully ignore the
|
|
|
|
|
* epoll event.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_EPOLL
|
|
|
|
|
debuglog("Encountered an epoll notification for %s that did not actually have "
|
|
|
|
|
"an associated request\n",
|
|
|
|
|
module->name);
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
/*
|
|
|
|
|
* Accept as many requests as possible, terminating when we would have blocked
|
|
|
|
|
* This inner loop is used in case there are more datagrams than epoll events for some reason
|
|
|
|
|
*/
|
|
|
|
|
while (true) {
|
|
|
|
|
int client_socket = accept4(module->socket_descriptor,
|
|
|
|
|
(struct sockaddr *)&client_address, &address_length,
|
|
|
|
|
SOCK_NONBLOCK);
|
|
|
|
|
if (client_socket < 0) {
|
|
|
|
|
if (errno == EWOULDBLOCK || errno == EAGAIN) break;
|
|
|
|
|
|
|
|
|
|
panic("accept4: %s", strerror(errno));
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* According to accept(2), it is possible that the the sockaddr structure client_address may be
|
|
|
|
|
* too small, resulting in data being truncated to fit. The appect call mutates the size value
|
|
|
|
|
* to indicate that this is the case.
|
|
|
|
|
*/
|
|
|
|
|
if (address_length > sizeof(client_address)) {
|
|
|
|
|
debuglog("A client address to %s has been truncated because buffer was too small\n",
|
|
|
|
|
module->name);
|
|
|
|
|
}
|
|
|
|
|
/*
|
|
|
|
|
* According to accept(2), it is possible that the the sockaddr structure client_address
|
|
|
|
|
* may be too small, resulting in data being truncated to fit. The appect call mutates
|
|
|
|
|
* the size value to indicate that this is the case.
|
|
|
|
|
*/
|
|
|
|
|
if (address_length > sizeof(client_address)) {
|
|
|
|
|
debuglog("A client address to %s has been truncated because buffer was too "
|
|
|
|
|
"small\n",
|
|
|
|
|
module->name);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_TOTAL_REQS_RESPS
|
|
|
|
|
runtime_total_requests++;
|
|
|
|
|
runtime_log_requests_responses();
|
|
|
|
|
runtime_total_requests++;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/* Perform Admission Control */
|
|
|
|
|
/* Perform Admission Control */
|
|
|
|
|
|
|
|
|
|
uint32_t estimated_execution = perf_window_get_percentile(&module->perf_window, 0.5);
|
|
|
|
|
/*
|
|
|
|
|
* If this is the first execution, assume a default execution
|
|
|
|
|
* TODO: Enhance module specification to provide "seed" value of estimated duration
|
|
|
|
|
*/
|
|
|
|
|
if (estimated_execution == -1) estimated_execution = 1000;
|
|
|
|
|
uint32_t estimated_execution = perf_window_get_percentile(&module->perf_window, 0.5);
|
|
|
|
|
/*
|
|
|
|
|
* If this is the first execution, assume a default execution
|
|
|
|
|
* TODO: Enhance module specification to provide "seed" value of estimated duration
|
|
|
|
|
*/
|
|
|
|
|
if (estimated_execution == -1) estimated_execution = 1000;
|
|
|
|
|
|
|
|
|
|
double admissions_estimate = (double)estimated_execution / module->relative_deadline;
|
|
|
|
|
double admissions_estimate = (double)estimated_execution / module->relative_deadline;
|
|
|
|
|
|
|
|
|
|
if (runtime_admitted + admissions_estimate >= runtime_worker_threads_count) {
|
|
|
|
|
listener_thread_reject(client_socket);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (runtime_admitted + admissions_estimate >= runtime_worker_threads_count) {
|
|
|
|
|
listener_thread_reject(client_socket);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Allocate a Sandbox Request */
|
|
|
|
|
struct sandbox_request *sandbox_request =
|
|
|
|
|
sandbox_request_allocate(module, module->name, client_socket,
|
|
|
|
|
(const struct sockaddr *)&client_address, request_arrival_timestamp,
|
|
|
|
|
admissions_estimate);
|
|
|
|
|
/* Allocate a Sandbox Request */
|
|
|
|
|
struct sandbox_request *sandbox_request =
|
|
|
|
|
sandbox_request_allocate(module, module->name, client_socket,
|
|
|
|
|
(const struct sockaddr *)&client_address,
|
|
|
|
|
request_arrival_timestamp, admissions_estimate);
|
|
|
|
|
|
|
|
|
|
/* Add to the Global Sandbox Request Scheduler */
|
|
|
|
|
global_request_scheduler_add(sandbox_request);
|
|
|
|
|
/* Clear the */
|
|
|
|
|
epoll_events[i].data.ptr = NULL;
|
|
|
|
|
|
|
|
|
|
/* Add to work accepted by the runtime */
|
|
|
|
|
runtime_admitted += admissions_estimate;
|
|
|
|
|
/* Add to the Global Sandbox Request Scheduler */
|
|
|
|
|
global_request_scheduler_add(sandbox_request);
|
|
|
|
|
|
|
|
|
|
/* Add to work accepted by the runtime */
|
|
|
|
|
runtime_admitted += admissions_estimate;
|
|
|
|
|
|
|
|
|
|
#ifdef LOG_ADMISSIONS_CONTROL
|
|
|
|
|
debuglog("Runtime Admitted: %f / %u\n", runtime_admitted, runtime_worker_threads_count);
|
|
|
|
|
debuglog("Runtime Admitted: %f / %u\n", runtime_admitted, runtime_worker_threads_count);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} /* while true */
|
|
|
|
|
} /* for loop */
|
|
|
|
|
} /* while true */
|
|
|
|
|
|
|
|
|
|
panic("Listener thread unexpectedly broke loop\n");
|
|
|
|
|
}
|
|
|
|
|
|
Sean McBride
5 years ago