Updated deferable server (#352)
* refactor: use var accross the Makefile instead of sledgert string * refactor bash libraries - remove hash symbol for scv_to_dat - add set_print_pretty for gdb mode - add logging for client - change printf format for perf log in table from float to int since usec * cleanup: generalize redundant files in the experiments: - remove individual gitignore, have single gitignore in parent (tests) - remove individual env files, have single copies in 'common' dir - remove individual install.sh, have install_tools.sh (in bash libs) * - add comment into install_tools.sh - rename mts to mtds * update the env files dir to common * move multi-tenancy env files to another directory (temp) * apply the deferable server to the new master * add the deferrable server env file to github CI test * clang-format * add deferable server attributes to all experiment specs * remove previously added generic interface that was only for Def Serv * Accomodated important changes Sean requested: - remove unnecessary assertions from pri_queue.h - set the runtime queue size to MAX_TENANT - add the scheduler updates back to the sandbox_interrupt state transitionmaster
Emil
3 years ago
committed by
GitHub
parent
9946f23eb7
commit
cb09ed51ea
@ -1 +1 @@
|
|||||||
Subproject commit 20a7c88816c8f8882e03d42c76ff8c1e72bfeaec
|
Subproject commit 0b9f67d75fd9dab652e1995e7adf91806080523b
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "global_request_scheduler.h"
|
||||||
|
|
||||||
|
void global_request_scheduler_mtds_initialize();
|
||||||
|
int global_request_scheduler_mtds_remove_with_mt_class(struct sandbox **, uint64_t, enum MULTI_TENANCY_CLASS);
|
||||||
|
uint64_t global_request_scheduler_mtds_guaranteed_peek();
|
||||||
|
uint64_t global_request_scheduler_mtds_default_peek();
|
||||||
|
void global_timeout_queue_add(struct tenant *);
|
||||||
|
void global_request_scheduler_mtds_promote_lock(struct tenant_global_request_queue *);
|
||||||
|
void global_request_scheduler_mtds_demote_nolock(struct tenant_global_request_queue *);
|
||||||
|
void global_timeout_queue_process_promotions();
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "tenant.h"
|
||||||
|
|
||||||
|
void local_runqueue_mtds_initialize();
|
||||||
|
void local_runqueue_mtds_promote(struct perworker_tenant_sandbox_queue *);
|
||||||
|
void local_runqueue_mtds_demote(struct perworker_tenant_sandbox_queue *);
|
||||||
|
void local_timeout_queue_add(struct tenant *);
|
||||||
|
void local_timeout_queue_process_promotions();
|
||||||
@ -0,0 +1,331 @@
|
|||||||
|
#include <assert.h>
|
||||||
|
#include <errno.h>
|
||||||
|
|
||||||
|
#include "global_request_scheduler.h"
|
||||||
|
#include "listener_thread.h"
|
||||||
|
#include "panic.h"
|
||||||
|
#include "priority_queue.h"
|
||||||
|
#include "runtime.h"
|
||||||
|
#include "tenant_functions.h"
|
||||||
|
|
||||||
|
static struct priority_queue *global_request_scheduler_mtds_guaranteed;
|
||||||
|
static struct priority_queue *global_request_scheduler_mtds_default;
|
||||||
|
static struct priority_queue *global_tenant_timeout_queue;
|
||||||
|
static lock_t global_lock;
|
||||||
|
|
||||||
|
static inline uint64_t
|
||||||
|
tenant_request_queue_get_priority(void *element)
|
||||||
|
{
|
||||||
|
struct tenant_global_request_queue *tgrq = (struct tenant_global_request_queue *)element;
|
||||||
|
struct sandbox *sandbox = NULL;
|
||||||
|
priority_queue_top_nolock(tgrq->sandbox_requests, (void **)&sandbox);
|
||||||
|
return (sandbox) ? sandbox->absolute_deadline : UINT64_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Demotes the given tenant's request queue, which means deletes the TGRQ from the Guaranteed queue
|
||||||
|
* and adds to the Default queue.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
global_request_scheduler_mtds_demote_nolock(struct tenant_global_request_queue *tgrq)
|
||||||
|
{
|
||||||
|
assert(tgrq != NULL);
|
||||||
|
|
||||||
|
if (tgrq->mt_class == MT_DEFAULT) return;
|
||||||
|
|
||||||
|
/* Delete the corresponding TGRQ from the Guaranteed queue */
|
||||||
|
int rc = priority_queue_delete_nolock(global_request_scheduler_mtds_guaranteed, tgrq);
|
||||||
|
if (rc == -1) {
|
||||||
|
panic("Tried to delete a non-present TGRQ from the Global Guaranteed queue. Already deleted?, ITS "
|
||||||
|
"SIZE: %d",
|
||||||
|
priority_queue_length_nolock(tgrq->sandbox_requests));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add the corresponding TGRQ to the Default queue */
|
||||||
|
rc = priority_queue_enqueue_nolock(global_request_scheduler_mtds_default, tgrq);
|
||||||
|
if (rc == -ENOSPC) panic("Global Default queue is full!\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pushes a sandbox request to the global runqueue
|
||||||
|
* @param sandbox
|
||||||
|
* @returns pointer to request if added. NULL otherwise
|
||||||
|
*/
|
||||||
|
static struct sandbox *
|
||||||
|
global_request_scheduler_mtds_add(struct sandbox *sandbox)
|
||||||
|
{
|
||||||
|
assert(sandbox);
|
||||||
|
assert(global_request_scheduler_mtds_guaranteed && global_request_scheduler_mtds_default);
|
||||||
|
if (unlikely(!listener_thread_is_running())) panic("%s is only callable by the listener thread\n", __func__);
|
||||||
|
|
||||||
|
struct tenant_global_request_queue *tgrq = sandbox->tenant->tgrq_requests;
|
||||||
|
|
||||||
|
LOCK_LOCK(&global_lock);
|
||||||
|
|
||||||
|
struct priority_queue *destination_queue = global_request_scheduler_mtds_default;
|
||||||
|
if (sandbox->tenant->tgrq_requests->mt_class == MT_GUARANTEED) {
|
||||||
|
destination_queue = global_request_scheduler_mtds_guaranteed;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t last_mrq_deadline = priority_queue_peek(tgrq->sandbox_requests);
|
||||||
|
|
||||||
|
int rc = priority_queue_enqueue_nolock(tgrq->sandbox_requests, sandbox);
|
||||||
|
if (rc == -ENOSPC) panic("Tenant's Request Queue is full\n");
|
||||||
|
// debuglog("Added a sandbox to the TGRQ");
|
||||||
|
|
||||||
|
/* Maintain the minheap structure by Removing and Adding the TGRQ from and to the global runqueue.
|
||||||
|
* Do this only when the TGRQ's priority is updated.
|
||||||
|
*/
|
||||||
|
if (priority_queue_peek(tgrq->sandbox_requests) < last_mrq_deadline) {
|
||||||
|
priority_queue_delete_nolock(destination_queue, tgrq);
|
||||||
|
|
||||||
|
rc = priority_queue_enqueue_nolock(destination_queue, tgrq);
|
||||||
|
if (rc == -ENOSPC) panic("Global Runqueue is full!\n");
|
||||||
|
// debuglog("Added the TGRQ back to the Global runqueue - %s to Heapify", QUEUE_NAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
LOCK_UNLOCK(&global_lock);
|
||||||
|
|
||||||
|
return sandbox;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param pointer to the pointer that we want to set to the address of the removed sandbox request
|
||||||
|
* @returns 0 if successful, -ENOENT if empty
|
||||||
|
*/
|
||||||
|
int
|
||||||
|
global_request_scheduler_mtds_remove(struct sandbox **removed_sandbox)
|
||||||
|
{
|
||||||
|
/* This function won't be used with the MTDS scheduler. Keeping merely for the polymorhism. */
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param removed_sandbox pointer to set to removed sandbox request
|
||||||
|
* @param target_deadline the deadline that the request must be earlier than to dequeue
|
||||||
|
* @returns 0 if successful, -ENOENT if empty or if request isn't earlier than target_deadline
|
||||||
|
*/
|
||||||
|
int
|
||||||
|
global_request_scheduler_mtds_remove_if_earlier(struct sandbox **removed_sandbox, uint64_t target_deadline)
|
||||||
|
{
|
||||||
|
/* This function won't be used with the MTDS scheduler. Keeping merely for the polymorhism. */
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param removed_sandbox pointer to set to removed sandbox request
|
||||||
|
* @param target_deadline the deadline that the request must be earlier than to dequeue
|
||||||
|
* @param mt_class the multi-tenancy class of the global request to compare the target deadline against
|
||||||
|
* @returns 0 if successful, -ENOENT if empty or if request isn't earlier than target_deadline
|
||||||
|
*/
|
||||||
|
int
|
||||||
|
global_request_scheduler_mtds_remove_with_mt_class(struct sandbox **removed_sandbox, uint64_t target_deadline,
|
||||||
|
enum MULTI_TENANCY_CLASS target_mt_class)
|
||||||
|
{
|
||||||
|
int rc = -ENOENT;
|
||||||
|
;
|
||||||
|
|
||||||
|
LOCK_LOCK(&global_lock);
|
||||||
|
|
||||||
|
/* Avoid unnessary locks when the target_deadline is tighter than the head of the Global runqueue */
|
||||||
|
uint64_t global_guaranteed_deadline = priority_queue_peek(global_request_scheduler_mtds_guaranteed);
|
||||||
|
uint64_t global_default_deadline = priority_queue_peek(global_request_scheduler_mtds_default);
|
||||||
|
|
||||||
|
switch (target_mt_class) {
|
||||||
|
case MT_GUARANTEED:
|
||||||
|
if (global_guaranteed_deadline >= target_deadline) goto done;
|
||||||
|
break;
|
||||||
|
case MT_DEFAULT:
|
||||||
|
if (global_guaranteed_deadline == UINT64_MAX && global_default_deadline >= target_deadline) goto done;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct tenant_global_request_queue *top_tgrq = NULL;
|
||||||
|
struct priority_queue *destination_queue = global_request_scheduler_mtds_guaranteed;
|
||||||
|
|
||||||
|
/* Spot the Tenant Global Request Queue (TGRQ) to remove the sandbox request from */
|
||||||
|
rc = priority_queue_top_nolock(destination_queue, (void **)&top_tgrq);
|
||||||
|
if (rc == -ENOENT) {
|
||||||
|
if (target_mt_class == MT_GUARANTEED) goto done;
|
||||||
|
|
||||||
|
destination_queue = global_request_scheduler_mtds_default;
|
||||||
|
|
||||||
|
rc = priority_queue_top_nolock(destination_queue, (void **)&top_tgrq);
|
||||||
|
if (rc == -ENOENT) goto done;
|
||||||
|
} else {
|
||||||
|
if (top_tgrq->mt_class == MT_GUARANTEED && top_tgrq->tenant->remaining_budget <= 0) {
|
||||||
|
global_request_scheduler_mtds_demote_nolock(top_tgrq);
|
||||||
|
// debuglog("Demoted '%s' GLOBALLY", top_tgrq->tenant->name);
|
||||||
|
top_tgrq->mt_class = MT_DEFAULT;
|
||||||
|
|
||||||
|
rc = -ENOENT;
|
||||||
|
goto done;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert(top_tgrq);
|
||||||
|
|
||||||
|
/* Remove the sandbox from the corresponding TGRQ */
|
||||||
|
rc = priority_queue_dequeue_nolock(top_tgrq->sandbox_requests, (void **)removed_sandbox);
|
||||||
|
assert(rc == 0);
|
||||||
|
|
||||||
|
/* Delete the TGRQ from the global runqueue completely if TGRQ is empty, re-add otherwise to heapify */
|
||||||
|
if (priority_queue_delete_nolock(destination_queue, top_tgrq) == -1) {
|
||||||
|
panic("Tried to delete an TGRQ from the Global runqueue, but was not present");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (priority_queue_length_nolock(top_tgrq->sandbox_requests) > 0) {
|
||||||
|
priority_queue_enqueue_nolock(destination_queue, top_tgrq);
|
||||||
|
}
|
||||||
|
|
||||||
|
done:
|
||||||
|
LOCK_UNLOCK(&global_lock);
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Peek at the priority of the highest priority task without having to take the lock
|
||||||
|
* Because this is a min-heap PQ, the highest priority is the lowest 64-bit integer
|
||||||
|
* This is used to store an absolute deadline
|
||||||
|
* @returns value of highest priority value in queue or ULONG_MAX if empty
|
||||||
|
*/
|
||||||
|
static uint64_t
|
||||||
|
global_request_scheduler_mtds_peek(void)
|
||||||
|
{
|
||||||
|
uint64_t val = priority_queue_peek(global_request_scheduler_mtds_guaranteed);
|
||||||
|
if (val == UINT64_MAX) val = priority_queue_peek(global_request_scheduler_mtds_default);
|
||||||
|
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
uint64_t
|
||||||
|
global_request_scheduler_mtds_guaranteed_peek(void)
|
||||||
|
{
|
||||||
|
return priority_queue_peek(global_request_scheduler_mtds_guaranteed);
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t
|
||||||
|
global_request_scheduler_mtds_default_peek(void)
|
||||||
|
{
|
||||||
|
return priority_queue_peek(global_request_scheduler_mtds_default);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initializes the variant and registers against the polymorphic interface
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
global_request_scheduler_mtds_initialize()
|
||||||
|
{
|
||||||
|
global_request_scheduler_mtds_guaranteed = priority_queue_initialize(RUNTIME_MAX_TENANT_COUNT, false,
|
||||||
|
tenant_request_queue_get_priority);
|
||||||
|
global_request_scheduler_mtds_default = priority_queue_initialize(RUNTIME_MAX_TENANT_COUNT, false,
|
||||||
|
tenant_request_queue_get_priority);
|
||||||
|
|
||||||
|
global_tenant_timeout_queue = priority_queue_initialize(RUNTIME_MAX_TENANT_COUNT, false,
|
||||||
|
tenant_timeout_get_priority);
|
||||||
|
|
||||||
|
LOCK_INIT(&global_lock);
|
||||||
|
|
||||||
|
struct global_request_scheduler_config config = {
|
||||||
|
.add_fn = global_request_scheduler_mtds_add,
|
||||||
|
.remove_fn = global_request_scheduler_mtds_remove,
|
||||||
|
.remove_if_earlier_fn = global_request_scheduler_mtds_remove_if_earlier,
|
||||||
|
.peek_fn = global_request_scheduler_mtds_peek
|
||||||
|
};
|
||||||
|
|
||||||
|
global_request_scheduler_initialize(&config);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
global_request_scheduler_mtds_free()
|
||||||
|
{
|
||||||
|
priority_queue_free(global_request_scheduler_mtds_guaranteed);
|
||||||
|
priority_queue_free(global_request_scheduler_mtds_default);
|
||||||
|
priority_queue_free(global_tenant_timeout_queue);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void
|
||||||
|
global_timeout_queue_add(struct tenant *tenant)
|
||||||
|
{
|
||||||
|
tenant->tgrq_requests->tenant_timeout.timeout = get_next_timeout_of_tenant(tenant->replenishment_period);
|
||||||
|
priority_queue_enqueue_nolock(global_tenant_timeout_queue, &tenant->tgrq_requests->tenant_timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Promotes the given tenant, which means deletes from the Default queue
|
||||||
|
* and adds to the Guaranteed queue.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
global_request_scheduler_mtds_promote_lock(struct tenant_global_request_queue *tgrq)
|
||||||
|
{
|
||||||
|
assert(tgrq != NULL);
|
||||||
|
// assert(priority_queue_length_nolock(tgrq->sandbox_requests) == 0);
|
||||||
|
|
||||||
|
LOCK_LOCK(&global_lock);
|
||||||
|
|
||||||
|
if (tgrq->mt_class == MT_GUARANTEED) goto done;
|
||||||
|
if (priority_queue_length_nolock(tgrq->sandbox_requests) == 0) goto done;
|
||||||
|
|
||||||
|
/* Delete the corresponding TGRQ from the Guaranteed queue */
|
||||||
|
int rc = priority_queue_delete_nolock(global_request_scheduler_mtds_default, tgrq);
|
||||||
|
if (rc == -1) {
|
||||||
|
panic("Tried to delete a non-present TGRQ from the Global Default queue. Already deleted?, ITS SIZE: "
|
||||||
|
"%d",
|
||||||
|
priority_queue_length_nolock(tgrq->sandbox_requests));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add the corresponding TGRQ to the Default queue */
|
||||||
|
rc = priority_queue_enqueue_nolock(global_request_scheduler_mtds_guaranteed, tgrq);
|
||||||
|
if (rc == -ENOSPC) panic("Global Guaranteed queue is full!\n");
|
||||||
|
|
||||||
|
done:
|
||||||
|
LOCK_UNLOCK(&global_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Checks if there are any tenant timers that run out in the GLOBAL queue,
|
||||||
|
* if so promote that tenant.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
global_timeout_queue_process_promotions()
|
||||||
|
{
|
||||||
|
struct tenant_timeout *top_tenant_timeout = NULL;
|
||||||
|
|
||||||
|
/* Check the timeout queue for a potential tenant to get PRomoted */
|
||||||
|
priority_queue_top_nolock(global_tenant_timeout_queue, (void **)&top_tenant_timeout);
|
||||||
|
if (top_tenant_timeout == NULL) return; // no guaranteed tenants
|
||||||
|
|
||||||
|
struct tenant *tenant = NULL;
|
||||||
|
struct tenant_global_request_queue *tgrq_to_promote = NULL;
|
||||||
|
uint64_t now = __getcycles();
|
||||||
|
int64_t prev_budget;
|
||||||
|
|
||||||
|
while (now >= top_tenant_timeout->timeout) {
|
||||||
|
tenant = top_tenant_timeout->tenant;
|
||||||
|
tgrq_to_promote = tenant->tgrq_requests;
|
||||||
|
|
||||||
|
if (tgrq_to_promote->mt_class == MT_DEFAULT) {
|
||||||
|
if (priority_queue_length_nolock(tgrq_to_promote->sandbox_requests) > 0)
|
||||||
|
global_request_scheduler_mtds_promote_lock(tgrq_to_promote);
|
||||||
|
tgrq_to_promote->mt_class = MT_GUARANTEED;
|
||||||
|
// debuglog("Promoted '%s' GLOBALLY", tenant->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: We need a smarter technique to reset budget to consider budget overusage:
|
||||||
|
prev_budget = atomic_load(&tenant->remaining_budget);
|
||||||
|
while (!atomic_compare_exchange_strong(&tenant->remaining_budget, &prev_budget, tenant->max_budget))
|
||||||
|
;
|
||||||
|
|
||||||
|
/* Reheapify the timeout queue with the updated timeout value of the tenant */
|
||||||
|
priority_queue_delete_nolock(global_tenant_timeout_queue, top_tenant_timeout);
|
||||||
|
top_tenant_timeout->timeout = get_next_timeout_of_tenant(tenant->replenishment_period);
|
||||||
|
priority_queue_enqueue_nolock(global_tenant_timeout_queue, top_tenant_timeout);
|
||||||
|
|
||||||
|
priority_queue_top_nolock(global_tenant_timeout_queue, (void **)&top_tenant_timeout);
|
||||||
|
now = __getcycles();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,262 @@
|
|||||||
|
#include <stdint.h>
|
||||||
|
#include <threads.h>
|
||||||
|
|
||||||
|
#include "arch/context.h"
|
||||||
|
#include "current_sandbox.h"
|
||||||
|
#include "debuglog.h"
|
||||||
|
#include "global_request_scheduler.h"
|
||||||
|
#include "local_runqueue.h"
|
||||||
|
#include "local_runqueue_mtds.h"
|
||||||
|
#include "panic.h"
|
||||||
|
#include "priority_queue.h"
|
||||||
|
#include "sandbox_functions.h"
|
||||||
|
#include "tenant_functions.h"
|
||||||
|
#include "runtime.h"
|
||||||
|
|
||||||
|
thread_local static struct priority_queue *local_runqueue_mtds_guaranteed;
|
||||||
|
thread_local static struct priority_queue *local_runqueue_mtds_default;
|
||||||
|
|
||||||
|
extern __thread struct priority_queue *worker_thread_timeout_queue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get Per-Worker-Tenant priority for Priority Queue ordering
|
||||||
|
* @param element the PWT
|
||||||
|
* @returns the priority of the head of the PWT
|
||||||
|
*/
|
||||||
|
static inline uint64_t
|
||||||
|
perworker_tenant_get_priority(void *element)
|
||||||
|
{
|
||||||
|
struct perworker_tenant_sandbox_queue *pwt = (struct perworker_tenant_sandbox_queue *)element;
|
||||||
|
struct sandbox *sandbox = NULL;
|
||||||
|
priority_queue_top_nolock(pwt->sandboxes, (void **)&sandbox);
|
||||||
|
return (sandbox) ? sandbox->absolute_deadline : UINT64_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if the run queue is empty
|
||||||
|
* @returns true if empty. false otherwise
|
||||||
|
*/
|
||||||
|
bool
|
||||||
|
local_runqueue_mtds_is_empty()
|
||||||
|
{
|
||||||
|
return priority_queue_length_nolock(local_runqueue_mtds_guaranteed) == 0
|
||||||
|
&& priority_queue_length_nolock(local_runqueue_mtds_default) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds a sandbox to the run queue
|
||||||
|
* @param sandbox
|
||||||
|
* @returns pointer to sandbox added
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
local_runqueue_mtds_add(struct sandbox *sandbox)
|
||||||
|
{
|
||||||
|
assert(sandbox != NULL);
|
||||||
|
|
||||||
|
struct perworker_tenant_sandbox_queue *pwt = &sandbox->tenant->pwt_sandboxes[worker_thread_idx];
|
||||||
|
struct priority_queue *destination_queue = pwt->mt_class == MT_GUARANTEED ? local_runqueue_mtds_guaranteed
|
||||||
|
: local_runqueue_mtds_default;
|
||||||
|
|
||||||
|
uint64_t prev_pwt_deadline = priority_queue_peek(pwt->sandboxes);
|
||||||
|
|
||||||
|
/* Add the sandbox to the per-worker-tenant (pwt) queue */
|
||||||
|
int rc = priority_queue_enqueue_nolock(pwt->sandboxes, sandbox);
|
||||||
|
if (rc == -ENOSPC) panic("Per Worker Tenant queue is full!\n");
|
||||||
|
// debuglog("Added a sandbox to the PWT of tenant '%s'", sandbox->tenant->name);
|
||||||
|
|
||||||
|
/* If the tenant was not in the local runqueue, then since we are the first sandbox for it in this worker, add
|
||||||
|
* the tenant. */
|
||||||
|
if (priority_queue_length_nolock(pwt->sandboxes) == 1) {
|
||||||
|
/* Add sandbox tenant to the worker's timeout queue if guaranteed tenant and only if first sandbox*/
|
||||||
|
if (tenant_is_paid(sandbox->tenant)) {
|
||||||
|
pwt->tenant_timeout.timeout = get_next_timeout_of_tenant(sandbox->tenant->replenishment_period);
|
||||||
|
priority_queue_enqueue_nolock(worker_thread_timeout_queue, &pwt->tenant_timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
rc = priority_queue_enqueue_nolock(destination_queue, pwt);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sandbox->absolute_deadline < prev_pwt_deadline) {
|
||||||
|
/* Maintain the minheap structure by deleting & adding the pwt.
|
||||||
|
* Do this only when the pwt's priority is updated. */
|
||||||
|
rc = priority_queue_delete_nolock(destination_queue, pwt);
|
||||||
|
assert(rc == 0);
|
||||||
|
rc = priority_queue_enqueue_nolock(destination_queue, pwt);
|
||||||
|
if (rc == -ENOSPC) panic("Worker Local Runqueue is full!\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes a sandbox from the runqueue
|
||||||
|
* @param sandbox to delete
|
||||||
|
*/
|
||||||
|
static void
|
||||||
|
local_runqueue_mtds_delete(struct sandbox *sandbox)
|
||||||
|
{
|
||||||
|
assert(sandbox != NULL);
|
||||||
|
struct perworker_tenant_sandbox_queue *pwt = &sandbox->tenant->pwt_sandboxes[worker_thread_idx];
|
||||||
|
|
||||||
|
/* Delete the sandbox from the corresponding Per-Worker-Tenant queue */
|
||||||
|
if (priority_queue_delete_nolock(pwt->sandboxes, sandbox) == -1) {
|
||||||
|
panic("Tried to delete sandbox %lu from PWT queue, but was not present\n", sandbox->id);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct priority_queue *destination_queue = local_runqueue_mtds_default;
|
||||||
|
|
||||||
|
if (pwt->mt_class == MT_GUARANTEED) { destination_queue = local_runqueue_mtds_guaranteed; }
|
||||||
|
|
||||||
|
|
||||||
|
/* Delete the PWT from the local runqueue completely if pwt is empty, re-add otherwise to heapify */
|
||||||
|
if (priority_queue_delete_nolock(destination_queue, pwt) == -1) {
|
||||||
|
// TODO: Apply the composite way of PQ deletion O(logn)
|
||||||
|
panic("Tried to delete a PWT of %s from local runqueue, but was not present\n", pwt->tenant->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add the PWT back to the local runqueue if it still has other sandboxes inside */
|
||||||
|
if (priority_queue_length_nolock(pwt->sandboxes) > 0) {
|
||||||
|
priority_queue_enqueue_nolock(destination_queue, pwt);
|
||||||
|
} else if (tenant_is_paid(pwt->tenant)) {
|
||||||
|
priority_queue_delete_nolock(worker_thread_timeout_queue, &pwt->tenant_timeout);
|
||||||
|
pwt->mt_class = MT_GUARANTEED;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This function determines the next sandbox to run.
|
||||||
|
* This is the head of the runqueue
|
||||||
|
*
|
||||||
|
* Execute the sandbox at the head of the thread local runqueue
|
||||||
|
* @return the sandbox to execute or NULL if none are available
|
||||||
|
*/
|
||||||
|
struct sandbox *
|
||||||
|
local_runqueue_mtds_get_next()
|
||||||
|
{
|
||||||
|
/* Get the deadline of the sandbox at the head of the local request queue */
|
||||||
|
struct perworker_tenant_sandbox_queue *next_pwt = NULL;
|
||||||
|
struct priority_queue *dq = local_runqueue_mtds_guaranteed;
|
||||||
|
|
||||||
|
/* Check the local guaranteed queue for any potential demotions */
|
||||||
|
int rc = priority_queue_top_nolock(dq, (void **)&next_pwt);
|
||||||
|
while (rc != -ENOENT && next_pwt->tenant->remaining_budget <= 0) { // next_pwt->mt_class==MT_DEFAULT){
|
||||||
|
local_runqueue_mtds_demote(next_pwt);
|
||||||
|
// debuglog("Demoted '%s' locally in GetNext", next_pwt->tenant->name);
|
||||||
|
next_pwt->mt_class = MT_DEFAULT;
|
||||||
|
rc = priority_queue_top_nolock(dq, (void **)&next_pwt);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rc == -ENOENT) {
|
||||||
|
dq = local_runqueue_mtds_default;
|
||||||
|
rc = priority_queue_top_nolock(dq, (void **)&next_pwt);
|
||||||
|
if (rc == -ENOENT) return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct sandbox *next_sandbox = NULL;
|
||||||
|
priority_queue_top_nolock(next_pwt->sandboxes, (void **)&next_sandbox);
|
||||||
|
assert(next_sandbox);
|
||||||
|
|
||||||
|
return next_sandbox;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Registers the PS variant with the polymorphic interface
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
local_runqueue_mtds_initialize()
|
||||||
|
{
|
||||||
|
/* Initialize local state */
|
||||||
|
local_runqueue_mtds_guaranteed = priority_queue_initialize(RUNTIME_MAX_TENANT_COUNT, false,
|
||||||
|
perworker_tenant_get_priority);
|
||||||
|
local_runqueue_mtds_default = priority_queue_initialize(RUNTIME_MAX_TENANT_COUNT, false,
|
||||||
|
perworker_tenant_get_priority);
|
||||||
|
|
||||||
|
/* Register Function Pointers for Abstract Scheduling API */
|
||||||
|
struct local_runqueue_config config = { .add_fn = local_runqueue_mtds_add,
|
||||||
|
.is_empty_fn = local_runqueue_mtds_is_empty,
|
||||||
|
.delete_fn = local_runqueue_mtds_delete,
|
||||||
|
.get_next_fn = local_runqueue_mtds_get_next };
|
||||||
|
|
||||||
|
local_runqueue_initialize(&config);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Promotes the given per-worker-tenant queue, which means deletes from the Default queue
|
||||||
|
* and adds to the Guaranteed queue.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
local_runqueue_mtds_promote(struct perworker_tenant_sandbox_queue *pwt)
|
||||||
|
{
|
||||||
|
assert(pwt != NULL);
|
||||||
|
|
||||||
|
/* Delete the corresponding PWT from the Guaranteed queue */
|
||||||
|
int rc = priority_queue_delete_nolock(local_runqueue_mtds_default, pwt);
|
||||||
|
if (rc == -1) {
|
||||||
|
panic("Tried to delete a non-present PWT of %s from the Local Default queue. Already deleted? , ITS "
|
||||||
|
"SIZE: %d",
|
||||||
|
pwt->tenant->name, priority_queue_length_nolock(pwt->sandboxes));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add the corresponding PWT to the Default queue */
|
||||||
|
rc = priority_queue_enqueue_nolock(local_runqueue_mtds_guaranteed, pwt);
|
||||||
|
if (rc == -ENOSPC) panic("Local Guaranteed queue is full!\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Demotes the given per-worker-tenant queue, which means deletes from the Guaranteed queue
|
||||||
|
* and adds to the Default queue.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
local_runqueue_mtds_demote(struct perworker_tenant_sandbox_queue *pwt)
|
||||||
|
{
|
||||||
|
assert(pwt != NULL);
|
||||||
|
|
||||||
|
/* Delete the corresponding PWT from the Guaranteed queue */
|
||||||
|
int rc = priority_queue_delete_nolock(local_runqueue_mtds_guaranteed, pwt);
|
||||||
|
if (rc == -1) {
|
||||||
|
panic("Tried to delete a non-present PWT of %s from the Local Guaranteed queue. Already deleted? , ITS "
|
||||||
|
"SIZE: %d",
|
||||||
|
pwt->tenant->name, priority_queue_length_nolock(pwt->sandboxes));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add the corresponding PWT to the Default queue */
|
||||||
|
rc = priority_queue_enqueue_nolock(local_runqueue_mtds_default, pwt);
|
||||||
|
if (rc == -ENOSPC) panic("Local Default queue is full!\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Checks if there are any tenant timers that run out in the LOCAL queue,
|
||||||
|
* if so promote that tenant.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
local_timeout_queue_process_promotions()
|
||||||
|
{
|
||||||
|
struct tenant_timeout *top_tenant_timeout = NULL;
|
||||||
|
|
||||||
|
/* Check the timeout queue for a potential tenant to get PRomoted */
|
||||||
|
priority_queue_top_nolock(worker_thread_timeout_queue, (void **)&top_tenant_timeout);
|
||||||
|
if (top_tenant_timeout == NULL) return; // no guaranteed tenants
|
||||||
|
|
||||||
|
struct perworker_tenant_sandbox_queue *pwt_to_promote = NULL;
|
||||||
|
uint64_t now = __getcycles();
|
||||||
|
|
||||||
|
while (now >= top_tenant_timeout->timeout) {
|
||||||
|
pwt_to_promote = top_tenant_timeout->pwt;
|
||||||
|
assert(priority_queue_length_nolock(pwt_to_promote->sandboxes) > 0);
|
||||||
|
|
||||||
|
if (pwt_to_promote->mt_class == MT_DEFAULT) {
|
||||||
|
local_runqueue_mtds_promote(pwt_to_promote);
|
||||||
|
pwt_to_promote->mt_class = MT_GUARANTEED;
|
||||||
|
// debuglog("Promoted '%s' locally", top_tenant_timeout->tenant->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Reheapify the timeout queue with the updated timeout value of the tenant */
|
||||||
|
priority_queue_delete_nolock(worker_thread_timeout_queue, top_tenant_timeout);
|
||||||
|
top_tenant_timeout->timeout = get_next_timeout_of_tenant(
|
||||||
|
top_tenant_timeout->tenant->replenishment_period);
|
||||||
|
priority_queue_enqueue_nolock(worker_thread_timeout_queue, top_tenant_timeout);
|
||||||
|
|
||||||
|
priority_queue_top_nolock(worker_thread_timeout_queue, (void **)&top_tenant_timeout);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
SLEDGE_SCHEDULER=MTDS
|
||||||
|
SLEDGE_DISABLE_PREEMPTION=false
|
||||||
|
SLEDGE_SANDBOX_PERF_LOG=perf.log
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
# Concurrency
|
||||||
|
|
||||||
|
## Question
|
||||||
|
|
||||||
|
_How does increasing levels of concurrent client requests affect tail latency, throughput, and the success/error rate of sandbox execution?_
|
||||||
|
|
||||||
|
## Independent Variable
|
||||||
|
|
||||||
|
- The number of concurrent client requests made at a given time
|
||||||
|
|
||||||
|
## Dependent Variables
|
||||||
|
|
||||||
|
- p50, p90, p99, and p100 latency measured in ms
|
||||||
|
- throughput measures in requests/second
|
||||||
|
- success rate, measures in % of requests that return a 200
|
||||||
|
|
||||||
|
## Assumptions about test environment
|
||||||
|
|
||||||
|
- You have a modern bash shell. My Linux environment shows version 4.4.20(1)-release
|
||||||
|
- `hey` (https://github.com/rakyll/hey) is available in your PATH
|
||||||
|
- You have compiled `sledgert` and the `empty.so` test workload
|
||||||
|
|
||||||
|
## TODO
|
||||||
|
|
||||||
|
- Harden scripts to validate assumptions
|
||||||
|
- Improve error handling in scripts. If `sledgrt` crashes, this charges forward until it hits a divide by error when attempting to clean data that doesn't exist
|
||||||
@ -0,0 +1,21 @@
|
|||||||
|
reset
|
||||||
|
|
||||||
|
set term jpeg
|
||||||
|
set output "latency.jpg"
|
||||||
|
|
||||||
|
set xlabel "Concurrency"
|
||||||
|
set ylabel "Latency (us)"
|
||||||
|
|
||||||
|
set key left top
|
||||||
|
|
||||||
|
set xrange [-5:]
|
||||||
|
set yrange [0:]
|
||||||
|
|
||||||
|
set style histogram columnstacked
|
||||||
|
|
||||||
|
plot 'latency.dat' using 1:8 title 'p100' linetype 0 linecolor 0 with linespoints, \
|
||||||
|
'latency.dat' using 1:7 title 'p99' lt 1 lc 1 w lp, \
|
||||||
|
'latency.dat' using 1:6 title 'p90' lt 2 lc 2 w lp, \
|
||||||
|
'latency.dat' using 1:5 title 'p50' lt 3 lc 3 w lp, \
|
||||||
|
'latency.dat' using 1:4 title 'mean' lt 4 lc 4 w lp, \
|
||||||
|
'latency.dat' using 1:3 title 'min' lt 5 lc 5 w lp
|
||||||
@ -0,0 +1,163 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success rate
|
||||||
|
# Success - The percentage of requests that complete by their deadlines
|
||||||
|
# Throughput - The mean number of successful requests per second
|
||||||
|
# Latency - the rount-trip resonse time (us) of successful requests at the p50, p90, p99, and p100 percentiles
|
||||||
|
|
||||||
|
# Add bash_libraries directory to path
|
||||||
|
__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
|
||||||
|
__run_sh__bash_libraries_relative_path="../../bash_libraries"
|
||||||
|
__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
|
||||||
|
export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
|
||||||
|
|
||||||
|
source csv_to_dat.sh || exit 1
|
||||||
|
source framework.sh || exit 1
|
||||||
|
source generate_gnuplots.sh || exit 1
|
||||||
|
source get_result_count.sh || exit 1
|
||||||
|
source panic.sh || exit 1
|
||||||
|
source path_join.sh || exit 1
|
||||||
|
source percentiles_table.sh || exit 1
|
||||||
|
|
||||||
|
validate_dependencies hey gnuplot jq
|
||||||
|
|
||||||
|
declare -gi iterations=10000
|
||||||
|
declare -gi duration_sec=5
|
||||||
|
declare -ga concurrency=(1 18 20 24 28 32 36 40 50)
|
||||||
|
|
||||||
|
# Execute the experiments concurrently
|
||||||
|
# $1 (hostname)
|
||||||
|
# $2 (results_directory) - a directory where we will store our results
|
||||||
|
run_experiments() {
|
||||||
|
if (($# != 2)); then
|
||||||
|
panic "invalid number of arguments \"$1\""
|
||||||
|
return 1
|
||||||
|
elif [[ -z "$1" ]]; then
|
||||||
|
panic "hostname \"$1\" was empty"
|
||||||
|
return 1
|
||||||
|
elif [[ ! -d "$2" ]]; then
|
||||||
|
panic "directory \"$2\" does not exist"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local hostname="$1"
|
||||||
|
local results_directory="$2"
|
||||||
|
|
||||||
|
|
||||||
|
printf "Running Experiments:\n"
|
||||||
|
for con in "${concurrency[@]}"; do
|
||||||
|
printf "\t%d Concurrency: " "$con"
|
||||||
|
hey -disable-compression -disable-keepalive -disable-redirects -z "$duration_sec"s -n "$iterations" -c "$con" -o csv -m GET -d "30\n" "http://$hostname:10030/fib" > "$results_directory/con$con.csv" 2> /dev/null || {
|
||||||
|
printf "[ERR]\n"
|
||||||
|
panic "experiment failed"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
get_result_count "$results_directory/con$con.csv" || {
|
||||||
|
printf "[ERR]\n"
|
||||||
|
panic "con$con.csv unexpectedly has zero requests"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
printf "[OK]\n"
|
||||||
|
done
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Process the experimental results and generate human-friendly results for success rate, throughput, and latency
|
||||||
|
process_client_results() {
|
||||||
|
if (($# != 1)); then
|
||||||
|
error_msg "invalid number of arguments ($#, expected 1)"
|
||||||
|
return 1
|
||||||
|
elif ! [[ -d "$1" ]]; then
|
||||||
|
error_msg "directory $1 does not exist"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local -r results_directory="$1"
|
||||||
|
|
||||||
|
printf "Processing Results: "
|
||||||
|
|
||||||
|
# Write headers to CSVs
|
||||||
|
printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv"
|
||||||
|
printf "Concurrency,Throughput\n" >> "$results_directory/throughput.csv"
|
||||||
|
percentiles_table_header "$results_directory/latency.csv" "Con"
|
||||||
|
|
||||||
|
for con in "${concurrency[@]}"; do
|
||||||
|
|
||||||
|
if [[ ! -f "$results_directory/con$con.csv" ]]; then
|
||||||
|
printf "[ERR]\n"
|
||||||
|
error_msg "Missing $results_directory/con$con.csv"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Calculate Success Rate for csv (percent of requests that return 200)
|
||||||
|
# P.S. When using hey -z option, this result is meaningless
|
||||||
|
awk -F, '
|
||||||
|
$7 == 200 {ok++}
|
||||||
|
END{printf "'"$con"',%3.2f\n", (ok / (NR - 1) * 100)}
|
||||||
|
' < "$results_directory/con$con.csv" >> "$results_directory/success.csv"
|
||||||
|
|
||||||
|
# Filter on 200s, convert from s to us, and sort
|
||||||
|
awk -F, '$7 == 200 {print ($1 * 1000000)}' < "$results_directory/con$con.csv" \
|
||||||
|
| sort -g > "$results_directory/con$con-response.csv"
|
||||||
|
|
||||||
|
# Get Number of 200s
|
||||||
|
oks=$(wc -l < "$results_directory/con$con-response.csv")
|
||||||
|
((oks == 0)) && continue # If all errors, skip line
|
||||||
|
|
||||||
|
# We determine duration by looking at the timestamp of the last complete request
|
||||||
|
# TODO: Should this instead just use the client-side synthetic duration_sec value?
|
||||||
|
duration=$(tail -n1 "$results_directory/con$con.csv" | cut -d, -f8)
|
||||||
|
|
||||||
|
# Throughput is calculated as the mean number of successful requests per second
|
||||||
|
throughput=$(echo "$oks/$duration" | bc)
|
||||||
|
printf "%d,%d\n" "$con" "$throughput" >> "$results_directory/throughput.csv"
|
||||||
|
|
||||||
|
# Generate Latency Data for csv
|
||||||
|
percentiles_table_row "$results_directory/con$con-response.csv" "$results_directory/latency.csv" "$con"
|
||||||
|
|
||||||
|
# Delete scratch file used for sorting/counting
|
||||||
|
rm -rf "$results_directory/con$con-response.csv"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Transform csvs to dat files for gnuplot
|
||||||
|
csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv"
|
||||||
|
rm "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv"
|
||||||
|
|
||||||
|
# Generate gnuplots
|
||||||
|
generate_gnuplots "$results_directory" "$__run_sh__base_path" || {
|
||||||
|
printf "[ERR]\n"
|
||||||
|
panic "failed to generate gnuplots"
|
||||||
|
}
|
||||||
|
|
||||||
|
printf "[OK]\n"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
experiment_server_post() {
|
||||||
|
local -r results_directory="$1"
|
||||||
|
|
||||||
|
# Only process data if SLEDGE_SANDBOX_PERF_LOG was set when running sledgert
|
||||||
|
if [[ -n "$SLEDGE_SANDBOX_PERF_LOG" ]]; then
|
||||||
|
if [[ -f "$__run_sh__base_path/$SLEDGE_SANDBOX_PERF_LOG" ]]; then
|
||||||
|
mv "$__run_sh__base_path/$SLEDGE_SANDBOX_PERF_LOG" "$results_directory/perf.log"
|
||||||
|
# process_server_results "$results_directory" || return 1
|
||||||
|
else
|
||||||
|
echo "Perf Log was set, but perf.log not found!"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Expected Symbol used by the framework
|
||||||
|
experiment_client() {
|
||||||
|
local -r target_hostname="$1"
|
||||||
|
local -r results_directory="$2"
|
||||||
|
|
||||||
|
#run_samples "$target_hostname" || return 1
|
||||||
|
run_experiments "$target_hostname" "$results_directory" || return 1
|
||||||
|
process_client_results "$results_directory" || return 1
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
framework_init "$@"
|
||||||
@ -0,0 +1,163 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# This experiment is intended to document how the level of concurrent requests influence the latency, throughput, and success/failure rate
|
||||||
|
# Success - The percentage of requests that complete out of the total expected
|
||||||
|
# Throughput - The mean number of successful requests per second
|
||||||
|
# Latency - the rount-trip resonse time (us) of successful requests at the p50, p90, p99, and p100 percentiles
|
||||||
|
|
||||||
|
# Add bash_libraries directory to path
|
||||||
|
__run_sh__base_path="$(dirname "$(realpath --logical "${BASH_SOURCE[0]}")")"
|
||||||
|
__run_sh__bash_libraries_relative_path="../../bash_libraries"
|
||||||
|
__run_sh__bash_libraries_absolute_path=$(cd "$__run_sh__base_path" && cd "$__run_sh__bash_libraries_relative_path" && pwd)
|
||||||
|
export PATH="$__run_sh__bash_libraries_absolute_path:$PATH"
|
||||||
|
|
||||||
|
source csv_to_dat.sh || exit 1
|
||||||
|
source framework.sh || exit 1
|
||||||
|
source generate_gnuplots.sh || exit 1
|
||||||
|
source get_result_count.sh || exit 1
|
||||||
|
source panic.sh || exit 1
|
||||||
|
source path_join.sh || exit 1
|
||||||
|
source percentiles_table.sh || exit 1
|
||||||
|
|
||||||
|
validate_dependencies loadtest gnuplot
|
||||||
|
|
||||||
|
# The global configs for the scripts
|
||||||
|
declare -gi iterations=10000
|
||||||
|
declare -gi duration_sec=5
|
||||||
|
declare -ga concurrency=(1 9 18 20 30 40 60 80 100)
|
||||||
|
declare -gi deadline_us=16000 #10ms for fib30
|
||||||
|
|
||||||
|
# Execute the experiments concurrently
|
||||||
|
# $1 (hostname)
|
||||||
|
# $2 (results_directory) - a directory where we will store our results
|
||||||
|
run_experiments() {
|
||||||
|
if (($# != 2)); then
|
||||||
|
panic "invalid number of arguments \"$1\""
|
||||||
|
return 1
|
||||||
|
elif [[ -z "$1" ]]; then
|
||||||
|
panic "hostname \"$1\" was empty"
|
||||||
|
return 1
|
||||||
|
elif [[ ! -d "$2" ]]; then
|
||||||
|
panic "directory \"$2\" does not exist"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local hostname="$1"
|
||||||
|
local results_directory="$2"
|
||||||
|
|
||||||
|
|
||||||
|
printf "Running Experiments:\n"
|
||||||
|
|
||||||
|
for con in "${concurrency[@]}"; do
|
||||||
|
printf "\t%d Concurrency: " "$con"
|
||||||
|
|
||||||
|
rps=$((1000000/$deadline_us*$con))
|
||||||
|
|
||||||
|
loadtest -t "$duration_sec" -d "$(($deadline_us/1000))" -c "$con" --rps "$rps" -P "30" "http://$hostname:10030/fib" > "$results_directory/con$con.txt" || { #-n "$iterations"
|
||||||
|
printf "[ERR]\n"
|
||||||
|
panic "experiment failed"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
get_result_count "$results_directory/con$con.txt" || {
|
||||||
|
printf "[ERR]\n"
|
||||||
|
panic "con$con unexpectedly has zero requests"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
printf "[OK]\n"
|
||||||
|
done
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Process the experimental results and generate human-friendly results for success rate, throughput, and latency
|
||||||
|
process_client_results() {
|
||||||
|
if (($# != 1)); then
|
||||||
|
error_msg "invalid number of arguments ($#, expected 1)"
|
||||||
|
return 1
|
||||||
|
elif ! [[ -d "$1" ]]; then
|
||||||
|
error_msg "directory $1 does not exist"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local -r results_directory="$1"
|
||||||
|
|
||||||
|
printf "Processing Results: "
|
||||||
|
|
||||||
|
# Write headers to CSVs
|
||||||
|
printf "Concurrency,Success_Rate\n" >> "$results_directory/success.csv"
|
||||||
|
printf "Concurrency,Throughput\n" >> "$results_directory/throughput.csv"
|
||||||
|
percentiles_table_header "$results_directory/latency.csv" "Con"
|
||||||
|
|
||||||
|
for con in "${concurrency[@]}"; do
|
||||||
|
|
||||||
|
if [[ ! -f "$results_directory/con$con.txt" ]]; then
|
||||||
|
printf "[ERR]\n"
|
||||||
|
error_msg "Missing $results_directory/con$con.txt"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get Number of 200s and then calculate Success Rate (percent of requests that return 200)
|
||||||
|
# If using loadtest -n option (not -t), then use ok/iterations instead of ok/total.
|
||||||
|
total=$(grep "Completed requests:" "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 14)
|
||||||
|
missed=$(grep "Total errors:" "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 13)
|
||||||
|
ok=$((total-missed))
|
||||||
|
((ok == 0)) && continue # If all errors, skip line
|
||||||
|
success_rate=$(echo "scale=2; $ok/$total*100"|bc)
|
||||||
|
printf "%d,%3.1f\n" "$con" "$success_rate" >> "$results_directory/success.csv"
|
||||||
|
|
||||||
|
# Throughput is calculated as the mean number of successful requests per second
|
||||||
|
throughput=$(grep "Requests per second" "$results_directory/con$con.txt" | cut -d ' ' -f 14 | tail -n 1)
|
||||||
|
printf "%d,%d\n" "$con" "$throughput" >> "$results_directory/throughput.csv"
|
||||||
|
|
||||||
|
# Generate Latency Data
|
||||||
|
min=0
|
||||||
|
p50=$(echo 1000*"$(grep 50% "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 12)" | bc)
|
||||||
|
p90=$(echo 1000*"$(grep 90% "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 12)" | bc)
|
||||||
|
p99=$(echo 1000*"$(grep 99% "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 12)" | bc)
|
||||||
|
p100=$(echo 1000*"$(grep 100% "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 12 | tail -n 1)" | bc)
|
||||||
|
mean=$(echo 1000*"$(grep "Mean latency:" "$results_directory/con$con.txt" | tr -s ' ' | cut -d ' ' -f 13)" | bc)
|
||||||
|
|
||||||
|
printf "%d,%d,%d,%.2f,%d,%d,%d,%d\n" "$con" "$ok" "$min" "$mean" "$p50" "$p90" "$p99" "$p100" >> "$results_directory/latency.csv"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Transform csvs to dat files for gnuplot
|
||||||
|
csv_to_dat "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv"
|
||||||
|
rm "$results_directory/success.csv" "$results_directory/throughput.csv" "$results_directory/latency.csv"
|
||||||
|
|
||||||
|
# Generate gnuplots
|
||||||
|
generate_gnuplots "$results_directory" "$__run_sh__base_path" || {
|
||||||
|
printf "[ERR]\n"
|
||||||
|
panic "failed to generate gnuplots"
|
||||||
|
}
|
||||||
|
|
||||||
|
printf "[OK]\n"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
experiment_server_post() {
|
||||||
|
local -r results_directory="$1"
|
||||||
|
|
||||||
|
# Only process data if SLEDGE_SANDBOX_PERF_LOG was set when running sledgert
|
||||||
|
if [[ -n "$SLEDGE_SANDBOX_PERF_LOG" ]]; then
|
||||||
|
if [[ -f "$__run_sh__base_path/$SLEDGE_SANDBOX_PERF_LOG" ]]; then
|
||||||
|
mv "$__run_sh__base_path/$SLEDGE_SANDBOX_PERF_LOG" "$results_directory/perf.log"
|
||||||
|
# process_server_results "$results_directory" || return 1
|
||||||
|
else
|
||||||
|
echo "Perf Log was set, but perf.log not found!"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Expected Symbol used by the framework
|
||||||
|
experiment_client() {
|
||||||
|
local -r target_hostname="$1"
|
||||||
|
local -r results_directory="$2"
|
||||||
|
|
||||||
|
#run_samples "$target_hostname" || return 1
|
||||||
|
run_experiments "$target_hostname" "$results_directory" || return 1
|
||||||
|
process_client_results "$results_directory" || return 1
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
framework_init "$@"
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "gwu",
|
||||||
|
"port": 10030,
|
||||||
|
"replenishment-period-us": 0,
|
||||||
|
"max-budget-us": 0,
|
||||||
|
"routes": [
|
||||||
|
{
|
||||||
|
"route": "/fib",
|
||||||
|
"path": "fibonacci.wasm.so",
|
||||||
|
"admissions-percentile": 70,
|
||||||
|
"expected-execution-us": 4000,
|
||||||
|
"relative-deadline-us": 16000,
|
||||||
|
"http-resp-size": 1024,
|
||||||
|
"http-resp-content-type": "text/plain"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
reset
|
||||||
|
|
||||||
|
set term jpeg
|
||||||
|
set output "success.jpg"
|
||||||
|
|
||||||
|
set xlabel "Concurrency"
|
||||||
|
set ylabel "Success (RC=200) rate %"
|
||||||
|
|
||||||
|
set xrange [0:]
|
||||||
|
set yrange [0:110]
|
||||||
|
|
||||||
|
plot 'success.dat' using 1:2 title 'Success rate' linetype 1 linecolor 1 with linespoints
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
reset
|
||||||
|
|
||||||
|
set term jpeg
|
||||||
|
set output "throughput.jpg"
|
||||||
|
|
||||||
|
set xlabel "Concurrency"
|
||||||
|
set ylabel "Requests/sec"
|
||||||
|
|
||||||
|
set xrange [-5:]
|
||||||
|
set yrange [0:]
|
||||||
|
|
||||||
|
plot 'throughput.dat' using 1:2 title 'Reqs/sec' linetype 1 linecolor 1 with linespoints
|
||||||
Loading…
Reference in new issue