ck/regressions/ck_ring/validate/ck_ring_mpmc.c

/*
 * Copyright 2011-2015 Samy Al Bahra.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <pthread.h>

#include <ck_barrier.h>
#include <ck_ring.h>
#include <ck_spinlock.h>
#include "../../common.h"

#ifndef ITERATIONS
#define ITERATIONS 128
#endif

struct context {
	unsigned int tid;
	unsigned int previous;
	unsigned int next;
	ck_ring_buffer_t *buffer;
};

struct entry {
	unsigned long value_long;
	unsigned int magic;
	unsigned int ref;
	int tid;
	int value;
};

static int nthr;
static ck_ring_t *ring;
static ck_ring_t ring_mpmc CK_CC_CACHELINE;
static ck_ring_t ring_mw CK_CC_CACHELINE;
static struct affinity a;
static int size;
static int eb;
static ck_barrier_centralized_t barrier = CK_BARRIER_CENTRALIZED_INITIALIZER;
static struct context *_context;

static unsigned int global_counter;

static void *
test_mpmc(void *c)
{
	unsigned int observed = 0;
	unsigned int enqueue = 0;
	unsigned int seed;
	int i, k, j, tid;
	struct context *context = c;
	ck_ring_buffer_t *buffer;
	unsigned int *csp;

	csp = malloc(sizeof(*csp) * nthr);
	assert(csp != NULL);

	memset(csp, 0, sizeof(*csp) * nthr);

	buffer = context->buffer;
        if (aff_iterate(&a)) {
                perror("ERROR: Could not affine thread");
                exit(EXIT_FAILURE);
        }

	tid = ck_pr_faa_int(&eb, 1);
	ck_pr_fence_memory();
	while (ck_pr_load_int(&eb) != nthr - 1);

	for (i = 0; i < ITERATIONS; i++) {
		for (j = 0; j < size; j++) {
			struct entry *o = NULL;
			int spin;

			/* Keep trying until we encounter at least one node. */
			if (j & 1) {
				if (ck_ring_dequeue_mpmc(&ring_mw, buffer, &o) == false)
					o = NULL;
			} else {
				if (ck_ring_trydequeue_mpmc(&ring_mw, buffer, &o) == false)
					o = NULL;
			}

			if (o == NULL) {
				o = malloc(sizeof(*o));
				if (o == NULL)
					continue;

				o->value_long = (unsigned long)ck_pr_faa_uint(&global_counter, 1) + 1;

				o->magic = 0xdead;
				o->ref = 0;
				o->tid = tid;

				if (ck_ring_enqueue_mpmc(&ring_mw, buffer, o) == false) {
					free(o);
				} else {
					enqueue++;
				}

				continue;
			}

			observed++;

			if (o->magic != 0xdead) {
				ck_error("[%p] (%x)\n",
					(void *)o, o->magic);
			}

			o->magic = 0xbeef;

			if (csp[o->tid] >= o->value_long)
				ck_error("queue semantics violated: %lu <= %lu\n", o->value_long, csp[o->tid]);

			csp[o->tid] = o->value_long;

			if (ck_pr_faa_uint(&o->ref, 1) != 0) {
				ck_error("[%p] We dequeued twice.\n", (void *)o);
			}

			if ((i % 4) == 0) {
				spin = common_rand_r(&seed) % 16384;
				for (k = 0; k < spin; k++) {
					ck_pr_stall();
				}
			}

			free(o);
		}
	}

	fprintf(stderr, "[%d] dequeue=%u enqueue=%u\n", tid, observed, enqueue);
	return NULL;
}

static void *
test_spmc(void *c)
{
	unsigned int observed = 0;
	unsigned long previous = 0;
	unsigned int seed;
	int i, k, j, tid;
	struct context *context = c;
	ck_ring_buffer_t *buffer;

	buffer = context->buffer;
        if (aff_iterate(&a)) {
                perror("ERROR: Could not affine thread");
                exit(EXIT_FAILURE);
        }

	tid = ck_pr_faa_int(&eb, 1);
	ck_pr_fence_memory();
	while (ck_pr_load_int(&eb) != nthr - 1);

	for (i = 0; i < ITERATIONS; i++) {
		for (j = 0; j < size; j++) {
			struct entry *o;
			int spin;

			/* Keep trying until we encounter at least one node. */
			if (j & 1) {
				while (ck_ring_dequeue_mpmc(&ring_mpmc, buffer,
				    &o) == false);
			} else {
				while (ck_ring_trydequeue_mpmc(&ring_mpmc, buffer,
				    &o) == false);
			}

			observed++;
			if (o->value < 0
			    || o->value != o->tid
			    || o->magic != 0xdead
			    || (previous != 0 && previous >= o->value_long)) {
				ck_error("[0x%p] (%x) (%d, %d) >< (0, %d)\n",
					(void *)o, o->magic, o->tid, o->value, size);
			}

			o->magic = 0xbeef;
			o->value = -31337;
			o->tid = -31338;
			previous = o->value_long;

			if (ck_pr_faa_uint(&o->ref, 1) != 0) {
				ck_error("[%p] We dequeued twice.\n", (void *)o);
			}

			if ((i % 4) == 0) {
				spin = common_rand_r(&seed) % 16384;
				for (k = 0; k < spin; k++) {
					ck_pr_stall();
				}
			}

			free(o);
		}
	}

	fprintf(stderr, "[%d] Observed %u\n", tid, observed);
	return NULL;
}

static void *
test(void *c)
{
	struct context *context = c;
	struct entry *entry;
	unsigned int s;
	int i, j;
	bool r;
	ck_ring_buffer_t *buffer = context->buffer;
	ck_barrier_centralized_state_t sense =
	    CK_BARRIER_CENTRALIZED_STATE_INITIALIZER;

        if (aff_iterate(&a)) {
                perror("ERROR: Could not affine thread");
                exit(EXIT_FAILURE);
        }

	if (context->tid == 0) {
		struct entry *entries;

		entries = malloc(sizeof(struct entry) * size);
		assert(entries != NULL);

		if (ck_ring_size(ring) != 0) {
			ck_error("More entries than expected: %u > 0\n",
				ck_ring_size(ring));
		}

		for (i = 0; i < size; i++) {
			entries[i].value = i;
			entries[i].tid = 0;

			if (true) {
				r = ck_ring_enqueue_mpmc(ring, buffer,
				    entries + i);
			} else {
				r = ck_ring_enqueue_mpmc_size(ring, buffer,
					entries + i, &s);

				if ((int)s != i) {
					ck_error("Size is %u, expected %d.\n",
					    s, size);
				}
			}

			assert(r != false);
		}

		if (ck_ring_size(ring) != (unsigned int)size) {
			ck_error("Less entries than expected: %u < %d\n",
				ck_ring_size(ring), size);
		}

		if (ck_ring_capacity(ring) != ck_ring_size(ring) + 1) {
			ck_error("Capacity less than expected: %u < %u\n",
				ck_ring_size(ring), ck_ring_capacity(ring));
		}
	}

	/*
	 * Wait for all threads. The idea here is to maximize the contention.
	 */
	ck_barrier_centralized(&barrier, &sense, nthr);

	for (i = 0; i < ITERATIONS; i++) {
		for (j = 0; j < size; j++) {
			buffer = _context[context->previous].buffer;
			while (ck_ring_dequeue_mpmc(ring + context->previous,
			    buffer, &entry) == false);

			if (context->previous != (unsigned int)entry->tid) {
				ck_error("[%u:%p] %u != %u\n",
					context->tid, (void *)entry, entry->tid, context->previous);
			}

			if (entry->value < 0 || entry->value >= size) {
				ck_error("[%u:%p] %u </> %u\n",
					context->tid, (void *)entry, entry->tid, context->previous);
			}

			entry->tid = context->tid;
			buffer = context->buffer;

			if (true) {
				r = ck_ring_enqueue_mpmc(ring + context->tid,
					buffer, entry);
			} else {
				r = ck_ring_enqueue_mpmc_size(ring + context->tid,
					buffer, entry, &s);

				if ((int)s >= size) {
					ck_error("Size %u out of range of %d\n",
					    s, size);
				}
			}
			assert(r == true);
		}
	}

	return NULL;
}

int
main(int argc, char *argv[])
{
	int i, r;
	unsigned long l;
	pthread_t *thread;
	ck_ring_buffer_t *buffer;

	if (argc != 4) {
		ck_error("Usage: validate <threads> <affinity delta> <size>\n");
	}

	a.request = 0;
	a.delta = atoi(argv[2]);

	nthr = atoi(argv[1]);
	assert(nthr >= 1);

	size = atoi(argv[3]);
	assert(size >= 4 && (size & size - 1) == 0);
	size -= 1;

	ring = malloc(sizeof(ck_ring_t) * nthr);
	assert(ring);

	_context = malloc(sizeof(*_context) * nthr);
	assert(_context);

	thread = malloc(sizeof(pthread_t) * nthr);
	assert(thread);
	fprintf(stderr, "SPSC test:");
	for (i = 0; i < nthr; i++) {
		_context[i].tid = i;
		if (i == 0) {
			_context[i].previous = nthr - 1;
			_context[i].next = i + 1;
		} else if (i == nthr - 1) {
			_context[i].next = 0;
			_context[i].previous = i - 1;
		} else {
			_context[i].next = i + 1;
			_context[i].previous = i - 1;
		}

		buffer = malloc(sizeof(ck_ring_buffer_t) * (size + 1));
		assert(buffer);
		memset(buffer, 0, sizeof(ck_ring_buffer_t) * (size + 1));
		_context[i].buffer = buffer;
		ck_ring_init(ring + i, size + 1);
		r = pthread_create(thread + i, NULL, test, _context + i);
		assert(r == 0);
	}

	for (i = 0; i < nthr; i++)
		pthread_join(thread[i], NULL);

	fprintf(stderr, " done\n");

	fprintf(stderr, "SPMC test:\n");
	buffer = malloc(sizeof(ck_ring_buffer_t) * (size + 1));
	assert(buffer);
	memset(buffer, 0, sizeof(void *) * (size + 1));
	ck_ring_init(&ring_mpmc, size + 1);
	for (i = 0; i < nthr - 1; i++) {
		_context[i].buffer = buffer;
		r = pthread_create(thread + i, NULL, test_spmc, _context + i);
		assert(r == 0);
	}

	for (l = 0; l < (unsigned long)size * ITERATIONS * (nthr - 1) ; l++) {
		struct entry *entry = malloc(sizeof *entry);

		assert(entry != NULL);
		entry->value_long = l;
		entry->value = (int)l;
		entry->tid = (int)l;
		entry->magic = 0xdead;
		entry->ref = 0;

		/* Wait until queue is not full. */
		if (l & 1) {
			while (ck_ring_enqueue_mpmc(&ring_mpmc,
			    buffer,
			    entry) == false)
				ck_pr_stall();
		} else {
			unsigned int s;

			while (ck_ring_enqueue_mpmc_size(&ring_mpmc,
			    buffer, entry, &s) == false) {
				ck_pr_stall();
			}

			if ((int)s >= (size * ITERATIONS * (nthr - 1))) {
				ck_error("MPMC: Unexpected size of %u\n", s);
			}
		}
	}

	for (i = 0; i < nthr - 1; i++)
		pthread_join(thread[i], NULL);
	ck_pr_store_int(&eb, 0);
	fprintf(stderr, "MPMC test:\n");
	buffer = malloc(sizeof(ck_ring_buffer_t) * (size + 1));
	assert(buffer);
	memset(buffer, 0, sizeof(void *) * (size + 1));
	ck_ring_init(&ring_mw, size + 1);
	for (i = 0; i < nthr - 1; i++) {
		_context[i].buffer = buffer;
		r = pthread_create(thread + i, NULL, test_mpmc, _context + i);
		assert(r == 0);
	}

	for (i = 0; i < nthr - 1; i++)
		pthread_join(thread[i], NULL);

	return (0);
}