tflock: Task-fair reader-writer locks reference implementation.

include/ck_tflock.h

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2014 Samy Al Bahra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _CK_TFLOCK_TICKET_H
+#define _CK_TFLOCK_TICKET_H
+
+/*
+ * This is an implementation of task-fair locks derived from the work
+ * described in:
+ *	John M. Mellor-Crummey and Michael L. Scott. 1991.
+ *	Scalable reader-writer synchronization for shared-memory
+ *	multiprocessors. SIGPLAN Not. 26, 7 (April 1991), 106-113. 
+ */
+
+#include <ck_cc.h>
+#include <ck_pr.h>
+
+struct ck_tflock_ticket {
+	uint32_t request;
+	uint32_t completion;
+};
+typedef struct ck_tflock_ticket ck_tflock_ticket_t;
+
+#define CK_TFLOCK_TICKET_INITIALIZER { 0, 0 }
+
+#define CK_TFLOCK_TICKET_RC_INCR	0x10000U	/* Read-side increment. */
+#define CK_TFLOCK_TICKET_WC_INCR	0x1U		/* Write-side increment. */
+#define CK_TFLOCK_TICKET_W_MASK		0xffffU		/* Write-side mask. */
+#define CK_TFLOCK_TICKET_WC_TOPMSK	0x8000U		/* Write clear mask for overflow. */
+#define CK_TFLOCK_TICKET_RC_TOPMSK	0x80000000U	/* Read clear mask for overflow. */
+
+CK_CC_INLINE static uint32_t
+ck_tflock_ticket_fca_32(uint32_t *target, uint32_t mask, uint32_t delta)
+{
+	uint32_t snapshot = ck_pr_load_32(target);
+	uint32_t goal;
+
+	for (;;) {
+		goal = (snapshot & ~mask) + delta;
+		if (ck_pr_cas_32_value(target, snapshot, goal, &snapshot) == true)
+			break;
+
+		ck_pr_stall();
+	}
+	
+	return snapshot;
+}
+
+CK_CC_INLINE static void
+ck_tflock_ticket_init(struct ck_tflock_ticket *pf)
+{
+
+	pf->request = pf->completion = 0;
+	return;
+}
+
+CK_CC_INLINE static void
+ck_tflock_ticket_write_lock(struct ck_tflock_ticket *lock)
+{
+	uint32_t previous;
+
+	previous = ck_tflock_ticket_fca_32(&lock->request, CK_TFLOCK_TICKET_WC_TOPMSK,
+	    CK_TFLOCK_TICKET_WC_INCR);
+	ck_pr_fence_atomic_load();
+	while (ck_pr_load_32(&lock->completion) != previous)
+		ck_pr_stall();
+
+	ck_pr_fence_acquire();
+	return;
+}
+
+CK_CC_INLINE static void
+ck_tflock_ticket_write_unlock(struct ck_tflock_ticket *lock)
+{
+
+	ck_pr_fence_release();
+	ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_WC_TOPMSK,
+	    CK_TFLOCK_TICKET_WC_INCR);
+	return;
+}
+
+CK_CC_INLINE static void
+ck_tflock_ticket_read_lock(struct ck_tflock_ticket *lock)
+{
+	uint32_t previous;
+
+	previous = ck_tflock_ticket_fca_32(&lock->request, CK_TFLOCK_TICKET_RC_TOPMSK,
+	    CK_TFLOCK_TICKET_RC_INCR) & CK_TFLOCK_TICKET_W_MASK;
+
+	ck_pr_fence_atomic_load();
+
+	while ((ck_pr_load_uint(&lock->completion) & CK_TFLOCK_TICKET_W_MASK) != previous)
+		ck_pr_stall();
+
+	ck_pr_fence_acquire();
+	return;
+}
+
+CK_CC_INLINE static void
+ck_tflock_ticket_read_unlock(struct ck_tflock_ticket *lock)
+{
+
+	ck_pr_fence_release();
+	ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_RC_TOPMSK,
+	    CK_TFLOCK_TICKET_RC_INCR);
+	return;
+}
+
+#endif /* _CK_TFLOCK_TICKET_H */
+

regressions/ck_tflock/benchmark/Makefile

@@ -0,0 +1,17 @@
+.PHONY: clean distribution
+
+OBJECTS=latency throughput
+
+all: $(OBJECTS)
+
+latency: latency.c ../../../include/ck_rwlock.h ../../../include/ck_elide.h
+	$(CC) $(CFLAGS) -o latency latency.c
+
+throughput: throughput.c ../../../include/ck_rwlock.h ../../../include/ck_elide.h
+	$(CC) $(CFLAGS) -o throughput throughput.c
+
+clean:
+	rm -rf *.dSYM *.exe *~ *.o $(OBJECTS)
+
+include ../../../build/regressions.build
+CFLAGS+=$(PTHREAD_CFLAGS) -D_GNU_SOURCE

regressions/ck_tflock/benchmark/latency.c

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2011-2014 Samy Al Bahra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <ck_tflock.h>
+#include <inttypes.h>
+#include <stdio.h>
+
+#include "../../common.h"
+
+#define CK_F_PR_RTM
+
+#ifndef STEPS
+#define STEPS 2000000
+#endif
+
+int
+main(void)
+{
+	uint64_t s_b, e_b, i;
+	ck_tflock_ticket_t tflock = CK_TFLOCK_TICKET_INITIALIZER;
+
+	for (i = 0; i < STEPS; i++) {
+		ck_tflock_ticket_write_lock(&tflock);
+		ck_tflock_ticket_write_unlock(&tflock);
+	}
+
+	s_b = rdtsc();
+	for (i = 0; i < STEPS; i++) {
+		ck_tflock_ticket_write_lock(&tflock);
+		ck_tflock_ticket_write_unlock(&tflock);
+	}
+	e_b = rdtsc();
+	printf("                WRITE: tflock   %15" PRIu64 "\n", (e_b - s_b) / STEPS);
+
+	for (i = 0; i < STEPS; i++) {
+		ck_tflock_ticket_read_lock(&tflock);
+		ck_tflock_ticket_read_unlock(&tflock);
+	}
+
+	s_b = rdtsc();
+	for (i = 0; i < STEPS; i++) {
+		ck_tflock_ticket_read_lock(&tflock);
+		ck_tflock_ticket_read_unlock(&tflock);
+	}
+	e_b = rdtsc();
+	printf("                READ:  tflock   %15" PRIu64 "\n", (e_b - s_b) / STEPS);
+
+	return 0;
+}
+

regressions/ck_tflock/benchmark/throughput.c

@@ -0,0 +1,182 @@
+/*
+ * Copyright 2011-2014 Samy Al Bahra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <ck_tflock.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "../../common.h"
+
+#ifndef STEPS
+#define STEPS 1000000
+#endif
+
+static int barrier;
+static int threads;
+static unsigned int flag CK_CC_CACHELINE;
+static struct {
+	ck_tflock_ticket_t lock;
+} rw CK_CC_CACHELINE = {
+	.lock = CK_TFLOCK_TICKET_INITIALIZER
+};
+
+static struct affinity affinity;
+
+static void *
+thread_lock(void *pun)
+{
+	uint64_t s_b, e_b, a, i;
+	uint64_t *value = pun;
+
+	if (aff_iterate(&affinity) != 0) {
+		perror("ERROR: Could not affine thread");
+		exit(EXIT_FAILURE);
+	}
+
+	ck_pr_inc_int(&barrier);
+	while (ck_pr_load_int(&barrier) != threads)
+		ck_pr_stall();
+
+	for (i = 1, a = 0;; i++) {
+		s_b = rdtsc();
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		ck_tflock_ticket_read_lock(&rw.lock);
+		ck_tflock_ticket_read_unlock(&rw.lock);
+		e_b = rdtsc();
+
+		a += (e_b - s_b) >> 4;
+
+		if (ck_pr_load_uint(&flag) == 1)
+			break;
+	}
+
+	ck_pr_inc_int(&barrier);
+	while (ck_pr_load_int(&barrier) != threads * 2)
+		ck_pr_stall();
+
+	*value = (a / i);
+	return NULL;
+}
+
+static void
+tflock_test(pthread_t *p, int d, uint64_t *latency, void *(*f)(void *), const char *label)
+{
+	int t;
+
+	ck_pr_store_int(&barrier, 0);
+	ck_pr_store_uint(&flag, 0);
+
+	affinity.delta = d;
+	affinity.request = 0;
+
+	fprintf(stderr, "Creating threads (%s)...", label);
+	for (t = 0; t < threads; t++) {
+		if (pthread_create(&p[t], NULL, f, latency + t) != 0) {
+			ck_error("ERROR: Could not create thread %d\n", t);
+		}
+	}
+	fprintf(stderr, "done\n");
+
+	common_sleep(10);
+	ck_pr_store_uint(&flag, 1);
+
+	fprintf(stderr, "Waiting for threads to finish acquisition regression...");
+	for (t = 0; t < threads; t++)
+		pthread_join(p[t], NULL);
+	fprintf(stderr, "done\n\n");
+
+	for (t = 1; t <= threads; t++)
+		printf("%10u %20" PRIu64 "\n", t, latency[t - 1]);
+
+	fprintf(stderr, "\n");
+	return;
+}
+
+
+int
+main(int argc, char *argv[])
+{
+	int d;
+	pthread_t *p;
+	uint64_t *latency;
+
+	if (argc != 3) {
+		ck_error("Usage: throughput <delta> <threads>\n");
+	}
+
+	threads = atoi(argv[2]);
+	if (threads <= 0) {
+		ck_error("ERROR: Threads must be a value > 0.\n");
+	}
+
+	p = malloc(sizeof(pthread_t) * threads);
+	if (p == NULL) {
+		ck_error("ERROR: Failed to initialize thread.\n");
+	}
+
+	latency = malloc(sizeof(uint64_t) * threads);
+	if (latency == NULL) {
+		ck_error("ERROR: Failed to create latency buffer.\n");
+	}
+
+	d = atoi(argv[1]);
+	tflock_test(p, d, latency, thread_lock, "tflock");
+	return 0;
+}
+

regressions/ck_tflock/validate/Makefile

@@ -0,0 +1,17 @@
+.PHONY: check clean distribution
+
+OBJECTS=validate
+
+all: $(OBJECTS)
+
+validate: validate.c ../../../include/ck_tflock.h ../../../include/ck_elide.h
+	$(CC) $(CFLAGS) -o validate validate.c
+
+check: all
+	./validate $(CORES) 1
+
+clean:
+	rm -rf *.dSYM *.exe *~ *.o $(OBJECTS)
+
+include ../../../build/regressions.build
+CFLAGS+=$(PTHREAD_CFLAGS) -D_GNU_SOURCE

regressions/ck_tflock/validate/validate.c

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2011-2014 Samy Al Bahra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+#include <ck_pr.h>
+#include <ck_tflock.h>
+
+#include "../../common.h"
+
+#ifndef ITERATE
+#define ITERATE 1000000
+#endif
+
+static struct affinity a;
+static unsigned int locked;
+static int nthr;
+static ck_tflock_ticket_t lock = CK_TFLOCK_TICKET_INITIALIZER;
+
+static void *
+thread(void *null CK_CC_UNUSED)
+{
+	unsigned int i = ITERATE;
+	unsigned int l;
+
+        if (aff_iterate(&a)) {
+                perror("ERROR: Could not affine thread");
+                exit(EXIT_FAILURE);
+        }
+
+	while (i--) {
+		ck_tflock_ticket_write_lock(&lock);
+		{
+			l = ck_pr_load_uint(&locked);
+			if (l != 0) {
+				ck_error("ERROR [WR:%d]: %u != 0\n", __LINE__, l);
+			}
+
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+
+			l = ck_pr_load_uint(&locked);
+			if (l != 8) {
+				ck_error("ERROR [WR:%d]: %u != 2\n", __LINE__, l);
+			}
+
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+
+			l = ck_pr_load_uint(&locked);
+			if (l != 0) {
+				ck_error("ERROR [WR:%d]: %u != 0\n", __LINE__, l);
+			}
+		}
+		ck_tflock_ticket_write_unlock(&lock);
+
+		ck_tflock_ticket_read_lock(&lock);
+		{
+			l = ck_pr_load_uint(&locked);
+			if (l != 0) {
+				ck_error("ERROR [RD:%d]: %u != 0\n", __LINE__, l);
+			}
+		}
+		ck_tflock_ticket_read_unlock(&lock);
+	}
+
+	return (NULL);
+}
+
+static void
+tflock_ticket_test(pthread_t *threads, void *(*f)(void *), const char *test)
+{
+	int i;
+
+	fprintf(stderr, "Creating threads (%s)...", test);
+	for (i = 0; i < nthr; i++) {
+		if (pthread_create(&threads[i], NULL, f, NULL)) {
+			ck_error("ERROR: Could not create thread %d\n", i);
+		}
+	}
+	fprintf(stderr, ".");
+
+	for (i = 0; i < nthr; i++)
+		pthread_join(threads[i], NULL);
+	fprintf(stderr, "done (passed)\n");
+	return;
+}
+
+int
+main(int argc, char *argv[])
+{
+	pthread_t *threads;
+
+	if (argc != 3) {
+		ck_error("Usage: validate <number of threads> <affinity delta>\n");
+	}
+
+	nthr = atoi(argv[1]);
+	if (nthr <= 0) {
+		ck_error("ERROR: Number of threads must be greater than 0\n");
+	}
+
+	threads = malloc(sizeof(pthread_t) * nthr);
+	if (threads == NULL) {
+		ck_error("ERROR: Could not allocate thread structures\n");
+	}
+
+	a.delta = atoi(argv[2]);
+
+	tflock_ticket_test(threads, thread, "regular");
+	return 0;
+}
+