ck_rwlock: Add ck_rwlock_recursive facility.

Allows for write-side recurion. Added upon request by
other users. Regression test has been updated.

include/ck_rwlock.h

@@ -37,7 +37,7 @@ struct ck_rwlock {
 };
 typedef struct ck_rwlock ck_rwlock_t;
 
-#define CK_RWLOCK_INITIALIZER {false, 0}
+#define CK_RWLOCK_INITIALIZER {0, 0}
 
 CK_CC_INLINE static void
 ck_rwlock_init(struct ck_rwlock *rw)
@@ -72,7 +72,7 @@ ck_rwlock_write_trylock(ck_rwlock_t *rw, unsigned int factor)
 {
 	unsigned int steps = 0;
 
-	while (ck_pr_fas_uint(&rw->writer, true) == true) {
+	while (ck_pr_fas_uint(&rw->writer, 1) != 0) {
 		if (++steps >= factor)
 			return false;
 
@@ -84,6 +84,7 @@ ck_rwlock_write_trylock(ck_rwlock_t *rw, unsigned int factor)
 			ck_rwlock_write_unlock(rw);
 			return false;
 		}
+
 		ck_pr_stall();
 	}
 
@@ -95,7 +96,7 @@ CK_CC_INLINE static void
 ck_rwlock_write_lock(ck_rwlock_t *rw)
 {
 
-	while (ck_pr_fas_uint(&rw->writer, true) == true)
+	while (ck_pr_fas_uint(&rw->writer, 1) != 0)
 		ck_pr_stall();
 
 	while (ck_pr_load_uint(&rw->n_readers) != 0)
@@ -111,7 +112,7 @@ ck_rwlock_read_trylock(ck_rwlock_t *rw, unsigned int factor)
 	unsigned int steps = 0;
 
 	for (;;) {
-		while (ck_pr_load_uint(&rw->writer) == true) {
+		while (ck_pr_load_uint(&rw->writer) != 0) {
 			if (++steps >= factor)
 				return false;
 
@@ -119,7 +120,7 @@ ck_rwlock_read_trylock(ck_rwlock_t *rw, unsigned int factor)
 		}
 
 		ck_pr_inc_uint(&rw->n_readers);
-		if (ck_pr_load_uint(&rw->writer) == false)
+		if (ck_pr_load_uint(&rw->writer) == 0)
 			break;
 		ck_pr_dec_uint(&rw->n_readers);
 
@@ -136,11 +137,11 @@ ck_rwlock_read_lock(ck_rwlock_t *rw)
 {
 
 	for (;;) {
-		while (ck_pr_load_uint(&rw->writer) == true)
+		while (ck_pr_load_uint(&rw->writer) != 0)
 			ck_pr_stall();
 
 		ck_pr_inc_uint(&rw->n_readers);
-		if (ck_pr_load_uint(&rw->writer) == false)
+		if (ck_pr_load_uint(&rw->writer) == 0)
 			break;
 		ck_pr_dec_uint(&rw->n_readers);
 	}
@@ -158,4 +159,63 @@ ck_rwlock_read_unlock(ck_rwlock_t *rw)
 	return;
 }
 
+/*
+ * Recursive writer reader-writer lock implementation.
+ */
+struct ck_rwlock_recursive {
+	struct ck_rwlock rw;
+	unsigned int wc;
+};
+typedef struct ck_rwlock_recursive ck_rwlock_recursive_t;
+
+#define CK_RWLOCK_RECURSIVE_INITIALIZER {CK_RWLOCK_INITIALIZER, 0}
+
+CK_CC_INLINE static void
+ck_rwlock_recursive_write_lock(ck_rwlock_recursive_t *rw, unsigned int tid)
+{
+	unsigned int o;
+
+	o = ck_pr_load_uint(&rw->rw.writer);
+	if (o == tid)
+		goto leave;
+
+	while (ck_pr_cas_uint(&rw->rw.writer, 0, tid) == false)
+		ck_pr_stall();
+
+	while (ck_pr_load_uint(&rw->rw.n_readers) != 0)
+		ck_pr_stall();
+
+	ck_pr_fence_store();
+
+leave:
+	rw->wc++;
+	return;
+}
+
+CK_CC_INLINE static void
+ck_rwlock_recursive_write_unlock(ck_rwlock_recursive_t *rw)
+{
+
+	if (--rw->wc == 0)
+		ck_pr_store_uint(&rw->rw.writer, 0);
+
+	return;
+}
+
+CK_CC_INLINE static void
+ck_rwlock_recursive_read_lock(ck_rwlock_recursive_t *rw)
+{
+
+	ck_rwlock_read_lock(&rw->rw);
+	return;
+}
+
+CK_CC_INLINE static void
+ck_rwlock_recursive_read_unlock(ck_rwlock_recursive_t *rw)
+{
+
+	ck_rwlock_read_unlock(&rw->rw);
+	return;
+}
+
 #endif /* _CK_RWLOCK_H */

regressions/ck_rwlock/validate/validate.c

@@ -45,9 +45,84 @@
 #endif
 
 static struct affinity a;
-static unsigned int locked = 0;
+static unsigned int locked;
+static unsigned int tid = 2;
 static int nthr;
 static ck_rwlock_t lock = CK_RWLOCK_INITIALIZER;
+static ck_rwlock_recursive_t r_lock = CK_RWLOCK_RECURSIVE_INITIALIZER;
+
+static void *
+thread_recursive(void *null CK_CC_UNUSED)
+{
+	int i = ITERATE;
+	unsigned int l;
+	unsigned int t = ck_pr_faa_uint(&tid, 1);
+
+        if (aff_iterate(&a)) {
+                perror("ERROR: Could not affine thread");
+                exit(EXIT_FAILURE);
+        }
+
+	while (i--) {
+		ck_rwlock_recursive_write_lock(&r_lock, t);
+		ck_rwlock_recursive_write_lock(&r_lock, t);
+		ck_rwlock_recursive_write_lock(&r_lock, t);
+		ck_rwlock_recursive_write_lock(&r_lock, t);
+		{
+			l = ck_pr_load_uint(&locked);
+			if (l != 0) {
+				fprintf(stderr, "ERROR [WR:%d]: %u != 0\n", __LINE__, l);
+				exit(EXIT_FAILURE);
+			}
+
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+			ck_pr_inc_uint(&locked);
+
+			l = ck_pr_load_uint(&locked);
+			if (l != 8) {
+				fprintf(stderr, "ERROR [WR:%d]: %u != 2\n", __LINE__, l);
+				exit(EXIT_FAILURE);
+			}
+
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+			ck_pr_dec_uint(&locked);
+
+			l = ck_pr_load_uint(&locked);
+			if (l != 0) {
+				fprintf(stderr, "ERROR [WR:%d]: %u != 0\n", __LINE__, l);
+				exit(EXIT_FAILURE);
+			}
+		}
+		ck_rwlock_recursive_write_unlock(&r_lock);
+		ck_rwlock_recursive_write_unlock(&r_lock);
+		ck_rwlock_recursive_write_unlock(&r_lock);
+		ck_rwlock_recursive_write_unlock(&r_lock);
+
+		ck_rwlock_recursive_read_lock(&r_lock);
+		{
+			l = ck_pr_load_uint(&locked);
+			if (l != 0) {
+				fprintf(stderr, "ERROR [RD:%d]: %u != 0\n", __LINE__, l);
+				exit(EXIT_FAILURE);
+			}
+		}
+		ck_rwlock_recursive_read_unlock(&r_lock);
+	}
+
+	return (NULL);
+}
 
 static void *
 thread(void *null CK_CC_UNUSED)
@@ -154,6 +229,20 @@ main(int argc, char *argv[])
 		pthread_join(threads[i], NULL);
 	fprintf(stderr, "done (passed)\n");
 
+	fprintf(stderr, "Creating threads (mutual exclusion, recursive)...");
+	for (i = 0; i < nthr; i++) {
+		if (pthread_create(&threads[i], NULL, thread_recursive, NULL)) {
+			fprintf(stderr, "ERROR: Could not create thread %d\n", i);
+			exit(EXIT_FAILURE);
+		}
+	}
+	fprintf(stderr, "done\n");
+
+	fprintf(stderr, "Waiting for threads to finish correctness regression...");
+	for (i = 0; i < nthr; i++)
+		pthread_join(threads[i], NULL);
+	fprintf(stderr, "done (passed)\n");
+
 	return (0);
 }