ck_array: Simple eliminating SPMC dense array for fast iteration.

The array is optimized for SPMC and fast iteration (though MPMC
transformation is also possible). This is an extremely simple
implementation with support for atomic in-place modification
through put -> remove elimination.

.gitignore

@@ -15,6 +15,7 @@ build/Makefile
 *.so
 *.dSYM
 .*.sw[op]
+regressions/ck_array/validate/serial
 regressions/ck_cohort/benchmark/ck_cohort.LATENCY
 regressions/ck_cohort/benchmark/ck_cohort.THROUGHPUT
 regressions/ck_pflock/benchmark/latency

include/ck_array.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2013 Samy Al Bahra
+ * Copyright 2013 AppNexus, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _CK_ARRAY_H
+#define _CK_ARRAY_H
+
+#include <ck_cc.h>
+#include <ck_malloc.h>
+#include <ck_pr.h>
+#include <stdbool.h>
+
+struct _ck_array {
+	unsigned int n_committed;
+	unsigned int length;
+	void *values[];
+};
+
+struct ck_array {
+	struct ck_malloc *allocator;
+	struct _ck_array *active;
+	unsigned int n_entries;
+	struct _ck_array *transaction;
+};
+typedef struct ck_array ck_array_t;
+
+struct ck_array_iterator {
+	struct _ck_array *snapshot;
+};
+typedef struct ck_array_iterator ck_array_iterator_t;
+
+#define CK_ARRAY_MODE_SPMC 0U
+#define CK_ARRAY_MODE_MPMC (void) /* Unsupported. */
+
+bool ck_array_init(ck_array_t *, unsigned int mode, struct ck_malloc *, unsigned int);
+bool ck_array_commit(ck_array_t *);
+bool ck_array_put(ck_array_t *, void *);
+int ck_array_put_unique(ck_array_t *, void *);
+bool ck_array_remove(ck_array_t *, void *);
+void ck_array_deinit(ck_array_t *, bool);
+
+CK_CC_INLINE static unsigned int
+ck_array_length(struct ck_array *array)
+{
+	struct _ck_array *a = ck_pr_load_ptr(&array->active);
+
+	ck_pr_fence_load();
+	return ck_pr_load_uint(&a->n_committed);
+}
+
+CK_CC_INLINE static void *
+ck_array_buffer(struct ck_array *array, unsigned int *length)
+{
+	struct _ck_array *a = ck_pr_load_ptr(&array->active);
+
+	ck_pr_fence_load();
+	*length = ck_pr_load_uint(&a->n_committed);
+	return a->values;
+}
+
+#define CK_ARRAY_FOREACH(a, i, b)		   	\
+	(i)->snapshot = ck_pr_load_ptr(&(a)->active);	\
+	ck_pr_fence_load();				\
+	for (unsigned int _ck_i = 0;		   	\
+	    _ck_i < (a)->active->n_committed &&		\
+	    ((*b) = (a)->active->values[_ck_i], 1);	\
+	    _ck_i++)
+
+#endif /* _CK_ARRAY_H */
+

include/ck_malloc.h

@@ -32,6 +32,7 @@
 
 struct ck_malloc {
 	void *(*malloc)(size_t);
+	void *(*realloc)(void *, size_t, size_t, bool);
 	void (*free)(void *, size_t, bool);
 };
 

regressions/Makefile

@@ -1,4 +1,5 @@
-DIR=backoff	\
+DIR=array	\
+    backoff	\
     bag		\
     barrier	\
     bitmap	\
@@ -22,6 +23,7 @@ DIR=backoff	\
 .PHONY: all clean check
 
 all:
+	$(MAKE) -C ./ck_array/validate all
 	$(MAKE) -C ./ck_cohort/validate all
 	$(MAKE) -C ./ck_cohort/benchmark all
 	$(MAKE) -C ./ck_bitmap/validate all
@@ -61,6 +63,7 @@ all:
 	$(MAKE) -C ./ck_bag/validate all
 
 clean:
+	$(MAKE) -C ./ck_array/validate clean
 	$(MAKE) -C ./ck_pflock/validate clean
 	$(MAKE) -C ./ck_pflock/benchmark clean
 	$(MAKE) -C ./ck_rwcohort/validate clean

regressions/ck_array/validate/Makefile

@@ -0,0 +1,17 @@
+.PHONY: check clean distribution
+
+OBJECTS=serial
+
+all: $(OBJECTS)
+
+serial: serial.c ../../../include/ck_array.h ../../../src/ck_array.c
+	$(CC) $(CFLAGS) -o serial serial.c ../../../src/ck_array.c
+
+check: all
+	./serial
+
+clean:
+	rm -rf *~ *.o $(OBJECTS) *.dSYM *.exe
+
+include ../../../build/regressions.build
+CFLAGS+=-D_GNU_SOURCE -ggdb

regressions/ck_array/validate/serial.c

@@ -0,0 +1,145 @@
+#include <ck_array.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "../../common.h"
+
+#ifndef ITERATION
+#define ITERATION 128
+#endif
+
+static void
+my_free(void *p, size_t m, bool d)
+{
+
+	(void)m;
+	(void)d;
+
+	free(p);
+	return;
+}
+
+static void *
+my_malloc(size_t b)
+{
+
+	return malloc(b);
+}
+
+static void *
+my_realloc(void *r, size_t a, size_t b, bool d)
+{
+
+	(void)a;
+	(void)d;
+
+	return realloc(r, b);
+}
+
+int
+main(void)
+{
+	void *r;
+	uintptr_t i;
+	ck_array_t array;
+	ck_array_iterator_t iterator;
+	struct ck_malloc m = {
+		.malloc = my_malloc,
+		.free = NULL,
+		.realloc = my_realloc
+	};
+
+	if (ck_array_init(&array, CK_ARRAY_MODE_SPMC, &m, 4) == true)
+		ck_error("ck_array_init with NULL free succeeded\n");
+
+	m.free = my_free;
+	if (ck_array_init(&array, CK_ARRAY_MODE_SPMC, &m, 4) == false)
+		ck_error("ck_array_init\n");
+
+	for (i = 0; i < ITERATION; i++) {
+		if (ck_array_put(&array, (void *)i) == false)
+			ck_error("ck_error_put\n");
+
+		if (ck_array_remove(&array, (void *)i) == false)
+			ck_error("ck_error_remove after put\n");
+	}
+
+	i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++;
+	if (i != 0)
+		ck_error("Non-empty array after put -> remove workload.\n");
+
+	ck_array_commit(&array);
+
+	i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++;
+	if (i != 0)
+		ck_error("Non-empty array after put -> remove -> commit workload.\n");
+
+	for (i = 0; i < ITERATION; i++) {
+		if (ck_array_put(&array, (void *)i) == false)
+			ck_error("ck_error_put\n");
+	}
+
+	i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++;
+	if (i != 0)
+		ck_error("Non-empty array after put workload.\n");
+
+	for (i = 0; i < ITERATION; i++) {
+		if (ck_array_remove(&array, (void *)i) == false)
+			ck_error("ck_error_remove after put\n");
+	}
+
+	i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++;
+	if (i != 0)
+		ck_error("Non-empty array after put -> remove workload.\n");
+
+	ck_array_commit(&array);
+
+	i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++;
+	if (i != 0)
+		ck_error("Non-empty array after put -> remove -> commit workload.\n");
+
+	for (i = 0; i < ITERATION; i++) {
+		if (ck_array_put(&array, (void *)i) == false)
+			ck_error("ck_error_put\n");
+	}
+
+	ck_array_commit(&array);
+
+	i = 0;
+	CK_ARRAY_FOREACH(&array, &iterator, &r) {
+		i++;
+	}
+
+	if (i != ITERATION)
+		ck_error("Incorrect item count in iteration\n");
+
+	ck_array_remove(&array, (void *)(uintptr_t)0);
+	ck_array_remove(&array, (void *)(uintptr_t)1);
+	ck_array_commit(&array);
+	i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++;
+	if (i != ITERATION - 2 || ck_array_length(&array) != ITERATION - 2)
+		ck_error("Incorrect item count in iteration after remove\n");
+
+	if (ck_array_put_unique(&array, (void *)UINTPTR_MAX) != 0)
+		ck_error("Unique value put failed.\n");
+
+	if (ck_array_put_unique(&array, (void *)(uintptr_t)4) != -1)
+		ck_error("put of 4 not detected as non-unique.\n");
+
+	if (ck_array_put_unique(&array, (void *)UINTPTR_MAX) != -1)
+		ck_error("put of UINTPTR_MAX not detected as non-unique.\n");
+
+	ck_array_commit(&array);
+	i = 0;
+	CK_ARRAY_FOREACH(&array, &iterator, &r) {
+		i++;
+	}
+	if (i != ITERATION - 1 || ck_array_length(&array) != ITERATION - 1)
+		ck_error("Incorrect item count in iteration after unique put\n");
+
+	ck_array_deinit(&array, false);
+
+	return 0;
+}
+

src/Makefile.in

@@ -16,7 +16,8 @@ OBJECTS=ck_barrier_centralized.o	\
 	ck_ht.o				\
 	ck_hp.o				\
 	ck_bag.o			\
-	ck_hs.o
+	ck_hs.o				\
+	ck_array.o
 
 all: libck.so libck.a
 
@@ -26,6 +27,9 @@ libck.so: $(OBJECTS)
 libck.a: $(OBJECTS)
 	ar rcs $(TARGET_DIR)/libck.a $(OBJECTS)
 
+ck_array.o: $(INCLUDE_DIR)/ck_array.h $(SDIR)/ck_array.c
+	$(CC) $(CFLAGS) -c -o $(TARGET_DIR)/ck_array.o $(SDIR)/ck_array.c
+
 ck_bag.o: $(INCLUDE_DIR)/ck_bag.h $(SDIR)/ck_bag.c
 	$(CC) $(CFLAGS) -c -o $(TARGET_DIR)/ck_bag.o $(SDIR)/ck_bag.c
 

src/ck_array.c

@@ -0,0 +1,243 @@
+/*
+ * Copyright 2013 Samy Al Bahra
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <ck_array.h>
+#include <ck_cc.h>
+#include <ck_pr.h>
+#include <stdbool.h>
+#include <string.h>
+
+static struct _ck_array *
+ck_array_create(struct ck_malloc *allocator, unsigned int length)
+{
+	struct _ck_array *active;
+
+	active = allocator->malloc(sizeof(struct _ck_array) + sizeof(void *) * length);
+	if (active == NULL)
+		return NULL;
+
+	active->n_committed = 0;
+	active->length = length;
+
+	return active;
+}
+
+bool
+ck_array_init(struct ck_array *array, unsigned int mode, struct ck_malloc *allocator, unsigned int length)
+{
+	struct _ck_array *active;
+
+	(void)mode;
+
+	if (allocator->realloc == NULL ||
+	    allocator->malloc == NULL ||
+	    allocator->free == NULL ||
+	    length == 0)
+		return false;
+
+	active = ck_array_create(allocator, length);
+	if (active == NULL)
+		return false;
+
+	array->n_entries = 0;
+	array->allocator = allocator;
+	array->active = active;
+	array->transaction = NULL;
+	return true;
+}
+
+bool
+ck_array_put(struct ck_array *array, void *value)
+{
+	struct _ck_array *target, *update;
+
+	/*
+	 * If no transaction copy has been necessary, attempt to do in-place
+	 * modification of the array.
+	 */
+	if (array->transaction == NULL) {
+		target = array->active;
+
+		if (array->n_entries == target->length) {
+			unsigned int size = target->length << 1;
+
+			update = ck_array_create(array->allocator, size);
+			if (update == NULL)
+				return false;
+
+			memcpy(update->values, target->values, sizeof(void *) * array->n_entries);
+			update->n_committed = target->n_committed;
+			update->length = size;
+
+			/* Serialize with respect to update contents. */
+			ck_pr_fence_store();
+			ck_pr_store_ptr(&array->active, update);
+
+			array->allocator->free(target,
+			    sizeof(struct _ck_array) + target->length * sizeof(void *), true);
+			target = update;
+		}
+
+		target->values[array->n_entries++] = value;
+		return true;
+	}
+
+	target = array->transaction;
+	if (array->n_entries == target->length) {
+		unsigned int size = target->length << 1;
+
+		update = array->allocator->realloc(array->transaction,
+		    sizeof(struct _ck_array) + sizeof(void *) * array->n_entries,
+		    sizeof(struct _ck_array) + sizeof(void *) * size,
+		    false);
+
+		if (update == NULL)
+			return false;
+
+		update->n_committed = target->n_committed;
+		update->length = size;
+		array->transaction = update;
+	}
+
+	target->values[array->n_entries++] = value;
+	return false;
+}
+
+int
+ck_array_put_unique(struct ck_array *array, void *value)
+{
+	unsigned int i, limit;
+	void **v;
+
+	limit = array->n_entries;
+	if (array->transaction != NULL) {
+		v = array->transaction->values;
+	} else {
+		v = array->active->values;
+	}
+
+	for (i = 0; i < limit; i++) {
+		if (v[i] == value)
+			return -1;
+	}
+
+	return !ck_array_put(array, value);
+}
+
+bool
+ck_array_remove(struct ck_array *array, void *value)
+{
+	struct _ck_array *target;
+	unsigned int i;
+
+	if (array->transaction != NULL) {
+		target = array->transaction;
+
+		for (i = 0; i < array->n_entries; i++) {
+			if (target->values[i] == value) {
+				target->values[i] = target->values[--array->n_entries];
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	target = array->active;
+
+	for (i = 0; i < array->n_entries; i++) {
+		if (target->values[i] == value)
+			break;
+	}
+
+	if (i == array->n_entries)
+		return false;
+
+	/* If there are pending additions, immediately eliminate the operation. */
+	if (target->n_committed != array->n_entries) {
+		target = array->active;
+		ck_pr_store_ptr(&target->values[i], target->values[--array->n_entries]);
+		return true;
+	}
+
+	/*
+	 * The assumption is that these allocations are small to begin with.
+	 * If there is no immediate opportunity for transaction, allocate a
+	 * transactional array which will be applied upon commit time.
+	 */
+	target = ck_array_create(array->allocator, array->n_entries);
+	if (target == NULL) {
+		return false;
+	}
+
+	memcpy(target->values, array->active->values, sizeof(void *) * array->n_entries);
+	target->length = array->n_entries;
+	target->n_committed = array->n_entries;
+	target->values[i] = target->values[--array->n_entries];
+
+	array->transaction = target;
+	return true;
+}
+
+bool
+ck_array_commit(ck_array_t *array)
+{
+	struct _ck_array *m = array->transaction;
+
+	if (m != NULL) {
+		struct _ck_array *p;
+
+		m->n_committed = array->n_entries;
+		ck_pr_fence_store();
+		p = array->active;
+		ck_pr_store_ptr(&array->active, m);
+		array->allocator->free(p, sizeof(struct _ck_array) +
+		    p->length * sizeof(void *), true);
+		array->transaction = NULL;
+
+		return true;
+	}
+
+	ck_pr_fence_store();
+	ck_pr_store_uint(&array->active->n_committed, array->n_entries);
+	return true;
+}
+
+void
+ck_array_deinit(struct ck_array *array, bool defer)
+{
+
+	array->allocator->free(array->active,
+	    sizeof(struct _ck_array) + sizeof(void *) * array->active->length, defer);
+
+	if (array->transaction != NULL) {
+		array->allocator->free(array->transaction,
+		    sizeof(struct _ck_array) + sizeof(void *) * array->transaction->length, defer);
+	}
+
+	return;
+}
+