From 2db9de633fd6d7592c3acf9904d5e31dc88426e1 Mon Sep 17 00:00:00 2001 From: Samy Al Bahra Date: Thu, 3 Oct 2013 17:06:59 -0400 Subject: [PATCH] ck_array: Simple eliminating SPMC dense array for fast iteration. The array is optimized for SPMC and fast iteration (though MPMC transformation is also possible). This is an extremely simple implementation with support for atomic in-place modification through put -> remove elimination. --- .gitignore | 1 + include/ck_array.h | 93 ++++++++++ include/ck_malloc.h | 1 + regressions/Makefile | 5 +- regressions/ck_array/validate/Makefile | 17 ++ regressions/ck_array/validate/serial.c | 145 +++++++++++++++ src/Makefile.in | 6 +- src/ck_array.c | 243 +++++++++++++++++++++++++ 8 files changed, 509 insertions(+), 2 deletions(-) create mode 100644 include/ck_array.h create mode 100644 regressions/ck_array/validate/Makefile create mode 100644 regressions/ck_array/validate/serial.c create mode 100644 src/ck_array.c diff --git a/.gitignore b/.gitignore index d91e51d..f9e9158 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ build/Makefile *.so *.dSYM .*.sw[op] +regressions/ck_array/validate/serial regressions/ck_cohort/benchmark/ck_cohort.LATENCY regressions/ck_cohort/benchmark/ck_cohort.THROUGHPUT regressions/ck_pflock/benchmark/latency diff --git a/include/ck_array.h b/include/ck_array.h new file mode 100644 index 0000000..a8a6476 --- /dev/null +++ b/include/ck_array.h @@ -0,0 +1,93 @@ +/* + * Copyright 2013 Samy Al Bahra + * Copyright 2013 AppNexus, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _CK_ARRAY_H +#define _CK_ARRAY_H + +#include +#include +#include +#include + +struct _ck_array { + unsigned int n_committed; + unsigned int length; + void *values[]; +}; + +struct ck_array { + struct ck_malloc *allocator; + struct _ck_array *active; + unsigned int n_entries; + struct _ck_array *transaction; +}; +typedef struct ck_array ck_array_t; + +struct ck_array_iterator { + struct _ck_array *snapshot; +}; +typedef struct ck_array_iterator ck_array_iterator_t; + +#define CK_ARRAY_MODE_SPMC 0U +#define CK_ARRAY_MODE_MPMC (void) /* Unsupported. */ + +bool ck_array_init(ck_array_t *, unsigned int mode, struct ck_malloc *, unsigned int); +bool ck_array_commit(ck_array_t *); +bool ck_array_put(ck_array_t *, void *); +int ck_array_put_unique(ck_array_t *, void *); +bool ck_array_remove(ck_array_t *, void *); +void ck_array_deinit(ck_array_t *, bool); + +CK_CC_INLINE static unsigned int +ck_array_length(struct ck_array *array) +{ + struct _ck_array *a = ck_pr_load_ptr(&array->active); + + ck_pr_fence_load(); + return ck_pr_load_uint(&a->n_committed); +} + +CK_CC_INLINE static void * +ck_array_buffer(struct ck_array *array, unsigned int *length) +{ + struct _ck_array *a = ck_pr_load_ptr(&array->active); + + ck_pr_fence_load(); + *length = ck_pr_load_uint(&a->n_committed); + return a->values; +} + +#define CK_ARRAY_FOREACH(a, i, b) \ + (i)->snapshot = ck_pr_load_ptr(&(a)->active); \ + ck_pr_fence_load(); \ + for (unsigned int _ck_i = 0; \ + _ck_i < (a)->active->n_committed && \ + ((*b) = (a)->active->values[_ck_i], 1); \ + _ck_i++) + +#endif /* _CK_ARRAY_H */ + diff --git a/include/ck_malloc.h b/include/ck_malloc.h index 93938af..57a08e4 100644 --- a/include/ck_malloc.h +++ b/include/ck_malloc.h @@ -32,6 +32,7 @@ struct ck_malloc { void *(*malloc)(size_t); + void *(*realloc)(void *, size_t, size_t, bool); void (*free)(void *, size_t, bool); }; diff --git a/regressions/Makefile b/regressions/Makefile index b098abf..d49f7bb 100644 --- a/regressions/Makefile +++ b/regressions/Makefile @@ -1,4 +1,5 @@ -DIR=backoff \ +DIR=array \ + backoff \ bag \ barrier \ bitmap \ @@ -22,6 +23,7 @@ DIR=backoff \ .PHONY: all clean check all: + $(MAKE) -C ./ck_array/validate all $(MAKE) -C ./ck_cohort/validate all $(MAKE) -C ./ck_cohort/benchmark all $(MAKE) -C ./ck_bitmap/validate all @@ -61,6 +63,7 @@ all: $(MAKE) -C ./ck_bag/validate all clean: + $(MAKE) -C ./ck_array/validate clean $(MAKE) -C ./ck_pflock/validate clean $(MAKE) -C ./ck_pflock/benchmark clean $(MAKE) -C ./ck_rwcohort/validate clean diff --git a/regressions/ck_array/validate/Makefile b/regressions/ck_array/validate/Makefile new file mode 100644 index 0000000..3c48167 --- /dev/null +++ b/regressions/ck_array/validate/Makefile @@ -0,0 +1,17 @@ +.PHONY: check clean distribution + +OBJECTS=serial + +all: $(OBJECTS) + +serial: serial.c ../../../include/ck_array.h ../../../src/ck_array.c + $(CC) $(CFLAGS) -o serial serial.c ../../../src/ck_array.c + +check: all + ./serial + +clean: + rm -rf *~ *.o $(OBJECTS) *.dSYM *.exe + +include ../../../build/regressions.build +CFLAGS+=-D_GNU_SOURCE -ggdb diff --git a/regressions/ck_array/validate/serial.c b/regressions/ck_array/validate/serial.c new file mode 100644 index 0000000..5897cf9 --- /dev/null +++ b/regressions/ck_array/validate/serial.c @@ -0,0 +1,145 @@ +#include +#include +#include +#include + +#include "../../common.h" + +#ifndef ITERATION +#define ITERATION 128 +#endif + +static void +my_free(void *p, size_t m, bool d) +{ + + (void)m; + (void)d; + + free(p); + return; +} + +static void * +my_malloc(size_t b) +{ + + return malloc(b); +} + +static void * +my_realloc(void *r, size_t a, size_t b, bool d) +{ + + (void)a; + (void)d; + + return realloc(r, b); +} + +int +main(void) +{ + void *r; + uintptr_t i; + ck_array_t array; + ck_array_iterator_t iterator; + struct ck_malloc m = { + .malloc = my_malloc, + .free = NULL, + .realloc = my_realloc + }; + + if (ck_array_init(&array, CK_ARRAY_MODE_SPMC, &m, 4) == true) + ck_error("ck_array_init with NULL free succeeded\n"); + + m.free = my_free; + if (ck_array_init(&array, CK_ARRAY_MODE_SPMC, &m, 4) == false) + ck_error("ck_array_init\n"); + + for (i = 0; i < ITERATION; i++) { + if (ck_array_put(&array, (void *)i) == false) + ck_error("ck_error_put\n"); + + if (ck_array_remove(&array, (void *)i) == false) + ck_error("ck_error_remove after put\n"); + } + + i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++; + if (i != 0) + ck_error("Non-empty array after put -> remove workload.\n"); + + ck_array_commit(&array); + + i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++; + if (i != 0) + ck_error("Non-empty array after put -> remove -> commit workload.\n"); + + for (i = 0; i < ITERATION; i++) { + if (ck_array_put(&array, (void *)i) == false) + ck_error("ck_error_put\n"); + } + + i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++; + if (i != 0) + ck_error("Non-empty array after put workload.\n"); + + for (i = 0; i < ITERATION; i++) { + if (ck_array_remove(&array, (void *)i) == false) + ck_error("ck_error_remove after put\n"); + } + + i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++; + if (i != 0) + ck_error("Non-empty array after put -> remove workload.\n"); + + ck_array_commit(&array); + + i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++; + if (i != 0) + ck_error("Non-empty array after put -> remove -> commit workload.\n"); + + for (i = 0; i < ITERATION; i++) { + if (ck_array_put(&array, (void *)i) == false) + ck_error("ck_error_put\n"); + } + + ck_array_commit(&array); + + i = 0; + CK_ARRAY_FOREACH(&array, &iterator, &r) { + i++; + } + + if (i != ITERATION) + ck_error("Incorrect item count in iteration\n"); + + ck_array_remove(&array, (void *)(uintptr_t)0); + ck_array_remove(&array, (void *)(uintptr_t)1); + ck_array_commit(&array); + i = 0; CK_ARRAY_FOREACH(&array, &iterator, &r) i++; + if (i != ITERATION - 2 || ck_array_length(&array) != ITERATION - 2) + ck_error("Incorrect item count in iteration after remove\n"); + + if (ck_array_put_unique(&array, (void *)UINTPTR_MAX) != 0) + ck_error("Unique value put failed.\n"); + + if (ck_array_put_unique(&array, (void *)(uintptr_t)4) != -1) + ck_error("put of 4 not detected as non-unique.\n"); + + if (ck_array_put_unique(&array, (void *)UINTPTR_MAX) != -1) + ck_error("put of UINTPTR_MAX not detected as non-unique.\n"); + + ck_array_commit(&array); + i = 0; + CK_ARRAY_FOREACH(&array, &iterator, &r) { + i++; + } + if (i != ITERATION - 1 || ck_array_length(&array) != ITERATION - 1) + ck_error("Incorrect item count in iteration after unique put\n"); + + ck_array_deinit(&array, false); + + return 0; +} + diff --git a/src/Makefile.in b/src/Makefile.in index 92db79c..0f39d02 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -16,7 +16,8 @@ OBJECTS=ck_barrier_centralized.o \ ck_ht.o \ ck_hp.o \ ck_bag.o \ - ck_hs.o + ck_hs.o \ + ck_array.o all: libck.so libck.a @@ -26,6 +27,9 @@ libck.so: $(OBJECTS) libck.a: $(OBJECTS) ar rcs $(TARGET_DIR)/libck.a $(OBJECTS) +ck_array.o: $(INCLUDE_DIR)/ck_array.h $(SDIR)/ck_array.c + $(CC) $(CFLAGS) -c -o $(TARGET_DIR)/ck_array.o $(SDIR)/ck_array.c + ck_bag.o: $(INCLUDE_DIR)/ck_bag.h $(SDIR)/ck_bag.c $(CC) $(CFLAGS) -c -o $(TARGET_DIR)/ck_bag.o $(SDIR)/ck_bag.c diff --git a/src/ck_array.c b/src/ck_array.c new file mode 100644 index 0000000..12743ea --- /dev/null +++ b/src/ck_array.c @@ -0,0 +1,243 @@ +/* + * Copyright 2013 Samy Al Bahra + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +static struct _ck_array * +ck_array_create(struct ck_malloc *allocator, unsigned int length) +{ + struct _ck_array *active; + + active = allocator->malloc(sizeof(struct _ck_array) + sizeof(void *) * length); + if (active == NULL) + return NULL; + + active->n_committed = 0; + active->length = length; + + return active; +} + +bool +ck_array_init(struct ck_array *array, unsigned int mode, struct ck_malloc *allocator, unsigned int length) +{ + struct _ck_array *active; + + (void)mode; + + if (allocator->realloc == NULL || + allocator->malloc == NULL || + allocator->free == NULL || + length == 0) + return false; + + active = ck_array_create(allocator, length); + if (active == NULL) + return false; + + array->n_entries = 0; + array->allocator = allocator; + array->active = active; + array->transaction = NULL; + return true; +} + +bool +ck_array_put(struct ck_array *array, void *value) +{ + struct _ck_array *target, *update; + + /* + * If no transaction copy has been necessary, attempt to do in-place + * modification of the array. + */ + if (array->transaction == NULL) { + target = array->active; + + if (array->n_entries == target->length) { + unsigned int size = target->length << 1; + + update = ck_array_create(array->allocator, size); + if (update == NULL) + return false; + + memcpy(update->values, target->values, sizeof(void *) * array->n_entries); + update->n_committed = target->n_committed; + update->length = size; + + /* Serialize with respect to update contents. */ + ck_pr_fence_store(); + ck_pr_store_ptr(&array->active, update); + + array->allocator->free(target, + sizeof(struct _ck_array) + target->length * sizeof(void *), true); + target = update; + } + + target->values[array->n_entries++] = value; + return true; + } + + target = array->transaction; + if (array->n_entries == target->length) { + unsigned int size = target->length << 1; + + update = array->allocator->realloc(array->transaction, + sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, + sizeof(struct _ck_array) + sizeof(void *) * size, + false); + + if (update == NULL) + return false; + + update->n_committed = target->n_committed; + update->length = size; + array->transaction = update; + } + + target->values[array->n_entries++] = value; + return false; +} + +int +ck_array_put_unique(struct ck_array *array, void *value) +{ + unsigned int i, limit; + void **v; + + limit = array->n_entries; + if (array->transaction != NULL) { + v = array->transaction->values; + } else { + v = array->active->values; + } + + for (i = 0; i < limit; i++) { + if (v[i] == value) + return -1; + } + + return !ck_array_put(array, value); +} + +bool +ck_array_remove(struct ck_array *array, void *value) +{ + struct _ck_array *target; + unsigned int i; + + if (array->transaction != NULL) { + target = array->transaction; + + for (i = 0; i < array->n_entries; i++) { + if (target->values[i] == value) { + target->values[i] = target->values[--array->n_entries]; + return true; + } + } + + return false; + } + + target = array->active; + + for (i = 0; i < array->n_entries; i++) { + if (target->values[i] == value) + break; + } + + if (i == array->n_entries) + return false; + + /* If there are pending additions, immediately eliminate the operation. */ + if (target->n_committed != array->n_entries) { + target = array->active; + ck_pr_store_ptr(&target->values[i], target->values[--array->n_entries]); + return true; + } + + /* + * The assumption is that these allocations are small to begin with. + * If there is no immediate opportunity for transaction, allocate a + * transactional array which will be applied upon commit time. + */ + target = ck_array_create(array->allocator, array->n_entries); + if (target == NULL) { + return false; + } + + memcpy(target->values, array->active->values, sizeof(void *) * array->n_entries); + target->length = array->n_entries; + target->n_committed = array->n_entries; + target->values[i] = target->values[--array->n_entries]; + + array->transaction = target; + return true; +} + +bool +ck_array_commit(ck_array_t *array) +{ + struct _ck_array *m = array->transaction; + + if (m != NULL) { + struct _ck_array *p; + + m->n_committed = array->n_entries; + ck_pr_fence_store(); + p = array->active; + ck_pr_store_ptr(&array->active, m); + array->allocator->free(p, sizeof(struct _ck_array) + + p->length * sizeof(void *), true); + array->transaction = NULL; + + return true; + } + + ck_pr_fence_store(); + ck_pr_store_uint(&array->active->n_committed, array->n_entries); + return true; +} + +void +ck_array_deinit(struct ck_array *array, bool defer) +{ + + array->allocator->free(array->active, + sizeof(struct _ck_array) + sizeof(void *) * array->active->length, defer); + + if (array->transaction != NULL) { + array->allocator->free(array->transaction, + sizeof(struct _ck_array) + sizeof(void *) * array->transaction->length, defer); + } + + return; +} +