From bf686c0a42071dcecd09a417ec91314feac042ba Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Fri, 3 Jan 2014 00:11:50 +0100 Subject: [PATCH 01/11] ck_rhs: Add. Introduce ck_rhs, a variant of ck_hs that uses the robin hood algorithm for insertion, and backward shift delete for deletion. --- include/ck_rhs.h | 125 ++ regressions/ck_rhs/benchmark/Makefile | 17 + .../ck_rhs/benchmark/parallel_bytestring.c | 599 ++++++++++ regressions/ck_rhs/benchmark/serial.c | 516 +++++++++ regressions/ck_rhs/validate/Makefile | 17 + regressions/ck_rhs/validate/serial.c | 245 ++++ src/Makefile.in | 1 + src/ck_rhs.c | 1026 +++++++++++++++++ 8 files changed, 2546 insertions(+) create mode 100644 include/ck_rhs.h create mode 100644 regressions/ck_rhs/benchmark/Makefile create mode 100644 regressions/ck_rhs/benchmark/parallel_bytestring.c create mode 100644 regressions/ck_rhs/benchmark/serial.c create mode 100644 regressions/ck_rhs/validate/Makefile create mode 100644 regressions/ck_rhs/validate/serial.c create mode 100644 src/ck_rhs.c diff --git a/include/ck_rhs.h b/include/ck_rhs.h new file mode 100644 index 0000000..67ac0ec --- /dev/null +++ b/include/ck_rhs.h @@ -0,0 +1,125 @@ +/* + * Copyright 2012-2014 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _CK_HS_H +#define _CK_HS_H + +#include +#include +#include +#include +#include +#include +#include + +/* + * Indicates a single-writer many-reader workload. Mutually + * exclusive with CK_RHS_MODE_MPMC + */ +#define CK_RHS_MODE_SPMC 1 + +/* + * Indicates that values to be stored are not pointers but + * values. Allows for full precision. Mutually exclusive + * with CK_RHS_MODE_OBJECT. + */ +#define CK_RHS_MODE_DIRECT 2 + +/* + * Indicates that the values to be stored are pointers. + * Allows for space optimizations in the presence of pointer + * packing. Mutually exclusive with CK_RHS_MODE_DIRECT. + */ +#define CK_RHS_MODE_OBJECT 8 + +/* Currently unsupported. */ +#define CK_RHS_MODE_MPMC (void) + +/* + * Hash callback function. + */ +typedef unsigned long ck_rhs_hash_cb_t(const void *, unsigned long); + +/* + * Returns pointer to object if objects are equivalent. + */ +typedef bool ck_rhs_compare_cb_t(const void *, const void *); + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_RHS_PP +#define CK_RHS_KEY_MASK ((1U << ((sizeof(void *) * 8) - CK_MD_VMA_BITS)) - 1) +#endif + +struct ck_rhs_map; +struct ck_rhs { + struct ck_malloc *m; + struct ck_rhs_map *map; + unsigned int mode; + unsigned long seed; + ck_rhs_hash_cb_t *hf; + ck_rhs_compare_cb_t *compare; +}; +typedef struct ck_rhs ck_rhs_t; + +struct ck_rhs_stat { + unsigned long n_entries; + unsigned int probe_maximum; +}; + +struct ck_rhs_iterator { + void **cursor; + unsigned long offset; +}; +typedef struct ck_rhs_iterator ck_rhs_iterator_t; + +#define CK_RHS_ITERATOR_INITIALIZER { NULL, 0 } + +/* Convenience wrapper to table hash function. */ +#define CK_RHS_HASH(T, F, K) F((K), (T)->seed) + +void ck_rhs_iterator_init(ck_rhs_iterator_t *); +bool ck_rhs_next(ck_rhs_t *, ck_rhs_iterator_t *, void **); +bool ck_rhs_move(ck_rhs_t *, ck_rhs_t *, ck_rhs_hash_cb_t *, + ck_rhs_compare_cb_t *, struct ck_malloc *); +bool ck_rhs_init(ck_rhs_t *, unsigned int, ck_rhs_hash_cb_t *, + ck_rhs_compare_cb_t *, struct ck_malloc *, unsigned long, unsigned long); +void ck_rhs_destroy(ck_rhs_t *); +void *ck_rhs_get(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_put(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_put_unique(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_set(ck_rhs_t *, unsigned long, const void *, void **); +bool ck_rhs_fas(ck_rhs_t *, unsigned long, const void *, void **); +void *ck_rhs_remove(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_grow(ck_rhs_t *, unsigned long); +bool ck_rhs_rebuild(ck_rhs_t *); +bool ck_rhs_gc(ck_rhs_t *); +unsigned long ck_rhs_count(ck_rhs_t *); +bool ck_rhs_reset(ck_rhs_t *); +bool ck_rhs_reset_size(ck_rhs_t *, unsigned long); +void ck_rhs_stat(ck_rhs_t *, struct ck_rhs_stat *); + +#endif /* _CK_HS_H */ + diff --git a/regressions/ck_rhs/benchmark/Makefile b/regressions/ck_rhs/benchmark/Makefile new file mode 100644 index 0000000..e997993 --- /dev/null +++ b/regressions/ck_rhs/benchmark/Makefile @@ -0,0 +1,17 @@ +.PHONY: clean distribution + +OBJECTS=serial parallel_bytestring + +all: $(OBJECTS) + +serial: serial.c ../../../include/ck_rhs.h ../../../src/ck_rhs.c + $(CC) $(CFLAGS) -o serial serial.c ../../../src/ck_rhs.c + +parallel_bytestring: parallel_bytestring.c ../../../include/ck_rhs.h ../../../src/ck_rhs.c ../../../src/ck_epoch.c + $(CC) $(PTHREAD_CFLAGS) $(CFLAGS) -o parallel_bytestring parallel_bytestring.c ../../../src/ck_rhs.c ../../../src/ck_epoch.c + +clean: + rm -rf *~ *.o $(OBJECTS) *.dSYM *.exe + +include ../../../build/regressions.build +CFLAGS+=-D_GNU_SOURCE diff --git a/regressions/ck_rhs/benchmark/parallel_bytestring.c b/regressions/ck_rhs/benchmark/parallel_bytestring.c new file mode 100644 index 0000000..f99cc89 --- /dev/null +++ b/regressions/ck_rhs/benchmark/parallel_bytestring.c @@ -0,0 +1,599 @@ +/* + * Copyright 2012 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyrighs + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyrighs + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include "../../common.h" +#include +#include "../../../src/ck_ht_hash.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static ck_rhs_t hs CK_CC_CACHELINE; +static char **keys; +static size_t keys_length = 0; +static size_t keys_capacity = 128; +static ck_epoch_t epoch_hs; +static ck_epoch_record_t epoch_wr; +static int n_threads; +static bool next_stage; + +enum state { + HS_STATE_STOP = 0, + HS_STATE_GET, + HS_STATE_STRICT_REPLACEMENT, + HS_STATE_DELETION, + HS_STATE_REPLACEMENT, + HS_STATE_COUNT +}; + +static ck_spinlock_t mtx = CK_SPINLOCK_INITIALIZER; +static struct affinity affinerator = AFFINITY_INITIALIZER; +static uint64_t accumulator[HS_STATE_COUNT]; +static int barrier[HS_STATE_COUNT]; +static int state; + +struct hs_epoch { + ck_epoch_entry_t epoch_entry; +}; + +COMMON_ALARM_DECLARE_GLOBAL(hs_alarm, alarm_event, next_stage) + +static void +alarm_handler(int s) +{ + + (void)s; + next_stage = true; + return; +} + +static unsigned long +hs_hash(const void *object, unsigned long seed) +{ + const char *c = object; + unsigned long h; + + h = (unsigned long)MurmurHash64A(c, strlen(c), seed); + return h; +} + +static bool +hs_compare(const void *previous, const void *compare) +{ + + return strcmp(previous, compare) == 0; +} + +static void +hs_destroy(ck_epoch_entry_t *e) +{ + + free(e); + return; +} + +static void * +hs_malloc(size_t r) +{ + ck_epoch_entry_t *b; + + b = malloc(sizeof(*b) + r); + return b + 1; +} + +static void +hs_free(void *p, size_t b, bool r) +{ + struct hs_epoch *e = p; + + (void)b; + + if (r == true) { + /* Destruction requires safe memory reclamation. */ + ck_epoch_call(&epoch_hs, &epoch_wr, &(--e)->epoch_entry, hs_destroy); + } else { + free(--e); + } + + return; +} + +static struct ck_malloc my_allocator = { + .malloc = hs_malloc, + .free = hs_free +}; + +static void +set_init(void) +{ + unsigned int mode = CK_RHS_MODE_OBJECT | CK_RHS_MODE_SPMC; + + + ck_epoch_init(&epoch_hs); + ck_epoch_register(&epoch_hs, &epoch_wr); + common_srand48((long int)time(NULL)); + if (ck_rhs_init(&hs, mode, hs_hash, hs_compare, &my_allocator, 65536, common_lrand48()) == false) { + perror("ck_rhs_init"); + exit(EXIT_FAILURE); + } + + return; +} + +static bool +set_remove(const char *value) +{ + unsigned long h; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return (bool)ck_rhs_remove(&hs, h, value); +} + +static bool +set_replace(const char *value) +{ + unsigned long h; + void *previous; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_set(&hs, h, value, &previous); +} + +static bool +set_swap(const char *value) +{ + unsigned long h; + void *previous; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_fas(&hs, h, value, &previous); +} + +static void * +set_get(const char *value) +{ + unsigned long h; + void *v; + + h = CK_RHS_HASH(&hs, hs_hash, value); + v = ck_rhs_get(&hs, h, value); + return v; +} + +static bool +set_insert(const char *value) +{ + unsigned long h; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_put(&hs, h, value); +} + +static size_t +set_count(void) +{ + + return ck_rhs_count(&hs); +} + +static bool +set_reset(void) +{ + + return ck_rhs_reset(&hs); +} + +static void * +reader(void *unused) +{ + size_t i; + ck_epoch_record_t epoch_record; + int state_previous = HS_STATE_STOP; + int n_state = 0; + uint64_t s, j, a; + + (void)unused; + if (aff_iterate(&affinerator) != 0) + perror("WARNING: Failed to affine thread"); + + s = j = a = 0; + ck_epoch_register(&epoch_hs, &epoch_record); + for (;;) { + j++; + ck_epoch_begin(&epoch_hs, &epoch_record); + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + char *r; + + r = set_get(keys[i]); + if (r == NULL) { + if (n_state == HS_STATE_STRICT_REPLACEMENT) { + ck_error("ERROR: Did not find during replacement: %s\n", keys[i]); + } + + continue; + } + + if (strcmp(r, keys[i]) == 0) + continue; + + ck_error("ERROR: Found invalid value: [%s] but expected [%s]\n", (char *)r, keys[i]); + } + a += rdtsc() - s; + ck_epoch_end(&epoch_hs, &epoch_record); + + n_state = ck_pr_load_int(&state); + if (n_state != state_previous) { + ck_spinlock_lock(&mtx); + accumulator[state_previous] += a / (j * keys_length); + ck_spinlock_unlock(&mtx); + + ck_pr_inc_int(&barrier[state_previous]); + while (ck_pr_load_int(&barrier[state_previous]) != n_threads + 1) + ck_pr_stall(); + + state_previous = n_state; + s = j = a = 0; + } + } + + return NULL; +} + +static uint64_t +acc(size_t i) +{ + uint64_t r; + + ck_spinlock_lock(&mtx); + r = accumulator[i]; + ck_spinlock_unlock(&mtx); + + return r; +} + +int +main(int argc, char *argv[]) +{ + FILE *fp; + char buffer[512]; + size_t i, j, r; + unsigned int d = 0; + uint64_t s, e, a, repeated; + char **t; + pthread_t *readers; + double p_r, p_d; + + COMMON_ALARM_DECLARE_LOCAL(hs_alarm, alarm_event) + + r = 20; + s = 8; + p_d = 0.5; + p_r = 0.5; + n_threads = CORES - 1; + + if (argc < 2) { + ck_error("Usage: parallel [ \n" + " ]\n"); + } + + if (argc >= 3) + r = atoi(argv[2]); + + if (argc >= 4) + s = (uint64_t)atoi(argv[3]); + + if (argc >= 5) { + n_threads = atoi(argv[4]); + if (n_threads < 1) { + ck_error("ERROR: Number of readers must be >= 1.\n"); + } + } + + if (argc >= 6) { + p_r = atof(argv[5]) / 100.00; + if (p_r < 0) { + ck_error("ERROR: Probability of replacement must be >= 0 and <= 100.\n"); + } + } + + if (argc >= 7) { + p_d = atof(argv[6]) / 100.00; + if (p_d < 0) { + ck_error("ERROR: Probability of deletion must be >= 0 and <= 100.\n"); + } + } + + COMMON_ALARM_INIT(hs_alarm, alarm_event, r) + + affinerator.delta = 1; + readers = malloc(sizeof(pthread_t) * n_threads); + assert(readers != NULL); + + keys = malloc(sizeof(char *) * keys_capacity); + assert(keys != NULL); + + fp = fopen(argv[1], "r"); + assert(fp != NULL); + + while (fgets(buffer, sizeof(buffer), fp) != NULL) { + buffer[strlen(buffer) - 1] = '\0'; + keys[keys_length++] = strdup(buffer); + assert(keys[keys_length - 1] != NULL); + + if (keys_length == keys_capacity) { + t = realloc(keys, sizeof(char *) * (keys_capacity *= 2)); + assert(t != NULL); + keys = t; + } + } + + t = realloc(keys, sizeof(char *) * keys_length); + assert(t != NULL); + keys = t; + + set_init(); + + for (i = 0; i < (size_t)n_threads; i++) { + if (pthread_create(&readers[i], NULL, reader, NULL) != 0) { + ck_error("ERROR: Failed to create thread %zu.\n", i); + } + } + + for (i = 0; i < keys_length; i++) + d += set_insert(keys[i]) == false; + + fprintf(stderr, " [S] %d readers, 1 writer.\n", n_threads); + fprintf(stderr, " [S] %zu entries stored and %u duplicates.\n\n", + set_count(), d); + + fprintf(stderr, " ,- BASIC TEST\n"); + fprintf(stderr, " | Executing SMR test..."); + a = 0; + for (j = 0; j < r; j++) { + if (set_reset() == false) { + ck_error("ERROR: Failed to reset hash table.\n"); + } + + s = rdtsc(); + for (i = 0; i < keys_length; i++) + d += set_insert(keys[i]) == false; + e = rdtsc(); + a += e - s; + } + fprintf(stderr, "done (%" PRIu64 " ticks)\n", a / (r * keys_length)); + + fprintf(stderr, " | Executing replacement test..."); + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_replace(keys[i]); + e = rdtsc(); + a += e - s; + } + fprintf(stderr, "done (%" PRIu64 " ticks)\n", a / (r * keys_length)); + + fprintf(stderr, " | Executing get test..."); + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + if (set_get(keys[i]) == NULL) { + ck_error("ERROR: Unexpected NULL value.\n"); + } + } + e = rdtsc(); + a += e - s; + } + fprintf(stderr, "done (%" PRIu64 " ticks)\n", a / (r * keys_length)); + + a = 0; + fprintf(stderr, " | Executing removal test..."); + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_remove(keys[i]); + e = rdtsc(); + a += e - s; + + for (i = 0; i < keys_length; i++) + set_insert(keys[i]); + } + fprintf(stderr, "done (%" PRIu64 " ticks)\n", a / (r * keys_length)); + + fprintf(stderr, " | Executing negative look-up test..."); + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + set_get("\x50\x03\x04\x05\x06\x10"); + } + e = rdtsc(); + a += e - s; + } + fprintf(stderr, "done (%" PRIu64 " ticks)\n", a / (r * keys_length)); + + ck_epoch_record_t epoch_temporary = epoch_wr; + ck_epoch_synchronize(&epoch_hs, &epoch_wr); + + fprintf(stderr, " '- Summary: %u pending, %u peak, %lu reclamations -> " + "%u pending, %u peak, %lu reclamations\n\n", + epoch_temporary.n_pending, epoch_temporary.n_peak, epoch_temporary.n_dispatch, + epoch_wr.n_pending, epoch_wr.n_peak, epoch_wr.n_dispatch); + + fprintf(stderr, " ,- READER CONCURRENCY\n"); + fprintf(stderr, " | Executing reader test..."); + + ck_pr_store_int(&state, HS_STATE_GET); + while (ck_pr_load_int(&barrier[HS_STATE_STOP]) != n_threads) + ck_pr_stall(); + ck_pr_inc_int(&barrier[HS_STATE_STOP]); + common_sleep(r); + ck_pr_store_int(&state, HS_STATE_STRICT_REPLACEMENT); + while (ck_pr_load_int(&barrier[HS_STATE_GET]) != n_threads) + ck_pr_stall(); + + fprintf(stderr, "done (reader = %" PRIu64 " ticks)\n", + acc(HS_STATE_GET) / n_threads); + + fprintf(stderr, " | Executing strict replacement test..."); + + a = repeated = 0; + common_alarm(alarm_handler, &alarm_event, r); + + ck_pr_inc_int(&barrier[HS_STATE_GET]); + for (;;) { + repeated++; + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + if (i & 1) { + set_replace(keys[i]); + } else { + set_swap(keys[i]); + } + } + e = rdtsc(); + a += e - s; + + if (next_stage == true) { + next_stage = false; + break; + } + } + + ck_pr_store_int(&state, HS_STATE_DELETION); + while (ck_pr_load_int(&barrier[HS_STATE_STRICT_REPLACEMENT]) != n_threads) + ck_pr_stall(); + set_reset(); + ck_epoch_synchronize(&epoch_hs, &epoch_wr); + fprintf(stderr, "done (writer = %" PRIu64 " ticks, reader = %" PRIu64 " ticks)\n", + a / (repeated * keys_length), acc(HS_STATE_STRICT_REPLACEMENT) / n_threads); + + common_alarm(alarm_handler, &alarm_event, r); + + fprintf(stderr, " | Executing deletion test (%.2f)...", p_d * 100); + a = repeated = 0; + ck_pr_inc_int(&barrier[HS_STATE_STRICT_REPLACEMENT]); + for (;;) { + double delete; + + repeated++; + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + set_insert(keys[i]); + if (p_d != 0.0) { + delete = common_drand48(); + if (delete <= p_d) + set_remove(keys[i]); + } + } + e = rdtsc(); + a += e - s; + + if (next_stage == true) { + next_stage = false; + break; + } + } + ck_pr_store_int(&state, HS_STATE_REPLACEMENT); + while (ck_pr_load_int(&barrier[HS_STATE_DELETION]) != n_threads) + ck_pr_stall(); + + set_reset(); + ck_epoch_synchronize(&epoch_hs, &epoch_wr); + fprintf(stderr, "done (writer = %" PRIu64 " ticks, reader = %" PRIu64 " ticks)\n", + a / (repeated * keys_length), acc(HS_STATE_DELETION) / n_threads); + + common_alarm(alarm_handler, &alarm_event, r); + + fprintf(stderr, " | Executing replacement test (%.2f)...", p_r * 100); + a = repeated = 0; + ck_pr_inc_int(&barrier[HS_STATE_DELETION]); + for (;;) { + double delete, replace; + + repeated++; + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + set_insert(keys[i]); + if (p_d != 0.0) { + delete = common_drand48(); + if (delete <= p_d) + set_remove(keys[i]); + } else { + delete = 0.0; + } + + if (p_r != 0.0) { + replace = common_drand48(); + if (replace <= p_r) { + if ((i & 1) || (delete <= p_d)) { + set_replace(keys[i]); + } else { + set_swap(keys[i]); + } + } + } + } + e = rdtsc(); + a += e - s; + + if (next_stage == true) { + next_stage = false; + break; + } + } + ck_pr_store_int(&state, HS_STATE_STOP); + while (ck_pr_load_int(&barrier[HS_STATE_REPLACEMENT]) != n_threads) + ck_pr_stall(); + set_reset(); + ck_epoch_synchronize(&epoch_hs, &epoch_wr); + fprintf(stderr, "done (writer = %" PRIu64 " ticks, reader = %" PRIu64 " ticks)\n", + a / (repeated * keys_length), acc(HS_STATE_REPLACEMENT) / n_threads); + + ck_pr_inc_int(&barrier[HS_STATE_REPLACEMENT]); + epoch_temporary = epoch_wr; + ck_epoch_synchronize(&epoch_hs, &epoch_wr); + + fprintf(stderr, " '- Summary: %u pending, %u peak, %lu reclamations -> " + "%u pending, %u peak, %lu reclamations\n\n", + epoch_temporary.n_pending, epoch_temporary.n_peak, epoch_temporary.n_dispatch, + epoch_wr.n_pending, epoch_wr.n_peak, epoch_wr.n_dispatch); + return 0; +} + diff --git a/regressions/ck_rhs/benchmark/serial.c b/regressions/ck_rhs/benchmark/serial.c new file mode 100644 index 0000000..94c7fde --- /dev/null +++ b/regressions/ck_rhs/benchmark/serial.c @@ -0,0 +1,516 @@ +/* + * Copyright 2012 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyrighs + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyrighs + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../common.h" +#include "../../../src/ck_ht_hash.h" + +static ck_rhs_t hs; +static char **keys; +static size_t keys_length = 0; +static size_t keys_capacity = 128; +static unsigned long global_seed; + +static void * +hs_malloc(size_t r) +{ + + return malloc(r); +} + +static void +hs_free(void *p, size_t b, bool r) +{ + + (void)b; + (void)r; + + free(p); + + return; +} + +static struct ck_malloc my_allocator = { + .malloc = hs_malloc, + .free = hs_free +}; + +static unsigned long +hs_hash(const void *object, unsigned long seed) +{ + const char *c = object; + unsigned long h; + + h = (unsigned long)MurmurHash64A(c, strlen(c), seed); + return h; +} + +static bool +hs_compare(const void *previous, const void *compare) +{ + + return strcmp(previous, compare) == 0; +} + +static void +set_destroy(void) +{ + + ck_rhs_destroy(&hs); + return; +} + +static void +set_init(unsigned int size, unsigned int mode) +{ + + if (ck_rhs_init(&hs, CK_RHS_MODE_OBJECT | CK_RHS_MODE_SPMC | mode, hs_hash, hs_compare, + &my_allocator, size, global_seed) == false) { + perror("ck_rhs_init"); + exit(EXIT_FAILURE); + } + + return; +} + +static bool +set_remove(const char *value) +{ + unsigned long h; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_remove(&hs, h, value) != NULL; +} + +static bool +set_swap(const char *value) +{ + unsigned long h; + void *previous; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_fas(&hs, h, value, &previous); +} + +static bool +set_replace(const char *value) +{ + unsigned long h; + void *previous; + + h = CK_RHS_HASH(&hs, hs_hash, value); + ck_rhs_set(&hs, h, value, &previous); + return previous != NULL; +} + +static void * +set_get(const char *value) +{ + unsigned long h; + void *v; + + h = CK_RHS_HASH(&hs, hs_hash, value); + v = ck_rhs_get(&hs, h, value); + return v; +} + +static bool +set_insert(const char *value) +{ + unsigned long h; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_put(&hs, h, value); +} + +static bool +set_insert_unique(const char *value) +{ + unsigned long h; + + h = CK_RHS_HASH(&hs, hs_hash, value); + return ck_rhs_put_unique(&hs, h, value); +} + +static size_t +set_count(void) +{ + + return ck_rhs_count(&hs); +} + +static bool +set_reset(void) +{ + + return ck_rhs_reset(&hs); +} + +static void +set_gc(void) +{ + + ck_rhs_gc(&hs); + return; +} + +static void +set_rebuild(void) +{ + + ck_rhs_rebuild(&hs); + return; +} + +static void +keys_shuffle(char **k) +{ + size_t i, j; + char *t; + + for (i = keys_length; i > 1; i--) { + j = rand() % (i - 1); + + if (j != i - 1) { + t = k[i - 1]; + k[i - 1] = k[j]; + k[j] = t; + } + } + + return; +} + +static void +run_test(const char *file, size_t r, unsigned int size, unsigned int mode) +{ + FILE *fp; + char buffer[512]; + size_t i, j; + unsigned int d = 0; + uint64_t s, e, a, ri, si, ai, sr, rg, sg, ag, sd, ng, ss, sts, su, sgc, sb; + struct ck_rhs_stat st; + char **t; + + keys = malloc(sizeof(char *) * keys_capacity); + assert(keys != NULL); + + fp = fopen(file, "r"); + assert(fp != NULL); + + while (fgets(buffer, sizeof(buffer), fp) != NULL) { + buffer[strlen(buffer) - 1] = '\0'; + keys[keys_length++] = strdup(buffer); + assert(keys[keys_length - 1] != NULL); + + if (keys_length == keys_capacity) { + t = realloc(keys, sizeof(char *) * (keys_capacity *= 2)); + assert(t != NULL); + keys = t; + } + } + + t = realloc(keys, sizeof(char *) * keys_length); + assert(t != NULL); + keys = t; + + set_init(size, mode); + for (i = 0; i < keys_length; i++) + d += set_insert(keys[i]) == false; + ck_rhs_stat(&hs, &st); + + fprintf(stderr, "# %zu entries stored, %u duplicates, %u probe.\n", + set_count(), d, st.probe_maximum); + + a = 0; + for (j = 0; j < r; j++) { + if (set_reset() == false) { + ck_error("ERROR: Failed to reset hash table.\n"); + } + + s = rdtsc(); + for (i = keys_length; i > 0; i--) + d += set_insert(keys[i - 1]) == false; + e = rdtsc(); + a += e - s; + } + ri = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + if (set_reset() == false) { + ck_error("ERROR: Failed to reset hash table.\n"); + } + + s = rdtsc(); + for (i = 0; i < keys_length; i++) + d += set_insert(keys[i]) == false; + e = rdtsc(); + a += e - s; + } + si = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + keys_shuffle(keys); + + if (set_reset() == false) { + ck_error("ERROR: Failed to reset hash table.\n"); + } + + s = rdtsc(); + for (i = 0; i < keys_length; i++) + d += set_insert(keys[i]) == false; + e = rdtsc(); + a += e - s; + } + ai = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_swap(keys[i]); + e = rdtsc(); + a += e - s; + } + ss = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_replace(keys[i]); + e = rdtsc(); + a += e - s; + } + sr = a / (r * keys_length); + + set_reset(); + for (i = 0; i < keys_length; i++) + set_insert(keys[i]); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = keys_length; i > 0; i--) { + if (set_get(keys[i - 1]) == NULL) { + ck_error("ERROR: Unexpected NULL value.\n"); + } + } + e = rdtsc(); + a += e - s; + } + rg = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + if (set_get(keys[i]) == NULL) { + ck_error("ERROR: Unexpected NULL value.\n"); + } + } + e = rdtsc(); + a += e - s; + } + sg = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + keys_shuffle(keys); + + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + if (set_get(keys[i]) == NULL) { + ck_error("ERROR: Unexpected NULL value.\n"); + } + } + e = rdtsc(); + a += e - s; + } + ag = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_remove(keys[i]); + e = rdtsc(); + a += e - s; + + for (i = 0; i < keys_length; i++) + set_insert(keys[i]); + } + sd = a / (r * keys_length); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) { + set_get("\x50\x03\x04\x05\x06\x10"); + } + e = rdtsc(); + a += e - s; + } + ng = a / (r * keys_length); + + set_reset(); + for (i = 0; i < keys_length; i++) + set_insert(keys[i]); + for (i = 0; i < keys_length; i++) + set_remove(keys[i]); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_insert(keys[i]); + e = rdtsc(); + a += e - s; + + for (i = 0; i < keys_length; i++) + set_remove(keys[i]); + } + sts = a / (r * keys_length); + + set_reset(); + + /* Prune duplicates. */ + for (i = 0; i < keys_length; i++) { + if (set_insert(keys[i]) == true) + continue; + + free(keys[i]); + keys[i] = keys[--keys_length]; + } + + for (i = 0; i < keys_length; i++) + set_remove(keys[i]); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + for (i = 0; i < keys_length; i++) + set_insert_unique(keys[i]); + e = rdtsc(); + a += e - s; + + for (i = 0; i < keys_length; i++) + set_remove(keys[i]); + } + su = a / (r * keys_length); + + for (i = 0; i < keys_length; i++) + set_insert_unique(keys[i]); + + for (i = 0; i < keys_length / 2; i++) + set_remove(keys[i]); + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + set_gc(); + e = rdtsc(); + a += e - s; + } + sgc = a / r; + + a = 0; + for (j = 0; j < r; j++) { + s = rdtsc(); + set_rebuild(); + e = rdtsc(); + a += e - s; + } + sb = a / r; + + printf("%zu " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 " " + "%" PRIu64 "\n", + keys_length, ri, si, ai, ss, sr, rg, sg, ag, sd, ng, sts, su, sgc, sb); + + fclose(fp); + + for (i = 0; i < keys_length; i++) { + free(keys[i]); + } + + free(keys); + keys_length = 0; + set_destroy(); + return; +} + +int +main(int argc, char *argv[]) +{ + unsigned int r, size; + + common_srand48((long int)time(NULL)); + if (argc < 2) { + ck_error("Usage: ck_rhs [ ]\n"); + } + + r = 16; + if (argc >= 3) + r = atoi(argv[2]); + + size = 8; + if (argc >= 4) + size = atoi(argv[3]); + + global_seed = common_lrand48(); + run_test(argv[1], r, size, 0); + fprintf(stderr, "# reverse_insertion serial_insertion random_insertion serial_swap " + "serial_replace reverse_get serial_get random_get serial_remove negative_get tombstone " + "set_unique gc rebuild\n\n"); + + return 0; +} + diff --git a/regressions/ck_rhs/validate/Makefile b/regressions/ck_rhs/validate/Makefile new file mode 100644 index 0000000..5987395 --- /dev/null +++ b/regressions/ck_rhs/validate/Makefile @@ -0,0 +1,17 @@ +.PHONY: check clean distribution + +OBJECTS=serial + +all: $(OBJECTS) + +serial: serial.c ../../../include/ck_rhs.h ../../../src/ck_rhs.c + $(CC) $(CFLAGS) -o serial serial.c ../../../src/ck_rhs.c + +check: all + ./serial + +clean: + rm -rf *~ *.o $(OBJECTS) *.dSYM *.exe + +include ../../../build/regressions.build +CFLAGS+=-D_GNU_SOURCE diff --git a/regressions/ck_rhs/validate/serial.c b/regressions/ck_rhs/validate/serial.c new file mode 100644 index 0000000..1dc4db1 --- /dev/null +++ b/regressions/ck_rhs/validate/serial.c @@ -0,0 +1,245 @@ +/* + * Copyright 2012 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyrighs + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyrighs + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "../../common.h" + +static void * +hs_malloc(size_t r) +{ + + return malloc(r); +} + +static void +hs_free(void *p, size_t b, bool r) +{ + + (void)b; + (void)r; + free(p); + return; +} + +static struct ck_malloc my_allocator = { + .malloc = hs_malloc, + .free = hs_free +}; + +const char *test[] = { "Samy", "Al", "Bahra", "dances", "in", "the", "wind.", "Once", + "upon", "a", "time", "his", "gypsy", "ate", "one", "itsy", + "bitsy", "spider.", "What", "goes", "up", "must", + "come", "down.", "What", "is", "down", "stays", + "down.", "A", "B", "C", "D", "E", "F", "G", "H", + "I", "J", "K", "L", "M", "N", "O", "P", "Q" }; + +const char *negative = "negative"; + +/* Purposefully crappy hash function. */ +static unsigned long +hs_hash(const void *object, unsigned long seed) +{ + const char *c = object; + unsigned long h; + + (void)seed; + h = c[0]; + return h; +} + +static bool +hs_compare(const void *previous, const void *compare) +{ + + return strcmp(previous, compare) == 0; +} + +static void +run_test(unsigned int is, unsigned int ad) +{ + ck_rhs_t hs[16]; + const size_t size = sizeof(hs) / sizeof(*hs); + size_t i, j; + const char *blob = "#blobs"; + unsigned long h; + + if (ck_rhs_init(&hs[0], CK_RHS_MODE_SPMC | CK_RHS_MODE_OBJECT | ad, hs_hash, hs_compare, &my_allocator, is, 6602834) == false) + ck_error("ck_rhs_init\n"); + + for (j = 0; j < size; j++) { + for (i = 0; i < sizeof(test) / sizeof(*test); i++) { + h = test[i][0]; + if (ck_rhs_get(&hs[j], h, test[i]) != NULL) { + continue; + } + + if (ck_rhs_put_unique(&hs[j], h, test[i]) == false) + ck_error("ERROR [%zu]: Failed to insert unique (%s)\n", j, test[i]); + + if (ck_rhs_remove(&hs[j], h, test[i]) == false) + ck_error("ERROR [%zu]: Failed to remove unique (%s)\n", j, test[i]); + + break; + } + + for (i = 0; i < sizeof(test) / sizeof(*test); i++) { + h = test[i][0]; + ck_rhs_put(&hs[j], h, test[i]); + if (ck_rhs_put(&hs[j], h, test[i]) == true) { + ck_error("ERROR [%u] [1]: put must fail on collision (%s).\n", is, test[i]); + } + if (ck_rhs_get(&hs[j], h, test[i]) == NULL) { + ck_error("ERROR [%u]: get must not fail after put\n", is); + } + } + + /* Test grow semantics. */ + ck_rhs_grow(&hs[j], 128); + for (i = 0; i < sizeof(test) / sizeof(*test); i++) { + h = test[i][0]; + if (ck_rhs_put(&hs[j], h, test[i]) == true) { + ck_error("ERROR [%u] [2]: put must fail on collision.\n", is); + } + + if (ck_rhs_get(&hs[j], h, test[i]) == NULL) { + ck_error("ERROR [%u]: get must not fail\n", is); + } + } + + h = blob[0]; + if (ck_rhs_get(&hs[j], h, blob) == NULL) { + if (j > 0) + ck_error("ERROR [%u]: Blob must always exist after first.\n", is); + + if (ck_rhs_put(&hs[j], h, blob) == false) { + ck_error("ERROR [%u]: A unique blob put failed.\n", is); + } + } else { + if (ck_rhs_put(&hs[j], h, blob) == true) { + ck_error("ERROR [%u]: Duplicate blob put succeeded.\n", is); + } + } + + /* Grow set and check get semantics. */ + ck_rhs_grow(&hs[j], 512); + for (i = 0; i < sizeof(test) / sizeof(*test); i++) { + h = test[i][0]; + if (ck_rhs_get(&hs[j], h, test[i]) == NULL) { + ck_error("ERROR [%u]: get must not fail\n", is); + } + } + + /* Delete and check negative membership. */ + for (i = 0; i < sizeof(test) / sizeof(*test); i++) { + void *r; + + h = test[i][0]; + if (ck_rhs_get(&hs[j], h, test[i]) == NULL) + continue; + + if (r = ck_rhs_remove(&hs[j], h, test[i]), r == NULL) { + ck_error("ERROR [%u]: remove must not fail\n", is); + } + + if (strcmp(r, test[i]) != 0) { + ck_error("ERROR [%u]: Removed incorrect node (%s != %s)\n", (char *)r, test[i], is); + } + } + + /* Test replacement semantics. */ + for (i = 0; i < sizeof(test) / sizeof(*test); i++) { + void *r; + bool d; + + h = test[i][0]; + d = ck_rhs_get(&hs[j], h, test[i]) != NULL; + if (ck_rhs_set(&hs[j], h, test[i], &r) == false) { + ck_error("ERROR [%u]: Failed to set\n", is); + } + + /* Expected replacement. */ + if (d == true && (r == NULL || strcmp(r, test[i]) != 0)) { + ck_error("ERROR [%u]: Incorrect previous value: %s != %s\n", + is, test[i], (char *)r); + } + + /* Replacement should succeed. */ + if (ck_rhs_fas(&hs[j], h, test[i], &r) == false) + ck_error("ERROR [%u]: ck_rhs_fas must succeed.\n", is); + + if (strcmp(r, test[i]) != 0) { + ck_error("ERROR [%u]: Incorrect replaced value: %s != %s\n", + is, test[i], (char *)r); + } + + if (ck_rhs_fas(&hs[j], h, negative, &r) == true) + ck_error("ERROR [%u]: Replacement of negative should fail.\n", is); + + if (ck_rhs_set(&hs[j], h, test[i], &r) == false) { + ck_error("ERROR [%u]: Failed to set [1]\n", is); + } + + if (strcmp(r, test[i]) != 0) { + ck_error("ERROR [%u]: Invalid &hs[j]: %s != %s\n", (char *)r, test[i], is); + } + } + + if (j == size - 1) + break; + + if (ck_rhs_move(&hs[j + 1], &hs[j], hs_hash, hs_compare, &my_allocator) == false) + ck_error("Failed to move hash table"); + + ck_rhs_gc(&hs[j + 1]); + + if (ck_rhs_rebuild(&hs[j + 1]) == false) + ck_error("Failed to rebuild"); + } + + return; +} + +int +main(void) +{ + unsigned int k; + + for (k = 16; k <= 64; k <<= 1) { + run_test(k, 0); + break; + } + + return 0; +} + diff --git a/src/Makefile.in b/src/Makefile.in index 5b3a66c..d1d6ae1 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -16,6 +16,7 @@ OBJECTS=ck_barrier_centralized.o \ ck_ht.o \ ck_hp.o \ ck_hs.o \ + ck_rhs.o \ ck_array.o all: libck.so libck.a diff --git a/src/ck_rhs.c b/src/ck_rhs.c new file mode 100644 index 0000000..0b33597 --- /dev/null +++ b/src/ck_rhs.c @@ -0,0 +1,1026 @@ +/* + * Copyright 2014 Olivier Houchard + * Copyright 2012-2014 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ck_internal.h" + +#ifndef CK_RHS_PROBE_L1_SHIFT +#define CK_RHS_PROBE_L1_SHIFT 3ULL +#endif /* CK_RHS_PROBE_L1_SHIFT */ + +#define CK_RHS_PROBE_L1 (1 << CK_RHS_PROBE_L1_SHIFT) +#define CK_RHS_PROBE_L1_MASK (CK_RHS_PROBE_L1 - 1) + +#ifndef CK_RHS_PROBE_L1_DEFAULT +#define CK_RHS_PROBE_L1_DEFAULT CK_MD_CACHELINE +#endif + +#define CK_RHS_VMA_MASK ((uintptr_t)((1ULL << CK_MD_VMA_BITS) - 1)) +#define CK_RHS_VMA(x) \ + ((void *)((uintptr_t)(x) & CK_RHS_VMA_MASK)) + +#define CK_RHS_EMPTY NULL +#define CK_RHS_G (2) +#define CK_RHS_G_MASK (CK_RHS_G - 1) + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8) +#define CK_RHS_WORD uint8_t +#define CK_RHS_WORD_MAX UINT8_MAX +#define CK_RHS_STORE(x, y) ck_pr_store_8(x, y) +#define CK_RHS_LOAD(x) ck_pr_load_8(x) +#elif defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_STORE_16) +#define CK_RHS_WORD uint16_t +#define CK_RHS_WORD_MAX UINT16_MAX +#define CK_RHS_STORE(x, y) ck_pr_store_16(x, y) +#define CK_RHS_LOAD(x) ck_pr_load_16(x) +#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32) +#define CK_RHS_WORD uint32_t +#define CK_RHS_WORD_MAX UINT32_MAX +#define CK_RHS_STORE(x, y) ck_pr_store_32(x, y) +#define CK_RHS_LOAD(x) ck_pr_load_32(x) +#else +#error "ck_rhs is not supported on your platform." +#endif + +#define CK_RHS_MAX_WANTED 0xffff + +enum ck_rhs_probe_behavior { + CK_RHS_PROBE = 0, /* Default behavior. */ + CK_RHS_PROBE_RH, /* Short-circuit if RH slot found. */ + CK_RHS_PROBE_INSERT, /* Short-circuit on probe bound if tombstone found. */ + + CK_RHS_PROBE_ROBIN_HOOD, /* Look for the first slot available for the entry we are about to replace, only used to internally implement Robin Hood */ + CK_RHS_PROBE_NO_RH, /* Don't do the RH dance */ +}; + +struct ck_rhs_entry_desc { + void *entry; + unsigned int probes; + unsigned short wanted; + CK_RHS_WORD probe_bound; + bool in_rh; +} __attribute__ ((__aligned__(16))); + +struct ck_rhs_map { + unsigned int generation[CK_RHS_G]; + unsigned int probe_maximum; + unsigned long mask; + unsigned long step; + unsigned int probe_limit; + unsigned long n_entries; + unsigned long capacity; + unsigned long size; + char offset_mask; + struct ck_rhs_entry_desc *descs; +}; + +void +ck_rhs_iterator_init(struct ck_rhs_iterator *iterator) +{ + + iterator->cursor = NULL; + iterator->offset = 0; + return; +} + +bool +ck_rhs_next(struct ck_rhs *hs, struct ck_rhs_iterator *i, void **key) +{ + struct ck_rhs_map *map = hs->map; + void *value; + + if (i->offset >= map->capacity) + return false; + + do { + value = map->descs[i->offset].entry; + if (value != CK_RHS_EMPTY) { +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + value = CK_RHS_VMA(value); +#endif + i->offset++; + *key = value; + return true; + } + } while (++i->offset < map->capacity); + + return false; +} + +void +ck_rhs_stat(struct ck_rhs *hs, struct ck_rhs_stat *st) +{ + struct ck_rhs_map *map = hs->map; + + st->n_entries = map->n_entries; + st->probe_maximum = map->probe_maximum; + return; +} + +unsigned long +ck_rhs_count(struct ck_rhs *hs) +{ + + return hs->map->n_entries; +} + +static void +ck_rhs_map_destroy(struct ck_malloc *m, struct ck_rhs_map *map, bool defer) +{ + + m->free(map, map->size, defer); + return; +} + +void +ck_rhs_destroy(struct ck_rhs *hs) +{ + + ck_rhs_map_destroy(hs->m, hs->map, false); + return; +} + +static struct ck_rhs_map * +ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries) +{ + struct ck_rhs_map *map; + unsigned long size, n_entries, limit; + + n_entries = ck_internal_power_2(entries); + if (n_entries < CK_RHS_PROBE_L1) + return NULL; + + size = sizeof(struct ck_rhs_map) + + (sizeof(struct ck_rhs_entry_desc) * n_entries + CK_MD_CACHELINE - 1); + + map = hs->m->malloc(size); + if (map == NULL) + return NULL; + + map->size = size; + map->offset_mask = (CK_MD_CACHELINE / sizeof(struct ck_rhs_entry_desc)) - 1; + /* We should probably use a more intelligent heuristic for default probe length. */ + limit = ck_internal_max(n_entries >> (CK_RHS_PROBE_L1_SHIFT + 2), CK_RHS_PROBE_L1_DEFAULT); + if (limit > UINT_MAX) + limit = UINT_MAX; + + map->probe_limit = (unsigned int)limit; + map->probe_maximum = 0; + map->capacity = n_entries; + map->step = ck_internal_bsf(n_entries); + map->mask = n_entries - 1; + map->n_entries = 0; + + /* Align map allocation to cache line. */ + map->descs = (void *)(((uintptr_t)&map[1] + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + memset(map->generation, 0, sizeof map->generation); + memset(map->descs, 0, sizeof(*map->descs) * n_entries); + + /* Commit entries purge with respect to map publication. */ + ck_pr_fence_store(); + return map; +} + +bool +ck_rhs_reset_size(struct ck_rhs *hs, unsigned long capacity) +{ + struct ck_rhs_map *map, *previous; + + previous = hs->map; + map = ck_rhs_map_create(hs, capacity); + if (map == NULL) + return false; + + ck_pr_store_ptr(&hs->map, map); + ck_rhs_map_destroy(hs->m, previous, true); + return true; +} + +bool +ck_rhs_reset(struct ck_rhs *hs) +{ + struct ck_rhs_map *previous; + + previous = hs->map; + return ck_rhs_reset_size(hs, previous->capacity); +} + +static inline unsigned long +ck_rhs_map_probe_next(struct ck_rhs_map *map, + unsigned long offset, + unsigned long probes) +{ + if (probes & map->offset_mask) { + offset = (offset &~ map->offset_mask) + ((offset + 1) & map->offset_mask); + return offset; + } else + return ((offset + probes) & map->mask); +} + +static inline unsigned long +ck_rhs_map_probe_prev(struct ck_rhs_map *map, unsigned long offset, + unsigned long probes) +{ + if (probes & map->offset_mask) { + offset = (offset &~ map->offset_mask) + ((offset - 1) & + map->offset_mask); + return offset; + } else + return ((offset - probes) & map->mask); +} + + +static inline void +ck_rhs_map_bound_set(struct ck_rhs_map *m, + unsigned long h, + unsigned long n_probes) +{ + unsigned long offset = h & m->mask; + + if (n_probes > m->probe_maximum) + ck_pr_store_uint(&m->probe_maximum, n_probes); + + if (m->descs[offset].probe_bound < n_probes) { + if (n_probes > CK_RHS_WORD_MAX) + n_probes = CK_RHS_WORD_MAX; + + CK_RHS_STORE(&m->descs[offset].probe_bound, n_probes); + ck_pr_fence_store(); + } + + return; +} + +static inline unsigned int +ck_rhs_map_bound_get(struct ck_rhs_map *m, unsigned long h) +{ + unsigned long offset = h & m->mask; + unsigned int r = CK_RHS_WORD_MAX; + + r = CK_RHS_LOAD(&m->descs[offset].probe_bound); + if (r == CK_RHS_WORD_MAX) + r = ck_pr_load_uint(&m->probe_maximum); + return r; +} + +bool +ck_rhs_grow(struct ck_rhs *hs, + unsigned long capacity) +{ + struct ck_rhs_map *map, *update; + void *previous, *prev_saved; + unsigned long k, offset, probes; + +restart: + map = hs->map; + if (map->capacity > capacity) + return false; + + update = ck_rhs_map_create(hs, capacity); + if (update == NULL) + return false; + + for (k = 0; k < map->capacity; k++) { + unsigned long h; + + prev_saved = previous = map->descs[k].entry; + if (previous == CK_RHS_EMPTY) + continue; + +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + previous = CK_RHS_VMA(previous); +#endif + + h = hs->hf(previous, hs->seed); + offset = h & update->mask; + probes = 0; + + for (;;) { + void **cursor = &update->descs[offset].entry; + + if (probes++ == update->probe_limit) { + /* + * We have hit the probe limit, map needs to be even larger. + */ + ck_rhs_map_destroy(hs->m, update, false); + capacity <<= 1; + goto restart; + } + + if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) { + *cursor = prev_saved; + update->n_entries++; + update->descs[offset].probes = probes; + ck_rhs_map_bound_set(update, h, probes); + break; + } else if (update->descs[offset].probes < probes) { + void *tmp = prev_saved; + unsigned int old_probes; + prev_saved = previous = *cursor; +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + previous = CK_RHS_VMA(previous); +#endif + *cursor = tmp; + ck_rhs_map_bound_set(update, h, probes); + h = hs->hf(previous, hs->seed); + old_probes = update->descs[offset].probes; + update->descs[offset].probes = probes; + probes = old_probes - 1; + continue; + } + update->descs[offset].wanted++; + + offset = ck_rhs_map_probe_next(update, offset, probes); + } + + } + + ck_pr_fence_store(); + ck_pr_store_ptr(&hs->map, update); + ck_rhs_map_destroy(hs->m, map, true); + return true; +} + +bool +ck_rhs_rebuild(struct ck_rhs *hs) +{ + + return ck_rhs_grow(hs, hs->map->capacity); +} + +static struct ck_rhs_entry_desc * +ck_rhs_map_probe(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + struct ck_rhs_entry_desc **priority, + unsigned long h, + const void *key, + void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior) +{ + + void *k; + const void *compare; + struct ck_rhs_entry_desc *cursor, *pr = NULL; + unsigned long offset, probes, opl; + +#ifdef CK_RHS_PP + /* If we are storing object pointers, then we may leverage pointer packing. */ + unsigned long hv = 0; + + if (hs->mode & CK_RHS_MODE_OBJECT) { + hv = (h >> 25) & CK_RHS_KEY_MASK; + compare = CK_RHS_VMA(key); + } else { + compare = key; + } +#else + compare = key; +#endif + + *object = NULL; + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { + probes = 0; + offset = h & map->mask; + } else { + /* Restart from the bucket we were previously in */ + probes = *n_probes; + offset = ck_rhs_map_probe_next(map, *priority - map->descs, + probes); + } + opl = probe_limit; + if (behavior == CK_RHS_PROBE_INSERT) + probe_limit = ck_rhs_map_bound_get(map, h); + + + for (;;) { + cursor = &map->descs[offset]; + if (probes++ == probe_limit) { + if (probe_limit == opl || pr != NULL) { + k = CK_RHS_EMPTY; + goto leave; + } + /* + * If no eligible slot has been found yet, continue probe + * sequence with original probe limit. + */ + probe_limit = opl; + } + + k = ck_pr_load_ptr(&cursor->entry); + if (k == CK_RHS_EMPTY) + goto leave; + + if ((behavior != CK_RHS_PROBE_NO_RH) && + (map->descs[offset].in_rh == false) && + map->descs[offset].probes < + probes) { + if (pr == NULL) { + pr = &map->descs[offset]; + *n_probes = probes; + + if (behavior == CK_RHS_PROBE_RH || + behavior == CK_RHS_PROBE_ROBIN_HOOD) { + k = CK_RHS_EMPTY; + goto leave; + } + } + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) { + if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + k = CK_RHS_VMA(k); + } +#endif + + if (k == compare) + goto leave; + + if (hs->compare == NULL) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + if (hs->compare(k, key) == true) + goto leave; + } + offset = ck_rhs_map_probe_next(map, offset, probes); + } + +leave: + if (probes > probe_limit) { + cursor = NULL; + } else { + *object = k; + } + + if (pr == NULL) + *n_probes = probes; + + *priority = pr; + return cursor; +} + +static inline void * +ck_rhs_marshal(unsigned int mode, const void *key, unsigned long h) +{ + void *insert; + +#ifdef CK_RHS_PP + if (mode & CK_RHS_MODE_OBJECT) { + insert = (void *)((uintptr_t)CK_RHS_VMA(key) | ((h >> 25) << CK_MD_VMA_BITS)); + } else { + insert = (void *)key; + } +#else + (void)mode; + (void)h; + insert = (void *)key; +#endif + + return insert; +} + +bool +ck_rhs_gc(struct ck_rhs *hs) +{ + unsigned long i; + struct ck_rhs_map *map = hs->map; + + unsigned int max_probes = 0; + for (i = 0; i < map->capacity; i++) { + if (map->descs[i].probes > max_probes) + max_probes = map->descs[i].probes; + } + map->probe_maximum = max_probes; + return true; +} + +static void +ck_rhs_add_wanted(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot, struct ck_rhs_entry_desc *old_slot, + unsigned long h) +{ + struct ck_rhs_map *map = hs->map; + long offset, end_offset; + unsigned int probes = 1; + bool found_slot = false; + + offset = h & map->mask; + end_offset = slot - map->descs; + + if (old_slot == NULL) + found_slot = true; + while (offset != end_offset) { + if (offset == old_slot - map->descs) + found_slot = true; + if (found_slot && map->descs[offset].wanted < CK_RHS_MAX_WANTED) + map->descs[offset].wanted++; + offset = ck_rhs_map_probe_next(map, offset, probes); + probes++; + } +} + +static unsigned long +ck_rhs_remove_wanted(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot, long limit) +{ + struct ck_rhs_map *map = hs->map; + int probes = slot->probes; + long offset = slot - map->descs; + bool do_remove = true; + + while (probes > 1) { + probes--; + offset = ck_rhs_map_probe_prev(map, offset, probes); + if (offset == limit) + do_remove = false; + if (map->descs[offset].wanted != CK_RHS_MAX_WANTED && do_remove) + map->descs[offset].wanted--; + + } + return offset; +} + +static long +ck_rhs_get_first_offset(struct ck_rhs_map *map, unsigned long offset, unsigned int probes) +{ + while (probes > 1) { + probes--; + offset = ck_rhs_map_probe_prev(map, offset, probes); + } + return (offset); +} + +#define CK_RHS_MAX_RH 512 + +static int +ck_rhs_put_robin_hood(struct ck_rhs *hs, + struct ck_rhs_entry_desc *orig_slot) +{ + struct ck_rhs_entry_desc *slot, *first; + void *object, *insert; + unsigned long orig_probes, n_probes; + struct ck_rhs_map *map; + unsigned long h = 0; + long prev; + void *key; + long prevs[512]; + unsigned int prevs_nb = 0; + + map = hs->map; + first = orig_slot; + n_probes = orig_slot->probes; +restart: + orig_probes = n_probes; + key = first->entry; + insert = key; +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + key = CK_RHS_VMA(key); +#endif + orig_slot = first; + orig_slot->in_rh = true; + + slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + map->probe_limit, prevs_nb == CK_RHS_MAX_RH ? + CK_RHS_PROBE_NO_RH : CK_RHS_PROBE_ROBIN_HOOD); + + if (slot == NULL && first == NULL) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) { + orig_slot->in_rh = false; + for (unsigned int i = 0; i < prevs_nb; i++) { + orig_slot = &map->descs[prevs[i]]; + orig_slot->in_rh = false; + } + return -1; + } + return 1; + } + + if (first != NULL) { + int old_probes = first->probes; + + first->probes = n_probes; + h = ck_rhs_get_first_offset(map, first - map->descs, n_probes); + ck_rhs_map_bound_set(map, h, n_probes); + prev = orig_slot - map->descs; + prevs[prevs_nb++] = prev; + n_probes = old_probes; + goto restart; + } else { + /* An empty slot was found. */ + h = ck_rhs_get_first_offset(map, slot - map->descs, n_probes); + ck_rhs_map_bound_set(map, h, n_probes); + ck_pr_store_ptr(&slot->entry, insert); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + slot->probes = n_probes; + orig_slot->in_rh = false; + ck_rhs_add_wanted(hs, slot, orig_slot, h); + } + while (prevs_nb > 0) { + prev = prevs[--prevs_nb]; + n_probes = orig_slot->probes; + ck_pr_store_ptr(&orig_slot->entry, map->descs[prev].entry); + h = ck_rhs_get_first_offset(map, orig_slot - map->descs, + orig_slot->probes); + ck_rhs_add_wanted(hs, orig_slot, &map->descs[prev], h); + + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + + ck_pr_fence_atomic_store(); + orig_slot = &map->descs[prev]; + + orig_slot->in_rh = false; + } + return 0; +} + +static void +ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot) +{ + struct ck_rhs_map *map = hs->map; + unsigned long h; + + h = ck_rhs_remove_wanted(hs, slot, -1); + + while (slot->wanted > 0) { + unsigned long offset = 0; + unsigned long wanted_probes = 1; + unsigned int probe = 0; + + + /* Find a successor */ + while (wanted_probes < map->probe_maximum) { + probe = wanted_probes; + offset = ck_rhs_map_probe_next(map, slot - map->descs, probe); + while (probe < map->probe_maximum) { + if (map->descs[offset].probes == probe + 1) + break; + probe++; + offset = ck_rhs_map_probe_next(map, offset, probe); + } + if (probe < map->probe_maximum) + break; + wanted_probes++; + } + if (wanted_probes == map->probe_maximum) { + slot->wanted = 0; + break; + } + + if (slot->wanted < CK_RHS_MAX_WANTED) + slot->wanted--; + slot->probes = wanted_probes; + + h = ck_rhs_remove_wanted(hs, &map->descs[offset], slot - map->descs); + //HASH_FOR_SLOT(slot) = map->rh[offset].hash; + ck_pr_store_ptr(&slot->entry, map->descs[offset].entry); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + slot = &map->descs[offset]; + } + ck_pr_store_ptr(&slot->entry, CK_RHS_EMPTY); + if ((slot->probes - 1) < CK_RHS_WORD_MAX) + CK_RHS_STORE(&map->descs[h].probe_bound, slot->probes - 1); + + slot->probes = 0; +} + +bool +ck_rhs_fas(struct ck_rhs *hs, + unsigned long h, + const void *key, + void **previous) +{ + struct ck_rhs_entry_desc *slot, *first; + void *object, *insert; + unsigned long n_probes; + struct ck_rhs_map *map = hs->map; + + *previous = NULL; +restart: + slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + ck_rhs_map_bound_get(map, h), CK_RHS_PROBE); + + /* Replacement semantics presume existence. */ + if (object == NULL) + return false; + + insert = ck_rhs_marshal(hs->mode, key, h); + + if (first != NULL) { + int ret; + + slot->in_rh = true; + ret = ck_rhs_put_robin_hood(hs, first); + slot->in_rh = false; + if (CK_CC_UNLIKELY(ret == 1)) + goto restart; + else if (CK_CC_UNLIKELY(ret != 0)) + return false; + + ck_pr_store_ptr(&first->entry, insert); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + first->probes = n_probes; + ck_rhs_add_wanted(hs, first, NULL, h); + ck_rhs_do_backward_shift_delete(hs, slot); + } else { + ck_pr_store_ptr(&slot->entry, insert); + slot->probes = n_probes; + } + *previous = object; + return true; +} + +bool +ck_rhs_set(struct ck_rhs *hs, + unsigned long h, + const void *key, + void **previous) +{ + struct ck_rhs_entry_desc *slot, *first; + void *object, *insert; + unsigned long n_probes; + struct ck_rhs_map *map; + + *previous = NULL; + +restart: + map = hs->map; + + slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT); + if (slot == NULL && first == NULL) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + + ck_rhs_map_bound_set(map, h, n_probes); + insert = ck_rhs_marshal(hs->mode, key, h); + + if (first != NULL) { + if (slot) + slot->in_rh = true; + int ret = ck_rhs_put_robin_hood(hs, first); + if (slot) + slot->in_rh = false; + + if (CK_CC_LIKELY(ret == 1)) + goto restart; + if (CK_CC_LIKELY(ret == -1)) + return false; + /* If an earlier bucket was found, then store entry there. */ + ck_pr_store_ptr(&first->entry, insert); + first->probes = n_probes; + /* + * If a duplicate key was found, then delete it after + * signaling concurrent probes to restart. Optionally, + * it is possible to install tombstone after grace + * period if we can guarantee earlier position of + * duplicate key. + */ + ck_rhs_add_wanted(hs, first, NULL, h); + if (object != NULL) { + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + ck_rhs_do_backward_shift_delete(hs, slot); + } + + } else { + /* + * If we are storing into same slot, then atomic store is sufficient + * for replacement. + */ + ck_pr_store_ptr(&slot->entry, insert); + slot->probes = n_probes; + if (object == NULL) + ck_rhs_add_wanted(hs, slot, NULL, h); + } + + if (object == NULL) { + map->n_entries++; + if ((map->n_entries << 1) > map->capacity) + ck_rhs_grow(hs, map->capacity << 1); + } + + *previous = object; + return true; +} + +CK_CC_INLINE static bool +ck_rhs_put_internal(struct ck_rhs *hs, + unsigned long h, + const void *key, + enum ck_rhs_probe_behavior behavior) +{ + struct ck_rhs_entry_desc *slot, *first; + void *object, *insert; + unsigned long n_probes; + struct ck_rhs_map *map; + struct ck_rhs_entry_desc *desc; + +restart: + map = hs->map; + + slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + map->probe_limit, behavior); + + if (slot == NULL && first == NULL) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + + /* Fail operation if a match was found. */ + if (object != NULL) + return false; + + ck_rhs_map_bound_set(map, h, n_probes); + insert = ck_rhs_marshal(hs->mode, key, h); + + if (first != NULL) { + int ret = ck_rhs_put_robin_hood(hs, first); + + if (CK_CC_UNLIKELY(ret == 1)) + return ck_rhs_put_internal(hs, h, key, behavior); + else if (CK_CC_UNLIKELY(ret == -1)) + return false; + /* Insert key into first bucket in probe sequence. */ + ck_pr_store_ptr(&first->entry, insert); + desc = first; + ck_rhs_add_wanted(hs, first, NULL, h); + + } else { + /* An empty slot was found. */ + ck_pr_store_ptr(&slot->entry, insert); + desc = slot; + ck_rhs_add_wanted(hs, slot, NULL, h); + } + desc->probes = n_probes; + + map->n_entries++; + if ((map->n_entries << 1) > map->capacity) + ck_rhs_grow(hs, map->capacity << 1); + + return true; +} + +bool +ck_rhs_put(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + + return ck_rhs_put_internal(hs, h, key, CK_RHS_PROBE_INSERT); +} + +bool +ck_rhs_put_unique(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + + return ck_rhs_put_internal(hs, h, key, CK_RHS_PROBE_RH); +} + +void * +ck_rhs_get(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + struct ck_rhs_entry_desc *first; + void *object; + struct ck_rhs_map *map; + unsigned long n_probes; + unsigned int g, g_p, probe; + unsigned int *generation; + + do { + map = ck_pr_load_ptr(&hs->map); + generation = &map->generation[h & CK_RHS_G_MASK]; + g = ck_pr_load_uint(generation); + probe = ck_rhs_map_bound_get(map, h); + ck_pr_fence_load(); + + first = NULL; + ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, probe, CK_RHS_PROBE_NO_RH); + + ck_pr_fence_load(); + g_p = ck_pr_load_uint(generation); + } while (g != g_p); + + return object; +} + +void * +ck_rhs_remove(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + struct ck_rhs_entry_desc *slot, *first; + void *object; + struct ck_rhs_map *map = hs->map; + unsigned long n_probes; + + slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + ck_rhs_map_bound_get(map, h), CK_RHS_PROBE_NO_RH); + if (object == NULL) + return NULL; + + map->n_entries--; + ck_rhs_do_backward_shift_delete(hs, slot); + return object; +} + +bool +ck_rhs_move(struct ck_rhs *hs, + struct ck_rhs *source, + ck_rhs_hash_cb_t *hf, + ck_rhs_compare_cb_t *compare, + struct ck_malloc *m) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL) + return false; + + hs->mode = source->mode; + hs->seed = source->seed; + hs->map = source->map; + hs->m = m; + hs->hf = hf; + hs->compare = compare; + return true; +} + +bool +ck_rhs_init(struct ck_rhs *hs, + unsigned int mode, + ck_rhs_hash_cb_t *hf, + ck_rhs_compare_cb_t *compare, + struct ck_malloc *m, + unsigned long n_entries, + unsigned long seed) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL) + return false; + + hs->m = m; + hs->mode = mode; + hs->seed = seed; + hs->hf = hf; + hs->compare = compare; + + hs->map = ck_rhs_map_create(hs, n_entries); + return hs->map != NULL; +} + From e47caab029da9c6f03be1e44aeba653d1fb4e8eb Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Fri, 3 Jan 2014 14:11:09 +0100 Subject: [PATCH 02/11] ck_rhs: Increase CK_RHS_G from 2 to 1024, since it's used a lot more. --- src/ck_rhs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index 0b33597..903ba66 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -52,7 +52,7 @@ ((void *)((uintptr_t)(x) & CK_RHS_VMA_MASK)) #define CK_RHS_EMPTY NULL -#define CK_RHS_G (2) +#define CK_RHS_G (1024) #define CK_RHS_G_MASK (CK_RHS_G - 1) #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8) From 0f908f1e31cb415f8ad2a05a9ecf9685e121b723 Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Sat, 4 Jan 2014 18:48:49 +0100 Subject: [PATCH 03/11] ck_hrs: Update probe_bound for every slot shifted. In ck_rhs_do_backward_shift_delete(), find if any entry with the same hash is stored further, and if not, update probe_bound for every entry being shifted, instead of just doing it for the slot being emptied. --- src/ck_rhs.c | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index 903ba66..ab6e2ba 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -589,11 +589,14 @@ ck_rhs_remove_wanted(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot, long lim static long ck_rhs_get_first_offset(struct ck_rhs_map *map, unsigned long offset, unsigned int probes) { - while (probes > 1) { - probes--; - offset = ck_rhs_map_probe_prev(map, offset, probes); + while (probes > 4) { + offset -= ((probes - 1) &~ map->offset_mask); + offset &= map->mask; + offset = (offset &~ map->offset_mask) + + ((offset - map->offset_mask) & map->offset_mask); + probes -= map->offset_mask + 1; } - return (offset); + return ((offset &~ map->offset_mask) + ((offset - (probes - 1)) & map->offset_mask)); } #define CK_RHS_MAX_RH 512 @@ -670,9 +673,7 @@ restart: h = ck_rhs_get_first_offset(map, orig_slot - map->descs, orig_slot->probes); ck_rhs_add_wanted(hs, orig_slot, &map->descs[prev], h); - ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); - ck_pr_fence_atomic_store(); orig_slot = &map->descs[prev]; @@ -690,9 +691,10 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, struct ck_rhs_entry_desc *slo h = ck_rhs_remove_wanted(hs, slot, -1); while (slot->wanted > 0) { - unsigned long offset = 0; + unsigned long offset = 0, tmp_offset; unsigned long wanted_probes = 1; unsigned int probe = 0; + unsigned int max_probes; /* Find a successor */ @@ -714,15 +716,41 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, struct ck_rhs_entry_desc *slo break; } - if (slot->wanted < CK_RHS_MAX_WANTED) - slot->wanted--; slot->probes = wanted_probes; h = ck_rhs_remove_wanted(hs, &map->descs[offset], slot - map->descs); - //HASH_FOR_SLOT(slot) = map->rh[offset].hash; ck_pr_store_ptr(&slot->entry, map->descs[offset].entry); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); + if (wanted_probes < CK_RHS_WORD_MAX) { + if (map->descs[h].wanted == 1) + CK_RHS_STORE(&map->descs[h].probe_bound, + wanted_probes); + else if (map->descs[h].probe_bound == CK_RHS_WORD_MAX || + map->descs[h].probe_bound == map->descs[offset].probes) { + probe++; + if (map->descs[h].probe_bound == CK_RHS_WORD_MAX) + max_probes = map->probe_maximum; + else { + max_probes = map->descs[h].probe_bound; + max_probes--; + } + tmp_offset = ck_rhs_map_probe_next(map, offset, probe); + + while (probe < max_probes) { + if (h == (unsigned long)ck_rhs_get_first_offset(map, tmp_offset, probe)) + break; + probe++; + tmp_offset = ck_rhs_map_probe_next(map, tmp_offset, probe); + } + if (probe >= max_probes) + CK_RHS_STORE(&map->descs[h].probe_bound, + wanted_probes); + } + } + if (slot->wanted < CK_RHS_MAX_WANTED) + slot->wanted--; + slot = &map->descs[offset]; } ck_pr_store_ptr(&slot->entry, CK_RHS_EMPTY); From 59cedf10c6eb691c6d06289da1d65b1061134c26 Mon Sep 17 00:00:00 2001 From: User Doginou Date: Mon, 6 Jan 2014 16:27:21 +0100 Subject: [PATCH 04/11] ck_rhs: Add a read mostly mode. Add a read-mostly mode, in which entries won't share cache lines with associated datas (probes, probe_bound, etc). This makes write operations slower, but make get faster. --- include/ck_rhs.h | 6 + regressions/ck_rhs/benchmark/serial.c | 1 + src/ck_rhs.c | 582 ++++++++++++++++++-------- 3 files changed, 424 insertions(+), 165 deletions(-) diff --git a/include/ck_rhs.h b/include/ck_rhs.h index 67ac0ec..8c8824d 100644 --- a/include/ck_rhs.h +++ b/include/ck_rhs.h @@ -55,6 +55,12 @@ */ #define CK_RHS_MODE_OBJECT 8 +/* + * Indicated that the load is read-mostly, so get should be optimized + * over put and delete + */ +#define CK_RHS_MODE_READ_MOSTLY 16 + /* Currently unsupported. */ #define CK_RHS_MODE_MPMC (void) diff --git a/regressions/ck_rhs/benchmark/serial.c b/regressions/ck_rhs/benchmark/serial.c index 94c7fde..18fa892 100644 --- a/regressions/ck_rhs/benchmark/serial.c +++ b/regressions/ck_rhs/benchmark/serial.c @@ -507,6 +507,7 @@ main(int argc, char *argv[]) global_seed = common_lrand48(); run_test(argv[1], r, size, 0); + run_test(argv[1], r, size, CK_RHS_MODE_READ_MOSTLY); fprintf(stderr, "# reverse_insertion serial_insertion random_insertion serial_swap " "serial_replace reverse_get serial_get random_get serial_remove negative_get tombstone " "set_unique gc rebuild\n\n"); diff --git a/src/ck_rhs.c b/src/ck_rhs.c index ab6e2ba..377b843 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -84,15 +84,31 @@ enum ck_rhs_probe_behavior { CK_RHS_PROBE_ROBIN_HOOD, /* Look for the first slot available for the entry we are about to replace, only used to internally implement Robin Hood */ CK_RHS_PROBE_NO_RH, /* Don't do the RH dance */ }; - struct ck_rhs_entry_desc { - void *entry; unsigned int probes; unsigned short wanted; CK_RHS_WORD probe_bound; bool in_rh; + void *entry; } __attribute__ ((__aligned__(16))); +struct ck_rhs_no_entry_desc { + unsigned int probes; + unsigned short wanted; + CK_RHS_WORD probe_bound; + bool in_rh; +} __attribute__ ((__aligned__(8))); + +typedef long (*probe_func) (struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior); + struct ck_rhs_map { unsigned int generation[CK_RHS_G]; unsigned int probe_maximum; @@ -103,9 +119,88 @@ struct ck_rhs_map { unsigned long capacity; unsigned long size; char offset_mask; - struct ck_rhs_entry_desc *descs; + union { + struct ck_rhs_entry_desc *descs; + struct ck_rhs_no_entry { + void **entries; + struct ck_rhs_no_entry_desc *descs; + } no_entries; + } entries; + bool read_mostly; + probe_func probe_func; }; +#define CK_RHS_ENTRY(map, offset) (map->read_mostly ? map->entries.no_entries.entries[offset] : map->entries.descs[offset].entry) +#define CK_RHS_ENTRY_ADDR(map, offset) (map->read_mostly ? &map->entries.no_entries.entries[offset] : &map->entries.descs[offset].entry) +#define CK_RHS_DESC(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? (struct ck_rhs_entry_desc *)(void *)&map->entries.no_entries.descs[offset] : &map->entries.descs[offset]) +#define CK_RHS_WANTED_INC(map, offset) do { \ + if (CK_CC_UNLIKELY(map->read_mostly)) \ + map->entries.no_entries.descs[offset].wanted++; \ + else \ + map->entries.descs[offset].wanted++; \ +} while (0) +#define CK_RHS_WANTED(map, offset) (CK_CC_UNLIKELY(map->read_mostly )? map->entries.no_entries.descs[offset].wanted : map->entries.descs[offset].wanted) +#define CK_RHS_SET_WANTED(map, offset, value) do { \ + if (CK_CC_UNLIKELY(map->read_mostly)) \ + map->entries.no_entries.descs[offset].wanted = value; \ + else \ + map->entries.descs[offset].probes = value; \ +} while (0) + +#define CK_RHS_WANTED_DEC(map, offset) do { \ + if (CK_CC_UNLIKELY(map->read_mostly)) \ + map->entries.no_entries.descs[offset].wanted--; \ + else \ + map->entries.descs[offset].wanted--; \ +} while (0) + +#define CK_RHS_PROBES(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? map->entries.no_entries.descs[offset].probes : map->entries.descs[offset].probes) +#define CK_RHS_SET_PROBES(map, offset, value) do { \ + if (CK_CC_UNLIKELY(map->read_mostly)) \ + map->entries.no_entries.descs[offset].probes = value; \ + else \ + map->entries.descs[offset].probes = value; \ +} while (0) +#define CK_RHS_PROBE_BOUND(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? map->entries.no_entries.descs[offset].probe_bound : map->entries.descs[offset].probe_bound) +#define CK_RHS_PROBE_BOUND_ADDR(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? (&map->entries.no_entries.descs[offset].probe_bound) : (&map->entries.descs[offset].probe_bound)) +#define CK_RHS_IN_RH(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? map->entries.no_entries.descs[offset].in_rh : map->entries.descs[offset].in_rh) +#define CK_RHS_SET_RH(map, offset) do { \ + if (CK_CC_UNLIKELY(map->read_mostly)) \ + map->entries.no_entries.descs[offset].in_rh = true; \ + else \ + map->entries.descs[offset].in_rh = true; \ +} while (0) +#define CK_RHS_UNSET_RH(map, offset) do { \ + if (CK_CC_UNLIKELY(map->read_mostly)) \ + map->entries.no_entries.descs[offset].in_rh = false; \ + else \ + map->entries.descs[offset].in_rh = false; \ +} while (0) + +#define CK_RHS_LOAD_FACTOR 50 + +static long +ck_rhs_map_probe(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior); + +static long +ck_rhs_map_probe_rm(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior); + void ck_rhs_iterator_init(struct ck_rhs_iterator *iterator) { @@ -125,7 +220,7 @@ ck_rhs_next(struct ck_rhs *hs, struct ck_rhs_iterator *i, void **key) return false; do { - value = map->descs[i->offset].entry; + value = CK_RHS_ENTRY(map, i->offset); if (value != CK_RHS_EMPTY) { #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) @@ -183,15 +278,18 @@ ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries) if (n_entries < CK_RHS_PROBE_L1) return NULL; - size = sizeof(struct ck_rhs_map) + - (sizeof(struct ck_rhs_entry_desc) * n_entries + CK_MD_CACHELINE - 1); - + if (hs->mode & CK_RHS_MODE_READ_MOSTLY) + size = sizeof(struct ck_rhs_map) + + (sizeof(void *) * n_entries + sizeof(struct ck_rhs_no_entry_desc) * n_entries + 2 * CK_MD_CACHELINE - 1); + else + size = sizeof(struct ck_rhs_map) + + (sizeof(struct ck_rhs_entry_desc) * n_entries + CK_MD_CACHELINE - 1); map = hs->m->malloc(size); if (map == NULL) return NULL; + map->read_mostly = !!(hs->mode & CK_RHS_MODE_READ_MOSTLY); map->size = size; - map->offset_mask = (CK_MD_CACHELINE / sizeof(struct ck_rhs_entry_desc)) - 1; /* We should probably use a more intelligent heuristic for default probe length. */ limit = ck_internal_max(n_entries >> (CK_RHS_PROBE_L1_SHIFT + 2), CK_RHS_PROBE_L1_DEFAULT); if (limit > UINT_MAX) @@ -205,10 +303,25 @@ ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries) map->n_entries = 0; /* Align map allocation to cache line. */ - map->descs = (void *)(((uintptr_t)&map[1] + - CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + if (map->read_mostly) { + map->entries.no_entries.entries = (void *)(((uintptr_t)&map[1] + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + map->entries.no_entries.descs = (void *)(((uintptr_t)map->entries.no_entries.entries + (sizeof(void *) * n_entries) + CK_MD_CACHELINE - 1) &~ (CK_MD_CACHELINE - 1)); + memset(map->entries.no_entries.entries, 0, + sizeof(void *) * n_entries); + memset(map->entries.no_entries.descs, 0, + sizeof(struct ck_rhs_no_entry_desc)); + map->offset_mask = (CK_MD_CACHELINE / sizeof(void *)) - 1; + map->probe_func = ck_rhs_map_probe_rm; + + } else { + map->entries.descs = (void *)(((uintptr_t)&map[1] + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + memset(map->entries.descs, 0, sizeof(struct ck_rhs_entry_desc) * n_entries); + map->offset_mask = (CK_MD_CACHELINE / sizeof(struct ck_rhs_entry_desc)) - 1; + map->probe_func = ck_rhs_map_probe; + } memset(map->generation, 0, sizeof map->generation); - memset(map->descs, 0, sizeof(*map->descs) * n_entries); /* Commit entries purge with respect to map publication. */ ck_pr_fence_store(); @@ -270,16 +383,20 @@ ck_rhs_map_bound_set(struct ck_rhs_map *m, unsigned long n_probes) { unsigned long offset = h & m->mask; + struct ck_rhs_entry_desc *desc; if (n_probes > m->probe_maximum) ck_pr_store_uint(&m->probe_maximum, n_probes); + if (!(m->read_mostly)) { + desc = &m->entries.descs[offset]; - if (m->descs[offset].probe_bound < n_probes) { - if (n_probes > CK_RHS_WORD_MAX) - n_probes = CK_RHS_WORD_MAX; + if (desc->probe_bound < n_probes) { + if (n_probes > CK_RHS_WORD_MAX) + n_probes = CK_RHS_WORD_MAX; - CK_RHS_STORE(&m->descs[offset].probe_bound, n_probes); - ck_pr_fence_store(); + CK_RHS_STORE(&desc->probe_bound, n_probes); + ck_pr_fence_store(); + } } return; @@ -291,9 +408,13 @@ ck_rhs_map_bound_get(struct ck_rhs_map *m, unsigned long h) unsigned long offset = h & m->mask; unsigned int r = CK_RHS_WORD_MAX; - r = CK_RHS_LOAD(&m->descs[offset].probe_bound); - if (r == CK_RHS_WORD_MAX) + if (m->read_mostly) r = ck_pr_load_uint(&m->probe_maximum); + else { + r = CK_RHS_LOAD(&m->entries.descs[offset].probe_bound); + if (r == CK_RHS_WORD_MAX) + r = ck_pr_load_uint(&m->probe_maximum); + } return r; } @@ -317,7 +438,7 @@ restart: for (k = 0; k < map->capacity; k++) { unsigned long h; - prev_saved = previous = map->descs[k].entry; + prev_saved = previous = CK_RHS_ENTRY(map, k); if (previous == CK_RHS_EMPTY) continue; @@ -331,7 +452,7 @@ restart: probes = 0; for (;;) { - void **cursor = &update->descs[offset].entry; + void **cursor = CK_RHS_ENTRY_ADDR(update, offset); if (probes++ == update->probe_limit) { /* @@ -345,10 +466,10 @@ restart: if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) { *cursor = prev_saved; update->n_entries++; - update->descs[offset].probes = probes; + CK_RHS_SET_PROBES(update, offset, probes); ck_rhs_map_bound_set(update, h, probes); break; - } else if (update->descs[offset].probes < probes) { + } else if (CK_RHS_PROBES(update, offset) < probes) { void *tmp = prev_saved; unsigned int old_probes; prev_saved = previous = *cursor; @@ -359,12 +480,12 @@ restart: *cursor = tmp; ck_rhs_map_bound_set(update, h, probes); h = hs->hf(previous, hs->seed); - old_probes = update->descs[offset].probes; - update->descs[offset].probes = probes; + old_probes = CK_RHS_PROBES(update, offset); + CK_RHS_SET_PROBES(update, offset, probes); probes = old_probes - 1; continue; - } - update->descs[offset].wanted++; + } + CK_RHS_WANTED_INC(update, offset); offset = ck_rhs_map_probe_next(update, offset, probes); } @@ -384,11 +505,129 @@ ck_rhs_rebuild(struct ck_rhs *hs) return ck_rhs_grow(hs, hs->map->capacity); } -static struct ck_rhs_entry_desc * +static long +ck_rhs_map_probe_rm(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior) +{ + + void *k; + const void *compare; + long pr = -1; + unsigned long offset, probes, opl; + +#ifdef CK_RHS_PP + /* If we are storing object pointers, then we may leverage pointer packing. */ + unsigned long hv = 0; + + if (hs->mode & CK_RHS_MODE_OBJECT) { + hv = (h >> 25) & CK_RHS_KEY_MASK; + compare = CK_RHS_VMA(key); + } else { + compare = key; + } +#else + compare = key; +#endif + + *object = NULL; + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { + probes = 0; + offset = h & map->mask; + } else { + /* Restart from the bucket we were previously in */ + probes = *n_probes; + offset = ck_rhs_map_probe_next(map, *priority, + probes); + } + opl = probe_limit; + + + for (;;) { + if (probes++ == probe_limit) { + if (probe_limit == opl || pr != -1) { + k = CK_RHS_EMPTY; + goto leave; + } + /* + * If no eligible slot has been found yet, continue probe + * sequence with original probe limit. + */ + probe_limit = opl; + } + + k = ck_pr_load_ptr(&map->entries.no_entries.entries[offset]); + if (k == CK_RHS_EMPTY) + goto leave; + + if ((behavior != CK_RHS_PROBE_NO_RH)) { + struct ck_rhs_entry_desc *desc = (void *)&map->entries.no_entries.descs[offset]; + + if (pr == -1 && + desc->in_rh == false && desc->probes < probes) { + pr = offset; + *n_probes = probes; + + if (behavior == CK_RHS_PROBE_RH || + behavior == CK_RHS_PROBE_ROBIN_HOOD) { + k = CK_RHS_EMPTY; + goto leave; + } + } + } + + + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) { + if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + k = CK_RHS_VMA(k); + } +#endif + + if (k == compare) + goto leave; + + if (hs->compare == NULL) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + if (hs->compare(k, key) == true) + goto leave; + } + offset = ck_rhs_map_probe_next(map, offset, probes); + } + +leave: + if (probes > probe_limit) { + offset = -1; + } else { + *object = k; + } + + if (pr == -1) + *n_probes = probes; + + *priority = pr; + return offset; +} + +static long ck_rhs_map_probe(struct ck_rhs *hs, struct ck_rhs_map *map, unsigned long *n_probes, - struct ck_rhs_entry_desc **priority, + long *priority, unsigned long h, const void *key, void **object, @@ -398,7 +637,7 @@ ck_rhs_map_probe(struct ck_rhs *hs, void *k; const void *compare; - struct ck_rhs_entry_desc *cursor, *pr = NULL; + long pr = -1; unsigned long offset, probes, opl; #ifdef CK_RHS_PP @@ -422,7 +661,7 @@ ck_rhs_map_probe(struct ck_rhs *hs, } else { /* Restart from the bucket we were previously in */ probes = *n_probes; - offset = ck_rhs_map_probe_next(map, *priority - map->descs, + offset = ck_rhs_map_probe_next(map, *priority, probes); } opl = probe_limit; @@ -431,9 +670,8 @@ ck_rhs_map_probe(struct ck_rhs *hs, for (;;) { - cursor = &map->descs[offset]; if (probes++ == probe_limit) { - if (probe_limit == opl || pr != NULL) { + if (probe_limit == opl || pr != -1) { k = CK_RHS_EMPTY; goto leave; } @@ -444,26 +682,24 @@ ck_rhs_map_probe(struct ck_rhs *hs, probe_limit = opl; } - k = ck_pr_load_ptr(&cursor->entry); + k = ck_pr_load_ptr(&map->entries.descs[offset].entry); if (k == CK_RHS_EMPTY) goto leave; - if ((behavior != CK_RHS_PROBE_NO_RH) && - (map->descs[offset].in_rh == false) && - map->descs[offset].probes < - probes) { - if (pr == NULL) { - pr = &map->descs[offset]; + if ((behavior != CK_RHS_PROBE_NO_RH)) { + struct ck_rhs_entry_desc *desc = &map->entries.descs[offset];; + + if (pr == -1 && + desc->in_rh == false && desc->probes < probes) { + pr = offset; *n_probes = probes; if (behavior == CK_RHS_PROBE_RH || - behavior == CK_RHS_PROBE_ROBIN_HOOD) { + behavior == CK_RHS_PROBE_ROBIN_HOOD) { k = CK_RHS_EMPTY; goto leave; } } - offset = ck_rhs_map_probe_next(map, offset, probes); - continue; } @@ -495,16 +731,16 @@ ck_rhs_map_probe(struct ck_rhs *hs, leave: if (probes > probe_limit) { - cursor = NULL; + offset = -1; } else { *object = k; } - if (pr == NULL) + if (pr == -1) *n_probes = probes; *priority = pr; - return cursor; + return offset; } static inline void * @@ -535,53 +771,58 @@ ck_rhs_gc(struct ck_rhs *hs) unsigned int max_probes = 0; for (i = 0; i < map->capacity; i++) { - if (map->descs[i].probes > max_probes) - max_probes = map->descs[i].probes; + if (CK_RHS_PROBES(map, i) > max_probes) + max_probes = CK_RHS_PROBES(map, i); } map->probe_maximum = max_probes; return true; } static void -ck_rhs_add_wanted(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot, struct ck_rhs_entry_desc *old_slot, +ck_rhs_add_wanted(struct ck_rhs *hs, long end_offset, long old_slot, unsigned long h) { struct ck_rhs_map *map = hs->map; - long offset, end_offset; + long offset; unsigned int probes = 1; bool found_slot = false; + struct ck_rhs_entry_desc *desc; offset = h & map->mask; - end_offset = slot - map->descs; - if (old_slot == NULL) + if (old_slot == -1) found_slot = true; while (offset != end_offset) { - if (offset == old_slot - map->descs) + if (offset == old_slot) found_slot = true; - if (found_slot && map->descs[offset].wanted < CK_RHS_MAX_WANTED) - map->descs[offset].wanted++; + if (found_slot) { + desc = CK_RHS_DESC(map, offset); + if (desc->wanted < CK_RHS_MAX_WANTED) + desc->wanted++; + } offset = ck_rhs_map_probe_next(map, offset, probes); probes++; } } static unsigned long -ck_rhs_remove_wanted(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot, long limit) +ck_rhs_remove_wanted(struct ck_rhs *hs, long offset, long limit) { struct ck_rhs_map *map = hs->map; - int probes = slot->probes; - long offset = slot - map->descs; + int probes = CK_RHS_PROBES(map, offset); bool do_remove = true; + struct ck_rhs_entry_desc *desc; while (probes > 1) { probes--; offset = ck_rhs_map_probe_prev(map, offset, probes); if (offset == limit) do_remove = false; - if (map->descs[offset].wanted != CK_RHS_MAX_WANTED && do_remove) - map->descs[offset].wanted--; - + if (do_remove) { + desc = CK_RHS_DESC(map, offset); + if (desc->wanted != CK_RHS_MAX_WANTED) + desc->wanted--; + } } return offset; } @@ -589,7 +830,7 @@ ck_rhs_remove_wanted(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot, long lim static long ck_rhs_get_first_offset(struct ck_rhs_map *map, unsigned long offset, unsigned int probes) { - while (probes > 4) { + while (probes > (unsigned long)map->offset_mask + 1) { offset -= ((probes - 1) &~ map->offset_mask); offset &= map->mask; offset = (offset &~ map->offset_mask) + @@ -603,11 +844,11 @@ ck_rhs_get_first_offset(struct ck_rhs_map *map, unsigned long offset, unsigned i static int ck_rhs_put_robin_hood(struct ck_rhs *hs, - struct ck_rhs_entry_desc *orig_slot) + long orig_slot, struct ck_rhs_entry_desc *desc) { - struct ck_rhs_entry_desc *slot, *first; + long slot, first; void *object, *insert; - unsigned long orig_probes, n_probes; + unsigned long n_probes; struct ck_rhs_map *map; unsigned long h = 0; long prev; @@ -617,80 +858,83 @@ ck_rhs_put_robin_hood(struct ck_rhs *hs, map = hs->map; first = orig_slot; - n_probes = orig_slot->probes; + n_probes = desc->probes; restart: - orig_probes = n_probes; - key = first->entry; + key = CK_RHS_ENTRY(map, first); insert = key; #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) key = CK_RHS_VMA(key); #endif orig_slot = first; - orig_slot->in_rh = true; + CK_RHS_SET_RH(map, orig_slot); - slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, prevs_nb == CK_RHS_MAX_RH ? CK_RHS_PROBE_NO_RH : CK_RHS_PROBE_ROBIN_HOOD); - if (slot == NULL && first == NULL) { + if (slot == -1 && first == -1) { + printf("prout!!\n"); if (ck_rhs_grow(hs, map->capacity << 1) == false) { - orig_slot->in_rh = false; + desc->in_rh = false; for (unsigned int i = 0; i < prevs_nb; i++) { - orig_slot = &map->descs[prevs[i]]; - orig_slot->in_rh = false; + CK_RHS_UNSET_RH(map, prevs[i]); } + printf("lolol\n"); return -1; } + printf("hehe\n"); return 1; } - if (first != NULL) { - int old_probes = first->probes; + if (first != -1) { + desc = CK_RHS_DESC(map, first); + int old_probes = desc->probes; - first->probes = n_probes; - h = ck_rhs_get_first_offset(map, first - map->descs, n_probes); + desc->probes = n_probes; + h = ck_rhs_get_first_offset(map, first, n_probes); ck_rhs_map_bound_set(map, h, n_probes); - prev = orig_slot - map->descs; + prev = orig_slot; prevs[prevs_nb++] = prev; n_probes = old_probes; goto restart; } else { /* An empty slot was found. */ - h = ck_rhs_get_first_offset(map, slot - map->descs, n_probes); + h = ck_rhs_get_first_offset(map, slot, n_probes); ck_rhs_map_bound_set(map, h, n_probes); - ck_pr_store_ptr(&slot->entry, insert); + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); - slot->probes = n_probes; - orig_slot->in_rh = false; + CK_RHS_SET_PROBES(map, slot, n_probes); + desc->in_rh = 0; ck_rhs_add_wanted(hs, slot, orig_slot, h); } while (prevs_nb > 0) { prev = prevs[--prevs_nb]; - n_probes = orig_slot->probes; - ck_pr_store_ptr(&orig_slot->entry, map->descs[prev].entry); - h = ck_rhs_get_first_offset(map, orig_slot - map->descs, - orig_slot->probes); - ck_rhs_add_wanted(hs, orig_slot, &map->descs[prev], h); + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, orig_slot), + CK_RHS_ENTRY(map, prev)); + h = ck_rhs_get_first_offset(map, orig_slot, + desc->probes); + ck_rhs_add_wanted(hs, orig_slot, prev, h); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); - orig_slot = &map->descs[prev]; - - orig_slot->in_rh = false; + orig_slot = prev; + desc->in_rh = false; + desc = CK_RHS_DESC(map, orig_slot); } return 0; } static void -ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, struct ck_rhs_entry_desc *slot) +ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) { struct ck_rhs_map *map = hs->map; + struct ck_rhs_entry_desc *desc = CK_RHS_DESC(map, slot), *new_desc = NULL; unsigned long h; h = ck_rhs_remove_wanted(hs, slot, -1); - while (slot->wanted > 0) { + while (desc->wanted > 0) { unsigned long offset = 0, tmp_offset; unsigned long wanted_probes = 1; unsigned int probe = 0; @@ -700,39 +944,42 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, struct ck_rhs_entry_desc *slo /* Find a successor */ while (wanted_probes < map->probe_maximum) { probe = wanted_probes; - offset = ck_rhs_map_probe_next(map, slot - map->descs, probe); + offset = ck_rhs_map_probe_next(map, slot, probe); while (probe < map->probe_maximum) { - if (map->descs[offset].probes == probe + 1) - break; + new_desc = CK_RHS_DESC(map, offset); + if (new_desc->probes == probe + 1) + break; probe++; offset = ck_rhs_map_probe_next(map, offset, probe); } if (probe < map->probe_maximum) - break; + break; wanted_probes++; } if (wanted_probes == map->probe_maximum) { - slot->wanted = 0; + desc->wanted = 0; break; } - slot->probes = wanted_probes; + desc->probes = wanted_probes; - h = ck_rhs_remove_wanted(hs, &map->descs[offset], slot - map->descs); - ck_pr_store_ptr(&slot->entry, map->descs[offset].entry); + h = ck_rhs_remove_wanted(hs, offset, slot); + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), + CK_RHS_ENTRY(map, offset)); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); if (wanted_probes < CK_RHS_WORD_MAX) { - if (map->descs[h].wanted == 1) - CK_RHS_STORE(&map->descs[h].probe_bound, + struct ck_rhs_entry_desc *hdesc = CK_RHS_DESC(map, h); + if (hdesc->wanted == 1) + CK_RHS_STORE(&hdesc->probe_bound, wanted_probes); - else if (map->descs[h].probe_bound == CK_RHS_WORD_MAX || - map->descs[h].probe_bound == map->descs[offset].probes) { + else if (hdesc->probe_bound == CK_RHS_WORD_MAX || + hdesc->probe_bound == new_desc->probes) { probe++; - if (map->descs[h].probe_bound == CK_RHS_WORD_MAX) + if (hdesc->probe_bound == CK_RHS_WORD_MAX) max_probes = map->probe_maximum; else { - max_probes = map->descs[h].probe_bound; + max_probes = hdesc->probe_bound; max_probes--; } tmp_offset = ck_rhs_map_probe_next(map, offset, probe); @@ -743,21 +990,22 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, struct ck_rhs_entry_desc *slo probe++; tmp_offset = ck_rhs_map_probe_next(map, tmp_offset, probe); } - if (probe >= max_probes) - CK_RHS_STORE(&map->descs[h].probe_bound, + if (probe == max_probes) + CK_RHS_STORE(&hdesc->probe_bound, wanted_probes); } } - if (slot->wanted < CK_RHS_MAX_WANTED) - slot->wanted--; - slot = &map->descs[offset]; + if (desc->wanted < CK_RHS_MAX_WANTED) + desc->wanted--; + slot = offset; + desc = new_desc; } - ck_pr_store_ptr(&slot->entry, CK_RHS_EMPTY); - if ((slot->probes - 1) < CK_RHS_WORD_MAX) - CK_RHS_STORE(&map->descs[h].probe_bound, slot->probes - 1); - - slot->probes = 0; + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), CK_RHS_EMPTY); + if ((desc->probes - 1) < CK_RHS_WORD_MAX) + CK_RHS_STORE(CK_RHS_PROBE_BOUND_ADDR(map, h), + desc->probes - 1); + desc->probes = 0; } bool @@ -766,14 +1014,15 @@ ck_rhs_fas(struct ck_rhs *hs, const void *key, void **previous) { - struct ck_rhs_entry_desc *slot, *first; + long slot, first; void *object, *insert; unsigned long n_probes; struct ck_rhs_map *map = hs->map; + struct ck_rhs_entry_desc *desc, *desc2; *previous = NULL; restart: - slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, ck_rhs_map_bound_get(map, h), CK_RHS_PROBE); /* Replacement semantics presume existence. */ @@ -782,26 +1031,28 @@ restart: insert = ck_rhs_marshal(hs->mode, key, h); - if (first != NULL) { + if (first != -1) { int ret; + desc = CK_RHS_DESC(map, slot); - slot->in_rh = true; - ret = ck_rhs_put_robin_hood(hs, first); - slot->in_rh = false; + desc2 = CK_RHS_DESC(map, first); + desc->in_rh = true; + ret = ck_rhs_put_robin_hood(hs, first, desc2); + desc->in_rh = false; if (CK_CC_UNLIKELY(ret == 1)) goto restart; else if (CK_CC_UNLIKELY(ret != 0)) return false; - ck_pr_store_ptr(&first->entry, insert); + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, first), insert); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); - first->probes = n_probes; - ck_rhs_add_wanted(hs, first, NULL, h); + desc2->probes = n_probes; + ck_rhs_add_wanted(hs, first, -1, h); ck_rhs_do_backward_shift_delete(hs, slot); } else { - ck_pr_store_ptr(&slot->entry, insert); - slot->probes = n_probes; + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); + CK_RHS_SET_PROBES(map, slot, n_probes); } *previous = object; return true; @@ -813,7 +1064,7 @@ ck_rhs_set(struct ck_rhs *hs, const void *key, void **previous) { - struct ck_rhs_entry_desc *slot, *first; + long slot, first; void *object, *insert; unsigned long n_probes; struct ck_rhs_map *map; @@ -823,8 +1074,8 @@ ck_rhs_set(struct ck_rhs *hs, restart: map = hs->map; - slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT); - if (slot == NULL && first == NULL) { + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT); + if (slot == -1 && first == -1) { if (ck_rhs_grow(hs, map->capacity << 1) == false) return false; @@ -834,20 +1085,24 @@ restart: ck_rhs_map_bound_set(map, h, n_probes); insert = ck_rhs_marshal(hs->mode, key, h); - if (first != NULL) { - if (slot) - slot->in_rh = true; - int ret = ck_rhs_put_robin_hood(hs, first); - if (slot) - slot->in_rh = false; + if (first != -1) { + struct ck_rhs_entry_desc *desc = NULL, *desc2; + if (slot != -1) { + desc = CK_RHS_DESC(map, slot); + desc->in_rh = true; + } + desc2 = CK_RHS_DESC(map, first); + int ret = ck_rhs_put_robin_hood(hs, first, desc2); + if (slot != -1) + desc->in_rh = false; if (CK_CC_LIKELY(ret == 1)) goto restart; if (CK_CC_LIKELY(ret == -1)) return false; /* If an earlier bucket was found, then store entry there. */ - ck_pr_store_ptr(&first->entry, insert); - first->probes = n_probes; + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, first), insert); + desc2->probes = n_probes; /* * If a duplicate key was found, then delete it after * signaling concurrent probes to restart. Optionally, @@ -855,7 +1110,7 @@ restart: * period if we can guarantee earlier position of * duplicate key. */ - ck_rhs_add_wanted(hs, first, NULL, h); + ck_rhs_add_wanted(hs, first, -1, h); if (object != NULL) { ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); @@ -867,15 +1122,15 @@ restart: * If we are storing into same slot, then atomic store is sufficient * for replacement. */ - ck_pr_store_ptr(&slot->entry, insert); - slot->probes = n_probes; + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); + CK_RHS_SET_PROBES(map, slot, n_probes); if (object == NULL) - ck_rhs_add_wanted(hs, slot, NULL, h); + ck_rhs_add_wanted(hs, slot, -1, h); } if (object == NULL) { map->n_entries++; - if ((map->n_entries << 1) > map->capacity) + if ((map->n_entries ) > ((map->capacity * CK_RHS_LOAD_FACTOR) / 100)) ck_rhs_grow(hs, map->capacity << 1); } @@ -889,19 +1144,18 @@ ck_rhs_put_internal(struct ck_rhs *hs, const void *key, enum ck_rhs_probe_behavior behavior) { - struct ck_rhs_entry_desc *slot, *first; + long slot, first; void *object, *insert; unsigned long n_probes; struct ck_rhs_map *map; - struct ck_rhs_entry_desc *desc; restart: map = hs->map; - slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, behavior); - if (slot == NULL && first == NULL) { + if (slot == -1 && first == -1) { if (ck_rhs_grow(hs, map->capacity << 1) == false) return false; @@ -915,30 +1169,28 @@ restart: ck_rhs_map_bound_set(map, h, n_probes); insert = ck_rhs_marshal(hs->mode, key, h); - if (first != NULL) { - int ret = ck_rhs_put_robin_hood(hs, first); - + if (first != -1) { + struct ck_rhs_entry_desc *desc = CK_RHS_DESC(map, first); + int ret = ck_rhs_put_robin_hood(hs, first, desc); if (CK_CC_UNLIKELY(ret == 1)) return ck_rhs_put_internal(hs, h, key, behavior); else if (CK_CC_UNLIKELY(ret == -1)) return false; /* Insert key into first bucket in probe sequence. */ - ck_pr_store_ptr(&first->entry, insert); - desc = first; - ck_rhs_add_wanted(hs, first, NULL, h); - + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, first), insert); + desc->probes = n_probes; + ck_rhs_add_wanted(hs, first, -1, h); } else { /* An empty slot was found. */ - ck_pr_store_ptr(&slot->entry, insert); - desc = slot; - ck_rhs_add_wanted(hs, slot, NULL, h); + ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); + CK_RHS_SET_PROBES(map, slot, n_probes); + ck_rhs_add_wanted(hs, slot, -1, h); } - desc->probes = n_probes; map->n_entries++; - if ((map->n_entries << 1) > map->capacity) + + if ((map->n_entries ) > ((map->capacity * CK_RHS_LOAD_FACTOR) / 100)) ck_rhs_grow(hs, map->capacity << 1); - return true; } @@ -965,7 +1217,7 @@ ck_rhs_get(struct ck_rhs *hs, unsigned long h, const void *key) { - struct ck_rhs_entry_desc *first; + long first; void *object; struct ck_rhs_map *map; unsigned long n_probes; @@ -979,8 +1231,8 @@ ck_rhs_get(struct ck_rhs *hs, probe = ck_rhs_map_bound_get(map, h); ck_pr_fence_load(); - first = NULL; - ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, probe, CK_RHS_PROBE_NO_RH); + first = -1; + map->probe_func(hs, map, &n_probes, &first, h, key, &object, probe, CK_RHS_PROBE_NO_RH); ck_pr_fence_load(); g_p = ck_pr_load_uint(generation); @@ -994,12 +1246,12 @@ ck_rhs_remove(struct ck_rhs *hs, unsigned long h, const void *key) { - struct ck_rhs_entry_desc *slot, *first; + long slot, first; void *object; struct ck_rhs_map *map = hs->map; unsigned long n_probes; - slot = ck_rhs_map_probe(hs, map, &n_probes, &first, h, key, &object, + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, ck_rhs_map_bound_get(map, h), CK_RHS_PROBE_NO_RH); if (object == NULL) return NULL; From eaff3ac2b3efad52b00ace9f3b3c857dcd57eafb Mon Sep 17 00:00:00 2001 From: User Doginou Date: Mon, 6 Jan 2014 16:47:44 +0100 Subject: [PATCH 05/11] ck_rhs: remove debugging printfs --- src/ck_rhs.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index 377b843..fb51915 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -874,16 +874,13 @@ restart: CK_RHS_PROBE_NO_RH : CK_RHS_PROBE_ROBIN_HOOD); if (slot == -1 && first == -1) { - printf("prout!!\n"); if (ck_rhs_grow(hs, map->capacity << 1) == false) { desc->in_rh = false; for (unsigned int i = 0; i < prevs_nb; i++) { CK_RHS_UNSET_RH(map, prevs[i]); } - printf("lolol\n"); return -1; } - printf("hehe\n"); return 1; } From 0e0fca638fda2fcb264b9e566b31397efdc9c5d4 Mon Sep 17 00:00:00 2001 From: Samy Al Bahra Date: Sat, 18 Jan 2014 17:05:40 -0500 Subject: [PATCH 06/11] ck_rhs: Various clean-up. --- src/ck_rhs.c | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index fb51915..c0c59d6 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -90,16 +90,16 @@ struct ck_rhs_entry_desc { CK_RHS_WORD probe_bound; bool in_rh; void *entry; -} __attribute__ ((__aligned__(16))); +} CK_CC_ALIGN(16); struct ck_rhs_no_entry_desc { unsigned int probes; unsigned short wanted; CK_RHS_WORD probe_bound; bool in_rh; -} __attribute__ ((__aligned__(8))); +} CK_CC_ALIGN(8); -typedef long (*probe_func) (struct ck_rhs *hs, +typedef long ck_hs_probe_cb_t(struct ck_rhs *hs, struct ck_rhs_map *map, unsigned long *n_probes, long *priority, @@ -127,7 +127,7 @@ struct ck_rhs_map { } no_entries; } entries; bool read_mostly; - probe_func probe_func; + ck_hs_probe_cb_t *probe_func; }; #define CK_RHS_ENTRY(map, offset) (map->read_mostly ? map->entries.no_entries.entries[offset] : map->entries.descs[offset].entry) @@ -179,27 +179,8 @@ struct ck_rhs_map { #define CK_RHS_LOAD_FACTOR 50 -static long -ck_rhs_map_probe(struct ck_rhs *hs, - struct ck_rhs_map *map, - unsigned long *n_probes, - long *priority, - unsigned long h, - const void *key, - void **object, - unsigned long probe_limit, - enum ck_rhs_probe_behavior behavior); - -static long -ck_rhs_map_probe_rm(struct ck_rhs *hs, - struct ck_rhs_map *map, - unsigned long *n_probes, - long *priority, - unsigned long h, - const void *key, - void **object, - unsigned long probe_limit, - enum ck_rhs_probe_behavior behavior); +static ck_hs_probe_cb_t *ck_rhs_map_probe; +static ck_hs_probe_cb_t *ck_rhs_map_probe_rm; void ck_rhs_iterator_init(struct ck_rhs_iterator *iterator) From 6d26a2b4daeeb4bd97bd65fbb0215130122d8687 Mon Sep 17 00:00:00 2001 From: Samy Al Bahra Date: Sat, 18 Jan 2014 17:11:29 -0500 Subject: [PATCH 07/11] ck_rhs: Move probe function to right namespace. Sleep deprivation... --- src/ck_rhs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index c0c59d6..8175dea 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -99,7 +99,7 @@ struct ck_rhs_no_entry_desc { bool in_rh; } CK_CC_ALIGN(8); -typedef long ck_hs_probe_cb_t(struct ck_rhs *hs, +typedef long ck_rhs_probe_cb_t(struct ck_rhs *hs, struct ck_rhs_map *map, unsigned long *n_probes, long *priority, @@ -127,7 +127,7 @@ struct ck_rhs_map { } no_entries; } entries; bool read_mostly; - ck_hs_probe_cb_t *probe_func; + ck_rhs_probe_cb_t *probe_func; }; #define CK_RHS_ENTRY(map, offset) (map->read_mostly ? map->entries.no_entries.entries[offset] : map->entries.descs[offset].entry) @@ -179,8 +179,8 @@ struct ck_rhs_map { #define CK_RHS_LOAD_FACTOR 50 -static ck_hs_probe_cb_t *ck_rhs_map_probe; -static ck_hs_probe_cb_t *ck_rhs_map_probe_rm; +static ck_rhs_probe_cb_t *ck_rhs_map_probe; +static ck_rhs_probe_cb_t *ck_rhs_map_probe_rm; void ck_rhs_iterator_init(struct ck_rhs_iterator *iterator) From ddab0f1820f3bb2f97eb612c752d9fb1a242b965 Mon Sep 17 00:00:00 2001 From: Samy Al Bahra Date: Sat, 18 Jan 2014 17:13:20 -0500 Subject: [PATCH 08/11] ck_rhs: Remove unnecessary parentheses. --- src/ck_rhs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index 8175dea..ec47d90 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -342,7 +342,7 @@ ck_rhs_map_probe_next(struct ck_rhs_map *map, offset = (offset &~ map->offset_mask) + ((offset + 1) & map->offset_mask); return offset; } else - return ((offset + probes) & map->mask); + return (offset + probes) & map->mask; } static inline unsigned long From 20b98dc271f0f270d0c338f2f37457cfe8dc2a49 Mon Sep 17 00:00:00 2001 From: Samy Al Bahra Date: Sat, 18 Jan 2014 17:16:09 -0500 Subject: [PATCH 09/11] ck_rhs: Additional clean-up. --- src/ck_rhs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index ec47d90..76bcc2d 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -179,8 +179,8 @@ struct ck_rhs_map { #define CK_RHS_LOAD_FACTOR 50 -static ck_rhs_probe_cb_t *ck_rhs_map_probe; -static ck_rhs_probe_cb_t *ck_rhs_map_probe_rm; +static ck_rhs_probe_cb_t ck_rhs_map_probe; +static ck_rhs_probe_cb_t ck_rhs_map_probe_rm; void ck_rhs_iterator_init(struct ck_rhs_iterator *iterator) @@ -547,7 +547,7 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, if (k == CK_RHS_EMPTY) goto leave; - if ((behavior != CK_RHS_PROBE_NO_RH)) { + if (behavior != CK_RHS_PROBE_NO_RH) { struct ck_rhs_entry_desc *desc = (void *)&map->entries.no_entries.descs[offset]; if (pr == -1 && From 8f1ea8043b8ee2c548bcb42fae0677b737a34026 Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Sun, 19 Jan 2014 13:55:45 +0100 Subject: [PATCH 10/11] ck_rhs: Use inlined functions instead of gruesome macros. --- src/ck_rhs.c | 250 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 157 insertions(+), 93 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index 76bcc2d..8ff7436 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -130,52 +130,117 @@ struct ck_rhs_map { ck_rhs_probe_cb_t *probe_func; }; -#define CK_RHS_ENTRY(map, offset) (map->read_mostly ? map->entries.no_entries.entries[offset] : map->entries.descs[offset].entry) -#define CK_RHS_ENTRY_ADDR(map, offset) (map->read_mostly ? &map->entries.no_entries.entries[offset] : &map->entries.descs[offset].entry) -#define CK_RHS_DESC(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? (struct ck_rhs_entry_desc *)(void *)&map->entries.no_entries.descs[offset] : &map->entries.descs[offset]) -#define CK_RHS_WANTED_INC(map, offset) do { \ - if (CK_CC_UNLIKELY(map->read_mostly)) \ - map->entries.no_entries.descs[offset].wanted++; \ - else \ - map->entries.descs[offset].wanted++; \ -} while (0) -#define CK_RHS_WANTED(map, offset) (CK_CC_UNLIKELY(map->read_mostly )? map->entries.no_entries.descs[offset].wanted : map->entries.descs[offset].wanted) -#define CK_RHS_SET_WANTED(map, offset, value) do { \ - if (CK_CC_UNLIKELY(map->read_mostly)) \ - map->entries.no_entries.descs[offset].wanted = value; \ - else \ - map->entries.descs[offset].probes = value; \ -} while (0) - -#define CK_RHS_WANTED_DEC(map, offset) do { \ - if (CK_CC_UNLIKELY(map->read_mostly)) \ - map->entries.no_entries.descs[offset].wanted--; \ - else \ - map->entries.descs[offset].wanted--; \ -} while (0) - -#define CK_RHS_PROBES(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? map->entries.no_entries.descs[offset].probes : map->entries.descs[offset].probes) -#define CK_RHS_SET_PROBES(map, offset, value) do { \ - if (CK_CC_UNLIKELY(map->read_mostly)) \ - map->entries.no_entries.descs[offset].probes = value; \ - else \ - map->entries.descs[offset].probes = value; \ -} while (0) -#define CK_RHS_PROBE_BOUND(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? map->entries.no_entries.descs[offset].probe_bound : map->entries.descs[offset].probe_bound) -#define CK_RHS_PROBE_BOUND_ADDR(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? (&map->entries.no_entries.descs[offset].probe_bound) : (&map->entries.descs[offset].probe_bound)) -#define CK_RHS_IN_RH(map, offset) (CK_CC_UNLIKELY(map->read_mostly) ? map->entries.no_entries.descs[offset].in_rh : map->entries.descs[offset].in_rh) -#define CK_RHS_SET_RH(map, offset) do { \ - if (CK_CC_UNLIKELY(map->read_mostly)) \ - map->entries.no_entries.descs[offset].in_rh = true; \ - else \ - map->entries.descs[offset].in_rh = true; \ -} while (0) -#define CK_RHS_UNSET_RH(map, offset) do { \ - if (CK_CC_UNLIKELY(map->read_mostly)) \ - map->entries.no_entries.descs[offset].in_rh = false; \ - else \ - map->entries.descs[offset].in_rh = false; \ -} while (0) +static CK_CC_INLINE void * +ck_rhs_entry(struct ck_rhs_map *map, long offset) +{ + + if (map->read_mostly) + return (map->entries.no_entries.entries[offset]); + else + return (map->entries.descs[offset].entry); +} + +static CK_CC_INLINE void ** +ck_rhs_entry_addr(struct ck_rhs_map *map, long offset) +{ + + if (map->read_mostly) + return (&map->entries.no_entries.entries[offset]); + else + return (&map->entries.descs[offset].entry); +} + +static CK_CC_INLINE struct ck_rhs_entry_desc * +ck_rhs_desc(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return ((struct ck_rhs_entry_desc *)(void *)&map->entries.no_entries.descs[offset]); + else + return (&map->entries.descs[offset]); +} + +static CK_CC_INLINE void +ck_rhs_wanted_inc(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].wanted++; + else + map->entries.descs[offset].wanted++; +} + +static CK_CC_INLINE unsigned int +ck_rhs_probes(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (map->entries.no_entries.descs[offset].probes); + else + return (map->entries.descs[offset].probes); +} + +static CK_CC_INLINE void +ck_rhs_set_probes(struct ck_rhs_map *map, long offset, unsigned int value) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].probes = value; + else + map->entries.descs[offset].probes = value; +} + +static CK_CC_INLINE CK_RHS_WORD +ck_rhs_probe_bound(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (map->entries.no_entries.descs[offset].probe_bound); + else + return (map->entries.descs[offset].probe_bound); +} + +static CK_CC_INLINE CK_RHS_WORD * +ck_rhs_probe_bound_addr(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (&map->entries.no_entries.descs[offset].probe_bound); + else + return (&map->entries.descs[offset].probe_bound); +} + + +static CK_CC_INLINE bool +ck_rhs_in_rh(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (map->entries.no_entries.descs[offset].in_rh); + else + return (map->entries.descs[offset].in_rh); +} + +static CK_CC_INLINE void +ck_rhs_set_rh(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].in_rh = true; + else + map->entries.descs[offset].in_rh = true; +} + +static CK_CC_INLINE void +ck_rhs_unset_rh(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].in_rh = false; + else + map->entries.descs[offset].in_rh = false; +} + #define CK_RHS_LOAD_FACTOR 50 @@ -201,7 +266,7 @@ ck_rhs_next(struct ck_rhs *hs, struct ck_rhs_iterator *i, void **key) return false; do { - value = CK_RHS_ENTRY(map, i->offset); + value = ck_rhs_entry(map, i->offset); if (value != CK_RHS_EMPTY) { #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) @@ -419,7 +484,7 @@ restart: for (k = 0; k < map->capacity; k++) { unsigned long h; - prev_saved = previous = CK_RHS_ENTRY(map, k); + prev_saved = previous = ck_rhs_entry(map, k); if (previous == CK_RHS_EMPTY) continue; @@ -433,7 +498,7 @@ restart: probes = 0; for (;;) { - void **cursor = CK_RHS_ENTRY_ADDR(update, offset); + void **cursor = ck_rhs_entry_addr(update, offset); if (probes++ == update->probe_limit) { /* @@ -447,10 +512,10 @@ restart: if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) { *cursor = prev_saved; update->n_entries++; - CK_RHS_SET_PROBES(update, offset, probes); + ck_rhs_set_probes(update, offset, probes); ck_rhs_map_bound_set(update, h, probes); break; - } else if (CK_RHS_PROBES(update, offset) < probes) { + } else if (ck_rhs_probes(update, offset) < probes) { void *tmp = prev_saved; unsigned int old_probes; prev_saved = previous = *cursor; @@ -461,12 +526,12 @@ restart: *cursor = tmp; ck_rhs_map_bound_set(update, h, probes); h = hs->hf(previous, hs->seed); - old_probes = CK_RHS_PROBES(update, offset); - CK_RHS_SET_PROBES(update, offset, probes); + old_probes = ck_rhs_probes(update, offset); + ck_rhs_set_probes(update, offset, probes); probes = old_probes - 1; continue; } - CK_RHS_WANTED_INC(update, offset); + ck_rhs_wanted_inc(update, offset); offset = ck_rhs_map_probe_next(update, offset, probes); } @@ -752,8 +817,8 @@ ck_rhs_gc(struct ck_rhs *hs) unsigned int max_probes = 0; for (i = 0; i < map->capacity; i++) { - if (CK_RHS_PROBES(map, i) > max_probes) - max_probes = CK_RHS_PROBES(map, i); + if (ck_rhs_probes(map, i) > max_probes) + max_probes = ck_rhs_probes(map, i); } map->probe_maximum = max_probes; return true; @@ -777,7 +842,7 @@ ck_rhs_add_wanted(struct ck_rhs *hs, long end_offset, long old_slot, if (offset == old_slot) found_slot = true; if (found_slot) { - desc = CK_RHS_DESC(map, offset); + desc = ck_rhs_desc(map, offset); if (desc->wanted < CK_RHS_MAX_WANTED) desc->wanted++; } @@ -790,7 +855,7 @@ static unsigned long ck_rhs_remove_wanted(struct ck_rhs *hs, long offset, long limit) { struct ck_rhs_map *map = hs->map; - int probes = CK_RHS_PROBES(map, offset); + int probes = ck_rhs_probes(map, offset); bool do_remove = true; struct ck_rhs_entry_desc *desc; @@ -800,7 +865,7 @@ ck_rhs_remove_wanted(struct ck_rhs *hs, long offset, long limit) if (offset == limit) do_remove = false; if (do_remove) { - desc = CK_RHS_DESC(map, offset); + desc = ck_rhs_desc(map, offset); if (desc->wanted != CK_RHS_MAX_WANTED) desc->wanted--; } @@ -841,14 +906,14 @@ ck_rhs_put_robin_hood(struct ck_rhs *hs, first = orig_slot; n_probes = desc->probes; restart: - key = CK_RHS_ENTRY(map, first); + key = ck_rhs_entry(map, first); insert = key; #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) key = CK_RHS_VMA(key); #endif orig_slot = first; - CK_RHS_SET_RH(map, orig_slot); + ck_rhs_set_rh(map, orig_slot); slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, prevs_nb == CK_RHS_MAX_RH ? @@ -857,16 +922,15 @@ restart: if (slot == -1 && first == -1) { if (ck_rhs_grow(hs, map->capacity << 1) == false) { desc->in_rh = false; - for (unsigned int i = 0; i < prevs_nb; i++) { - CK_RHS_UNSET_RH(map, prevs[i]); - } + for (unsigned int i = 0; i < prevs_nb; i++) + ck_rhs_unset_rh(map, prevs[i]); return -1; } return 1; } if (first != -1) { - desc = CK_RHS_DESC(map, first); + desc = ck_rhs_desc(map, first); int old_probes = desc->probes; desc->probes = n_probes; @@ -880,17 +944,17 @@ restart: /* An empty slot was found. */ h = ck_rhs_get_first_offset(map, slot, n_probes); ck_rhs_map_bound_set(map, h, n_probes); - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); - CK_RHS_SET_PROBES(map, slot, n_probes); + ck_rhs_set_probes(map, slot, n_probes); desc->in_rh = 0; ck_rhs_add_wanted(hs, slot, orig_slot, h); } while (prevs_nb > 0) { prev = prevs[--prevs_nb]; - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, orig_slot), - CK_RHS_ENTRY(map, prev)); + ck_pr_store_ptr(ck_rhs_entry_addr(map, orig_slot), + ck_rhs_entry(map, prev)); h = ck_rhs_get_first_offset(map, orig_slot, desc->probes); ck_rhs_add_wanted(hs, orig_slot, prev, h); @@ -898,7 +962,7 @@ restart: ck_pr_fence_atomic_store(); orig_slot = prev; desc->in_rh = false; - desc = CK_RHS_DESC(map, orig_slot); + desc = ck_rhs_desc(map, orig_slot); } return 0; } @@ -907,7 +971,7 @@ static void ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) { struct ck_rhs_map *map = hs->map; - struct ck_rhs_entry_desc *desc = CK_RHS_DESC(map, slot), *new_desc = NULL; + struct ck_rhs_entry_desc *desc = ck_rhs_desc(map, slot), *new_desc = NULL; unsigned long h; h = ck_rhs_remove_wanted(hs, slot, -1); @@ -924,7 +988,7 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) probe = wanted_probes; offset = ck_rhs_map_probe_next(map, slot, probe); while (probe < map->probe_maximum) { - new_desc = CK_RHS_DESC(map, offset); + new_desc = ck_rhs_desc(map, offset); if (new_desc->probes == probe + 1) break; probe++; @@ -942,12 +1006,12 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) desc->probes = wanted_probes; h = ck_rhs_remove_wanted(hs, offset, slot); - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), - CK_RHS_ENTRY(map, offset)); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), + ck_rhs_entry(map, offset)); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); if (wanted_probes < CK_RHS_WORD_MAX) { - struct ck_rhs_entry_desc *hdesc = CK_RHS_DESC(map, h); + struct ck_rhs_entry_desc *hdesc = ck_rhs_desc(map, h); if (hdesc->wanted == 1) CK_RHS_STORE(&hdesc->probe_bound, wanted_probes); @@ -979,9 +1043,9 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) slot = offset; desc = new_desc; } - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), CK_RHS_EMPTY); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), CK_RHS_EMPTY); if ((desc->probes - 1) < CK_RHS_WORD_MAX) - CK_RHS_STORE(CK_RHS_PROBE_BOUND_ADDR(map, h), + CK_RHS_STORE(ck_rhs_probe_bound_addr(map, h), desc->probes - 1); desc->probes = 0; } @@ -1011,9 +1075,9 @@ restart: if (first != -1) { int ret; - desc = CK_RHS_DESC(map, slot); - desc2 = CK_RHS_DESC(map, first); + desc = ck_rhs_desc(map, slot); + desc2 = ck_rhs_desc(map, first); desc->in_rh = true; ret = ck_rhs_put_robin_hood(hs, first, desc2); desc->in_rh = false; @@ -1022,15 +1086,15 @@ restart: else if (CK_CC_UNLIKELY(ret != 0)) return false; - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, first), insert); + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); desc2->probes = n_probes; ck_rhs_add_wanted(hs, first, -1, h); ck_rhs_do_backward_shift_delete(hs, slot); } else { - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); - CK_RHS_SET_PROBES(map, slot, n_probes); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); } *previous = object; return true; @@ -1066,20 +1130,20 @@ restart: if (first != -1) { struct ck_rhs_entry_desc *desc = NULL, *desc2; if (slot != -1) { - desc = CK_RHS_DESC(map, slot); + desc = ck_rhs_desc(map, slot); desc->in_rh = true; } - desc2 = CK_RHS_DESC(map, first); + desc2 = ck_rhs_desc(map, first); int ret = ck_rhs_put_robin_hood(hs, first, desc2); if (slot != -1) desc->in_rh = false; - if (CK_CC_LIKELY(ret == 1)) + if (CK_CC_UNLIKELY(ret == 1)) goto restart; - if (CK_CC_LIKELY(ret == -1)) + if (CK_CC_UNLIKELY(ret == -1)) return false; /* If an earlier bucket was found, then store entry there. */ - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, first), insert); + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); desc2->probes = n_probes; /* * If a duplicate key was found, then delete it after @@ -1100,8 +1164,8 @@ restart: * If we are storing into same slot, then atomic store is sufficient * for replacement. */ - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); - CK_RHS_SET_PROBES(map, slot, n_probes); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); if (object == NULL) ck_rhs_add_wanted(hs, slot, -1, h); } @@ -1148,20 +1212,20 @@ restart: insert = ck_rhs_marshal(hs->mode, key, h); if (first != -1) { - struct ck_rhs_entry_desc *desc = CK_RHS_DESC(map, first); + struct ck_rhs_entry_desc *desc = ck_rhs_desc(map, first); int ret = ck_rhs_put_robin_hood(hs, first, desc); if (CK_CC_UNLIKELY(ret == 1)) return ck_rhs_put_internal(hs, h, key, behavior); else if (CK_CC_UNLIKELY(ret == -1)) return false; /* Insert key into first bucket in probe sequence. */ - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, first), insert); + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); desc->probes = n_probes; ck_rhs_add_wanted(hs, first, -1, h); } else { /* An empty slot was found. */ - ck_pr_store_ptr(CK_RHS_ENTRY_ADDR(map, slot), insert); - CK_RHS_SET_PROBES(map, slot, n_probes); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); ck_rhs_add_wanted(hs, slot, -1, h); } From efc88a5af57f5efe6af32490597cb91c11039f38 Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Sun, 19 Jan 2014 14:01:23 +0100 Subject: [PATCH 11/11] ck_rhs: style(9) --- src/ck_rhs.c | 42 ++++++++++++++---------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/ck_rhs.c b/src/ck_rhs.c index 8ff7436..84e4ee1 100644 --- a/src/ck_rhs.c +++ b/src/ck_rhs.c @@ -326,10 +326,13 @@ ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries) if (hs->mode & CK_RHS_MODE_READ_MOSTLY) size = sizeof(struct ck_rhs_map) + - (sizeof(void *) * n_entries + sizeof(struct ck_rhs_no_entry_desc) * n_entries + 2 * CK_MD_CACHELINE - 1); + (sizeof(void *) * n_entries + + sizeof(struct ck_rhs_no_entry_desc) * n_entries + + 2 * CK_MD_CACHELINE - 1); else size = sizeof(struct ck_rhs_map) + - (sizeof(struct ck_rhs_entry_desc) * n_entries + CK_MD_CACHELINE - 1); + (sizeof(struct ck_rhs_entry_desc) * n_entries + + CK_MD_CACHELINE - 1); map = hs->m->malloc(size); if (map == NULL) return NULL; @@ -404,7 +407,8 @@ ck_rhs_map_probe_next(struct ck_rhs_map *map, unsigned long probes) { if (probes & map->offset_mask) { - offset = (offset &~ map->offset_mask) + ((offset + 1) & map->offset_mask); + offset = (offset &~ map->offset_mask) + + ((offset + 1) & map->offset_mask); return offset; } else return (offset + probes) & map->mask; @@ -532,7 +536,6 @@ restart: continue; } ck_rhs_wanted_inc(update, offset); - offset = ck_rhs_map_probe_next(update, offset, probes); } @@ -562,7 +565,6 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, unsigned long probe_limit, enum ck_rhs_probe_behavior behavior) { - void *k; const void *compare; long pr = -1; @@ -581,7 +583,6 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, #else compare = key; #endif - *object = NULL; if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { probes = 0; @@ -594,7 +595,6 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, } opl = probe_limit; - for (;;) { if (probes++ == probe_limit) { if (probe_limit == opl || pr != -1) { @@ -607,7 +607,6 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, */ probe_limit = opl; } - k = ck_pr_load_ptr(&map->entries.no_entries.entries[offset]); if (k == CK_RHS_EMPTY) goto leave; @@ -628,7 +627,6 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, } } - if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) { @@ -654,7 +652,6 @@ ck_rhs_map_probe_rm(struct ck_rhs *hs, } offset = ck_rhs_map_probe_next(map, offset, probes); } - leave: if (probes > probe_limit) { offset = -1; @@ -714,7 +711,6 @@ ck_rhs_map_probe(struct ck_rhs *hs, if (behavior == CK_RHS_PROBE_INSERT) probe_limit = ck_rhs_map_bound_get(map, h); - for (;;) { if (probes++ == probe_limit) { if (probe_limit == opl || pr != -1) { @@ -727,13 +723,11 @@ ck_rhs_map_probe(struct ck_rhs *hs, */ probe_limit = opl; } - k = ck_pr_load_ptr(&map->entries.descs[offset].entry); if (k == CK_RHS_EMPTY) goto leave; - if ((behavior != CK_RHS_PROBE_NO_RH)) { - struct ck_rhs_entry_desc *desc = &map->entries.descs[offset];; + struct ck_rhs_entry_desc *desc = &map->entries.descs[offset]; if (pr == -1 && desc->in_rh == false && desc->probes < probes) { @@ -748,7 +742,6 @@ ck_rhs_map_probe(struct ck_rhs *hs, } } - if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) { @@ -774,7 +767,6 @@ ck_rhs_map_probe(struct ck_rhs *hs, } offset = ck_rhs_map_probe_next(map, offset, probes); } - leave: if (probes > probe_limit) { offset = -1; @@ -971,18 +963,17 @@ static void ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) { struct ck_rhs_map *map = hs->map; - struct ck_rhs_entry_desc *desc = ck_rhs_desc(map, slot), *new_desc = NULL; + struct ck_rhs_entry_desc *desc, *new_desc = NULL; unsigned long h; + desc = ck_rhs_desc(map, slot); h = ck_rhs_remove_wanted(hs, slot, -1); - while (desc->wanted > 0) { unsigned long offset = 0, tmp_offset; unsigned long wanted_probes = 1; unsigned int probe = 0; unsigned int max_probes; - /* Find a successor */ while (wanted_probes < map->probe_maximum) { probe = wanted_probes; @@ -992,7 +983,8 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) if (new_desc->probes == probe + 1) break; probe++; - offset = ck_rhs_map_probe_next(map, offset, probe); + offset = ck_rhs_map_probe_next(map, offset, + probe); } if (probe < map->probe_maximum) break; @@ -1002,9 +994,7 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) desc->wanted = 0; break; } - desc->probes = wanted_probes; - h = ck_rhs_remove_wanted(hs, offset, slot); ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), ck_rhs_entry(map, offset)); @@ -1024,8 +1014,8 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) max_probes = hdesc->probe_bound; max_probes--; } - tmp_offset = ck_rhs_map_probe_next(map, offset, probe); - + tmp_offset = ck_rhs_map_probe_next(map, offset, + probe); while (probe < max_probes) { if (h == (unsigned long)ck_rhs_get_first_offset(map, tmp_offset, probe)) break; @@ -1037,7 +1027,6 @@ ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) wanted_probes); } } - if (desc->wanted < CK_RHS_MAX_WANTED) desc->wanted--; slot = offset; @@ -1085,7 +1074,6 @@ restart: goto restart; else if (CK_CC_UNLIKELY(ret != 0)) return false; - ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); ck_pr_fence_atomic_store(); @@ -1123,7 +1111,6 @@ restart: goto restart; } - ck_rhs_map_bound_set(map, h, n_probes); insert = ck_rhs_marshal(hs->mode, key, h); @@ -1230,7 +1217,6 @@ restart: } map->n_entries++; - if ((map->n_entries ) > ((map->capacity * CK_RHS_LOAD_FACTOR) / 100)) ck_rhs_grow(hs, map->capacity << 1); return true;