From 3ad9f7e63d60656049219ab8a66a4d76e0bbc43e Mon Sep 17 00:00:00 2001 From: Samy Al Bahra Date: Mon, 2 Apr 2012 18:49:08 -0400 Subject: [PATCH] ck_ht: Add support for other 64-bit platforms. Specifically, any platform that has CK support for 64-bit load/store operations. Additional improvements have been made to the unit tests to disambiguate put/get failures. --- include/ck_ht.h | 36 +++++++++- regressions/ck_ht/benchmark/parallel.c | 5 +- regressions/ck_ht/benchmark/serial.c | 6 +- regressions/ck_ht/validate/serial.c | 9 ++- src/ck_ht.c | 98 ++++++++++++++++++++++---- 5 files changed, 127 insertions(+), 27 deletions(-) diff --git a/include/ck_ht.h b/include/ck_ht.h index 672cfb3..2237229 100644 --- a/include/ck_ht.h +++ b/include/ck_ht.h @@ -27,10 +27,11 @@ #ifndef _CK_HT_H #define _CK_HT_H -#ifdef __x86_64__ +#include + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) #include -#include #include #include #include @@ -47,8 +48,15 @@ enum ck_ht_mode { }; struct ck_ht_entry { +#ifdef __x86_64__ uintptr_t key; uintptr_t value CK_CC_PACKED; +#else + uintptr_t key; + uintptr_t value; + uint64_t key_length; + uint64_t hash; +#endif } CK_CC_ALIGNED; typedef struct ck_ht_entry ck_ht_entry_t; @@ -99,7 +107,13 @@ CK_CC_INLINE static void ck_ht_entry_key_set(ck_ht_entry_t *entry, const void *key, uint16_t key_length) { +#ifdef __x86_64__ entry->key = (uintptr_t)key | ((uintptr_t)key_length << 48); +#else + entry->key = (uintptr_t)key; + entry->key_length = key_length; +#endif + return; } @@ -114,14 +128,22 @@ CK_CC_INLINE static uint16_t ck_ht_entry_key_length(ck_ht_entry_t *entry) { +#ifdef __x86_64__ return entry->key >> 48; +#else + return entry->key_length; +#endif } CK_CC_INLINE static void * ck_ht_entry_value(ck_ht_entry_t *entry) { +#ifdef __x86_64__ return (void *)(entry->value & (((uintptr_t)1 << 48) - 1)); +#else + return (void *)entry->value; +#endif } CK_CC_INLINE static void @@ -132,8 +154,16 @@ ck_ht_entry_set(struct ck_ht_entry *entry, const void *value) { +#ifdef __x86_64__ entry->key = (uintptr_t)key | ((uintptr_t)key_length << 48); entry->value = (uintptr_t)value | ((uintptr_t)(h.value >> 32) << 48); +#else + entry->key = (uintptr_t)key; + entry->value = (uintptr_t)value; + entry->key_length = key_length; + entry->hash = h.value; +#endif + return; } @@ -181,5 +211,5 @@ bool ck_ht_remove_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); bool ck_ht_reset_spmc(ck_ht_t *); uint64_t ck_ht_count(ck_ht_t *); -#endif /* __x86_64__ */ +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_STORE_64 */ #endif /* _CK_HT_H */ diff --git a/regressions/ck_ht/benchmark/parallel.c b/regressions/ck_ht/benchmark/parallel.c index 0abeff5..265bf88 100644 --- a/regressions/ck_ht/benchmark/parallel.c +++ b/regressions/ck_ht/benchmark/parallel.c @@ -24,7 +24,7 @@ * SUCH DAMAGE. */ -#ifdef __x86_64__ +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) #include #include @@ -570,4 +570,5 @@ main(void) return 0; } -#endif /* __x86_64__ */ +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_STORE_64 */ + diff --git a/regressions/ck_ht/benchmark/serial.c b/regressions/ck_ht/benchmark/serial.c index e4bf7d7..9e4d1a1 100644 --- a/regressions/ck_ht/benchmark/serial.c +++ b/regressions/ck_ht/benchmark/serial.c @@ -24,8 +24,7 @@ * SUCH DAMAGE. */ -#ifdef __x86_64__ - +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) #include #include #include @@ -272,4 +271,5 @@ main(void) return 0; } -#endif /* __x86_64__ */ +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_STORE_64 */ + diff --git a/regressions/ck_ht/validate/serial.c b/regressions/ck_ht/validate/serial.c index 101f715..29f893b 100644 --- a/regressions/ck_ht/validate/serial.c +++ b/regressions/ck_ht/validate/serial.c @@ -24,8 +24,7 @@ * SUCH DAMAGE. */ -#ifdef __x86_64__ - +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) #include #include #include @@ -100,7 +99,7 @@ main(void) ck_ht_hash(&h, &ht, test[i], l); ck_ht_entry_key_set(&entry, test[i], l); if (ck_ht_get_spmc(&ht, h, &entry) == false) { - fprintf(stderr, "ERROR: Failed to find [%s]\n", test[i]); + fprintf(stderr, "ERROR (put): Failed to find [%s]\n", test[i]); } else { void *k, *v; @@ -177,7 +176,7 @@ main(void) ck_ht_hash(&h, &ht, test[i], l); ck_ht_entry_key_set(&entry, test[i], l); if (ck_ht_get_spmc(&ht, h, &entry) == false) { - fprintf(stderr, "ERROR: Failed to find [%s]\n", test[i]); + fprintf(stderr, "ERROR (set): Failed to find [%s]\n", test[i]); } else { void *k, *v; @@ -289,5 +288,5 @@ main(void) return 0; } -#endif +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_STORE_64 */ diff --git a/src/ck_ht.c b/src/ck_ht.c index 07e40a1..e2b2f29 100644 --- a/src/ck_ht.c +++ b/src/ck_ht.c @@ -24,21 +24,19 @@ * SUCH DAMAGE. */ +#include + +#if defined(CK_F_PR_STORE_64) && defined(CK_F_PR_LOAD_64) /* * This implementation borrows several techniques from Josh Dybnis's * nbds library which can be found at http://code.google.com/p/nbds + * + * This release currently only includes support for 64-bit platforms. + * We can address 32-bit platforms in a future release. */ - -/* - * This release currently only includes support for x86_64 as I haven't - * had the time to put pressure on other platforms. The portability issues - * will be addressed next release. - */ -#ifdef __x86_64__ - #include #include -#include +#include #include #include #include @@ -47,7 +45,13 @@ #include "ck_internal.h" #ifndef CK_HT_BUCKET_LENGTH + +#ifdef __x86_64__ #define CK_HT_BUCKET_SHIFT 2ULL +#else +#define CK_HT_BUCKET_SHIFT 1ULL +#endif + #define CK_HT_BUCKET_LENGTH (1 << CK_HT_BUCKET_SHIFT) #define CK_HT_BUCKET_MASK (CK_HT_BUCKET_LENGTH - 1) #endif @@ -191,6 +195,12 @@ ck_ht_map_probe(struct ck_ht_map *map, uint64_t probes = 0; uint64_t probe_maximum; +#ifndef __x86_64__ + uint64_t d = 0; + uint64_t d_prime = 0; +retry: +#endif + probe_maximum = ck_pr_load_64(&map->probe_maximum); offset = h.value & map->mask; @@ -220,12 +230,20 @@ ck_ht_map_probe(struct ck_ht_map *map, * it is worth the code duplication. */ if (probe_limit == NULL) { +#ifdef __x86_64__ + snapshot->key = (uintptr_t)ck_pr_load_ptr(&cursor->key); + ck_pr_fence_load(); + snapshot->value = (uintptr_t)ck_pr_load_ptr(&cursor->value); +#else + d = ck_pr_load_64(&map->deletions); snapshot->key = (uintptr_t)ck_pr_load_ptr(&cursor->key); ck_pr_fence_load(); + snapshot->key_length = ck_pr_load_64(&cursor->key_length); + snapshot->hash = ck_pr_load_64(&cursor->hash); snapshot->value = (uintptr_t)ck_pr_load_ptr(&cursor->value); +#endif } else { - snapshot->key = cursor->key; - snapshot->value = cursor->value; + *snapshot = *cursor; } /* @@ -233,8 +251,10 @@ ck_ht_map_probe(struct ck_ht_map *map, * in order to prevent a complete reprobe sequence in * the case of intermittent writers. */ - if (first == NULL && snapshot->key == CK_HT_KEY_TOMBSTONE) { - first = cursor; + if (snapshot->key == CK_HT_KEY_TOMBSTONE) { + if (first == NULL) + first = cursor; + continue; } @@ -247,6 +267,19 @@ ck_ht_map_probe(struct ck_ht_map *map, if (map->mode == CK_HT_MODE_BYTESTRING) { void *pointer; +#ifndef __x86_64__ + if (probe_limit == NULL) { + d_prime = ck_pr_load_64(&map->deletions); + + /* + * It is possible that the slot was + * replaced, initiate a re-probe. + */ + if (d != d_prime) + goto retry; + } +#endif + /* * Check memoized portion of hash value before * expensive full-length comparison. @@ -255,8 +288,13 @@ ck_ht_map_probe(struct ck_ht_map *map, if (k != key_length) continue; +#ifdef __x86_64__ if (snapshot->value >> 48 != ((h.value >> 32) & 0xFFFF)) continue; +#else + if (snapshot->hash != h.value) + continue; +#endif pointer = ck_ht_entry_key(snapshot); if (memcmp(pointer, key, key_length) == 0) @@ -465,12 +503,21 @@ ck_ht_get_spmc(ck_ht_t *table, { struct ck_ht_entry *candidate, snapshot; struct ck_ht_map *map; + +#ifdef __x86_64__ uint64_t d, d_prime; restart: +#endif map = ck_pr_load_ptr(&table->map); +#ifdef __x86_64__ + /* + * Platforms that cannot read key and key_length atomically must reprobe + * on the scan of any single entry. + */ d = ck_pr_load_64(&map->deletions); +#endif if (table->mode == CK_HT_MODE_BYTESTRING) { candidate = ck_ht_map_probe(map, h, &snapshot, NULL, @@ -480,6 +527,7 @@ restart: (void *)entry->key, sizeof(entry->key), NULL); } +#ifdef __x86_64__ d_prime = ck_pr_load_64(&map->deletions); if (d != d_prime) { /* @@ -489,6 +537,7 @@ restart: */ goto restart; } +#endif if (candidate == NULL || snapshot.key == CK_HT_KEY_EMPTY) return false; @@ -541,6 +590,10 @@ ck_ht_set_spmc(ck_ht_t *table, * before transitioning from K to T. (K, B) implies (K, B, D') * so we will reprobe successfully from this transient state. */ +#ifndef __x86_64__ + ck_pr_store_64(&priority->key_length, entry->key_length); + ck_pr_store_64(&priority->hash, entry->hash); +#endif ck_pr_store_ptr(&priority->value, (void *)entry->value); ck_pr_fence_store(); ck_pr_store_ptr(&priority->key, (void *)entry->key); @@ -558,9 +611,17 @@ ck_ht_set_spmc(ck_ht_t *table, if (priority != NULL) candidate = priority; +#ifdef __x86_64__ + ck_pr_store_ptr(&candidate->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr(&candidate->key, (void *)entry->key); +#else + ck_pr_store_64(&candidate->key_length, entry->key_length); + ck_pr_store_64(&candidate->hash, entry->hash); ck_pr_store_ptr(&candidate->value, (void *)entry->value); ck_pr_fence_store(); ck_pr_store_ptr(&candidate->key, (void *)entry->key); +#endif /* * If we are insert a new entry then increment number @@ -627,9 +688,18 @@ ck_ht_put_spmc(ck_ht_t *table, if (priority != NULL) candidate = priority; +#ifdef __x86_64__ + ck_pr_store_ptr(&candidate->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr(&candidate->key, (void *)entry->key); +#else + ck_pr_store_64(&candidate->key_length, entry->key_length); + ck_pr_store_64(&candidate->hash, entry->hash); ck_pr_store_ptr(&candidate->value, (void *)entry->value); ck_pr_fence_store(); ck_pr_store_ptr(&candidate->key, (void *)entry->key); +#endif + ck_pr_store_64(&map->n_entries, map->n_entries + 1); /* Enforce a load factor of 0.5. */ @@ -647,4 +717,4 @@ ck_ht_destroy(struct ck_ht *table) return; } -#endif /* __x86_64__ */ +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_STORE_64 */