Merge branch 'master' of github.com:concurrencykit/ck

include/ck_ht.h

@@ -33,8 +33,19 @@
 
 #include <ck_pr.h>
 
-#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64)
 #define CK_F_HT
+#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64)
+#define CK_HT_TYPE uint64_t
+#define CK_HT_TYPE_LOAD		ck_pr_load_64
+#define CK_HT_TYPE_STORE 	ck_pr_store_64
+#define CK_HT_TYPE_MAX		UINT64_MAX
+#else
+#define CK_HT_TYPE uint32_t
+#define CK_HT_TYPE_LOAD		ck_pr_load_32
+#define CK_HT_TYPE_STORE	ck_pr_store_32
+#define CK_HT_TYPE_MAX		UINT32_MAX
+#endif
+
 
 #include <ck_cc.h>
 #include <ck_malloc.h>
@@ -68,8 +79,8 @@ struct ck_ht_entry {
 #else
 	uintptr_t key;
 	uintptr_t value;
-	uint64_t key_length;
+	CK_HT_TYPE key_length;
-	uint64_t hash;
+	CK_HT_TYPE hash;
 } CK_CC_ALIGN(32);
 #endif
 typedef struct ck_ht_entry ck_ht_entry_t;
@@ -249,17 +260,16 @@ void ck_ht_stat(ck_ht_t *, struct ck_ht_stat *);
 void ck_ht_hash(ck_ht_hash_t *, ck_ht_t *, const void *, uint16_t);
 void ck_ht_hash_direct(ck_ht_hash_t *, ck_ht_t *, uintptr_t);
 bool ck_ht_init(ck_ht_t *, unsigned int, ck_ht_hash_cb_t *,
-    struct ck_malloc *, uint64_t, uint64_t);
+    struct ck_malloc *, CK_HT_TYPE, uint64_t);
 void ck_ht_destroy(ck_ht_t *);
 bool ck_ht_set_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *);
 bool ck_ht_put_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *);
 bool ck_ht_get_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *);
 bool ck_ht_gc(struct ck_ht *, unsigned long, unsigned long);
-bool ck_ht_grow_spmc(ck_ht_t *, uint64_t);
+bool ck_ht_grow_spmc(ck_ht_t *, CK_HT_TYPE);
 bool ck_ht_remove_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *);
 bool ck_ht_reset_spmc(ck_ht_t *);
-bool ck_ht_reset_size_spmc(ck_ht_t *, uint64_t);
+bool ck_ht_reset_size_spmc(ck_ht_t *, CK_HT_TYPE);
-uint64_t ck_ht_count(ck_ht_t *);
+CK_HT_TYPE ck_ht_count(ck_ht_t *);
 
-#endif /* CK_F_PR_LOAD_64 && CK_F_PR_STORE_64 */
 #endif /* CK_HT_H */

include/ck_limits.h

@@ -26,6 +26,20 @@
 
 #if defined(__linux__) && defined(__KERNEL__)
 #include <linux/kernel.h>
+
+#ifndef UINT8_MAX
+#define UINT8_MAX ((u8)(~0U))
+#endif
+#ifndef UINT16_MAX
+#define UINT16_MAX USHRT_MAX
+#endif
+#ifndef UINT32_MAX
+#define UINT32_MAX UINT_MAX
+#endif
+#ifndef UINT64_MAX
+#define UINT64_MAX ULLONG_MAX
+#endif
+
 #else
 #include <limits.h>
 #endif /* __linux__ && __KERNEL__ */

include/gcc/ck_pr.h

@@ -53,7 +53,7 @@ ck_pr_barrier(void)
  */
 #include "ck_f_pr.h"
 
-#define CK_PR_ACCESS(x) (*(volatile typeof(x) *)&(x))
+#define CK_PR_ACCESS(x) (*(volatile __typeof__(x) *)&(x))
 
 #define CK_PR_LOAD(S, M, T)		 			\
 	CK_CC_INLINE static T					\

regressions/ck_ht/benchmark/parallel_bytestring.c

@@ -26,7 +26,6 @@
 
 #include <ck_ht.h>
 
-#ifdef CK_F_HT
 
 #include <assert.h>
 #include <ck_epoch.h>
@@ -558,12 +557,3 @@ main(int argc, char *argv[])
 	    epoch_wr.n_pending, epoch_wr.n_peak, epoch_wr.n_dispatch);
 	return 0;
 }
-#else
-int
-main(void)
-{
-
-	return 0;
-}
-#endif /* CK_F_HT */
-

regressions/ck_ht/benchmark/parallel_direct.c

@@ -26,8 +26,6 @@
 
 #include <ck_ht.h>
 
-#ifdef CK_F_HT
-
 #include <assert.h>
 #include <ck_epoch.h>
 #include <ck_malloc.h>
@@ -545,12 +543,3 @@ main(int argc, char *argv[])
 	    epoch_wr.n_pending, epoch_wr.n_peak, epoch_wr.n_dispatch);
 	return 0;
 }
-#else
-int
-main(void)
-{
-
-	return 0;
-}
-#endif /* CK_F_HT */
-

regressions/ck_ht/benchmark/serial.c

@@ -26,8 +26,6 @@
 
 #include <ck_ht.h>
 
-#ifdef CK_F_HT
-
 #include <assert.h>
 #include <ck_malloc.h>
 #include <errno.h>
@@ -387,12 +385,3 @@ main(int argc, char *argv[])
 
 	return 0;
 }
-#else
-int
-main(void)
-{
-
-	return 0;
-}
-#endif /* CK_F_HT */
-

regressions/ck_ht/validate/serial.c

@@ -26,7 +26,6 @@
 
 #include <ck_ht.h>
 
-#ifdef CK_F_HT
 #include <assert.h>
 #include <ck_malloc.h>
 #include <errno.h>
@@ -281,12 +280,3 @@ main(void)
 	ck_ht_destroy(&ht);
 	return 0;
 }
-#else
-int
-main(void)
-{
-
-	return 0;
-}
-#endif /* CK_F_HT */
-

src/ck_ht.c

@@ -27,7 +27,6 @@
 #define CK_HT_IM
 #include <ck_ht.h>
 
-#ifdef CK_F_HT
 /*
  * This implementation borrows several techniques from Josh Dybnis's
  * nbds library which can be found at http://code.google.com/p/nbds
@@ -82,15 +81,15 @@
 
 struct ck_ht_map {
 	unsigned int mode;
-	uint64_t deletions;
+	CK_HT_TYPE deletions;
-	uint64_t probe_maximum;
+	CK_HT_TYPE probe_maximum;
-	uint64_t probe_length;
+	CK_HT_TYPE probe_length;
-	uint64_t probe_limit;
+	CK_HT_TYPE probe_limit;
-	uint64_t size;
+	CK_HT_TYPE size;
-	uint64_t n_entries;
+	CK_HT_TYPE n_entries;
-	uint64_t mask;
+	CK_HT_TYPE mask;
-	uint64_t capacity;
+	CK_HT_TYPE capacity;
-	uint64_t step;
+	CK_HT_TYPE step;
 	CK_HT_WORD *probe_bound;
 	struct ck_ht_entry *entries;
 };
@@ -134,15 +133,15 @@ ck_ht_hash_wrapper(struct ck_ht_hash *h,
     uint64_t seed)
 {
 
-	h->value = MurmurHash64A(key, length, seed);
+	h->value = (unsigned long)MurmurHash64A(key, length, seed);
 	return;
 }
 
 static struct ck_ht_map *
-ck_ht_map_create(struct ck_ht *table, uint64_t entries)
+ck_ht_map_create(struct ck_ht *table, CK_HT_TYPE entries)
 {
 	struct ck_ht_map *map;
-	uint64_t size;
+	CK_HT_TYPE size;
 	uintptr_t prefix;
 	uint32_t n_entries;
 
@@ -193,12 +192,12 @@ ck_ht_map_create(struct ck_ht *table, uint64_t entries)
 static inline void
 ck_ht_map_bound_set(struct ck_ht_map *m,
     struct ck_ht_hash h,
-    uint64_t n_probes)
+    CK_HT_TYPE n_probes)
 {
-	uint64_t offset = h.value & m->mask;
+	CK_HT_TYPE offset = h.value & m->mask;
 
 	if (n_probes > m->probe_maximum)
-		ck_pr_store_64(&m->probe_maximum, n_probes);
+		CK_HT_TYPE_STORE(&m->probe_maximum, n_probes);
 
 	if (m->probe_bound != NULL && m->probe_bound[offset] < n_probes) {
 		if (n_probes >= CK_HT_WORD_MAX)
@@ -211,18 +210,18 @@ ck_ht_map_bound_set(struct ck_ht_map *m,
 	return;
 }
 
-static inline uint64_t
+static inline CK_HT_TYPE
 ck_ht_map_bound_get(struct ck_ht_map *m, struct ck_ht_hash h)
 {
-	uint64_t offset = h.value & m->mask;
+	CK_HT_TYPE offset = h.value & m->mask;
-	uint64_t r = CK_HT_WORD_MAX;
+	CK_HT_TYPE r = CK_HT_WORD_MAX;
 
 	if (m->probe_bound != NULL) {
 		r = CK_HT_LOAD(&m->probe_bound[offset]);
 		if (r == CK_HT_WORD_MAX)
-			r = ck_pr_load_64(&m->probe_maximum);
+			r = CK_HT_TYPE_LOAD(&m->probe_maximum);
 	} else {
-		r = ck_pr_load_64(&m->probe_maximum);
+		r = CK_HT_TYPE_LOAD(&m->probe_maximum);
 	}
 
 	return r;
@@ -256,7 +255,7 @@ ck_ht_init(struct ck_ht *table,
     unsigned int mode,
     ck_ht_hash_cb_t *h,
     struct ck_malloc *m,
-    uint64_t entries,
+    CK_HT_TYPE entries,
     uint64_t seed)
 {
 
@@ -284,19 +283,19 @@ ck_ht_map_probe_wr(struct ck_ht_map *map,
     ck_ht_entry_t **available,
     const void *key,
     uint16_t key_length,
-    uint64_t *probe_limit,
+    CK_HT_TYPE *probe_limit,
-    uint64_t *probe_wr)
+    CK_HT_TYPE *probe_wr)
 {
 	struct ck_ht_entry *bucket, *cursor;
 	struct ck_ht_entry *first = NULL;
 	size_t offset, i, j;
-	uint64_t probes = 0;
+	CK_HT_TYPE probes = 0;
-	uint64_t limit;
+	CK_HT_TYPE limit;
 
 	if (probe_limit == NULL) {
 		limit = ck_ht_map_bound_get(map, h);
 	} else {
-		limit = UINT64_MAX;
+		limit = CK_HT_TYPE_MAX;
 	}
 
 	offset = h.value & map->mask;
@@ -388,11 +387,11 @@ ck_ht_gc(struct ck_ht *ht, unsigned long cycles, unsigned long seed)
 {
 	CK_HT_WORD *bounds = NULL;
 	struct ck_ht_map *map = ht->map;
-	uint64_t maximum, i;
+	CK_HT_TYPE maximum, i;
-	uint64_t size = 0;
+	CK_HT_TYPE size = 0;
 
 	if (map->n_entries == 0) {
-		ck_pr_store_64(&map->probe_maximum, 0);
+		CK_HT_TYPE_STORE(&map->probe_maximum, 0);
 		if (map->probe_bound != NULL)
 			memset(map->probe_bound, 0, sizeof(CK_HT_WORD) * map->capacity);
 
@@ -417,8 +416,8 @@ ck_ht_gc(struct ck_ht *ht, unsigned long cycles, unsigned long seed)
 	for (i = 0; i < map->capacity; i++) {
 		struct ck_ht_entry *entry, *priority, snapshot;
 		struct ck_ht_hash h;
-		uint64_t probes_wr;
+		CK_HT_TYPE probes_wr;
-		uint64_t offset;
+		CK_HT_TYPE offset;
 
 		entry = &map->entries[(i + seed) & map->mask];
 		if (entry->key == CK_HT_KEY_EMPTY ||
@@ -452,17 +451,17 @@ ck_ht_gc(struct ck_ht *ht, unsigned long cycles, unsigned long seed)
 		offset = h.value & map->mask;
 
 		if (priority != NULL) {
-			ck_pr_store_64(&map->deletions, map->deletions + 1);
+			CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1);
 			ck_pr_fence_store();
 #ifndef CK_HT_PP
-			ck_pr_store_64(&priority->key_length, entry->key_length);
+			CK_HT_TYPE_STORE(&priority->key_length, entry->key_length);
-			ck_pr_store_64(&priority->hash, entry->hash);
+			CK_HT_TYPE_STORE(&priority->hash, entry->hash);
 #endif
 			ck_pr_store_ptr_unsafe(&priority->value, (void *)entry->value);
 			ck_pr_fence_store();
 			ck_pr_store_ptr_unsafe(&priority->key, (void *)entry->key);
 			ck_pr_fence_store();
-			ck_pr_store_64(&map->deletions, map->deletions + 1);
+			CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1);
 			ck_pr_fence_store();
 			ck_pr_store_ptr_unsafe(&entry->key, (void *)CK_HT_KEY_TOMBSTONE);
 			ck_pr_fence_store();
@@ -482,7 +481,7 @@ ck_ht_gc(struct ck_ht *ht, unsigned long cycles, unsigned long seed)
 	}
 
 	if (maximum != map->probe_maximum)
-		ck_pr_store_64(&map->probe_maximum, maximum);
+		CK_HT_TYPE_STORE(&map->probe_maximum, maximum);
 
 	if (bounds != NULL) {
 		for (i = 0; i < map->capacity; i++)
@@ -503,12 +502,12 @@ ck_ht_map_probe_rd(struct ck_ht_map *map,
 {
 	struct ck_ht_entry *bucket, *cursor;
 	size_t offset, i, j;
-	uint64_t probes = 0;
+	CK_HT_TYPE probes = 0;
-	uint64_t probe_maximum;
+	CK_HT_TYPE probe_maximum;
 
 #ifndef CK_HT_PP
-	uint64_t d = 0;
+	CK_HT_TYPE d = 0;
-	uint64_t d_prime = 0;
+	CK_HT_TYPE d_prime = 0;
 retry:
 #endif
 
@@ -537,11 +536,11 @@ retry:
 			ck_pr_fence_load();
 			snapshot->value = (uintptr_t)ck_pr_load_ptr(&cursor->value);
 #else
-			d = ck_pr_load_64(&map->deletions);
+			d = CK_HT_TYPE_LOAD(&map->deletions);
 			snapshot->key = (uintptr_t)ck_pr_load_ptr(&cursor->key);
 			ck_pr_fence_load();
-			snapshot->key_length = ck_pr_load_64(&cursor->key_length);
+			snapshot->key_length = CK_HT_TYPE_LOAD(&cursor->key_length);
-			snapshot->hash = ck_pr_load_64(&cursor->hash);
+			snapshot->hash = CK_HT_TYPE_LOAD(&cursor->hash);
 			snapshot->value = (uintptr_t)ck_pr_load_ptr(&cursor->value);
 #endif
 
@@ -576,7 +575,7 @@ retry:
 				if (snapshot->hash != h.value)
 					continue;
 
-				d_prime = ck_pr_load_64(&map->deletions);
+				d_prime = CK_HT_TYPE_LOAD(&map->deletions);
 
 				/*
 				 * It is possible that the slot was
@@ -601,12 +600,12 @@ leave:
 	return cursor;
 }
 
-uint64_t
+CK_HT_TYPE
 ck_ht_count(struct ck_ht *table)
 {
 	struct ck_ht_map *map = ck_pr_load_ptr(&table->map);
 
-	return ck_pr_load_64(&map->n_entries);
+	return CK_HT_TYPE_LOAD(&map->n_entries);
 }
 
 bool
@@ -634,7 +633,7 @@ ck_ht_next(struct ck_ht *table,
 }
 
 bool
-ck_ht_reset_size_spmc(struct ck_ht *table, uint64_t size)
+ck_ht_reset_size_spmc(struct ck_ht *table, CK_HT_TYPE size)
 {
 	struct ck_ht_map *map, *update;
 
@@ -657,13 +656,13 @@ ck_ht_reset_spmc(struct ck_ht *table)
 }
 
 bool
-ck_ht_grow_spmc(struct ck_ht *table, uint64_t capacity)
+ck_ht_grow_spmc(struct ck_ht *table, CK_HT_TYPE capacity)
 {
 	struct ck_ht_map *map, *update;
 	struct ck_ht_entry *bucket, *previous;
 	struct ck_ht_hash h;
 	size_t k, i, j, offset;
-	uint64_t probes;
+	CK_HT_TYPE probes;
 
 restart:
 	map = table->map;
@@ -772,7 +771,7 @@ ck_ht_remove_spmc(struct ck_ht *table,
 
 	ck_pr_store_ptr_unsafe(&candidate->key, (void *)CK_HT_KEY_TOMBSTONE);
 	ck_pr_fence_store();
-	ck_pr_store_64(&map->n_entries, map->n_entries - 1);
+	CK_HT_TYPE_STORE(&map->n_entries, map->n_entries - 1);
 	return true;
 }
 
@@ -783,7 +782,7 @@ ck_ht_get_spmc(struct ck_ht *table,
 {
 	struct ck_ht_entry *candidate, snapshot;
 	struct ck_ht_map *map;
-	uint64_t d, d_prime;
+	CK_HT_TYPE d, d_prime;
 
 restart:
 	map = ck_pr_load_ptr(&table->map);
@@ -792,7 +791,7 @@ restart:
 	 * Platforms that cannot read key and key_length atomically must reprobe
 	 * on the scan of any single entry.
 	 */
-	d = ck_pr_load_64(&map->deletions);
+	d = CK_HT_TYPE_LOAD(&map->deletions);
 
 	if (table->mode & CK_HT_MODE_BYTESTRING) {
 		candidate = ck_ht_map_probe_rd(map, h, &snapshot,
@@ -802,7 +801,7 @@ restart:
 		    (void *)entry->key, sizeof(entry->key));
 	}
 
-	d_prime = ck_pr_load_64(&map->deletions);
+	d_prime = CK_HT_TYPE_LOAD(&map->deletions);
 	if (d != d_prime) {
 		/*
 		 * It is possible we have read (K, V'). Only valid states are
@@ -826,7 +825,7 @@ ck_ht_set_spmc(struct ck_ht *table,
 {
 	struct ck_ht_entry snapshot, *candidate, *priority;
 	struct ck_ht_map *map;
-	uint64_t probes, probes_wr;
+	CK_HT_TYPE probes, probes_wr;
 	bool empty = false;
 
 	for (;;) {
@@ -873,8 +872,8 @@ ck_ht_set_spmc(struct ck_ht *table,
 		probes = probes_wr;
 
 #ifndef CK_HT_PP
-		ck_pr_store_64(&priority->key_length, entry->key_length);
+		CK_HT_TYPE_STORE(&priority->key_length, entry->key_length);
-		ck_pr_store_64(&priority->hash, entry->hash);
+		CK_HT_TYPE_STORE(&priority->hash, entry->hash);
 #endif
 
 		/*
@@ -883,7 +882,7 @@ ck_ht_set_spmc(struct ck_ht *table,
 		 * a tombstone.
 		 */
 		if (priority->value == CK_HT_KEY_TOMBSTONE) {
-			ck_pr_store_64(&map->deletions, map->deletions + 1);
+			CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1);
 			ck_pr_fence_store();
 		}
 
@@ -896,7 +895,7 @@ ck_ht_set_spmc(struct ck_ht *table,
 		 * Make sure that readers who observe the tombstone would
 		 * also observe counter change.
 		 */
-		ck_pr_store_64(&map->deletions, map->deletions + 1);
+		CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1);
 		ck_pr_fence_store();
 
 		ck_pr_store_ptr_unsafe(&candidate->key, (void *)CK_HT_KEY_TOMBSTONE);
@@ -913,7 +912,7 @@ ck_ht_set_spmc(struct ck_ht *table,
 
 		if (priority != NULL) {
 			if (priority->key == CK_HT_KEY_TOMBSTONE) {
-				ck_pr_store_64(&map->deletions, map->deletions + 1);
+				CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1);
 				ck_pr_fence_store();
 			}
 
@@ -926,8 +925,8 @@ ck_ht_set_spmc(struct ck_ht *table,
 		ck_pr_fence_store();
 		ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key);
 #else
-		ck_pr_store_64(&candidate->key_length, entry->key_length);
+		CK_HT_TYPE_STORE(&candidate->key_length, entry->key_length);
-		ck_pr_store_64(&candidate->hash, entry->hash);
+		CK_HT_TYPE_STORE(&candidate->hash, entry->hash);
 		ck_pr_store_ptr_unsafe(&candidate->value, (void *)entry->value);
 		ck_pr_fence_store();
 		ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key);
@@ -938,7 +937,7 @@ ck_ht_set_spmc(struct ck_ht *table,
 		 * of entries associated with map.
 		 */
 		if (replace == false)
-			ck_pr_store_64(&map->n_entries, map->n_entries + 1);
+			CK_HT_TYPE_STORE(&map->n_entries, map->n_entries + 1);
 	}
 
 	ck_ht_map_bound_set(map, h, probes);
@@ -963,7 +962,7 @@ ck_ht_put_spmc(struct ck_ht *table,
 {
 	struct ck_ht_entry snapshot, *candidate, *priority;
 	struct ck_ht_map *map;
-	uint64_t probes, probes_wr;
+	CK_HT_TYPE probes, probes_wr;
 
 	for (;;) {
 		map = table->map;
@@ -989,7 +988,7 @@ ck_ht_put_spmc(struct ck_ht *table,
 
 	if (priority != NULL) {
 		/* Version counter is updated before re-use. */
-		ck_pr_store_64(&map->deletions, map->deletions + 1);
+		CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1);
 		ck_pr_fence_store();
 
 		/* Re-use tombstone if one was found. */
@@ -1012,14 +1011,14 @@ ck_ht_put_spmc(struct ck_ht *table,
 	ck_pr_fence_store();
 	ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key);
 #else
-	ck_pr_store_64(&candidate->key_length, entry->key_length);
+	CK_HT_TYPE_STORE(&candidate->key_length, entry->key_length);
-	ck_pr_store_64(&candidate->hash, entry->hash);
+	CK_HT_TYPE_STORE(&candidate->hash, entry->hash);
 	ck_pr_store_ptr_unsafe(&candidate->value, (void *)entry->value);
 	ck_pr_fence_store();
 	ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key);
 #endif
 
-	ck_pr_store_64(&map->n_entries, map->n_entries + 1);
+	CK_HT_TYPE_STORE(&map->n_entries, map->n_entries + 1);
 
 	/* Enforce a load factor of 0.5. */
 	if (map->n_entries * 2 > map->capacity)
@@ -1035,5 +1034,3 @@ ck_ht_destroy(struct ck_ht *table)
 	ck_ht_map_destroy(table->m, table->map, false);
 	return;
 }
-
-#endif /* CK_F_HT */