Merge branch 'master' of git.concurrencykit.org:ck

include/ck_hs.h

@@ -35,9 +35,32 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-#define CK_HS_MODE_SPMC     1
-#define CK_HS_MODE_DIRECT   2
-#define CK_HS_MODE_OBJECT   8
+/*
+ * Indicates a single-writer many-reader workload. Mutually
+ * exclusive with CK_HS_MODE_MPMC
+ */
+#define CK_HS_MODE_SPMC		1
+
+/*
+ * Indicates that values to be stored are not pointers but
+ * values. Allows for full precision. Mutually exclusive
+ * with CK_HS_MODE_OBJECT.
+ */
+#define CK_HS_MODE_DIRECT	2
+
+/* 
+ * Indicates that the values to be stored are pointers.
+ * Allows for space optimizations in the presence of pointer
+ * packing. Mutually exclusive with CK_HS_MODE_DIRECT.
+ */
+#define CK_HS_MODE_OBJECT	8
+
+/*
+ * Indicates a delete-heavy workload. This will reduce the
+ * need for garbage collection at the cost of approximately
+ * 12% to 20% increased memory usage.
+ */
+#define CK_HS_MODE_DELETE	16
 
 /* Currently unsupported. */
 #define CK_HS_MODE_MPMC    (void)
@@ -99,6 +122,8 @@ bool ck_hs_set(ck_hs_t *, unsigned long, const void *, void **);
 bool ck_hs_fas(ck_hs_t *, unsigned long, const void *, void **);
 void *ck_hs_remove(ck_hs_t *, unsigned long, const void *);
 bool ck_hs_grow(ck_hs_t *, unsigned long);
+bool ck_hs_rebuild(ck_hs_t *);
+bool ck_hs_gc(ck_hs_t *, unsigned long, unsigned long);
 unsigned long ck_hs_count(ck_hs_t *);
 bool ck_hs_reset(ck_hs_t *);
 bool ck_hs_reset_size(ck_hs_t *, unsigned long);

regressions/ck_hs/benchmark/serial.c

@@ -42,6 +42,7 @@ static ck_hs_t hs;
 static char **keys;
 static size_t keys_length = 0;
 static size_t keys_capacity = 128;
+static unsigned long global_seed;
 
 static void *
 hs_malloc(size_t r)
@@ -85,11 +86,19 @@ hs_compare(const void *previous, const void *compare)
 }
 
 static void
-set_init(void)
+set_destroy(void)
 {
 
-	common_srand48((long int)time(NULL));
-	if (ck_hs_init(&hs, CK_HS_MODE_OBJECT | CK_HS_MODE_SPMC, hs_hash, hs_compare, &my_allocator, 8, common_lrand48()) == false) {
+	ck_hs_destroy(&hs);
+	return;
+}
+
+static void
+set_init(unsigned int size, unsigned int mode)
+{
+
+	if (ck_hs_init(&hs, CK_HS_MODE_OBJECT | CK_HS_MODE_SPMC | mode, hs_hash, hs_compare,
+	    &my_allocator, size, global_seed) == false) {
 		perror("ck_hs_init");
 		exit(EXIT_FAILURE);
 	}
@@ -170,6 +179,22 @@ set_reset(void)
 	return ck_hs_reset(&hs);
 }
 
+static void
+set_gc(void)
+{
+
+	ck_hs_gc(&hs, 0, 0);
+	return;
+}
+
+static void
+set_rebuild(void)
+{
+
+	ck_hs_rebuild(&hs);
+	return;
+}
+
 static void
 keys_shuffle(char **k)
 {
@@ -189,35 +214,21 @@ keys_shuffle(char **k)
 	return;
 }
 
-int
-main(int argc, char *argv[])
+static void
+run_test(const char *file, size_t r, unsigned int size, unsigned int mode)
 {
 	FILE *fp;
 	char buffer[512];
-	size_t i, j, r;
+	size_t i, j;
 	unsigned int d = 0;
-	uint64_t s, e, a, ri, si, ai, sr, rg, sg, ag, sd, ng, ss, sts, su;
+	uint64_t s, e, a, ri, si, ai, sr, rg, sg, ag, sd, ng, ss, sts, su, sgc, sb;
 	struct ck_hs_stat st;
 	char **t;
 
-	r = 20;
-	s = 8;
-	srand(time(NULL));
-
-	if (argc < 2) {
-		ck_error("Usage: ck_hs <dictionary> [<repetitions> <initial size>]\n");
-	}
-
-	if (argc >= 3)
-		r = atoi(argv[2]);
-
-	if (argc >= 4)
-		s = (uint64_t)atoi(argv[3]);
-
 	keys = malloc(sizeof(char *) * keys_capacity);
 	assert(keys != NULL);
 
-	fp = fopen(argv[1], "r");
+	fp = fopen(file, "r");
 	assert(fp != NULL);
 
 	while (fgets(buffer, sizeof(buffer), fp) != NULL) {
@@ -236,7 +247,7 @@ main(int argc, char *argv[])
 	assert(t != NULL);
 	keys = t;
 
-	set_init();
+	set_init(size, mode);
 	for (i = 0; i < keys_length; i++)
 		d += set_insert(keys[i]) == false;
 	ck_hs_stat(&hs, &st);
@@ -244,10 +255,6 @@ main(int argc, char *argv[])
 	fprintf(stderr, "# %zu entries stored, %u duplicates, %u probe.\n",
 	    set_count(), d, st.probe_maximum);
 
-	fprintf(stderr, "#    reverse_insertion serial_insertion random_insertion serial_swap "
-	    "serial_replace reverse_get serial_get random_get serial_remove negative_get tombstone "
-	    "set_unique\n\n");
-
 	a = 0;
 	for (j = 0; j < r; j++) {
 		if (set_reset() == false) {
@@ -407,6 +414,7 @@ main(int argc, char *argv[])
 		if (set_insert(keys[i]) == true)
 			continue;
 
+		free(keys[i]);
 		keys[i] = keys[--keys_length];
 	}
 
@@ -426,6 +434,24 @@ main(int argc, char *argv[])
 	}
 	su = a / (r * keys_length);
 
+	a = 0;
+	for (j = 0; j < r; j++) {
+		s = rdtsc();
+		set_gc();
+		e = rdtsc();
+		a += e - s;
+	}
+	sgc = a / r;
+
+	a = 0;
+	for (j = 0; j < r; j++) {
+		s = rdtsc();
+		set_rebuild();
+		e = rdtsc();
+		a += e - s;
+	}
+	sb = a / r;
+
 	printf("%zu "
 	    "%" PRIu64 " "
 	    "%" PRIu64 " "
@@ -438,8 +464,47 @@ main(int argc, char *argv[])
 	    "%" PRIu64 " "
 	    "%" PRIu64 " "
 	    "%" PRIu64 " "
+	    "%" PRIu64 " "
+	    "%" PRIu64 " "
 	    "%" PRIu64 "\n",
-	    keys_length, ri, si, ai, ss, sr, rg, sg, ag, sd, ng, sts, su);
+	    keys_length, ri, si, ai, ss, sr, rg, sg, ag, sd, ng, sts, su, sgc, sb);
+
+	fclose(fp);
+
+	for (i = 0; i < keys_length; i++) {
+		free(keys[i]);
+	}
+
+	free(keys);
+	keys_length = 0;
+	set_destroy();
+	return;
+}
+
+int
+main(int argc, char *argv[])
+{
+	unsigned int r, size;
+
+	common_srand48((long int)time(NULL));
+	if (argc < 2) {
+		ck_error("Usage: ck_hs <dictionary> [<repetitions> <initial size>]\n");
+	}
+
+	r = 16;
+	if (argc >= 3)
+		r = atoi(argv[2]);
+
+	size = 8;
+	if (argc >= 4)
+		size = atoi(argv[3]);
+
+	global_seed = common_lrand48();
+	run_test(argv[1], r, size, 0);
+	run_test(argv[1], r, size, CK_HS_MODE_DELETE);
+	fprintf(stderr, "#    reverse_insertion serial_insertion random_insertion serial_swap "
+	    "serial_replace reverse_get serial_get random_get serial_remove negative_get tombstone "
+	    "set_unique gc rebuild\n\n");
 
 	return 0;
 }

regressions/ck_hs/validate/serial.c

@@ -86,7 +86,7 @@ hs_compare(const void *previous, const void *compare)
 }
 
 static void
-run_test(unsigned int is)
+run_test(unsigned int is, unsigned int ad)
 {
 	ck_hs_t hs[16];
 	const size_t size = sizeof(hs) / sizeof(*hs);
@@ -94,10 +94,8 @@ run_test(unsigned int is)
 	const char *blob = "#blobs";
 	unsigned long h;
 
-	if (ck_hs_init(&hs[0], CK_HS_MODE_SPMC | CK_HS_MODE_OBJECT, hs_hash, hs_compare, &my_allocator, is, 6602834) == false) {
-		perror("ck_hs_init");
-		exit(EXIT_FAILURE);
-	}
+	if (ck_hs_init(&hs[0], CK_HS_MODE_SPMC | CK_HS_MODE_OBJECT | ad, hs_hash, hs_compare, &my_allocator, is, 6602834) == false)
+		ck_error("ck_hs_init\n");
 
 	for (j = 0; j < size; j++) {
 		for (i = 0; i < sizeof(test) / sizeof(*test); i++) {
@@ -111,6 +109,8 @@ run_test(unsigned int is)
 
 			if (ck_hs_remove(&hs[j], h, test[i]) == false)
 				ck_error("ERROR [%zu]: Failed to remove unique (%s)\n", j, test[i]);
+
+			break;
 		}
 
 		for (i = 0; i < sizeof(test) / sizeof(*test); i++) {
@@ -220,6 +220,15 @@ run_test(unsigned int is)
 
 		if (ck_hs_move(&hs[j + 1], &hs[j], hs_hash, hs_compare, &my_allocator) == false)
 			ck_error("Failed to move hash table");
+
+		if (j & 1) {
+			ck_hs_gc(&hs[j + 1], 0, 0);
+		} else {
+			ck_hs_gc(&hs[j + 1], 26, 26);
+		}
+
+		if (ck_hs_rebuild(&hs[j + 1]) == false)
+			ck_error("Failed to rebuild");
 	}
 
 	return;
@@ -230,8 +239,10 @@ main(void)
 {
 	unsigned int k;
 
-	for (k = 4; k <= 32; k <<= 1) {
-		run_test(k);
+	for (k = 16; k <= 64; k <<= 1) {
+		run_test(k, 0);
+		run_test(k, CK_HS_MODE_DELETE);
+		break;
 	}
 
 	return 0;

src/ck_hs.c

@@ -55,6 +55,25 @@
 #define CK_HS_G		(2)
 #define CK_HS_G_MASK	(CK_HS_G - 1)
 
+#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8)
+#define CK_HS_WORD          uint8_t
+#define CK_HS_WORD_MAX	    UINT8_MAX
+#define CK_HS_STORE(x, y)   ck_pr_store_8(x, y)
+#define CK_HS_LOAD(x)       ck_pr_load_8(x)
+#elif defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_STORE_16)
+#define CK_HS_WORD          uint16_t
+#define CK_HS_WORD_MAX	    UINT16_MAX
+#define CK_HS_STORE(x, y)   ck_pr_store_16(x, y)
+#define CK_HS_LOAD(x)       ck_pr_load_16(x)
+#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32)
+#define CK_HS_WORD          uint32_t
+#define CK_HS_WORD_MAX	    UINT32_MAX
+#define CK_HS_STORE(x, y)   ck_pr_store_32(x, y)
+#define CK_HS_LOAD(x)       ck_pr_load_32(x)
+#else
+#error "ck_hs is not supported on your platform."
+#endif
+
 enum ck_hs_probe_behavior {
 	CK_HS_PROBE = 0,	/* Default behavior. */
 	CK_HS_PROBE_TOMBSTONE	/* Short-circuit on tombstone. */
@@ -70,6 +89,7 @@ struct ck_hs_map {
 	unsigned long n_entries;
 	unsigned long capacity;
 	unsigned long size;
+	CK_HS_WORD *probe_bound;
 	void **entries;
 };
 
@@ -145,11 +165,21 @@ static struct ck_hs_map *
 ck_hs_map_create(struct ck_hs *hs, unsigned long entries)
 {
 	struct ck_hs_map *map;
-	unsigned long size, n_entries, limit;
+	unsigned long size, n_entries, prefix, limit;
 
 	n_entries = ck_internal_power_2(entries);
+	if (n_entries < CK_HS_PROBE_L1)
+		return NULL;
+
 	size = sizeof(struct ck_hs_map) + (sizeof(void *) * n_entries + CK_MD_CACHELINE - 1);
 
+	if (hs->mode & CK_HS_MODE_DELETE) {
+		prefix = sizeof(CK_HS_WORD) * n_entries;
+		size += prefix;
+	} else {
+		prefix = 0;
+	}
+
 	map = hs->m->malloc(size);
 	if (map == NULL)
 		return NULL;
@@ -169,10 +199,19 @@ ck_hs_map_create(struct ck_hs *hs, unsigned long entries)
 	map->n_entries = 0;
 
 	/* Align map allocation to cache line. */
-	map->entries = (void *)(((uintptr_t)(map + 1) + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1));
+	map->entries = (void *)(((uintptr_t)&map[1] + prefix +
+	    CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1));
+
 	memset(map->entries, 0, sizeof(void *) * n_entries);
 	memset(map->generation, 0, sizeof map->generation);
 
+	if (hs->mode & CK_HS_MODE_DELETE) {
+		map->probe_bound = (CK_HS_WORD *)&map[1];
+		memset(map->probe_bound, 0, prefix);
+	} else {
+		map->probe_bound = NULL;
+	}
+
 	/* Commit entries purge with respect to map publication. */
 	ck_pr_fence_store();
 	return map;
@@ -218,6 +257,44 @@ ck_hs_map_probe_next(struct ck_hs_map *map,
 	    (stride | CK_HS_PROBE_L1)) & map->mask;
 }
 
+static inline void
+ck_hs_map_bound_set(struct ck_hs_map *m,
+    unsigned long h,
+    unsigned long n_probes)
+{
+	unsigned long offset = h & m->mask;
+
+	if (n_probes > m->probe_maximum)
+		ck_pr_store_uint(&m->probe_maximum, n_probes);
+
+	if (m->probe_bound != NULL && m->probe_bound[offset] < n_probes) {
+		if (n_probes > CK_HS_WORD_MAX)
+			n_probes = CK_HS_WORD_MAX;
+
+		CK_HS_STORE(&m->probe_bound[offset], n_probes);
+		ck_pr_fence_store();
+	}
+
+	return;
+}
+
+static inline unsigned int
+ck_hs_map_bound_get(struct ck_hs_map *m, unsigned long h)
+{
+	unsigned long offset = h & m->mask;
+	unsigned int r = CK_HS_WORD_MAX;
+
+	if (m->probe_bound != NULL) {
+		r = CK_HS_LOAD(&m->probe_bound[offset]);
+		if (r == CK_HS_WORD_MAX)
+			r = ck_pr_load_uint(&m->probe_maximum);
+	} else {
+		r = ck_pr_load_uint(&m->probe_maximum);
+	}
+
+	return r;
+}
+
 bool
 ck_hs_grow(struct ck_hs *hs,
     unsigned long capacity)
@@ -265,9 +342,7 @@ restart:
 					*cursor = map->entries[k];
 					update->n_entries++;
 
-					if (probes > update->probe_maximum)
-						update->probe_maximum = probes;
-
+					ck_hs_map_bound_set(update, h, probes);
 					break;
 				}
 			}
@@ -294,6 +369,13 @@ restart:
 	return true;
 }
 
+bool
+ck_hs_rebuild(struct ck_hs *hs)
+{
+
+	return ck_hs_grow(hs, hs->map->capacity);
+}
+
 static void **
 ck_hs_map_probe(struct ck_hs *hs,
     struct ck_hs_map *map,
@@ -413,6 +495,79 @@ ck_hs_marshal(unsigned int mode, const void *key, unsigned long h)
 	return insert;
 }
 
+bool
+ck_hs_gc(struct ck_hs *hs, unsigned long cycles, unsigned long seed)
+{
+	unsigned long size, i;
+	struct ck_hs_map *map = hs->map;
+	unsigned int maximum = map->probe_maximum;
+	CK_HS_WORD *bounds = NULL;
+
+	if (cycles == 0 && map->probe_bound != NULL) {
+		size = sizeof(CK_HS_WORD) * map->capacity;
+		bounds = hs->m->malloc(size);
+		if (bounds == NULL)
+			return false;
+
+		memset(bounds, 0, size);
+	}
+
+	for (i = 0; i < map->capacity; i++) {
+		void **first, *object, *entry, **slot;
+		unsigned long n_probes, offset, h;
+
+		entry = map->entries[(i + seed) & map->mask];
+		if (entry == CK_HS_EMPTY || entry == CK_HS_TOMBSTONE)
+			continue;
+
+#ifdef CK_HS_PP
+		if (hs->mode & CK_HS_MODE_OBJECT)
+			entry = CK_HS_VMA(entry);
+#endif
+
+		h = hs->hf(entry, hs->seed);
+		offset = h & map->mask;
+
+		slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, entry, &object,
+		    ck_hs_map_bound_get(map, h), CK_HS_PROBE);
+
+		if (first != NULL) {
+			void *insert = ck_hs_marshal(hs->mode, entry, h);
+
+			ck_pr_store_ptr(first, insert);
+			ck_pr_inc_uint(&map->generation[h & CK_HS_G_MASK]);
+			ck_pr_fence_atomic_store();
+			ck_pr_store_ptr(slot, CK_HS_TOMBSTONE);
+		}
+
+		if (cycles == 0) {
+			if (n_probes > CK_HS_WORD_MAX)
+				n_probes = CK_HS_WORD_MAX;
+
+			if (bounds != NULL && n_probes > bounds[offset])
+				bounds[offset] = n_probes;
+
+			if (n_probes > maximum)
+				maximum = n_probes;
+		} else if (--cycles == 0)
+			break;
+	}
+
+	if (bounds != NULL) { 
+		for (i = 0; i < map->capacity; i++) {
+			if (bounds[i] == 0 && map->probe_bound[i] != 0)
+				continue;
+
+			CK_HS_STORE(&map->probe_bound[i], bounds[i]);
+		}
+
+		hs->m->free(bounds, size, false);
+	}
+
+	ck_pr_store_uint(&map->probe_maximum, maximum);
+	return true;
+}
+
 bool
 ck_hs_fas(struct ck_hs *hs,
     unsigned long h,
@@ -468,9 +623,7 @@ restart:
 		goto restart;
 	}
 
-	if (n_probes > map->probe_maximum)
-		ck_pr_store_uint(&map->probe_maximum, n_probes);
-
+	ck_hs_map_bound_set(map, h, n_probes);
 	insert = ck_hs_marshal(hs->mode, key, h);
 
 	if (first != NULL) {
@@ -534,9 +687,7 @@ restart:
 	if (object != NULL)
 		return false;
 
-	if (n_probes > map->probe_maximum)
-		ck_pr_store_uint(&map->probe_maximum, n_probes);
-
+	ck_hs_map_bound_set(map, h, n_probes);
 	insert = ck_hs_marshal(hs->mode, key, h);
 
 	if (first != NULL) {
@@ -587,7 +738,7 @@ ck_hs_get(struct ck_hs *hs,
 		map = ck_pr_load_ptr(&hs->map);
 		generation = &map->generation[h & CK_HS_G_MASK];
 		g = ck_pr_load_uint(generation);
-		probe = ck_pr_load_uint(&map->probe_maximum);
+		probe  = ck_hs_map_bound_get(map, h);
 		ck_pr_fence_load();
 
 		ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, probe, CK_HS_PROBE);
@@ -608,7 +759,8 @@ ck_hs_remove(struct ck_hs *hs,
 	struct ck_hs_map *map = hs->map;
 	unsigned long n_probes;
 
-	slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_maximum, CK_HS_PROBE);
+	slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object,
+	    ck_hs_map_bound_get(map, h), CK_HS_PROBE);
 	if (object == NULL)
 		return NULL;