Merge branch 'master' of https://github.com/sbahra/ck

configure

@@ -33,7 +33,7 @@ EXIT_FAILURE=1
 
 P_PWD=`pwd`
 MAINTAINER='sbahra@repnop.org'
-VERSION=${VERSION:-'0.2.18'}
+VERSION=${VERSION:-'0.2.19'}
 VERSION_MAJOR='0'
 BUILD="$PWD/build/ck.build"
 PREFIX=${PREFIX:-"/usr/local"}

doc/Makefile.in

@@ -74,6 +74,7 @@ OBJECTS=ck_ht_count 	  		\
 	ck_hs_get			\
 	ck_hs_put			\
 	ck_hs_set			\
+	ck_hs_fas			\
 	ck_hs_remove			\
 	ck_hs_grow			\
 	ck_hs_count			\

doc/ck_hs_count

@@ -56,6 +56,7 @@ thread.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_reset 3 ,

doc/ck_hs_destroy

@@ -62,6 +62,7 @@ This function is guaranteed not to fail.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_fas

@@ -0,0 +1,94 @@
+.\"
+.\" Copyright 2013 Samy Al Bahra.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"
+.Dd June 20, 2013
+.Dt CK_HS_FAS 3
+.Sh NAME
+.Nm ck_hs_fas
+.Nd fetch and store key in hash set
+.Sh LIBRARY
+Concurrency Kit (libck, \-lck)
+.Sh SYNOPSIS
+.In ck_hs.h
+.Ft bool
+.Fn ck_hs_fas "ck_hs_t *hs" "unsigned long hash" "const void *key" "void **previous"
+.Sh DESCRIPTION
+The
+.Fn ck_hs_fas 3
+function will fetch and store the key specified by the
+.Fa key
+argument in the hash set pointed to by the
+.Fa hs
+argument. The key specified by
+.Fa key
+is expected to have the hash value specified by the
+.Fa hash
+argument (which was previously generated using the
+.Xr CK_HS_HASH 3
+macro).
+.Pp
+If the call to
+.Fn ck_hs_fas 3
+was successful then the key specified by
+.Fa key
+was successfully stored in the hash set pointed to by
+.Fa hs .
+The key must already exist in the hash set, and is
+replaced by 
+.Fa key
+and the previous value is stored into the void pointer
+pointed to by the
+.Fa previous
+argument. If the key does not exist in the hash set
+then the function will return false and the hash set
+is unchanged. This function
+is guaranteed to be stable with respect to memory usage.
+.Sh RETURN VALUES
+Upon successful completion,
+.Fn ck_hs_fas 3
+returns true and otherwise returns false on failure.
+.Sh ERRORS
+Behavior is undefined if
+.Fa key
+or
+.Fa hs
+are uninitialized. 
+.Sh SEE ALSO
+.Xr ck_hs_init 3 ,
+.Xr ck_hs_destroy 3 ,
+.Xr CK_HS_HASH 3 ,
+.Xr ck_hs_iterator_init 3 ,
+.Xr ck_hs_next 3 ,
+.Xr ck_hs_get 3 ,
+.Xr ck_hs_put 3 ,
+.Xr ck_hs_remove 3 ,
+.Xr ck_hs_grow 3 ,
+.Xr ck_hs_count 3 ,
+.Xr ck_hs_reset 3 ,
+.Xr ck_hs_reset_size 3 ,
+.Xr ck_hs_stat 3
+.Pp
+Additional information available at http://concurrencykit.org/

doc/ck_hs_get

@@ -73,6 +73,7 @@ are uninitialized.
 .Xr ck_hs_next 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_grow

@@ -67,6 +67,7 @@ failures.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_count 3 ,
 .Xr ck_hs_reset 3 ,

doc/ck_hs_init

@@ -144,6 +144,7 @@ object.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_iterator_init

@@ -63,6 +63,7 @@ This function will not fail.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_next

@@ -77,6 +77,7 @@ are uninitialized.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_put

@@ -45,7 +45,7 @@ argument in the hash set pointed to by the
 argument. The key specified by
 .Fa key
 is expected to have the hash value specified by the
-.Fa h
+.Fa hash
 argument (which was previously generated using the
 .Xr CK_HS_HASH 3
 macro).
@@ -83,6 +83,7 @@ to accomodate key insertion.
 .Xr ck_hs_next 3 ,
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_remove

@@ -78,6 +78,7 @@ are uninitialized.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,
 .Xr ck_hs_reset 3 ,

doc/ck_hs_reset

@@ -63,6 +63,7 @@ thread.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_reset_size 3 ,
 .Xr ck_hs_grow 3 ,

doc/ck_hs_reset_size

@@ -66,6 +66,7 @@ thread.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_set

@@ -45,7 +45,7 @@ argument in the hash set pointed to by the
 argument. The key specified by
 .Fa key
 is expected to have the hash value specified by the
-.Fa h
+.Fa hash
 argument (which was previously generated using the
 .Xr CK_HS_HASH 3
 macro).
@@ -87,6 +87,7 @@ to accomodate key insertion.
 .Xr ck_hs_next 3 ,
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

doc/ck_hs_stat

@@ -67,6 +67,7 @@ thread.
 .Xr ck_hs_get 3 ,
 .Xr ck_hs_put 3 ,
 .Xr ck_hs_set 3 ,
+.Xr ck_hs_fas 3 ,
 .Xr ck_hs_remove 3 ,
 .Xr ck_hs_grow 3 ,
 .Xr ck_hs_count 3 ,

include/ck_cc.h

@@ -77,4 +77,12 @@
 #define CK_CC_CACHELINE
 #endif
 
+#ifndef CK_CC_LIKELY
+#define CK_CC_LIKELY(x)
+#endif
+
+#ifndef CK_CC_UNLIKELY
+#define CK_CC_UNLIKELY(x)
+#endif
+
 #endif /* _CK_CC_H */

include/ck_hs.h

@@ -94,6 +94,7 @@ void ck_hs_destroy(ck_hs_t *);
 void *ck_hs_get(ck_hs_t *, unsigned long, const void *);
 bool ck_hs_put(ck_hs_t *, unsigned long, const void *);
 bool ck_hs_set(ck_hs_t *, unsigned long, const void *, void **);
+bool ck_hs_fas(ck_hs_t *, unsigned long, const void *, void **);
 void *ck_hs_remove(ck_hs_t *, unsigned long, const void *);
 bool ck_hs_grow(ck_hs_t *, unsigned long);
 unsigned long ck_hs_count(ck_hs_t *);

include/ck_spinlock.h

@@ -492,7 +492,7 @@ ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c)
 		position = CK_SPINLOCK_TICKET_LOAD(&ticket->value) &
 		    CK_SPINLOCK_TICKET_MASK;
 
-		backoff = request - position;
+		backoff = (request - position) & CK_SPINLOCK_TICKET_MASK;
 		backoff <<= c;
 		ck_backoff_eb(&backoff);
 	}
@@ -596,8 +596,7 @@ ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c)
 		if (position == request)
 			break;
 
-		/* Overflow is handled fine, assuming 2s complement. */
-		backoff = (request - position);
+		backoff = request - position;
 		backoff <<= c;
 
 		/*

include/gcc/ck_cc.h

@@ -75,6 +75,12 @@
 #pragma GCC poison malloc free
 #endif
 
+/*
+ * Branch execution hints.
+ */
+#define CK_CC_LIKELY(x) (__builtin_expect(!!(x), 1))
+#define CK_CC_UNLIKELY(x) (__builtin_expect(!!(x), 0))
+
 /*
  * Some compilers are overly strict regarding aliasing semantics.
  * Unfortunately, in many cases it makes more sense to pay aliasing

regressions/ck_hs/benchmark/parallel_bytestring.c

@@ -171,6 +171,16 @@ set_replace(const char *value)
 	return ck_hs_set(&hs, h, value, &previous);
 }
 
+static bool
+set_swap(const char *value)
+{
+	unsigned long h;
+	void *previous;
+
+	h = CK_HS_HASH(&hs, hs_hash, value);
+	return ck_hs_fas(&hs, h, value, &previous);
+}
+
 static void *
 set_get(const char *value)
 {
@@ -205,7 +215,6 @@ set_reset(void)
 	return ck_hs_reset(&hs);
 }
 
-
 static void *
 reader(void *unused)
 {
@@ -469,8 +478,13 @@ main(int argc, char *argv[])
 	for (;;) {
 		repeated++;
 		s = rdtsc();
-		for (i = 0; i < keys_length; i++)
-			set_replace(keys[i]);
+		for (i = 0; i < keys_length; i++) {
+			if (i & 1) {
+				set_replace(keys[i]);
+			} else {
+				set_swap(keys[i]);
+			}
+		}
 		e = rdtsc();
 		a += e - s;
 
@@ -539,11 +553,19 @@ main(int argc, char *argv[])
 				delete = common_drand48();
 				if (delete <= p_d)
 					set_remove(keys[i]);
+			} else {
+				delete = 0.0;
 			}
+
 			if (p_r != 0.0) {
 				replace = common_drand48();
-				if (replace <= p_r)
-					set_replace(keys[i]);
+				if (replace <= p_r) {
+					if ((i & 1) || (delete <= p_d)) {
+						set_replace(keys[i]);
+					} else {
+						set_swap(keys[i]);
+					}
+				}
 			}
 		}
 		e = rdtsc();

regressions/ck_hs/benchmark/serial.c

@@ -107,6 +107,16 @@ set_remove(const char *value)
 	return true;
 }
 
+static bool
+set_swap(const char *value)
+{
+	unsigned long h;
+	void *previous;
+
+	h = CK_HS_HASH(&hs, hs_hash, value);
+	return ck_hs_fas(&hs, h, value, &previous);
+}
+
 static bool
 set_replace(const char *value)
 {
@@ -178,7 +188,7 @@ main(int argc, char *argv[])
 	char buffer[512];
 	size_t i, j, r;
 	unsigned int d = 0;
-	uint64_t s, e, a, ri, si, ai, sr, rg, sg, ag, sd, ng;
+	uint64_t s, e, a, ri, si, ai, sr, rg, sg, ag, sd, ng, ss;
 	struct ck_hs_stat st;
 	char **t;
 
@@ -226,7 +236,7 @@ main(int argc, char *argv[])
 	fprintf(stderr, "# %zu entries stored, %u duplicates, %u probe.\n",
 	    set_count(), d, st.probe_maximum);
 
-	fprintf(stderr, "#    reverse_insertion serial_insertion random_insertion serial_replace reverse_get serial_get random_get serial_remove negative_get\n\n");
+	fprintf(stderr, "#    reverse_insertion serial_insertion random_insertion serial_swap serial_replace reverse_get serial_get random_get serial_remove negative_get\n\n");
 
 	a = 0;
 	for (j = 0; j < r; j++) {
@@ -272,6 +282,16 @@ main(int argc, char *argv[])
 	}
 	ai = a / (r * keys_length);
 
+	a = 0;
+	for (j = 0; j < r; j++) {
+		s = rdtsc();
+		for (i = 0; i < keys_length; i++)
+			set_swap(keys[i]);
+		e = rdtsc();
+		a += e - s;
+	}
+	ss = a / (r * keys_length);
+
 	a = 0;
 	for (j = 0; j < r; j++) {
 		s = rdtsc();
@@ -360,8 +380,9 @@ main(int argc, char *argv[])
 	    "%" PRIu64 " "
 	    "%" PRIu64 " "
 	    "%" PRIu64 " "
+	    "%" PRIu64 " "
 	    "%" PRIu64 "\n",
-	    keys_length, ri, si, ai, sr, rg, sg, ag, sd, ng);
+	    keys_length, ri, si, ai, ss, sr, rg, sg, ag, sd, ng);
 
 	return 0;
 }

regressions/ck_hs/validate/serial.c

@@ -171,6 +171,18 @@ main(void)
 			    test[i], (char *)r);
 		}
 
+		/* Replacement should succeed. */
+		if (ck_hs_fas(&hs, h, test[i], &r) == false)
+			ck_error("ERROR: ck_hs_fas must succeed.\n");
+
+		if (strcmp(r, test[i]) != 0) {
+			ck_error("ERROR: Incorrect replaced value: %s != %s\n",
+			    test[i], (char *)r);
+		}
+
+		if (ck_hs_fas(&hs, h, negative, &r) == true)
+			ck_error("ERROR: Replacement of negative should fail.\n");
+
 		if (ck_hs_set(&hs, h, test[i], &r) == false) {
 			ck_error("ERROR: Failed to set [1]\n");
 		}

regressions/ck_spinlock/ck_ticket_pb.h

@@ -1,5 +1,5 @@
 #define LOCK_NAME "ck_ticket_pb"
 #define LOCK_DEFINE static ck_spinlock_ticket_t CK_CC_CACHELINE lock = CK_SPINLOCK_TICKET_INITIALIZER
-#define LOCK ck_spinlock_ticket_lock_pb(&lock, 5)
+#define LOCK ck_spinlock_ticket_lock_pb(&lock, 0)
 #define UNLOCK ck_spinlock_ticket_unlock(&lock)
 

src/ck_hs.c

@@ -241,16 +241,18 @@ restart:
 
 		h = hs->hf(previous, hs->seed);
 		offset = h & update->mask;
-		probes = 0;
+		i = probes = 0;
 
-		for (i = 0; i < update->probe_limit; i++) {
+		for (;;) {
 			bucket = (void *)((uintptr_t)&update->entries[offset] & ~(CK_MD_CACHELINE - 1));
 
 			for (j = 0; j < CK_HS_PROBE_L1; j++) {
 				void **cursor = bucket + ((j + offset) & (CK_HS_PROBE_L1 - 1));
 
-				probes++;
-				if (*cursor == CK_HS_EMPTY) {
+				if (probes++ == update->probe_limit)
+					break;
+
+				if (CK_CC_LIKELY(*cursor == CK_HS_EMPTY)) {
 					*cursor = map->entries[k];
 					update->n_entries++;
 
@@ -264,10 +266,10 @@ restart:
 			if (j < CK_HS_PROBE_L1)
 				break;
 
-			offset = ck_hs_map_probe_next(update, offset, h, i, probes);
+			offset = ck_hs_map_probe_next(update, offset, h, i++, probes);
 		}
 
-		if (i == update->probe_limit) {
+		if (probes > update->probe_limit) {
 			/*
 			 * We have hit the probe limit, map needs to be even larger.
 			 */
@@ -296,8 +298,7 @@ ck_hs_map_probe(struct ck_hs *hs,
 	void **bucket, **cursor, *k;
 	const void *compare;
 	void **pr = NULL;
-	unsigned long offset, i, j;
-	unsigned long probes = 0;
+	unsigned long offset, j, i, probes;
 
 #ifdef CK_HS_PP
 	/* If we are storing object pointers, then we may leverage pointer packing. */
@@ -315,13 +316,18 @@ ck_hs_map_probe(struct ck_hs *hs,
 
 	offset = h & map->mask;
 	*object = NULL;
+	i = probes = 0;
 
-	for (i = 0; i < probe_limit; i++) {
+	for (;;) {
 		bucket = (void **)((uintptr_t)&map->entries[offset] & ~(CK_MD_CACHELINE - 1));
 
 		for (j = 0; j < CK_HS_PROBE_L1; j++) {
 			cursor = bucket + ((j + offset) & (CK_HS_PROBE_L1 - 1));
-			probes++;
+
+			if (probes++ == probe_limit) {
+				k = CK_HS_EMPTY;
+				goto leave;
+			}
 
 			k = ck_pr_load_ptr(cursor);
 			if (k == CK_HS_EMPTY)
@@ -355,14 +361,14 @@ ck_hs_map_probe(struct ck_hs *hs,
 				goto leave;
 		}
 
-		offset = ck_hs_map_probe_next(map, offset, h, i, probes);
+		offset = ck_hs_map_probe_next(map, offset, h, i++, probes);
 	}
 
 leave:
-	if (i != probe_limit) {
-		*object = k;
-	} else {
+	if (probes > probe_limit) {
 		cursor = NULL;
+	} else {
+		*object = k;
 	}
 
 	if (pr == NULL)
@@ -372,6 +378,58 @@ leave:
 	return cursor;
 }
 
+static inline void *
+ck_hs_marshal(unsigned int mode, const void *key, unsigned long h)
+{
+	void *insert;
+
+#ifdef CK_HS_PP
+	if (mode & CK_HS_MODE_OBJECT) {
+		insert = (void *)((uintptr_t)key | ((h >> 25) << CK_MD_VMA_BITS));
+	} else {
+		insert = (void *)key;
+	}
+#else
+	(void)mode;
+	(void)h;
+	insert = (void *)key;
+#endif
+
+	return insert;
+}
+
+bool
+ck_hs_fas(struct ck_hs *hs,
+    unsigned long h,
+    const void *key,
+    void **previous)
+{
+	void **slot, **first, *object, *insert;
+	unsigned long n_probes;
+	struct ck_hs_map *map = hs->map;
+
+	*previous = NULL;
+	slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_maximum);
+
+	/* Replacement semantics presume existence. */
+	if (object == NULL)
+		return false;
+
+	insert = ck_hs_marshal(hs->mode, key, h);
+
+	if (first != NULL) {
+		ck_pr_store_ptr(first, insert);
+		ck_pr_inc_uint(&map->generation[h & CK_HS_G_MASK]);
+		ck_pr_fence_atomic_store();
+		ck_pr_store_ptr(slot, CK_HS_TOMBSTONE);
+	} else {
+		ck_pr_store_ptr(slot, insert);
+	}
+
+	*previous = object;
+	return true;
+}
+
 bool
 ck_hs_set(struct ck_hs *hs,
     unsigned long h,
@@ -395,19 +453,11 @@ restart:
 		goto restart;
 	}
 
-#ifdef CK_HS_PP
-	if (hs->mode & CK_HS_MODE_OBJECT) {
-		insert = (void *)((uintptr_t)key | ((h >> 25) << CK_MD_VMA_BITS));
-	} else {
-		insert = (void *)key;
-	}
-#else
-	insert = (void *)key;
-#endif
-
 	if (n_probes > map->probe_maximum)
 		ck_pr_store_uint(&map->probe_maximum, n_probes);
 
+	insert = ck_hs_marshal(hs->mode, key, h);
+
 	if (first != NULL) {
 		/* If an earlier bucket was found, then store entry there. */
 		ck_pr_store_ptr(first, insert);
@@ -419,7 +469,7 @@ restart:
 		 * period if we can guarantee earlier position of
 		 * duplicate key.
 		 */
-		if (slot != NULL && *slot != CK_HS_EMPTY) {
+		if (object != NULL) {
 			ck_pr_inc_uint(&map->generation[h & CK_HS_G_MASK]);
 			ck_pr_fence_atomic_store();
 			ck_pr_store_ptr(slot, CK_HS_TOMBSTONE);
@@ -463,22 +513,14 @@ restart:
 	}
 
 	/* Fail operation if a match was found. */
-	if (slot != NULL && *slot != CK_HS_EMPTY)
+	if (object != NULL)
 		return false;
 
-#ifdef CK_HS_PP
-	if (hs->mode & CK_HS_MODE_OBJECT) {
-		insert = (void *)((uintptr_t)key | ((h >> 25) << CK_MD_VMA_BITS));
-	} else {
-		insert = (void *)key;
-	}
-#else
-	insert = (void *)key;
-#endif
-
 	if (n_probes > map->probe_maximum)
 		ck_pr_store_uint(&map->probe_maximum, n_probes);
 
+	insert = ck_hs_marshal(hs->mode, key, h);
+
 	if (first != NULL) {
 		/* Insert key into first bucket in probe sequence. */
 		ck_pr_store_ptr(first, insert);
@@ -499,7 +541,7 @@ ck_hs_get(struct ck_hs *hs,
     unsigned long h,
     const void *key)
 {
-	void **slot, **first, *object;
+	void **first, *object;
 	struct ck_hs_map *map;
 	unsigned long n_probes;
 	unsigned int g, g_p, probe;
@@ -512,9 +554,7 @@ ck_hs_get(struct ck_hs *hs,
 		probe = ck_pr_load_uint(&map->probe_maximum);
 		ck_pr_fence_load();
 
-		slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, probe);
-		if (slot == NULL)
-			return NULL;
+		ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, probe);
 
 		ck_pr_fence_load();
 		g_p = ck_pr_load_uint(generation);
@@ -533,7 +573,7 @@ ck_hs_remove(struct ck_hs *hs,
 	unsigned long n_probes;
 
 	slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_maximum);
-	if (slot == NULL || object == NULL)
+	if (object == NULL)
 		return NULL;
 
 	ck_pr_store_ptr(slot, CK_HS_TOMBSTONE);