Merge remote-tracking branch 'upstream/master'

configure

@@ -93,6 +93,8 @@ generate()
 	    -e "s#@GZIP@#$GZIP#g"				\
 	    -e "s#@GZIP_SUFFIX@#$GZIP_SUFFIX#g"			\
 	    -e "s#@POINTER_PACK_ENABLE@#$POINTER_PACK_ENABLE#g"	\
+	    -e "s#@VMA_BITS@#$VMA_BITS_R#g"			\
+	    -e "s#@VMA_BITS_VALUE@#$VMA_BITS_VALUE_R#g"		\
 		$1 > $2
 }
 
@@ -109,8 +111,9 @@ generate_stdout()
 	echo "                LD = $LD"
 	echo "           LDFLAGS = $LDFLAGS"
 	echo "              GZIP = $GZIP"
-	echo "      POINTER_PACK = $POINTER_PACK_ENABLE"
 	echo "             CORES = $CORES"
+	echo "      POINTER_PACK = $POINTER_PACK_ENABLE"
+	echo "          VMA_BITS = $VMA_BITS"
 	echo
 	echo "Headers will be installed in $HEADERS"
 	echo "Libraries will be installed in $LIBRARY"
@@ -136,6 +139,7 @@ for option in $*; do
 		echo "The following options will modify code generation."
 		echo "  --cores=N                Specify number of cores available on target machine"
 		echo "  --enable-pointer-packing Assumes 48-bit address encoding"
+		echo "  --vma-bits=N             Specify valid number of VMA bits"
 		echo
 		echo "The following environment variables may be used:"
 		echo "   CC       C compiler command"
@@ -146,6 +150,9 @@ for option in $*; do
 		echo "Report bugs to ${MAINTAINER}."
 		exit $EXIT_SUCCESS
 		;;
+	--vma-bits=*)
+		VMA_BITS=$value
+		;;
 	--enable-pointer-packing)
 		POINTER_PACK_ENABLE="CK_MD_POINTER_PACK_ENABLE"
 		;;
@@ -336,6 +343,38 @@ esac
 
 assert "$PLATFORM" "$PLATFORM" "unsupported"
 
+printf "Detecting VMA bits..............."
+if test "$VMA_BITS" = ""; then
+	if test "$PLATFORM" = "x86" || test $PLATFORM = "x86_64"; then
+		case $SYSTEM in
+			darwin)
+				VMA=`sysctl -n machdep.cpu.address_bits.virtual`
+				;;
+			linux)
+				VMA=`awk '/address sizes/ {print $7;exit}' /proc/cpuinfo`
+				;;
+			*)
+				VMA="unknown"
+				;;
+		esac
+	fi
+
+	VMA_BITS=$VMA
+else
+	VMA=$VMA_BITS
+fi
+
+if test "$VMA" = "unknown"; then
+	echo "unknown"
+	VMA_BITS_R="CK_MD_VMA_BITS_UNKNOWN"
+	VMA_BITS_VALUE_R=""
+	POINTER_PACK_ENABLE="CK_MD_POINTER_PACK_DISABLE"
+else
+	echo "success [$VMA]"
+	VMA_BITS_R="CK_MD_VMA_BITS"
+	VMA_BITS_VALUE_R="${VMA_BITS}ULL"
+fi
+
 # Platform will be used as a macro.
 PROFILE="${PROFILE:-$PLATFORM}"
 PLATFORM="__${PLATFORM}__"
@@ -380,7 +419,6 @@ if test -z "$CC" -o ! -x "$CC"; then
 fi
 assert "$CC" "not found"
 
-# Make sure GCC 4.X, the only supported compiler, is being used.
 cat << EOF > .1.c
 #include <stdio.h>
 int main(void) {

doc/ck_ht_entry_key_set

@@ -44,6 +44,9 @@ with a key pointed to by the
 .Fa key
 argument. The length of the key is specified by
 .Fa key_length.
+The maximum value of
+.Fa key_length
+is defined by the CK_HT_KEY_LENGTH macro.
 This function is typically used to initialize an
 entry for
 .Xr ck_ht_get_spmc 3

doc/ck_ht_entry_set

@@ -46,6 +46,9 @@ argument and a value pointed to by the
 .Fa value
 argument. The length of the key is specified by
 .Fa key_length.
+The maximum value of
+.Fa key_length
+is defined by the CK_HT_KEY_LENGTH macro.
 This function is typically used to initialize an
 entry for
 .Xr ck_ht_set_spmc 3

include/ck_epoch.h

@@ -93,11 +93,20 @@ ck_epoch_begin(ck_epoch_t *epoch, ck_epoch_record_t *record)
 	 */
 	if (record->active == 0) {
 		unsigned int g_epoch = ck_pr_load_uint(&epoch->epoch);
+
+		/*
+		 * It is possible for loads to be re-ordered before the store
+		 * is committed into the caller's epoch and active fields.
+		 * Execute a full barrier to serialize stores with respect to
+		 * loads
+		 */
 		ck_pr_store_uint(&record->epoch, g_epoch);
+		ck_pr_store_uint(&record->active, 1);
+		ck_pr_fence_strict_memory();
+		return;
 	}
 
 	ck_pr_store_uint(&record->active, record->active + 1);
-	ck_pr_fence_memory();
 	return;
 }
 

include/ck_ht.h

@@ -49,8 +49,12 @@ enum ck_ht_mode {
 	CK_HT_MODE_BYTESTRING
 };
 
-#if defined(__x86_64__) && defined(CK_MD_POINTER_PACK_ENABLE)
+#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS)
 #define CK_HT_PP
+#define CK_HT_KEY_LENGTH ((sizeof(void *) * 8) - CK_MD_VMA_BITS)
+#define CK_HT_KEY_MASK   ((1U << CK_HT_KEY_LENGTH) - 1)
+#else
+#define CK_HT_KEY_LENGTH 65535U
 #endif
 
 struct ck_ht_entry {
@@ -132,7 +136,7 @@ ck_ht_entry_key_set(ck_ht_entry_t *entry, const void *key, uint16_t key_length)
 {
 
 #ifdef CK_HT_PP
-	entry->key = (uintptr_t)key | ((uintptr_t)key_length << 48);
+	entry->key = (uintptr_t)key | ((uintptr_t)key_length << CK_MD_VMA_BITS);
 #else
 	entry->key = (uintptr_t)key;
 	entry->key_length = key_length;
@@ -146,7 +150,7 @@ ck_ht_entry_key(ck_ht_entry_t *entry)
 {
 
 #ifdef CK_HT_PP
-	return (void *)(entry->key & (((uintptr_t)1 << 48) - 1));
+	return (void *)(entry->key & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1));
 #else
 	return (void *)entry->key;
 #endif
@@ -157,7 +161,7 @@ ck_ht_entry_key_length(ck_ht_entry_t *entry)
 {
 
 #ifdef CK_HT_PP
-	return entry->key >> 48;
+	return entry->key >> CK_MD_VMA_BITS;
 #else
 	return entry->key_length;
 #endif
@@ -168,7 +172,7 @@ ck_ht_entry_value(ck_ht_entry_t *entry)
 {
 
 #ifdef CK_HT_PP
-	return (void *)(entry->value & (((uintptr_t)1 << 48) - 1));
+	return (void *)(entry->value & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1));
 #else
 	return (void *)entry->value;
 #endif
@@ -183,8 +187,8 @@ ck_ht_entry_set(struct ck_ht_entry *entry,
 {
 
 #ifdef CK_HT_PP
-	entry->key = (uintptr_t)key | ((uintptr_t)key_length << 48);
+	entry->key = (uintptr_t)key | ((uintptr_t)key_length << CK_MD_VMA_BITS);
-	entry->value = (uintptr_t)value | ((uintptr_t)(h.value >> 32) << 48);
+	entry->value = (uintptr_t)value | ((uintptr_t)(h.value >> 32) << CK_MD_VMA_BITS);
 #else
 	entry->key = (uintptr_t)key;
 	entry->value = (uintptr_t)value;

include/ck_md.h.in

@@ -39,4 +39,8 @@
 #define @POINTER_PACK_ENABLE@
 #endif
 
+#ifndef @VMA_BITS@ 
+#define @VMA_BITS@ @VMA_BITS_VALUE@
+#endif
+
 #endif /* _CK_MD_H */

regressions/ck_epoch/validate/ck_epoch_poll.c

@@ -45,17 +45,20 @@
 
 #include "../../common.h"
 
+static unsigned int n_rd;
+static unsigned int n_wr;
 static unsigned int n_threads;
 static unsigned int barrier;
 static unsigned int e_barrier;
 static unsigned int readers;
+static unsigned int writers;
 
-#ifndef PAIRS
+#ifndef PAIRS_S
-#define PAIRS 100000
+#define PAIRS_S 100000
 #endif
 
-#ifndef ITERATE
+#ifndef ITERATE_S
-#define ITERATE 20
+#define ITERATE_S 20
 #endif
 
 struct node {
@@ -83,7 +86,7 @@ static void *
 read_thread(void *unused CK_CC_UNUSED)
 {
 	unsigned int j;
-	ck_epoch_record_t record;
+	ck_epoch_record_t record CK_CC_CACHELINE;
 	ck_stack_entry_t *cursor;
 
 	/*
@@ -138,10 +141,10 @@ read_thread(void *unused CK_CC_UNUSED)
 }
 
 static void *
-thread(void *unused CK_CC_UNUSED)
+write_thread(void *unused CK_CC_UNUSED)
 {
 	struct node **entry, *e;
-	unsigned int i, j;
+	unsigned int i, j, tid;
 	ck_epoch_record_t record;
 	ck_stack_entry_t *s;
 
@@ -152,17 +155,18 @@ thread(void *unused CK_CC_UNUSED)
 		exit(EXIT_FAILURE);
 	}
 
+	tid = ck_pr_faa_uint(&writers, 1);
 	ck_pr_inc_uint(&barrier);
 	while (ck_pr_load_uint(&barrier) < n_threads);
 
-	entry = malloc(sizeof(struct node *) * PAIRS);
+	entry = malloc(sizeof(struct node *) * PAIRS_S);
 	if (entry == NULL) {
 		fprintf(stderr, "Failed allocation.\n");
 		exit(EXIT_FAILURE);
 	}
 
-	for (j = 0; j < ITERATE; j++) {
+	for (j = 0; j < ITERATE_S; j++) {
-		for (i = 0; i < PAIRS; i++) {
+		for (i = 0; i < PAIRS_S; i++) {
 			entry[i] = malloc(sizeof(struct node));
 			if (entry == NULL) {
 				fprintf(stderr, "Failed individual allocation\n");
@@ -170,34 +174,37 @@ thread(void *unused CK_CC_UNUSED)
 			}
 		}
 
-		for (i = 0; i < PAIRS; i++) {
+		for (i = 0; i < PAIRS_S; i++) {
 			ck_stack_push_upmc(&stack, &entry[i]->stack_entry);
 		}
 
 		while (ck_pr_load_uint(&readers) == 0)
 			ck_pr_stall();
 
-		fprintf(stderr, "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[W] %2.2f: %c", (double)j / ITERATE, animate[i % strlen(animate)]);
+		if (tid == 0) {
+			fprintf(stderr, "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[W] %2.2f: %c",
+			    (double)j / ITERATE_S, animate[i % strlen(animate)]);
+		}
 
-		for (i = 0; i < PAIRS; i++) {
+		for (i = 0; i < PAIRS_S; i++) {
+			ck_epoch_begin(&stack_epoch, &record);
 			s = ck_stack_pop_upmc(&stack);
 			e = stack_container(s);
+			ck_epoch_end(&stack_epoch, &record);
 
 			ck_epoch_call(&stack_epoch, &record, &e->epoch_entry, destructor);
-
-			if (i % 1024)
 			ck_epoch_poll(&stack_epoch, &record);
-
-			if (i % 8192)
-				fprintf(stderr, "\b%c", animate[i % strlen(animate)]);
 		}
 	}
 
 	ck_epoch_synchronize(&stack_epoch, &record);
+
+	if (tid == 0) {
 		fprintf(stderr, "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[W] Peak: %u (%2.2f%%)\n    Reclamations: %lu\n\n",
 			record.n_peak,
-			(double)record.n_peak / ((double)PAIRS * ITERATE) * 100,
+			(double)record.n_peak / ((double)PAIRS_S * ITERATE_S) * 100,
 			record.n_dispatch);
+	}
 
 	ck_pr_inc_uint(&e_barrier);
 	while (ck_pr_load_uint(&e_barrier) < n_threads);
@@ -210,23 +217,27 @@ main(int argc, char *argv[])
 	unsigned int i;
 	pthread_t *threads;
 
-	if (argc != 3) {
+	if (argc != 4) {
-		fprintf(stderr, "Usage: stack <threads> <affinity delta>\n");
+		fprintf(stderr, "Usage: stack <#readers> <#writers> <affinity delta>\n");
 		exit(EXIT_FAILURE);
 	}
 
-	n_threads = atoi(argv[1]);
+	n_rd = atoi(argv[1]);
-	a.delta = atoi(argv[2]);
+	n_wr = atoi(argv[2]);
+	n_threads = n_wr + n_rd;
+
+	a.delta = atoi(argv[3]);
 	a.request = 0;
 
 	threads = malloc(sizeof(pthread_t) * n_threads);
-
 	ck_epoch_init(&stack_epoch);
 
-	for (i = 0; i < n_threads - 1; i++)
+	for (i = 0; i < n_rd; i++)
 		pthread_create(threads + i, NULL, read_thread, NULL);
 
-	pthread_create(threads + i, NULL, thread, NULL);
+	do {
+		pthread_create(threads + i, NULL, write_thread, NULL);
+	} while (++i < n_wr + n_rd);
 
 	for (i = 0; i < n_threads; i++)
 		pthread_join(threads[i], NULL);

src/ck_ht.c

@@ -258,7 +258,7 @@ retry:
 					continue;
 
 #ifdef CK_HT_PP
-				if (cursor->value >> 48 != ((h.value >> 32) & 0xFFFF))
+				if ((cursor->value >> CK_MD_VMA_BITS) != ((h.value >> 32) & CK_HT_KEY_MASK))
 					continue;
 #else
 				if (cursor->hash != h.value)
@@ -369,7 +369,7 @@ retry:
 				if (k != key_length)
 					continue;
 #ifdef CK_HT_PP
-				if (snapshot->value >> 48 != ((h.value >> 32) & 0xFFFF))
+				if ((snapshot->value >> CK_MD_VMA_BITS) != ((h.value >> 32) & CK_HT_KEY_MASK))
 					continue;
 #else
 				if (snapshot->hash != h.value)