ck_stack: Add wait-free trypop and trypush operations.

Regression tests have been updated to test these operations as
equivalents to regular looping push/pop operations.

include/ck_stack.h

@@ -65,6 +65,24 @@ ck_stack_push_upmc(struct ck_stack *target, struct ck_stack_entry *entry)
 }
 #endif /* CK_F_STACK_PUSH_UPMC */
 
+#ifndef CK_F_STACK_TRYPUSH_UPMC
+#define CK_F_STACK_TRYPUSH_UPMC
+/*
+ * Stack producer operation for multiple unique producers and multiple consumers.
+ * Returns true on success and false on failure.
+ */
+CK_CC_INLINE static bool
+ck_stack_trypush_upmc(struct ck_stack *target, struct ck_stack_entry *entry)
+{
+	struct ck_stack_entry *stack;
+
+	stack = ck_pr_load_ptr(&target->head);
+	ck_pr_store_ptr(&entry->next, stack);
+
+	return ck_pr_cas_ptr(&target->head, stack, entry);
+}
+#endif /* CK_F_STACK_TRYPUSH_UPMC */
+
 #ifndef CK_F_STACK_POP_UPMC
 #define CK_F_STACK_POP_UPMC
 /*
@@ -88,6 +106,32 @@ ck_stack_pop_upmc(struct ck_stack *target)
 }
 #endif
 
+#ifndef CK_F_STACK_TRYPOP_UPMC
+#define CK_F_STACK_TRYPOP_UPMC
+/*
+ * Stack production operation for multiple unique producers and multiple consumers.
+ * Returns true on success and false on failure. The value pointed to by the second
+ * argument is set to a valid ck_stack_entry_t reference if true is returned. If
+ * false is returned, then the value pointed to by the second argument is undefined.
+ */
+CK_CC_INLINE static bool
+ck_stack_trypop_upmc(struct ck_stack *target, struct ck_stack_entry **r)
+{
+	struct ck_stack_entry *entry;
+
+	entry = ck_pr_load_ptr(&target->head);
+	if (entry == NULL)
+		return false;
+
+	if (ck_pr_cas_ptr(&target->head, entry, entry->next) == true) {
+		*r = entry;
+		return true;
+	}
+
+	return false;
+}
+#endif /* CK_F_STACK_TRYPOP_UPMC */
+
 #ifndef CK_F_STACK_BATCH_POP_UPMC
 #define CK_F_STACK_BATCH_POP_UPMC
 /*
@@ -117,6 +161,19 @@ ck_stack_push_mpmc(struct ck_stack *target, struct ck_stack_entry *entry)
 }
 #endif /* CK_F_STACK_PUSH_MPMC */
 
+#ifndef CK_F_STACK_TRYPUSH_MPMC
+#define CK_F_STACK_TRYPUSH_MPMC
+/*
+ * Stack producer operation safe for multiple producers and multiple consumers.
+ */
+CK_CC_INLINE static bool
+ck_stack_trypush_mpmc(struct ck_stack *target, struct ck_stack_entry *entry)
+{
+
+	return ck_stack_trypush_upmc(target, entry);
+}
+#endif /* CK_F_STACK_TRYPUSH_MPMC */
+
 #ifdef CK_F_PR_CAS_PTR_2_VALUE
 #ifndef CK_F_STACK_POP_MPMC
 #define CK_F_STACK_POP_MPMC
@@ -147,6 +204,30 @@ ck_stack_pop_mpmc(struct ck_stack *target)
 	return (original.head);
 }
 #endif /* CK_F_STACK_POP_MPMC */
+
+#ifndef CK_F_STACK_TRYPOP_MPMC
+#define CK_F_STACK_TRYPOP_MPMC
+CK_CC_INLINE static bool
+ck_stack_trypop_mpmc(struct ck_stack *target, struct ck_stack_entry **r)
+{
+	struct ck_stack original, update;
+
+	original.generation = ck_pr_load_ptr(&target->generation);
+	original.head = ck_pr_load_ptr(&target->head);
+	if (original.head == NULL)
+		return false;
+
+	update.generation = original.generation + 1;
+	update.head = original.head->next;
+
+	if (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == true) {
+		*r = original.head;
+		return true;
+	}
+
+	return false;
+}
+#endif /* CK_F_STACK_TRYPOP_MPMC */
 #endif /* CK_F_PR_CAS_PTR_2_VALUE */
 
 #ifndef CK_F_STACK_BATCH_POP_MPMC

regressions/ck_stack/validate/Makefile

@@ -2,6 +2,7 @@
 
 OBJECTS=serial mpnc_push mpmc_push upmc_push spinlock_push spinlock_eb_push \
 	mpmc_pop upmc_pop spinlock_pop spinlock_eb_pop			    \
+	upmc_trypop mpmc_trypop mpmc_trypair				    \
 	mpmc_pair spinlock_pair spinlock_eb_pair pthreads_pair
 
 all: $(OBJECTS)
@@ -9,20 +10,25 @@ all: $(OBJECTS)
 serial: serial.c
 	$(CC) $(CFLAGS) -o serial serial.c
 
-mpnc_push mpmc_push upmc_push spinlock_push spinlock_eb_push: push.c
+mpmc_trypush upmc_trypush mpnc_push mpmc_push upmc_push spinlock_push spinlock_eb_push: push.c
+	$(CC) -DTRYUPMC $(CFLAGS) -o upmc_trypush push.c
+	$(CC) -DTRYMPMC $(CFLAGS) -o mpmc_trypush push.c
 	$(CC) -DMPNC $(CFLAGS) -o mpnc_push push.c
 	$(CC) -DMPMC $(CFLAGS) -o mpmc_push push.c
 	$(CC) -DUPMC $(CFLAGS) -o upmc_push push.c
 	$(CC) -DSPINLOCK $(CFLAGS) -o spinlock_push push.c
 	$(CC) -DSPINLOCK -DEB $(CFLAGS) -o spinlock_eb_push push.c
 
-mpmc_pop upmc_pop spinlock_pop spinlock_eb_pop: pop.c
+upmc_trypop mpmc_trypop mpmc_pop tryupmc_pop upmc_pop spinlock_pop spinlock_eb_pop: pop.c
+	$(CC) -DTRYMPMC $(CFLAGS) -o mpmc_trypop pop.c
+	$(CC) -DTRYUPMC $(CFLAGS) -o upmc_trypop pop.c
 	$(CC) -DMPMC $(CFLAGS) -o mpmc_pop pop.c
 	$(CC) -DUPMC $(CFLAGS) -o upmc_pop pop.c
 	$(CC) -DSPINLOCK $(CFLAGS) -o spinlock_pop pop.c
 	$(CC) -DEB -DSPINLOCK $(CFLAGS) -o spinlock_eb_pop pop.c
 
-mpmc_pair spinlock_pair spinlock_eb_pair pthreads_pair: pair.c
+mpmc_trypair mpmc_pair spinlock_pair spinlock_eb_pair pthreads_pair: pair.c
+	$(CC) -DTRYMPMC $(CFLAGS) -o mpmc_trypair pair.c
 	$(CC) -DMPMC $(CFLAGS) -o mpmc_pair pair.c
 	$(CC) -DSPINLOCK $(CFLAGS) -o spinlock_pair pair.c
 	$(CC) -DEB -DSPINLOCK $(CFLAGS) -o spinlock_eb_pair pair.c

regressions/ck_stack/validate/pair.c

@@ -84,7 +84,7 @@ static unsigned int critical;
 static void *
 stack_thread(void *buffer)
 {
-#if (defined(MPMC) && defined(CK_F_STACK_POP_MPMC)) || (defined(UPMC) && defined(CK_F_STACK_POP_UPMC))
+#if (defined(MPMC) && defined(CK_F_STACK_POP_MPMC)) || (defined(UPMC) && defined(CK_F_STACK_POP_UPMC)) || (defined(TRYUPMC) && defined(CK_F_STACK_TRYPOP_UPMC)) || (defined(TRYMPMC) && defined(CK_F_STACK_TRYPOP_MPMC))
 	ck_stack_entry_t *ref;
 #endif
 	struct entry *entry = buffer;
@@ -102,8 +102,14 @@ stack_thread(void *buffer)
 	for (i = 0; i < n; i++) {
 #if defined(MPMC)
                 ck_stack_push_mpmc(&stack, &entry->next);
+#elif defined(TRYMPMC)
+		while (ck_stack_trypush_mpmc(&stack, &entry->next) == false)
+			ck_pr_stall();
 #elif defined(UPMC)
                 ck_stack_push_upmc(&stack, &entry->next);
+#elif defined(TRYUPMC)
+		while (ck_stack_trypush_upmc(&stack, &entry->next) == false)
+			ck_pr_stall();
 #elif defined(SPINLOCK) || defined(PTHREADS)
 		LOCK(&stack_spinlock);
 		ck_pr_store_ptr(&entry->next, stack);
@@ -124,6 +130,12 @@ stack_thread(void *buffer)
 		ref = ck_stack_pop_mpmc(&stack);
 		entry = getvalue(ref);
 #endif
+#elif defined(TRYMPMC)
+#ifdef CK_F_STACK_TRYPOP_MPMC
+		while (ck_stack_trypop_mpmc(&stack, &ref) == false)
+			ck_pr_stall();
+		entry = getvalue(ref);
+#endif /* CK_F_STACK_TRYPOP_MPMC */
 #elif defined(UPMC)
 		ref = ck_stack_pop_upmc(&stack);
 		entry = getvalue(ref);

regressions/ck_stack/validate/pop.c

@@ -79,7 +79,7 @@ static unsigned int critical;
 static void *
 stack_thread(void *unused CK_CC_UNUSED)
 {
-#if (defined(MPMC) && defined(CK_F_STACK_POP_MPMC)) || (defined(UPMC) && defined(CK_F_STACK_POP_UPMC))
+#if (defined(MPMC) && defined(CK_F_STACK_POP_MPMC)) || (defined(UPMC) && defined(CK_F_STACK_POP_UPMC)) || (defined(TRYMPMC) && defined(CK_F_STACK_TRYPOP_MPMC)) || (defined(TRYUPMC) && defined(CK_F_STACK_TRYPOP_UPMC))
 	ck_stack_entry_t *ref;
 #endif
 	struct entry *entry = NULL;
@@ -100,11 +100,23 @@ stack_thread(void *unused CK_CC_UNUSED)
 		ref = ck_stack_pop_mpmc(&stack);
 		assert(ref);
 		entry = getvalue(ref);
-#endif
+#endif /* CK_F_STACK_POP_MPMC */
+#elif defined(TRYMPMC)
+#ifdef CK_F_STACK_TRYPOP_MPMC
+		while (ck_stack_trypop_mpmc(&stack, &ref) == false)
+			ck_pr_stall();
+		assert(ref);
+		entry = getvalue(ref);
+#endif /* CK_F_STACK_TRYPOP_MPMC */
 #elif defined(UPMC)
 		ref = ck_stack_pop_upmc(&stack);
 		assert(ref);
 		entry = getvalue(ref);
+#elif defined(TRYUPMC)
+		while (ck_stack_trypop_upmc(&stack, &ref) == false)
+			ck_pr_stall();
+		assert(ref);
+		entry = getvalue(ref);
 #elif defined(SPINLOCK)
 		LOCK(&stack_spinlock);
 		entry = stack;

regressions/ck_stack/validate/push.c

@@ -104,6 +104,12 @@ stack_thread(void *buffer)
 		ck_stack_push_mpnc(&stack, &bucket[i].next);
 #elif defined(MPMC)
 		ck_stack_push_mpmc(&stack, &bucket[i].next);
+#elif defined(TRYMPMC)
+		while (ck_stack_trypush_mpmc(&stack, &bucket[i].next) == false)
+			ck_pr_stall();
+#elif defined(TRYUPMC)
+		while (ck_stack_trypush_upmc(&stack, &bucket[i].next) == false)
+			ck_pr_stall();
 #elif defined(UPMC)
 		ck_stack_push_upmc(&stack, &bucket[i].next);
 #elif defined(SPINLOCK) || defined(PTHREADS)