ck_fifo: Add tryenqueue/trydequeue wait-free operations.

Pending regressions tests, taking snapshot while I move to cafe.

include/ck_fifo.h

@@ -285,10 +285,60 @@ ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo,
 	return;
 }
 
+CK_CC_INLINE static bool
+ck_fifo_mpmc_tryenqueue(struct ck_fifo_mpmc *fifo,
+		        struct ck_fifo_mpmc_entry *entry,
+		        void *value)
+{
+	struct ck_fifo_mpmc_pointer tail, next, update;
+
+	entry->value = value;
+	entry->next.pointer = NULL;
+	entry->next.generation = 0;
+
+	ck_pr_fence_store();
+
+	tail.generation = ck_pr_load_ptr(&fifo->tail.generation);
+	tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);
+
+	next.generation = ck_pr_load_ptr(&tail.pointer->next.generation);
+	next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer);
+
+	if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation)
+		return false;
+
+	if (next.pointer != NULL) {
+		/*
+		 * If the tail pointer has an entry following it then
+		 * it needs to be forwarded to the next entry. This
+		 * helps us guarantee we are always operating on the
+		 * last entry.
+		 */
+		update.pointer = next.pointer;
+		update.generation = tail.generation + 1;
+		ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);
+		return false;
+	} else {
+		/*
+		 * Attempt to commit new entry to the end of the
+		 * current tail.
+		 */
+		update.pointer = entry;
+		update.generation = next.generation + 1;
+		if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == false)
+			return false;
+	}
+
+	/* After a successful insert, forward the tail to the new entry. */
+	update.generation = tail.generation + 1;
+	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);
+	return true;
+}
+
 CK_CC_INLINE static bool
 ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo,
 		     void *value,
-		     ck_fifo_mpmc_entry_t **garbage)
+		     struct ck_fifo_mpmc_entry **garbage)
 {
 	struct ck_fifo_mpmc_pointer head, tail, next, update;
 
@@ -330,6 +380,50 @@ ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo,
 	return (true);
 }
 
+CK_CC_INLINE static bool
+ck_fifo_mpmc_trydequeue(struct ck_fifo_mpmc *fifo,
+			void *value,
+			struct ck_fifo_mpmc_entry **garbage)
+{
+	struct ck_fifo_mpmc_pointer head, tail, next, update;
+
+	head.generation = ck_pr_load_ptr(&fifo->head.generation);
+	head.pointer = ck_pr_load_ptr(&fifo->head.pointer);
+
+	tail.generation = ck_pr_load_ptr(&fifo->tail.generation);
+	tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);
+
+	next.generation = ck_pr_load_ptr(&head.pointer->next.generation);
+	next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer);
+
+	update.pointer = next.pointer;
+	if (head.pointer == tail.pointer) {
+		/*
+		 * The head is guaranteed to always point at a stub
+		 * entry. If the stub entry has no references then the
+		 * queue is empty.
+		 */
+		if (next.pointer == NULL)
+			return false;
+
+		/* Forward the tail pointer if necessary. */
+		update.generation = tail.generation + 1;
+		ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);
+		return false;
+	} else {
+		/* Save value before commit. */
+		*(void **)value = ck_pr_load_ptr(&next.pointer->value);
+
+		/* Forward the head pointer to the next entry. */
+		update.generation = head.generation + 1;
+		if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == false)
+			return false;
+	}
+
+	*garbage = head.pointer;
+	return true;
+}
+
 #define CK_FIFO_MPMC_ISEMPTY(f)	((f)->head.pointer->next.pointer == NULL)
 #define CK_FIFO_MPMC_FIRST(f)	((f)->head.pointer->next.pointer)
 #define CK_FIFO_MPMC_NEXT(m)	((m)->next.pointer)