Merge branch 'master' of git.concurrencykit.org:ck

include/ck_fifo.h

@@ -105,11 +105,9 @@ ck_fifo_spsc_init(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *stub)
 
 	ck_spinlock_init(&fifo->m_head);
 	ck_spinlock_init(&fifo->m_tail);
-	ck_pr_store_ptr(&stub->next, NULL);
-	ck_pr_store_ptr(&fifo->head, stub);
-	ck_pr_store_ptr(&fifo->tail, stub);
-	ck_pr_store_ptr(&fifo->head_snapshot, stub);
-	ck_pr_store_ptr(&fifo->garbage, stub);
+
+	stub->next = NULL;
+	fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub;
 	return;
 }
 
@@ -119,33 +117,33 @@ ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo,
 		     void *value)
 {
 
-	ck_pr_store_ptr(&entry->value, value);
-	ck_pr_store_ptr(&entry->next, NULL);
-	ck_pr_store_ptr(&fifo->tail->next, entry);
+	entry->value = value;
+	entry->next = NULL;
+
+	/* If stub->next is visible, guarantee that entry is consistent. */
 	ck_pr_fence_store();
-	ck_pr_store_ptr(&fifo->tail, entry);
+	ck_pr_store_ptr(&fifo->tail->next, entry);
+	fifo->tail = entry;
 	return;
 }
 
 CK_CC_INLINE static bool
 ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value)
 {
-	struct ck_fifo_spsc_entry *stub, *entry;
-	void *store;
+	struct ck_fifo_spsc_entry *entry;
 
 	/*
 	 * The head pointer is guaranteed to always point to a stub entry.
 	 * If the stub entry does not point to an entry, then the queue is
 	 * empty.
 	 */
-	stub = ck_pr_load_ptr(&fifo->head);
-	ck_pr_fence_load();
-	entry = stub->next;
+	entry = ck_pr_load_ptr(&fifo->head->next);
 	if (entry == NULL)
 		return (false);
 
-	store = ck_pr_load_ptr(&entry->value);
-	ck_pr_store_ptr(value, store);
+	/* If entry is visible, guarantee store to value is visible. */
+	ck_pr_fence_load();
+	ck_pr_store_ptr(value, entry->value);
 	ck_pr_store_ptr(&fifo->head, entry);
 	return (true);
 }
@@ -157,21 +155,17 @@ ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value)
 CK_CC_INLINE static struct ck_fifo_spsc_entry *
 ck_fifo_spsc_recycle(struct ck_fifo_spsc *fifo)
 {
-	struct ck_fifo_spsc_entry *p, *garbage;
-
-	garbage = ck_pr_load_ptr(&fifo->garbage);
-	p = ck_pr_load_ptr(&fifo->head_snapshot);
+	struct ck_fifo_spsc_entry *garbage;
 
-	if (garbage == p) {
-		p = ck_pr_load_ptr(&fifo->head);
-		ck_pr_store_ptr(&fifo->head_snapshot, p);
-		if (garbage == p)
-			return (NULL);
+	if (fifo->head_snapshot == fifo->garbage) {
+		fifo->head_snapshot = ck_pr_load_ptr(&fifo->head);
+		if (fifo->head_snapshot == fifo->garbage)
+			return NULL;
 	}
 
-	p = garbage;
-	ck_pr_store_ptr(&fifo->garbage, garbage->next);
-	return (p);
+	garbage = fifo->garbage;
+	fifo->garbage = garbage->next;
+	return (garbage);
 }
 
 CK_CC_INLINE static bool
@@ -222,12 +216,10 @@ CK_CC_INLINE static void
 ck_fifo_mpmc_init(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *stub)
 {
 
-	ck_pr_store_ptr(&fifo->head.pointer, stub);
-	ck_pr_store_ptr(&fifo->head.generation, 0);
-	ck_pr_store_ptr(&fifo->tail.pointer, stub);
-	ck_pr_store_ptr(&fifo->tail.generation, 0);
-	ck_pr_store_ptr(&stub->next.pointer, NULL);
-	ck_pr_store_ptr(&stub->next.generation, 0);
+	stub->next.pointer = NULL;
+	stub->next.generation = NULL;
+	fifo->head.pointer = fifo->tail.pointer = stub;
+	fifo->head.generation = fifo->tail.generation = NULL;
 	return;
 }
 
@@ -282,6 +274,7 @@ ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo,
 
 	/* After a successful insert, forward the tail to the new entry. */
 	update.generation = tail.generation + 1;
+	ck_pr_fence_store();
 	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);
 	return;
 }
@@ -332,6 +325,7 @@ ck_fifo_mpmc_tryenqueue(struct ck_fifo_mpmc *fifo,
 	}
 
 	/* After a successful insert, forward the tail to the new entry. */
+	ck_pr_fence_store();
 	update.generation = tail.generation + 1;
 	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);
 	return true;

include/ck_hp_fifo.h

@@ -93,6 +93,7 @@ ck_hp_fifo_enqueue_mpmc(ck_hp_record_t *record,
 			break;
 	}
 
+	ck_pr_fence_store();
 	ck_pr_cas_ptr(&fifo->tail, tail, entry);
 	return;
 }
@@ -122,6 +123,7 @@ ck_hp_fifo_tryenqueue_mpmc(ck_hp_record_t *record,
 	} else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == false)
 		return false;
 
+	ck_pr_fence_store();
 	ck_pr_cas_ptr(&fifo->tail, tail, entry);
 	return true;
 }

include/ck_stack.h

@@ -56,10 +56,13 @@ ck_stack_push_upmc(struct ck_stack *target, struct ck_stack_entry *entry)
 	struct ck_stack_entry *stack;
 
 	stack = ck_pr_load_ptr(&target->head);
-	ck_pr_store_ptr(&entry->next, stack);
+	entry->next = stack;
+	ck_pr_fence_store();
 
-	while (ck_pr_cas_ptr_value(&target->head, stack, entry, &stack) == false)
-		ck_pr_store_ptr(&entry->next, stack);
+	while (ck_pr_cas_ptr_value(&target->head, stack, entry, &stack) == false) {
+		entry->next = stack;
+		ck_pr_fence_store();
+	}
 
 	return;
 }
@@ -77,7 +80,8 @@ ck_stack_trypush_upmc(struct ck_stack *target, struct ck_stack_entry *entry)
 	struct ck_stack_entry *stack;
 
 	stack = ck_pr_load_ptr(&target->head);
-	ck_pr_store_ptr(&entry->next, stack);
+	entry->next = stack;
+	ck_pr_fence_store();
 
 	return ck_pr_cas_ptr(&target->head, stack, entry);
 }
@@ -91,15 +95,20 @@ ck_stack_trypush_upmc(struct ck_stack *target, struct ck_stack_entry *entry)
 CK_CC_INLINE static struct ck_stack_entry *
 ck_stack_pop_upmc(struct ck_stack *target)
 {
-	struct ck_stack_entry *entry;
+	struct ck_stack_entry *entry, *next;
 
 	entry = ck_pr_load_ptr(&target->head);
 	if (entry == NULL)
 		return (NULL);
 
-	while (ck_pr_cas_ptr_value(&target->head, entry, entry->next, &entry) == false) {
+	ck_pr_fence_load();
+	next = entry->next;
+	while (ck_pr_cas_ptr_value(&target->head, entry, next, &entry) == false) {
 		if (entry == NULL)
 			break;
+
+		ck_pr_fence_load();
+		next = entry->next;
 	}
 
 	return (entry);
@@ -123,6 +132,7 @@ ck_stack_trypop_upmc(struct ck_stack *target, struct ck_stack_entry **r)
 	if (entry == NULL)
 		return false;
 
+	ck_pr_fence_load();
 	if (ck_pr_cas_ptr(&target->head, entry, entry->next) == true) {
 		*r = entry;
 		return true;
@@ -143,6 +153,7 @@ ck_stack_batch_pop_upmc(struct ck_stack *target)
 	struct ck_stack_entry *entry;
 
 	entry = ck_pr_fas_ptr(&target->head, NULL);
+	ck_pr_fence_load();
 	return (entry);
 }
 #endif /* CK_F_STACK_BATCH_POP_UPMC */
@@ -190,6 +201,8 @@ ck_stack_pop_mpmc(struct ck_stack *target)
 	if (original.head == NULL)
 		return (NULL);
 
+	ck_pr_fence_load();
+
 	update.generation = original.generation + 1;
 	update.head = original.head->next;
 
@@ -197,8 +210,10 @@ ck_stack_pop_mpmc(struct ck_stack *target)
 		if (original.head == NULL)
 			return (NULL);
 
-		ck_pr_store_ptr(&update.generation, original.generation + 1);
-		ck_pr_store_ptr(&update.head, original.head->next);
+		update.generation = original.generation + 1;
+
+		ck_pr_fence_load();
+		update.head = original.head->next;
 	}
 
 	return (original.head);
@@ -218,6 +233,7 @@ ck_stack_trypop_mpmc(struct ck_stack *target, struct ck_stack_entry **r)
 		return false;
 
 	update.generation = original.generation + 1;
+	ck_pr_fence_load();
 	update.head = original.head->next;
 
 	if (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == true) {
@@ -316,9 +332,7 @@ ck_stack_init(struct ck_stack *stack)
 {
 
 	stack->head = NULL;
-	stack->generation = 0;
-
-	ck_pr_fence_store();
+	stack->generation = NULL;
 	return;
 }
 

regressions/ck_fifo/validate/Makefile

@@ -10,16 +10,16 @@ check: all
 	./ck_fifo_spsc_iterator
 	./ck_fifo_mpmc_iterator
 
-ck_fifo_spsc: ck_fifo_spsc.c
+ck_fifo_spsc: ck_fifo_spsc.c ../../../include/ck_fifo.h
 	$(CC) $(CFLAGS) -o ck_fifo_spsc ck_fifo_spsc.c
 
-ck_fifo_mpmc: ck_fifo_mpmc.c
+ck_fifo_mpmc: ck_fifo_mpmc.c ../../../include/ck_fifo.h
 	$(CC) $(CFLAGS) -o ck_fifo_mpmc ck_fifo_mpmc.c
 
-ck_fifo_spsc_iterator: ck_fifo_spsc_iterator.c
+ck_fifo_spsc_iterator: ck_fifo_spsc_iterator.c ../../../include/ck_fifo.h
 	$(CC) $(CFLAGS) -o ck_fifo_spsc_iterator ck_fifo_spsc_iterator.c
 
-ck_fifo_mpmc_iterator: ck_fifo_mpmc_iterator.c
+ck_fifo_mpmc_iterator: ck_fifo_mpmc_iterator.c ../../../include/ck_fifo.h
 	$(CC) $(CFLAGS) -o ck_fifo_mpmc_iterator ck_fifo_mpmc_iterator.c
 
 clean:

regressions/ck_stack/validate/Makefile

@@ -3,7 +3,8 @@
 OBJECTS=serial mpnc_push mpmc_push upmc_push spinlock_push spinlock_eb_push \
 	mpmc_pop upmc_pop spinlock_pop spinlock_eb_pop			    \
 	upmc_trypop mpmc_trypop mpmc_trypair				    \
-	mpmc_pair spinlock_pair spinlock_eb_pair pthreads_pair
+	mpmc_pair spinlock_pair spinlock_eb_pair pthreads_pair		    \
+	mpmc_trypush upmc_trypush
 
 all: $(OBJECTS)
 
@@ -18,6 +19,8 @@ check: all
 	./mpnc_push $(CORES) 1 0
 	./mpmc_push $(CORES) 1 0
 	./upmc_push $(CORES) 1 0
+	./mpmc_trypush $(CORES) 1 0
+	./upmc_trypush $(CORES) 1 0
 
 serial: serial.c
 	$(CC) $(CFLAGS) -o serial serial.c