diff --git a/include/ck_fifo.h b/include/ck_fifo.h index f3ac243..a0eb548 100644 --- a/include/ck_fifo.h +++ b/include/ck_fifo.h @@ -105,11 +105,9 @@ ck_fifo_spsc_init(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *stub) ck_spinlock_init(&fifo->m_head); ck_spinlock_init(&fifo->m_tail); - ck_pr_store_ptr(&stub->next, NULL); - ck_pr_store_ptr(&fifo->head, stub); - ck_pr_store_ptr(&fifo->tail, stub); - ck_pr_store_ptr(&fifo->head_snapshot, stub); - ck_pr_store_ptr(&fifo->garbage, stub); + + stub->next = NULL; + fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub; return; } @@ -119,11 +117,13 @@ ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo, void *value) { - ck_pr_store_ptr(&entry->value, value); - ck_pr_store_ptr(&entry->next, NULL); - ck_pr_store_ptr(&fifo->tail->next, entry); + entry->value = value; + entry->next = NULL; + + /* If stub->next is visible, guarantee that entry is consistent. */ ck_pr_fence_store(); - ck_pr_store_ptr(&fifo->tail, entry); + ck_pr_store_ptr(&fifo->tail->next, entry); + fifo->tail = entry; return; } @@ -138,15 +138,14 @@ ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value) * If the stub entry does not point to an entry, then the queue is * empty. */ - stub = ck_pr_load_ptr(&fifo->head); - ck_pr_fence_load(); - entry = stub->next; + entry = ck_pr_load_ptr(&fifo->head->next); if (entry == NULL) return (false); - store = ck_pr_load_ptr(&entry->value); - ck_pr_store_ptr(value, store); - ck_pr_store_ptr(&fifo->head, entry); + /* If entry is visible, guarantee store to value is visible. */ + ck_pr_fence_load(); + ck_pr_store_ptr(value, entry->value); + fifo->head = entry; return (true); } @@ -157,21 +156,17 @@ ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value) CK_CC_INLINE static struct ck_fifo_spsc_entry * ck_fifo_spsc_recycle(struct ck_fifo_spsc *fifo) { - struct ck_fifo_spsc_entry *p, *garbage; - - garbage = ck_pr_load_ptr(&fifo->garbage); - p = ck_pr_load_ptr(&fifo->head_snapshot); + struct ck_fifo_spsc_entry *garbage; - if (garbage == p) { - p = ck_pr_load_ptr(&fifo->head); - ck_pr_store_ptr(&fifo->head_snapshot, p); - if (garbage == p) - return (NULL); + if (fifo->head_snapshot == fifo->garbage) { + fifo->head_snapshot = ck_pr_load_ptr(&fifo->head); + if (fifo->head_snapshot == fifo->garbage) + return NULL; } - p = garbage; - ck_pr_store_ptr(&fifo->garbage, garbage->next); - return (p); + garbage = fifo->garbage; + fifo->garbage = garbage->next; + return (garbage); } CK_CC_INLINE static bool