|
|
@ -81,7 +81,7 @@ ck_hp_fifo_enqueue_mpmc(ck_hp_record_t *record,
|
|
|
|
for (;;) {
|
|
|
|
for (;;) {
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
ck_hp_set(record, 0, tail);
|
|
|
|
ck_hp_set(record, 0, tail);
|
|
|
|
ck_pr_fence_strict_memory();
|
|
|
|
ck_pr_fence_store_load();
|
|
|
|
if (tail != ck_pr_load_ptr(&fifo->tail))
|
|
|
|
if (tail != ck_pr_load_ptr(&fifo->tail))
|
|
|
|
continue;
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
|
@ -112,7 +112,7 @@ ck_hp_fifo_tryenqueue_mpmc(ck_hp_record_t *record,
|
|
|
|
|
|
|
|
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
ck_hp_set(record, 0, tail);
|
|
|
|
ck_hp_set(record, 0, tail);
|
|
|
|
ck_pr_fence_strict_memory();
|
|
|
|
ck_pr_fence_store_load();
|
|
|
|
if (tail != ck_pr_load_ptr(&fifo->tail))
|
|
|
|
if (tail != ck_pr_load_ptr(&fifo->tail))
|
|
|
|
return false;
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
|
@ -140,13 +140,13 @@ ck_hp_fifo_dequeue_mpmc(ck_hp_record_t *record,
|
|
|
|
ck_pr_fence_load();
|
|
|
|
ck_pr_fence_load();
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
ck_hp_set(record, 0, head);
|
|
|
|
ck_hp_set(record, 0, head);
|
|
|
|
ck_pr_fence_strict_memory();
|
|
|
|
ck_pr_fence_store_load();
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
continue;
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
|
|
next = ck_pr_load_ptr(&head->next);
|
|
|
|
next = ck_pr_load_ptr(&head->next);
|
|
|
|
ck_hp_set(record, 1, next);
|
|
|
|
ck_hp_set(record, 1, next);
|
|
|
|
ck_pr_fence_strict_memory();
|
|
|
|
ck_pr_fence_store_load();
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
continue;
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
|
@ -175,13 +175,13 @@ ck_hp_fifo_trydequeue_mpmc(ck_hp_record_t *record,
|
|
|
|
ck_pr_fence_load();
|
|
|
|
ck_pr_fence_load();
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
tail = ck_pr_load_ptr(&fifo->tail);
|
|
|
|
ck_hp_set(record, 0, head);
|
|
|
|
ck_hp_set(record, 0, head);
|
|
|
|
ck_pr_fence_strict_memory();
|
|
|
|
ck_pr_fence_store_load();
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
return NULL;
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
|
|
|
|
next = ck_pr_load_ptr(&head->next);
|
|
|
|
next = ck_pr_load_ptr(&head->next);
|
|
|
|
ck_hp_set(record, 1, next);
|
|
|
|
ck_hp_set(record, 1, next);
|
|
|
|
ck_pr_fence_strict_memory();
|
|
|
|
ck_pr_fence_store_load();
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
if (head != ck_pr_load_ptr(&fifo->head))
|
|
|
|
return NULL;
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
|
|
|
|