haiwan
/
ck
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ck_bag: Various bug fixes.

Browse Source 
Add necessary load fence to iterator.
Initialize iterator appropriately for empty bags.
Improve unit test.
Fix bag linkage bug for non x86_64 targets.
Fix block accounting on removal.
ck_pring
Abel Mathew 13 years ago
parent f1293b379b
commit eaa8ad1d4d
3 changed files with 124 additions and 78 deletions
Show Stats Download Patch File Download Diff File

35
include/ck_bag.h
Unescape View File

@ -31,6 +31,7 @@
#include <ck_cc.h> #include <ck_cc.h>
#include <ck_pr.h> #include <ck_pr.h>
#include <ck_malloc.h> #include <ck_malloc.h>
#include <ck_stdint.h>
#include <stdbool.h> #include <stdbool.h>
#include <stddef.h> #include <stddef.h>
@ -53,7 +54,7 @@
/* /*
* Growth strategies * Bag growth strategies.
*/ */
enum ck_bag_allocation_strategy { enum ck_bag_allocation_strategy {
CK_BAG_ALLOCATE_GEOMETRIC = 0, CK_BAG_ALLOCATE_GEOMETRIC = 0,
@ -61,9 +62,9 @@ enum ck_bag_allocation_strategy {
}; };
/* /*
* max: max n_entries per block * max: max n_entries per block
* bytes: sizeof(ck_bag_block) + sizeof(flex. array member) * bytes: sizeof(ck_bag_block) + sizeof(flex. array member)
* + inline allocator overhead * + inline allocator overhead
*/ */
struct ck_bag_block_info { struct ck_bag_block_info {
size_t max; size_t max;
@ -92,12 +93,9 @@ struct ck_bag {
struct ck_bag_block *head; struct ck_bag_block *head;
struct ck_bag_block *avail_head; struct ck_bag_block *avail_head;
struct ck_bag_block *avail_tail; struct ck_bag_block *avail_tail;
unsigned int n_entries; unsigned int n_entries;
unsigned int n_blocks; unsigned int n_blocks;
enum ck_bag_allocation_strategy alloc_strat; enum ck_bag_allocation_strategy alloc_strat;
struct ck_bag_block_info info; struct ck_bag_block_info info;
}; };
typedef struct ck_bag ck_bag_t; typedef struct ck_bag ck_bag_t;
@ -109,7 +107,9 @@ struct ck_bag_iterator {
}; };
typedef struct ck_bag_iterator ck_bag_iterator_t; typedef struct ck_bag_iterator ck_bag_iterator_t;
#ifdef __x86_64__
#define CK_BAG_BLOCK_ENTRIES_MASK ((uintptr_t)0xFFFF << 48) #define CK_BAG_BLOCK_ENTRIES_MASK ((uintptr_t)0xFFFF << 48)
#endif
CK_CC_INLINE static struct ck_bag_block * CK_CC_INLINE static struct ck_bag_block *
ck_bag_block_next(struct ck_bag_block *block) ck_bag_block_next(struct ck_bag_block *block)
@ -144,13 +144,10 @@ CK_CC_INLINE static void
ck_bag_iterator_init(ck_bag_iterator_t *iterator, ck_bag_t *bag) ck_bag_iterator_init(ck_bag_iterator_t *iterator, ck_bag_t *bag)
{ {
iterator->index = 0;
iterator->block = ck_pr_load_ptr(&bag->head); iterator->block = ck_pr_load_ptr(&bag->head);
if (iterator->block == NULL) { iterator->index = 0;
iterator->n_entries = 0; if (iterator->block != NULL)
} else {
iterator->n_entries = ck_bag_block_count(iterator->block); iterator->n_entries = ck_bag_block_count(iterator->block);
}
return; return;
} }
@ -158,7 +155,6 @@ ck_bag_iterator_init(ck_bag_iterator_t *iterator, ck_bag_t *bag)
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_bag_next(struct ck_bag_iterator *iterator, void **entry) ck_bag_next(struct ck_bag_iterator *iterator, void **entry)
{ {
uint16_t n_entries;
struct ck_bag_block *next; struct ck_bag_block *next;
if (iterator->block == NULL) if (iterator->block == NULL)
@ -167,20 +163,17 @@ ck_bag_next(struct ck_bag_iterator *iterator, void **entry)
if (iterator->index >= iterator->n_entries) { if (iterator->index >= iterator->n_entries) {
next = ck_pr_load_ptr(&iterator->block->next.ptr); next = ck_pr_load_ptr(&iterator->block->next.ptr);
iterator->block = ck_bag_block_next(next); iterator->block = ck_bag_block_next(next);
n_entries = (iterator->block != NULL) ? if (iterator->block == NULL)
ck_bag_block_count(iterator->block) : 0; return false;
if (n_entries == 0) iterator->n_entries = ck_bag_block_count(iterator->block);
if (iterator->n_entries == 0)
return false; return false;
ck_pr_fence_load();
iterator->index = 0; iterator->index = 0;
iterator->n_entries = n_entries; ck_pr_fence_load();
} }
if (iterator->block == NULL)
return false;
*entry = ck_pr_load_ptr(&iterator->block->array[iterator->index++]); *entry = ck_pr_load_ptr(&iterator->block->array[iterator->index++]);
return true; return true;
} }

89
regressions/ck_bag/validate/order.c
Unescape View File

@ -35,7 +35,6 @@
#include "../../common.h" #include "../../common.h"
#define NUM_READER_THREADS 8 #define NUM_READER_THREADS 8
#define WRITER_MAX (1 << 17)
#define READ_LATENCY 8 #define READ_LATENCY 8
static ck_bag_t bag; static ck_bag_t bag;
@ -43,6 +42,7 @@ static ck_epoch_t epoch_bag;
static ck_epoch_record_t epoch_wr; static ck_epoch_record_t epoch_wr;
static int leave; static int leave;
static unsigned int barrier; static unsigned int barrier;
static unsigned int writer_max = 131072;
struct bag_epoch { struct bag_epoch {
ck_epoch_entry_t epoch_entry; ck_epoch_entry_t epoch_entry;
@ -106,11 +106,12 @@ reader(void *arg)
* guarantee across the bag. * guarantee across the bag.
*/ */
for (;;) { for (;;) {
ck_epoch_read_begin(&epoch_record);
ck_bag_iterator_init(&iterator, &bag); ck_bag_iterator_init(&iterator, &bag);
curr_max = prev_max = prev = -1; curr_max = prev_max = prev = -1;
block = NULL;
while (ck_bag_next(&iterator, &curr_ptr)) { while (ck_bag_next(&iterator, &curr_ptr) == true) {
if (block != iterator.block) { if (block != iterator.block) {
prev = -1; prev = -1;
curr = 0; curr = 0;
@ -122,11 +123,13 @@ reader(void *arg)
curr = (uintptr_t)(curr_ptr); curr = (uintptr_t)(curr_ptr);
if (curr < prev) { if (curr < prev) {
/* Ascending order within block violated */ /* Ascending order within block violated */
fprintf(stderr, "%p: %p: %ju\n", (void *)&epoch_record, (void *)iterator.block, (uintmax_t)curr);
fprintf(stderr, "ERROR: %ju < %ju \n", fprintf(stderr, "ERROR: %ju < %ju \n",
(uintmax_t)curr, (uintmax_t)prev); (uintmax_t)curr, (uintmax_t)prev);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} else if (prev_max != -1 && curr > prev_max) { } else if (prev_max != -1 && curr > prev_max) {
/* Max of prev block > max of current block */ /* Max of prev block > max of current block */
fprintf(stderr, "%p: %p: %ju\n", (void *)&epoch_record, (void *)iterator.block, (uintmax_t)curr);
fprintf(stderr, "ERROR: %ju > prev_max: %ju\n", fprintf(stderr, "ERROR: %ju > prev_max: %ju\n",
(uintmax_t)curr, (uintmax_t)prev_max); (uintmax_t)curr, (uintmax_t)prev_max);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -138,6 +141,8 @@ reader(void *arg)
n_entries++; n_entries++;
} }
ck_epoch_read_end(&epoch_record);
iterations++; iterations++;
if (ck_pr_load_int(&leave) == 1) if (ck_pr_load_int(&leave) == 1)
break; break;
@ -146,7 +151,7 @@ reader(void *arg)
fprintf(stderr, "Read %llu entries in %llu iterations.\n", n_entries, iterations); fprintf(stderr, "Read %llu entries in %llu iterations.\n", n_entries, iterations);
ck_pr_inc_uint(&barrier); ck_pr_inc_uint(&barrier);
while (ck_pr_load_uint(&barrier) != NUM_READER_THREADS) while (ck_pr_load_uint(&barrier) != NUM_READER_THREADS + 1)
ck_pr_stall(); ck_pr_stall();
return NULL; return NULL;
@ -157,36 +162,57 @@ static void *
writer_thread(void *unused) writer_thread(void *unused)
{ {
unsigned int i; unsigned int i;
unsigned int iteration = 0;
(void)unused; (void)unused;
for (;;) { for (;;) {
for (i = 0; i < WRITER_MAX; i++) iteration++;
ck_bag_put_spmc(&bag, (void *)(uintptr_t)i); ck_epoch_write_begin(&epoch_wr);
for (i = 1; i <= writer_max; i++) {
if (ck_bag_put_spmc(&bag, (void *)(uintptr_t)i) == false) {
perror("ck_bag_put_spmc");
exit(EXIT_FAILURE);
}
if (ck_bag_member_spmc(&bag, (void *)(uintptr_t)i) == false) {
fprintf(stderr, "ck_bag_put_spmc [%u]: %u\n", iteration, i);
exit(EXIT_FAILURE);
}
}
if (ck_pr_load_int(&leave) == 1) if (ck_pr_load_int(&leave) == 1)
break; break;
for (i = 0; i < WRITER_MAX >> 1; i++) { //fprintf(stderr, "set: %d", iteration);
for (i = 1; i < writer_max; i++) {
void *replace = (void *)(uintptr_t)i; void *replace = (void *)(uintptr_t)i;
if (ck_bag_set_spmc(&bag, (void *)(uintptr_t)i, replace) == false || if (ck_bag_set_spmc(&bag, (void *)(uintptr_t)i, replace) == false) {
replace != (void *)(uintptr_t)i) {
fprintf(stderr, "ERROR: set %ju != %ju", fprintf(stderr, "ERROR: set %ju != %ju",
(uintmax_t)(uintptr_t)replace, (uintmax_t)i); (uintmax_t)(uintptr_t)replace, (uintmax_t)i);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
if (ck_pr_load_int(&leave) == 1) for (i = writer_max; i > 0; i--) {
break; if (ck_bag_member_spmc(&bag, (void *)(uintptr_t)i) == false) {
fprintf(stderr, "ck_bag_member_spmc [%u]: %u\n", iteration, i);
exit(EXIT_FAILURE);
}
if (ck_bag_remove_spmc(&bag, (void *)(uintptr_t)i) == false) {
fprintf(stderr, "ck_bag_remove_spmc [%u]: %u\n", iteration, i);
exit(EXIT_FAILURE);
}
}
for (i = WRITER_MAX; i > 1; i--) ck_epoch_write_end(&epoch_wr);
ck_bag_remove_spmc(&bag, (void *)((uintptr_t)i - 1));
} }
while (ck_pr_load_uint(&barrier) != NUM_READER_THREADS) while (ck_pr_load_uint(&barrier) != NUM_READER_THREADS)
ck_pr_stall(); ck_pr_stall();
ck_pr_inc_uint(&barrier);
return NULL; return NULL;
} }
@ -200,7 +226,7 @@ main(int argc, char **argv)
ck_bag_iterator_t bag_it; ck_bag_iterator_t bag_it;
size_t b = CK_BAG_DEFAULT; size_t b = CK_BAG_DEFAULT;
if (argc == 2) { if (argc >= 2) {
int r = atoi(argv[1]); int r = atoi(argv[1]);
if (r <= 0) { if (r <= 0) {
fprintf(stderr, "# entries in block must be > 0\n"); fprintf(stderr, "# entries in block must be > 0\n");
@ -210,21 +236,46 @@ main(int argc, char **argv)
b = (size_t)r; b = (size_t)r;
} }
if (argc >= 3) {
int r = atoi(argv[2]);
if (r < 16) {
fprintf(stderr, "# entries must be >= 16\n");
exit(EXIT_FAILURE);
}
writer_max = (unsigned int)r;
}
ck_epoch_init(&epoch_bag, 100); ck_epoch_init(&epoch_bag, 100);
ck_epoch_register(&epoch_bag, &epoch_wr); ck_epoch_register(&epoch_bag, &epoch_wr);
ck_bag_allocator_set(&allocator, sizeof(struct bag_epoch)); ck_bag_allocator_set(&allocator, sizeof(struct bag_epoch));
ck_bag_init(&bag, b, CK_BAG_ALLOCATE_GEOMETRIC); ck_bag_init(&bag, b, CK_BAG_ALLOCATE_GEOMETRIC);
fprintf(stderr, "Configuration: %zu bytes/block, %zu entries/block\n", bag.info.bytes, bag.info.max); fprintf(stderr, "Configuration: %u entries, %zu bytes/block, %zu entries/block\n", writer_max, bag.info.bytes, bag.info.max);
i = 1;
/* Basic Test */
ck_bag_put_spmc(&bag, (void *)(uintptr_t)i);
ck_bag_remove_spmc(&bag, (void *)(uintptr_t)i);
ck_bag_put_spmc(&bag, (void *)(uintptr_t)i);
/* Sequential test */ /* Sequential test */
for (i = 0; i < 10; i++) for (i = 1; i <= 10; i++)
ck_bag_put_spmc(&bag, (void *)(uintptr_t)i);
for (i = 1; i <= 10; i++)
ck_bag_remove_spmc(&bag, (void *)(uintptr_t)i);
for (i = 10; i > 0; i--)
ck_bag_remove_spmc(&bag, (void *)(uintptr_t)i);
for (i = 1; i <= 10; i++)
ck_bag_put_spmc(&bag, (void *)(uintptr_t)i); ck_bag_put_spmc(&bag, (void *)(uintptr_t)i);
ck_bag_iterator_init(&bag_it, &bag); ck_bag_iterator_init(&bag_it, &bag);
while (ck_bag_next(&bag_it, &curr_ptr)) { while (ck_bag_next(&bag_it, &curr_ptr)) {
curr = (uintptr_t)(curr_ptr); curr = (uintptr_t)(curr_ptr);
if (curr > (uintptr_t)i) if (curr > (uintptr_t)i)
fprintf(stderr, "ERROR: %ju != %u", (uintmax_t)curr, i); fprintf(stderr, "ERROR: %ju != %u\n", (uintmax_t)curr, i);
ck_bag_remove_spmc(&bag, curr_ptr); ck_bag_remove_spmc(&bag, curr_ptr);
} }
@ -236,14 +287,14 @@ main(int argc, char **argv)
pthread_create(&readers[i], NULL, reader, NULL); pthread_create(&readers[i], NULL, reader, NULL);
} }
sleep(20); sleep(10);
ck_pr_store_int(&leave, 1); ck_pr_store_int(&leave, 1);
for (i = 0; i < NUM_READER_THREADS; i++) for (i = 0; i < NUM_READER_THREADS; i++)
pthread_join(readers[i], NULL); pthread_join(readers[i], NULL);
pthread_join(writer, NULL); pthread_join(writer, NULL);
fprintf(stderr, "Current Entries: %u\n", ck_bag_count(&bag)); fprintf(stderr, "Current entries: %u\n", ck_bag_count(&bag));
ck_bag_destroy(&bag); ck_bag_destroy(&bag);
return 0; return 0;
} }

78
src/ck_bag.c
Unescape View File

@ -34,10 +34,7 @@
#include <stdbool.h> #include <stdbool.h>
#include <string.h> #include <string.h>
#ifndef CK_BAG_PAGESIZE
#define CK_BAG_PAGESIZE CK_MD_PAGESIZE #define CK_BAG_PAGESIZE CK_MD_PAGESIZE
#endif
#define CK_BAG_MAX_N_ENTRIES (1 << 12) #define CK_BAG_MAX_N_ENTRIES (1 << 12)
static struct ck_malloc allocator; static struct ck_malloc allocator;
@ -67,11 +64,9 @@ ck_bag_init(struct ck_bag *bag,
block_overhead = sizeof(struct ck_bag_block) + allocator_overhead; block_overhead = sizeof(struct ck_bag_block) + allocator_overhead;
if (n_block_entries == CK_BAG_DEFAULT) { if (n_block_entries == CK_BAG_DEFAULT) {
bag->info.max = (CK_BAG_PAGESIZE - block_overhead) / sizeof(void *); bag->info.max = ((CK_BAG_PAGESIZE - block_overhead) / sizeof(void *));
} else { } else {
bag->info.max = (n_block_entries * CK_MD_CACHELINE - block_overhead) / sizeof(void *); bag->info.max = ((n_block_entries * CK_MD_CACHELINE - block_overhead) / sizeof(void *));
if (n_block_entries > bag->info.max)
bag->info.max = n_block_entries;
} }
bag->info.bytes = block_overhead + sizeof(void *) * bag->info.max; bag->info.bytes = block_overhead + sizeof(void *) * bag->info.max;
@ -111,7 +106,7 @@ ck_bag_put_spmc(struct ck_bag *bag, void *entry)
struct ck_bag_block *cursor, *new_block, *new_block_prev, *new_tail; struct ck_bag_block *cursor, *new_block, *new_block_prev, *new_tail;
uint16_t n_entries_block; uint16_t n_entries_block;
size_t blocks_alloc, i; size_t blocks_alloc, i;
uintptr_t next; uintptr_t next = 0;
new_block = new_block_prev = new_tail = NULL; new_block = new_block_prev = new_tail = NULL;
@ -142,6 +137,10 @@ ck_bag_put_spmc(struct ck_bag *bag, void *entry)
if (i == 0) if (i == 0)
new_tail = new_block; new_tail = new_block;
#ifndef __x86_64__
new_block->next.n_entries = 0;
#endif
new_block->next.ptr = new_block_prev; new_block->next.ptr = new_block_prev;
new_block->avail_next = new_block_prev; new_block->avail_next = new_block_prev;
if (new_block_prev != NULL) if (new_block_prev != NULL)
@ -162,7 +161,7 @@ ck_bag_put_spmc(struct ck_bag *bag, void *entry)
bag->n_blocks += blocks_alloc; /* n_blocks and n_avail_blocks? */ bag->n_blocks += blocks_alloc; /* n_blocks and n_avail_blocks? */
} }
/* Update the Available List */ /* Update the available list */
if (new_block != NULL) { if (new_block != NULL) {
if (bag->avail_tail != NULL) { if (bag->avail_tail != NULL) {
cursor->avail_prev = bag->avail_tail; cursor->avail_prev = bag->avail_tail;
@ -173,56 +172,59 @@ ck_bag_put_spmc(struct ck_bag *bag, void *entry)
} }
bag->avail_tail = new_tail; bag->avail_tail = new_tail;
} else if (n_entries_block == bag->info.max - 1) { } else if (n_entries_block == bag->info.max-1) {
/* New entry will fill up block, remove from avail list */ /* New entry will fill up block, remove from avail list */
if (cursor->avail_prev != NULL) if (cursor->avail_prev != NULL)
cursor->avail_prev->avail_next = cursor->avail_next; cursor->avail_prev->avail_next = cursor->avail_next;
if (cursor->avail_next != NULL) if (cursor->avail_next != NULL)
cursor->avail_next->avail_prev = cursor->avail_prev; cursor->avail_next->avail_prev = cursor->avail_prev;
if (bag->avail_head == cursor) if (bag->avail_head == cursor)
bag->avail_head = cursor->avail_next; bag->avail_head = cursor->avail_next;
if (bag->avail_tail == cursor) if (bag->avail_tail == cursor)
bag->avail_tail = cursor->avail_prev; bag->avail_tail = cursor->avail_prev;
/* For debugging purposes */ /* For debugging purposes */
cursor->avail_next = NULL; cursor->avail_next = NULL;
cursor->avail_prev = NULL; cursor->avail_prev = NULL;
} }
/* Update array and block->n_entries */ /* Update array and block->n_entries */
cursor->array[n_entries_block++] = entry; cursor->array[n_entries_block++] = entry;
ck_pr_fence_store();
#ifdef __x86_64__ #ifdef __x86_64__
next = ((uintptr_t)n_entries_block << 48); next = ((uintptr_t)n_entries_block << 48);
#endif #endif
ck_pr_fence_store();
/* Update bag's list */ /* Update bag's list */
if (n_entries_block == 1) { if (n_entries_block == 1) {
if (bag->head != NULL) {
#ifdef __x86_64__ #ifdef __x86_64__
if (bag->head != NULL)
next += ((uintptr_t)(void *)ck_bag_block_next(bag->head)); next += ((uintptr_t)(void *)ck_bag_block_next(bag->head));
#else #else
next = (uintptr_t)(void *)ck_bag_block_next(bag->head);
#endif
}
#ifndef __x86_64__
ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block); ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block);
if (bag->head != NULL)
next = (uintptr_t)ck_bag_block_next(bag->head->next.ptr);
else
next = 0;
#endif #endif
ck_pr_store_ptr(&cursor->next.ptr, (void *)next); ck_pr_store_ptr(&cursor->next.ptr, (void *)next);
ck_pr_store_ptr(&bag->head, cursor); ck_pr_store_ptr(&bag->head, cursor);
} else { } else {
#ifdef __x86_64__ #ifdef __x86_64__
next += ((uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr)); next += ((uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr));
ck_pr_store_ptr(&cursor->next, (void *)next); ck_pr_store_ptr(&cursor->next, (void *)next);
#else #else
ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block); ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block);
#endif #endif
} }
ck_pr_store_uint(&bag->n_entries, bag->n_entries + 1); ck_pr_store_uint(&bag->n_entries, bag->n_entries + 1);
@ -230,10 +232,7 @@ ck_bag_put_spmc(struct ck_bag *bag, void *entry)
} }
/* /*
* Set * Replace prev_entry with new entry if exists, otherwise insert into bag.
* Replace prev_entry with new entry if exists, otherwise insert into bag
*
* On return, new_entry = prev_entry on replacement, NULL on insertion.
*/ */
bool bool
ck_bag_set_spmc(struct ck_bag *bag, void *compare, void *update) ck_bag_set_spmc(struct ck_bag *bag, void *compare, void *update)
@ -269,11 +268,13 @@ ck_bag_remove_spmc(struct ck_bag *bag, void *entry)
prev = NULL; prev = NULL;
while (cursor != NULL) { while (cursor != NULL) {
n_entries = ck_bag_block_count(cursor); n_entries = ck_bag_block_count(cursor);
for (block_index = 0; block_index < n_entries; block_index++) { for (block_index = 0; block_index < n_entries; block_index++) {
if (cursor->array[block_index] == entry) if (cursor->array[block_index] == entry)
goto found; goto found;
} }
prev = cursor; prev = cursor;
cursor = ck_bag_block_next(cursor->next.ptr); cursor = ck_bag_block_next(cursor->next.ptr);
} }
@ -293,7 +294,7 @@ found:
next = ((uintptr_t)prev->next.ptr & (CK_BAG_BLOCK_ENTRIES_MASK)) | next = ((uintptr_t)prev->next.ptr & (CK_BAG_BLOCK_ENTRIES_MASK)) |
(uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr); (uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr);
#else #else
next = (uintptr_t)cursor->next.ptr; next = (uintptr_t)(void *)cursor->next.ptr;
#endif #endif
ck_pr_store_ptr(&prev->next.ptr, (struct ck_bag_block *)next); ck_pr_store_ptr(&prev->next.ptr, (struct ck_bag_block *)next);
} }
@ -305,27 +306,28 @@ found:
if (cursor->avail_next != NULL) if (cursor->avail_next != NULL)
cursor->avail_next->avail_prev = cursor->avail_prev; cursor->avail_next->avail_prev = cursor->avail_prev;
bag->n_blocks--;
copy = cursor->avail_next; copy = cursor->avail_next;
} else { } else {
uintptr_t next_ptr; uintptr_t next_ptr;
copy = allocator.malloc(bag->info.bytes);
copy = allocator.malloc(bag->info.bytes);
if (copy == NULL) if (copy == NULL)
return false; return false;
memcpy(copy, cursor, bag->info.bytes); memcpy(copy, cursor, bag->info.bytes);
copy->array[block_index] = copy->array[--n_entries]; copy->array[block_index] = copy->array[--n_entries];
ck_pr_fence_store();
next_ptr = (uintptr_t)(void *)ck_bag_block_next(copy->next.ptr); next_ptr = (uintptr_t)(void *)ck_bag_block_next(copy->next.ptr);
#ifdef __x86_64__ #ifdef __x86_64__
copy->next.ptr = (void *)(((uintptr_t)n_entries << 48) | next_ptr); copy->next.ptr = (void *)(((uintptr_t)n_entries << 48) | next_ptr);
#else #else
copy->next.n_entries = n_entries; copy->next.n_entries = n_entries;
copy->next.ptr = (void *)next_ptr; copy->next.ptr = (struct ck_bag_block *)next_ptr;
#endif #endif
ck_pr_fence_store();
if (prev == NULL) { if (prev == NULL) {
ck_pr_store_ptr(&bag->head, copy); ck_pr_store_ptr(&bag->head, copy);
} else { } else {

Write Preview
Loading…
Cancel
Save