|
|
|
@ -32,8 +32,8 @@
|
|
|
|
|
#include <ck_pr.h>
|
|
|
|
|
#include <ck_spinlock.h>
|
|
|
|
|
|
|
|
|
|
#ifndef CK_BARRIER_CENTRALIZED
|
|
|
|
|
#define CK_BARRIER_CENTRALIZED
|
|
|
|
|
#ifndef CK_F_BARRIER_CENTRALIZED
|
|
|
|
|
#define CK_F_BARRIER_CENTRALIZED
|
|
|
|
|
|
|
|
|
|
struct ck_barrier_centralized {
|
|
|
|
|
unsigned int value;
|
|
|
|
@ -46,7 +46,7 @@ struct ck_barrier_centralized_state {
|
|
|
|
|
};
|
|
|
|
|
typedef struct ck_barrier_centralized_state ck_barrier_centralized_state_t;
|
|
|
|
|
|
|
|
|
|
#define CK_BARRIER_CENTRALIZED_INITIALIZER {0, 0}
|
|
|
|
|
#define CK_BARRIER_CENTRALIZED_INITIALIZER {0, 0}
|
|
|
|
|
#define CK_BARRIER_CENTRALIZED_STATE_INITIALIZER {0}
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
@ -70,10 +70,10 @@ ck_barrier_centralized(struct ck_barrier_centralized *barrier,
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif /* CK_BARRIER_CENTRALIZED */
|
|
|
|
|
#endif /* CK_F_BARRIER_CENTRALIZED */
|
|
|
|
|
|
|
|
|
|
#ifndef CK_BARRIER_COMBINING
|
|
|
|
|
#define CK_BARRIER_COMBINING
|
|
|
|
|
#ifndef CK_F_BARRIER_COMBINING
|
|
|
|
|
#define CK_F_BARRIER_COMBINING
|
|
|
|
|
|
|
|
|
|
struct ck_barrier_combining_entry {
|
|
|
|
|
unsigned int k;
|
|
|
|
@ -97,32 +97,32 @@ typedef struct ck_barrier_combining_state ck_barrier_combining_state_t;
|
|
|
|
|
|
|
|
|
|
struct ck_barrier_combining {
|
|
|
|
|
struct ck_barrier_combining_entry *root;
|
|
|
|
|
ck_spinlock_cas_t mutex;
|
|
|
|
|
ck_spinlock_fas_t mutex;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
typedef struct ck_barrier_combining ck_barrier_combining_t;
|
|
|
|
|
|
|
|
|
|
struct ck_barrier_ct_queue {
|
|
|
|
|
struct ck_barrier_combining_queue {
|
|
|
|
|
struct ck_barrier_combining_entry *head;
|
|
|
|
|
struct ck_barrier_combining_entry *tail;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
|
ck_barrier_ct_queue_init(struct ck_barrier_ct_queue *queue)
|
|
|
|
|
ck_barrier_combining_queue_init(struct ck_barrier_combining_queue *queue)
|
|
|
|
|
{
|
|
|
|
|
queue->head = queue->tail = NULL;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
|
ck_barrier_ct_queue_enqueue(struct ck_barrier_ct_queue *queue,
|
|
|
|
|
struct ck_barrier_combining_entry *node_value)
|
|
|
|
|
ck_barrier_combining_queue_enqueue(struct ck_barrier_combining_queue *queue,
|
|
|
|
|
struct ck_barrier_combining_entry *node_value)
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
node_value->next = NULL;
|
|
|
|
|
|
|
|
|
|
if (queue->head == NULL) {
|
|
|
|
|
queue->head = queue->tail = node_value;
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -133,7 +133,7 @@ ck_barrier_ct_queue_enqueue(struct ck_barrier_ct_queue *queue,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static struct ck_barrier_combining_entry *
|
|
|
|
|
ck_barrier_ct_queue_dequeue(struct ck_barrier_ct_queue *queue)
|
|
|
|
|
ck_barrier_combining_queue_dequeue(struct ck_barrier_combining_queue *queue)
|
|
|
|
|
{
|
|
|
|
|
struct ck_barrier_combining_entry *front = NULL;
|
|
|
|
|
|
|
|
|
@ -145,24 +145,17 @@ ck_barrier_ct_queue_dequeue(struct ck_barrier_ct_queue *queue)
|
|
|
|
|
return (front);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static bool
|
|
|
|
|
ck_barrier_ct_queue_is_empty(struct ck_barrier_ct_queue *queue)
|
|
|
|
|
{
|
|
|
|
|
return (queue->head == NULL);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
|
ck_barrier_combining_init(struct ck_barrier_combining *root,
|
|
|
|
|
struct ck_barrier_combining_entry *init_root)
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
init_root->k = 0;
|
|
|
|
|
init_root->count = 0;
|
|
|
|
|
init_root->sense = 0;
|
|
|
|
|
init_root->parent = init_root->lchild = init_root->rchild = NULL;
|
|
|
|
|
|
|
|
|
|
ck_spinlock_cas_init(&root->mutex);
|
|
|
|
|
ck_spinlock_fas_init(&root->mutex);
|
|
|
|
|
root->root = init_root;
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -171,6 +164,7 @@ ck_barrier_combining_try_insert(struct ck_barrier_combining_entry *parent,
|
|
|
|
|
struct ck_barrier_combining_entry *tnode,
|
|
|
|
|
struct ck_barrier_combining_entry **child)
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
if (*child == NULL) {
|
|
|
|
|
*child = tnode;
|
|
|
|
|
tnode->parent = parent;
|
|
|
|
@ -187,31 +181,33 @@ ck_barrier_combining_entry_init(struct ck_barrier_combining *root,
|
|
|
|
|
struct ck_barrier_combining_entry *tnode)
|
|
|
|
|
{
|
|
|
|
|
struct ck_barrier_combining_entry *node;
|
|
|
|
|
struct ck_barrier_ct_queue queue;
|
|
|
|
|
struct ck_barrier_combining_queue queue;
|
|
|
|
|
|
|
|
|
|
ck_barrier_ct_queue_init(&queue);
|
|
|
|
|
ck_barrier_combining_queue_init(&queue);
|
|
|
|
|
|
|
|
|
|
tnode->k = 1;
|
|
|
|
|
tnode->count = 0;
|
|
|
|
|
tnode->sense = 0;
|
|
|
|
|
tnode->lchild = tnode->rchild = NULL;
|
|
|
|
|
|
|
|
|
|
ck_spinlock_cas_lock(&root->mutex);
|
|
|
|
|
|
|
|
|
|
ck_barrier_ct_queue_enqueue(&queue, root->root);
|
|
|
|
|
while (!ck_barrier_ct_queue_is_empty(&queue)) {
|
|
|
|
|
node = ck_barrier_ct_queue_dequeue(&queue);
|
|
|
|
|
if (ck_barrier_combining_try_insert(node, tnode, &node->lchild) == true) {
|
|
|
|
|
ck_spinlock_cas_unlock(&root->mutex);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (ck_barrier_combining_try_insert(node, tnode, &node->rchild) == true) {
|
|
|
|
|
ck_spinlock_cas_unlock(&root->mutex);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
ck_barrier_ct_queue_enqueue(&queue, node->lchild);
|
|
|
|
|
ck_barrier_ct_queue_enqueue(&queue, node->rchild);
|
|
|
|
|
ck_spinlock_fas_lock(&root->mutex);
|
|
|
|
|
ck_barrier_combining_queue_enqueue(&queue, root->root);
|
|
|
|
|
while (queue.head != NULL) {
|
|
|
|
|
node = ck_barrier_combining_queue_dequeue(&queue);
|
|
|
|
|
|
|
|
|
|
if (ck_barrier_combining_try_insert(node, tnode, &node->lchild) == true)
|
|
|
|
|
goto leave;
|
|
|
|
|
|
|
|
|
|
if (ck_barrier_combining_try_insert(node, tnode, &node->rchild) == true)
|
|
|
|
|
goto leave;
|
|
|
|
|
|
|
|
|
|
ck_barrier_combining_queue_enqueue(&queue, node->lchild);
|
|
|
|
|
ck_barrier_combining_queue_enqueue(&queue, node->rchild);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
leave:
|
|
|
|
|
ck_spinlock_fas_unlock(&root->mutex);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
@ -219,6 +215,7 @@ ck_barrier_combining_aux(struct ck_barrier_combining *barrier,
|
|
|
|
|
struct ck_barrier_combining_entry *tnode,
|
|
|
|
|
unsigned int sense)
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
/* Incrementing a leaf's count is unnecessary. */
|
|
|
|
|
if (tnode->lchild == NULL) {
|
|
|
|
|
ck_barrier_combining_aux(barrier, tnode->parent, sense);
|
|
|
|
@ -229,12 +226,13 @@ ck_barrier_combining_aux(struct ck_barrier_combining *barrier,
|
|
|
|
|
if (ck_pr_faa_uint(&tnode->count, 1) == tnode->k - 1) {
|
|
|
|
|
if (tnode->parent != NULL)
|
|
|
|
|
ck_barrier_combining_aux(barrier, tnode->parent, sense);
|
|
|
|
|
|
|
|
|
|
ck_pr_store_uint(&tnode->count, 0);
|
|
|
|
|
ck_pr_store_uint(&tnode->sense, ~tnode->sense);
|
|
|
|
|
} else
|
|
|
|
|
while (sense != ck_pr_load_uint(&tnode->sense)) {
|
|
|
|
|
} else {
|
|
|
|
|
while (sense != ck_pr_load_uint(&tnode->sense))
|
|
|
|
|
ck_pr_stall();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
@ -247,8 +245,7 @@ ck_barrier_combining(struct ck_barrier_combining *barrier,
|
|
|
|
|
ck_barrier_combining_aux(barrier, tnode, state->sense);
|
|
|
|
|
state->sense = ~state->sense;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif /* CK_BARRIER_COMBINING */
|
|
|
|
|
#endif /* CK_F_BARRIER_COMBINING */
|
|
|
|
|
|
|
|
|
|
#endif /* _CK_BARRIER_H */
|
|
|
|
|
|
|
|
|
|