diff --git a/include/ck_barrier.h b/include/ck_barrier.h index ba1b2b3..c3bf059 100644 --- a/include/ck_barrier.h +++ b/include/ck_barrier.h @@ -133,8 +133,11 @@ struct ck_barrier_tournament_state { }; typedef struct ck_barrier_tournament_state ck_barrier_tournament_state_t; -void ck_barrier_tournament_subscribe(ck_barrier_tournament_t *, ck_barrier_tournament_state_t *); -void ck_barrier_tournament_init(ck_barrier_tournament_t *, ck_barrier_tournament_round_t **, unsigned int); +void ck_barrier_tournament_subscribe(ck_barrier_tournament_t *, + ck_barrier_tournament_state_t *); +void ck_barrier_tournament_init(ck_barrier_tournament_t *, + ck_barrier_tournament_round_t **, + unsigned int); unsigned int ck_barrier_tournament_size(unsigned int); void ck_barrier_tournament(ck_barrier_tournament_t *, ck_barrier_tournament_state_t *); diff --git a/include/ck_epoch.h b/include/ck_epoch.h index e8923aa..c223d4c 100644 --- a/include/ck_epoch.h +++ b/include/ck_epoch.h @@ -154,10 +154,12 @@ ck_epoch_update(struct ck_epoch *global, struct ck_epoch_record *record) CK_STACK_FOREACH(&global->records, cursor) { c_record = ck_epoch_record_container(cursor); - if (ck_pr_load_uint(&c_record->status) == CK_EPOCH_FREE || c_record == record) + if (ck_pr_load_uint(&c_record->status) == CK_EPOCH_FREE || + c_record == record) continue; - if (ck_pr_load_uint(&c_record->active) == true && ck_pr_load_uint(&c_record->epoch) != g_epoch) + if (ck_pr_load_uint(&c_record->active) == true && + ck_pr_load_uint(&c_record->epoch) != g_epoch) return; } @@ -196,9 +198,10 @@ ck_epoch_start(struct ck_epoch_record *record) unsigned int epoch = record->epoch & (CK_EPOCH_LENGTH - 1); /* - * This means all threads with a potential reference to a hazard pointer - * will have a view as new as or newer than the calling thread. No active - * reference should exist to any object in the record's pending list. + * This means all threads with a potential reference to a + * hazard pointer will have a view as new as or newer than + * the calling thread. No active reference should exist to + * any object in the record's pending list. */ CK_STACK_FOREACH_SAFE(&record->pending[epoch], cursor, next) global->destroy(cursor); diff --git a/include/ck_fifo.h b/include/ck_fifo.h index 42a9a0c..7dd63ee 100644 --- a/include/ck_fifo.h +++ b/include/ck_fifo.h @@ -64,7 +64,9 @@ ck_fifo_spsc_init(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *stub) } CK_CC_INLINE static void -ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *entry, void *value) +ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo, + struct ck_fifo_spsc_entry *entry, + void *value) { ck_pr_store_ptr(&entry->value, value); @@ -172,7 +174,9 @@ ck_fifo_mpmc_init(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *stub) } CK_CC_INLINE static void -ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *entry, void *value) +ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo, + struct ck_fifo_mpmc_entry *entry, + void *value) { struct ck_fifo_mpmc_pointer tail, next, update; ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; @@ -229,7 +233,9 @@ ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *entry } CK_CC_INLINE static bool -ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo, void *value, ck_fifo_mpmc_entry_t **garbage) +ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo, + void *value, + ck_fifo_mpmc_entry_t **garbage) { struct ck_fifo_mpmc_pointer head, tail, next, update; ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; diff --git a/include/ck_pr.h b/include/ck_pr.h index f269a0c..fc39d06 100644 --- a/include/ck_pr.h +++ b/include/ck_pr.h @@ -47,18 +47,21 @@ #error Your platform is unsupported #endif -#define CK_PR_BIN(K, S, M, T, P, C) \ - CK_CC_INLINE static void \ - ck_pr_##K##_##S(M *target, T value) \ - { \ - T previous; \ - C punt; \ - punt = ck_pr_load_##S(target); \ - previous = (T)punt; \ - while (ck_pr_cas_##S##_value(target, (C)previous, (C)(previous P value), &previous) == false) \ - ck_pr_stall(); \ - \ - return; \ +#define CK_PR_BIN(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T value) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P value), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ } #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) @@ -298,18 +301,18 @@ CK_PR_BIN_S(or, 8, uint8_t, |) #undef CK_PR_BIN_S #undef CK_PR_BIN -#define CK_PR_BTX(K, S, M, T, P, C, R) \ - CK_CC_INLINE static bool \ - ck_pr_##K##_##S(M *target, unsigned int offset) \ - { \ - T previous; \ - C punt; \ - punt = ck_pr_load_##S(target); \ - previous = (T)punt; \ - while (ck_pr_cas_##S##_value(target, (C)previous, \ - (C)(previous P (R ((T)1 << offset))), &previous) == false) \ - ck_pr_stall(); \ - return ((previous >> offset) & 1); \ +#define CK_PR_BTX(K, S, M, T, P, C, R) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(M *target, unsigned int offset) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, (C)previous, \ + (C)(previous P (R ((T)1 << offset))), &previous) == false) \ + ck_pr_stall(); \ + return ((previous >> offset) & 1); \ } #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) @@ -431,26 +434,29 @@ CK_PR_BTX_S(bts, 16, uint16_t, |,) #undef CK_PR_BTX_S #undef CK_PR_BTX -#define CK_PR_UNARY(K, X, S, M, T) \ - CK_CC_INLINE static void \ - ck_pr_##K##_##S(M *target) \ - { \ - ck_pr_##X##_##S(target, (T)1); \ - return; \ +#define CK_PR_UNARY(K, X, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + ck_pr_##X##_##S(target, (T)1); \ + return; \ } -#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ - CK_CC_INLINE static void \ - ck_pr_##K##_##S##_zero(M *target, bool *zero) \ - { \ - T previous; \ - C punt; \ - punt = (C)ck_pr_load_##S(target); \ - previous = (T)punt; \ - while (ck_pr_cas_##S##_value(target, (C)previous, (C)(previous P 1), &previous) == false) \ - ck_pr_stall(); \ - *zero = previous == (T)Z; \ - return; \ +#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P 1), \ + &previous) == false) \ + ck_pr_stall(); \ + *zero = previous == (T)Z; \ + return; \ } #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) @@ -653,33 +659,39 @@ CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) #undef CK_PR_UNARY_Z #undef CK_PR_UNARY -#define CK_PR_N(K, S, M, T, P, C) \ - CK_CC_INLINE static void \ - ck_pr_##K##_##S(M *target) \ - { \ - T previous; \ - C punt; \ - punt = (C)ck_pr_load_##S(target); \ - previous = (T)punt; \ - while (ck_pr_cas_##S##_value(target, (C)previous, (C)(P previous), &previous) == false) \ - ck_pr_stall(); \ - \ - return; \ +#define CK_PR_N(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(P previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ } -#define CK_PR_N_Z(S, M, T, C) \ - CK_CC_INLINE static void \ - ck_pr_neg_##S##_zero(M *target, bool *zero) \ - { \ - T previous; \ - C punt; \ - punt = (C)ck_pr_load_##S(target); \ - previous = (T)punt; \ - while (ck_pr_cas_##S##_value(target, (C)previous, (C)(-previous), &previous) == false) \ - ck_pr_stall(); \ - \ - *zero = previous == 0; \ - return; \ +#define CK_PR_N_Z(S, M, T, C) \ + CK_CC_INLINE static void \ + ck_pr_neg_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(-previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + *zero = previous == 0; \ + return; \ } #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) @@ -842,30 +854,36 @@ CK_PR_N_Z_S(8, uint8_t) #undef CK_PR_N_Z #undef CK_PR_N -#define CK_PR_FAA(S, M, T, C) \ - CK_CC_INLINE static C \ - ck_pr_faa_##S(M *target, T delta) \ - { \ - T previous; \ - C punt; \ - punt = (C)ck_pr_load_##S(target); \ - previous = (T)punt; \ - while (ck_pr_cas_##S##_value(target, (C)previous, (C)(previous + delta), &previous) == false) \ - ck_pr_stall(); \ - \ - return ((C)previous); \ +#define CK_PR_FAA(S, M, T, C) \ + CK_CC_INLINE static C \ + ck_pr_faa_##S(M *target, T delta) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous + delta), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return ((C)previous); \ } -#define CK_PR_FAS(S, M, C) \ - CK_CC_INLINE static C \ - ck_pr_fas_##S(M *target, C update) \ - { \ - C previous; \ - previous = ck_pr_load_##S(target); \ - while (ck_pr_cas_##S##_value(target, previous, update, &previous) == false) \ - ck_pr_stall(); \ - \ - return (previous); \ +#define CK_PR_FAS(S, M, C) \ + CK_CC_INLINE static C \ + ck_pr_fas_##S(M *target, C update) \ + { \ + C previous; \ + previous = ck_pr_load_##S(target); \ + while (ck_pr_cas_##S##_value(target, \ + previous, \ + update, \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return (previous); \ } #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) diff --git a/include/ck_spinlock.h b/include/ck_spinlock.h index 5716bc1..1cd7e89 100644 --- a/include/ck_spinlock.h +++ b/include/ck_spinlock.h @@ -107,7 +107,8 @@ ck_spinlock_anderson_lock(struct ck_spinlock_anderson *lock, next = lock->wrap; else next = position + 1; - } while (ck_pr_cas_uint_value(&lock->next, position, next, &position) == false); + } while (ck_pr_cas_uint_value(&lock->next, position, + next, &position) == false); position %= count; } else { @@ -505,7 +506,8 @@ ck_spinlock_mcs_unlock(struct ck_spinlock_mcs **queue, struct ck_spinlock_mcs *n * that we are the current tail. In this case, we may just * mark the spinlock queue as empty. */ - if (ck_pr_load_ptr(queue) == node && ck_pr_cas_ptr(queue, node, NULL) == true) + if (ck_pr_load_ptr(queue) == node && + ck_pr_cas_ptr(queue, node, NULL) == true) return; /* diff --git a/include/gcc/ck_pr.h b/include/gcc/ck_pr.h index 5cb5e92..1f1acf4 100644 --- a/include/gcc/ck_pr.h +++ b/include/gcc/ck_pr.h @@ -111,13 +111,13 @@ CK_PR_FENCE(memory) /* * Atomic compare and swap. */ -#define CK_PR_CAS(S, M, T) \ - CK_CC_INLINE static bool \ - ck_pr_cas_##S(M *target, T compare, T set) \ - { \ - bool z; \ - z = __sync_bool_compare_and_swap((T *)target, compare, set); \ - return z; \ +#define CK_PR_CAS(S, M, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + z = __sync_bool_compare_and_swap((T *)target, compare, set); \ + return z; \ } CK_PR_CAS(ptr, void, void *) @@ -146,13 +146,13 @@ ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *v) return (set == compare); } -#define CK_PR_CAS_O(S, T) \ - CK_CC_INLINE static bool \ - ck_pr_cas_##S##_value(T *target, T compare, T set, T *v) \ - { \ - set = __sync_val_compare_and_swap(target, compare, set); \ - *v = set; \ - return (set == compare); \ +#define CK_PR_CAS_O(S, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(T *target, T compare, T set, T *v) \ + { \ + set = __sync_val_compare_and_swap(target, compare, set);\ + *v = set; \ + return (set == compare); \ } CK_PR_CAS_O(char, char) @@ -194,12 +194,12 @@ CK_PR_FAA_S(8, uint8_t) /* * Atomic store-only binary operations. */ -#define CK_PR_BINARY(K, S, M, T) \ - CK_CC_INLINE static void \ - ck_pr_##K##_##S(M *target, T d) \ - { \ - d = __sync_fetch_and_##K((T *)target, d); \ - return; \ +#define CK_PR_BINARY(K, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_##K((T *)target, d); \ + return; \ } #define CK_PR_BINARY_S(K, S, T) CK_PR_BINARY(K, S, T, T) diff --git a/include/gcc/x86_64/ck_pr.h b/include/gcc/x86_64/ck_pr.h index d9fc965..fdd3c62 100644 --- a/include/gcc/x86_64/ck_pr.h +++ b/include/gcc/x86_64/ck_pr.h @@ -433,7 +433,10 @@ ck_pr_cas_ptr_2(void *t, void *c, void *s) } CK_CC_INLINE static bool -ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t v[2]) +ck_pr_cas_64_2_value(uint64_t target[2], + uint64_t compare[2], + uint64_t set[2], + uint64_t v[2]) { bool z;