diff --git a/include/ck_swlock.h b/include/ck_swlock.h index 29ac407..3719b01 100644 --- a/include/ck_swlock.h +++ b/include/ck_swlock.h @@ -49,6 +49,7 @@ ck_swlock_init(struct ck_swlock *rw) { rw->value = 0; + ck_pr_barrier(); return; } @@ -56,6 +57,7 @@ CK_CC_INLINE static void ck_swlock_write_unlock(ck_swlock_t *rw) { + ck_pr_fence_release(); ck_pr_and_32(&rw->value, CK_SWLOCK_READER_BITS); return; } @@ -64,7 +66,8 @@ CK_CC_INLINE static bool ck_swlock_locked_writer(ck_swlock_t *rw) { - return (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT); + ck_pr_fence_load(); + return ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT; } CK_CC_INLINE static void @@ -80,6 +83,7 @@ CK_CC_INLINE static bool ck_swlock_locked(ck_swlock_t *rw) { + ck_pr_fence_load(); return ck_pr_load_32(&rw->value); } @@ -94,6 +98,7 @@ ck_swlock_write_trylock(ck_swlock_t *rw) return false; } + ck_pr_fence_acquire(); return true; } @@ -105,6 +110,7 @@ ck_swlock_write_lock(ck_swlock_t *rw) { ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT); + ck_pr_fence_acquire(); while ((ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_BITS) != 0) ck_pr_stall(); @@ -117,9 +123,11 @@ ck_swlock_write_latch(ck_swlock_t *rw) { ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT); - + ck_pr_fence_acquire(); + /* Stall until readers have seen the latch and cleared. */ - while (ck_pr_cas_32(&rw->value, CK_SWLOCK_WRITER_BIT, (CK_SWLOCK_WRITER_BIT | CK_SWLOCK_LATCH_BIT)) == false) { + while (ck_pr_cas_32(&rw->value, CK_SWLOCK_WRITER_BIT, + (CK_SWLOCK_WRITER_BIT | CK_SWLOCK_LATCH_BIT)) == false) { do { ck_pr_stall(); } while (ck_pr_load_uint(&rw->value) != CK_SWLOCK_WRITER_BIT); @@ -132,6 +140,7 @@ CK_CC_INLINE static void ck_swlock_write_unlatch(ck_swlock_t *rw) { + ck_pr_fence_release(); ck_pr_store_32(&rw->value, 0); return; } @@ -148,7 +157,8 @@ ck_swlock_read_trylock(ck_swlock_t *rw) ck_pr_dec_32(&rw->value); return false; } - + + ck_pr_fence_acquire(); return true; } @@ -158,7 +168,6 @@ CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_read, ck_swlock_t, CK_CC_INLINE static void ck_swlock_read_lock(ck_swlock_t *rw) { - uint32_t l; for (;;) { while (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT) @@ -166,6 +175,8 @@ ck_swlock_read_lock(ck_swlock_t *rw) l = ck_pr_faa_32(&rw->value, 1); + ck_pr_fence_acquire(); + if (!(l & CK_SWLOCK_WRITER_BIT)) return; @@ -178,14 +189,15 @@ ck_swlock_read_lock(ck_swlock_t *rw) CK_CC_INLINE static bool ck_swlock_read_trylatchlock(ck_swlock_t *rw) { - uint32_t l = ck_pr_load_32(&rw->value); if (l & CK_SWLOCK_WRITER_BIT) return false; l = ck_pr_faa_32(&rw->value, 1); - + + ck_pr_fence_acquire(); + if (!(l & CK_SWLOCK_WRITER_BIT)) return true; @@ -199,14 +211,15 @@ ck_swlock_read_trylatchlock(ck_swlock_t *rw) CK_CC_INLINE static void ck_swlock_read_latchlock(ck_swlock_t *rw) { - uint32_t l; for (;;) { while (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT) ck_pr_stall(); l = ck_pr_faa_32(&rw->value, 1); - + + ck_pr_fence_acquire(); + if (!(l & (CK_SWLOCK_LATCH_BIT | CK_SWLOCK_WRITER_BIT))) return; @@ -222,6 +235,7 @@ CK_CC_INLINE static bool ck_swlock_locked_reader(ck_swlock_t *rw) { + ck_pr_fence_load(); return ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_BITS; } @@ -229,6 +243,7 @@ CK_CC_INLINE static void ck_swlock_read_unlock(ck_swlock_t *rw) { + ck_pr_fence_release(); ck_pr_dec_32(&rw->value); return; }