haiwan
/
ck
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ck_*lock: Migrate all locking primitives to lock / unlock.

Browse Source 
This only affects RMO. This adds stricter semantics for critical section
serialization. In addition to this, asymmetric synchronization primitives will
now provide load ordering with respect to readers.

This also modifies locked operations to have acquire semantics
(they're there for elision predicates, and this doesn't impact them
in any way). There are several performance improvements included in this
as well (redundant fence was removed from days of wanting to support
Alpha).
ck_pring
Samy Al Bahra 10 years ago
parent 1537c8091d
commit e693cb9afe
14 changed files with 141 additions and 120 deletions
Show Stats Download Patch File Download Diff File

12
include/ck_brlock.h
Unescape View File

@ -91,7 +91,7 @@ ck_brlock_write_lock(struct ck_brlock *br)
ck_pr_stall(); ck_pr_stall();
} }
/* Already acquired with respect to other writers. */ ck_pr_fence_lock();
return; return;
} }
@ -99,7 +99,7 @@ CK_CC_INLINE static void
ck_brlock_write_unlock(struct ck_brlock *br) ck_brlock_write_unlock(struct ck_brlock *br)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&br->writer, false); ck_pr_store_uint(&br->writer, false);
return; return;
} }
@ -134,7 +134,7 @@ ck_brlock_write_trylock(struct ck_brlock *br, unsigned int factor)
} }
} }
/* Already acquired with respect to other writers. */ ck_pr_fence_lock();
return true; return true;
} }
@ -212,7 +212,7 @@ ck_brlock_read_lock(struct ck_brlock *br, struct ck_brlock_reader *reader)
ck_pr_store_uint(&reader->n_readers, 0); ck_pr_store_uint(&reader->n_readers, 0);
} }
ck_pr_fence_load(); ck_pr_fence_lock();
return; return;
} }
@ -263,7 +263,7 @@ ck_brlock_read_trylock(struct ck_brlock *br,
return false; return false;
} }
ck_pr_fence_load(); ck_pr_fence_lock();
return true; return true;
} }
@ -271,7 +271,7 @@ CK_CC_INLINE static void
ck_brlock_read_unlock(struct ck_brlock_reader *reader) ck_brlock_read_unlock(struct ck_brlock_reader *reader)
{ {
ck_pr_fence_load_store(); ck_pr_fence_unlock();
ck_pr_store_uint(&reader->n_readers, reader->n_readers - 1); ck_pr_store_uint(&reader->n_readers, reader->n_readers - 1);
return; return;
} }

9
include/ck_bytelock.h
Unescape View File

@ -107,6 +107,7 @@ ck_bytelock_write_lock(struct ck_bytelock *bytelock, unsigned int slot)
while (ck_pr_load_uint(&bytelock->n_readers) != 0) while (ck_pr_load_uint(&bytelock->n_readers) != 0)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_lock();
return; return;
} }
@ -118,7 +119,7 @@ CK_CC_INLINE static void
ck_bytelock_write_unlock(struct ck_bytelock *bytelock) ck_bytelock_write_unlock(struct ck_bytelock *bytelock)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&bytelock->owner, 0); ck_pr_store_uint(&bytelock->owner, 0);
return; return;
} }
@ -147,7 +148,7 @@ ck_bytelock_read_lock(struct ck_bytelock *bytelock, unsigned int slot)
ck_pr_stall(); ck_pr_stall();
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -169,7 +170,7 @@ ck_bytelock_read_lock(struct ck_bytelock *bytelock, unsigned int slot)
ck_pr_stall(); ck_pr_stall();
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -177,7 +178,7 @@ CK_CC_INLINE static void
ck_bytelock_read_unlock(struct ck_bytelock *bytelock, unsigned int slot) ck_bytelock_read_unlock(struct ck_bytelock *bytelock, unsigned int slot)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
if (slot > sizeof bytelock->readers) if (slot > sizeof bytelock->readers)
ck_pr_dec_uint(&bytelock->n_readers); ck_pr_dec_uint(&bytelock->n_readers);

8
include/ck_pflock.h
Unescape View File

@ -71,7 +71,7 @@ CK_CC_INLINE static void
ck_pflock_write_unlock(ck_pflock_t *pf) ck_pflock_write_unlock(ck_pflock_t *pf)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
/* Migrate from write phase to read phase. */ /* Migrate from write phase to read phase. */
ck_pr_and_32(&pf->rin, CK_PFLOCK_LSB); ck_pr_and_32(&pf->rin, CK_PFLOCK_LSB);
@ -103,7 +103,7 @@ ck_pflock_write_lock(ck_pflock_t *pf)
while (ck_pr_load_32(&pf->rout) != ticket) while (ck_pr_load_32(&pf->rout) != ticket)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -111,7 +111,7 @@ CK_CC_INLINE static void
ck_pflock_read_unlock(ck_pflock_t *pf) ck_pflock_read_unlock(ck_pflock_t *pf)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_faa_32(&pf->rout, CK_PFLOCK_RINC); ck_pr_faa_32(&pf->rout, CK_PFLOCK_RINC);
return; return;
} }
@ -135,7 +135,7 @@ ck_pflock_read_lock(ck_pflock_t *pf)
leave: leave:
/* Acquire semantics with respect to readers. */ /* Acquire semantics with respect to readers. */
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }

27
include/ck_rwlock.h
Unescape View File

@ -54,7 +54,7 @@ CK_CC_INLINE static void
ck_rwlock_write_unlock(ck_rwlock_t *rw) ck_rwlock_write_unlock(ck_rwlock_t *rw)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&rw->writer, 0); ck_pr_store_uint(&rw->writer, 0);
return; return;
} }
@ -62,9 +62,11 @@ ck_rwlock_write_unlock(ck_rwlock_t *rw)
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_rwlock_locked_writer(ck_rwlock_t *rw) ck_rwlock_locked_writer(ck_rwlock_t *rw)
{ {
bool r;
ck_pr_fence_load(); r = ck_pr_load_uint(&rw->writer);
return ck_pr_load_uint(&rw->writer); ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -79,13 +81,12 @@ ck_rwlock_write_downgrade(ck_rwlock_t *rw)
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_rwlock_locked(ck_rwlock_t *rw) ck_rwlock_locked(ck_rwlock_t *rw)
{ {
unsigned int r; bool l;
ck_pr_fence_load();
r = ck_pr_load_uint(&rw->writer);
ck_pr_fence_load();
return ck_pr_load_uint(&rw->n_readers) | r; l = ck_pr_load_uint(&rw->n_readers) |
ck_pr_load_uint(&rw->writer);
ck_pr_fence_acquire();
return l;
} }
CK_CC_INLINE static bool CK_CC_INLINE static bool
@ -102,6 +103,7 @@ ck_rwlock_write_trylock(ck_rwlock_t *rw)
return false; return false;
} }
ck_pr_fence_lock();
return true; return true;
} }
@ -120,6 +122,7 @@ ck_rwlock_write_lock(ck_rwlock_t *rw)
while (ck_pr_load_uint(&rw->n_readers) != 0) while (ck_pr_load_uint(&rw->n_readers) != 0)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_lock();
return; return;
} }
@ -143,7 +146,7 @@ ck_rwlock_read_trylock(ck_rwlock_t *rw)
ck_pr_fence_atomic_load(); ck_pr_fence_atomic_load();
if (ck_pr_load_uint(&rw->writer) == 0) { if (ck_pr_load_uint(&rw->writer) == 0) {
ck_pr_fence_load(); ck_pr_fence_lock();
return true; return true;
} }
@ -230,6 +233,7 @@ ck_rwlock_recursive_write_lock(ck_rwlock_recursive_t *rw, unsigned int tid)
while (ck_pr_load_uint(&rw->rw.n_readers) != 0) while (ck_pr_load_uint(&rw->rw.n_readers) != 0)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_lock();
leave: leave:
rw->wc++; rw->wc++;
return; return;
@ -254,6 +258,7 @@ ck_rwlock_recursive_write_trylock(ck_rwlock_recursive_t *rw, unsigned int tid)
return false; return false;
} }
ck_pr_fence_lock();
leave: leave:
rw->wc++; rw->wc++;
return true; return true;
@ -264,7 +269,7 @@ ck_rwlock_recursive_write_unlock(ck_rwlock_recursive_t *rw)
{ {
if (--rw->wc == 0) { if (--rw->wc == 0) {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&rw->rw.writer, 0); ck_pr_store_uint(&rw->rw.writer, 0);
} }

39
include/ck_swlock.h
Unescape View File

@ -58,7 +58,7 @@ CK_CC_INLINE static void
ck_swlock_write_unlock(ck_swlock_t *rw) ck_swlock_write_unlock(ck_swlock_t *rw)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_and_32(&rw->value, CK_SWLOCK_READER_MASK); ck_pr_and_32(&rw->value, CK_SWLOCK_READER_MASK);
return; return;
} }
@ -66,9 +66,11 @@ ck_swlock_write_unlock(ck_swlock_t *rw)
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_swlock_locked_writer(ck_swlock_t *rw) ck_swlock_locked_writer(ck_swlock_t *rw)
{ {
bool r;
ck_pr_fence_load(); r = ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT;
return ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT; ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -83,17 +85,21 @@ ck_swlock_write_downgrade(ck_swlock_t *rw)
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_swlock_locked(ck_swlock_t *rw) ck_swlock_locked(ck_swlock_t *rw)
{ {
bool r;
ck_pr_fence_load(); r = ck_pr_load_32(&rw->value);
return ck_pr_load_32(&rw->value); ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_swlock_write_trylock(ck_swlock_t *rw) ck_swlock_write_trylock(ck_swlock_t *rw)
{ {
bool r;
ck_pr_fence_acquire(); r = ck_pr_cas_32(&rw->value, 0, CK_SWLOCK_WRITER_BIT);
return ck_pr_cas_32(&rw->value, 0, CK_SWLOCK_WRITER_BIT); ck_pr_fence_lock();
return r;
} }
CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_write, ck_swlock_t, CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_write, ck_swlock_t,
@ -107,7 +113,7 @@ ck_swlock_write_lock(ck_swlock_t *rw)
while (ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK) while (ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -126,7 +132,7 @@ ck_swlock_write_latch(ck_swlock_t *rw)
} while (ck_pr_load_32(&rw->value) != CK_SWLOCK_WRITER_BIT); } while (ck_pr_load_32(&rw->value) != CK_SWLOCK_WRITER_BIT);
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -134,7 +140,7 @@ CK_CC_INLINE static void
ck_swlock_write_unlatch(ck_swlock_t *rw) ck_swlock_write_unlatch(ck_swlock_t *rw)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_32(&rw->value, 0); ck_pr_store_32(&rw->value, 0);
return; return;
} }
@ -155,15 +161,11 @@ ck_swlock_read_trylock(ck_swlock_t *rw)
return false; return false;
l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK; l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK;
if (l == 0) {
ck_pr_fence_acquire();
return true;
}
if (l == CK_SWLOCK_WRITER_BIT) if (l == CK_SWLOCK_WRITER_BIT)
ck_pr_dec_32(&rw->value); ck_pr_dec_32(&rw->value);
return false; ck_pr_fence_lock();
return l == 0;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -188,11 +190,10 @@ ck_swlock_read_lock(ck_swlock_t *rw)
ck_pr_dec_32(&rw->value); ck_pr_dec_32(&rw->value);
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_swlock_locked_reader(ck_swlock_t *rw) ck_swlock_locked_reader(ck_swlock_t *rw)
{ {
@ -205,7 +206,7 @@ CK_CC_INLINE static void
ck_swlock_read_unlock(ck_swlock_t *rw) ck_swlock_read_unlock(ck_swlock_t *rw)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_dec_32(&rw->value); ck_pr_dec_32(&rw->value);
return; return;
} }

17
include/ck_tflock.h
Unescape View File

@ -89,7 +89,7 @@ ck_tflock_ticket_write_lock(struct ck_tflock_ticket *lock)
while (ck_pr_load_32(&lock->completion) != previous) while (ck_pr_load_32(&lock->completion) != previous)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -97,7 +97,7 @@ CK_CC_INLINE static void
ck_tflock_ticket_write_unlock(struct ck_tflock_ticket *lock) ck_tflock_ticket_write_unlock(struct ck_tflock_ticket *lock)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_WC_TOPMSK, ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_WC_TOPMSK,
CK_TFLOCK_TICKET_WC_INCR); CK_TFLOCK_TICKET_WC_INCR);
return; return;
@ -108,15 +108,18 @@ ck_tflock_ticket_read_lock(struct ck_tflock_ticket *lock)
{ {
uint32_t previous; uint32_t previous;
previous = ck_tflock_ticket_fca_32(&lock->request, CK_TFLOCK_TICKET_RC_TOPMSK, previous = ck_tflock_ticket_fca_32(&lock->request,
CK_TFLOCK_TICKET_RC_INCR) & CK_TFLOCK_TICKET_W_MASK; CK_TFLOCK_TICKET_RC_TOPMSK, CK_TFLOCK_TICKET_RC_INCR) &
CK_TFLOCK_TICKET_W_MASK;
ck_pr_fence_atomic_load(); ck_pr_fence_atomic_load();
while ((ck_pr_load_uint(&lock->completion) & CK_TFLOCK_TICKET_W_MASK) != previous) while ((ck_pr_load_32(&lock->completion) &
CK_TFLOCK_TICKET_W_MASK) != previous) {
ck_pr_stall(); ck_pr_stall();
}
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -124,7 +127,7 @@ CK_CC_INLINE static void
ck_tflock_ticket_read_unlock(struct ck_tflock_ticket *lock) ck_tflock_ticket_read_unlock(struct ck_tflock_ticket *lock)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_RC_TOPMSK, ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_RC_TOPMSK,
CK_TFLOCK_TICKET_RC_INCR); CK_TFLOCK_TICKET_RC_INCR);
return; return;

17
include/spinlock/anderson.h
Unescape View File

@ -91,12 +91,12 @@ CK_CC_INLINE static bool
ck_spinlock_anderson_locked(struct ck_spinlock_anderson *lock) ck_spinlock_anderson_locked(struct ck_spinlock_anderson *lock)
{ {
unsigned int position; unsigned int position;
bool r;
ck_pr_fence_load();
position = ck_pr_load_uint(&lock->next) & lock->mask; position = ck_pr_load_uint(&lock->next) & lock->mask;
ck_pr_fence_load(); r = ck_pr_load_uint(&lock->slots[position].locked);
ck_pr_fence_acquire();
return ck_pr_load_uint(&lock->slots[position].locked); return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -131,13 +131,16 @@ ck_spinlock_anderson_lock(struct ck_spinlock_anderson *lock,
/* Serialize with respect to previous thread's store. */ /* Serialize with respect to previous thread's store. */
ck_pr_fence_load(); ck_pr_fence_load();
/* Spin until slot is marked as unlocked. First slot is initialized to false. */ /*
* Spin until slot is marked as unlocked. First slot is initialized to
* false.
*/
while (ck_pr_load_uint(&lock->slots[position].locked) == true) while (ck_pr_load_uint(&lock->slots[position].locked) == true)
ck_pr_stall(); ck_pr_stall();
/* Prepare slot for potential re-use by another thread. */ /* Prepare slot for potential re-use by another thread. */
ck_pr_store_uint(&lock->slots[position].locked, true); ck_pr_store_uint(&lock->slots[position].locked, true);
ck_pr_fence_acquire(); ck_pr_fence_lock();
*slot = lock->slots + position; *slot = lock->slots + position;
return; return;
@ -149,7 +152,7 @@ ck_spinlock_anderson_unlock(struct ck_spinlock_anderson *lock,
{ {
unsigned int position; unsigned int position;
ck_pr_fence_release(); ck_pr_fence_unlock();
/* Mark next slot as available. */ /* Mark next slot as available. */
if (lock->wrap == 0) if (lock->wrap == 0)

15
include/spinlock/cas.h
Unescape View File

@ -60,18 +60,17 @@ ck_spinlock_cas_trylock(struct ck_spinlock_cas *lock)
unsigned int value; unsigned int value;
value = ck_pr_fas_uint(&lock->value, true); value = ck_pr_fas_uint(&lock->value, true);
if (value == false) ck_pr_fence_lock();
ck_pr_fence_acquire();
return !value; return !value;
} }
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_spinlock_cas_locked(struct ck_spinlock_cas *lock) ck_spinlock_cas_locked(struct ck_spinlock_cas *lock)
{ {
bool r = ck_pr_load_uint(&lock->value);
ck_pr_fence_load(); ck_pr_fence_acquire();
return ck_pr_load_uint(&lock->value); return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -83,7 +82,7 @@ ck_spinlock_cas_lock(struct ck_spinlock_cas *lock)
ck_pr_stall(); ck_pr_stall();
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -95,7 +94,7 @@ ck_spinlock_cas_lock_eb(struct ck_spinlock_cas *lock)
while (ck_pr_cas_uint(&lock->value, false, true) == false) while (ck_pr_cas_uint(&lock->value, false, true) == false)
ck_backoff_eb(&backoff); ck_backoff_eb(&backoff);
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -104,7 +103,7 @@ ck_spinlock_cas_unlock(struct ck_spinlock_cas *lock)
{ {
/* Set lock state to unlocked. */ /* Set lock state to unlocked. */
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&lock->value, false); ck_pr_store_uint(&lock->value, false);
return; return;
} }

20
include/spinlock/clh.h
Unescape View File

@ -57,11 +57,12 @@ CK_CC_INLINE static bool
ck_spinlock_clh_locked(struct ck_spinlock_clh **queue) ck_spinlock_clh_locked(struct ck_spinlock_clh **queue)
{ {
struct ck_spinlock_clh *head; struct ck_spinlock_clh *head;
bool r;
ck_pr_fence_load();
head = ck_pr_load_ptr(queue); head = ck_pr_load_ptr(queue);
ck_pr_fence_load(); r = ck_pr_load_uint(&head->wait);
return ck_pr_load_uint(&head->wait); ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -73,7 +74,10 @@ ck_spinlock_clh_lock(struct ck_spinlock_clh **queue, struct ck_spinlock_clh *thr
thread->wait = true; thread->wait = true;
ck_pr_fence_store_atomic(); ck_pr_fence_store_atomic();
/* Mark current request as last request. Save reference to previous request. */ /*
* Mark current request as last request. Save reference to previous
* request.
*/
previous = ck_pr_fas_ptr(queue, thread); previous = ck_pr_fas_ptr(queue, thread);
thread->previous = previous; thread->previous = previous;
@ -82,7 +86,7 @@ ck_spinlock_clh_lock(struct ck_spinlock_clh **queue, struct ck_spinlock_clh *thr
while (ck_pr_load_uint(&previous->wait) == true) while (ck_pr_load_uint(&previous->wait) == true)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -100,8 +104,10 @@ ck_spinlock_clh_unlock(struct ck_spinlock_clh **thread)
*/ */
previous = thread[0]->previous; previous = thread[0]->previous;
/* We have to pay this cost anyways, use it as a compiler barrier too. */ /*
ck_pr_fence_release(); * We have to pay this cost anyways, use it as a compiler barrier too.
*/
ck_pr_fence_unlock();
ck_pr_store_uint(&(*thread)->wait, false); ck_pr_store_uint(&(*thread)->wait, false);
/* /*

25
include/spinlock/dec.h
Unescape View File

@ -62,20 +62,18 @@ ck_spinlock_dec_trylock(struct ck_spinlock_dec *lock)
unsigned int value; unsigned int value;
value = ck_pr_fas_uint(&lock->value, 0); value = ck_pr_fas_uint(&lock->value, 0);
if (value == 1) { ck_pr_fence_lock();
ck_pr_fence_acquire(); return value == 1;
return true;
}
return false;
} }
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_spinlock_dec_locked(struct ck_spinlock_dec *lock) ck_spinlock_dec_locked(struct ck_spinlock_dec *lock)
{ {
bool r;
ck_pr_fence_load(); r = ck_pr_load_uint(&lock->value) != 1;
return ck_pr_load_uint(&lock->value) != 1; ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -98,7 +96,7 @@ ck_spinlock_dec_lock(struct ck_spinlock_dec *lock)
ck_pr_stall(); ck_pr_stall();
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -116,7 +114,7 @@ ck_spinlock_dec_lock_eb(struct ck_spinlock_dec *lock)
ck_backoff_eb(&backoff); ck_backoff_eb(&backoff);
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -124,9 +122,12 @@ CK_CC_INLINE static void
ck_spinlock_dec_unlock(struct ck_spinlock_dec *lock) ck_spinlock_dec_unlock(struct ck_spinlock_dec *lock)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
/* Unconditionally set lock value to 1 so someone can decrement lock to 0. */ /*
* Unconditionally set lock value to 1 so someone can decrement lock
* to 0.
*/
ck_pr_store_uint(&lock->value, 1); ck_pr_store_uint(&lock->value, 1);
return; return;
} }

15
include/spinlock/fas.h
Unescape View File

@ -58,8 +58,7 @@ ck_spinlock_fas_trylock(struct ck_spinlock_fas *lock)
bool value; bool value;
value = ck_pr_fas_uint(&lock->value, true); value = ck_pr_fas_uint(&lock->value, true);
if (value == false) ck_pr_fence_lock();
ck_pr_fence_acquire();
return !value; return !value;
} }
@ -67,9 +66,11 @@ ck_spinlock_fas_trylock(struct ck_spinlock_fas *lock)
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_spinlock_fas_locked(struct ck_spinlock_fas *lock) ck_spinlock_fas_locked(struct ck_spinlock_fas *lock)
{ {
bool r;
ck_pr_fence_load(); r = ck_pr_load_uint(&lock->value);
return ck_pr_load_uint(&lock->value); ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -81,7 +82,7 @@ ck_spinlock_fas_lock(struct ck_spinlock_fas *lock)
ck_pr_stall(); ck_pr_stall();
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -93,7 +94,7 @@ ck_spinlock_fas_lock_eb(struct ck_spinlock_fas *lock)
while (ck_pr_fas_uint(&lock->value, true) == true) while (ck_pr_fas_uint(&lock->value, true) == true)
ck_backoff_eb(&backoff); ck_backoff_eb(&backoff);
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -101,7 +102,7 @@ CK_CC_INLINE static void
ck_spinlock_fas_unlock(struct ck_spinlock_fas *lock) ck_spinlock_fas_unlock(struct ck_spinlock_fas *lock)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&lock->value, false); ck_pr_store_uint(&lock->value, false);
return; return;
} }

11
include/spinlock/hclh.h
Unescape View File

@ -62,11 +62,12 @@ CK_CC_INLINE static bool
ck_spinlock_hclh_locked(struct ck_spinlock_hclh **queue) ck_spinlock_hclh_locked(struct ck_spinlock_hclh **queue)
{ {
struct ck_spinlock_hclh *head; struct ck_spinlock_hclh *head;
bool r;
ck_pr_fence_load();
head = ck_pr_load_ptr(queue); head = ck_pr_load_ptr(queue);
ck_pr_fence_load(); r = ck_pr_load_uint(&head->wait);
return ck_pr_load_uint(&head->wait); ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -110,7 +111,7 @@ ck_spinlock_hclh_lock(struct ck_spinlock_hclh **glob_queue,
while (ck_pr_load_uint(&previous->wait) == true) while (ck_pr_load_uint(&previous->wait) == true)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -129,7 +130,7 @@ ck_spinlock_hclh_unlock(struct ck_spinlock_hclh **thread)
previous = thread[0]->previous; previous = thread[0]->previous;
/* We have to pay this cost anyways, use it as a compiler barrier too. */ /* We have to pay this cost anyways, use it as a compiler barrier too. */
ck_pr_fence_release(); ck_pr_fence_unlock();
ck_pr_store_uint(&(*thread)->wait, false); ck_pr_store_uint(&(*thread)->wait, false);
/* /*

20
include/spinlock/mcs.h
Unescape View File

@ -57,25 +57,25 @@ CK_CC_INLINE static bool
ck_spinlock_mcs_trylock(struct ck_spinlock_mcs **queue, ck_spinlock_mcs_trylock(struct ck_spinlock_mcs **queue,
struct ck_spinlock_mcs *node) struct ck_spinlock_mcs *node)
{ {
bool r;
node->locked = true; node->locked = true;
node->next = NULL; node->next = NULL;
ck_pr_fence_store_atomic(); ck_pr_fence_store_atomic();
if (ck_pr_cas_ptr(queue, NULL, node) == true) { r = ck_pr_cas_ptr(queue, NULL, node);
ck_pr_fence_acquire(); ck_pr_fence_lock();
return true; return r;
}
return false;
} }
CK_CC_INLINE static bool CK_CC_INLINE static bool
ck_spinlock_mcs_locked(struct ck_spinlock_mcs **queue) ck_spinlock_mcs_locked(struct ck_spinlock_mcs **queue)
{ {
bool r;
ck_pr_fence_load(); r = ck_pr_load_ptr(queue) != NULL;
return ck_pr_load_ptr(queue) != NULL; ck_pr_fence_acquire();
return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -108,7 +108,7 @@ ck_spinlock_mcs_lock(struct ck_spinlock_mcs **queue,
ck_pr_stall(); ck_pr_stall();
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -118,7 +118,7 @@ ck_spinlock_mcs_unlock(struct ck_spinlock_mcs **queue,
{ {
struct ck_spinlock_mcs *next; struct ck_spinlock_mcs *next;
ck_pr_fence_release(); ck_pr_fence_unlock();
next = ck_pr_load_ptr(&node->next); next = ck_pr_load_ptr(&node->next);
if (next == NULL) { if (next == NULL) {

26
include/spinlock/ticket.h
Unescape View File

@ -88,12 +88,11 @@ ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket)
{ {
CK_SPINLOCK_TICKET_TYPE request, position; CK_SPINLOCK_TICKET_TYPE request, position;
ck_pr_fence_load();
request = CK_SPINLOCK_TICKET_LOAD(&ticket->value); request = CK_SPINLOCK_TICKET_LOAD(&ticket->value);
position = request & CK_SPINLOCK_TICKET_MASK; position = request & CK_SPINLOCK_TICKET_MASK;
request >>= CK_SPINLOCK_TICKET_SHIFT; request >>= CK_SPINLOCK_TICKET_SHIFT;
ck_pr_fence_acquire();
return request != position; return request != position;
} }
@ -115,7 +114,7 @@ ck_spinlock_ticket_lock(struct ck_spinlock_ticket *ticket)
CK_SPINLOCK_TICKET_MASK; CK_SPINLOCK_TICKET_MASK;
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -142,7 +141,7 @@ ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c)
ck_backoff_eb(&backoff); ck_backoff_eb(&backoff);
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -163,7 +162,7 @@ ck_spinlock_ticket_trylock(struct ck_spinlock_ticket *ticket)
return false; return false;
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return true; return true;
} }
@ -171,7 +170,7 @@ CK_CC_INLINE static void
ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket) ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket)
{ {
ck_pr_fence_release(); ck_pr_fence_unlock();
CK_SPINLOCK_TICKET_INC((CK_SPINLOCK_TICKET_TYPE_BASE *)(void *)&ticket->value); CK_SPINLOCK_TICKET_INC((CK_SPINLOCK_TICKET_TYPE_BASE *)(void *)&ticket->value);
return; return;
} }
@ -212,11 +211,12 @@ CK_CC_INLINE static bool
ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket) ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket)
{ {
unsigned int request; unsigned int request;
bool r;
ck_pr_fence_load(); r = ck_pr_load_uint(&ticket->position) !=
request = ck_pr_load_uint(&ticket->next); ck_pr_load_uint(&ticket->next);
ck_pr_fence_load(); ck_pr_fence_acquire();
return ck_pr_load_uint(&ticket->position) != request; return r;
} }
CK_CC_INLINE static void CK_CC_INLINE static void
@ -235,7 +235,7 @@ ck_spinlock_ticket_lock(struct ck_spinlock_ticket *ticket)
while (ck_pr_load_uint(&ticket->position) != request) while (ck_pr_load_uint(&ticket->position) != request)
ck_pr_stall(); ck_pr_stall();
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -264,7 +264,7 @@ ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c)
ck_backoff_eb(&backoff); ck_backoff_eb(&backoff);
} }
ck_pr_fence_acquire(); ck_pr_fence_lock();
return; return;
} }
@ -273,7 +273,7 @@ ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket)
{ {
unsigned int update; unsigned int update;
ck_pr_fence_release(); ck_pr_fence_unlock();
/* /*
* Update current ticket value so next lock request can proceed. * Update current ticket value so next lock request can proceed.

Write Preview
Loading…
Cancel
Save