|
|
|
@ -408,7 +408,7 @@ CK_PR_GENERATE(xor)
|
|
|
|
|
#undef CK_PR_BINARY
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Atomic compare and swap.
|
|
|
|
|
* Atomic compare and swap, with a variant that sets *v to the old value of target.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef __GCC_ASM_FLAG_OUTPUTS__
|
|
|
|
|
#define CK_PR_CAS(S, M, T, C, I) \
|
|
|
|
@ -424,6 +424,20 @@ CK_PR_GENERATE(xor)
|
|
|
|
|
: "q" (set) \
|
|
|
|
|
: "memory", "cc"); \
|
|
|
|
|
return z; \
|
|
|
|
|
} \
|
|
|
|
|
\
|
|
|
|
|
CK_CC_INLINE static bool \
|
|
|
|
|
ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
|
|
|
|
|
{ \
|
|
|
|
|
bool z; \
|
|
|
|
|
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %3, %0;" \
|
|
|
|
|
: "+m" (*(C *)target), \
|
|
|
|
|
"=@ccz" (z), \
|
|
|
|
|
"+a" (compare) \
|
|
|
|
|
: "q" (set) \
|
|
|
|
|
: "memory", "cc"); \
|
|
|
|
|
*(T *)v = compare; \
|
|
|
|
|
return z; \
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
#define CK_PR_CAS(S, M, T, C, I) \
|
|
|
|
@ -438,52 +452,13 @@ CK_PR_GENERATE(xor)
|
|
|
|
|
"a" (compare) \
|
|
|
|
|
: "memory", "cc"); \
|
|
|
|
|
return z; \
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
CK_PR_CAS(ptr, void, void *, char, "cmpxchgq")
|
|
|
|
|
|
|
|
|
|
#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I)
|
|
|
|
|
|
|
|
|
|
CK_PR_CAS_S(char, char, "cmpxchgb")
|
|
|
|
|
CK_PR_CAS_S(int, int, "cmpxchgl")
|
|
|
|
|
CK_PR_CAS_S(uint, unsigned int, "cmpxchgl")
|
|
|
|
|
#ifndef CK_PR_DISABLE_DOUBLE
|
|
|
|
|
CK_PR_CAS_S(double, double, "cmpxchgq")
|
|
|
|
|
#endif
|
|
|
|
|
CK_PR_CAS_S(64, uint64_t, "cmpxchgq")
|
|
|
|
|
CK_PR_CAS_S(32, uint32_t, "cmpxchgl")
|
|
|
|
|
CK_PR_CAS_S(16, uint16_t, "cmpxchgw")
|
|
|
|
|
CK_PR_CAS_S(8, uint8_t, "cmpxchgb")
|
|
|
|
|
|
|
|
|
|
#undef CK_PR_CAS_S
|
|
|
|
|
#undef CK_PR_CAS
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Compare and swap, set *v to old value of target.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef __GCC_ASM_FLAG_OUTPUTS__
|
|
|
|
|
#define CK_PR_CAS_O(S, M, T, C, I) \
|
|
|
|
|
CK_CC_INLINE static bool \
|
|
|
|
|
ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
|
|
|
|
|
{ \
|
|
|
|
|
bool z; \
|
|
|
|
|
__asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \
|
|
|
|
|
: "+m" (*(C *)target), \
|
|
|
|
|
"=@ccz" (z), \
|
|
|
|
|
"+a" (compare) \
|
|
|
|
|
: "q" (set) \
|
|
|
|
|
: "memory", "cc"); \
|
|
|
|
|
*(T *)v = compare; \
|
|
|
|
|
return z; \
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
#define CK_PR_CAS_O(S, M, T, C, I) \
|
|
|
|
|
} \
|
|
|
|
|
\
|
|
|
|
|
CK_CC_INLINE static bool \
|
|
|
|
|
ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
|
|
|
|
|
{ \
|
|
|
|
|
bool z; \
|
|
|
|
|
__asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \
|
|
|
|
|
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %3, %0;" \
|
|
|
|
|
"setz %1;" \
|
|
|
|
|
: "+m" (*(C *)target), \
|
|
|
|
|
"=q" (z), \
|
|
|
|
@ -495,24 +470,23 @@ CK_PR_CAS_S(8, uint8_t, "cmpxchgb")
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
CK_PR_CAS_O(ptr, void, void *, char, "q")
|
|
|
|
|
CK_PR_CAS(ptr, void, void *, char, "cmpxchgq")
|
|
|
|
|
|
|
|
|
|
#define CK_PR_CAS_O_S(S, T, I) \
|
|
|
|
|
CK_PR_CAS_O(S, T, T, T, I)
|
|
|
|
|
#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I)
|
|
|
|
|
|
|
|
|
|
CK_PR_CAS_O_S(char, char, "b")
|
|
|
|
|
CK_PR_CAS_O_S(int, int, "l")
|
|
|
|
|
CK_PR_CAS_O_S(uint, unsigned int, "l")
|
|
|
|
|
CK_PR_CAS_S(char, char, "cmpxchgb")
|
|
|
|
|
CK_PR_CAS_S(int, int, "cmpxchgl")
|
|
|
|
|
CK_PR_CAS_S(uint, unsigned int, "cmpxchgl")
|
|
|
|
|
#ifndef CK_PR_DISABLE_DOUBLE
|
|
|
|
|
CK_PR_CAS_O_S(double, double, "q")
|
|
|
|
|
CK_PR_CAS_S(double, double, "cmpxchgq")
|
|
|
|
|
#endif
|
|
|
|
|
CK_PR_CAS_O_S(64, uint64_t, "q")
|
|
|
|
|
CK_PR_CAS_O_S(32, uint32_t, "l")
|
|
|
|
|
CK_PR_CAS_O_S(16, uint16_t, "w")
|
|
|
|
|
CK_PR_CAS_O_S(8, uint8_t, "b")
|
|
|
|
|
CK_PR_CAS_S(64, uint64_t, "cmpxchgq")
|
|
|
|
|
CK_PR_CAS_S(32, uint32_t, "cmpxchgl")
|
|
|
|
|
CK_PR_CAS_S(16, uint16_t, "cmpxchgw")
|
|
|
|
|
CK_PR_CAS_S(8, uint8_t, "cmpxchgb")
|
|
|
|
|
|
|
|
|
|
#undef CK_PR_CAS_O_S
|
|
|
|
|
#undef CK_PR_CAS_O
|
|
|
|
|
#undef CK_PR_CAS_S
|
|
|
|
|
#undef CK_PR_CAS
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Contrary to C-interface, alignment requirements are that of uint64_t[2].
|
|
|
|
|