|
|
@ -144,7 +144,7 @@ CK_PR_FAS_S(8, uint8_t, "xchgb")
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
#define CK_PR_LOAD(S, M, T, C, I) \
|
|
|
|
#define CK_PR_LOAD(S, M, T, C, I) \
|
|
|
|
CK_CC_INLINE static T \
|
|
|
|
CK_CC_INLINE static T \
|
|
|
|
ck_pr_load_##S(M *target) \
|
|
|
|
ck_pr_load_##S(const M *target) \
|
|
|
|
{ \
|
|
|
|
{ \
|
|
|
|
T r; \
|
|
|
|
T r; \
|
|
|
|
__asm__ __volatile__(I " %1, %0" \
|
|
|
|
__asm__ __volatile__(I " %1, %0" \
|
|
|
@ -171,15 +171,14 @@ CK_PR_LOAD_S(8, uint8_t, "movb")
|
|
|
|
#undef CK_PR_LOAD
|
|
|
|
#undef CK_PR_LOAD
|
|
|
|
|
|
|
|
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
CK_CC_INLINE static void
|
|
|
|
ck_pr_load_64_2(uint64_t target[2], uint64_t v[2])
|
|
|
|
ck_pr_load_64_2(const uint64_t target[2], uint64_t v[2])
|
|
|
|
{
|
|
|
|
{
|
|
|
|
__asm__ __volatile__("movq %%rdx, %%rcx;"
|
|
|
|
__asm__ __volatile__("movq %%rdx, %%rcx;"
|
|
|
|
"movq %%rax, %%rbx;"
|
|
|
|
"movq %%rax, %%rbx;"
|
|
|
|
CK_PR_LOCK_PREFIX "cmpxchg16b %0;"
|
|
|
|
CK_PR_LOCK_PREFIX "cmpxchg16b %0;"
|
|
|
|
: "+m" (*(uint64_t *)target),
|
|
|
|
: "=a" (v[0]),
|
|
|
|
"=a" (v[0]),
|
|
|
|
|
|
|
|
"=d" (v[1])
|
|
|
|
"=d" (v[1])
|
|
|
|
:
|
|
|
|
: "m" (*(const uint64_t *)target)
|
|
|
|
: "rbx", "rcx", "memory", "cc");
|
|
|
|
: "rbx", "rcx", "memory", "cc");
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -193,9 +192,10 @@ ck_pr_load_ptr_2(void *t, void *v)
|
|
|
|
|
|
|
|
|
|
|
|
#define CK_PR_LOAD_2(S, W, T) \
|
|
|
|
#define CK_PR_LOAD_2(S, W, T) \
|
|
|
|
CK_CC_INLINE static void \
|
|
|
|
CK_CC_INLINE static void \
|
|
|
|
ck_pr_load_##S##_##W(T t[2], T v[2]) \
|
|
|
|
ck_pr_load_##S##_##W(const T t[2], T v[2]) \
|
|
|
|
{ \
|
|
|
|
{ \
|
|
|
|
ck_pr_load_64_2((uint64_t *)(void *)t, (uint64_t *)(void *)v); \
|
|
|
|
ck_pr_load_64_2((const uint64_t *)(const void *)t, \
|
|
|
|
|
|
|
|
(uint64_t *)(void *)v); \
|
|
|
|
return; \
|
|
|
|
return; \
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -222,7 +222,7 @@ CK_PR_LOAD_2(8, 16, uint8_t)
|
|
|
|
return; \
|
|
|
|
return; \
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
CK_PR_STORE(ptr, void, void *, char, "movq")
|
|
|
|
CK_PR_STORE(ptr, void, const void *, char, "movq")
|
|
|
|
|
|
|
|
|
|
|
|
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
|
|
|
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
|
|
|
|
|
|
|
|
|
|
|