|
|
@ -54,17 +54,27 @@ ck_pr_stall(void)
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
|
|
|
* isb, dsb and dmb instructions only appeared with armv7, so use the old
|
|
|
|
#define __CK_ISB __asm __volatile("isb" : : "r" (0) : "memory")
|
|
|
|
* notation
|
|
|
|
#define _CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory")
|
|
|
|
*/
|
|
|
|
#define _CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory")
|
|
|
|
|
|
|
|
/* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */
|
|
|
|
|
|
|
|
#ifdef __FreeBSD__
|
|
|
|
|
|
|
|
#define _CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory")
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
#define _CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory")
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
#define _ISB \
|
|
|
|
#else
|
|
|
|
|
|
|
|
/* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */
|
|
|
|
|
|
|
|
#define _CK_ISB \
|
|
|
|
__asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
|
|
|
|
__asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
|
|
|
|
#define _DSB \
|
|
|
|
#define _CK_DSB \
|
|
|
|
__asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
|
|
|
|
__asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
|
|
|
|
#define _DMB \
|
|
|
|
#define _CK_DMB \
|
|
|
|
__asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
|
|
|
|
__asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
|
|
|
|
|
|
|
|
#define _CK_DMB_ST _CK_DMB
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
#define CK_PR_FENCE(T, I) \
|
|
|
|
#define CK_PR_FENCE(T, I) \
|
|
|
|
CK_CC_INLINE static void \
|
|
|
|
CK_CC_INLINE static void \
|
|
|
@ -73,22 +83,23 @@ ck_pr_stall(void)
|
|
|
|
I; \
|
|
|
|
I; \
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
CK_PR_FENCE(atomic, _DMB)
|
|
|
|
CK_PR_FENCE(atomic, _CK_DMB_ST)
|
|
|
|
CK_PR_FENCE(atomic_store, _DMB)
|
|
|
|
CK_PR_FENCE(atomic_store, _CK_DMB_ST)
|
|
|
|
CK_PR_FENCE(atomic_load, _DMB)
|
|
|
|
CK_PR_FENCE(atomic_load, _CK_DMB_ST)
|
|
|
|
CK_PR_FENCE(store_atomic, _DMB)
|
|
|
|
CK_PR_FENCE(store_atomic, _CK_DMB_ST)
|
|
|
|
CK_PR_FENCE(load_atomic, _DMB)
|
|
|
|
CK_PR_FENCE(load_atomic, _CK_DMB)
|
|
|
|
CK_PR_FENCE(store, _DMB)
|
|
|
|
CK_PR_FENCE(store, _CK_DMB_ST)
|
|
|
|
CK_PR_FENCE(store_load, _DMB)
|
|
|
|
CK_PR_FENCE(store_load, _CK_DMB_ST)
|
|
|
|
CK_PR_FENCE(load, _DMB)
|
|
|
|
CK_PR_FENCE(load, _CK_DMB)
|
|
|
|
CK_PR_FENCE(load_store, _DMB)
|
|
|
|
CK_PR_FENCE(load_store, _CK_DMB)
|
|
|
|
CK_PR_FENCE(memory, _DMB)
|
|
|
|
CK_PR_FENCE(memory, _CK_DMB)
|
|
|
|
|
|
|
|
|
|
|
|
#undef CK_PR_FENCE
|
|
|
|
#undef CK_PR_FENCE
|
|
|
|
|
|
|
|
|
|
|
|
#undef _ISB
|
|
|
|
#undef _CK_ISB
|
|
|
|
#undef _DSB
|
|
|
|
#undef _CK_DSB
|
|
|
|
#undef _DMB
|
|
|
|
#undef _CK_DMB
|
|
|
|
|
|
|
|
#undef _CK_DMB_ST
|
|
|
|
|
|
|
|
|
|
|
|
#define CK_PR_LOAD(S, M, T, C, I) \
|
|
|
|
#define CK_PR_LOAD(S, M, T, C, I) \
|
|
|
|
CK_CC_INLINE static T \
|
|
|
|
CK_CC_INLINE static T \
|
|
|
|