Commit 61b7cddf authored by Will Deacon's avatar Will Deacon
Browse files

Merge branch 'for-next/atomics' into for-next/core 

* for-next/atomics: (10 commits)
  Rework LSE instruction selection to use static keys instead of alternatives
parents ac12cf85 03adcbd9

arch/arm64/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -1290,6 +1290,7 @@ config ARM64_PAN 


config ARM64_LSE_ATOMICS

	bool "Atomic instructions"

	depends on JUMP_LABEL

	default y

	help

	  As part of the Large System Extensions, ARMv8.1 introduces new

arch/arm64/Makefile

+8 −1
Original line number Diff line number Diff line
@@ -39,6 +39,12 @@ $(warning LSE atomics not supported by binutils) 
  endif

endif



cc_has_k_constraint := $(call try-run,echo				\

	'int main(void) {						\

		asm volatile("and w0, w0, %w0" :: "K" (4294967295));	\

		return 0;						\

	}' | $(CC) -S -x c -o "$$TMP" -,,-DCONFIG_CC_HAS_K_CONSTRAINT=1)



ifeq ($(CONFIG_ARM64), y)

brokengasinst := $(call as-instr,1:\n.inst 0\n.rept . - 1b\n\nnop\n.endr\n,,-DCONFIG_BROKEN_GAS_INST=1)
@@ -63,7 +69,8 @@ ifeq ($(CONFIG_GENERIC_COMPAT_VDSO), y) 
  endif

endif



KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr) $(brokengasinst) $(compat_vdso)

KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr) $(brokengasinst)	\

		   $(compat_vdso) $(cc_has_k_constraint)

KBUILD_CFLAGS	+= -fno-asynchronous-unwind-tables

KBUILD_CFLAGS	+= $(call cc-disable-warning, psabi)

KBUILD_AFLAGS	+= $(lseinstr) $(brokengasinst) $(compat_vdso)

arch/arm64/include/asm/atomic.h

+83 −11
Original line number Diff line number Diff line
@@ -13,19 +13,91 @@ 
#include <linux/types.h>



#include <asm/barrier.h>

#include <asm/cmpxchg.h>

#include <asm/lse.h>



#define __ARM64_IN_ATOMIC_IMPL



#if defined(CONFIG_ARM64_LSE_ATOMICS) && defined(CONFIG_AS_LSE)

#include <asm/atomic_lse.h>

#else

#include <asm/atomic_ll_sc.h>

#endif



#undef __ARM64_IN_ATOMIC_IMPL



#include <asm/cmpxchg.h>

#define ATOMIC_OP(op)							\

static inline void arch_##op(int i, atomic_t *v)			\

{									\

	__lse_ll_sc_body(op, i, v);					\

}



ATOMIC_OP(atomic_andnot)

ATOMIC_OP(atomic_or)

ATOMIC_OP(atomic_xor)

ATOMIC_OP(atomic_add)

ATOMIC_OP(atomic_and)

ATOMIC_OP(atomic_sub)



#undef ATOMIC_OP



#define ATOMIC_FETCH_OP(name, op)					\

static inline int arch_##op##name(int i, atomic_t *v)			\

{									\

	return __lse_ll_sc_body(op##name, i, v);			\

}



#define ATOMIC_FETCH_OPS(op)						\

	ATOMIC_FETCH_OP(_relaxed, op)					\

	ATOMIC_FETCH_OP(_acquire, op)					\

	ATOMIC_FETCH_OP(_release, op)					\

	ATOMIC_FETCH_OP(        , op)



ATOMIC_FETCH_OPS(atomic_fetch_andnot)

ATOMIC_FETCH_OPS(atomic_fetch_or)

ATOMIC_FETCH_OPS(atomic_fetch_xor)

ATOMIC_FETCH_OPS(atomic_fetch_add)

ATOMIC_FETCH_OPS(atomic_fetch_and)

ATOMIC_FETCH_OPS(atomic_fetch_sub)

ATOMIC_FETCH_OPS(atomic_add_return)

ATOMIC_FETCH_OPS(atomic_sub_return)



#undef ATOMIC_FETCH_OP

#undef ATOMIC_FETCH_OPS



#define ATOMIC64_OP(op)							\

static inline void arch_##op(long i, atomic64_t *v)			\

{									\

	__lse_ll_sc_body(op, i, v);					\

}



ATOMIC64_OP(atomic64_andnot)

ATOMIC64_OP(atomic64_or)

ATOMIC64_OP(atomic64_xor)

ATOMIC64_OP(atomic64_add)

ATOMIC64_OP(atomic64_and)

ATOMIC64_OP(atomic64_sub)



#undef ATOMIC64_OP



#define ATOMIC64_FETCH_OP(name, op)					\

static inline long arch_##op##name(long i, atomic64_t *v)		\

{									\

	return __lse_ll_sc_body(op##name, i, v);			\

}



#define ATOMIC64_FETCH_OPS(op)						\

	ATOMIC64_FETCH_OP(_relaxed, op)					\

	ATOMIC64_FETCH_OP(_acquire, op)					\

	ATOMIC64_FETCH_OP(_release, op)					\

	ATOMIC64_FETCH_OP(        , op)



ATOMIC64_FETCH_OPS(atomic64_fetch_andnot)

ATOMIC64_FETCH_OPS(atomic64_fetch_or)

ATOMIC64_FETCH_OPS(atomic64_fetch_xor)

ATOMIC64_FETCH_OPS(atomic64_fetch_add)

ATOMIC64_FETCH_OPS(atomic64_fetch_and)

ATOMIC64_FETCH_OPS(atomic64_fetch_sub)

ATOMIC64_FETCH_OPS(atomic64_add_return)

ATOMIC64_FETCH_OPS(atomic64_sub_return)



#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_FETCH_OPS



static inline long arch_atomic64_dec_if_positive(atomic64_t *v)

{

	return __lse_ll_sc_body(atomic64_dec_if_positive, v);

}



#define ATOMIC_INIT(i)	{ (i) }

arch/arm64/include/asm/atomic_ll_sc.h

+121 −96
Original line number Diff line number Diff line
@@ -10,83 +10,92 @@ 
#ifndef __ASM_ATOMIC_LL_SC_H

#define __ASM_ATOMIC_LL_SC_H



#ifndef __ARM64_IN_ATOMIC_IMPL

#error "please don't include this file directly"

#include <linux/stringify.h>



#if IS_ENABLED(CONFIG_ARM64_LSE_ATOMICS) && IS_ENABLED(CONFIG_AS_LSE)

#define __LL_SC_FALLBACK(asm_ops)					\

"	b	3f\n"							\

"	.subsection	1\n"						\

"3:\n"									\

asm_ops "\n"								\

"	b	4f\n"							\

"	.previous\n"							\

"4:\n"

#else

#define __LL_SC_FALLBACK(asm_ops) asm_ops

#endif



#ifndef CONFIG_CC_HAS_K_CONSTRAINT

#define K

#endif



/*

 * AArch64 UP and SMP safe atomic ops.  We use load exclusive and

 * store exclusive to ensure that these are atomic.  We may loop

 * to ensure that the update happens.

 *

 * NOTE: these functions do *not* follow the PCS and must explicitly

 * save any clobbered registers other than x0 (regardless of return

 * value).  This is achieved through -fcall-saved-* compiler flags for

 * this file, which unfortunately don't work on a per-function basis

 * (the optimize attribute silently ignores these options).

 */



#define ATOMIC_OP(op, asm_op)						\

__LL_SC_INLINE void							\

__LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v))			\

#define ATOMIC_OP(op, asm_op, constraint)				\

static inline void							\

__ll_sc_atomic_##op(int i, atomic_t *v)					\

{									\

	unsigned long tmp;						\

	int result;							\

									\

	asm volatile("// atomic_" #op "\n"				\

	__LL_SC_FALLBACK(						\

"	prfm	pstl1strm, %2\n"					\

"1:	ldxr	%w0, %2\n"						\

"	" #asm_op "	%w0, %w0, %w3\n"				\

"	stxr	%w1, %w0, %2\n"						\

"	cbnz	%w1, 1b"						\

"	cbnz	%w1, 1b\n")						\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i));							\

}									\

__LL_SC_EXPORT(arch_atomic_##op);

	: __stringify(constraint) "r" (i));				\

}



#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op)		\

__LL_SC_INLINE int							\

__LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v))	\

#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\

static inline int							\

__ll_sc_atomic_##op##_return##name(int i, atomic_t *v)			\

{									\

	unsigned long tmp;						\

	int result;							\

									\

	asm volatile("// atomic_" #op "_return" #name "\n"		\

	__LL_SC_FALLBACK(						\

"	prfm	pstl1strm, %2\n"					\

"1:	ld" #acq "xr	%w0, %2\n"					\

"	" #asm_op "	%w0, %w0, %w3\n"				\

"	st" #rel "xr	%w1, %w0, %2\n"					\

"	cbnz	%w1, 1b\n"						\

"	" #mb								\

"	" #mb )								\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i)							\

	: __stringify(constraint) "r" (i)				\

	: cl);								\

									\

	return result;							\

}									\

__LL_SC_EXPORT(arch_atomic_##op##_return##name);

}



#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op)		\

__LL_SC_INLINE int							\

__LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v))	\

#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \

static inline int							\

__ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)			\

{									\

	unsigned long tmp;						\

	int val, result;						\

									\

	asm volatile("// atomic_fetch_" #op #name "\n"			\

	__LL_SC_FALLBACK(						\

"	prfm	pstl1strm, %3\n"					\

"1:	ld" #acq "xr	%w0, %3\n"					\

"	" #asm_op "	%w1, %w0, %w4\n"				\

"	st" #rel "xr	%w2, %w1, %3\n"					\

"	cbnz	%w2, 1b\n"						\

"	" #mb								\

"	" #mb )								\

	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\

	: "Ir" (i)							\

	: __stringify(constraint) "r" (i)				\

	: cl);								\

									\

	return result;							\

}									\

__LL_SC_EXPORT(arch_atomic_fetch_##op##name);

}



#define ATOMIC_OPS(...)							\

	ATOMIC_OP(__VA_ARGS__)						\
@@ -99,8 +108,8 @@ __LL_SC_EXPORT(arch_atomic_fetch_##op##name); 
	ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)



ATOMIC_OPS(add, add)

ATOMIC_OPS(sub, sub)

ATOMIC_OPS(add, add, I)

ATOMIC_OPS(sub, sub, J)



#undef ATOMIC_OPS

#define ATOMIC_OPS(...)							\
@@ -110,77 +119,82 @@ ATOMIC_OPS(sub, sub) 
	ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)



ATOMIC_OPS(and, and)

ATOMIC_OPS(andnot, bic)

ATOMIC_OPS(or, orr)

ATOMIC_OPS(xor, eor)

ATOMIC_OPS(and, and, K)

ATOMIC_OPS(or, orr, K)

ATOMIC_OPS(xor, eor, K)

/*

 * GAS converts the mysterious and undocumented BIC (immediate) alias to

 * an AND (immediate) instruction with the immediate inverted. We don't

 * have a constraint for this, so fall back to register.

 */

ATOMIC_OPS(andnot, bic, )



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#define ATOMIC64_OP(op, asm_op)						\

__LL_SC_INLINE void							\

__LL_SC_PREFIX(arch_atomic64_##op(s64 i, atomic64_t *v))		\

#define ATOMIC64_OP(op, asm_op, constraint)				\

static inline void							\

__ll_sc_atomic64_##op(s64 i, atomic64_t *v)				\

{									\

	s64 result;							\

	unsigned long tmp;						\

									\

	asm volatile("// atomic64_" #op "\n"				\

	__LL_SC_FALLBACK(						\

"	prfm	pstl1strm, %2\n"					\

"1:	ldxr	%0, %2\n"						\

"	" #asm_op "	%0, %0, %3\n"					\

"	stxr	%w1, %0, %2\n"						\

"	cbnz	%w1, 1b"						\

"	cbnz	%w1, 1b")						\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i));							\

}									\

__LL_SC_EXPORT(arch_atomic64_##op);

	: __stringify(constraint) "r" (i));				\

}



#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op)		\

__LL_SC_INLINE s64							\

__LL_SC_PREFIX(arch_atomic64_##op##_return##name(s64 i, atomic64_t *v))\

#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\

static inline long							\

__ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v)		\

{									\

	s64 result;							\

	unsigned long tmp;						\

									\

	asm volatile("// atomic64_" #op "_return" #name "\n"		\

	__LL_SC_FALLBACK(						\

"	prfm	pstl1strm, %2\n"					\

"1:	ld" #acq "xr	%0, %2\n"					\

"	" #asm_op "	%0, %0, %3\n"					\

"	st" #rel "xr	%w1, %0, %2\n"					\

"	cbnz	%w1, 1b\n"						\

"	" #mb								\

"	" #mb )								\

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\

	: "Ir" (i)							\

	: __stringify(constraint) "r" (i)				\

	: cl);								\

									\

	return result;							\

}									\

__LL_SC_EXPORT(arch_atomic64_##op##_return##name);

}



#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op)		\

__LL_SC_INLINE s64							\

__LL_SC_PREFIX(arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v))	\

#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\

static inline long							\

__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v)		\

{									\

	s64 result, val;						\

	unsigned long tmp;						\

									\

	asm volatile("// atomic64_fetch_" #op #name "\n"		\

	__LL_SC_FALLBACK(						\

"	prfm	pstl1strm, %3\n"					\

"1:	ld" #acq "xr	%0, %3\n"					\

"	" #asm_op "	%1, %0, %4\n"					\

"	st" #rel "xr	%w2, %1, %3\n"					\

"	cbnz	%w2, 1b\n"						\

"	" #mb								\

"	" #mb )								\

	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\

	: "Ir" (i)							\

	: __stringify(constraint) "r" (i)				\

	: cl);								\

									\

	return result;							\

}									\

__LL_SC_EXPORT(arch_atomic64_fetch_##op##name);

}



#define ATOMIC64_OPS(...)						\

	ATOMIC64_OP(__VA_ARGS__)					\
@@ -193,8 +207,8 @@ __LL_SC_EXPORT(arch_atomic64_fetch_##op##name); 
	ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)



ATOMIC64_OPS(add, add)

ATOMIC64_OPS(sub, sub)

ATOMIC64_OPS(add, add, I)

ATOMIC64_OPS(sub, sub, J)



#undef ATOMIC64_OPS

#define ATOMIC64_OPS(...)						\
@@ -204,23 +218,29 @@ ATOMIC64_OPS(sub, sub) 
	ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)



ATOMIC64_OPS(and, and)

ATOMIC64_OPS(andnot, bic)

ATOMIC64_OPS(or, orr)

ATOMIC64_OPS(xor, eor)

ATOMIC64_OPS(and, and, L)

ATOMIC64_OPS(or, orr, L)

ATOMIC64_OPS(xor, eor, L)

/*

 * GAS converts the mysterious and undocumented BIC (immediate) alias to

 * an AND (immediate) instruction with the immediate inverted. We don't

 * have a constraint for this, so fall back to register.

 */

ATOMIC64_OPS(andnot, bic, )



#undef ATOMIC64_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



__LL_SC_INLINE s64

__LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))

static inline s64

__ll_sc_atomic64_dec_if_positive(atomic64_t *v)

{

	s64 result;

	unsigned long tmp;



	asm volatile("// atomic64_dec_if_positive\n"

	__LL_SC_FALLBACK(

"	prfm	pstl1strm, %2\n"

"1:	ldxr	%0, %2\n"

"	subs	%0, %0, #1\n"
@@ -228,20 +248,19 @@ __LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v)) 
"	stlxr	%w1, %0, %2\n"

"	cbnz	%w1, 1b\n"

"	dmb	ish\n"

"2:"

"2:")

	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)

	:

	: "cc", "memory");



	return result;

}

__LL_SC_EXPORT(arch_atomic64_dec_if_positive);



#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl)		\

__LL_SC_INLINE u##sz							\

__LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr,		\

#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)	\

static inline u##sz							\

__ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\

					 unsigned long old,		\

					 u##sz new))			\

					 u##sz new)			\

{									\

	unsigned long tmp;						\

	u##sz oldval;							\
@@ -255,6 +274,7 @@ __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \ 
		old = (u##sz)old;					\

									\

	asm volatile(							\

	__LL_SC_FALLBACK(						\

	"	prfm	pstl1strm, %[v]\n"				\

	"1:	ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"		\

	"	eor	%" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"	\
@@ -262,46 +282,51 @@ __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \ 
	"	st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"	\

	"	cbnz	%w[tmp], 1b\n"					\

	"	" #mb "\n"						\

	"2:"								\

	"2:")								\

	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),			\

	  [v] "+Q" (*(u##sz *)ptr)					\

	: [old] "Kr" (old), [new] "r" (new)				\

	: [old] __stringify(constraint) "r" (old), [new] "r" (new)	\

	: cl);								\

									\

	return oldval;							\

}									\

__LL_SC_EXPORT(__cmpxchg_case_##name##sz);

}



__CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         )

__CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         )

__CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         )

__CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         )

__CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory")

__CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory")

__CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory")

__CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory")

__CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory")

__CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory")

__CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory")

__CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory")

__CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory")

__CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory")

__CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory")

__CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory")

/*

 * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly

 * handle the 'K' constraint for the value 4294967295 - thus we use no

 * constraint for 32 bit operations.

 */

__CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , K)

__CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , K)

__CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , K)

__CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)

__CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", K)

__CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", K)

__CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", K)

__CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)

__CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", K)

__CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", K)

__CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", K)

__CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)

__CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", K)

__CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", K)

__CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", K)

__CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)



#undef __CMPXCHG_CASE



#define __CMPXCHG_DBL(name, mb, rel, cl)				\

__LL_SC_INLINE long							\

__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,		\

static inline long							\

__ll_sc__cmpxchg_double##name(unsigned long old1,			\

				      unsigned long old2,		\

				      unsigned long new1,		\

				      unsigned long new2,		\

				      volatile void *ptr))		\

				      volatile void *ptr)		\

{									\

	unsigned long tmp, ret;						\

									\

	asm volatile("// __cmpxchg_double" #name "\n"			\

	__LL_SC_FALLBACK(						\

	"	prfm	pstl1strm, %2\n"				\

	"1:	ldxp	%0, %1, %2\n"					\

	"	eor	%0, %0, %3\n"					\
@@ -311,18 +336,18 @@ __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \ 
	"	st" #rel "xp	%w0, %5, %6, %2\n"			\

	"	cbnz	%w0, 1b\n"					\

	"	" #mb "\n"						\

	"2:"								\

	"2:")								\

	: "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)	\

	: "r" (old1), "r" (old2), "r" (new1), "r" (new2)		\

	: cl);								\

									\

	return ret;							\

}									\

__LL_SC_EXPORT(__cmpxchg_double##name);

}



__CMPXCHG_DBL(   ,        ,  ,         )

__CMPXCHG_DBL(_mb, dmb ish, l, "memory")



#undef __CMPXCHG_DBL

#undef K



#endif	/* __ASM_ATOMIC_LL_SC_H */

arch/arm64/include/asm/atomic_lse.h

+135 −260

File changed.

Preview size limit exceeded, changes collapsed.