Commit 29f006fd authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

asm-generic/atomic: Add try_cmpxchg() fallbacks 



Only x86 provides try_cmpxchg() outside of the atomic_t interfaces,
provide generic fallbacks to create this interface from the widely
available cmpxchg() function.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarMasami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarWill Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/159870621515.1229682.15506193091065001742.stgit@devnote2
parent d741bf41

arch/x86/include/asm/atomic.h

+1 −1
Original line number Diff line number Diff line
@@ -199,7 +199,7 @@ static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) 


static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)

{

	return try_cmpxchg(&v->counter, old, new);

	return arch_try_cmpxchg(&v->counter, old, new);

}

#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg

arch/x86/include/asm/atomic64_64.h

+1 −1
Original line number Diff line number Diff line
@@ -187,7 +187,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 


static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)

{

	return try_cmpxchg(&v->counter, old, new);

	return arch_try_cmpxchg(&v->counter, old, new);

}

#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg

arch/x86/include/asm/cmpxchg.h

+1 −1
Original line number Diff line number Diff line
@@ -221,7 +221,7 @@ extern void __add_wrong_size(void) 
#define __try_cmpxchg(ptr, pold, new, size)				\

	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)



#define try_cmpxchg(ptr, pold, new) 					\

#define arch_try_cmpxchg(ptr, pold, new) 				\

	__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))



/*

include/asm-generic/atomic-instrumented.h

+130 −86
Original line number Diff line number Diff line
@@ -1749,6 +1749,50 @@ atomic64_dec_if_positive(atomic64_t *v) 
})

#endif



#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg)

#define try_cmpxchg(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#if defined(arch_try_cmpxchg_acquire)

#define try_cmpxchg_acquire(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#if defined(arch_try_cmpxchg_release)

#define try_cmpxchg_release(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#if defined(arch_try_cmpxchg_relaxed)

#define try_cmpxchg_relaxed(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#define cmpxchg_local(ptr, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \
@@ -1786,4 +1830,4 @@ atomic64_dec_if_positive(atomic64_t *v) 
})



#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */

// 89bf97f3a7509b740845e51ddf31055b48a81f40
 
// ff0fe7f81ee97f01f13bb78b0e3ce800bc56d9dd

include/linux/atomic-arch-fallback.h

+80 −10
Original line number Diff line number Diff line
@@ -9,9 +9,9 @@ 
#include <linux/compiler.h>



#ifndef arch_xchg_relaxed

#define arch_xchg_relaxed		arch_xchg

#define arch_xchg_acquire arch_xchg

#define arch_xchg_release arch_xchg

#define arch_xchg_relaxed arch_xchg

#else /* arch_xchg_relaxed */



#ifndef arch_xchg_acquire
@@ -32,9 +32,9 @@ 
#endif /* arch_xchg_relaxed */



#ifndef arch_cmpxchg_relaxed

#define arch_cmpxchg_relaxed		arch_cmpxchg

#define arch_cmpxchg_acquire arch_cmpxchg

#define arch_cmpxchg_release arch_cmpxchg

#define arch_cmpxchg_relaxed arch_cmpxchg

#else /* arch_cmpxchg_relaxed */



#ifndef arch_cmpxchg_acquire
@@ -55,9 +55,9 @@ 
#endif /* arch_cmpxchg_relaxed */



#ifndef arch_cmpxchg64_relaxed

#define arch_cmpxchg64_relaxed		arch_cmpxchg64

#define arch_cmpxchg64_acquire arch_cmpxchg64

#define arch_cmpxchg64_release arch_cmpxchg64

#define arch_cmpxchg64_relaxed arch_cmpxchg64

#else /* arch_cmpxchg64_relaxed */



#ifndef arch_cmpxchg64_acquire
@@ -77,6 +77,76 @@ 


#endif /* arch_cmpxchg64_relaxed */



#ifndef arch_try_cmpxchg_relaxed

#ifdef arch_try_cmpxchg

#define arch_try_cmpxchg_acquire arch_try_cmpxchg

#define arch_try_cmpxchg_release arch_try_cmpxchg

#define arch_try_cmpxchg_relaxed arch_try_cmpxchg

#endif /* arch_try_cmpxchg */



#ifndef arch_try_cmpxchg

#define arch_try_cmpxchg(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg */



#ifndef arch_try_cmpxchg_acquire

#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_acquire */



#ifndef arch_try_cmpxchg_release

#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_release */



#ifndef arch_try_cmpxchg_relaxed

#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_relaxed */



#else /* arch_try_cmpxchg_relaxed */



#ifndef arch_try_cmpxchg_acquire

#define arch_try_cmpxchg_acquire(...) \

	__atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)

#endif



#ifndef arch_try_cmpxchg_release

#define arch_try_cmpxchg_release(...) \

	__atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)

#endif



#ifndef arch_try_cmpxchg

#define arch_try_cmpxchg(...) \

	__atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)

#endif



#endif /* arch_try_cmpxchg_relaxed */



#ifndef arch_atomic_read_acquire

static __always_inline int

arch_atomic_read_acquire(const atomic_t *v)
@@ -2288,4 +2358,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) 
#endif



#endif /* _LINUX_ATOMIC_FALLBACK_H */

// 90cd26cfd69d2250303d654955a0cc12620fb91b
 
// cca554917d7ea73d5e3e7397dd70c484cad9b2c4