Commit e6ce9d74 authored by Uros Bizjak's avatar Uros Bizjak Committed by Ingo Molnar
Browse files

locking/atomic: Add generic try_cmpxchg{,64}_local() support 



Add generic support for try_cmpxchg{,64}_local() and their falbacks.

These provides the generic try_cmpxchg_local family of functions
from the arch_ prefixed version, also adding explicit instrumentation.

Signed-off-by: default avatarUros Bizjak <ubizjak@gmail.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarMark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20230405141710.3551-2-ubizjak@gmail.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
parent 286deb7e

include/linux/atomic/atomic-arch-fallback.h

+23 −1
Original line number Diff line number Diff line
@@ -217,6 +217,28 @@ 


#endif /* arch_try_cmpxchg64_relaxed */



#ifndef arch_try_cmpxchg_local

#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_local */



#ifndef arch_try_cmpxchg64_local

#define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg64_local */



#ifndef arch_atomic_read_acquire

static __always_inline int

arch_atomic_read_acquire(const atomic_t *v)
@@ -2646,4 +2668,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) 
#endif



#endif /* _LINUX_ATOMIC_FALLBACK_H */

// 00071fffa021cec66f6290d706d69c91df87bade
 
// ad2e2b4d168dbc60a73922616047a9bfa446af36

include/linux/atomic/atomic-instrumented.h

+19 −1
Original line number Diff line number Diff line
@@ -2132,6 +2132,24 @@ atomic_long_dec_if_positive(atomic_long_t *v) 
	arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \

})



#define try_cmpxchg_local(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})



#define try_cmpxchg64_local(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})



#define cmpxchg_double(ptr, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \
@@ -2149,4 +2167,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) 
})



#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */

// 1b485de9cbaa4900de59e14ee2084357eaeb1c3a
 
// 97fe4d79aa058d2164df824632cbc4f716d2a407

scripts/atomic/gen-atomic-fallback.sh

+4 −0
Original line number Diff line number Diff line
@@ -225,6 +225,10 @@ for cmpxchg in "cmpxchg" "cmpxchg64"; do 
	gen_try_cmpxchg_fallbacks "${cmpxchg}"

done



for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do

	gen_try_cmpxchg_fallback "${cmpxchg}" ""

done



grep '^[a-z]' "$1" | while read name meta args; do

	gen_proto "${meta}" "${name}" "atomic" "int" ${args}

done

scripts/atomic/gen-atomic-instrumented.sh

+1 −1
Original line number Diff line number Diff line
@@ -173,7 +173,7 @@ for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do 
	done

done



for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do

for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do

	gen_xchg "${xchg}" "" ""

	printf "\n"

done