Commit 76d4acf2 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'perf-kprobes-2020-12-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull perf/kprobes updates from Thomas Gleixner:
 "Make kretprobes lockless to avoid the rp->lock performance and
  potential lock ordering issues"

* tag 'perf-kprobes-2020-12-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  locking/atomics: Regenerate the atomics-check SHA1's
  kprobes: Replace rp->free_instance with freelist
  freelist: Implement lockless freelist
  asm-generic/atomic: Add try_cmpxchg() fallbacks
  kprobes: Remove kretprobe hash
  llist: Add nonatomic __llist_add() and __llist_dell_all()
parents 8a8ca83e a70a04b3

arch/x86/include/asm/atomic.h

+1 −1
Original line number Diff line number Diff line
@@ -199,7 +199,7 @@ static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) 


static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)

{

	return try_cmpxchg(&v->counter, old, new);

	return arch_try_cmpxchg(&v->counter, old, new);

}

#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg

arch/x86/include/asm/atomic64_64.h

+1 −1
Original line number Diff line number Diff line
@@ -187,7 +187,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 


static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)

{

	return try_cmpxchg(&v->counter, old, new);

	return arch_try_cmpxchg(&v->counter, old, new);

}

#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg

arch/x86/include/asm/cmpxchg.h

+1 −1
Original line number Diff line number Diff line
@@ -221,7 +221,7 @@ extern void __add_wrong_size(void) 
#define __try_cmpxchg(ptr, pold, new, size)				\

	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)



#define try_cmpxchg(ptr, pold, new) 					\

#define arch_try_cmpxchg(ptr, pold, new) 				\

	__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))



/*

include/asm-generic/atomic-instrumented.h

+130 −86
Original line number Diff line number Diff line
@@ -1749,6 +1749,50 @@ atomic64_dec_if_positive(atomic64_t *v) 
})

#endif



#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg)

#define try_cmpxchg(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#if defined(arch_try_cmpxchg_acquire)

#define try_cmpxchg_acquire(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#if defined(arch_try_cmpxchg_release)

#define try_cmpxchg_release(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#if defined(arch_try_cmpxchg_relaxed)

#define try_cmpxchg_relaxed(ptr, oldp, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \

	typeof(oldp) __ai_oldp = (oldp); \

	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \

	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \

	arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \

})

#endif



#define cmpxchg_local(ptr, ...) \

({ \

	typeof(ptr) __ai_ptr = (ptr); \
@@ -1786,4 +1830,4 @@ atomic64_dec_if_positive(atomic64_t *v) 
})



#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */

// 9d5e6a315fb1335d02f0ccd3655a91c3dafcc63e
 
// 4bec382e44520f4d8267e42620054db26a659ea3

include/linux/atomic-arch-fallback.h

+80 −10
Original line number Diff line number Diff line
@@ -9,9 +9,9 @@ 
#include <linux/compiler.h>



#ifndef arch_xchg_relaxed

#define arch_xchg_relaxed		arch_xchg

#define arch_xchg_acquire arch_xchg

#define arch_xchg_release arch_xchg

#define arch_xchg_relaxed arch_xchg

#else /* arch_xchg_relaxed */



#ifndef arch_xchg_acquire
@@ -32,9 +32,9 @@ 
#endif /* arch_xchg_relaxed */



#ifndef arch_cmpxchg_relaxed

#define arch_cmpxchg_relaxed		arch_cmpxchg

#define arch_cmpxchg_acquire arch_cmpxchg

#define arch_cmpxchg_release arch_cmpxchg

#define arch_cmpxchg_relaxed arch_cmpxchg

#else /* arch_cmpxchg_relaxed */



#ifndef arch_cmpxchg_acquire
@@ -55,9 +55,9 @@ 
#endif /* arch_cmpxchg_relaxed */



#ifndef arch_cmpxchg64_relaxed

#define arch_cmpxchg64_relaxed		arch_cmpxchg64

#define arch_cmpxchg64_acquire arch_cmpxchg64

#define arch_cmpxchg64_release arch_cmpxchg64

#define arch_cmpxchg64_relaxed arch_cmpxchg64

#else /* arch_cmpxchg64_relaxed */



#ifndef arch_cmpxchg64_acquire
@@ -77,6 +77,76 @@ 


#endif /* arch_cmpxchg64_relaxed */



#ifndef arch_try_cmpxchg_relaxed

#ifdef arch_try_cmpxchg

#define arch_try_cmpxchg_acquire arch_try_cmpxchg

#define arch_try_cmpxchg_release arch_try_cmpxchg

#define arch_try_cmpxchg_relaxed arch_try_cmpxchg

#endif /* arch_try_cmpxchg */



#ifndef arch_try_cmpxchg

#define arch_try_cmpxchg(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg */



#ifndef arch_try_cmpxchg_acquire

#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_acquire */



#ifndef arch_try_cmpxchg_release

#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_release */



#ifndef arch_try_cmpxchg_relaxed

#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \

({ \

	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \

	___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \

	if (unlikely(___r != ___o)) \

		*___op = ___r; \

	likely(___r == ___o); \

})

#endif /* arch_try_cmpxchg_relaxed */



#else /* arch_try_cmpxchg_relaxed */



#ifndef arch_try_cmpxchg_acquire

#define arch_try_cmpxchg_acquire(...) \

	__atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)

#endif



#ifndef arch_try_cmpxchg_release

#define arch_try_cmpxchg_release(...) \

	__atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)

#endif



#ifndef arch_try_cmpxchg

#define arch_try_cmpxchg(...) \

	__atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)

#endif



#endif /* arch_try_cmpxchg_relaxed */



#ifndef arch_atomic_read_acquire

static __always_inline int

arch_atomic_read_acquire(const atomic_t *v)
@@ -2288,4 +2358,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) 
#endif



#endif /* _LINUX_ATOMIC_FALLBACK_H */

// 90cd26cfd69d2250303d654955a0cc12620fb91b
 
// cca554917d7ea73d5e3e7397dd70c484cad9b2c4