Commit 2319be13 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'locking-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull locking updates from Ingo Molnar:

 - rwsem cleanups & optimizations/fixes:
    - Conditionally wake waiters in reader/writer slowpaths
    - Always try to wake waiters in out_nolock path

 - Add try_cmpxchg64() implementation, with arch optimizations - and use
   it to micro-optimize sched_clock_{local,remote}()

 - Various force-inlining fixes to address objdump instrumentation-check
   warnings

 - Add lock contention tracepoints:

    lock:contention_begin
    lock:contention_end

 - Misc smaller fixes & cleanups

* tag 'locking-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched/clock: Use try_cmpxchg64 in sched_clock_{local,remote}
  locking/atomic/x86: Introduce arch_try_cmpxchg64
  locking/atomic: Add generic try_cmpxchg64 support
  futex: Remove a PREEMPT_RT_FULL reference.
  locking/qrwlock: Change "queue rwlock" to "queued rwlock"
  lockdep: Delete local_irq_enable_in_hardirq()
  locking/mutex: Make contention tracepoints more consistent wrt adaptive spinning
  locking: Apply contention tracepoints in the slow path
  locking: Add lock contention tracepoints
  locking/rwsem: Always try to wake waiters in out_nolock path
  locking/rwsem: Conditionally wake waiters in reader/writer slowpaths
  locking/rwsem: No need to check for handoff bit if wait queue empty
  lockdep: Fix -Wunused-parameter for _THIS_IP_
  x86/mm: Force-inline __phys_addr_nodebug()
  x86/kvm/svm: Force-inline GHCB accessors
  task_stack, x86/cea: Force-inline stack helpers
parents 143a6252 8491d1bd

arch/arm64/kernel/entry-common.c

+4 −4
Original line number Diff line number Diff line
@@ -75,7 +75,7 @@ static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) 
	if (interrupts_enabled(regs)) {

		if (regs->exit_rcu) {

			trace_hardirqs_on_prepare();

			lockdep_hardirqs_on_prepare(CALLER_ADDR0);

			lockdep_hardirqs_on_prepare();

			rcu_irq_exit();

			lockdep_hardirqs_on(CALLER_ADDR0);

			return;
@@ -121,7 +121,7 @@ static __always_inline void enter_from_user_mode(struct pt_regs *regs) 
static __always_inline void __exit_to_user_mode(void)

{

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	lockdep_hardirqs_on_prepare();

	user_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);

}
@@ -179,7 +179,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs) 
	ftrace_nmi_exit();

	if (restore) {

		trace_hardirqs_on_prepare();

		lockdep_hardirqs_on_prepare(CALLER_ADDR0);

		lockdep_hardirqs_on_prepare();

	}



	rcu_nmi_exit();
@@ -215,7 +215,7 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) 


	if (restore) {

		trace_hardirqs_on_prepare();

		lockdep_hardirqs_on_prepare(CALLER_ADDR0);

		lockdep_hardirqs_on_prepare();

	}



	rcu_nmi_exit();

arch/x86/include/asm/cmpxchg_32.h

+21 −0
Original line number Diff line number Diff line
@@ -42,6 +42,9 @@ static inline void set_64bit(volatile u64 *ptr, u64 value) 
#define arch_cmpxchg64_local(ptr, o, n)					\

	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \

					       (unsigned long long)(n)))

#define arch_try_cmpxchg64(ptr, po, n)					\

	__try_cmpxchg64((ptr), (unsigned long long *)(po), \

			(unsigned long long)(n))

#endif



static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
@@ -70,6 +73,24 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 
	return prev;

}



static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)

{

	bool success;

	u64 old = *pold;

	asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]"

		     CC_SET(z)

		     : CC_OUT(z) (success),

		       [ptr] "+m" (*ptr),

		       "+A" (old)

		     : "b" ((u32)new),

		       "c" ((u32)(new >> 32))

		     : "memory");



	if (unlikely(!success))

		*pold = old;

	return success;

}



#ifndef CONFIG_X86_CMPXCHG64

/*

 * Building a kernel capable running on 80386 and 80486. It may be necessary

arch/x86/include/asm/cmpxchg_64.h

+6 −0
Original line number Diff line number Diff line
@@ -19,6 +19,12 @@ static inline void set_64bit(volatile u64 *ptr, u64 val) 
	arch_cmpxchg_local((ptr), (o), (n));				\

})



#define arch_try_cmpxchg64(ptr, po, n)					\

({									\

	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\

	arch_try_cmpxchg((ptr), (po), (n));				\

})



#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16)



#endif /* _ASM_X86_CMPXCHG_64_H */

arch/x86/include/asm/cpu_entry_area.h

+1 −1
Original line number Diff line number Diff line
@@ -143,7 +143,7 @@ extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags); 


extern struct cpu_entry_area *get_cpu_entry_area(int cpu);



static inline struct entry_stack *cpu_entry_stack(int cpu)

static __always_inline struct entry_stack *cpu_entry_stack(int cpu)

{

	return &get_cpu_entry_area(cpu)->entry_stack_page.stack;

}

arch/x86/include/asm/page_64.h

+1 −1
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ extern unsigned long page_offset_base; 
extern unsigned long vmalloc_base;

extern unsigned long vmemmap_base;



static inline unsigned long __phys_addr_nodebug(unsigned long x)

static __always_inline unsigned long __phys_addr_nodebug(unsigned long x)

{

	unsigned long y = x - __START_KERNEL_map;