Commit 501f7f69 authored by Namhyung Kim's avatar Namhyung Kim Committed by Peter Zijlstra
Browse files

locking: Add __lockfunc to slow path functions 



So that we can skip the functions in the perf lock contention and other
places like /proc/PID/wchan.

Signed-off-by: default avatarNamhyung Kim <namhyung@kernel.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarWaiman Long <longman@redhat.com>
Link: https://lore.kernel.org/r/20220810220346.1919485-1-namhyung@kernel.org
parent 9aeaf5bc

arch/x86/include/asm/qspinlock_paravirt.h

+7 −6
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
 */

#ifdef CONFIG_64BIT



PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);

__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");

#define __pv_queued_spin_unlock	__pv_queued_spin_unlock

#define PV_UNLOCK		"__raw_callee_save___pv_queued_spin_unlock"

#define PV_UNLOCK_SLOWPATH	"__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath); 
/*

 * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock

 * which combines the registers saving trunk and the body of the following

 * C code:

 * C code.  Note that it puts the code in the .spinlock.text section which

 * is equivalent to adding __lockfunc in the C code:

 *

 * void __pv_queued_spin_unlock(struct qspinlock *lock)

 * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)

 * {

 *	u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);

 *
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath); 
 *   rsi = lockval           (second argument)

 *   rdx = internal variable (set to 0)

 */

asm    (".pushsection .text;"

asm    (".pushsection .spinlock.text;"

	".globl " PV_UNLOCK ";"

	".type " PV_UNLOCK ", @function;"

	".align 4,0x90;"
@@ -65,8 +66,8 @@ asm (".pushsection .text;" 


#else /* CONFIG_64BIT */



extern void __pv_queued_spin_unlock(struct qspinlock *lock);

PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);

extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);

__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");



#endif /* CONFIG_64BIT */

#endif

kernel/locking/qrwlock.c

+2 −2
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ 
 * queued_read_lock_slowpath - acquire read lock of a queued rwlock

 * @lock: Pointer to queued rwlock structure

 */

void queued_read_lock_slowpath(struct qrwlock *lock)

void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)

{

	/*

	 * Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath); 
 * queued_write_lock_slowpath - acquire write lock of a queued rwlock

 * @lock : Pointer to queued rwlock structure

 */

void queued_write_lock_slowpath(struct qrwlock *lock)

void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)

{

	int cnts;

kernel/locking/qspinlock.c

+1 −1
Original line number Diff line number Diff line
@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, 
 * contended             :    (*,x,y) +--> (*,0,0) ---> (*,0,1) -'  :

 *   queue               :         ^--'                             :

 */

void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)

void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)

{

	struct mcs_spinlock *prev, *next, *node;

	u32 old, tail;

kernel/locking/qspinlock_paravirt.h

+2 −2
Original line number Diff line number Diff line
@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node) 
 * PV versions of the unlock fastpath and slowpath functions to be used

 * instead of queued_spin_unlock().

 */

__visible void

__visible __lockfunc void

__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)

{

	struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked) 
#include <asm/qspinlock_paravirt.h>



#ifndef __pv_queued_spin_unlock

__visible void __pv_queued_spin_unlock(struct qspinlock *lock)

__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)

{

	u8 locked;