Commit d9ed39df authored by Sven Schnelle's avatar Sven Schnelle Committed by liwei
Browse files

entry: Move exit to usermode functions to header file 

mainline inclusion
from mainline-v6.8-rc1
commit d68019471995ba47e56a9da355df13a1cdb5bf7e
category: performance
bugzilla: https://gitee.com/openeuler/kernel/issues/I9OJ0P
CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d68019471995ba47e56a9da355df13a1cdb5bf7e



------------------------------------------------------

To allow inlining, move exit_to_user_mode() to
entry-common.h.

Signed-off-by: default avatarSven Schnelle <svens@linux.ibm.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20231218074520.1998026-2-svens@linux.ibm.com


Signed-off-by: default avatarliwei <liwei728@huawei.com>
parent a00ba8b5

include/linux/entry-common.h

+52 −1
Original line number Diff line number Diff line
@@ -7,6 +7,10 @@ 
#include <linux/syscalls.h>

#include <linux/seccomp.h>

#include <linux/sched.h>

#include <linux/context_tracking.h>

#include <linux/livepatch.h>

#include <linux/resume_user_mode.h>

#include <linux/tick.h>



#include <asm/entry-common.h>
@@ -258,6 +262,43 @@ static __always_inline void arch_exit_to_user_mode(void) { } 
 */

void arch_do_signal_or_restart(struct pt_regs *regs);



/**

 * exit_to_user_mode_loop - do any pending work before leaving to user space

 */

unsigned long exit_to_user_mode_loop(struct pt_regs *regs,

				     unsigned long ti_work);



/**

 * exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required

 * @regs:	Pointer to pt_regs on entry stack

 *

 * 1) check that interrupts are disabled

 * 2) call tick_nohz_user_enter_prepare()

 * 3) call exit_to_user_mode_loop() if any flags from

 *    EXIT_TO_USER_MODE_WORK are set

 * 4) check that interrupts are still disabled

 */

static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)

{

	unsigned long ti_work;



	lockdep_assert_irqs_disabled();



	/* Flush pending rcuog wakeup before the last need_resched() check */

	tick_nohz_user_enter_prepare();



	ti_work = read_thread_flags();

	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))

		ti_work = exit_to_user_mode_loop(regs, ti_work);



	arch_exit_to_user_mode_prepare(regs, ti_work);



	/* Ensure that kernel state is sane for a return to userspace */

	kmap_assert_nomap();

	lockdep_assert_irqs_disabled();

	lockdep_sys_exit();

}



/**

 * exit_to_user_mode - Fixup state when exiting to user mode

 *
@@ -276,7 +317,17 @@ void arch_do_signal_or_restart(struct pt_regs *regs); 
 * non-instrumentable.

 * The caller has to invoke syscall_exit_to_user_mode_work() before this.

 */

void exit_to_user_mode(void);

static __always_inline void exit_to_user_mode(void)

{

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare();

	instrumentation_end();



	user_enter_irqoff();

	arch_exit_to_user_mode();

	lockdep_hardirqs_on(CALLER_ADDR0);

}



/**

 * syscall_exit_to_user_mode_work - Handle work before returning to user mode

kernel/entry/common.c

+9 −43
Original line number Diff line number Diff line
@@ -129,28 +129,15 @@ noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs) 
	instrumentation_end();

}



/* See comment for exit_to_user_mode() in entry-common.h */

static __always_inline void __exit_to_user_mode(void)

{

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare();

	instrumentation_end();



	user_enter_irqoff();

	arch_exit_to_user_mode();

	lockdep_hardirqs_on(CALLER_ADDR0);

}



void noinstr exit_to_user_mode(void)

{

	__exit_to_user_mode();

}



/* Workaround to allow gradual conversion of architecture code */

void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }



static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,

/**

 * exit_to_user_mode_loop - do any pending work before leaving to user space

 * @regs:	Pointer to pt_regs on entry stack

 * @ti_work:	TIF work flags as read by the caller

 */

__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,

						     unsigned long ti_work)

{

	/*
@@ -196,27 +183,6 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 
	return ti_work;

}



static void exit_to_user_mode_prepare(struct pt_regs *regs)

{

	unsigned long ti_work;



	lockdep_assert_irqs_disabled();



	/* Flush pending rcuog wakeup before the last need_resched() check */

	tick_nohz_user_enter_prepare();



	ti_work = read_thread_flags();

	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))

		ti_work = exit_to_user_mode_loop(regs, ti_work);



	arch_exit_to_user_mode_prepare(regs, ti_work);



	/* Ensure that kernel state is sane for a return to userspace */

	kmap_assert_nomap();

	lockdep_assert_irqs_disabled();

	lockdep_sys_exit();

}



/*

 * If SYSCALL_EMU is set, then the only reason to report is when

 * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP).  This syscall
@@ -301,7 +267,7 @@ __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs) 
	instrumentation_begin();

	__syscall_exit_to_user_mode_work(regs);

	instrumentation_end();

	__exit_to_user_mode();

	exit_to_user_mode();

}



noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
@@ -314,7 +280,7 @@ noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) 
	instrumentation_begin();

	exit_to_user_mode_prepare(regs);

	instrumentation_end();

	__exit_to_user_mode();

	exit_to_user_mode();

}



noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)