Commit 167fd210 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

x86/entry: Use generic syscall exit functionality 



Replace the x86 variant with the generic version. Provide the relevant
architecture specific helper functions and defines.

Use a temporary define for idtentry_exit_user which will be cleaned up
seperately.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarKees Cook <keescook@chromium.org>
Link: https://lkml.kernel.org/r/20200722220520.494648601@linutronix.de
parent 27d6b4d1

arch/x86/entry/common.c

+5 −216
Original line number Diff line number Diff line
@@ -15,15 +15,8 @@ 
#include <linux/smp.h>

#include <linux/errno.h>

#include <linux/ptrace.h>

#include <linux/tracehook.h>

#include <linux/audit.h>

#include <linux/signal.h>

#include <linux/export.h>

#include <linux/context_tracking.h>

#include <linux/user-return-notifier.h>

#include <linux/nospec.h>

#include <linux/uprobes.h>

#include <linux/livepatch.h>

#include <linux/syscalls.h>

#include <linux/uaccess.h>
@@ -42,191 +35,6 @@ 
#include <asm/syscall.h>

#include <asm/irq_stack.h>



#include <trace/events/syscalls.h>



/**

 * exit_to_user_mode - Fixup state when exiting to user mode

 *

 * Syscall exit enables interrupts, but the kernel state is interrupts

 * disabled when this is invoked. Also tell RCU about it.

 *

 * 1) Trace interrupts on state

 * 2) Invoke context tracking if enabled to adjust RCU state

 * 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on.

 * 4) Tell lockdep that interrupts are enabled

 */

static __always_inline void exit_to_user_mode(void)

{

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	instrumentation_end();



	user_enter_irqoff();

	mds_user_clear_cpu_buffers();

	lockdep_hardirqs_on(CALLER_ADDR0);

}



#define EXIT_TO_USERMODE_LOOP_FLAGS				\

	(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |	\

	 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING)



static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)

{

	/*

	 * In order to return to user mode, we need to have IRQs off with

	 * none of EXIT_TO_USERMODE_LOOP_FLAGS set.  Several of these flags

	 * can be set at any time on preemptible kernels if we have IRQs on,

	 * so we need to loop.  Disabling preemption wouldn't help: doing the

	 * work to clear some of the flags can sleep.

	 */

	while (true) {

		/* We have work to do. */

		local_irq_enable();



		if (cached_flags & _TIF_NEED_RESCHED)

			schedule();



		if (cached_flags & _TIF_UPROBE)

			uprobe_notify_resume(regs);



		if (cached_flags & _TIF_PATCH_PENDING)

			klp_update_patch_state(current);



		/* deal with pending signal delivery */

		if (cached_flags & _TIF_SIGPENDING)

			do_signal(regs);



		if (cached_flags & _TIF_NOTIFY_RESUME) {

			clear_thread_flag(TIF_NOTIFY_RESUME);

			tracehook_notify_resume(regs);

			rseq_handle_notify_resume(NULL, regs);

		}



		/* Disable IRQs and retry */

		local_irq_disable();



		cached_flags = READ_ONCE(current_thread_info()->flags);



		if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))

			break;

	}

}



static void __prepare_exit_to_usermode(struct pt_regs *regs)

{

	struct thread_info *ti = current_thread_info();

	u32 cached_flags;



	addr_limit_user_check();



	lockdep_assert_irqs_disabled();

	lockdep_sys_exit();



	cached_flags = READ_ONCE(ti->flags);



	if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))

		exit_to_usermode_loop(regs, cached_flags);



	/* Reload ti->flags; we may have rescheduled above. */

	cached_flags = READ_ONCE(ti->flags);



	if (cached_flags & _TIF_USER_RETURN_NOTIFY)

		fire_user_return_notifiers();



	if (unlikely(cached_flags & _TIF_IO_BITMAP))

		tss_update_io_bitmap();



	fpregs_assert_state_consistent();

	if (unlikely(cached_flags & _TIF_NEED_FPU_LOAD))

		switch_fpu_return();



#ifdef CONFIG_COMPAT

	/*

	 * Compat syscalls set TS_COMPAT.  Make sure we clear it before

	 * returning to user mode.  We need to clear it *after* signal

	 * handling, because syscall restart has a fixup for compat

	 * syscalls.  The fixup is exercised by the ptrace_syscall_32

	 * selftest.

	 *

	 * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer

	 * special case only applies after poking regs and before the

	 * very next return to user mode.

	 */

	ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

#endif

}



static noinstr void prepare_exit_to_usermode(struct pt_regs *regs)

{

	instrumentation_begin();

	__prepare_exit_to_usermode(regs);

	instrumentation_end();

	exit_to_user_mode();

}



#define SYSCALL_EXIT_WORK_FLAGS				\

	(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |	\

	 _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)



static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)

{

	bool step;



	audit_syscall_exit(regs);



	if (cached_flags & _TIF_SYSCALL_TRACEPOINT)

		trace_sys_exit(regs, regs->ax);



	/*

	 * If TIF_SYSCALL_EMU is set, we only get here because of

	 * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).

	 * We already reported this syscall instruction in

	 * syscall_trace_enter().

	 */

	step = unlikely(

		(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))

		== _TIF_SINGLESTEP);

	if (step || cached_flags & _TIF_SYSCALL_TRACE)

		tracehook_report_syscall_exit(regs, step);

}



static void __syscall_return_slowpath(struct pt_regs *regs)

{

	struct thread_info *ti = current_thread_info();

	u32 cached_flags = READ_ONCE(ti->flags);



	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);



	if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&

	    WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))

		local_irq_enable();



	rseq_syscall(regs);



	/*

	 * First do one-time work.  If these work items are enabled, we

	 * want to run them exactly once per syscall exit with IRQs on.

	 */

	if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))

		syscall_slow_exit_work(regs, cached_flags);



	local_irq_disable();

	__prepare_exit_to_usermode(regs);

}



/*

 * Called with IRQs on and fully valid regs.  Returns with IRQs off in a

 * state such that we can immediately switch to user mode.

 */

__visible noinstr void syscall_return_slowpath(struct pt_regs *regs)

{

	instrumentation_begin();

	__syscall_return_slowpath(regs);

	instrumentation_end();

	exit_to_user_mode();

}



#ifdef CONFIG_X86_64

__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)

{
@@ -245,7 +53,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) 
#endif

	}

	instrumentation_end();

	syscall_return_slowpath(regs);

	syscall_exit_to_user_mode(regs);

}

#endif
@@ -284,7 +92,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) 
	unsigned int nr = syscall_32_enter(regs);



	do_syscall_32_irqs_on(regs, nr);

	syscall_return_slowpath(regs);

	syscall_exit_to_user_mode(regs);

}



static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
@@ -310,13 +118,13 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) 
	if (res) {

		/* User code screwed up. */

		regs->ax = -EFAULT;

		syscall_return_slowpath(regs);

		syscall_exit_to_user_mode(regs);

		return false;

	}



	/* Now this is just like a normal syscall. */

	do_syscall_32_irqs_on(regs, nr);

	syscall_return_slowpath(regs);

	syscall_exit_to_user_mode(regs);

	return true;

}
@@ -524,7 +332,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state) 


	/* Check whether this returns to user mode */

	if (user_mode(regs)) {

		prepare_exit_to_usermode(regs);

		irqentry_exit_to_user_mode(regs);

	} else if (regs->flags & X86_EFLAGS_IF) {

		/*

		 * If RCU was not watching on entry this needs to be done
@@ -555,25 +363,6 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state) 
	}

}



/**

 * idtentry_exit_user - Handle return from exception to user mode

 * @regs:	Pointer to pt_regs (exception entry regs)

 *

 * Runs the necessary preemption and work checks and returns to the caller

 * with interrupts disabled and no further work pending.

 *

 * This is the last action before returning to the low level ASM code which

 * just needs to return to the appropriate context.

 *

 * Counterpart to idtentry_enter_user().

 */

void noinstr idtentry_exit_user(struct pt_regs *regs)

{

	lockdep_assert_irqs_disabled();



	prepare_exit_to_usermode(regs);

}



#ifdef CONFIG_XEN_PV

#ifndef CONFIG_PREEMPTION

/*

arch/x86/entry/entry_32.S

+1 −1
Original line number Diff line number Diff line
@@ -846,7 +846,7 @@ SYM_CODE_START(ret_from_fork) 
2:

	/* When we fork, we trace the syscall return in the child, too. */

	movl    %esp, %eax

	call    syscall_return_slowpath

	call    syscall_exit_to_user_mode

	jmp     .Lsyscall_32_done



	/* kernel thread */

arch/x86/entry/entry_64.S

+1 −1
Original line number Diff line number Diff line
@@ -283,7 +283,7 @@ SYM_CODE_START(ret_from_fork) 
2:

	UNWIND_HINT_REGS

	movq	%rsp, %rdi

	call	syscall_return_slowpath	/* returns with IRQs disabled */

	call	syscall_exit_to_user_mode	/* returns with IRQs disabled */

	jmp	swapgs_restore_regs_and_return_to_usermode



1:

arch/x86/include/asm/entry-common.h

+44 −0
Original line number Diff line number Diff line
@@ -2,6 +2,12 @@ 
#ifndef _ASM_X86_ENTRY_COMMON_H

#define _ASM_X86_ENTRY_COMMON_H



#include <linux/user-return-notifier.h>



#include <asm/nospec-branch.h>

#include <asm/io_bitmap.h>

#include <asm/fpu/api.h>



/* Check that the stack and regs on entry from user mode are sane. */

static __always_inline void arch_check_user_regs(struct pt_regs *regs)

{
@@ -29,4 +35,42 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs) 
}

#define arch_check_user_regs arch_check_user_regs



#define ARCH_SYSCALL_EXIT_WORK		(_TIF_SINGLESTEP)



static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,

						  unsigned long ti_work)

{

	if (ti_work & _TIF_USER_RETURN_NOTIFY)

		fire_user_return_notifiers();



	if (unlikely(ti_work & _TIF_IO_BITMAP))

		tss_update_io_bitmap();



	fpregs_assert_state_consistent();

	if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))

		switch_fpu_return();



#ifdef CONFIG_COMPAT

	/*

	 * Compat syscalls set TS_COMPAT.  Make sure we clear it before

	 * returning to user mode.  We need to clear it *after* signal

	 * handling, because syscall restart has a fixup for compat

	 * syscalls.  The fixup is exercised by the ptrace_syscall_32

	 * selftest.

	 *

	 * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer

	 * special case only applies after poking regs and before the

	 * very next return to user mode.

	 */

	current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);

#endif

}

#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare



static __always_inline void arch_exit_to_user_mode(void)

{

	mds_user_clear_cpu_buffers();

}

#define arch_exit_to_user_mode arch_exit_to_user_mode



#endif

arch/x86/include/asm/idtentry.h

+1 −2
Original line number Diff line number Diff line
@@ -13,8 +13,7 @@ 


/* Temporary define */

#define idtentry_enter_user	irqentry_enter_from_user_mode



void idtentry_exit_user(struct pt_regs *regs);

#define idtentry_exit_user	irqentry_exit_to_user_mode



typedef struct idtentry_state {

	bool exit_rcu;