Commit 5b90b966 authored by Christophe Leroy's avatar Christophe Leroy Committed by Michael Ellerman
Browse files

powerpc/syscall: Avoid storing 'current' in another pointer 



By saving the pointer pointing to thread_info.flags, gcc copies r2
in a non-volatile register.

We know 'current' doesn't change, so avoid that intermediaite pointer.

Reduces null_syscall benchmark by 2 cycles (322 => 320 cycles)

On PPC64, gcc seems to know that 'current' is not changing, and it keeps
it in a non volatile register to avoid multiple read of 'current' in paca.

Signed-off-by: default avatarChristophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/ad0363ff0ff8c125f40e1cdc589a85bbd7e31693.1612946484.git.christophe.leroy@csgroup.eu
parent d524dda7

arch/powerpc/kernel/interrupt.c

+9 −12
Original line number Diff line number Diff line
@@ -241,7 +241,6 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, 
					   struct pt_regs *regs,

					   long scv)

{

	unsigned long *ti_flagsp = &current_thread_info()->flags;

	unsigned long ti_flags;

	unsigned long ret = 0;

	bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv;
@@ -257,7 +256,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, 
	/* Check whether the syscall is issued inside a restartable sequence */

	rseq_syscall(regs);



	ti_flags = *ti_flagsp;

	ti_flags = current_thread_info()->flags;



	if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {

		if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
@@ -271,7 +270,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, 
			ret = _TIF_RESTOREALL;

		else

			regs->gpr[3] = r3;

		clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp);

		clear_bits(_TIF_PERSYSCALL_MASK, &current_thread_info()->flags);

	} else {

		regs->gpr[3] = r3;

	}
@@ -284,7 +283,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, 
	local_irq_disable();



again:

	ti_flags = READ_ONCE(*ti_flagsp);

	ti_flags = READ_ONCE(current_thread_info()->flags);

	while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {

		local_irq_enable();

		if (ti_flags & _TIF_NEED_RESCHED) {
@@ -300,7 +299,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, 
			do_notify_resume(regs, ti_flags);

		}

		local_irq_disable();

		ti_flags = READ_ONCE(*ti_flagsp);

		ti_flags = READ_ONCE(current_thread_info()->flags);

	}



	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
@@ -357,7 +356,6 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, 
#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */

notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)

{

	unsigned long *ti_flagsp = &current_thread_info()->flags;

	unsigned long ti_flags;

	unsigned long flags;

	unsigned long ret = 0;
@@ -380,7 +378,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned 
	local_irq_save(flags);



again:

	ti_flags = READ_ONCE(*ti_flagsp);

	ti_flags = READ_ONCE(current_thread_info()->flags);

	while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {

		local_irq_enable(); /* returning to user: may enable */

		if (ti_flags & _TIF_NEED_RESCHED) {
@@ -391,7 +389,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned 
			do_notify_resume(regs, ti_flags);

		}

		local_irq_disable();

		ti_flags = READ_ONCE(*ti_flagsp);

		ti_flags = READ_ONCE(current_thread_info()->flags);

	}



	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
@@ -443,7 +441,6 @@ void preempt_schedule_irq(void); 


notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr)

{

	unsigned long *ti_flagsp = &current_thread_info()->flags;

	unsigned long flags;

	unsigned long ret = 0;

#ifdef CONFIG_PPC64
@@ -466,8 +463,8 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign 
	amr = kuap_get_and_check_amr();

#endif



	if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) {

		clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp);

	if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {

		clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);

		ret = 1;

	}
@@ -479,7 +476,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign 
again:

		if (IS_ENABLED(CONFIG_PREEMPT)) {

			/* Return to preemptible kernel context */

			if (unlikely(*ti_flagsp & _TIF_NEED_RESCHED)) {

			if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) {

				if (preempt_count() == 0)

					preempt_schedule_irq();

			}