Commit 105fc335 authored by Mark Rutland's avatar Mark Rutland Committed by Will Deacon
Browse files

arm64: entry: move el1 irq/nmi logic to C 



In preparation for reworking the EL1 irq/nmi entry code, move the
existing logic to C. We no longer need the asm_nmi_enter() and
asm_nmi_exit() wrappers, so these are removed. The new C functions are
marked noinstr, which prevents compiler instrumentation and runtime
probing.

In subsequent patches we'll want the new C helpers to be called in all
cases, so we don't bother wrapping the calls with ifdeferry. Even when
the new C functions are stubs the trivial calls are unlikely to have a
measurable impact on the IRQ or NMI paths anyway.

Prototypes are added to <asm/exception.h> as otherwise (in some
configurations) GCC will complain about the lack of a forward
declaration. We already do this for existing function, e.g.
enter_from_user_mode().

The new helpers are marked as noinstr (which prevents all
instrumentation, tracing, and kprobes). Otherwise, there should be no
functional change as a result of this patch.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201130115950.22492-7-mark.rutland@arm.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 3cb5ed4d

arch/arm64/include/asm/exception.h

+2 −0
Original line number Diff line number Diff line
@@ -31,6 +31,8 @@ static inline u32 disr_to_esr(u64 disr) 
	return esr;

}



asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs);

asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs);

asmlinkage void enter_from_user_mode(void);

void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);

void do_undefinstr(struct pt_regs *regs);

arch/arm64/kernel/entry-common.c

+16 −0
Original line number Diff line number Diff line
@@ -17,6 +17,22 @@ 
#include <asm/mmu.h>

#include <asm/sysreg.h>



asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)

{

	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))

		nmi_enter();



	trace_hardirqs_off();

}



asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)

{

	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))

		nmi_exit();

	else

		trace_hardirqs_on();

}



static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);

arch/arm64/kernel/entry.S

+4 −30
Original line number Diff line number Diff line
@@ -637,16 +637,8 @@ SYM_CODE_START_LOCAL_NOALIGN(el1_irq) 
	gic_prio_irq_setup pmr=x20, tmp=x1

	enable_da_f



#ifdef CONFIG_ARM64_PSEUDO_NMI

	test_irqs_unmasked	res=x0, pmr=x20

	cbz	x0, 1f

	bl	asm_nmi_enter

1:

#endif



#ifdef CONFIG_TRACE_IRQFLAGS

	bl	trace_hardirqs_off

#endif

	mov	x0, sp

	bl	enter_el1_irq_or_nmi



	irq_handler
@@ -665,26 +657,8 @@ alternative_else_nop_endif 
1:

#endif



#ifdef CONFIG_ARM64_PSEUDO_NMI

	/*

	 * When using IRQ priority masking, we can get spurious interrupts while

	 * PMR is set to GIC_PRIO_IRQOFF. An NMI might also have occurred in a

	 * section with interrupts disabled. Skip tracing in those cases.

	 */

	test_irqs_unmasked	res=x0, pmr=x20

	cbz	x0, 1f

	bl	asm_nmi_exit

1:

#endif



#ifdef CONFIG_TRACE_IRQFLAGS

#ifdef CONFIG_ARM64_PSEUDO_NMI

	test_irqs_unmasked	res=x0, pmr=x20

	cbnz	x0, 1f

#endif

	bl	trace_hardirqs_on

1:

#endif

	mov	x0, sp

	bl	exit_el1_irq_or_nmi



	kernel_exit 1

SYM_CODE_END(el1_irq)

arch/arm64/kernel/irq.c

+0 −15
Original line number Diff line number Diff line
@@ -67,18 +67,3 @@ void __init init_IRQ(void) 
		local_daif_restore(DAIF_PROCCTX_NOIRQ);

	}

}
 


/*

 * Stubs to make nmi_enter/exit() code callable from ASM

 */

asmlinkage void notrace asm_nmi_enter(void)

{

	nmi_enter();

}

NOKPROBE_SYMBOL(asm_nmi_enter);



asmlinkage void notrace asm_nmi_exit(void)

{

	nmi_exit();

}

NOKPROBE_SYMBOL(asm_nmi_exit);