Commit 9b69d48c authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/64e: remove implicit soft-masking and interrupt exit restart logic 



The implicit soft-masking to speed up interrupt return was going to be
used by 64e as well, but it has not been extensively tested on that
platform and is not considered ready. It was intended to be disabled
before merge. Disable it for now.

Most of the restart code is common with 64s, so with more correctness
and performance testing this could be re-enabled again by adding the
extra soft-mask checks to interrupt handlers and flipping
exit_must_hard_disable().

Fixes: 9d1988ca ("powerpc/64: treat low kernel text as irqs soft-masked")
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210630074621.2109197-4-npiggin@gmail.com
parent fce01acf

arch/powerpc/include/asm/interrupt.h

+24 −9
Original line number Diff line number Diff line
@@ -73,20 +73,34 @@ 
#include <asm/kprobes.h>

#include <asm/runlatch.h>



#ifdef CONFIG_PPC64

#ifdef CONFIG_PPC_BOOK3S_64

extern char __end_soft_masked[];

unsigned long search_kernel_restart_table(unsigned long addr);

#endif



#ifdef CONFIG_PPC_BOOK3S_64

DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);



static inline bool is_implicit_soft_masked(struct pt_regs *regs)

{

	if (regs->msr & MSR_PR)

		return false;



	if (regs->nip >= (unsigned long)__end_soft_masked)

		return false;



	return true;

}



static inline void srr_regs_clobbered(void)

{

	local_paca->srr_valid = 0;

	local_paca->hsrr_valid = 0;

}

#else

static inline bool is_implicit_soft_masked(struct pt_regs *regs)

{

	return false;

}



static inline void srr_regs_clobbered(void)

{

}
@@ -150,11 +164,13 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup 
		 */

		if (TRAP(regs) != INTERRUPT_PROGRAM) {

			CT_WARN_ON(ct_state() != CONTEXT_KERNEL);

			BUG_ON(regs->nip < (unsigned long)__end_soft_masked);

			BUG_ON(is_implicit_soft_masked(regs));

		}

#ifdef CONFIG_PPC_BOOK3S

		/* Move this under a debugging check */

		if (arch_irq_disabled_regs(regs))

			BUG_ON(search_kernel_restart_table(regs->nip));

#endif

	}

#endif
@@ -244,10 +260,9 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte 
	local_paca->irq_soft_mask = IRQS_ALL_DISABLED;

	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;



	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !(regs->msr & MSR_PR) &&

				regs->nip < (unsigned long)__end_soft_masked) {

		// Kernel code running below __end_soft_masked is

		// implicitly soft-masked.

	if (is_implicit_soft_masked(regs)) {

		// Adjust regs->softe soft implicit soft-mask, so

		// arch_irq_disabled_regs(regs) behaves as expected.

		regs->softe = IRQS_ALL_DISABLED;

	}
@@ -282,6 +297,7 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter 
	 */



#ifdef CONFIG_PPC64

#ifdef CONFIG_PPC_BOOK3S

	if (arch_irq_disabled_regs(regs)) {

		unsigned long rst = search_kernel_restart_table(regs->nip);

		if (rst)
@@ -289,7 +305,6 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter 
	}

#endif



#ifdef CONFIG_PPC64

	if (nmi_disables_ftrace(regs))

		this_cpu_set_ftrace_enabled(state->ftrace_enabled);

arch/powerpc/kernel/exceptions-64e.S

+1 −11
Original line number Diff line number Diff line
@@ -342,17 +342,7 @@ ret_from_mc_except: 
#define PROLOG_ADDITION_MASKABLE_GEN(n)					    \

	lbz	r10,PACAIRQSOFTMASK(r13);	/* are irqs soft-masked? */ \

	andi.	r10,r10,IRQS_DISABLED;	/* yes -> go out of line */ \

	bne	masked_interrupt_book3e_##n;				    \

	/* Kernel code below __end_soft_masked is implicitly masked */	    \

	andi.	r10,r11,MSR_PR;						    \

	bne	1f;			/* user -> not masked */	    \

	std	r14,PACA_EXGEN+EX_R14(r13);				    \

	LOAD_REG_IMMEDIATE_SYM(r14, r10, __end_soft_masked);		    \

	mfspr	r10,SPRN_SRR0;						    \

	cmpld	r10,r14;						    \

	ld	r14,PACA_EXGEN+EX_R14(r13);				    \

	blt	masked_interrupt_book3e_##n;				    \

1:

	bne	masked_interrupt_book3e_##n



/*

 * Additional regs must be re-loaded from paca before EXCEPTION_COMMON* is

arch/powerpc/kernel/interrupt.c

+1 −1
Original line number Diff line number Diff line
@@ -36,7 +36,7 @@ static inline bool exit_must_hard_disable(void) 
#else

static inline bool exit_must_hard_disable(void)

{

	return IS_ENABLED(CONFIG_PPC32);

	return true;

}

#endif

arch/powerpc/kernel/interrupt_64.S

+14 −2
Original line number Diff line number Diff line
@@ -231,7 +231,7 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate) 
	li	r10,IRQS_ALL_DISABLED

	stb	r10,PACAIRQSOFTMASK(r13)

	b	system_call_vectored_common

#endif

#endif /* CONFIG_PPC_BOOK3S */



	.balign IFETCH_ALIGN_BYTES

	.globl system_call_common_real
@@ -320,10 +320,12 @@ END_BTB_FLUSH_SECTION 
	li	r5,0 /* !scv */

	bl	syscall_exit_prepare

	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */

#ifdef CONFIG_PPC_BOOK3S

.Lsyscall_rst_start:

	lbz	r11,PACAIRQHAPPENED(r13)

	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l

	bne-	syscall_restart

#endif

	li	r11,IRQS_ENABLED

	stb	r11,PACAIRQSOFTMASK(r13)

	li	r11,0
@@ -396,6 +398,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 
	b	.Lsyscall_restore_regs_cont

.Lsyscall_rst_end:



#ifdef CONFIG_PPC_BOOK3S

syscall_restart:

	GET_PACA(r13)

	ld	r1,PACA_EXIT_SAVE_R1(r13)
@@ -409,6 +412,7 @@ syscall_restart: 
	b	.Lsyscall_rst_start



RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)

#endif



#ifdef CONFIG_PPC_TRANSACTIONAL_MEM

tabort_syscall:
@@ -504,10 +508,12 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()) 
	bne-	.Lrestore_nvgprs_\srr

.Lrestore_nvgprs_\srr\()_cont:

	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */

#ifdef CONFIG_PPC_BOOK3S

.Linterrupt_return_\srr\()_user_rst_start:

	lbz	r11,PACAIRQHAPPENED(r13)

	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l

	bne-	interrupt_return_\srr\()_user_restart

#endif

	li	r11,IRQS_ENABLED

	stb	r11,PACAIRQSOFTMASK(r13)

	li	r11,0
@@ -590,6 +596,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 
	REST_NVGPRS(r1)

	b	.Lrestore_nvgprs_\srr\()_cont



#ifdef CONFIG_PPC_BOOK3S

interrupt_return_\srr\()_user_restart:

	GET_PACA(r13)

	ld	r1,PACA_EXIT_SAVE_R1(r13)
@@ -602,6 +609,7 @@ interrupt_return_\srr\()_user_restart: 
	b	.Linterrupt_return_\srr\()_user_rst_start



RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)

#endif



	.balign IFETCH_ALIGN_BYTES

.Lkernel_interrupt_return_\srr\():
@@ -615,9 +623,11 @@ RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr 
	cmpwi	r11,IRQS_ENABLED

	stb	r11,PACAIRQSOFTMASK(r13)

	bne	1f

#ifdef CONFIG_PPC_BOOK3S

	lbz	r11,PACAIRQHAPPENED(r13)

	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l

	bne-	interrupt_return_\srr\()_kernel_restart

#endif

	li	r11,0

	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS

1:
@@ -717,6 +727,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 
	b	.	/* prevent speculative execution */

.Linterrupt_return_\srr\()_kernel_rst_end:



#ifdef CONFIG_PPC_BOOK3S

interrupt_return_\srr\()_kernel_restart:

	GET_PACA(r13)

	ld	r1,PACA_EXIT_SAVE_R1(r13)
@@ -729,14 +740,15 @@ interrupt_return_\srr\()_kernel_restart: 
	b	.Linterrupt_return_\srr\()_kernel_rst_start



RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)

#endif



.endm



interrupt_return_macro srr

#ifdef CONFIG_PPC_BOOK3S

interrupt_return_macro hsrr

#endif /* CONFIG_PPC_BOOK3S */



	.globl __end_soft_masked

__end_soft_masked:

DEFINE_FIXED_SYMBOL(__end_soft_masked)

#endif /* CONFIG_PPC_BOOK3S */