Commit 13799748 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/64: use interrupt restart table to speed up return from interrupt



Use the restart table facility to return from interrupt or system calls
without disabling MSR[EE] or MSR[RI].

Interrupt return asm is put into the low soft-masked region, to prevent
interrupts being processed here, although they are still taken as masked
interrupts which causes SRRs to be clobbered, and a pending soft-masked
interrupt to require replaying.

The return code uses restart table regions to redirct to a fixup handler
rather than continue with the exit, if such an interrupt happens. In
this case the interrupt return is redirected to a fixup handler which
reloads r1 for the interrupt stack and reloads registers and sets state
up to replay the soft-masked interrupt and try the exit again.

Some types of security exit fallback flushes and barriers are currently
unable to cope with reentrant interrupts, e.g., because they store some
state in the scratch SPR which would be clobbered even by masked
interrupts. For now the interrupts-enabled exits are disabled when these
flushes are used.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
[mpe: Guard unused exit_must_hard_disable() as reported by lkp]
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210617155116.2167984-13-npiggin@gmail.com
parent 9d1988ca
Loading
Loading
Loading
Loading
+5 −0
Original line number Diff line number Diff line
@@ -73,6 +73,11 @@ long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8,
notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv);
notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs);
notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs);
#ifdef CONFIG_PPC64
unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs);
unsigned long interrupt_exit_user_restart(struct pt_regs *regs);
unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs);
#endif

long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
		      u32 len_high, u32 len_low);
+11 −2
Original line number Diff line number Diff line
@@ -18,8 +18,17 @@
 * PACA flags in paca->irq_happened.
 *
 * This bits are set when interrupts occur while soft-disabled
 * and allow a proper replay. Additionally, PACA_IRQ_HARD_DIS
 * is set whenever we manually hard disable.
 * and allow a proper replay.
 *
 * The PACA_IRQ_HARD_DIS is set whenever we hard disable. It is almost
 * always in synch with the MSR[EE] state, except:
 * - A window in interrupt entry, where hardware disables MSR[EE] and that
 *   must be "reconciled" with the soft mask state.
 * - NMI interrupts that hit in awkward places, until they fix the state.
 * - When local irqs are being enabled and state is being fixed up.
 * - When returning from an interrupt there are some windows where this
 *   can become out of synch, but gets fixed before the RFI or before
 *   executing the next user instruction (see arch/powerpc/kernel/interrupt.c).
 */
#define PACA_IRQ_HARD_DIS	0x01
#define PACA_IRQ_DBELL		0x02
+2 −0
Original line number Diff line number Diff line
@@ -79,6 +79,8 @@ unsigned long search_kernel_restart_table(unsigned long addr);
#endif

#ifdef CONFIG_PPC_BOOK3S_64
DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);

static inline void srr_regs_clobbered(void)
{
	local_paca->srr_valid = 0;
+3 −0
Original line number Diff line number Diff line
@@ -165,6 +165,9 @@ struct paca_struct {
	u64 kstack;			/* Saved Kernel stack addr */
	u64 saved_r1;			/* r1 save for RTAS calls or PM or EE=0 */
	u64 saved_msr;			/* MSR saved here by enter_rtas */
#ifdef CONFIG_PPC64
	u64 exit_save_r1;		/* Syscall/interrupt R1 save */
#endif
#ifdef CONFIG_PPC_BOOK3E
	u16 trap_save;			/* Used when bad stack is encountered */
#endif
+1 −0
Original line number Diff line number Diff line
@@ -53,6 +53,7 @@ struct pt_regs
		struct {
#ifdef CONFIG_PPC64
			unsigned long ppr;
			unsigned long exit_result;
#endif
			union {
#ifdef CONFIG_PPC_KUAP
Loading