Commit 1df7d5e4 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/64s: introduce different functions to return from SRR vs HSRR interrupts 



This makes no real difference yet except that HSRR type interrupts will
use hrfid to return. This is important for the next patch.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210617155116.2167984-4-npiggin@gmail.com
parent bf9155f1

arch/powerpc/kernel/entry_64.S

+47 −18
Original line number Diff line number Diff line
@@ -635,51 +635,57 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 
	 * touched, no exit work created, then this can be used.

	 */

	.balign IFETCH_ALIGN_BYTES

	.globl fast_interrupt_return

fast_interrupt_return:

_ASM_NOKPROBE_SYMBOL(fast_interrupt_return)

	.globl fast_interrupt_return_srr

fast_interrupt_return_srr:

_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)

	kuap_check_amr r3, r4

	ld	r5,_MSR(r1)

	andi.	r0,r5,MSR_PR

#ifdef CONFIG_PPC_BOOK3S

	bne	.Lfast_user_interrupt_return_amr

	bne	.Lfast_user_interrupt_return_amr_srr

	kuap_kernel_restore r3, r4

	andi.	r0,r5,MSR_RI

	li	r3,0 /* 0 return value, no EMULATE_STACK_STORE */

	bne+	.Lfast_kernel_interrupt_return

	bne+	.Lfast_kernel_interrupt_return_srr

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	unrecoverable_exception

	b	. /* should not get here */

#else

	bne	.Lfast_user_interrupt_return

	b	.Lfast_kernel_interrupt_return

	bne	.Lfast_user_interrupt_return_srr

	b	.Lfast_kernel_interrupt_return_srr

#endif



.macro interrupt_return_macro srr

	.balign IFETCH_ALIGN_BYTES

	.globl interrupt_return

interrupt_return:

_ASM_NOKPROBE_SYMBOL(interrupt_return)

	.globl interrupt_return_\srr

interrupt_return_\srr\():

_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())

	ld	r4,_MSR(r1)

	andi.	r0,r4,MSR_PR

	beq	.Lkernel_interrupt_return

	beq	.Lkernel_interrupt_return_\srr

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	interrupt_exit_user_prepare

	cmpdi	r3,0

	bne-	.Lrestore_nvgprs

	bne-	.Lrestore_nvgprs_\srr



#ifdef CONFIG_PPC_BOOK3S

.Lfast_user_interrupt_return_amr:

.Lfast_user_interrupt_return_amr_\srr\():

	kuap_user_restore r3, r4

#endif

.Lfast_user_interrupt_return:

.Lfast_user_interrupt_return_\srr\():

	ld	r11,_NIP(r1)

	ld	r12,_MSR(r1)

BEGIN_FTR_SECTION

	ld	r10,_PPR(r1)

	mtspr	SPRN_PPR,r10

END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

	.ifc \srr,srr

	mtspr	SPRN_SRR0,r11

	mtspr	SPRN_SRR1,r12

	.else

	mtspr	SPRN_HSRR0,r11

	mtspr	SPRN_HSRR1,r12

	.endif



BEGIN_FTR_SECTION

	stdcx.	r0,0,r1		/* to clear the reservation */
@@ -706,24 +712,33 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 
	REST_GPR(6, r1)

	REST_GPR(0, r1)

	REST_GPR(1, r1)

	.ifc \srr,srr

	RFI_TO_USER

	.else

	HRFI_TO_USER

	.endif

	b	.	/* prevent speculative execution */



.Lrestore_nvgprs:

.Lrestore_nvgprs_\srr\():

	REST_NVGPRS(r1)

	b	.Lfast_user_interrupt_return

	b	.Lfast_user_interrupt_return_\srr



	.balign IFETCH_ALIGN_BYTES

.Lkernel_interrupt_return:

.Lkernel_interrupt_return_\srr\():

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	interrupt_exit_kernel_prepare



.Lfast_kernel_interrupt_return:

.Lfast_kernel_interrupt_return_\srr\():

	cmpdi	cr1,r3,0

	ld	r11,_NIP(r1)

	ld	r12,_MSR(r1)

	.ifc \srr,srr

	mtspr	SPRN_SRR0,r11

	mtspr	SPRN_SRR1,r12

	.else

	mtspr	SPRN_HSRR0,r11

	mtspr	SPRN_HSRR1,r12

	.endif



BEGIN_FTR_SECTION

	stdcx.	r0,0,r1		/* to clear the reservation */
@@ -757,7 +772,11 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 
	REST_GPR(6, r1)

	REST_GPR(0, r1)

	REST_GPR(1, r1)

	.ifc \srr,srr

	RFI_TO_KERNEL

	.else

	HRFI_TO_KERNEL

	.endif

	b	.	/* prevent speculative execution */



1:	/*
@@ -777,8 +796,18 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 
	std	r9,0(r1) /* perform store component of stdu */

	ld	r9,PACA_EXGEN+0(r13)



	.ifc \srr,srr

	RFI_TO_KERNEL

	.else

	HRFI_TO_KERNEL

	.endif

	b	.	/* prevent speculative execution */

.endm



interrupt_return_macro srr

#ifdef CONFIG_PPC_BOOK3S

interrupt_return_macro hsrr

#endif



#ifdef CONFIG_PPC_RTAS

/*

arch/powerpc/kernel/exceptions-64e.S

+4 −0
Original line number Diff line number Diff line
@@ -26,6 +26,10 @@ 
#include <asm/feature-fixups.h>

#include <asm/context_tracking.h>



/* 64e interrupt returns always use SRR registers */

#define fast_interrupt_return fast_interrupt_return_srr

#define interrupt_return interrupt_return_srr



/* XXX This will ultimately add space for a special exception save

 *     structure used to save things like SRR0/SRR1, SPRGs, MAS, etc...

 *     when taking special interrupts. For now we don't support that,

arch/powerpc/kernel/exceptions-64s.S

+40 −36
Original line number Diff line number Diff line
@@ -1149,7 +1149,7 @@ EXC_COMMON_BEGIN(machine_check_common) 
	mtmsrd 	r10,1

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	machine_check_exception

	b	interrupt_return

	b	interrupt_return_srr





#ifdef CONFIG_PPC_P7_NAP
@@ -1275,7 +1275,7 @@ BEGIN_MMU_FTR_SECTION 
MMU_FTR_SECTION_ELSE

	bl	do_page_fault

ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)

	b	interrupt_return

	b	interrupt_return_srr



1:	bl	do_break

	/*
@@ -1283,7 +1283,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 
	 * If so, we need to restore them with their updated values.

	 */

	REST_NVGPRS(r1)

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1323,7 +1323,7 @@ BEGIN_MMU_FTR_SECTION 
	bl	do_slb_fault

	cmpdi	r3,0

	bne-	1f

	b	fast_interrupt_return

	b	fast_interrupt_return_srr

1:	/* Error case */

MMU_FTR_SECTION_ELSE

	/* Radix case, access is outside page table range */
@@ -1332,7 +1332,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 
	std	r3,RESULT(r1)

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	do_bad_slb_fault

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1368,7 +1368,7 @@ BEGIN_MMU_FTR_SECTION 
MMU_FTR_SECTION_ELSE

	bl	do_page_fault

ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1403,7 +1403,7 @@ BEGIN_MMU_FTR_SECTION 
	bl	do_slb_fault

	cmpdi	r3,0

	bne-	1f

	b	fast_interrupt_return

	b	fast_interrupt_return_srr

1:	/* Error case */

MMU_FTR_SECTION_ELSE

	/* Radix case, access is outside page table range */
@@ -1412,7 +1412,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 
	std	r3,RESULT(r1)

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	do_bad_slb_fault

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1456,7 +1456,11 @@ EXC_COMMON_BEGIN(hardware_interrupt_common) 
	GEN_COMMON hardware_interrupt

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	do_IRQ

	b	interrupt_return

	BEGIN_FTR_SECTION

	b	interrupt_return_hsrr

	FTR_SECTION_ELSE

	b	interrupt_return_srr

	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)





/**
@@ -1483,7 +1487,7 @@ EXC_COMMON_BEGIN(alignment_common) 
	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	alignment_exception

	REST_NVGPRS(r1) /* instruction emulation may change GPRs */

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1590,7 +1594,7 @@ EXC_COMMON_BEGIN(program_check_common) 
	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	program_check_exception

	REST_NVGPRS(r1) /* instruction emulation may change GPRs */

	b	interrupt_return

	b	interrupt_return_srr





/*
@@ -1633,12 +1637,12 @@ BEGIN_FTR_SECTION 
END_FTR_SECTION_IFSET(CPU_FTR_TM)

#endif

	bl	load_up_fpu

	b	fast_interrupt_return

	b	fast_interrupt_return_srr

#ifdef CONFIG_PPC_TRANSACTIONAL_MEM

2:	/* User process was in a transaction */

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	fp_unavailable_tm

	b	interrupt_return

	b	interrupt_return_srr

#endif
@@ -1677,7 +1681,7 @@ EXC_COMMON_BEGIN(decrementer_common) 
	GEN_COMMON decrementer

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	timer_interrupt

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1761,7 +1765,7 @@ EXC_COMMON_BEGIN(doorbell_super_common) 
#else

	bl	unknown_async_exception

#endif

	b	interrupt_return

	b	interrupt_return_srr





EXC_REAL_NONE(0xb00, 0x100)
@@ -1925,7 +1929,7 @@ EXC_COMMON_BEGIN(single_step_common) 
	GEN_COMMON single_step

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	single_step_exception

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -1963,7 +1967,7 @@ BEGIN_MMU_FTR_SECTION 
MMU_FTR_SECTION_ELSE

	bl      unknown_exception

ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)

	b       interrupt_return

	b       interrupt_return_hsrr





/**
@@ -1988,7 +1992,7 @@ EXC_COMMON_BEGIN(h_instr_storage_common) 
	GEN_COMMON h_instr_storage

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	unknown_exception

	b	interrupt_return

	b	interrupt_return_hsrr





/**
@@ -2012,7 +2016,7 @@ EXC_COMMON_BEGIN(emulation_assist_common) 
	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	emulation_assist_interrupt

	REST_NVGPRS(r1) /* instruction emulation may change GPRs */

	b	interrupt_return

	b	interrupt_return_hsrr





/**
@@ -2089,7 +2093,7 @@ EXC_COMMON_BEGIN(hmi_exception_common) 
	GEN_COMMON hmi_exception

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	handle_hmi_exception

	b	interrupt_return

	b	interrupt_return_hsrr





/**
@@ -2119,7 +2123,7 @@ EXC_COMMON_BEGIN(h_doorbell_common) 
#else

	bl	unknown_async_exception

#endif

	b	interrupt_return

	b	interrupt_return_hsrr





/**
@@ -2145,7 +2149,7 @@ EXC_COMMON_BEGIN(h_virt_irq_common) 
	GEN_COMMON h_virt_irq

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	do_IRQ

	b	interrupt_return

	b	interrupt_return_hsrr





EXC_REAL_NONE(0xec0, 0x20)
@@ -2188,7 +2192,7 @@ EXC_COMMON_BEGIN(performance_monitor_common) 
	GEN_COMMON performance_monitor

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	performance_monitor_exception

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -2225,19 +2229,19 @@ BEGIN_FTR_SECTION 
  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)

#endif

	bl	load_up_altivec

	b	fast_interrupt_return

	b	fast_interrupt_return_srr

#ifdef CONFIG_PPC_TRANSACTIONAL_MEM

2:	/* User process was in a transaction */

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	altivec_unavailable_tm

	b	interrupt_return

	b	interrupt_return_srr

#endif

1:

END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)

#endif

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	altivec_unavailable_exception

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -2278,14 +2282,14 @@ BEGIN_FTR_SECTION 
2:	/* User process was in a transaction */

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	vsx_unavailable_tm

	b	interrupt_return

	b	interrupt_return_srr

#endif

1:

END_FTR_SECTION_IFSET(CPU_FTR_VSX)

#endif

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	vsx_unavailable_exception

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -2313,7 +2317,7 @@ EXC_COMMON_BEGIN(facility_unavailable_common) 
	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	facility_unavailable_exception

	REST_NVGPRS(r1) /* instruction emulation may change GPRs */

	b	interrupt_return

	b	interrupt_return_srr





/**
@@ -2341,7 +2345,7 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common) 
	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	facility_unavailable_exception

	REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */

	b	interrupt_return

	b	interrupt_return_hsrr





EXC_REAL_NONE(0xfa0, 0x20)
@@ -2370,7 +2374,7 @@ EXC_COMMON_BEGIN(cbe_system_error_common) 
	GEN_COMMON cbe_system_error

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	cbe_system_error_exception

	b	interrupt_return

	b	interrupt_return_hsrr



#else /* CONFIG_CBE_RAS */

EXC_REAL_NONE(0x1200, 0x100)
@@ -2401,7 +2405,7 @@ EXC_COMMON_BEGIN(instruction_breakpoint_common) 
	GEN_COMMON instruction_breakpoint

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	instruction_breakpoint_exception

	b	interrupt_return

	b	interrupt_return_srr





EXC_REAL_NONE(0x1400, 0x100)
@@ -2521,7 +2525,7 @@ EXC_COMMON_BEGIN(denorm_exception_common) 
	GEN_COMMON denorm_exception

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	unknown_exception

	b	interrupt_return

	b	interrupt_return_hsrr





#ifdef CONFIG_CBE_RAS
@@ -2538,7 +2542,7 @@ EXC_COMMON_BEGIN(cbe_maintenance_common) 
	GEN_COMMON cbe_maintenance

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	cbe_maintenance_exception

	b	interrupt_return

	b	interrupt_return_hsrr



#else /* CONFIG_CBE_RAS */

EXC_REAL_NONE(0x1600, 0x100)
@@ -2568,7 +2572,7 @@ EXC_COMMON_BEGIN(altivec_assist_common) 
#else

	bl	unknown_exception

#endif

	b	interrupt_return

	b	interrupt_return_srr





#ifdef CONFIG_CBE_RAS
@@ -2585,7 +2589,7 @@ EXC_COMMON_BEGIN(cbe_thermal_common) 
	GEN_COMMON cbe_thermal

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	cbe_thermal_exception

	b	interrupt_return

	b	interrupt_return_hsrr



#else /* CONFIG_CBE_RAS */

EXC_REAL_NONE(0x1800, 0x100)

arch/powerpc/kernel/vector.S

+1 −1
Original line number Diff line number Diff line
@@ -131,7 +131,7 @@ _GLOBAL(load_up_vsx) 
	/* enable use of VSX after return */

	oris	r12,r12,MSR_VSX@h

	std	r12,_MSR(r1)

	b	fast_interrupt_return

	b	fast_interrupt_return_srr



#endif /* CONFIG_VSX */