Commit 133d0518 authored by Julien Thierry's avatar Julien Thierry Committed by Catalin Marinas
Browse files

arm64: Make PMR part of task context 



In order to replace PSR.I interrupt disabling/enabling with ICC_PMR_EL1
interrupt masking, ICC_PMR_EL1 needs to be saved/restored when
taking/returning from an exception. This mimics the way hardware saves
and restores PSR.I bit in spsr_el1 for exceptions and ERET.

Add PMR to the registers to save in the pt_regs struct upon kernel entry,
and restore it before ERET. Also, initialize it to a sane value when
creating new tasks.

Signed-off-by: default avatarJulien Thierry <julien.thierry@arm.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Reviewed-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent cdbc81dd

arch/arm64/include/asm/processor.h

+3 −0
Original line number Original line Diff line number Diff line
@@ -191,6 +191,9 @@ static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) 
	memset(regs, 0, sizeof(*regs));

	memset(regs, 0, sizeof(*regs));

	forget_syscall(regs);

	forget_syscall(regs);

	regs->pc = pc;

	regs->pc = pc;



	if (system_uses_irq_prio_masking())

		regs->pmr_save = GIC_PRIO_IRQON;

}

}





static inline void start_thread(struct pt_regs *regs, unsigned long pc,

static inline void start_thread(struct pt_regs *regs, unsigned long pc,

arch/arm64/include/asm/ptrace.h

+11 −3
Original line number Original line Diff line number Diff line
@@ -19,6 +19,8 @@ 
#ifndef __ASM_PTRACE_H

#ifndef __ASM_PTRACE_H

#define __ASM_PTRACE_H

#define __ASM_PTRACE_H





#include <asm/cpufeature.h>



#include <uapi/asm/ptrace.h>

#include <uapi/asm/ptrace.h>





/* Current Exception Level values, as contained in CurrentEL */

/* Current Exception Level values, as contained in CurrentEL */
@@ -179,7 +181,8 @@ struct pt_regs { 
#endif

#endif





	u64 orig_addr_limit;

	u64 orig_addr_limit;

	u64 unused;	// maintain 16 byte alignment

	/* Only valid when ARM64_HAS_IRQ_PRIO_MASKING is enabled. */

	u64 pmr_save;

	u64 stackframe[2];

	u64 stackframe[2];

};

};
@@ -214,8 +217,13 @@ static inline void forget_syscall(struct pt_regs *regs) 
#define processor_mode(regs) \

#define processor_mode(regs) \

	((regs)->pstate & PSR_MODE_MASK)

	((regs)->pstate & PSR_MODE_MASK)





#define irqs_priority_unmasked(regs)					\

	(system_uses_irq_prio_masking() ?				\

		(regs)->pmr_save == GIC_PRIO_IRQON :			\

		true)



#define interrupts_enabled(regs)			\

#define interrupts_enabled(regs)			\

	(!((regs)->pstate & PSR_I_BIT))

	(!((regs)->pstate & PSR_I_BIT) && irqs_priority_unmasked(regs))





#define fast_interrupts_enabled(regs) \

#define fast_interrupts_enabled(regs) \

	(!((regs)->pstate & PSR_F_BIT))

	(!((regs)->pstate & PSR_F_BIT))

arch/arm64/kernel/asm-offsets.c

+1 −0
Original line number Original line Diff line number Diff line
@@ -73,6 +73,7 @@ int main(void) 
  DEFINE(S_PC,			offsetof(struct pt_regs, pc));

  DEFINE(S_PC,			offsetof(struct pt_regs, pc));

  DEFINE(S_SYSCALLNO,		offsetof(struct pt_regs, syscallno));

  DEFINE(S_SYSCALLNO,		offsetof(struct pt_regs, syscallno));

  DEFINE(S_ORIG_ADDR_LIMIT,	offsetof(struct pt_regs, orig_addr_limit));

  DEFINE(S_ORIG_ADDR_LIMIT,	offsetof(struct pt_regs, orig_addr_limit));

  DEFINE(S_PMR_SAVE,		offsetof(struct pt_regs, pmr_save));

  DEFINE(S_STACKFRAME,		offsetof(struct pt_regs, stackframe));

  DEFINE(S_STACKFRAME,		offsetof(struct pt_regs, stackframe));

  DEFINE(S_FRAME_SIZE,		sizeof(struct pt_regs));

  DEFINE(S_FRAME_SIZE,		sizeof(struct pt_regs));

  BLANK();

  BLANK();

arch/arm64/kernel/entry.S

+14 −0
Original line number Original line Diff line number Diff line
@@ -249,6 +249,12 @@ alternative_else_nop_endif 
	msr	sp_el0, tsk

	msr	sp_el0, tsk

	.endif

	.endif





	/* Save pmr */

alternative_if ARM64_HAS_IRQ_PRIO_MASKING

	mrs_s	x20, SYS_ICC_PMR_EL1

	str	x20, [sp, #S_PMR_SAVE]

alternative_else_nop_endif



	/*

	/*

	 * Registers that may be useful after this macro is invoked:

	 * Registers that may be useful after this macro is invoked:

	 *

	 *
@@ -269,6 +275,14 @@ alternative_else_nop_endif 
	/* No need to restore UAO, it will be restored from SPSR_EL1 */

	/* No need to restore UAO, it will be restored from SPSR_EL1 */

	.endif

	.endif





	/* Restore pmr */

alternative_if ARM64_HAS_IRQ_PRIO_MASKING

	ldr	x20, [sp, #S_PMR_SAVE]

	msr_s	SYS_ICC_PMR_EL1, x20

	/* Ensure priority change is seen by redistributor */

	dsb	sy

alternative_else_nop_endif



	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR

	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR

	.if	\el == 0

	.if	\el == 0

	ct_user_enter

	ct_user_enter

arch/arm64/kernel/process.c

+6 −0
Original line number Original line Diff line number Diff line
@@ -232,6 +232,9 @@ void __show_regs(struct pt_regs *regs) 




	printk("sp : %016llx\n", sp);

	printk("sp : %016llx\n", sp);





	if (system_uses_irq_prio_masking())

		printk("pmr_save: %08llx\n", regs->pmr_save);



	i = top_reg;

	i = top_reg;





	while (i >= 0) {

	while (i >= 0) {
@@ -363,6 +366,9 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, 
		if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)

		if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)

			childregs->pstate |= PSR_SSBS_BIT;

			childregs->pstate |= PSR_SSBS_BIT;





		if (system_uses_irq_prio_masking())

			childregs->pmr_save = GIC_PRIO_IRQON;



		p->thread.cpu_context.x19 = stack_start;

		p->thread.cpu_context.x19 = stack_start;

		p->thread.cpu_context.x20 = stk_sz;

		p->thread.cpu_context.x20 = stk_sz;

	}

	}