Commit 3bb61aa6 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux 

Pull arm64 fixes from Will Deacon:
 "I'm sad to say that we've got an unusually large arm64 fixes pull for
  rc7 which addresses numerous significant instrumentation issues with
  our entry code.

  Without these patches, lockdep is hopelessly unreliable in some
  configurations [1,2] and syzkaller is therefore not a lot of use
  because it's so noisy.

  Although much of this has always been broken, it appears to have been
  exposed more readily by other changes such as 044d0d6d ("lockdep:
  Only trace IRQ edges") and general lockdep improvements around IRQ
  tracing and NMIs.

  Fixing this properly required moving much of the instrumentation hooks
  from our entry assembly into C, which Mark has been working on for the
  last few weeks. We're not quite ready to move to the recently added
  generic functions yet, but the code here has been deliberately written
  to mimic that closely so we can look at cleaning things up once we
  have a bit more breathing room.

  Having said all that, the second version of these patches was posted
  last week and I pushed it into our CI (kernelci and cki) along with a
  commit which forced on PROVE_LOCKING, NOHZ_FULL and
  CONTEXT_TRACKING_FORCE. The result? We found a real bug in the
  md/raid10 code [3].

  Oh, and there's also a really silly typo patch that's unrelated.

  Summary:

   - Fix numerous issues with instrumentation and exception entry

   - Fix hideous typo in unused register field definition"

[1] https://lore.kernel.org/r/CACT4Y+aAzoJ48Mh1wNYD17pJqyEcDnrxGfApir=-j171TnQXhw@mail.gmail.com
[2] https://lore.kernel.org/r/20201119193819.GA2601289@elver.google.com
[3] https://lore.kernel.org/r/94c76d5e-466a-bc5f-e6c2-a11b65c39f83@redhat.com

* tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux:
  arm64: mte: Fix typo in macro definition
  arm64: entry: fix EL1 debug transitions
  arm64: entry: fix NMI {user, kernel}->kernel transitions
  arm64: entry: fix non-NMI kernel<->kernel transitions
  arm64: ptrace: prepare for EL1 irq/rcu tracking
  arm64: entry: fix non-NMI user<->kernel transitions
  arm64: entry: move el1 irq/nmi logic to C
  arm64: entry: prepare ret_to_user for function call
  arm64: entry: move enter_from_user_mode to entry-common.c
  arm64: entry: mark entry code as noinstr
  arm64: mark idle code as noinstr
  arm64: syscall: exit userspace before unmasking exceptions
parents 2c6ffa9e 9e5344e0

arch/arm64/include/asm/daifflags.h

+3 −0
Original line number Diff line number Diff line
@@ -128,6 +128,9 @@ static inline void local_daif_inherit(struct pt_regs *regs) 
{

	unsigned long flags = regs->pstate & DAIF_MASK;



	if (interrupts_enabled(regs))

		trace_hardirqs_on();



	/*

	 * We can't use local_daif_restore(regs->pstate) here as

	 * system_has_prio_mask_debugging() won't restore the I bit if it can

arch/arm64/include/asm/exception.h

+5 −0
Original line number Diff line number Diff line
@@ -31,7 +31,12 @@ static inline u32 disr_to_esr(u64 disr) 
	return esr;

}



asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs);

asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs);

asmlinkage void enter_from_user_mode(void);

asmlinkage void exit_to_user_mode(void);

void arm64_enter_nmi(struct pt_regs *regs);

void arm64_exit_nmi(struct pt_regs *regs);

void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);

void do_undefinstr(struct pt_regs *regs);

void do_bti(struct pt_regs *regs);

arch/arm64/include/asm/ptrace.h

+4 −0
Original line number Diff line number Diff line
@@ -193,6 +193,10 @@ struct pt_regs { 
	/* Only valid when ARM64_HAS_IRQ_PRIO_MASKING is enabled. */

	u64 pmr_save;

	u64 stackframe[2];



	/* Only valid for some EL1 exceptions. */

	u64 lockdep_hardirqs;

	u64 exit_rcu;

};



static inline bool in_syscall(struct pt_regs const *regs)

arch/arm64/include/asm/sysreg.h

+1 −1
Original line number Diff line number Diff line
@@ -987,7 +987,7 @@ 
#define SYS_TFSR_EL1_TF0_SHIFT	0

#define SYS_TFSR_EL1_TF1_SHIFT	1

#define SYS_TFSR_EL1_TF0	(UL(1) << SYS_TFSR_EL1_TF0_SHIFT)

#define SYS_TFSR_EL1_TF1	(UK(2) << SYS_TFSR_EL1_TF1_SHIFT)

#define SYS_TFSR_EL1_TF1	(UL(1) << SYS_TFSR_EL1_TF1_SHIFT)



/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */

#define SYS_MPIDR_SAFE_VAL	(BIT(31))

arch/arm64/kernel/entry-common.c

+190 −64
Original line number Diff line number Diff line
@@ -17,40 +17,164 @@ 
#include <asm/mmu.h>

#include <asm/sysreg.h>



static void notrace el1_abort(struct pt_regs *regs, unsigned long esr)

/*

 * This is intended to match the logic in irqentry_enter(), handling the kernel

 * mode transitions only.

 */

static void noinstr enter_from_kernel_mode(struct pt_regs *regs)

{

	regs->exit_rcu = false;



	if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {

		lockdep_hardirqs_off(CALLER_ADDR0);

		rcu_irq_enter();

		trace_hardirqs_off_finish();



		regs->exit_rcu = true;

		return;

	}



	lockdep_hardirqs_off(CALLER_ADDR0);

	rcu_irq_enter_check_tick();

	trace_hardirqs_off_finish();

}



/*

 * This is intended to match the logic in irqentry_exit(), handling the kernel

 * mode transitions only, and with preemption handled elsewhere.

 */

static void noinstr exit_to_kernel_mode(struct pt_regs *regs)

{

	lockdep_assert_irqs_disabled();



	if (interrupts_enabled(regs)) {

		if (regs->exit_rcu) {

			trace_hardirqs_on_prepare();

			lockdep_hardirqs_on_prepare(CALLER_ADDR0);

			rcu_irq_exit();

			lockdep_hardirqs_on(CALLER_ADDR0);

			return;

		}



		trace_hardirqs_on();

	} else {

		if (regs->exit_rcu)

			rcu_irq_exit();

	}

}



void noinstr arm64_enter_nmi(struct pt_regs *regs)

{

	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();



	__nmi_enter();

	lockdep_hardirqs_off(CALLER_ADDR0);

	lockdep_hardirq_enter();

	rcu_nmi_enter();



	trace_hardirqs_off_finish();

	ftrace_nmi_enter();

}



void noinstr arm64_exit_nmi(struct pt_regs *regs)

{

	bool restore = regs->lockdep_hardirqs;



	ftrace_nmi_exit();

	if (restore) {

		trace_hardirqs_on_prepare();

		lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	}



	rcu_nmi_exit();

	lockdep_hardirq_exit();

	if (restore)

		lockdep_hardirqs_on(CALLER_ADDR0);

	__nmi_exit();

}



asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)

{

	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))

		arm64_enter_nmi(regs);

	else

		enter_from_kernel_mode(regs);

}



asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)

{

	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))

		arm64_exit_nmi(regs);

	else

		exit_to_kernel_mode(regs);

}



static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);



	enter_from_kernel_mode(regs);

	local_daif_inherit(regs);

	far = untagged_addr(far);

	do_mem_abort(far, esr, regs);

	local_daif_mask();

	exit_to_kernel_mode(regs);

}

NOKPROBE_SYMBOL(el1_abort);



static void notrace el1_pc(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);



	enter_from_kernel_mode(regs);

	local_daif_inherit(regs);

	do_sp_pc_abort(far, esr, regs);

	local_daif_mask();

	exit_to_kernel_mode(regs);

}

NOKPROBE_SYMBOL(el1_pc);



static void notrace el1_undef(struct pt_regs *regs)

static void noinstr el1_undef(struct pt_regs *regs)

{

	enter_from_kernel_mode(regs);

	local_daif_inherit(regs);

	do_undefinstr(regs);

	local_daif_mask();

	exit_to_kernel_mode(regs);

}

NOKPROBE_SYMBOL(el1_undef);



static void notrace el1_inv(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)

{

	enter_from_kernel_mode(regs);

	local_daif_inherit(regs);

	bad_mode(regs, 0, esr);

	local_daif_mask();

	exit_to_kernel_mode(regs);

}

NOKPROBE_SYMBOL(el1_inv);



static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)

static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)

{

	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();



	lockdep_hardirqs_off(CALLER_ADDR0);

	rcu_nmi_enter();



	trace_hardirqs_off_finish();

}



static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)

{

	bool restore = regs->lockdep_hardirqs;



	if (restore) {

		trace_hardirqs_on_prepare();

		lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	}



	rcu_nmi_exit();

	if (restore)

		lockdep_hardirqs_on(CALLER_ADDR0);

}



static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -62,18 +186,21 @@ static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) 
	if (system_uses_irq_prio_masking())

		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);



	arm64_enter_el1_dbg(regs);

	do_debug_exception(far, esr, regs);

	arm64_exit_el1_dbg(regs);

}

NOKPROBE_SYMBOL(el1_dbg);



static void notrace el1_fpac(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)

{

	enter_from_kernel_mode(regs);

	local_daif_inherit(regs);

	do_ptrauth_fault(regs, esr);

	local_daif_mask();

	exit_to_kernel_mode(regs);

}

NOKPROBE_SYMBOL(el1_fpac);



asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)

asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)

{

	unsigned long esr = read_sysreg(esr_el1);
@@ -106,20 +233,34 @@ asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) 
		el1_inv(regs, esr);

	}

}

NOKPROBE_SYMBOL(el1_sync_handler);



static void notrace el0_da(struct pt_regs *regs, unsigned long esr)

asmlinkage void noinstr enter_from_user_mode(void)

{

	lockdep_hardirqs_off(CALLER_ADDR0);

	CT_WARN_ON(ct_state() != CONTEXT_USER);

	user_exit_irqoff();

	trace_hardirqs_off_finish();

}



asmlinkage void noinstr exit_to_user_mode(void)

{

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	user_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);

}



static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);



	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	far = untagged_addr(far);

	do_mem_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el0_da);



static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -131,90 +272,80 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) 
	if (!is_ttbr0_addr(far))

		arm64_apply_bp_hardening();



	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_mem_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el0_ia);



static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_fpsimd_acc(esr, regs);

}

NOKPROBE_SYMBOL(el0_fpsimd_acc);



static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_sve_acc(esr, regs);

}

NOKPROBE_SYMBOL(el0_sve_acc);



static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_fpsimd_exc(esr, regs);

}

NOKPROBE_SYMBOL(el0_fpsimd_exc);



static void notrace el0_sys(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_sysinstr(esr, regs);

}

NOKPROBE_SYMBOL(el0_sys);



static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);



	if (!is_ttbr0_addr(instruction_pointer(regs)))

		arm64_apply_bp_hardening();



	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_sp_pc_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el0_pc);



static void notrace el0_sp(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_sp_pc_abort(regs->sp, esr, regs);

}

NOKPROBE_SYMBOL(el0_sp);



static void notrace el0_undef(struct pt_regs *regs)

static void noinstr el0_undef(struct pt_regs *regs)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_undefinstr(regs);

}

NOKPROBE_SYMBOL(el0_undef);



static void notrace el0_bti(struct pt_regs *regs)

static void noinstr el0_bti(struct pt_regs *regs)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_bti(regs);

}

NOKPROBE_SYMBOL(el0_bti);



static void notrace el0_inv(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	bad_el0_sync(regs, 0, esr);

}

NOKPROBE_SYMBOL(el0_inv);



static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)

{

	/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */

	unsigned long far = read_sysreg(far_el1);
@@ -222,30 +353,28 @@ static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) 
	if (system_uses_irq_prio_masking())

		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);



	user_exit_irqoff();

	enter_from_user_mode();

	do_debug_exception(far, esr, regs);

	local_daif_restore(DAIF_PROCCTX_NOIRQ);

}

NOKPROBE_SYMBOL(el0_dbg);



static void notrace el0_svc(struct pt_regs *regs)

static void noinstr el0_svc(struct pt_regs *regs)

{

	if (system_uses_irq_prio_masking())

		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);



	enter_from_user_mode();

	do_el0_svc(regs);

}

NOKPROBE_SYMBOL(el0_svc);



static void notrace el0_fpac(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_ptrauth_fault(regs, esr);

}

NOKPROBE_SYMBOL(el0_fpac);



asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)

asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)

{

	unsigned long esr = read_sysreg(esr_el1);
@@ -297,27 +426,25 @@ asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) 
		el0_inv(regs, esr);

	}

}

NOKPROBE_SYMBOL(el0_sync_handler);



#ifdef CONFIG_COMPAT

static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	enter_from_user_mode();

	local_daif_restore(DAIF_PROCCTX);

	do_cp15instr(esr, regs);

}

NOKPROBE_SYMBOL(el0_cp15);



static void notrace el0_svc_compat(struct pt_regs *regs)

static void noinstr el0_svc_compat(struct pt_regs *regs)

{

	if (system_uses_irq_prio_masking())

		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);



	enter_from_user_mode();

	do_el0_svc_compat(regs);

}

NOKPROBE_SYMBOL(el0_svc_compat);



asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)

asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)

{

	unsigned long esr = read_sysreg(esr_el1);
@@ -360,5 +487,4 @@ asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) 
		el0_inv(regs, esr);

	}

}

NOKPROBE_SYMBOL(el0_sync_compat_handler);

#endif /* CONFIG_COMPAT */