Commit 2e77a62c authored by Mark Rutland's avatar Mark Rutland Committed by Will Deacon
Browse files

arm64: extable: add a dedicated uaccess handler 



For inline assembly, we place exception fixups out-of-line in the
`.fixup` section such that these are out of the way of the fast path.
This has a few drawbacks:

* Since the fixup code is anonymous, backtraces will symbolize fixups as
  offsets from the nearest prior symbol, currently
  `__entry_tramp_text_end`. This is confusing, and painful to debug
  without access to the relevant vmlinux.

* Since the exception handler adjusts the PC to execute the fixup, and
  the fixup uses a direct branch back into the function it fixes,
  backtraces of fixups miss the original function. This is confusing,
  and violates requirements for RELIABLE_STACKTRACE (and therefore
  LIVEPATCH).

* Inline assembly and associated fixups are generated from templates,
  and we have many copies of logically identical fixups which only
  differ in which specific registers are written to and which address is
  branched to at the end of the fixup. This is potentially wasteful of
  I-cache resources, and makes it hard to add additional logic to fixups
  without significant bloat.

This patch address all three concerns for inline uaccess fixups by
adding a dedicated exception handler which updates registers in
exception context and subsequent returns back into the function which
faulted, removing the need for fixups specialized to each faulting
instruction.

Other than backtracing, there should be no functional change as a result
of this patch.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211019160219.5202-12-mark.rutland@arm.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent d6e2cc56

arch/arm64/include/asm/asm-extable.h

+24 −0
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#define EX_TYPE_NONE			0

#define EX_TYPE_FIXUP			1

#define EX_TYPE_BPF			2

#define EX_TYPE_UACCESS_ERR_ZERO	3



#ifdef __ASSEMBLY__
@@ -37,8 +38,11 @@ 


#else /* __ASSEMBLY__ */



#include <linux/bits.h>

#include <linux/stringify.h>



#include <asm/gpr-num.h>



#define __ASM_EXTABLE_RAW(insn, fixup, type, data)	\

	".pushsection	__ex_table, \"a\"\n"		\

	".align		2\n"				\
@@ -51,6 +55,26 @@ 
#define _ASM_EXTABLE(insn, fixup) \

	__ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")



#define EX_DATA_REG_ERR_SHIFT	0

#define EX_DATA_REG_ERR		GENMASK(4, 0)

#define EX_DATA_REG_ZERO_SHIFT	5

#define EX_DATA_REG_ZERO	GENMASK(9, 5)



#define EX_DATA_REG(reg, gpr)						\

	"((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")"



#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero)		\

	__DEFINE_ASM_GPR_NUMS						\

	__ASM_EXTABLE_RAW(#insn, #fixup, 				\

			  __stringify(EX_TYPE_UACCESS_ERR_ZERO),	\

			  "("						\

			    EX_DATA_REG(ERR, err) " | "			\

			    EX_DATA_REG(ZERO, zero)			\

			  ")")



#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err)			\

	_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)



#endif /* __ASSEMBLY__ */



#endif /* __ASM_ASM_EXTABLE_H */

arch/arm64/include/asm/futex.h

+8 −17
Original line number Diff line number Diff line
@@ -25,19 +25,14 @@ do { \ 
"	cbz	%w0, 3f\n"						\

"	sub	%w4, %w4, %w0\n"					\

"	cbnz	%w4, 1b\n"						\

"	mov	%w0, %w7\n"						\

"	mov	%w0, %w6\n"						\

"3:\n"									\

"	dmb	ish\n"							\

"	.pushsection .fixup,\"ax\"\n"					\

"	.align	2\n"							\

"4:	mov	%w0, %w6\n"						\

"	b	3b\n"							\

"	.popsection\n"							\

	_ASM_EXTABLE(1b, 4b)						\

	_ASM_EXTABLE(2b, 4b)						\

	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0)				\

	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0)				\

	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),	\

	  "+r" (loops)							\

	: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN)			\

	: "r" (oparg), "Ir" (-EAGAIN)					\

	: "memory");							\

	uaccess_disable_privileged();					\

} while (0)
@@ -105,18 +100,14 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr, 
"	cbz	%w3, 3f\n"

"	sub	%w4, %w4, %w3\n"

"	cbnz	%w4, 1b\n"

"	mov	%w0, %w8\n"

"	mov	%w0, %w7\n"

"3:\n"

"	dmb	ish\n"

"4:\n"

"	.pushsection .fixup,\"ax\"\n"

"5:	mov	%w0, %w7\n"

"	b	4b\n"

"	.popsection\n"

	_ASM_EXTABLE(1b, 5b)

	_ASM_EXTABLE(2b, 5b)

	_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)

	_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)

	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)

	: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)

	: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)

	: "memory");

	uaccess_disable_privileged();

arch/arm64/include/asm/uaccess.h

+4 −15
Original line number Diff line number Diff line
@@ -255,15 +255,9 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr) 
	asm volatile(							\

	"1:	" load "	" reg "1, [%2]\n"			\

	"2:\n"								\

	"	.section .fixup, \"ax\"\n"				\

	"	.align	2\n"						\

	"3:	mov	%w0, %3\n"					\

	"	mov	%1, #0\n"					\

	"	b	2b\n"						\

	"	.previous\n"						\

	_ASM_EXTABLE(1b, 3b)						\

	_ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %w0, %w1)			\

	: "+r" (err), "=&r" (x)						\

	: "r" (addr), "i" (-EFAULT))

	: "r" (addr))



#define __raw_get_mem(ldr, x, ptr, err)					\

do {									\
@@ -332,14 +326,9 @@ do { \ 
	asm volatile(							\

	"1:	" store "	" reg "1, [%2]\n"			\

	"2:\n"								\

	"	.section .fixup,\"ax\"\n"				\

	"	.align	2\n"						\

	"3:	mov	%w0, %3\n"					\

	"	b	2b\n"						\

	"	.previous\n"						\

	_ASM_EXTABLE(1b, 3b)						\

	_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0)				\

	: "+r" (err)							\

	: "r" (x), "r" (addr), "i" (-EFAULT))

	: "r" (x), "r" (addr))



#define __raw_put_mem(str, x, ptr, err)					\

do {									\

arch/arm64/kernel/armv8_deprecated.c

+3 −9
Original line number Diff line number Diff line
@@ -279,7 +279,7 @@ static void __init register_insn_emulation_sysctl(void) 
do {								\

	uaccess_enable_privileged();				\

	__asm__ __volatile__(					\

	"	mov		%w3, %w7\n"			\

	"	mov		%w3, %w6\n"			\

	"0:	ldxr"B"		%w2, [%4]\n"			\

	"1:	stxr"B"		%w0, %w1, [%4]\n"		\

	"	cbz		%w0, 2f\n"			\
@@ -290,16 +290,10 @@ do { \ 
	"2:\n"							\

	"	mov		%w1, %w2\n"			\

	"3:\n"							\

	"	.pushsection	 .fixup,\"ax\"\n"		\

	"	.align		2\n"				\

	"4:	mov		%w0, %w6\n"			\

	"	b		3b\n"				\

	"	.popsection"					\

	_ASM_EXTABLE(0b, 4b)					\

	_ASM_EXTABLE(1b, 4b)					\

	_ASM_EXTABLE_UACCESS_ERR(0b, 3b, %w0)			\

	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0)			\

	: "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2)	\

	: "r" ((unsigned long)addr), "i" (-EAGAIN),		\

	  "i" (-EFAULT),					\

	  "i" (__SWP_LL_SC_LOOPS)				\

	: "memory");						\

	uaccess_disable_privileged();				\

arch/arm64/kernel/traps.c

+2 −7
Original line number Diff line number Diff line
@@ -527,14 +527,9 @@ NOKPROBE_SYMBOL(do_ptrauth_fault); 
			"1:	" insn ", %1\n"			\

			"	mov	%w0, #0\n"		\

			"2:\n"					\

			"	.pushsection .fixup,\"ax\"\n"	\

			"	.align	2\n"			\

			"3:	mov	%w0, %w2\n"		\

			"	b	2b\n"			\

			"	.popsection\n"			\

			_ASM_EXTABLE(1b, 3b)			\

			_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0)	\

			: "=r" (res)				\

			: "r" (address), "i" (-EFAULT));	\

			: "r" (address));			\

		uaccess_ttbr0_disable();			\

	}