Commit 99fe09c8 authored by Will Deacon's avatar Will Deacon
Browse files

Merge branch 'for-next/extable' into for-next/core 

* for-next/extable:
  arm64: vmlinux.lds.S: remove `.fixup` section
  arm64: extable: add load_unaligned_zeropad() handler
  arm64: extable: add a dedicated uaccess handler
  arm64: extable: add `type` and `data` fields
  arm64: extable: use `ex` for `exception_table_entry`
  arm64: extable: make fixup_exception() return bool
  arm64: extable: consolidate definitions
  arm64: gpr-num: support W registers
  arm64: factor out GPR numbering helpers
  arm64: kvm: use kvm_exception_table_entry
  arm64: lib: __arch_copy_to_user(): fold fixups into body
  arm64: lib: __arch_copy_from_user(): fold fixups into body
  arm64: lib: __arch_clear_user(): fold fixups into body
parents a69483ee bf6e667f

arch/arm64/include/asm/asm-extable.h

0 → 100644
+95 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

#ifndef __ASM_ASM_EXTABLE_H

#define __ASM_ASM_EXTABLE_H



#define EX_TYPE_NONE			0

#define EX_TYPE_FIXUP			1

#define EX_TYPE_BPF			2

#define EX_TYPE_UACCESS_ERR_ZERO	3

#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD	4



#ifdef __ASSEMBLY__



#define __ASM_EXTABLE_RAW(insn, fixup, type, data)	\

	.pushsection	__ex_table, "a";		\

	.align		2;				\

	.long		((insn) - .);			\

	.long		((fixup) - .);			\

	.short		(type);				\

	.short		(data);				\

	.popsection;



/*

 * Create an exception table entry for `insn`, which will branch to `fixup`

 * when an unhandled fault is taken.

 */

	.macro		_asm_extable, insn, fixup

	__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)

	.endm



/*

 * Create an exception table entry for `insn` if `fixup` is provided. Otherwise

 * do nothing.

 */

	.macro		_cond_extable, insn, fixup

	.ifnc		\fixup,

	_asm_extable	\insn, \fixup

	.endif

	.endm



#else /* __ASSEMBLY__ */



#include <linux/bits.h>

#include <linux/stringify.h>



#include <asm/gpr-num.h>



#define __ASM_EXTABLE_RAW(insn, fixup, type, data)	\

	".pushsection	__ex_table, \"a\"\n"		\

	".align		2\n"				\

	".long		((" insn ") - .)\n"		\

	".long		((" fixup ") - .)\n"		\

	".short		(" type ")\n"			\

	".short		(" data ")\n"			\

	".popsection\n"



#define _ASM_EXTABLE(insn, fixup) \

	__ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")



#define EX_DATA_REG_ERR_SHIFT	0

#define EX_DATA_REG_ERR		GENMASK(4, 0)

#define EX_DATA_REG_ZERO_SHIFT	5

#define EX_DATA_REG_ZERO	GENMASK(9, 5)



#define EX_DATA_REG(reg, gpr)						\

	"((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")"



#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero)		\

	__DEFINE_ASM_GPR_NUMS						\

	__ASM_EXTABLE_RAW(#insn, #fixup, 				\

			  __stringify(EX_TYPE_UACCESS_ERR_ZERO),	\

			  "("						\

			    EX_DATA_REG(ERR, err) " | "			\

			    EX_DATA_REG(ZERO, zero)			\

			  ")")



#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err)			\

	_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)



#define EX_DATA_REG_DATA_SHIFT	0

#define EX_DATA_REG_DATA	GENMASK(4, 0)

#define EX_DATA_REG_ADDR_SHIFT	5

#define EX_DATA_REG_ADDR	GENMASK(9, 5)



#define _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(insn, fixup, data, addr)		\

	__DEFINE_ASM_GPR_NUMS							\

	__ASM_EXTABLE_RAW(#insn, #fixup,					\

			  __stringify(EX_TYPE_LOAD_UNALIGNED_ZEROPAD),		\

			  "("							\

			    EX_DATA_REG(DATA, data) " | "			\

			    EX_DATA_REG(ADDR, addr)				\

			  ")")



#endif /* __ASSEMBLY__ */



#endif /* __ASM_ASM_EXTABLE_H */

arch/arm64/include/asm/asm-uaccess.h

+6 −1
Original line number Diff line number Diff line
@@ -3,10 +3,11 @@ 
#define __ASM_ASM_UACCESS_H



#include <asm/alternative-macros.h>

#include <asm/asm-extable.h>

#include <asm/assembler.h>

#include <asm/kernel-pgtable.h>

#include <asm/mmu.h>

#include <asm/sysreg.h>

#include <asm/assembler.h>



/*

 * User access enabling/disabling macros.
@@ -58,6 +59,10 @@ alternative_else_nop_endif 
	.endm

#endif



#define USER(l, x...)				\

9999:	x;					\

	_asm_extable	9999b, l



/*

 * Generate the assembly for LDTR/STTR with exception table entries.

 * This is complicated as there is no post-increment or pair versions of the

arch/arm64/include/asm/assembler.h

+2 −27
Original line number Diff line number Diff line
@@ -14,9 +14,10 @@ 


#include <asm-generic/export.h>



#include <asm/asm-offsets.h>

#include <asm/alternative.h>

#include <asm/asm-bug.h>

#include <asm/asm-extable.h>

#include <asm/asm-offsets.h>

#include <asm/cpufeature.h>

#include <asm/cputype.h>

#include <asm/debug-monitors.h>
@@ -129,32 +130,6 @@ alternative_endif 
	.endr

	.endm



/*

 * Create an exception table entry for `insn`, which will branch to `fixup`

 * when an unhandled fault is taken.

 */

	.macro		_asm_extable, insn, fixup

	.pushsection	__ex_table, "a"

	.align		3

	.long		(\insn - .), (\fixup - .)

	.popsection

	.endm



/*

 * Create an exception table entry for `insn` if `fixup` is provided. Otherwise

 * do nothing.

 */

	.macro		_cond_extable, insn, fixup

	.ifnc		\fixup,

	_asm_extable	\insn, \fixup

	.endif

	.endm





#define USER(l, x...)				\

9999:	x;					\

	_asm_extable	9999b, l



/*

 * Register aliases.

 */

arch/arm64/include/asm/extable.h

+17 −6
Original line number Diff line number Diff line
@@ -18,10 +18,21 @@ 
struct exception_table_entry

{

	int insn, fixup;

	short type, data;

};



#define ARCH_HAS_RELATIVE_EXTABLE



#define swap_ex_entry_fixup(a, b, tmp, delta)		\

do {							\

	(a)->fixup = (b)->fixup + (delta);		\

	(b)->fixup = (tmp).fixup - (delta);		\

	(a)->type = (b)->type;				\

	(b)->type = (tmp).type;				\

	(a)->data = (b)->data;				\

	(b)->data = (tmp).data;				\

} while (0)



static inline bool in_bpf_jit(struct pt_regs *regs)

{

	if (!IS_ENABLED(CONFIG_BPF_JIT))
@@ -32,16 +43,16 @@ static inline bool in_bpf_jit(struct pt_regs *regs) 
}



#ifdef CONFIG_BPF_JIT

int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,

bool ex_handler_bpf(const struct exception_table_entry *ex,

		    struct pt_regs *regs);

#else /* !CONFIG_BPF_JIT */

static inline

int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,

bool ex_handler_bpf(const struct exception_table_entry *ex,

		    struct pt_regs *regs)

{

	return 0;

	return false;

}

#endif /* !CONFIG_BPF_JIT */



extern int fixup_exception(struct pt_regs *regs);

bool fixup_exception(struct pt_regs *regs);

#endif

arch/arm64/include/asm/futex.h

+8 −17
Original line number Diff line number Diff line
@@ -25,19 +25,14 @@ do { \ 
"	cbz	%w0, 3f\n"						\

"	sub	%w4, %w4, %w0\n"					\

"	cbnz	%w4, 1b\n"						\

"	mov	%w0, %w7\n"						\

"	mov	%w0, %w6\n"						\

"3:\n"									\

"	dmb	ish\n"							\

"	.pushsection .fixup,\"ax\"\n"					\

"	.align	2\n"							\

"4:	mov	%w0, %w6\n"						\

"	b	3b\n"							\

"	.popsection\n"							\

	_ASM_EXTABLE(1b, 4b)						\

	_ASM_EXTABLE(2b, 4b)						\

	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0)				\

	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0)				\

	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),	\

	  "+r" (loops)							\

	: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN)			\

	: "r" (oparg), "Ir" (-EAGAIN)					\

	: "memory");							\

	uaccess_disable_privileged();					\

} while (0)
@@ -105,18 +100,14 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr, 
"	cbz	%w3, 3f\n"

"	sub	%w4, %w4, %w3\n"

"	cbnz	%w4, 1b\n"

"	mov	%w0, %w8\n"

"	mov	%w0, %w7\n"

"3:\n"

"	dmb	ish\n"

"4:\n"

"	.pushsection .fixup,\"ax\"\n"

"5:	mov	%w0, %w7\n"

"	b	4b\n"

"	.popsection\n"

	_ASM_EXTABLE(1b, 5b)

	_ASM_EXTABLE(2b, 5b)

	_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)

	_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)

	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)

	: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)

	: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)

	: "memory");

	uaccess_disable_privileged();