Commit d6e2cc56 authored by Mark Rutland's avatar Mark Rutland Committed by Will Deacon
Browse files

arm64: extable: add `type` and `data` fields 



Subsequent patches will add specialized handlers for fixups, in addition
to the simple PC fixup and BPF handlers we have today. In preparation,
this patch adds a new `type` field to struct exception_table_entry, and
uses this to distinguish the fixup and BPF cases. A `data` field is also
added so that subsequent patches can associate data specific to each
exception site (e.g. register numbers).

Handlers are named ex_handler_*() for consistency, following the exmaple
of x86. At the same time, get_ex_fixup() is split out into a helper so
that it can be used by other ex_handler_*() functions ins subsequent
patches.

This patch will increase the size of the exception tables, which will be
remedied by subsequent patches removing redundant fixup code. There
should be no functional change as a result of this patch.

Since each entry is now 12 bytes in size, we must reduce the alignment
of each entry from `.align 3` (i.e. 8 bytes) to `.align 2` (i.e. 4
bytes), which is the natrual alignment of the `insn` and `fixup` fields.
The current 8-byte alignment is a holdover from when the `insn` and
`fixup` fields was 8 bytes, and while not harmful has not been necessary
since commit:

  6c94f27a ("arm64: switch to relative exception tables")

Similarly, RO_EXCEPTION_TABLE_ALIGN is dropped to 4 bytes.

Concurrently with this patch, x86's exception table entry format is
being updated (similarly to a 12-byte format, with 32-bytes of absolute
data). Once both have been merged it should be possible to unify the
sorttable logic for the two.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: James Morse <james.morse@arm.com>
Cc: Jean-Philippe Brucker <jean-philippe@linaro.org>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211019160219.5202-11-mark.rutland@arm.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 5d0e7905

arch/arm64/include/asm/asm-extable.h

+20 −12
Original line number Diff line number Diff line
@@ -2,13 +2,19 @@ 
#ifndef __ASM_ASM_EXTABLE_H

#define __ASM_ASM_EXTABLE_H



#define EX_TYPE_NONE			0

#define EX_TYPE_FIXUP			1

#define EX_TYPE_BPF			2



#ifdef __ASSEMBLY__



#define __ASM_EXTABLE_RAW(insn, fixup)		\

#define __ASM_EXTABLE_RAW(insn, fixup, type, data)	\

	.pushsection	__ex_table, "a";		\

	.align		3;			\

	.align		2;				\

	.long		((insn) - .);			\

	.long		((fixup) - .);			\

	.short		(type);				\

	.short		(data);				\

	.popsection;



/*
@@ -16,7 +22,7 @@ 
 * when an unhandled fault is taken.

 */

	.macro		_asm_extable, insn, fixup

	__ASM_EXTABLE_RAW(\insn, \fixup)

	__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)

	.endm



/*
@@ -33,15 +39,17 @@ 


#include <linux/stringify.h>



#define __ASM_EXTABLE_RAW(insn, fixup)		\

#define __ASM_EXTABLE_RAW(insn, fixup, type, data)	\

	".pushsection	__ex_table, \"a\"\n"		\

	".align		3\n"			\

	".align		2\n"				\

	".long		((" insn ") - .)\n"		\

	".long		((" fixup ") - .)\n"		\

	".short		(" type ")\n"			\

	".short		(" data ")\n"			\

	".popsection\n"



#define _ASM_EXTABLE(insn, fixup) \

	__ASM_EXTABLE_RAW(#insn, #fixup)

	__ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")



#endif /* __ASSEMBLY__ */

arch/arm64/include/asm/extable.h

+15 −4
Original line number Diff line number Diff line
@@ -18,10 +18,21 @@ 
struct exception_table_entry

{

	int insn, fixup;

	short type, data;

};



#define ARCH_HAS_RELATIVE_EXTABLE



#define swap_ex_entry_fixup(a, b, tmp, delta)		\

do {							\

	(a)->fixup = (b)->fixup + (delta);		\

	(b)->fixup = (tmp).fixup - (delta);		\

	(a)->type = (b)->type;				\

	(b)->type = (tmp).type;				\

	(a)->data = (b)->data;				\

	(b)->data = (tmp).data;				\

} while (0)



static inline bool in_bpf_jit(struct pt_regs *regs)

{

	if (!IS_ENABLED(CONFIG_BPF_JIT))
@@ -32,11 +43,11 @@ static inline bool in_bpf_jit(struct pt_regs *regs) 
}



#ifdef CONFIG_BPF_JIT

bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,

bool ex_handler_bpf(const struct exception_table_entry *ex,

		    struct pt_regs *regs);

#else /* !CONFIG_BPF_JIT */

static inline

bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,

bool ex_handler_bpf(const struct exception_table_entry *ex,

		    struct pt_regs *regs)

{

	return false;

arch/arm64/kernel/vmlinux.lds.S

+1 −1
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ 
#define SBSS_ALIGN			0

#endif



#define RO_EXCEPTION_TABLE_ALIGN	8

#define RO_EXCEPTION_TABLE_ALIGN	4

#define RUNTIME_DISCARD_EXIT



#include <asm-generic/vmlinux.lds.h>

arch/arm64/mm/extable.c

+25 −4
Original line number Diff line number Diff line
@@ -6,6 +6,24 @@ 
#include <linux/extable.h>

#include <linux/uaccess.h>



#include <asm/asm-extable.h>



typedef bool (*ex_handler_t)(const struct exception_table_entry *,

			     struct pt_regs *);



static inline unsigned long

get_ex_fixup(const struct exception_table_entry *ex)

{

	return ((unsigned long)&ex->fixup + ex->fixup);

}



static bool ex_handler_fixup(const struct exception_table_entry *ex,

			     struct pt_regs *regs)

{

	regs->pc = get_ex_fixup(ex);

	return true;

}



bool fixup_exception(struct pt_regs *regs)

{

	const struct exception_table_entry *ex;
@@ -14,9 +32,12 @@ bool fixup_exception(struct pt_regs *regs) 
	if (!ex)

		return false;



	if (in_bpf_jit(regs))

		return arm64_bpf_fixup_exception(ex, regs);

	switch (ex->type) {

	case EX_TYPE_FIXUP:

		return ex_handler_fixup(ex, regs);

	case EX_TYPE_BPF:

		return ex_handler_bpf(ex, regs);

	}



	regs->pc = (unsigned long)&ex->fixup + ex->fixup;

	return true;

	BUG();

}

arch/arm64/net/bpf_jit_comp.c

+5 −2
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include <linux/printk.h>

#include <linux/slab.h>



#include <asm/asm-extable.h>

#include <asm/byteorder.h>

#include <asm/cacheflush.h>

#include <asm/debug-monitors.h>
@@ -358,7 +359,7 @@ static void build_epilogue(struct jit_ctx *ctx) 
#define BPF_FIXUP_OFFSET_MASK	GENMASK(26, 0)

#define BPF_FIXUP_REG_MASK	GENMASK(31, 27)



bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,

bool ex_handler_bpf(const struct exception_table_entry *ex,

		    struct pt_regs *regs)

{

	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
@@ -412,6 +413,8 @@ static int add_exception_handler(const struct bpf_insn *insn, 
	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |

		    FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);



	ex->type = EX_TYPE_BPF;



	ctx->exentry_idx++;

	return 0;

}