Commit 8f93402b authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

x86/ibt,entry: Sprinkle ENDBR dust 



Kernel entry points should be having ENDBR on for IBT configs.

The SYSCALL entry points are found through taking their respective
address in order to program them in the MSRs, while the exception
entry points are found through UNWIND_HINT_IRET_REGS.

The rule is that any UNWIND_HINT_IRET_REGS at sym+0 should have an
ENDBR, see the later objtool ibt validation patch.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20220308154317.933157479@infradead.org
parent 5b2fc515

arch/x86/entry/entry_64.S

+7 −0
Original line number Diff line number Diff line
@@ -86,6 +86,7 @@ 


SYM_CODE_START(entry_SYSCALL_64)

	UNWIND_HINT_EMPTY

	ENDBR



	swapgs

	/* tss.sp2 is scratch space. */
@@ -350,6 +351,7 @@ SYM_CODE_END(ret_from_fork) 
.macro idtentry vector asmsym cfunc has_error_code:req

SYM_CODE_START(\asmsym)

	UNWIND_HINT_IRET_REGS offset=\has_error_code*8

	ENDBR

	ASM_CLAC



	.if \has_error_code == 0
@@ -417,6 +419,7 @@ SYM_CODE_END(\asmsym) 
.macro idtentry_mce_db vector asmsym cfunc

SYM_CODE_START(\asmsym)

	UNWIND_HINT_IRET_REGS

	ENDBR

	ASM_CLAC



	pushq	$-1			/* ORIG_RAX: no syscall to restart */
@@ -472,6 +475,7 @@ SYM_CODE_END(\asmsym) 
.macro idtentry_vc vector asmsym cfunc

SYM_CODE_START(\asmsym)

	UNWIND_HINT_IRET_REGS

	ENDBR

	ASM_CLAC



	/*
@@ -533,6 +537,7 @@ SYM_CODE_END(\asmsym) 
.macro idtentry_df vector asmsym cfunc

SYM_CODE_START(\asmsym)

	UNWIND_HINT_IRET_REGS offset=8

	ENDBR

	ASM_CLAC



	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
@@ -1069,6 +1074,7 @@ SYM_CODE_END(error_return) 
 */

SYM_CODE_START(asm_exc_nmi)

	UNWIND_HINT_IRET_REGS

	ENDBR



	/*

	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
@@ -1427,6 +1433,7 @@ SYM_CODE_END(asm_exc_nmi) 
 */

SYM_CODE_START(ignore_sysret)

	UNWIND_HINT_EMPTY

	ENDBR

	mov	$-ENOSYS, %eax

	sysretl

SYM_CODE_END(ignore_sysret)

arch/x86/entry/entry_64_compat.S

+3 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ 
 */

SYM_CODE_START(entry_SYSENTER_compat)

	UNWIND_HINT_EMPTY

	ENDBR

	/* Interrupts are off on entry. */

	SWAPGS
@@ -198,6 +199,7 @@ SYM_CODE_END(entry_SYSENTER_compat) 
 */

SYM_CODE_START(entry_SYSCALL_compat)

	UNWIND_HINT_EMPTY

	ENDBR

	/* Interrupts are off on entry. */

	swapgs
@@ -340,6 +342,7 @@ SYM_CODE_END(entry_SYSCALL_compat) 
 */

SYM_CODE_START(entry_INT80_compat)

	UNWIND_HINT_EMPTY

	ENDBR

	/*

	 * Interrupts are off on entry.

	 */

arch/x86/include/asm/idtentry.h

+11 −9
Original line number Diff line number Diff line
@@ -5,6 +5,8 @@ 
/* Interrupts/Exceptions */

#include <asm/trapnr.h>



#define IDT_ALIGN	(8 * (1 + HAS_KERNEL_IBT))



#ifndef __ASSEMBLY__

#include <linux/entry-common.h>

#include <linux/hardirq.h>
@@ -480,7 +482,7 @@ __visible noinstr void func(struct pt_regs *regs, \ 


/*

 * ASM code to emit the common vector entry stubs where each stub is

 * packed into 8 bytes.

 * packed into IDT_ALIGN bytes.

 *

 * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because

 * GCC treats the local vector variable as unsigned int and would expand
@@ -492,33 +494,33 @@ __visible noinstr void func(struct pt_regs *regs, \ 
 * point is to mask off the bits above bit 7 because the push is sign

 * extending.

 */

	.align 8

	.align IDT_ALIGN

SYM_CODE_START(irq_entries_start)

    vector=FIRST_EXTERNAL_VECTOR

    .rept NR_EXTERNAL_VECTORS

	UNWIND_HINT_IRET_REGS

0 :

	ENDBR

	.byte	0x6a, vector

	jmp	asm_common_interrupt

	nop

	/* Ensure that the above is 8 bytes max */

	. = 0b + 8

	/* Ensure that the above is IDT_ALIGN bytes max */

	.fill 0b + IDT_ALIGN - ., 1, 0xcc

	vector = vector+1

    .endr

SYM_CODE_END(irq_entries_start)



#ifdef CONFIG_X86_LOCAL_APIC

	.align 8

	.align IDT_ALIGN

SYM_CODE_START(spurious_entries_start)

    vector=FIRST_SYSTEM_VECTOR

    .rept NR_SYSTEM_VECTORS

	UNWIND_HINT_IRET_REGS

0 :

	ENDBR

	.byte	0x6a, vector

	jmp	asm_spurious_interrupt

	nop

	/* Ensure that the above is 8 bytes max */

	. = 0b + 8

	/* Ensure that the above is IDT_ALIGN bytes max */

	.fill 0b + IDT_ALIGN - ., 1, 0xcc

	vector = vector+1

    .endr

SYM_CODE_END(spurious_entries_start)

arch/x86/include/asm/segment.h

+2 −1
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 


#include <linux/const.h>

#include <asm/alternative.h>

#include <asm/ibt.h>



/*

 * Constructor for a conventional segment GDT (or LDT) entry.
@@ -275,7 +276,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) 
 * vector has no error code (two bytes), a 'push $vector_number' (two

 * bytes), and a jump to the common entry code (up to five bytes).

 */

#define EARLY_IDT_HANDLER_SIZE 9

#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)



/*

 * xen_early_idt_handler_array is for Xen pv guests: for each entry in

arch/x86/kernel/head_64.S

+3 −1
Original line number Diff line number Diff line
@@ -371,9 +371,11 @@ SYM_CODE_START(early_idt_handler_array) 
	.rept NUM_EXCEPTION_VECTORS

	.if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0

		UNWIND_HINT_IRET_REGS

		ENDBR

		pushq $0	# Dummy error code, to make stack frame uniform

	.else

		UNWIND_HINT_IRET_REGS offset=8

		ENDBR

	.endif

	pushq $i		# 72(%rsp) Vector number

	jmp early_idt_handler_common
@@ -381,11 +383,11 @@ SYM_CODE_START(early_idt_handler_array) 
	i = i + 1

	.fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc

	.endr

	UNWIND_HINT_IRET_REGS offset=16

SYM_CODE_END(early_idt_handler_array)

	ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS]



SYM_CODE_START_LOCAL(early_idt_handler_common)

	UNWIND_HINT_IRET_REGS offset=16

	/*

	 * The stack is the hardware frame, an error code or zero, and the

	 * vector number.