Commit 43aa0a19 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'objtool-core-2021-10-31' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull objtool updates from Thomas Gleixner:

 - Improve retpoline code patching by separating it from alternatives
   which reduces memory footprint and allows to do better optimizations
   in the actual runtime patching.

 - Add proper retpoline support for x86/BPF

 - Address noinstr warnings in x86/kvm, lockdep and paravirtualization
   code

 - Add support to handle pv_opsindirect calls in the noinstr analysis

 - Classify symbols upfront and cache the result to avoid redundant
   str*cmp() invocations.

 - Add a CFI hash to reduce memory consumption which also reduces
   runtime on a allyesconfig by ~50%

 - Adjust XEN code to make objtool handling more robust and as a side
   effect to prevent text fragmentation due to placement of the
   hypercall page.

* tag 'objtool-core-2021-10-31' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (41 commits)
  bpf,x86: Respect X86_FEATURE_RETPOLINE*
  bpf,x86: Simplify computing label offsets
  x86,bugs: Unconditionally allow spectre_v2=retpoline,amd
  x86/alternative: Add debug prints to apply_retpolines()
  x86/alternative: Try inline spectre_v2=retpoline,amd
  x86/alternative: Handle Jcc __x86_indirect_thunk_\reg
  x86/alternative: Implement .retpoline_sites support
  x86/retpoline: Create a retpoline thunk array
  x86/retpoline: Move the retpoline thunk declarations to nospec-branch.h
  x86/asm: Fixup odd GEN-for-each-reg.h usage
  x86/asm: Fix register order
  x86/retpoline: Remove unused replacement symbols
  objtool,x86: Replace alternatives with .retpoline_sites
  objtool: Shrink struct instruction
  objtool: Explicitly avoid self modifying code in .altinstr_replacement
  objtool: Classify symbols
  objtool: Support pv_opsindirect calls for noinstr
  x86/xen: Rework the xen_{cpu,irq,mmu}_opsarrays
  x86/xen: Mark xen_force_evtchn_callback() noinstr
  x86/xen: Make irq_disable() noinstr
  ...
parents 595b28fb 87c87ecd

arch/um/kernel/um_arch.c

+4 −0
Original line number Diff line number Diff line
@@ -421,6 +421,10 @@ void __init check_bugs(void) 
	os_check_bugs();

}



void apply_retpolines(s32 *start, s32 *end)

{

}



void apply_alternatives(struct alt_instr *start, struct alt_instr *end)

{

}

arch/x86/include/asm/GEN-for-each-reg.h

+10 −4
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * These are in machine order; things rely on that.

 */

#ifdef CONFIG_64BIT

GEN(rax)

GEN(rbx)

GEN(rcx)

GEN(rdx)

GEN(rbx)

GEN(rsp)

GEN(rbp)

GEN(rsi)

GEN(rdi)

GEN(rbp)

GEN(r8)

GEN(r9)

GEN(r10)
@@ -16,10 +21,11 @@ GEN(r14) 
GEN(r15)

#else

GEN(eax)

GEN(ebx)

GEN(ecx)

GEN(edx)

GEN(ebx)

GEN(esp)

GEN(ebp)

GEN(esi)

GEN(edi)

GEN(ebp)

#endif

arch/x86/include/asm/alternative.h

+1 −0
Original line number Diff line number Diff line
@@ -75,6 +75,7 @@ extern int alternatives_patched; 


extern void alternative_instructions(void);

extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);

extern void apply_retpolines(s32 *start, s32 *end);



struct module;

arch/x86/include/asm/asm-prototypes.h

+0 −18
Original line number Diff line number Diff line
@@ -17,21 +17,3 @@ 
extern void cmpxchg8b_emu(void);

#endif

#ifdef CONFIG_RETPOLINE



#undef GEN

#define GEN(reg) \

	extern asmlinkage void __x86_indirect_thunk_ ## reg (void);

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) \

	extern asmlinkage void __x86_indirect_alt_call_ ## reg (void);

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) \

	extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);

#include <asm/GEN-for-each-reg.h>



#endif /* CONFIG_RETPOLINE */

arch/x86/include/asm/nospec-branch.h

+13 −59
Original line number Diff line number Diff line
@@ -5,12 +5,15 @@ 


#include <linux/static_key.h>

#include <linux/objtool.h>

#include <linux/linkage.h>



#include <asm/alternative.h>

#include <asm/cpufeatures.h>

#include <asm/msr-index.h>

#include <asm/unwind_hints.h>



#define RETPOLINE_THUNK_SIZE	32



/*

 * Fill the CPU return stack buffer.

 *
@@ -118,6 +121,16 @@ 
	".popsection\n\t"



#ifdef CONFIG_RETPOLINE



typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];



#define GEN(reg) \

	extern retpoline_thunk_t __x86_indirect_thunk_ ## reg;

#include <asm/GEN-for-each-reg.h>

#undef GEN



extern retpoline_thunk_t __x86_indirect_thunk_array[];



#ifdef CONFIG_X86_64



/*
@@ -303,63 +316,4 @@ static inline void mds_idle_clear_cpu_buffers(void) 


#endif /* __ASSEMBLY__ */



/*

 * Below is used in the eBPF JIT compiler and emits the byte sequence

 * for the following assembly:

 *

 * With retpolines configured:

 *

 *    callq do_rop

 *  spec_trap:

 *    pause

 *    lfence

 *    jmp spec_trap

 *  do_rop:

 *    mov %rcx,(%rsp) for x86_64

 *    mov %edx,(%esp) for x86_32

 *    retq

 *

 * Without retpolines configured:

 *

 *    jmp *%rcx for x86_64

 *    jmp *%edx for x86_32

 */

#ifdef CONFIG_RETPOLINE

# ifdef CONFIG_X86_64

#  define RETPOLINE_RCX_BPF_JIT_SIZE	17

#  define RETPOLINE_RCX_BPF_JIT()				\

do {								\

	EMIT1_off32(0xE8, 7);	 /* callq do_rop */		\

	/* spec_trap: */					\

	EMIT2(0xF3, 0x90);       /* pause */			\

	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\

	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\

	/* do_rop: */						\

	EMIT4(0x48, 0x89, 0x0C, 0x24); /* mov %rcx,(%rsp) */	\

	EMIT1(0xC3);             /* retq */			\

} while (0)

# else /* !CONFIG_X86_64 */

#  define RETPOLINE_EDX_BPF_JIT()				\

do {								\

	EMIT1_off32(0xE8, 7);	 /* call do_rop */		\

	/* spec_trap: */					\

	EMIT2(0xF3, 0x90);       /* pause */			\

	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\

	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\

	/* do_rop: */						\

	EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */		\

	EMIT1(0xC3);             /* ret */			\

} while (0)

# endif

#else /* !CONFIG_RETPOLINE */

# ifdef CONFIG_X86_64

#  define RETPOLINE_RCX_BPF_JIT_SIZE	2

#  define RETPOLINE_RCX_BPF_JIT()				\

	EMIT2(0xFF, 0xE1);       /* jmp *%rcx */

# else /* !CONFIG_X86_64 */

#  define RETPOLINE_EDX_BPF_JIT()				\

	EMIT2(0xFF, 0xE2)        /* jmp *%edx */

# endif

#endif



#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */