Commit 9bc0bb50 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

objtool/x86: Rewrite retpoline thunk calls 



When the compiler emits: "CALL __x86_indirect_thunk_\reg" for an
indirect call, have objtool rewrite it to:

	ALTERNATIVE "call __x86_indirect_thunk_\reg",
		    "call *%reg", ALT_NOT(X86_FEATURE_RETPOLINE)

Additionally, in order to not emit endless identical
.altinst_replacement chunks, use a global symbol for them, see
__x86_indirect_alt_*.

This also avoids objtool from having to do code generation.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarMiroslav Benes <mbenes@suse.cz>
Link: https://lkml.kernel.org/r/20210326151300.320177914@infradead.org
parent 50e7b4a1

arch/x86/include/asm/asm-prototypes.h

+10 −2
Original line number Diff line number Diff line
@@ -19,11 +19,19 @@ extern void cmpxchg8b_emu(void); 


#ifdef CONFIG_RETPOLINE



#define DECL_INDIRECT_THUNK(reg) \

#undef GEN

#define GEN(reg) \

	extern asmlinkage void __x86_indirect_thunk_ ## reg (void);

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) \

	extern asmlinkage void __x86_indirect_alt_call_ ## reg (void);

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) DECL_INDIRECT_THUNK(reg)

#define GEN(reg) \

	extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);

#include <asm/GEN-for-each-reg.h>



#endif /* CONFIG_RETPOLINE */

arch/x86/lib/retpoline.S

+40 −1
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ 
#include <asm/unwind_hints.h>

#include <asm/frame.h>



	.section .text.__x86.indirect_thunk



.macro RETPOLINE reg

	ANNOTATE_INTRA_FUNCTION_CALL

	call    .Ldo_rop_\@
@@ -25,9 +27,9 @@ 
.endm



.macro THUNK reg

	.section .text.__x86.indirect_thunk



	.align 32



SYM_FUNC_START(__x86_indirect_thunk_\reg)



	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
@@ -38,6 +40,32 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg) 


.endm



/*

 * This generates .altinstr_replacement symbols for use by objtool. They,

 * however, must not actually live in .altinstr_replacement since that will be

 * discarded after init, but module alternatives will also reference these

 * symbols.

 *

 * Their names matches the "__x86_indirect_" prefix to mark them as retpolines.

 */

.macro ALT_THUNK reg



	.align 1



SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg)

	ANNOTATE_RETPOLINE_SAFE

1:	call	*%\reg

2:	.skip	5-(2b-1b), 0x90

SYM_FUNC_END(__x86_indirect_alt_call_\reg)



SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg)

	ANNOTATE_RETPOLINE_SAFE

1:	jmp	*%\reg

2:	.skip	5-(2b-1b), 0x90

SYM_FUNC_END(__x86_indirect_alt_jmp_\reg)



.endm



/*

 * Despite being an assembler file we can't just use .irp here

 * because __KSYM_DEPS__ only uses the C preprocessor and would
@@ -61,3 +89,14 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg) 
#define GEN(reg) EXPORT_THUNK(reg)

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) ALT_THUNK reg

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg)

#include <asm/GEN-for-each-reg.h>



#undef GEN

#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg)

#include <asm/GEN-for-each-reg.h>

tools/objtool/arch/x86/decode.c

+117 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <objtool/elf.h>

#include <objtool/arch.h>

#include <objtool/warn.h>

#include <arch/elf.h>



static unsigned char op_to_cfi_reg[][2] = {

	{CFI_AX, CFI_R8},
@@ -613,6 +614,122 @@ const char *arch_nop_insn(int len) 
	return nops[len-1];

}



/* asm/alternative.h ? */



#define ALTINSTR_FLAG_INV	(1 << 15)

#define ALT_NOT(feat)		((feat) | ALTINSTR_FLAG_INV)



struct alt_instr {

	s32 instr_offset;	/* original instruction */

	s32 repl_offset;	/* offset to replacement instruction */

	u16 cpuid;		/* cpuid bit set for replacement */

	u8  instrlen;		/* length of original instruction */

	u8  replacementlen;	/* length of new instruction */

} __packed;



static int elf_add_alternative(struct elf *elf,

			       struct instruction *orig, struct symbol *sym,

			       int cpuid, u8 orig_len, u8 repl_len)

{

	const int size = sizeof(struct alt_instr);

	struct alt_instr *alt;

	struct section *sec;

	Elf_Scn *s;



	sec = find_section_by_name(elf, ".altinstructions");

	if (!sec) {

		sec = elf_create_section(elf, ".altinstructions",

					 SHF_WRITE, size, 0);



		if (!sec) {

			WARN_ELF("elf_create_section");

			return -1;

		}

	}



	s = elf_getscn(elf->elf, sec->idx);

	if (!s) {

		WARN_ELF("elf_getscn");

		return -1;

	}



	sec->data = elf_newdata(s);

	if (!sec->data) {

		WARN_ELF("elf_newdata");

		return -1;

	}



	sec->data->d_size = size;

	sec->data->d_align = 1;



	alt = sec->data->d_buf = malloc(size);

	if (!sec->data->d_buf) {

		perror("malloc");

		return -1;

	}

	memset(sec->data->d_buf, 0, size);



	if (elf_add_reloc_to_insn(elf, sec, sec->sh.sh_size,

				  R_X86_64_PC32, orig->sec, orig->offset)) {

		WARN("elf_create_reloc: alt_instr::instr_offset");

		return -1;

	}



	if (elf_add_reloc(elf, sec, sec->sh.sh_size + 4,

			  R_X86_64_PC32, sym, 0)) {

		WARN("elf_create_reloc: alt_instr::repl_offset");

		return -1;

	}



	alt->cpuid = cpuid;

	alt->instrlen = orig_len;

	alt->replacementlen = repl_len;



	sec->sh.sh_size += size;

	sec->changed = true;



	return 0;

}



#define X86_FEATURE_RETPOLINE                ( 7*32+12)



int arch_rewrite_retpolines(struct objtool_file *file)

{

	struct instruction *insn;

	struct reloc *reloc;

	struct symbol *sym;

	char name[32] = "";



	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {



		if (!strcmp(insn->sec->name, ".text.__x86.indirect_thunk"))

			continue;



		reloc = insn->reloc;



		sprintf(name, "__x86_indirect_alt_%s_%s",

			insn->type == INSN_JUMP_DYNAMIC ? "jmp" : "call",

			reloc->sym->name + 21);



		sym = find_symbol_by_name(file->elf, name);

		if (!sym) {

			sym = elf_create_undef_symbol(file->elf, name);

			if (!sym) {

				WARN("elf_create_undef_symbol");

				return -1;

			}

		}



		if (elf_add_alternative(file->elf, insn, sym,

					ALT_NOT(X86_FEATURE_RETPOLINE), 5, 5)) {

			WARN("elf_add_alternative");

			return -1;

		}

	}



	return 0;

}



int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg)

{

	struct cfi_reg *cfa = &insn->cfi.cfa;