Commit b6c881b2 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Borislav Petkov (AMD)
Browse files

x86/alternative: Complicate optimize_nops() some more 



Because:

  SMP alternatives: ffffffff810026dc: [2:44) optimized NOPs: eb 2a eb 28 cc cc
    cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc
    cc cc cc cc cc cc cc cc cc cc cc cc cc

is quite daft, make things more complicated and have the NOP runlength
detection eat the preceding JMP if they both end at the same target.

  SMP alternatives: ffffffff810026dc: [0:44) optimized NOPs: eb 2a cc cc cc cc
    cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc
    cc cc cc cc cc cc cc cc cc cc cc cc cc

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20230208171431.433132442@infradead.org
parent 6c480f22

arch/x86/kernel/alternative.c

+50 −10
Original line number Diff line number Diff line
@@ -114,6 +114,8 @@ const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 
};



/*

 * Fill the buffer with a single effective instruction of size @len.

 *

 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info)

 * for every single-byte NOP, try to generate the maximally available NOP of

 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for
@@ -152,6 +154,9 @@ extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 
extern s32 __smp_locks[], __smp_locks_end[];

void text_poke_early(void *addr, const void *opcode, size_t len);



/*

 * Matches NOP and NOPL, not any of the other possible NOPs.

 */

static bool insn_is_nop(struct insn *insn)

{

	if (insn->opcode.bytes[0] == 0x90)
@@ -165,6 +170,10 @@ static bool insn_is_nop(struct insn *insn) 
	return false;

}



/*

 * Find the offset of the first non-NOP instruction starting at @offset

 * but no further than @len.

 */

static int skip_nops(u8 *instr, int offset, int len)

{

	struct insn insn;
@@ -180,12 +189,47 @@ static int skip_nops(u8 *instr, int offset, int len) 
	return offset;

}



/*

 * Optimize a sequence of NOPs, possibly preceded by an unconditional jump

 * to the end of the NOP sequence into a single NOP.

 */

static bool __optimize_nops(u8 *instr, size_t len, struct insn *insn,

			    int *next, int *prev, int *target)

{

	int i = *next - insn->length;



	switch (insn->opcode.bytes[0]) {

	case JMP8_INSN_OPCODE:

	case JMP32_INSN_OPCODE:

		*prev = i;

		*target = *next + insn->immediate.value;

		return false;

	}



	if (insn_is_nop(insn)) {

		int nop = i;



		*next = skip_nops(instr, *next, len);

		if (*target && *next == *target)

			nop = *prev;



		add_nop(instr + nop, *next - nop);

		DUMP_BYTES(ALT, instr, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, *next);

		return true;

	}



	*target = 0;

	return false;

}



/*

 * "noinline" to cause control flow change and thus invalidate I$ and

 * cause refetch after modification.

 */

static void __init_or_module noinline optimize_nops(u8 *instr, size_t len)

{

	int prev, target = 0;



	for (int next, i = 0; i < len; i = next) {

		struct insn insn;
@@ -194,11 +238,7 @@ static void __init_or_module noinline optimize_nops(u8 *instr, size_t len) 


		next = i + insn.length;



		if (insn_is_nop(&insn)) {

			next = skip_nops(instr, next, len);

			add_nop(instr + i, next - i);

			DUMP_BYTES(ALT, instr, len, "%px: [%d:%d) optimized NOPs: ", instr, i, next);

		}

		__optimize_nops(instr, len, &insn, &next, &prev, &target);

	}

}
@@ -275,6 +315,8 @@ bool need_reloc(unsigned long offset, u8 *src, size_t src_len) 
static void __init_or_module noinline

apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len)

{

	int prev, target = 0;



	for (int next, i = 0; i < len; i = next) {

		struct insn insn;
@@ -283,6 +325,9 @@ apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len) 


		next = i + insn.length;



		if (__optimize_nops(buf, len, &insn, &next, &prev, &target))

			continue;



		switch (insn.opcode.bytes[0]) {

		case 0x0f:

			if (insn.opcode.bytes[1] < 0x80 ||
@@ -324,11 +369,6 @@ apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len) 
					    src - dest);

			}

		}



		if (insn_is_nop(&insn)) {

			next = skip_nops(buf, next, len);

			add_nop(buf + i, next - i);

		}

	}

}