Commit e6d6c071 authored by Josh Poimboeuf's avatar Josh Poimboeuf Committed by Ingo Molnar
Browse files

x86/static_call: Add out-of-line static call implementation 



Add the x86 out-of-line static call implementation.  For each key, a
permanent trampoline is created which is the destination for all static
calls for the given key.  The trampoline has a direct jump which gets
patched by static_call_update() when the destination function changes.

[peterz: fixed trampoline, rewrote patching code]

Signed-off-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20200818135804.804315175@infradead.org
parent 6333e8f7

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -215,6 +215,7 @@ config X86 
	select HAVE_FUNCTION_ARG_ACCESS_API

	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR

	select HAVE_STACK_VALIDATION		if X86_64

	select HAVE_STATIC_CALL

	select HAVE_RSEQ

	select HAVE_SYSCALL_TRACEPOINTS

	select HAVE_UNSTABLE_SCHED_CLOCK

arch/x86/include/asm/static_call.h

0 → 100644
+23 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _ASM_STATIC_CALL_H

#define _ASM_STATIC_CALL_H



#include <asm/text-patching.h>



/*

 * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which

 * does a direct jump to the function.  The direct jump gets patched by

 * static_call_update().

 */

#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\

	asm(".pushsection .text, \"ax\"				\n"	\

	    ".align 4						\n"	\

	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\

	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\

	    "	.byte 0xe9 # jmp.d32				\n"	\

	    "	.long " #func " - (. + 4)			\n"	\

	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\

	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \

	    ".popsection					\n")



#endif /* _ASM_STATIC_CALL_H */

arch/x86/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -68,6 +68,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rtc.o 
obj-y			+= pci-iommu_table.o

obj-y			+= resource.o

obj-y			+= irqflags.o

obj-y			+= static_call.o



obj-y				+= process.o

obj-y				+= fpu/

arch/x86/kernel/static_call.c

0 → 100644
+31 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

#include <linux/static_call.h>

#include <linux/memory.h>

#include <linux/bug.h>

#include <asm/text-patching.h>



static void __static_call_transform(void *insn, u8 opcode, void *func)

{

	const void *code = text_gen_insn(opcode, insn, func);



	if (WARN_ONCE(*(u8 *)insn != opcode,

		      "unexpected static call insn opcode 0x%x at %pS\n",

		      opcode, insn))

		return;



	if (memcmp(insn, code, CALL_INSN_SIZE) == 0)

		return;



	text_poke_bp(insn, code, CALL_INSN_SIZE, NULL);

}



void arch_static_call_transform(void *site, void *tramp, void *func)

{

	mutex_lock(&text_mutex);



	if (tramp)

		__static_call_transform(tramp, JMP32_INSN_OPCODE, func);



	mutex_unlock(&text_mutex);

}

EXPORT_SYMBOL_GPL(arch_static_call_transform);