Commit 75caf33e authored by Masami Hiramatsu's avatar Masami Hiramatsu Committed by Alexei Starovoitov
Browse files

rethook: x86: Add rethook x86 implementation 



Add rethook for x86 implementation. Most of the code has been copied from
kretprobes on x86.

Signed-off-by: default avatarMasami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Tested-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/164735286243.1084943.7477055110527046644.stgit@devnote2
parent 54ecbe6f

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -221,6 +221,7 @@ config X86 
	select HAVE_KPROBES_ON_FTRACE

	select HAVE_FUNCTION_ERROR_INJECTION

	select HAVE_KRETPROBES

	select HAVE_RETHOOK

	select HAVE_KVM

	select HAVE_LIVEPATCH			if X86_64

	select HAVE_MIXED_BREAKPOINTS_REGS

arch/x86/include/asm/unwind.h

+7 −1
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#include <linux/sched.h>

#include <linux/ftrace.h>

#include <linux/kprobes.h>

#include <linux/rethook.h>

#include <asm/ptrace.h>

#include <asm/stacktrace.h>
@@ -16,7 +17,7 @@ struct unwind_state { 
	unsigned long stack_mask;

	struct task_struct *task;

	int graph_idx;

#ifdef CONFIG_KRETPROBES

#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK)

	struct llist_node *kr_cur;

#endif

	bool error;
@@ -107,6 +108,11 @@ static inline 
unsigned long unwind_recover_kretprobe(struct unwind_state *state,

				       unsigned long addr, unsigned long *addr_p)

{

#ifdef CONFIG_RETHOOK

	if (is_rethook_trampoline(addr))

		return rethook_find_ret_addr(state->task, (unsigned long)addr_p,

					     &state->kr_cur);

#endif

#ifdef CONFIG_KRETPROBES

	return is_kretprobe_trampoline(addr) ?

		kretprobe_find_ret_addr(state->task, addr_p, &state->kr_cur) :

arch/x86/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -106,6 +106,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o 
obj-$(CONFIG_FTRACE_SYSCALLS)	+= ftrace.o

obj-$(CONFIG_X86_TSC)		+= trace_clock.o

obj-$(CONFIG_TRACING)		+= trace.o

obj-$(CONFIG_RETHOOK)		+= rethook.o

obj-$(CONFIG_CRASH_CORE)	+= crash_core_$(BITS).o

obj-$(CONFIG_KEXEC_CORE)	+= machine_kexec_$(BITS).o

obj-$(CONFIG_KEXEC_CORE)	+= relocate_kernel_$(BITS).o crash.o

arch/x86/kernel/kprobes/common.h

+1 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 


#include <asm/asm.h>

#include <asm/frame.h>

#include <asm/insn.h>



#ifdef CONFIG_X86_64

arch/x86/kernel/rethook.c

0 → 100644
+119 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

/*

 * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c.

 */

#include <linux/bug.h>

#include <linux/rethook.h>

#include <linux/kprobes.h>



#include "kprobes/common.h"



__visible void arch_rethook_trampoline_callback(struct pt_regs *regs);



/*

 * When a target function returns, this code saves registers and calls

 * arch_rethook_trampoline_callback(), which calls the rethook handler.

 */

asm(

	".text\n"

	".global arch_rethook_trampoline\n"

	".type arch_rethook_trampoline, @function\n"

	"arch_rethook_trampoline:\n"

#ifdef CONFIG_X86_64

	/* Push a fake return address to tell the unwinder it's a kretprobe. */

	"	pushq $arch_rethook_trampoline\n"

	UNWIND_HINT_FUNC

	/* Save the 'sp - 8', this will be fixed later. */

	"	pushq %rsp\n"

	"	pushfq\n"

	SAVE_REGS_STRING

	"	movq %rsp, %rdi\n"

	"	call arch_rethook_trampoline_callback\n"

	RESTORE_REGS_STRING

	/* In the callback function, 'regs->flags' is copied to 'regs->sp'. */

	"	addq $8, %rsp\n"

	"	popfq\n"

#else

	/* Push a fake return address to tell the unwinder it's a kretprobe. */

	"	pushl $arch_rethook_trampoline\n"

	UNWIND_HINT_FUNC

	/* Save the 'sp - 4', this will be fixed later. */

	"	pushl %esp\n"

	"	pushfl\n"

	SAVE_REGS_STRING

	"	movl %esp, %eax\n"

	"	call arch_rethook_trampoline_callback\n"

	RESTORE_REGS_STRING

	/* In the callback function, 'regs->flags' is copied to 'regs->sp'. */

	"	addl $4, %esp\n"

	"	popfl\n"

#endif

	"	ret\n"

	".size arch_rethook_trampoline, .-arch_rethook_trampoline\n"

);

NOKPROBE_SYMBOL(arch_rethook_trampoline);



/*

 * Called from arch_rethook_trampoline

 */

__used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs)

{

	unsigned long *frame_pointer;



	/* fixup registers */

	regs->cs = __KERNEL_CS;

#ifdef CONFIG_X86_32

	regs->gs = 0;

#endif

	regs->ip = (unsigned long)&arch_rethook_trampoline;

	regs->orig_ax = ~0UL;

	regs->sp += sizeof(long);

	frame_pointer = &regs->sp + 1;



	/*

	 * The return address at 'frame_pointer' is recovered by the

	 * arch_rethook_fixup_return() which called from this

	 * rethook_trampoline_handler().

	 */

	rethook_trampoline_handler(regs, (unsigned long)frame_pointer);



	/*

	 * Copy FLAGS to 'pt_regs::sp' so that arch_rethook_trapmoline()

	 * can do RET right after POPF.

	 */

	regs->sp = regs->flags;

}

NOKPROBE_SYMBOL(arch_rethook_trampoline_callback);



/*

 * arch_rethook_trampoline() skips updating frame pointer. The frame pointer

 * saved in arch_rethook_trampoline_callback() points to the real caller

 * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have

 * a standard stack frame with CONFIG_FRAME_POINTER=y.

 * Let's mark it non-standard function. Anyway, FP unwinder can correctly

 * unwind without the hint.

 */

STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline);



/* This is called from rethook_trampoline_handler(). */

void arch_rethook_fixup_return(struct pt_regs *regs,

			       unsigned long correct_ret_addr)

{

	unsigned long *frame_pointer = &regs->sp + 1;



	/* Replace fake return address with real one. */

	*frame_pointer = correct_ret_addr;

}

NOKPROBE_SYMBOL(arch_rethook_fixup_return);



void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount)

{

	unsigned long *stack = (unsigned long *)regs->sp;



	rh->ret_addr = stack[0];

	rh->frame = regs->sp;



	/* Replace the return addr with trampoline addr */

	stack[0] = (unsigned long) arch_rethook_trampoline;

}

NOKPROBE_SYMBOL(arch_rethook_prepare);