Commit f3a112c0 authored by Masami Hiramatsu's avatar Masami Hiramatsu Committed by Alexei Starovoitov
Browse files

x86,rethook,kprobes: Replace kretprobe with rethook on x86 



Replaces the kretprobe code with rethook on x86. With this patch,
kretprobe on x86 uses the rethook instead of kretprobe specific
trampoline code.

Signed-off-by: default avatarMasami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Tested-by: default avatarJiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/bpf/164826163692.2455864.13745421016848209527.stgit@devnote2
parent 73f9b911

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -224,6 +224,7 @@ config X86 
	select HAVE_KPROBES_ON_FTRACE

	select HAVE_FUNCTION_ERROR_INJECTION

	select HAVE_KRETPROBES

	select HAVE_RETHOOK

	select HAVE_KVM

	select HAVE_LIVEPATCH			if X86_64

	select HAVE_MIXED_BREAKPOINTS_REGS

arch/x86/include/asm/unwind.h

+11 −12
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ 


#include <linux/sched.h>

#include <linux/ftrace.h>

#include <linux/kprobes.h>

#include <linux/rethook.h>

#include <asm/ptrace.h>

#include <asm/stacktrace.h>
@@ -16,7 +16,7 @@ struct unwind_state { 
	unsigned long stack_mask;

	struct task_struct *task;

	int graph_idx;

#ifdef CONFIG_KRETPROBES

#if defined(CONFIG_RETHOOK)

	struct llist_node *kr_cur;

#endif

	bool error;
@@ -104,19 +104,18 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, 
#endif



static inline

unsigned long unwind_recover_kretprobe(struct unwind_state *state,

unsigned long unwind_recover_rethook(struct unwind_state *state,

				     unsigned long addr, unsigned long *addr_p)

{

#ifdef CONFIG_KRETPROBES

	return is_kretprobe_trampoline(addr) ?

		kretprobe_find_ret_addr(state->task, addr_p, &state->kr_cur) :

		addr;

#else

	return addr;

#ifdef CONFIG_RETHOOK

	if (is_rethook_trampoline(addr))

		return rethook_find_ret_addr(state->task, (unsigned long)addr_p,

					     &state->kr_cur);

#endif

	return addr;

}



/* Recover the return address modified by kretprobe and ftrace_graph. */

/* Recover the return address modified by rethook and ftrace_graph. */

static inline

unsigned long unwind_recover_ret_addr(struct unwind_state *state,

				     unsigned long addr, unsigned long *addr_p)
@@ -125,7 +124,7 @@ unsigned long unwind_recover_ret_addr(struct unwind_state *state, 


	ret = ftrace_graph_ret_addr(state->task, &state->graph_idx,

				    addr, addr_p);

	return unwind_recover_kretprobe(state, ret, addr_p);

	return unwind_recover_rethook(state, ret, addr_p);

}



/*

arch/x86/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -103,6 +103,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o 
obj-$(CONFIG_FTRACE_SYSCALLS)	+= ftrace.o

obj-$(CONFIG_X86_TSC)		+= trace_clock.o

obj-$(CONFIG_TRACING)		+= trace.o

obj-$(CONFIG_RETHOOK)		+= rethook.o

obj-$(CONFIG_CRASH_CORE)	+= crash_core_$(BITS).o

obj-$(CONFIG_KEXEC_CORE)	+= machine_kexec_$(BITS).o

obj-$(CONFIG_KEXEC_CORE)	+= relocate_kernel_$(BITS).o crash.o

arch/x86/kernel/kprobes/common.h

+1 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 


#include <asm/asm.h>

#include <asm/frame.h>

#include <asm/insn.h>



#ifdef CONFIG_X86_64

arch/x86/kernel/kprobes/core.c

+0 −107
Original line number Diff line number Diff line
@@ -811,18 +811,6 @@ set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 
		= (regs->flags & X86_EFLAGS_IF);

}



void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)

{

	unsigned long *sara = stack_addr(regs);



	ri->ret_addr = (kprobe_opcode_t *) *sara;

	ri->fp = sara;



	/* Replace the return addr with trampoline addr */

	*sara = (unsigned long) &__kretprobe_trampoline;

}

NOKPROBE_SYMBOL(arch_prepare_kretprobe);



static void kprobe_post_process(struct kprobe *cur, struct pt_regs *regs,

			       struct kprobe_ctlblk *kcb)

{
@@ -1023,101 +1011,6 @@ int kprobe_int3_handler(struct pt_regs *regs) 
}

NOKPROBE_SYMBOL(kprobe_int3_handler);



/*

 * When a retprobed function returns, this code saves registers and

 * calls trampoline_handler() runs, which calls the kretprobe's handler.

 */

asm(

	".text\n"

	".global __kretprobe_trampoline\n"

	".type __kretprobe_trampoline, @function\n"

	"__kretprobe_trampoline:\n"

#ifdef CONFIG_X86_64

	ANNOTATE_NOENDBR

	/* Push a fake return address to tell the unwinder it's a kretprobe. */

	"	pushq $__kretprobe_trampoline\n"

	UNWIND_HINT_FUNC

	/* Save the 'sp - 8', this will be fixed later. */

	"	pushq %rsp\n"

	"	pushfq\n"

	SAVE_REGS_STRING

	"	movq %rsp, %rdi\n"

	"	call trampoline_handler\n"

	RESTORE_REGS_STRING

	/* In trampoline_handler(), 'regs->flags' is copied to 'regs->sp'. */

	"	addq $8, %rsp\n"

	"	popfq\n"

#else

	/* Push a fake return address to tell the unwinder it's a kretprobe. */

	"	pushl $__kretprobe_trampoline\n"

	UNWIND_HINT_FUNC

	/* Save the 'sp - 4', this will be fixed later. */

	"	pushl %esp\n"

	"	pushfl\n"

	SAVE_REGS_STRING

	"	movl %esp, %eax\n"

	"	call trampoline_handler\n"

	RESTORE_REGS_STRING

	/* In trampoline_handler(), 'regs->flags' is copied to 'regs->sp'. */

	"	addl $4, %esp\n"

	"	popfl\n"

#endif

	ASM_RET

	".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"

);

NOKPROBE_SYMBOL(__kretprobe_trampoline);

/*

 * __kretprobe_trampoline() skips updating frame pointer. The frame pointer

 * saved in trampoline_handler() points to the real caller function's

 * frame pointer. Thus the __kretprobe_trampoline() doesn't have a

 * standard stack frame with CONFIG_FRAME_POINTER=y.

 * Let's mark it non-standard function. Anyway, FP unwinder can correctly

 * unwind without the hint.

 */

STACK_FRAME_NON_STANDARD_FP(__kretprobe_trampoline);



/* This is called from kretprobe_trampoline_handler(). */

void arch_kretprobe_fixup_return(struct pt_regs *regs,

				 kprobe_opcode_t *correct_ret_addr)

{

	unsigned long *frame_pointer = &regs->sp + 1;



	/* Replace fake return address with real one. */

	*frame_pointer = (unsigned long)correct_ret_addr;

}



/*

 * Called from __kretprobe_trampoline

 */

__used __visible void trampoline_handler(struct pt_regs *regs)

{

	unsigned long *frame_pointer;



	/* fixup registers */

	regs->cs = __KERNEL_CS;

#ifdef CONFIG_X86_32

	regs->gs = 0;

#endif

	regs->ip = (unsigned long)&__kretprobe_trampoline;

	regs->orig_ax = ~0UL;

	regs->sp += sizeof(long);

	frame_pointer = &regs->sp + 1;



	/*

	 * The return address at 'frame_pointer' is recovered by the

	 * arch_kretprobe_fixup_return() which called from the

	 * kretprobe_trampoline_handler().

	 */

	kretprobe_trampoline_handler(regs, frame_pointer);



	/*

	 * Copy FLAGS to 'pt_regs::sp' so that __kretprobe_trapmoline()

	 * can do RET right after POPF.

	 */

	regs->sp = regs->flags;

}

NOKPROBE_SYMBOL(trampoline_handler);



int kprobe_fault_handler(struct pt_regs *regs, int trapnr)

{

	struct kprobe *cur = kprobe_running();