Commit 1dd7dcb6 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'trace-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace 

Pull tracing updates from Steven Rostedt:
 "There was a lot of clean ups and minor fixes.  One of those clean ups
  was to the trace_seq code.  It also removed the return values to the
  trace_seq_*() functions and use trace_seq_has_overflowed() to see if
  the buffer filled up or not.  This is similar to work being done to
  the seq_file code as well in another tree.

  Some of the other goodies include:

   - Added some "!" (NOT) logic to the tracing filter.

   - Fixed the frame pointer logic to the x86_64 mcount trampolines

   - Added the logic for dynamic trampolines on !CONFIG_PREEMPT systems.
     That is, the ftrace trampoline can be dynamically allocated and be
     called directly by functions that only have a single hook to them"

* tag 'trace-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace: (55 commits)
  tracing: Truncated output is better than nothing
  tracing: Add additional marks to signal very large time deltas
  Documentation: describe trace_buf_size parameter more accurately
  tracing: Allow NOT to filter AND and OR clauses
  tracing: Add NOT to filtering logic
  ftrace/fgraph/x86: Have prepare_ftrace_return() take ip as first parameter
  ftrace/x86: Get rid of ftrace_caller_setup
  ftrace/x86: Have save_mcount_regs macro also save stack frames if needed
  ftrace/x86: Add macro MCOUNT_REG_SIZE for amount of stack used to save mcount regs
  ftrace/x86: Simplify save_mcount_regs on getting RIP
  ftrace/x86: Have save_mcount_regs store RIP in %rdi for first parameter
  ftrace/x86: Rename MCOUNT_SAVE_FRAME and add more detailed comments
  ftrace/x86: Move MCOUNT_SAVE_FRAME out of header file
  ftrace/x86: Have static tracing also use ftrace_caller_setup
  ftrace/x86: Have static function tracing always test for function graph
  kprobes: Add IPMODIFY flag to kprobe_ftrace_ops
  ftrace, kprobes: Support IPMODIFY flag to find IP modify conflict
  kprobes/ftrace: Recover original IP if pre_handler doesn't change it
  tracing/trivial: Fix typos and make an int into a bool
  tracing: Deletion of an unnecessary check before iput()
  ...
parents b6da0076 3558a5ac

Documentation/kernel-parameters.txt

+1 −1
Original line number Diff line number Diff line
@@ -3520,7 +3520,7 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
			are saved.



	trace_buf_size=nn[KMG]

			[FTRACE] will set tracing buffer size.

			[FTRACE] will set tracing buffer size on each cpu.



	trace_event=[event-list]

			[FTRACE] Set and start specified trace events in order

Documentation/trace/ftrace.txt

+12 −3
Original line number Diff line number Diff line
@@ -234,6 +234,11 @@ of ftrace. Here is a list of some of the key files: 
	will be displayed on the same line as the function that

	is returning registers.



	If the callback registered to be traced by a function with

	the "ip modify" attribute (thus the regs->ip can be changed),

	an 'I' will be displayed on the same line as the function that

	can be overridden.



  function_profile_enabled:



	When set it will enable all functions with either the function
@@ -680,9 +685,11 @@ The above is mostly meaningful for kernel developers. 
	 needs to be fixed to be only relative to the same CPU.

	 The marks are determined by the difference between this

	 current trace and the next trace.

	  '!' - greater than preempt_mark_thresh (default 100)

	  '+' - greater than 1 microsecond

	  ' ' - less than or equal to 1 microsecond.

	  '$' - greater than 1 second

	  '#' - greater than 1000 microsecond

	  '!' - greater than 100 microsecond

	  '+' - greater than 10 microsecond

	  ' ' - less than or equal to 10 microsecond.



  The rest is the same as the 'trace' file.
@@ -1951,6 +1958,8 @@ want, depending on your needs. 


  + means that the function exceeded 10 usecs.

  ! means that the function exceeded 100 usecs.

  # means that the function exceeded 1000 usecs.

  $ means that the function exceeded 1 sec.





- The task/pid field displays the thread cmdline and pid which

arch/powerpc/kernel/ftrace.c

+1 −1
Original line number Diff line number Diff line
@@ -449,7 +449,7 @@ void ftrace_replace_code(int enable) 
		rec = ftrace_rec_iter_record(iter);

		ret = __ftrace_replace_code(rec, enable);

		if (ret) {

			ftrace_bug(ret, rec->ip);

			ftrace_bug(ret, rec);

			return;

		}

	}

arch/x86/include/asm/ftrace.h

+0 −33
Original line number Diff line number Diff line 
#ifndef _ASM_X86_FTRACE_H

#define _ASM_X86_FTRACE_H



#ifdef __ASSEMBLY__



	/* skip is set if the stack was already partially adjusted */

	.macro MCOUNT_SAVE_FRAME skip=0

	 /*

	  * We add enough stack to save all regs.

	  */

	subq $(SS+8-\skip), %rsp

	movq %rax, RAX(%rsp)

	movq %rcx, RCX(%rsp)

	movq %rdx, RDX(%rsp)

	movq %rsi, RSI(%rsp)

	movq %rdi, RDI(%rsp)

	movq %r8, R8(%rsp)

	movq %r9, R9(%rsp)

	 /* Move RIP to its proper location */

	movq SS+8(%rsp), %rdx

	movq %rdx, RIP(%rsp)

	.endm



	.macro MCOUNT_RESTORE_FRAME skip=0

	movq R9(%rsp), %r9

	movq R8(%rsp), %r8

	movq RDI(%rsp), %rdi

	movq RSI(%rsp), %rsi

	movq RDX(%rsp), %rdx

	movq RCX(%rsp), %rcx

	movq RAX(%rsp), %rax

	addq $(SS+8-\skip), %rsp

	.endm



#endif



#ifdef CONFIG_FUNCTION_TRACER

#ifdef CC_USING_FENTRY

# define MCOUNT_ADDR		((long)(__fentry__))

arch/x86/kernel/ftrace.c

+279 −9
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include <linux/ftrace.h>

#include <linux/percpu.h>

#include <linux/sched.h>

#include <linux/slab.h>

#include <linux/init.h>

#include <linux/list.h>

#include <linux/module.h>
@@ -47,7 +48,7 @@ int ftrace_arch_code_modify_post_process(void) 
union ftrace_code_union {

	char code[MCOUNT_INSN_SIZE];

	struct {

		char e8;

		unsigned char e8;

		int offset;

	} __attribute__((packed));

};
@@ -582,7 +583,7 @@ void ftrace_replace_code(int enable) 


 remove_breakpoints:

	pr_warn("Failed on %s (%d):\n", report, count);

	ftrace_bug(ret, rec ? rec->ip : 0);

	ftrace_bug(ret, rec);

	for_ftrace_rec_iter(iter) {

		rec = ftrace_rec_iter_record(iter);

		/*
@@ -644,13 +645,8 @@ int __init ftrace_dyn_arch_init(void) 
{

	return 0;

}

#endif



#ifdef CONFIG_FUNCTION_GRAPH_TRACER



#ifdef CONFIG_DYNAMIC_FTRACE

extern void ftrace_graph_call(void);



#if defined(CONFIG_X86_64) || defined(CONFIG_FUNCTION_GRAPH_TRACER)

static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)

{

	static union ftrace_code_union calc;
@@ -664,6 +660,280 @@ static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr) 
	 */

	return calc.code;

}

#endif



/* Currently only x86_64 supports dynamic trampolines */

#ifdef CONFIG_X86_64



#ifdef CONFIG_MODULES

#include <linux/moduleloader.h>

/* Module allocation simplifies allocating memory for code */

static inline void *alloc_tramp(unsigned long size)

{

	return module_alloc(size);

}

static inline void tramp_free(void *tramp)

{

	module_free(NULL, tramp);

}

#else

/* Trampolines can only be created if modules are supported */

static inline void *alloc_tramp(unsigned long size)

{

	return NULL;

}

static inline void tramp_free(void *tramp) { }

#endif



/* Defined as markers to the end of the ftrace default trampolines */

extern void ftrace_caller_end(void);

extern void ftrace_regs_caller_end(void);

extern void ftrace_return(void);

extern void ftrace_caller_op_ptr(void);

extern void ftrace_regs_caller_op_ptr(void);



/* movq function_trace_op(%rip), %rdx */

/* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */

#define OP_REF_SIZE	7



/*

 * The ftrace_ops is passed to the function callback. Since the

 * trampoline only services a single ftrace_ops, we can pass in

 * that ops directly.

 *

 * The ftrace_op_code_union is used to create a pointer to the

 * ftrace_ops that will be passed to the callback function.

 */

union ftrace_op_code_union {

	char code[OP_REF_SIZE];

	struct {

		char op[3];

		int offset;

	} __attribute__((packed));

};



static unsigned long

create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)

{

	unsigned const char *jmp;

	unsigned long start_offset;

	unsigned long end_offset;

	unsigned long op_offset;

	unsigned long offset;

	unsigned long size;

	unsigned long ip;

	unsigned long *ptr;

	void *trampoline;

	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */

	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };

	union ftrace_op_code_union op_ptr;

	int ret;



	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {

		start_offset = (unsigned long)ftrace_regs_caller;

		end_offset = (unsigned long)ftrace_regs_caller_end;

		op_offset = (unsigned long)ftrace_regs_caller_op_ptr;

	} else {

		start_offset = (unsigned long)ftrace_caller;

		end_offset = (unsigned long)ftrace_caller_end;

		op_offset = (unsigned long)ftrace_caller_op_ptr;

	}



	size = end_offset - start_offset;



	/*

	 * Allocate enough size to store the ftrace_caller code,

	 * the jmp to ftrace_return, as well as the address of

	 * the ftrace_ops this trampoline is used for.

	 */

	trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));

	if (!trampoline)

		return 0;



	*tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);



	/* Copy ftrace_caller onto the trampoline memory */

	ret = probe_kernel_read(trampoline, (void *)start_offset, size);

	if (WARN_ON(ret < 0)) {

		tramp_free(trampoline);

		return 0;

	}



	ip = (unsigned long)trampoline + size;



	/* The trampoline ends with a jmp to ftrace_return */

	jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_return);

	memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);



	/*

	 * The address of the ftrace_ops that is used for this trampoline

	 * is stored at the end of the trampoline. This will be used to

	 * load the third parameter for the callback. Basically, that

	 * location at the end of the trampoline takes the place of

	 * the global function_trace_op variable.

	 */



	ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);

	*ptr = (unsigned long)ops;



	op_offset -= start_offset;

	memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);



	/* Are we pointing to the reference? */

	if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) {

		tramp_free(trampoline);

		return 0;

	}



	/* Load the contents of ptr into the callback parameter */

	offset = (unsigned long)ptr;

	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;



	op_ptr.offset = offset;



	/* put in the new offset to the ftrace_ops */

	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);



	/* ALLOC_TRAMP flags lets us know we created it */

	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;



	return (unsigned long)trampoline;

}



static unsigned long calc_trampoline_call_offset(bool save_regs)

{

	unsigned long start_offset;

	unsigned long call_offset;



	if (save_regs) {

		start_offset = (unsigned long)ftrace_regs_caller;

		call_offset = (unsigned long)ftrace_regs_call;

	} else {

		start_offset = (unsigned long)ftrace_caller;

		call_offset = (unsigned long)ftrace_call;

	}



	return call_offset - start_offset;

}



void arch_ftrace_update_trampoline(struct ftrace_ops *ops)

{

	ftrace_func_t func;

	unsigned char *new;

	unsigned long offset;

	unsigned long ip;

	unsigned int size;

	int ret;



	if (ops->trampoline) {

		/*

		 * The ftrace_ops caller may set up its own trampoline.

		 * In such a case, this code must not modify it.

		 */

		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))

			return;

	} else {

		ops->trampoline = create_trampoline(ops, &size);

		if (!ops->trampoline)

			return;

		ops->trampoline_size = size;

	}



	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);

	ip = ops->trampoline + offset;



	func = ftrace_ops_get_func(ops);



	/* Do a safe modify in case the trampoline is executing */

	new = ftrace_call_replace(ip, (unsigned long)func);

	ret = update_ftrace_func(ip, new);



	/* The update should never fail */

	WARN_ON(ret);

}



/* Return the address of the function the trampoline calls */

static void *addr_from_call(void *ptr)

{

	union ftrace_code_union calc;

	int ret;



	ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);

	if (WARN_ON_ONCE(ret < 0))

		return NULL;



	/* Make sure this is a call */

	if (WARN_ON_ONCE(calc.e8 != 0xe8)) {

		pr_warn("Expected e8, got %x\n", calc.e8);

		return NULL;

	}



	return ptr + MCOUNT_INSN_SIZE + calc.offset;

}



void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,

			   unsigned long frame_pointer);



/*

 * If the ops->trampoline was not allocated, then it probably

 * has a static trampoline func, or is the ftrace caller itself.

 */

static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)

{

	unsigned long offset;

	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;

	void *ptr;



	if (ops && ops->trampoline) {

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

		/*

		 * We only know about function graph tracer setting as static

		 * trampoline.

		 */

		if (ops->trampoline == FTRACE_GRAPH_ADDR)

			return (void *)prepare_ftrace_return;

#endif

		return NULL;

	}



	offset = calc_trampoline_call_offset(save_regs);



	if (save_regs)

		ptr = (void *)FTRACE_REGS_ADDR + offset;

	else

		ptr = (void *)FTRACE_ADDR + offset;



	return addr_from_call(ptr);

}



void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)

{

	unsigned long offset;



	/* If we didn't allocate this trampoline, consider it static */

	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))

		return static_tramp_func(ops, rec);



	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);

	return addr_from_call((void *)ops->trampoline + offset);

}



void arch_ftrace_trampoline_free(struct ftrace_ops *ops)

{

	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))

		return;



	tramp_free((void *)ops->trampoline);

	ops->trampoline = 0;

}



#endif /* CONFIG_X86_64 */

#endif /* CONFIG_DYNAMIC_FTRACE */



#ifdef CONFIG_FUNCTION_GRAPH_TRACER



#ifdef CONFIG_DYNAMIC_FTRACE

extern void ftrace_graph_call(void);



static int ftrace_mod_jmp(unsigned long ip, void *func)

{
@@ -694,7 +964,7 @@ int ftrace_disable_ftrace_graph_caller(void) 
 * Hook the return address and push it in the stack of return addrs

 * in current thread info.

 */

void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,

void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,

			   unsigned long frame_pointer)

{

	unsigned long old;