Commit cb411545 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: Speed up symbol resolving in kprobe multi link' 

Jiri Olsa says:

====================

hi,
sending additional fix for symbol resolving in kprobe multi link
requested by Alexei and Andrii [1].

This speeds up bpftrace kprobe attachment, when using pure symbols
(3344 symbols) to attach:

Before:

  # perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* {  } i:ms:1 { exit(); }'
  ...
  6.5681 +- 0.0225 seconds time elapsed  ( +-  0.34% )

After:

  # perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* {  } i:ms:1 { exit(); }'
  ...
  0.5661 +- 0.0275 seconds time elapsed  ( +-  4.85% )

v6 changes:
  - rewrote patch 1 changelog and fixed the line length [Christoph]

v5 changes:
  - added acks [Masami]
  - workaround in selftest for RCU warning by filtering out several
    functions to attach

v4 changes:
  - fix compile issue [kernel test robot]
  - added acks [Andrii]

v3 changes:
  - renamed kallsyms_lookup_names to ftrace_lookup_symbols
    and moved it to ftrace.c [Masami]
  - added ack [Andrii]
  - couple small test fixes [Andrii]

v2 changes (first version [2]):
  - removed the 2 seconds check [Alexei]
  - moving/forcing symbols sorting out of kallsyms_lookup_names function [Alexei]
  - skipping one array allocation and copy_from_user [Andrii]
  - several small fixes [Masami,Andrii]
  - build fix [kernel test robot]

thanks,
jirka

[1] https://lore.kernel.org/bpf/CAEf4BzZtQaiUxQ-sm_hH2qKPRaqGHyOfEsW96DxtBHRaKLoL3Q@mail.gmail.com/
[2] https://lore.kernel.org/bpf/20220407125224.310255-1-jolsa@kernel.org/


====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 9376d389 5b6c7e5c

include/linux/ftrace.h

+6 −0
Original line number Diff line number Diff line
@@ -303,6 +303,8 @@ int unregister_ftrace_function(struct ftrace_ops *ops); 
extern void ftrace_stub(unsigned long a0, unsigned long a1,

			struct ftrace_ops *op, struct ftrace_regs *fregs);





int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs);

#else /* !CONFIG_FUNCTION_TRACER */

/*

 * (un)register_ftrace_function must be a macro since the ops parameter
@@ -313,6 +315,10 @@ extern void ftrace_stub(unsigned long a0, unsigned long a1, 
static inline void ftrace_kill(void) { }

static inline void ftrace_free_init_mem(void) { }

static inline void ftrace_free_mem(struct module *mod, void *start, void *end) { }

static inline int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs)

{

	return -EOPNOTSUPP;

}

#endif /* CONFIG_FUNCTION_TRACER */



struct ftrace_func_entry {

include/linux/kallsyms.h

+6 −1
Original line number Diff line number Diff line
@@ -65,11 +65,11 @@ static inline void *dereference_symbol_descriptor(void *ptr) 
	return ptr;

}



#ifdef CONFIG_KALLSYMS

int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *,

				      unsigned long),

			    void *data);



#ifdef CONFIG_KALLSYMS

/* Lookup the address for a symbol. Returns 0 if not found. */

unsigned long kallsyms_lookup_name(const char *name);
@@ -163,6 +163,11 @@ static inline bool kallsyms_show_value(const struct cred *cred) 
	return false;

}



static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *,

					  unsigned long), void *data)

{

	return -EOPNOTSUPP;

}

#endif /*CONFIG_KALLSYMS*/



static inline void print_ip_sym(const char *loglvl, unsigned long ip)

kernel/kallsyms.c

+1 −2
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <linux/compiler.h>

#include <linux/module.h>

#include <linux/kernel.h>

#include <linux/bsearch.h>



/*

 * These will be re-linked against their real values
@@ -228,7 +229,6 @@ unsigned long kallsyms_lookup_name(const char *name) 
	return module_kallsyms_lookup_name(name);

}



#ifdef CONFIG_LIVEPATCH

/*

 * Iterate over all symbols in vmlinux.  For symbols from modules use

 * module_kallsyms_on_each_symbol instead.
@@ -251,7 +251,6 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, 
	}

	return 0;

}

#endif /* CONFIG_LIVEPATCH */



static unsigned long get_symbol_pos(unsigned long addr,

				    unsigned long *symbolsize,

kernel/trace/bpf_trace.c

+66 −46
Original line number Diff line number Diff line
@@ -2229,6 +2229,59 @@ struct bpf_kprobe_multi_run_ctx { 
	unsigned long entry_ip;

};



struct user_syms {

	const char **syms;

	char *buf;

};



static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt)

{

	unsigned long __user usymbol;

	const char **syms = NULL;

	char *buf = NULL, *p;

	int err = -ENOMEM;

	unsigned int i;



	syms = kvmalloc(cnt * sizeof(*syms), GFP_KERNEL);

	if (!syms)

		goto error;



	buf = kvmalloc(cnt * KSYM_NAME_LEN, GFP_KERNEL);

	if (!buf)

		goto error;



	for (p = buf, i = 0; i < cnt; i++) {

		if (__get_user(usymbol, usyms + i)) {

			err = -EFAULT;

			goto error;

		}

		err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN);

		if (err == KSYM_NAME_LEN)

			err = -E2BIG;

		if (err < 0)

			goto error;

		syms[i] = p;

		p += err + 1;

	}



	us->syms = syms;

	us->buf = buf;

	return 0;



error:

	if (err) {

		kvfree(syms);

		kvfree(buf);

	}

	return err;

}



static void free_user_syms(struct user_syms *us)

{

	kvfree(us->syms);

	kvfree(us->buf);

}



static void bpf_kprobe_multi_link_release(struct bpf_link *link)

{

	struct bpf_kprobe_multi_link *kmulti_link;
@@ -2349,53 +2402,12 @@ kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip, 
	kprobe_multi_link_prog_run(link, entry_ip, regs);

}



static int

kprobe_multi_resolve_syms(const void __user *usyms, u32 cnt,

			  unsigned long *addrs)

static int symbols_cmp(const void *a, const void *b)

{

	unsigned long addr, size;

	const char __user **syms;

	int err = -ENOMEM;

	unsigned int i;

	char *func;



	size = cnt * sizeof(*syms);

	syms = kvzalloc(size, GFP_KERNEL);

	if (!syms)

		return -ENOMEM;



	func = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);

	if (!func)

		goto error;



	if (copy_from_user(syms, usyms, size)) {

		err = -EFAULT;

		goto error;

	}



	for (i = 0; i < cnt; i++) {

		err = strncpy_from_user(func, syms[i], KSYM_NAME_LEN);

		if (err == KSYM_NAME_LEN)

			err = -E2BIG;

		if (err < 0)

			goto error;

		err = -EINVAL;

		addr = kallsyms_lookup_name(func);

		if (!addr)

			goto error;

		if (!kallsyms_lookup_size_offset(addr, &size, NULL))

			goto error;

		addr = ftrace_location_range(addr, addr + size - 1);

		if (!addr)

			goto error;

		addrs[i] = addr;

	}

	const char **str_a = (const char **) a;

	const char **str_b = (const char **) b;



	err = 0;

error:

	kvfree(syms);

	kfree(func);

	return err;

	return strcmp(*str_a, *str_b);

}



int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
@@ -2441,7 +2453,15 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr 
			goto error;

		}

	} else {

		err = kprobe_multi_resolve_syms(usyms, cnt, addrs);

		struct user_syms us;



		err = copy_user_syms(&us, usyms, cnt);

		if (err)

			goto error;



		sort(us.syms, cnt, sizeof(*us.syms), symbols_cmp, NULL);

		err = ftrace_lookup_symbols(us.syms, cnt, addrs);

		free_user_syms(&us);

		if (err)

			goto error;

	}

kernel/trace/fprobe.c

+12 −20
Original line number Diff line number Diff line
@@ -85,39 +85,31 @@ static void fprobe_exit_handler(struct rethook_node *rh, void *data, 
}

NOKPROBE_SYMBOL(fprobe_exit_handler);



static int symbols_cmp(const void *a, const void *b)

{

	const char **str_a = (const char **) a;

	const char **str_b = (const char **) b;



	return strcmp(*str_a, *str_b);

}



/* Convert ftrace location address from symbols */

static unsigned long *get_ftrace_locations(const char **syms, int num)

{

	unsigned long addr, size;

	unsigned long *addrs;

	int i;



	/* Convert symbols to symbol address */

	addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);

	if (!addrs)

		return ERR_PTR(-ENOMEM);



	for (i = 0; i < num; i++) {

		addr = kallsyms_lookup_name(syms[i]);

		if (!addr)	/* Maybe wrong symbol */

			goto error;



		/* Convert symbol address to ftrace location. */

		if (!kallsyms_lookup_size_offset(addr, &size, NULL) || !size)

			goto error;



		addr = ftrace_location_range(addr, addr + size - 1);

		if (!addr) /* No dynamic ftrace there. */

			goto error;



		addrs[i] = addr;

	}

	/* ftrace_lookup_symbols expects sorted symbols */

	sort(syms, num, sizeof(*syms), symbols_cmp, NULL);



	if (!ftrace_lookup_symbols(syms, num, addrs))

		return addrs;



error:

	kfree(addrs);



	return ERR_PTR(-ENOENT);

}