Commit 0236fec5 authored by Jiri Olsa's avatar Jiri Olsa Committed by Alexei Starovoitov
Browse files

bpf: Resolve symbols with ftrace_lookup_symbols for kprobe multi link 



Using kallsyms_lookup_names function to speed up symbols lookup in
kprobe multi link attachment and replacing with it the current
kprobe_multi_resolve_syms function.

This speeds up bpftrace kprobe attachment:

  # perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* {  } i:ms:1 { exit(); }'
  ...
  6.5681 +- 0.0225 seconds time elapsed  ( +-  0.34% )

After:

  # perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* {  } i:ms:1 { exit(); }'
  ...
  0.5661 +- 0.0275 seconds time elapsed  ( +-  4.85% )

Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20220510122616.2652285-5-jolsa@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 8be92533

kernel/trace/bpf_trace.c

+66 −46
Original line number Diff line number Diff line
@@ -2229,6 +2229,59 @@ struct bpf_kprobe_multi_run_ctx { 
	unsigned long entry_ip;

};



struct user_syms {

	const char **syms;

	char *buf;

};



static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt)

{

	unsigned long __user usymbol;

	const char **syms = NULL;

	char *buf = NULL, *p;

	int err = -ENOMEM;

	unsigned int i;



	syms = kvmalloc(cnt * sizeof(*syms), GFP_KERNEL);

	if (!syms)

		goto error;



	buf = kvmalloc(cnt * KSYM_NAME_LEN, GFP_KERNEL);

	if (!buf)

		goto error;



	for (p = buf, i = 0; i < cnt; i++) {

		if (__get_user(usymbol, usyms + i)) {

			err = -EFAULT;

			goto error;

		}

		err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN);

		if (err == KSYM_NAME_LEN)

			err = -E2BIG;

		if (err < 0)

			goto error;

		syms[i] = p;

		p += err + 1;

	}



	us->syms = syms;

	us->buf = buf;

	return 0;



error:

	if (err) {

		kvfree(syms);

		kvfree(buf);

	}

	return err;

}



static void free_user_syms(struct user_syms *us)

{

	kvfree(us->syms);

	kvfree(us->buf);

}



static void bpf_kprobe_multi_link_release(struct bpf_link *link)

{

	struct bpf_kprobe_multi_link *kmulti_link;
@@ -2349,53 +2402,12 @@ kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip, 
	kprobe_multi_link_prog_run(link, entry_ip, regs);

}



static int

kprobe_multi_resolve_syms(const void __user *usyms, u32 cnt,

			  unsigned long *addrs)

static int symbols_cmp(const void *a, const void *b)

{

	unsigned long addr, size;

	const char __user **syms;

	int err = -ENOMEM;

	unsigned int i;

	char *func;



	size = cnt * sizeof(*syms);

	syms = kvzalloc(size, GFP_KERNEL);

	if (!syms)

		return -ENOMEM;



	func = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);

	if (!func)

		goto error;



	if (copy_from_user(syms, usyms, size)) {

		err = -EFAULT;

		goto error;

	}



	for (i = 0; i < cnt; i++) {

		err = strncpy_from_user(func, syms[i], KSYM_NAME_LEN);

		if (err == KSYM_NAME_LEN)

			err = -E2BIG;

		if (err < 0)

			goto error;

		err = -EINVAL;

		addr = kallsyms_lookup_name(func);

		if (!addr)

			goto error;

		if (!kallsyms_lookup_size_offset(addr, &size, NULL))

			goto error;

		addr = ftrace_location_range(addr, addr + size - 1);

		if (!addr)

			goto error;

		addrs[i] = addr;

	}

	const char **str_a = (const char **) a;

	const char **str_b = (const char **) b;



	err = 0;

error:

	kvfree(syms);

	kfree(func);

	return err;

	return strcmp(*str_a, *str_b);

}



int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
@@ -2441,7 +2453,15 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr 
			goto error;

		}

	} else {

		err = kprobe_multi_resolve_syms(usyms, cnt, addrs);

		struct user_syms us;



		err = copy_user_syms(&us, usyms, cnt);

		if (err)

			goto error;



		sort(us.syms, cnt, sizeof(*us.syms), symbols_cmp, NULL);

		err = ftrace_lookup_symbols(us.syms, cnt, addrs);

		free_user_syms(&us);

		if (err)

			goto error;

	}