Commit a7d24d95 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: enable task local storage for tracing' 



Song Liu says:

====================

This set enables task local storage for non-BPF_LSM programs.

It is common for tracing BPF program to access per-task data. Currently,
these data are stored in hash tables with pid as the key. In
bcc/libbpftools [1], 9 out of 23 tools use such hash tables. However,
hash table is not ideal for many use case. Task local storage provides
better usability and performance for BPF programs. Please refer to 6/6 for
some performance comparison of task local storage vs. hash table.

Changes v5 => v6:
1. Add inc/dec bpf_task_storage_busy in bpf_local_storage_map_free().

Changes v4 => v5:
1. Fix build w/o CONFIG_NET. (kernel test robot)
2. Remove unnecessary check for !task_storage_ptr(). (Martin)
3. Small changes in commit logs.

Changes v3 => v4:
1. Prevent deadlock from recursive calls of bpf_task_storage_[get|delete].
   (2/6 checks potential deadlock and fails over, 4/6 adds a selftest).

Changes v2 => v3:
1. Make the selftest more robust. (Andrii)
2. Small changes with runqslower. (Andrii)
3. Shortern CC list to make it easy for vger.

Changes v1 => v2:
1. Do not allocate task local storage when the task is being freed.
2. Revise the selftest and added a new test for a task being freed.
3. Minor changes in runqslower.
====================

Acked-by: default avatarMartin KaFai Lau <kafai@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 9c8f21e6 ced47e30

include/linux/bpf.h

+7 −0
Original line number Diff line number Diff line
@@ -1499,6 +1499,7 @@ struct bpf_prog *bpf_prog_by_id(u32 id); 
struct bpf_link *bpf_link_by_id(u32 id);



const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);

void bpf_task_storage_free(struct task_struct *task);

#else /* !CONFIG_BPF_SYSCALL */

static inline struct bpf_prog *bpf_prog_get(u32 ufd)

{
@@ -1684,6 +1685,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) 
{

	return NULL;

}



static inline void bpf_task_storage_free(struct task_struct *task)

{

}

#endif /* CONFIG_BPF_SYSCALL */



void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
@@ -1886,6 +1891,8 @@ extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; 
extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;

extern const struct bpf_func_proto bpf_sock_from_file_proto;

extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;

extern const struct bpf_func_proto bpf_task_storage_get_proto;

extern const struct bpf_func_proto bpf_task_storage_delete_proto;



const struct bpf_func_proto *bpf_tracing_func_proto(

	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_local_storage.h

+2 −1
Original line number Diff line number Diff line
@@ -126,7 +126,8 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, 
			 struct bpf_local_storage_map *smap,

			 bool cacheit_lockit);



void bpf_local_storage_map_free(struct bpf_local_storage_map *smap);

void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,

				int __percpu *busy_counter);



int bpf_local_storage_map_check_btf(const struct bpf_map *map,

				    const struct btf *btf,

include/linux/bpf_lsm.h

+0 −22
Original line number Diff line number Diff line
@@ -38,21 +38,9 @@ static inline struct bpf_storage_blob *bpf_inode( 
	return inode->i_security + bpf_lsm_blob_sizes.lbs_inode;

}



static inline struct bpf_storage_blob *bpf_task(

	const struct task_struct *task)

{

	if (unlikely(!task->security))

		return NULL;



	return task->security + bpf_lsm_blob_sizes.lbs_task;

}



extern const struct bpf_func_proto bpf_inode_storage_get_proto;

extern const struct bpf_func_proto bpf_inode_storage_delete_proto;

extern const struct bpf_func_proto bpf_task_storage_get_proto;

extern const struct bpf_func_proto bpf_task_storage_delete_proto;

void bpf_inode_storage_free(struct inode *inode);

void bpf_task_storage_free(struct task_struct *task);



#else /* !CONFIG_BPF_LSM */
@@ -73,20 +61,10 @@ static inline struct bpf_storage_blob *bpf_inode( 
	return NULL;

}



static inline struct bpf_storage_blob *bpf_task(

	const struct task_struct *task)

{

	return NULL;

}



static inline void bpf_inode_storage_free(struct inode *inode)

{

}



static inline void bpf_task_storage_free(struct task_struct *task)

{

}



#endif /* CONFIG_BPF_LSM */



#endif /* _LINUX_BPF_LSM_H */

include/linux/bpf_types.h

+1 −1
Original line number Diff line number Diff line
@@ -109,8 +109,8 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops) 
#endif

#ifdef CONFIG_BPF_LSM

BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)

#endif

BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)

#if defined(CONFIG_XDP_SOCKETS)

BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops)

include/linux/sched.h

+5 −0
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ struct audit_context; 
struct backing_dev_info;

struct bio_list;

struct blk_plug;

struct bpf_local_storage;

struct capture_control;

struct cfs_rq;

struct fs_struct;
@@ -1348,6 +1349,10 @@ struct task_struct { 
	/* Used by LSM modules for access restriction: */

	void				*security;

#endif

#ifdef CONFIG_BPF_SYSCALL

	/* Used by BPF task local storage */

	struct bpf_local_storage __rcu	*bpf_storage;

#endif



#ifdef CONFIG_GCC_PLUGIN_STACKLEAK

	unsigned long			lowest_stack;