Commit a10787e6 authored by Song Liu's avatar Song Liu Committed by Alexei Starovoitov
Browse files

bpf: Enable task local storage for tracing programs 



To access per-task data, BPF programs usually creates a hash table with
pid as the key. This is not ideal because:
 1. The user need to estimate the proper size of the hash table, which may
    be inaccurate;
 2. Big hash tables are slow;
 3. To clean up the data properly during task terminations, the user need
    to write extra logic.

Task local storage overcomes these issues and offers a better option for
these per-task data. Task local storage is only available to BPF_LSM. Now
enable it for tracing programs.

Unlike LSM programs, tracing programs can be called in IRQ contexts.
Helpers that access task local storage are updated to use
raw_spin_lock_irqsave() instead of raw_spin_lock_bh().

Tracing programs can attach to functions on the task free path, e.g.
exit_creds(). To avoid allocating task local storage after
bpf_task_storage_free(). bpf_task_storage_get() is updated to not allocate
new storage when the task is not refcounted (task->usage == 0).

Signed-off-by: default avatarSong Liu <songliubraving@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarKP Singh <kpsingh@kernel.org>
Acked-by: default avatarMartin KaFai Lau <kafai@fb.com>
Link: https://lore.kernel.org/bpf/20210225234319.336131-2-songliubraving@fb.com
parent 9c8f21e6

include/linux/bpf.h

+7 −0
Original line number Diff line number Diff line
@@ -1499,6 +1499,7 @@ struct bpf_prog *bpf_prog_by_id(u32 id); 
struct bpf_link *bpf_link_by_id(u32 id);



const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);

void bpf_task_storage_free(struct task_struct *task);

#else /* !CONFIG_BPF_SYSCALL */

static inline struct bpf_prog *bpf_prog_get(u32 ufd)

{
@@ -1684,6 +1685,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) 
{

	return NULL;

}



static inline void bpf_task_storage_free(struct task_struct *task)

{

}

#endif /* CONFIG_BPF_SYSCALL */



void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
@@ -1886,6 +1891,8 @@ extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; 
extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;

extern const struct bpf_func_proto bpf_sock_from_file_proto;

extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;

extern const struct bpf_func_proto bpf_task_storage_get_proto;

extern const struct bpf_func_proto bpf_task_storage_delete_proto;



const struct bpf_func_proto *bpf_tracing_func_proto(

	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_lsm.h

+0 −22
Original line number Diff line number Diff line
@@ -38,21 +38,9 @@ static inline struct bpf_storage_blob *bpf_inode( 
	return inode->i_security + bpf_lsm_blob_sizes.lbs_inode;

}



static inline struct bpf_storage_blob *bpf_task(

	const struct task_struct *task)

{

	if (unlikely(!task->security))

		return NULL;



	return task->security + bpf_lsm_blob_sizes.lbs_task;

}



extern const struct bpf_func_proto bpf_inode_storage_get_proto;

extern const struct bpf_func_proto bpf_inode_storage_delete_proto;

extern const struct bpf_func_proto bpf_task_storage_get_proto;

extern const struct bpf_func_proto bpf_task_storage_delete_proto;

void bpf_inode_storage_free(struct inode *inode);

void bpf_task_storage_free(struct task_struct *task);



#else /* !CONFIG_BPF_LSM */
@@ -73,20 +61,10 @@ static inline struct bpf_storage_blob *bpf_inode( 
	return NULL;

}



static inline struct bpf_storage_blob *bpf_task(

	const struct task_struct *task)

{

	return NULL;

}



static inline void bpf_inode_storage_free(struct inode *inode)

{

}



static inline void bpf_task_storage_free(struct task_struct *task)

{

}



#endif /* CONFIG_BPF_LSM */



#endif /* _LINUX_BPF_LSM_H */

include/linux/bpf_types.h

+1 −1
Original line number Diff line number Diff line
@@ -109,8 +109,8 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops) 
#endif

#ifdef CONFIG_BPF_LSM

BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)

#endif

BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)

#if defined(CONFIG_XDP_SOCKETS)

BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops)

include/linux/sched.h

+5 −0
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ struct audit_context; 
struct backing_dev_info;

struct bio_list;

struct blk_plug;

struct bpf_local_storage;

struct capture_control;

struct cfs_rq;

struct fs_struct;
@@ -1348,6 +1349,10 @@ struct task_struct { 
	/* Used by LSM modules for access restriction: */

	void				*security;

#endif

#ifdef CONFIG_BPF_SYSCALL

	/* Used by BPF task local storage */

	struct bpf_local_storage __rcu	*bpf_storage;

#endif



#ifdef CONFIG_GCC_PLUGIN_STACKLEAK

	unsigned long			lowest_stack;

kernel/bpf/Makefile

+1 −2
Original line number Diff line number Diff line
@@ -9,8 +9,8 @@ CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) 
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o

obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o

obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o

obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o

obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o

obj-${CONFIG_BPF_LSM}	  += bpf_task_storage.o

obj-$(CONFIG_BPF_SYSCALL) += disasm.o

obj-$(CONFIG_BPF_JIT) += trampoline.o

obj-$(CONFIG_BPF_SYSCALL) += btf.o
@@ -18,7 +18,6 @@ obj-$(CONFIG_BPF_JIT) += dispatcher.o 
ifeq ($(CONFIG_NET),y)

obj-$(CONFIG_BPF_SYSCALL) += devmap.o

obj-$(CONFIG_BPF_SYSCALL) += cpumap.o

obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o

obj-$(CONFIG_BPF_SYSCALL) += offload.o

obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o

endif