bpf: Implement bpf_local_storage for inodes (8ea63684) · Commits · EulixOS / Software / Kernel

include/linux/bpf_lsm.h

+29 −0

Original line number	Diff line number	Diff line
		@@ -17,9 +17,28 @@
		#include <linux/lsm_hook_defs.h>
		#undef LSM_HOOK

		struct bpf_storage_blob {
		struct bpf_local_storage __rcu *storage;
		};

		extern struct lsm_blob_sizes bpf_lsm_blob_sizes;

		int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
		const struct bpf_prog *prog);

		static inline struct bpf_storage_blob *bpf_inode(
		const struct inode *inode)
		{
		if (unlikely(!inode->i_security))
		return NULL;

		return inode->i_security + bpf_lsm_blob_sizes.lbs_inode;
		}

		extern const struct bpf_func_proto bpf_inode_storage_get_proto;
		extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
		void bpf_inode_storage_free(struct inode *inode);

		#else /* !CONFIG_BPF_LSM */

		static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
		@@ -28,6 +47,16 @@ static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
		return -EOPNOTSUPP;
		}

		static inline struct bpf_storage_blob *bpf_inode(
		const struct inode *inode)
		{
		return NULL;
		}

		static inline void bpf_inode_storage_free(struct inode *inode)
		{
		}

		#endif /* CONFIG_BPF_LSM */

		#endif /* _LINUX_BPF_LSM_H */

include/linux/bpf_types.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -107,6 +107,9 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_SK_STORAGE, sk_storage_map_ops)
		BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)
		BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops)
		#endif
		#ifdef CONFIG_BPF_LSM
		BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops)
		#endif
		BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
		#if defined(CONFIG_XDP_SOCKETS)
		BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops)

include/uapi/linux/bpf.h

+39 −1

Original line number	Diff line number	Diff line
		@@ -155,6 +155,7 @@ enum bpf_map_type {
		BPF_MAP_TYPE_DEVMAP_HASH,
		BPF_MAP_TYPE_STRUCT_OPS,
		BPF_MAP_TYPE_RINGBUF,
		BPF_MAP_TYPE_INODE_STORAGE,
		};

		/* Note that tracing related programs such as
		@@ -3509,6 +3510,41 @@ union bpf_attr {
		*
		* -EPERM This helper cannot be used under the
		* current sock_ops->op.
		* void bpf_inode_storage_get(struct bpf_map map, void inode, void value, u64 flags)
		* Description
		* Get a bpf_local_storage from an inode.
		*
		* Logically, it could be thought of as getting the value from
		* a map with inode as the key. From this
		* perspective, the usage is not much different from
		* bpf_map_lookup_elem\ (map, &\ inode) except this
		* helper enforces the key must be an inode and the map must also
		* be a BPF_MAP_TYPE_INODE_STORAGE.
		*
		* Underneath, the value is stored locally at inode instead of
		* the map. The map is used as the bpf-local-storage
		* "type". The bpf-local-storage "type" (i.e. the map) is
		* searched against all bpf_local_storage residing at inode.
		*
		* An optional flags (BPF_LOCAL_STORAGE_GET_F_CREATE) can be
		* used such that a new bpf_local_storage will be
		* created if one does not exist. value can be used
		* together with BPF_LOCAL_STORAGE_GET_F_CREATE to specify
		* the initial value of a bpf_local_storage. If value is
		* NULL, the new bpf_local_storage will be zero initialized.
		* Return
		* A bpf_local_storage pointer is returned on success.
		*
		* NULL if not found or there was an error in adding
		* a new bpf_local_storage.
		*
		* int bpf_inode_storage_delete(struct bpf_map map, void inode)
		* Description
		* Delete a bpf_local_storage from an inode.
		* Return
		* 0 on success.
		*
		* -ENOENT if the bpf_local_storage cannot be found.
		*/
		#define __BPF_FUNC_MAPPER(FN) \
		FN(unspec), \
		@@ -3655,7 +3691,9 @@ union bpf_attr {
		FN(get_task_stack), \
		FN(load_hdr_opt), \
		FN(store_hdr_opt), \
		FN(reserve_hdr_opt),
		FN(reserve_hdr_opt), \
		FN(inode_storage_get), \
		FN(inode_storage_delete), \
		/* */

		/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/Makefile

+1 −0

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ CFLAGS_core.o += $(call cc-disable-warning, override-init)
		obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o
		obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
		obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
		obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
		obj-$(CONFIG_BPF_SYSCALL) += disasm.o
		obj-$(CONFIG_BPF_JIT) += trampoline.o
		obj-$(CONFIG_BPF_SYSCALL) += btf.o

kernel/bpf/bpf_inode_storage.c

0 → 100644

+273 −0

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0
		/*
		* Copyright (c) 2019 Facebook
		* Copyright 2020 Google LLC.
		*/

		#include <linux/rculist.h>
		#include <linux/list.h>
		#include <linux/hash.h>
		#include <linux/types.h>
		#include <linux/spinlock.h>
		#include <linux/bpf.h>
		#include <linux/bpf_local_storage.h>
		#include <net/sock.h>
		#include <uapi/linux/sock_diag.h>
		#include <uapi/linux/btf.h>
		#include <linux/bpf_lsm.h>
		#include <linux/btf_ids.h>
		#include <linux/fdtable.h>

		DEFINE_BPF_STORAGE_CACHE(inode_cache);

		static struct bpf_local_storage __rcu **
		inode_storage_ptr(void *owner)
		{
		struct inode *inode = owner;
		struct bpf_storage_blob *bsb;

		bsb = bpf_inode(inode);
		if (!bsb)
		return NULL;
		return &bsb->storage;
		}

		static struct bpf_local_storage_data inode_storage_lookup(struct inode inode,
		struct bpf_map *map,
		bool cacheit_lockit)
		{
		struct bpf_local_storage *inode_storage;
		struct bpf_local_storage_map *smap;
		struct bpf_storage_blob *bsb;

		bsb = bpf_inode(inode);
		if (!bsb)
		return NULL;

		inode_storage = rcu_dereference(bsb->storage);
		if (!inode_storage)
		return NULL;

		smap = (struct bpf_local_storage_map *)map;
		return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
		}

		void bpf_inode_storage_free(struct inode *inode)
		{
		struct bpf_local_storage_elem *selem;
		struct bpf_local_storage *local_storage;
		bool free_inode_storage = false;
		struct bpf_storage_blob *bsb;
		struct hlist_node *n;

		bsb = bpf_inode(inode);
		if (!bsb)
		return;

		rcu_read_lock();

		local_storage = rcu_dereference(bsb->storage);
		if (!local_storage) {
		rcu_read_unlock();
		return;
		}

		/* Netiher the bpf_prog nor the bpf-map's syscall
		* could be modifying the local_storage->list now.
		* Thus, no elem can be added-to or deleted-from the
		* local_storage->list by the bpf_prog or by the bpf-map's syscall.
		*
		* It is racing with bpf_local_storage_map_free() alone
		* when unlinking elem from the local_storage->list and
		* the map's bucket->list.
		*/
		raw_spin_lock_bh(&local_storage->lock);
		hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
		/* Always unlink from map before unlinking from
		* local_storage.
		*/
		bpf_selem_unlink_map(selem);
		free_inode_storage = bpf_selem_unlink_storage_nolock(
		local_storage, selem, false);
		}
		raw_spin_unlock_bh(&local_storage->lock);
		rcu_read_unlock();

		/* free_inoode_storage should always be true as long as
		* local_storage->list was non-empty.
		*/
		if (free_inode_storage)
		kfree_rcu(local_storage, rcu);
		}

		static void bpf_fd_inode_storage_lookup_elem(struct bpf_map map, void *key)
		{
		struct bpf_local_storage_data *sdata;
		struct file *f;
		int fd;

		fd = (int )key;
		f = fget_raw(fd);
		if (!f)
		return NULL;

		sdata = inode_storage_lookup(f->f_inode, map, true);
		fput(f);
		return sdata ? sdata->data : NULL;
		}

		static int bpf_fd_inode_storage_update_elem(struct bpf_map map, void key,
		void *value, u64 map_flags)
		{
		struct bpf_local_storage_data *sdata;
		struct file *f;
		int fd;

		fd = (int )key;
		f = fget_raw(fd);
		if (!f \|\| !inode_storage_ptr(f->f_inode))
		return -EBADF;

		sdata = bpf_local_storage_update(f->f_inode,
		(struct bpf_local_storage_map *)map,
		value, map_flags);
		fput(f);
		return PTR_ERR_OR_ZERO(sdata);
		}

		static int inode_storage_delete(struct inode inode, struct bpf_map map)
		{
		struct bpf_local_storage_data *sdata;

		sdata = inode_storage_lookup(inode, map, false);
		if (!sdata)
		return -ENOENT;

		bpf_selem_unlink(SELEM(sdata));

		return 0;
		}

		static int bpf_fd_inode_storage_delete_elem(struct bpf_map map, void key)
		{
		struct file *f;
		int fd, err;

		fd = (int )key;
		f = fget_raw(fd);
		if (!f)
		return -EBADF;

		err = inode_storage_delete(f->f_inode, map);
		fput(f);
		return err;
		}

		BPF_CALL_4(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode,
		void *, value, u64, flags)
		{
		struct bpf_local_storage_data *sdata;

		if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
		return (unsigned long)NULL;

		/* explicitly check that the inode_storage_ptr is not
		* NULL as inode_storage_lookup returns NULL in this case and
		* bpf_local_storage_update expects the owner to have a
		* valid storage pointer.
		*/
		if (!inode_storage_ptr(inode))
		return (unsigned long)NULL;

		sdata = inode_storage_lookup(inode, map, true);
		if (sdata)
		return (unsigned long)sdata->data;

		/* This helper must only called from where the inode is gurranteed
		* to have a refcount and cannot be freed.
		*/
		if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
		sdata = bpf_local_storage_update(
		inode, (struct bpf_local_storage_map *)map, value,
		BPF_NOEXIST);
		return IS_ERR(sdata) ? (unsigned long)NULL :
		(unsigned long)sdata->data;
		}

		return (unsigned long)NULL;
		}

		BPF_CALL_2(bpf_inode_storage_delete,
		struct bpf_map , map, struct inode , inode)
		{
		/* This helper must only called from where the inode is gurranteed
		* to have a refcount and cannot be freed.
		*/
		return inode_storage_delete(inode, map);
		}

		static int notsupp_get_next_key(struct bpf_map map, void key,
		void *next_key)
		{
		return -ENOTSUPP;
		}

		static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)
		{
		struct bpf_local_storage_map *smap;

		smap = bpf_local_storage_map_alloc(attr);
		if (IS_ERR(smap))
		return ERR_CAST(smap);

		smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
		return &smap->map;
		}

		static void inode_storage_map_free(struct bpf_map *map)
		{
		struct bpf_local_storage_map *smap;

		smap = (struct bpf_local_storage_map *)map;
		bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
		bpf_local_storage_map_free(smap);
		}

		static int inode_storage_map_btf_id;
		const struct bpf_map_ops inode_storage_map_ops = {
		.map_alloc_check = bpf_local_storage_map_alloc_check,
		.map_alloc = inode_storage_map_alloc,
		.map_free = inode_storage_map_free,
		.map_get_next_key = notsupp_get_next_key,
		.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
		.map_update_elem = bpf_fd_inode_storage_update_elem,
		.map_delete_elem = bpf_fd_inode_storage_delete_elem,
		.map_check_btf = bpf_local_storage_map_check_btf,
		.map_btf_name = "bpf_local_storage_map",
		.map_btf_id = &inode_storage_map_btf_id,
		.map_owner_storage_ptr = inode_storage_ptr,
		};

		BTF_ID_LIST(bpf_inode_storage_btf_ids)
		BTF_ID_UNUSED
		BTF_ID(struct, inode)

		const struct bpf_func_proto bpf_inode_storage_get_proto = {
		.func = bpf_inode_storage_get,
		.gpl_only = false,
		.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
		.arg1_type = ARG_CONST_MAP_PTR,
		.arg2_type = ARG_PTR_TO_BTF_ID,
		.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
		.arg4_type = ARG_ANYTHING,
		.btf_id = bpf_inode_storage_btf_ids,
		};

		const struct bpf_func_proto bpf_inode_storage_delete_proto = {
		.func = bpf_inode_storage_delete,
		.gpl_only = false,
		.ret_type = RET_INTEGER,
		.arg1_type = ARG_CONST_MAP_PTR,
		.arg2_type = ARG_PTR_TO_BTF_ID,
		.btf_id = bpf_inode_storage_btf_ids,
		};