Commit 3d76a4d3 authored by Stanislav Fomichev's avatar Stanislav Fomichev Committed by Martin KaFai Lau
Browse files

bpf: XDP metadata RX kfuncs 



Define a new kfunc set (xdp_metadata_kfunc_ids) which implements all possible
XDP metatada kfuncs. Not all devices have to implement them. If kfunc is not
supported by the target device, the default implementation is called instead.
The verifier, at load time, replaces a call to the generic kfunc with a call
to the per-device one. Per-device kfunc pointers are stored in separate
struct xdp_metadata_ops.

Cc: John Fastabend <john.fastabend@gmail.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Martin KaFai Lau <martin.lau@linux.dev>
Cc: Jakub Kicinski <kuba@kernel.org>
Cc: Willem de Bruijn <willemb@google.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Anatoly Burakov <anatoly.burakov@intel.com>
Cc: Alexander Lobakin <alexandr.lobakin@intel.com>
Cc: Magnus Karlsson <magnus.karlsson@gmail.com>
Cc: Maryam Tahhan <mtahhan@redhat.com>
Cc: xdp-hints@xdp-project.net
Cc: netdev@vger.kernel.org
Signed-off-by: default avatarStanislav Fomichev <sdf@google.com>
Link: https://lore.kernel.org/r/20230119221536.3349901-8-sdf@google.com


Signed-off-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
parent 40535704

include/linux/bpf.h

+16 −1
Original line number Diff line number Diff line
@@ -2480,6 +2480,9 @@ bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 
void unpriv_ebpf_notify(int new_state);



#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)

int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,

			      struct bpf_prog_aux *prog_aux);

void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);

int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);

void bpf_dev_bound_netdev_unregister(struct net_device *dev);
@@ -2514,6 +2517,18 @@ void sock_map_unhash(struct sock *sk); 
void sock_map_destroy(struct sock *sk);

void sock_map_close(struct sock *sk, long timeout);

#else

static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,

					    struct bpf_prog_aux *prog_aux)

{

	return -EOPNOTSUPP;

}



static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,

						u32 func_id)

{

	return NULL;

}



static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,

					  union bpf_attr *attr)

{

include/linux/netdevice.h

+8 −0
Original line number Diff line number Diff line
@@ -74,6 +74,7 @@ struct udp_tunnel_nic_info; 
struct udp_tunnel_nic;

struct bpf_prog;

struct xdp_buff;

struct xdp_md;



void synchronize_net(void);

void netdev_set_default_ethtool_ops(struct net_device *dev,
@@ -1618,6 +1619,11 @@ struct net_device_ops { 
						  bool cycles);

};



struct xdp_metadata_ops {

	int	(*xmo_rx_timestamp)(const struct xdp_md *ctx, u64 *timestamp);

	int	(*xmo_rx_hash)(const struct xdp_md *ctx, u32 *hash);

};



/**

 * enum netdev_priv_flags - &struct net_device priv_flags

 *
@@ -1801,6 +1807,7 @@ enum netdev_ml_priv_type { 
 *

 *	@netdev_ops:	Includes several pointers to callbacks,

 *			if one wants to override the ndo_*() functions

 *	@xdp_metadata_ops:	Includes pointers to XDP metadata callbacks.

 *	@ethtool_ops:	Management operations

 *	@l3mdev_ops:	Layer 3 master device operations

 *	@ndisc_ops:	Includes callbacks for different IPv6 neighbour
@@ -2050,6 +2057,7 @@ struct net_device { 
	unsigned int		flags;

	unsigned long long	priv_flags;

	const struct net_device_ops *netdev_ops;

	const struct xdp_metadata_ops *xdp_metadata_ops;

	int			ifindex;

	unsigned short		gflags;

	unsigned short		hard_header_len;

include/net/xdp.h

+21 −0
Original line number Diff line number Diff line
@@ -409,4 +409,25 @@ void xdp_attachment_setup(struct xdp_attachment_info *info, 


#define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE



#define XDP_METADATA_KFUNC_xxx	\

	XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_TIMESTAMP, \

			   bpf_xdp_metadata_rx_timestamp) \

	XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_HASH, \

			   bpf_xdp_metadata_rx_hash) \



enum {

#define XDP_METADATA_KFUNC(name, _) name,

XDP_METADATA_KFUNC_xxx

#undef XDP_METADATA_KFUNC

MAX_XDP_METADATA_KFUNC,

};



#ifdef CONFIG_NET

u32 bpf_xdp_metadata_kfunc_id(int id);

bool bpf_dev_bound_kfunc_id(u32 btf_id);

#else

static inline u32 bpf_xdp_metadata_kfunc_id(int id) { return 0; }

static inline bool bpf_dev_bound_kfunc_id(u32 btf_id) { return false; }

#endif



#endif /* __LINUX_NET_XDP_H__ */

kernel/bpf/core.c

+8 −0
Original line number Diff line number Diff line
@@ -2096,6 +2096,14 @@ bool bpf_prog_map_compatible(struct bpf_map *map, 
	if (fp->kprobe_override)

		return false;



	/* XDP programs inserted into maps are not guaranteed to run on

	 * a particular netdev (and can run outside driver context entirely

	 * in the case of devmap and cpumap). Until device checks

	 * are implemented, prohibit adding dev-bound programs to program maps.

	 */

	if (bpf_prog_is_dev_bound(fp->aux))

		return false;



	spin_lock(&map->owner.lock);

	if (!map->owner.type) {

		/* There's no owner yet where we could check for

kernel/bpf/offload.c

+44 −0
Original line number Diff line number Diff line
@@ -755,6 +755,50 @@ void bpf_dev_bound_netdev_unregister(struct net_device *dev) 
	up_write(&bpf_devs_lock);

}



int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,

			      struct bpf_prog_aux *prog_aux)

{

	if (!bpf_prog_is_dev_bound(prog_aux)) {

		bpf_log(log, "metadata kfuncs require device-bound program\n");

		return -EINVAL;

	}



	if (bpf_prog_is_offloaded(prog_aux)) {

		bpf_log(log, "metadata kfuncs can't be offloaded\n");

		return -EINVAL;

	}



	return 0;

}



void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id)

{

	const struct xdp_metadata_ops *ops;

	void *p = NULL;



	/* We don't hold bpf_devs_lock while resolving several

	 * kfuncs and can race with the unregister_netdevice().

	 * We rely on bpf_dev_bound_match() check at attach

	 * to render this program unusable.

	 */

	down_read(&bpf_devs_lock);

	if (!prog->aux->offload)

		goto out;



	ops = prog->aux->offload->netdev->xdp_metadata_ops;

	if (!ops)

		goto out;



	if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_TIMESTAMP))

		p = ops->xmo_rx_timestamp;

	else if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_HASH))

		p = ops->xmo_rx_hash;

out:

	up_read(&bpf_devs_lock);



	return p;

}



static int __init bpf_offload_init(void)

{

	return rhashtable_init(&offdevs, &offdevs_params);