Commit d53ad5d8 authored by Toke Høiland-Jørgensen's avatar Toke Høiland-Jørgensen Committed by Alexei Starovoitov
Browse files

xdp: Move conversion to xdp_frame out of map functions 



All map redirect functions except XSK maps convert xdp_buff to xdp_frame
before enqueueing it. So move this conversion of out the map functions
and into xdp_do_redirect(). This removes a bit of duplicated code, but more
importantly it makes it possible to support caller-allocated xdp_frame
structures, which will be added in a subsequent commit.

Signed-off-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20220103150812.87914-5-toke@redhat.com
parent 64693ec7

include/linux/bpf.h

+10 −10
Original line number Diff line number Diff line
@@ -1669,17 +1669,17 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 
struct btf *bpf_get_btf_vmlinux(void);



/* Map specifics */

struct xdp_buff;

struct xdp_frame;

struct sk_buff;

struct bpf_dtab_netdev;

struct bpf_cpu_map_entry;



void __dev_flush(void);

int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,

		    struct net_device *dev_rx);

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,

		    struct net_device *dev_rx);

int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,

int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,

			  struct bpf_map *map, bool exclude_ingress);

int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,

			     struct bpf_prog *xdp_prog);
@@ -1688,7 +1688,7 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 
			   bool exclude_ingress);



void __cpu_map_flush(void);

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,

		    struct net_device *dev_rx);

int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,

			     struct sk_buff *skb);
@@ -1866,26 +1866,26 @@ static inline void __dev_flush(void) 
{

}



struct xdp_buff;

struct xdp_frame;

struct bpf_dtab_netdev;

struct bpf_cpu_map_entry;



static inline

int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,

		    struct net_device *dev_rx)

{

	return 0;

}



static inline

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,

		    struct net_device *dev_rx)

{

	return 0;

}



static inline

int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,

int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,

			  struct bpf_map *map, bool exclude_ingress)

{

	return 0;
@@ -1913,7 +1913,7 @@ static inline void __cpu_map_flush(void) 
}



static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,

				  struct xdp_buff *xdp,

				  struct xdp_frame *xdpf,

				  struct net_device *dev_rx)

{

	return 0;

kernel/bpf/cpumap.c

+1 −7
Original line number Diff line number Diff line
@@ -746,15 +746,9 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) 
		list_add(&bq->flush_node, flush_list);

}



int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,

		    struct net_device *dev_rx)

{

	struct xdp_frame *xdpf;



	xdpf = xdp_convert_buff_to_frame(xdp);

	if (unlikely(!xdpf))

		return -EOVERFLOW;



	/* Info needed when constructing SKB on remote CPU */

	xdpf->dev_rx = dev_rx;

kernel/bpf/devmap.c

+11 −21
Original line number Diff line number Diff line
@@ -467,24 +467,19 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 
	bq->q[bq->count++] = xdpf;

}



static inline int __xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,

				struct net_device *dev_rx,

				struct bpf_prog *xdp_prog)

{

	struct xdp_frame *xdpf;

	int err;



	if (!dev->netdev_ops->ndo_xdp_xmit)

		return -EOPNOTSUPP;



	err = xdp_ok_fwd_dev(dev, xdp->data_end - xdp->data);

	err = xdp_ok_fwd_dev(dev, xdpf->len);

	if (unlikely(err))

		return err;



	xdpf = xdp_convert_buff_to_frame(xdp);

	if (unlikely(!xdpf))

		return -EOVERFLOW;



	bq_enqueue(dev, xdpf, dev_rx, xdp_prog);

	return 0;

}
@@ -520,27 +515,27 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev 
	return act;

}



int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,

		    struct net_device *dev_rx)

{

	return __xdp_enqueue(dev, xdp, dev_rx, NULL);

	return __xdp_enqueue(dev, xdpf, dev_rx, NULL);

}



int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,

		    struct net_device *dev_rx)

{

	struct net_device *dev = dst->dev;



	return __xdp_enqueue(dev, xdp, dev_rx, dst->xdp_prog);

	return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);

}



static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_buff *xdp)

static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf)

{

	if (!obj ||

	    !obj->dev->netdev_ops->ndo_xdp_xmit)

		return false;



	if (xdp_ok_fwd_dev(obj->dev, xdp->data_end - xdp->data))

	if (xdp_ok_fwd_dev(obj->dev, xdpf->len))

		return false;



	return true;
@@ -586,14 +581,13 @@ static int get_upper_ifindexes(struct net_device *dev, int *indexes) 
	return n;

}



int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,

int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,

			  struct bpf_map *map, bool exclude_ingress)

{

	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);

	struct bpf_dtab_netdev *dst, *last_dst = NULL;

	int excluded_devices[1+MAX_NEST_DEV];

	struct hlist_head *head;

	struct xdp_frame *xdpf;

	int num_excluded = 0;

	unsigned int i;

	int err;
@@ -603,15 +597,11 @@ int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, 
		excluded_devices[num_excluded++] = dev_rx->ifindex;

	}



	xdpf = xdp_convert_buff_to_frame(xdp);

	if (unlikely(!xdpf))

		return -EOVERFLOW;



	if (map->map_type == BPF_MAP_TYPE_DEVMAP) {

		for (i = 0; i < map->max_entries; i++) {

			dst = rcu_dereference_check(dtab->netdev_map[i],

						    rcu_read_lock_bh_held());

			if (!is_valid_dst(dst, xdp))

			if (!is_valid_dst(dst, xdpf))

				continue;



			if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))
@@ -634,7 +624,7 @@ int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, 
			head = dev_map_index_hash(dtab, i);

			hlist_for_each_entry_rcu(dst, head, index_hlist,

						 lockdep_is_held(&dtab->index_lock)) {

				if (!is_valid_dst(dst, xdp))

				if (!is_valid_dst(dst, xdpf))

					continue;



				if (is_ifindex_excluded(excluded_devices, num_excluded,

net/core/filter.c

+17 −7
Original line number Diff line number Diff line
@@ -3964,12 +3964,24 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, 
	enum bpf_map_type map_type = ri->map_type;

	void *fwd = ri->tgt_value;

	u32 map_id = ri->map_id;

	struct xdp_frame *xdpf;

	struct bpf_map *map;

	int err;



	ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */

	ri->map_type = BPF_MAP_TYPE_UNSPEC;



	if (map_type == BPF_MAP_TYPE_XSKMAP) {

		err = __xsk_map_redirect(fwd, xdp);

		goto out;

	}



	xdpf = xdp_convert_buff_to_frame(xdp);

	if (unlikely(!xdpf)) {

		err = -EOVERFLOW;

		goto err;

	}



	switch (map_type) {

	case BPF_MAP_TYPE_DEVMAP:

		fallthrough;
@@ -3977,17 +3989,14 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, 
		map = READ_ONCE(ri->map);

		if (unlikely(map)) {

			WRITE_ONCE(ri->map, NULL);

			err = dev_map_enqueue_multi(xdp, dev, map,

			err = dev_map_enqueue_multi(xdpf, dev, map,

						    ri->flags & BPF_F_EXCLUDE_INGRESS);

		} else {

			err = dev_map_enqueue(fwd, xdp, dev);

			err = dev_map_enqueue(fwd, xdpf, dev);

		}

		break;

	case BPF_MAP_TYPE_CPUMAP:

		err = cpu_map_enqueue(fwd, xdp, dev);

		break;

	case BPF_MAP_TYPE_XSKMAP:

		err = __xsk_map_redirect(fwd, xdp);

		err = cpu_map_enqueue(fwd, xdpf, dev);

		break;

	case BPF_MAP_TYPE_UNSPEC:

		if (map_id == INT_MAX) {
@@ -3996,7 +4005,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, 
				err = -EINVAL;

				break;

			}

			err = dev_xdp_enqueue(fwd, xdp, dev);

			err = dev_xdp_enqueue(fwd, xdpf, dev);

			break;

		}

		fallthrough;
@@ -4004,6 +4013,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, 
		err = -EBADRQC;

	}



out:

	if (unlikely(err))

		goto err;