xsk: Add generic xdp multi-buffer recv support (930f681f) · Commits · EulixOS / Software / Kernel

arch/x86/configs/openeuler_defconfig

+1 −0

Original line number	Diff line number	Diff line
		@@ -1154,6 +1154,7 @@ CONFIG_SMC=m
		CONFIG_SMC_DIAG=m
		CONFIG_XDP_SOCKETS=y
		# CONFIG_XDP_SOCKETS_DIAG is not set
		# CONFIG_XSK_MULTI_BUF is not set
		CONFIG_INET=y
		CONFIG_IP_MULTICAST=y
		CONFIG_IP_ADVANCED_ROUTER=y

+7 −0

Original line number	Diff line number	Diff line
		@@ -1620,6 +1620,8 @@ struct net_device_ops {
		* @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
		* @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
		* @IFF_LIVE_RENAME_OK: rename is allowed while device is up and running
		* @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with
		* skb_headlen(skb) == 0 (data starts from frag0)
		*/
		enum netdev_priv_flags {
		IFF_802_1Q_VLAN = 1<<0,
		@@ -1653,6 +1655,7 @@ enum netdev_priv_flags {
		IFF_FAILOVER_SLAVE = 1<<28,
		IFF_L3MDEV_RX_HANDLER = 1<<29,
		IFF_LIVE_RENAME_OK = 1<<30,
		IFF_TX_SKB_NO_LINEAR = 1<<31,
		};

		#define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
		@@ -1685,6 +1688,7 @@ enum netdev_priv_flags {
		#define IFF_FAILOVER_SLAVE IFF_FAILOVER_SLAVE
		#define IFF_L3MDEV_RX_HANDLER IFF_L3MDEV_RX_HANDLER
		#define IFF_LIVE_RENAME_OK IFF_LIVE_RENAME_OK
		#define IFF_TX_SKB_NO_LINEAR IFF_TX_SKB_NO_LINEAR

		/* Specifies the type of the struct net_device::ml_priv pointer */
		enum netdev_ml_priv_type {
		@@ -3953,6 +3957,9 @@ static inline void dev_consume_skb_any(struct sk_buff *skb)

		void generic_xdp_tx(struct sk_buff skb, struct bpf_prog xdp_prog);
		int do_xdp_generic(struct bpf_prog xdp_prog, struct sk_buff skb);
		#ifdef CONFIG_XSK_MULTI_BUF
		int do_xdp_generic_multi(struct bpf_prog xdp_prog, struct sk_buff *pskb);
		#endif
		int netif_rx(struct sk_buff *skb);
		int netif_rx_ni(struct sk_buff *skb);
		int netif_rx_any_context(struct sk_buff *skb);

+6 −0

Original line number	Diff line number	Diff line
		@@ -530,6 +530,9 @@ struct skb_shared_info {
		* Warning : all fields before dataref are cleared in __alloc_skb()
		*/
		atomic_t dataref;
		#ifdef CONFIG_XSK_MULTI_BUF
		KABI_FILL_HOLE(unsigned int xdp_frags_size)
		#endif

		/* Intermediate layers must ensure that destructor_arg
		* remains valid until skb destructor */
		@@ -3110,6 +3113,9 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f)
		__skb_frag_ref(&skb_shinfo(skb)->frags[f]);
		}

		#ifdef CONFIG_XSK_MULTI_BUF
		int skb_cow_data_for_xdp(struct sk_buff *pskb, struct bpf_prog prog);
		#endif
		/**
		* __skb_frag_unref - release a reference on a paged fragment.
		* @frag: the paged fragment

+43 −0

Original line number	Diff line number	Diff line
		@@ -71,6 +71,12 @@ struct xdp_txq_info {
		struct net_device *dev;
		};

		#ifdef CONFIG_XSK_MULTI_BUF
		enum xdp_buff_flags {
		XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
		};
		#endif

		struct xdp_buff {
		void *data;
		void *data_end;
		@@ -79,8 +85,28 @@ struct xdp_buff {
		struct xdp_rxq_info *rxq;
		struct xdp_txq_info *txq;
		u32 frame_sz; /* frame size to deduce data_hard_end/reserved tailroom*/
		#ifdef CONFIG_XSK_MULTI_BUF
		KABI_FILL_HOLE(u32 flags) /* supported values defined in xdp_buff_flags */
		#endif
		};

		#ifdef CONFIG_XSK_MULTI_BUF
		static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp)
		{
		return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS);
		}

		static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp)
		{
		xdp->flags \|= XDP_FLAGS_HAS_FRAGS;
		}

		static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp)
		{
		xdp->flags &= ~XDP_FLAGS_HAS_FRAGS;
		}
		#endif

		/* Reserve memory area at end-of data area.
		*
		* This macro reserves tailroom in the XDP buffer by limiting the
		@@ -97,6 +123,23 @@ xdp_get_shared_info_from_buff(struct xdp_buff *xdp)
		return (struct skb_shared_info *)xdp_data_hard_end(xdp);
		}

		#ifdef CONFIG_XSK_MULTI_BUF
		static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp)
		{
		unsigned int len = xdp->data_end - xdp->data;
		struct skb_shared_info *sinfo;

		if (likely(!xdp_buff_has_frags(xdp)))
		goto out;

		sinfo = xdp_get_shared_info_from_buff(xdp);

		len += sinfo->xdp_frags_size;
		out:
		return len;
		}
		#endif

		struct xdp_frame {
		void *data;
		u16 len;

+20 −0

Original line number	Diff line number	Diff line
		@@ -43,6 +43,13 @@ struct xsk_map {
		struct xdp_sock {
		/* struct sock must be the first member of struct xdp_sock */
		struct sock sk;
		#ifdef CONFIG_XSK_MULTI_BUF
		/* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
		* packet, the partially built skb is saved here so that packet building can resume in next
		* call of __xsk_generic_xmit().
		*/
		KABI_FILL_HOLE(struct sk_buff *skb)
		#endif
		struct xsk_queue *rx ____cacheline_aligned_in_smp;
		struct net_device *dev;
		struct xdp_umem *umem;
		@@ -50,6 +57,9 @@ struct xdp_sock {
		struct xsk_buff_pool *pool;
		u16 queue_id;
		bool zc;
		#ifdef CONFIG_XSK_MULTI_BUF
		KABI_FILL_HOLE(bool sg)
		#endif
		enum {
		XSK_READY = 0,
		XSK_BOUND,
		@@ -77,6 +87,9 @@ struct xdp_sock {
		#ifdef CONFIG_XDP_SOCKETS

		int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp);
		#ifdef CONFIG_XSK_MULTI_BUF
		int xsk_generic_rcv_multi(struct xdp_sock xs, struct xdp_buff xdp);
		#endif
		int __xsk_map_redirect(struct xdp_sock xs, struct xdp_buff xdp);
		void __xsk_map_flush(void);

		@@ -100,6 +113,13 @@ static inline int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp)
		return -ENOTSUPP;
		}

		#ifdef CONFIG_XSK_MULTI_BUF
		static inline int xsk_generic_rcv_multi(struct xdp_sock xs, struct xdp_buff xdp)
		{
		return -ENOTSUPP;
		}
		#endif

		static inline int __xsk_map_redirect(struct xdp_sock xs, struct xdp_buff xdp)
		{
		return -EOPNOTSUPP;