Commit 578b0546 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'shrink-struct-ubuf_info' 

Pavel Begunkov says:

====================
shrink struct ubuf_info

struct ubuf_info is large but not all fields are needed for all
cases. We have limited space in io_uring for it and large ubuf_info
prevents some struct embedding, even though we use only a subset
of the fields. It's also not very clean trying to use this typeless
extra space.

Shrink struct ubuf_info to only necessary fields used in generic paths,
namely ->callback, ->refcnt and ->flags, which take only 16 bytes. And
make MSG_ZEROCOPY and some other users to embed it into a larger struct
ubuf_info_msgzc mimicking the former ubuf_info.

Note, xen/vhost may also have some cleaning on top by creating
new structs containing ubuf_info but with proper types.
====================

Link: https://lore.kernel.org/r/cover.1663892211.git.asml.silence@gmail.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 929a6cdf e7d2b510

drivers/net/xen-netback/common.h

+1 −1
Original line number Diff line number Diff line
@@ -62,7 +62,7 @@ struct pending_tx_info { 
	 * ubuf_to_vif is a helper which finds the struct xenvif from a pointer

	 * to this field.

	 */

	struct ubuf_info callback_struct;

	struct ubuf_info_msgzc callback_struct;

};



#define XEN_NETIF_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE)

drivers/net/xen-netback/interface.c

+2 −2
Original line number Diff line number Diff line
@@ -591,8 +591,8 @@ int xenvif_init_queue(struct xenvif_queue *queue) 
	}



	for (i = 0; i < MAX_PENDING_REQS; i++) {

		queue->pending_tx_info[i].callback_struct = (struct ubuf_info)

			{ .callback = xenvif_zerocopy_callback,

		queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)

			{ { .callback = xenvif_zerocopy_callback },

			  { { .ctx = NULL,

			      .desc = i } } };

		queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;

drivers/net/xen-netback/netback.c

+4 −3
Original line number Diff line number Diff line
@@ -133,7 +133,7 @@ static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue, 


/* Find the containing VIF's structure from a pointer in pending_tx_info array

 */

static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info *ubuf)

static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info_msgzc *ubuf)

{

	u16 pending_idx = ubuf->desc;

	struct pending_tx_info *temp =
@@ -1228,11 +1228,12 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) 
	return work_done;

}



void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,

void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,

			      bool zerocopy_success)

{

	unsigned long flags;

	pending_ring_idx_t index;

	struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);

	struct xenvif_queue *queue = ubuf_to_queue(ubuf);



	/* This is the only place where we grab this lock, to protect callbacks
@@ -1241,7 +1242,7 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf, 
	spin_lock_irqsave(&queue->callback_lock, flags);

	do {

		u16 pending_idx = ubuf->desc;

		ubuf = (struct ubuf_info *) ubuf->ctx;

		ubuf = (struct ubuf_info_msgzc *) ubuf->ctx;

		BUG_ON(queue->dealloc_prod - queue->dealloc_cons >=

			MAX_PENDING_REQS);

		index = pending_index(queue->dealloc_prod);

drivers/vhost/net.c

+8 −7
Original line number Diff line number Diff line
@@ -118,7 +118,7 @@ struct vhost_net_virtqueue { 
	/* Number of XDP frames batched */

	int batched_xdp;

	/* an array of userspace buffers info */

	struct ubuf_info *ubuf_info;

	struct ubuf_info_msgzc *ubuf_info;

	/* Reference counting for outstanding ubufs.

	 * Protected by vq mutex. Writers must also take device mutex. */

	struct vhost_net_ubuf_ref *ubufs;
@@ -382,8 +382,9 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net, 
}



static void vhost_zerocopy_callback(struct sk_buff *skb,

				    struct ubuf_info *ubuf, bool success)

				    struct ubuf_info *ubuf_base, bool success)

{

	struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);

	struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;

	struct vhost_virtqueue *vq = ubufs->vq;

	int cnt;
@@ -871,7 +872,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) 
	size_t len, total_len = 0;

	int err;

	struct vhost_net_ubuf_ref *ubufs;

	struct ubuf_info *ubuf;

	struct ubuf_info_msgzc *ubuf;

	bool zcopy_used;

	int sent_pkts = 0;
@@ -907,14 +908,14 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) 
			ubuf = nvq->ubuf_info + nvq->upend_idx;

			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);

			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;

			ubuf->callback = vhost_zerocopy_callback;

			ubuf->ctx = nvq->ubufs;

			ubuf->desc = nvq->upend_idx;

			ubuf->flags = SKBFL_ZEROCOPY_FRAG;

			refcount_set(&ubuf->refcnt, 1);

			ubuf->ubuf.callback = vhost_zerocopy_callback;

			ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;

			refcount_set(&ubuf->ubuf.refcnt, 1);

			msg.msg_control = &ctl;

			ctl.type = TUN_MSG_UBUF;

			ctl.ptr = ubuf;

			ctl.ptr = &ubuf->ubuf;

			msg.msg_controllen = sizeof(ctl);

			ubufs = nvq->ubufs;

			atomic_inc(&ubufs->refcount);

include/linux/skbuff.h

+9 −2
Original line number Diff line number Diff line
@@ -533,6 +533,13 @@ enum { 
struct ubuf_info {

	void (*callback)(struct sk_buff *, struct ubuf_info *,

			 bool zerocopy_success);

	refcount_t refcnt;

	u8 flags;

};



struct ubuf_info_msgzc {

	struct ubuf_info ubuf;



	union {

		struct {

			unsigned long desc;
@@ -545,8 +552,6 @@ struct ubuf_info { 
			u32 bytelen;

		};

	};

	refcount_t refcnt;

	u8 flags;



	struct mmpin {

		struct user_struct *user;
@@ -555,6 +560,8 @@ struct ubuf_info { 
};



#define skb_uarg(SKB)	((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))

#define uarg_to_msgzc(ubuf_ptr)	container_of((ubuf_ptr), struct ubuf_info_msgzc, \

					     ubuf)



int mm_account_pinned_pages(struct mmpin *mmp, size_t size);

void mm_unaccount_pinned_pages(struct mmpin *mmp);