Commit ebe73a28 authored by David Ahern's avatar David Ahern Committed by Jakub Kicinski
Browse files

net: Allow custom iter handler in msghdr 



Add support for custom iov_iter handling to msghdr. The idea is that
in-kernel subsystems want control over how an SG is split.

Signed-off-by: default avatarDavid Ahern <dsahern@kernel.org>
[pavel: move callback into msghdr]
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 7c701d92

include/linux/skbuff.h

+4 −3
Original line number Diff line number Diff line
@@ -1776,13 +1776,14 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); 
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,

			   bool success);



int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,

			    struct iov_iter *from, size_t length);

int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,

			    struct sk_buff *skb, struct iov_iter *from,

			    size_t length);



static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,

					  struct msghdr *msg, int len)

{

	return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);

	return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len);

}



int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,

include/linux/socket.h

+4 −0
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ struct file; 
struct pid;

struct cred;

struct socket;

struct sock;

struct sk_buff;



#define __sockaddr_check_size(size)	\

	BUILD_BUG_ON(((size) > sizeof(struct __kernel_sockaddr_storage)))
@@ -70,6 +72,8 @@ struct msghdr { 
	__kernel_size_t	msg_controllen;	/* ancillary data buffer length */

	struct kiocb	*msg_iocb;	/* ptr to iocb for async requests */

	struct ubuf_info *msg_ubuf;

	int (*sg_from_iter)(struct sock *sk, struct sk_buff *skb,

			    struct iov_iter *from, size_t length);

};



struct user_msghdr {

net/core/datagram.c

+10 −4
Original line number Diff line number Diff line
@@ -613,10 +613,16 @@ int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset, 
}

EXPORT_SYMBOL(skb_copy_datagram_from_iter);



int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,

			    struct iov_iter *from, size_t length)

int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,

			    struct sk_buff *skb, struct iov_iter *from,

			    size_t length)

{

	int frag = skb_shinfo(skb)->nr_frags;

	int frag;



	if (msg && msg->sg_from_iter)

		return msg->sg_from_iter(sk, skb, from, length);



	frag = skb_shinfo(skb)->nr_frags;



	while (length && iov_iter_count(from)) {

		struct page *pages[MAX_SKB_FRAGS];
@@ -702,7 +708,7 @@ int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *from) 
	if (skb_copy_datagram_from_iter(skb, 0, from, copy))

		return -EFAULT;



	return __zerocopy_sg_from_iter(NULL, skb, from, ~0U);

	return __zerocopy_sg_from_iter(NULL, NULL, skb, from, ~0U);

}

EXPORT_SYMBOL(zerocopy_sg_from_iter);

net/core/skbuff.c

+1 −1
Original line number Diff line number Diff line
@@ -1358,7 +1358,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 
	if (orig_uarg && uarg != orig_uarg)

		return -EEXIST;



	err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);

	err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);

	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {

		struct sock *save_sk = skb->sk;