Commit 8a3854c7 authored by Paolo Abeni's avatar Paolo Abeni Committed by David S. Miller
Browse files

udp: track the forward memory release threshold in an hot cacheline 



When the receiver process and the BH runs on different cores,
udp_rmem_release() experience a cache miss while accessing sk_rcvbuf,
as the latter shares the same cacheline with sk_forward_alloc, written
by the BH.

With this patch, UDP tracks the rcvbuf value and its update via custom
SOL_SOCKET socket options, and copies the forward memory threshold value
used by udp_rmem_release() in a different cacheline, already accessed by
the above function and uncontended.

Since the UDP socket init operation grown a bit, factor out the common
code between v4 and v6 in a shared helper.

Overall the above give a 10% peek throughput increase under UDP flood.

Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
Reviewed-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarKuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent a5ef058d

include/linux/udp.h

+3 −0
Original line number Diff line number Diff line
@@ -87,6 +87,9 @@ struct udp_sock { 


	/* This field is dirtied by udp_recvmsg() */

	int		forward_deficit;



	/* This fields follows rcvbuf value, and is touched by udp_recvmsg */

	int		forward_threshold;

};



#define UDP_MAX_SEGMENTS	(1 << 6UL)

include/net/udp.h

+9 −0
Original line number Diff line number Diff line
@@ -174,6 +174,15 @@ INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *)); 
struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,

				  netdev_features_t features, bool is_ipv6);



static inline void udp_lib_init_sock(struct sock *sk)

{

	struct udp_sock *up = udp_sk(sk);



	skb_queue_head_init(&up->reader_queue);

	up->forward_threshold = sk->sk_rcvbuf >> 2;

	set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);

}



/* hash routines shared between UDPv4/6 and UDP-Litev4/6 */

static inline int udp_lib_hash(struct sock *sk)

{

net/ipv4/udp.c

+15 −3
Original line number Diff line number Diff line
@@ -1448,7 +1448,7 @@ static void udp_rmem_release(struct sock *sk, int size, int partial, 
	if (likely(partial)) {

		up->forward_deficit += size;

		size = up->forward_deficit;

		if (size < (sk->sk_rcvbuf >> 2) &&

		if (size < READ_ONCE(up->forward_threshold) &&

		    !skb_queue_empty(&up->reader_queue))

			return;

	} else {
@@ -1622,7 +1622,7 @@ static void udp_destruct_sock(struct sock *sk) 


int udp_init_sock(struct sock *sk)

{

	skb_queue_head_init(&udp_sk(sk)->reader_queue);

	udp_lib_init_sock(sk);

	sk->sk_destruct = udp_destruct_sock;

	return 0;

}
@@ -2671,6 +2671,18 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname, 
	int err = 0;

	int is_udplite = IS_UDPLITE(sk);



	if (level == SOL_SOCKET) {

		err = sk_setsockopt(sk, level, optname, optval, optlen);



		if (optname == SO_RCVBUF || optname == SO_RCVBUFFORCE) {

			sockopt_lock_sock(sk);

			/* paired with READ_ONCE in udp_rmem_release() */

			WRITE_ONCE(up->forward_threshold, sk->sk_rcvbuf >> 2);

			sockopt_release_sock(sk);

		}

		return err;

	}



	if (optlen < sizeof(int))

		return -EINVAL;
@@ -2784,7 +2796,7 @@ EXPORT_SYMBOL(udp_lib_setsockopt); 
int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,

		   unsigned int optlen)

{

	if (level == SOL_UDP  ||  level == SOL_UDPLITE)

	if (level == SOL_UDP  ||  level == SOL_UDPLITE || level == SOL_SOCKET)

		return udp_lib_setsockopt(sk, level, optname,

					  optval, optlen,

					  udp_push_pending_frames);

net/ipv6/udp.c

+2 −2
Original line number Diff line number Diff line
@@ -64,7 +64,7 @@ static void udpv6_destruct_sock(struct sock *sk) 


int udpv6_init_sock(struct sock *sk)

{

	skb_queue_head_init(&udp_sk(sk)->reader_queue);

	udp_lib_init_sock(sk);

	sk->sk_destruct = udpv6_destruct_sock;

	return 0;

}
@@ -1669,7 +1669,7 @@ void udpv6_destroy_sock(struct sock *sk) 
int udpv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,

		     unsigned int optlen)

{

	if (level == SOL_UDP  ||  level == SOL_UDPLITE)

	if (level == SOL_UDP  ||  level == SOL_UDPLITE || level == SOL_SOCKET)

		return udp_lib_setsockopt(sk, level, optname,

					  optval, optlen,

					  udp_v6_push_pending_frames);