Commit ef57c161 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

ipv6: move inet6_sk(sk)->rx_dst_cookie to sk->sk_rx_dst_cookie 



Increase cache locality by moving rx_dst_coookie next to sk->sk_rx_dst

This removes one or two cache line misses in IPv6 early demux (TCP/UDP)

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 0c0a5ef8

include/linux/ipv6.h

+0 −1
Original line number Diff line number Diff line
@@ -282,7 +282,6 @@ struct ipv6_pinfo { 
	__be32			rcv_flowinfo;



	__u32			dst_cookie;

	__u32			rx_dst_cookie;



	struct ipv6_mc_socklist	__rcu *ipv6_mc_list;

	struct ipv6_ac_socklist	*ipv6_ac_list;

include/net/sock.h

+2 −0
Original line number Diff line number Diff line
@@ -260,6 +260,7 @@ struct bpf_local_storage; 
  *	@sk_wq: sock wait queue and async head

  *	@sk_rx_dst: receive input route used by early demux

  *	@sk_rx_dst_ifindex: ifindex for @sk_rx_dst

  *	@sk_rx_dst_cookie: cookie for @sk_rx_dst

  *	@sk_dst_cache: destination cache

  *	@sk_dst_pending_confirm: need to confirm neighbour

  *	@sk_policy: flow policy
@@ -432,6 +433,7 @@ struct sock { 
#endif

	struct dst_entry	*sk_rx_dst;

	int			sk_rx_dst_ifindex;

	u32			sk_rx_dst_cookie;



	struct dst_entry __rcu	*sk_dst_cache;

	atomic_t		sk_omem_alloc;

net/ipv6/tcp_ipv6.c

+3 −3
Original line number Diff line number Diff line
@@ -109,7 +109,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 


		sk->sk_rx_dst = dst;

		sk->sk_rx_dst_ifindex = skb->skb_iif;

		tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);

		sk->sk_rx_dst_cookie = rt6_get_cookie(rt);

	}

}
@@ -1511,7 +1511,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 
		if (dst) {

			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||

			    INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,

					    dst, np->rx_dst_cookie) == NULL) {

					    dst, sk->sk_rx_dst_cookie) == NULL) {

				dst_release(dst);

				sk->sk_rx_dst = NULL;

			}
@@ -1872,7 +1872,7 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb) 
			struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);



			if (dst)

				dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);

				dst = dst_check(dst, sk->sk_rx_dst_cookie);

			if (dst &&

			    sk->sk_rx_dst_ifindex == skb->skb_iif)

				skb_dst_set_noref(skb, dst);

net/ipv6/udp.c

+2 −2
Original line number Diff line number Diff line
@@ -884,7 +884,7 @@ static void udp6_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst) 
	if (udp_sk_rx_dst_set(sk, dst)) {

		const struct rt6_info *rt = (const struct rt6_info *)dst;



		inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);

		sk->sk_rx_dst_cookie = rt6_get_cookie(rt);

	}

}
@@ -1073,7 +1073,7 @@ INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb) 
	dst = READ_ONCE(sk->sk_rx_dst);



	if (dst)

		dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie);

		dst = dst_check(dst, sk->sk_rx_dst_cookie);

	if (dst) {

		/* set noref for now.

		 * any place which wants to hold dst has to call