Commit 0c0a5ef8 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

tcp: move inet->rx_dst_ifindex to sk->sk_rx_dst_ifindex 



Increase cache locality by moving rx_dst_ifindex next to sk->sk_rx_dst

This is part of an effort to reduce cache line misses in TCP fast path.

This removes one cache line miss in early demux.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent fd559a94

include/net/inet_sock.h

+1 −2
Original line number Diff line number Diff line
@@ -207,11 +207,10 @@ struct inet_sock { 
	__be32			inet_saddr;

	__s16			uc_ttl;

	__u16			cmsg_flags;

	struct ip_options_rcu __rcu	*inet_opt;

	__be16			inet_sport;

	__u16			inet_id;



	struct ip_options_rcu __rcu	*inet_opt;

	int			rx_dst_ifindex;

	__u8			tos;

	__u8			min_ttl;

	__u8			mc_ttl;

include/net/sock.h

+3 −0
Original line number Diff line number Diff line
@@ -259,6 +259,7 @@ struct bpf_local_storage; 
  *	@sk_rcvbuf: size of receive buffer in bytes

  *	@sk_wq: sock wait queue and async head

  *	@sk_rx_dst: receive input route used by early demux

  *	@sk_rx_dst_ifindex: ifindex for @sk_rx_dst

  *	@sk_dst_cache: destination cache

  *	@sk_dst_pending_confirm: need to confirm neighbour

  *	@sk_policy: flow policy
@@ -430,6 +431,8 @@ struct sock { 
	struct xfrm_policy __rcu *sk_policy[2];

#endif

	struct dst_entry	*sk_rx_dst;

	int			sk_rx_dst_ifindex;



	struct dst_entry __rcu	*sk_dst_cache;

	atomic_t		sk_omem_alloc;

	int			sk_sndbuf;

net/ipv4/tcp_ipv4.c

+3 −3
Original line number Diff line number Diff line
@@ -1703,7 +1703,7 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb) 
		sock_rps_save_rxhash(sk, skb);

		sk_mark_napi_id(sk, skb);

		if (dst) {

			if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||

			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||

			    !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,

					     dst, 0)) {

				dst_release(dst);
@@ -1788,7 +1788,7 @@ int tcp_v4_early_demux(struct sk_buff *skb) 
			if (dst)

				dst = dst_check(dst, 0);

			if (dst &&

			    inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)

			    sk->sk_rx_dst_ifindex == skb->skb_iif)

				skb_dst_set_noref(skb, dst);

		}

	}
@@ -2195,7 +2195,7 @@ void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 


	if (dst && dst_hold_safe(dst)) {

		sk->sk_rx_dst = dst;

		inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;

		sk->sk_rx_dst_ifindex = skb->skb_iif;

	}

}

EXPORT_SYMBOL(inet_sk_rx_dst_set);

net/ipv6/tcp_ipv6.c

+3 −3
Original line number Diff line number Diff line
@@ -108,7 +108,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 
		const struct rt6_info *rt = (const struct rt6_info *)dst;



		sk->sk_rx_dst = dst;

		inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;

		sk->sk_rx_dst_ifindex = skb->skb_iif;

		tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);

	}

}
@@ -1509,7 +1509,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 
		sock_rps_save_rxhash(sk, skb);

		sk_mark_napi_id(sk, skb);

		if (dst) {

			if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||

			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||

			    INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,

					    dst, np->rx_dst_cookie) == NULL) {

				dst_release(dst);
@@ -1874,7 +1874,7 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb) 
			if (dst)

				dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);

			if (dst &&

			    inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)

			    sk->sk_rx_dst_ifindex == skb->skb_iif)

				skb_dst_set_noref(skb, dst);

		}

	}