Commit 4fa37e49 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'tcp-introduce-optional-per-netns-ehash' 

Kuniyuki Iwashima says:

====================
tcp: Introduce optional per-netns ehash.

The more sockets we have in the hash table, the longer we spend looking
up the socket.  While running a number of small workloads on the same
host, they penalise each other and cause performance degradation.

The root cause might be a single workload that consumes much more
resources than the others.  It often happens on a cloud service where
different workloads share the same computing resource.

On EC2 c5.24xlarge instance (196 GiB memory and 524288 (1Mi / 2) ehash
entries), after running iperf3 in different netns, creating 24Mi sockets
without data transfer in the root netns causes about 10% performance
regression for the iperf3's connection.

 thash_entries		sockets		length		Gbps
	524288		      1		     1		50.7
			   24Mi		    48		45.1

It is basically related to the length of the list of each hash bucket.
For testing purposes to see how performance drops along the length,
I set 131072 (1Mi / 8) to thash_entries, and here's the result.

 thash_entries		sockets		length		Gbps
        131072		      1		     1		50.7
			    1Mi		     8		49.9
			    2Mi		    16		48.9
			    4Mi		    32		47.3
			    8Mi		    64		44.6
			   16Mi		   128		40.6
			   24Mi		   192		36.3
			   32Mi		   256		32.5
			   40Mi		   320		27.0
			   48Mi		   384		25.0

To resolve the socket lookup degradation, we introduce an optional
per-netns hash table for TCP, but it's just ehash, and we still share
the global bhash, bhash2 and lhash2.

With a smaller ehash, we can look up non-listener sockets faster and
isolate such noisy neighbours.  Also, we can reduce lock contention.

For details, please see the last patch.

  patch 1 - 4: prep for per-netns ehash
  patch     5: small optimisation for netns dismantle without TIME_WAIT sockets
  patch     6: add per-netns ehash

Many thanks to Eric Dumazet for reviewing and advising.
====================

Link: https://lore.kernel.org/r/20220908011022.45342-1-kuniyu@amazon.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 17df341d d1e5e640

Documentation/networking/ip-sysctl.rst

+29 −0
Original line number Diff line number Diff line
@@ -1040,6 +1040,35 @@ tcp_challenge_ack_limit - INTEGER 
	TCP stack implements per TCP socket limits anyway.

	Default: INT_MAX (unlimited)



tcp_ehash_entries - INTEGER

	Show the number of hash buckets for TCP sockets in the current

	networking namespace.



	A negative value means the networking namespace does not own its

	hash buckets and shares the initial networking namespace's one.



tcp_child_ehash_entries - INTEGER

	Control the number of hash buckets for TCP sockets in the child

	networking namespace, which must be set before clone() or unshare().



	If the value is not 0, the kernel uses a value rounded up to 2^n

	as the actual hash bucket size.  0 is a special value, meaning

	the child networking namespace will share the initial networking

	namespace's hash buckets.



	Note that the child will use the global one in case the kernel

	fails to allocate enough memory.  In addition, the global hash

	buckets are spread over available NUMA nodes, but the allocation

	of the child hash table depends on the current process's NUMA

	policy, which could result in performance differences.



	Note also that the default value of tcp_max_tw_buckets and

	tcp_max_syn_backlog depend on the hash bucket size.



	Possible values: 0, 2^n (n: 0 - 24 (16Mi))



	Default: 0



UDP variables

=============

drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c

+2 −3
Original line number Diff line number Diff line
@@ -1069,8 +1069,7 @@ static void chtls_pass_accept_rpl(struct sk_buff *skb, 
	cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);

}



static void inet_inherit_port(struct inet_hashinfo *hash_info,

			      struct sock *lsk, struct sock *newsk)

static void inet_inherit_port(struct sock *lsk, struct sock *newsk)

{

	local_bh_disable();

	__inet_inherit_port(lsk, newsk);
@@ -1240,7 +1239,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk, 
						     ipv4.sysctl_tcp_window_scaling),

					   tp->window_clamp);

	neigh_release(n);

	inet_inherit_port(&tcp_hashinfo, lsk, newsk);

	inet_inherit_port(lsk, newsk);

	csk_set_flag(csk, CSK_CONN_INLINE);

	bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c

+3 −2
Original line number Diff line number Diff line
@@ -461,6 +461,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) 
{

	struct ethhdr *eth = (struct ethhdr *)(skb->data);

	struct net_device *netdev = rq->netdev;

	struct net *net = dev_net(netdev);

	struct sock *sk = NULL;

	unsigned int datalen;

	struct iphdr *iph;
@@ -475,7 +476,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) 
		depth += sizeof(struct iphdr);

		th = (void *)iph + sizeof(struct iphdr);



		sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,

		sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,

					     iph->saddr, th->source, iph->daddr,

					     th->dest, netdev->ifindex);

#if IS_ENABLED(CONFIG_IPV6)
@@ -485,7 +486,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) 
		depth += sizeof(struct ipv6hdr);

		th = (void *)ipv6h + sizeof(struct ipv6hdr);



		sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,

		sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,

						&ipv6h->saddr, th->source,

						&ipv6h->daddr, ntohs(th->dest),

						netdev->ifindex, 0);

drivers/net/ethernet/netronome/nfp/crypto/tls.c

+3 −2
Original line number Diff line number Diff line
@@ -474,6 +474,7 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev, 
{

	struct nfp_net *nn = netdev_priv(netdev);

	struct nfp_net_tls_offload_ctx *ntls;

	struct net *net = dev_net(netdev);

	struct ipv6hdr *ipv6h;

	struct tcphdr *th;

	struct iphdr *iph;
@@ -494,13 +495,13 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev, 


	switch (ipv6h->version) {

	case 4:

		sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,

		sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,

					     iph->saddr, th->source, iph->daddr,

					     th->dest, netdev->ifindex);

		break;

#if IS_ENABLED(CONFIG_IPV6)

	case 6:

		sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,

		sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,

						&ipv6h->saddr, th->source,

						&ipv6h->daddr, ntohs(th->dest),

						netdev->ifindex, 0);

include/net/inet_hashtables.h

+16 −0
Original line number Diff line number Diff line
@@ -168,8 +168,20 @@ struct inet_hashinfo { 
	/* The 2nd listener table hashed by local port and address */

	unsigned int			lhash2_mask;

	struct inet_listen_hashbucket	*lhash2;



	bool				pernet;

};



static inline struct inet_hashinfo *tcp_or_dccp_get_hashinfo(const struct sock *sk)

{

#if IS_ENABLED(CONFIG_IP_DCCP)

	return sk->sk_prot->h.hashinfo ? :

		sock_net(sk)->ipv4.tcp_death_row.hashinfo;

#else

	return sock_net(sk)->ipv4.tcp_death_row.hashinfo;

#endif

}



static inline struct inet_listen_hashbucket *

inet_lhash2_bucket(struct inet_hashinfo *h, u32 hash)

{
@@ -204,6 +216,10 @@ static inline void inet_ehash_locks_free(struct inet_hashinfo *hashinfo) 
	hashinfo->ehash_locks = NULL;

}



struct inet_hashinfo *inet_pernet_hashinfo_alloc(struct inet_hashinfo *hashinfo,

						 unsigned int ehash_entries);

void inet_pernet_hashinfo_free(struct inet_hashinfo *hashinfo);



struct inet_bind_bucket *

inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net,

			struct inet_bind_hashbucket *head,