Commit 405ac7fd authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'tcp-avoid-indirect-call-in-__sk_stream_memory_free' 

Eric Dumazet says:

====================
tcp: avoid indirect call in __sk_stream_memory_free()

Small improvement for CONFIG_RETPOLINE=y, when dealing with TCP sockets.
====================

Link: https://lore.kernel.org/r/20201113150809.3443527-1-eric.dumazet@gmail.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents ae8cb932 1c5f2ced

include/net/sock.h

+6 −2
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ 
#include <linux/rculist_nulls.h>

#include <linux/poll.h>

#include <linux/sockptr.h>



#include <linux/indirect_call_wrapper.h>

#include <linux/atomic.h>

#include <linux/refcount.h>

#include <net/dst.h>
@@ -1264,13 +1264,17 @@ static inline void sk_refcnt_debug_release(const struct sock *sk) 
#define sk_refcnt_debug_release(sk) do { } while (0)

#endif /* SOCK_REFCNT_DEBUG */



INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));



static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)

{

	if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))

		return false;



	return sk->sk_prot->stream_memory_free ?

		sk->sk_prot->stream_memory_free(sk, wake) : true;

		INDIRECT_CALL_1(sk->sk_prot->stream_memory_free,

			        tcp_stream_memory_free,

				sk, wake) : true;

}



static inline bool sk_stream_memory_free(const struct sock *sk)

include/net/tcp.h

+1 −12
Original line number Diff line number Diff line
@@ -1965,18 +1965,7 @@ static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 
	return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;

}



/* @wake is one when sk_stream_write_space() calls us.

 * This sends EPOLLOUT only if notsent_bytes is half the limit.

 * This mimics the strategy used in sock_def_write_space().

 */

static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)

{

	const struct tcp_sock *tp = tcp_sk(sk);

	u32 notsent_bytes = READ_ONCE(tp->write_seq) -

			    READ_ONCE(tp->snd_nxt);



	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);

}

bool tcp_stream_memory_free(const struct sock *sk, int wake);



#ifdef CONFIG_PROC_FS

int tcp4_proc_init(void);

net/ipv4/tcp_ipv4.c

+14 −0
Original line number Diff line number Diff line
@@ -2740,6 +2740,20 @@ void tcp4_proc_exit(void) 
}

#endif /* CONFIG_PROC_FS */



/* @wake is one when sk_stream_write_space() calls us.

 * This sends EPOLLOUT only if notsent_bytes is half the limit.

 * This mimics the strategy used in sock_def_write_space().

 */

bool tcp_stream_memory_free(const struct sock *sk, int wake)

{

	const struct tcp_sock *tp = tcp_sk(sk);

	u32 notsent_bytes = READ_ONCE(tp->write_seq) -

			    READ_ONCE(tp->snd_nxt);



	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);

}

EXPORT_SYMBOL(tcp_stream_memory_free);



struct proto tcp_prot = {

	.name			= "TCP",

	.owner			= THIS_MODULE,