Commit 4915d50e authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

inet: add READ_ONCE(sk->sk_bound_dev_if) in INET_MATCH() 



INET_MATCH() runs without holding a lock on the socket.

We probably need to annotate most reads.

This patch makes INET_MATCH() an inline function
to ease our changes.

v2:

We remove the 32bit version of it, as modern compilers
should generate the same code really, no need to
try to be smarter.

Also make 'struct net *net' the first argument.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 49bb39bd

include/net/inet_hashtables.h

+16 −19
Original line number Diff line number Diff line
@@ -255,7 +255,6 @@ static inline struct sock *inet_lookup_listener(struct net *net, 
	((__force __portpair)(((__u32)(__dport) << 16) | (__force __u32)(__be16)(__sport)))

#endif



#if (BITS_PER_LONG == 64)

#ifdef __BIG_ENDIAN

#define INET_ADDR_COOKIE(__name, __saddr, __daddr) \

	const __addrpair __name = (__force __addrpair) ( \
@@ -267,24 +266,22 @@ static inline struct sock *inet_lookup_listener(struct net *net, 
				   (((__force __u64)(__be32)(__daddr)) << 32) | \

				   ((__force __u64)(__be32)(__saddr)))

#endif /* __BIG_ENDIAN */

#define INET_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif, __sdif) \

	(((__sk)->sk_portpair == (__ports))			&&	\

	 ((__sk)->sk_addrpair == (__cookie))			&&	\

	 (((__sk)->sk_bound_dev_if == (__dif))			||	\

	  ((__sk)->sk_bound_dev_if == (__sdif)))		&&	\

	 net_eq(sock_net(__sk), (__net)))

#else /* 32-bit arch */

#define INET_ADDR_COOKIE(__name, __saddr, __daddr) \

	const int __name __deprecated __attribute__((unused))



#define INET_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif, __sdif) \

	(((__sk)->sk_portpair == (__ports))		&&		\

	 ((__sk)->sk_daddr	== (__saddr))		&&		\

	 ((__sk)->sk_rcv_saddr	== (__daddr))		&&		\

	 (((__sk)->sk_bound_dev_if == (__dif))		||		\

	  ((__sk)->sk_bound_dev_if == (__sdif)))	&&		\

	 net_eq(sock_net(__sk), (__net)))

#endif /* 64-bit arch */



static inline bool INET_MATCH(struct net *net, const struct sock *sk,

			      const __addrpair cookie, const __portpair ports,

			      int dif, int sdif)

{

	int bound_dev_if;



	if (!net_eq(sock_net(sk), net) ||

	    sk->sk_portpair != ports ||

	    sk->sk_addrpair != cookie)

	        return false;



	/* Paired with WRITE_ONCE() from sock_bindtoindex_locked() */

	bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);

	return bound_dev_if == dif || bound_dev_if == sdif;

}



/* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need

 * not check it for lookups anymore, thanks Alexey. -DaveM

include/net/sock.h

+0 −3
Original line number Diff line number Diff line
@@ -161,9 +161,6 @@ typedef __u64 __bitwise __addrpair; 
 *	for struct sock and struct inet_timewait_sock.

 */

struct sock_common {

	/* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned

	 * address on 64bit arches : cf INET_MATCH()

	 */

	union {

		__addrpair	skc_addrpair;

		struct {

net/ipv4/inet_hashtables.c

+5 −10
Original line number Diff line number Diff line
@@ -373,13 +373,11 @@ struct sock *__inet_lookup_established(struct net *net, 
	sk_nulls_for_each_rcu(sk, node, &head->chain) {

		if (sk->sk_hash != hash)

			continue;

		if (likely(INET_MATCH(sk, net, acookie,

				      saddr, daddr, ports, dif, sdif))) {

		if (likely(INET_MATCH(net, sk, acookie, ports, dif, sdif))) {

			if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))

				goto out;

			if (unlikely(!INET_MATCH(sk, net, acookie,

						 saddr, daddr, ports,

						 dif, sdif))) {

			if (unlikely(!INET_MATCH(net, sk, acookie,

						 ports, dif, sdif))) {

				sock_gen_put(sk);

				goto begin;

			}
@@ -428,8 +426,7 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row, 
		if (sk2->sk_hash != hash)

			continue;



		if (likely(INET_MATCH(sk2, net, acookie,

					 saddr, daddr, ports, dif, sdif))) {

		if (likely(INET_MATCH(net, sk2, acookie, ports, dif, sdif))) {

			if (sk2->sk_state == TCP_TIME_WAIT) {

				tw = inet_twsk(sk2);

				if (twsk_unique(sk, sk2, twp))
@@ -495,9 +492,7 @@ static bool inet_ehash_lookup_by_sk(struct sock *sk, 
		if (esk->sk_hash != sk->sk_hash)

			continue;

		if (sk->sk_family == AF_INET) {

			if (unlikely(INET_MATCH(esk, net, acookie,

						sk->sk_daddr,

						sk->sk_rcv_saddr,

			if (unlikely(INET_MATCH(net, esk, acookie,

						ports, dif, sdif))) {

				return true;

			}

net/ipv4/udp.c

+1 −2
Original line number Diff line number Diff line
@@ -2563,8 +2563,7 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net, 
	struct sock *sk;



	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {

		if (INET_MATCH(sk, net, acookie, rmt_addr,

			       loc_addr, ports, dif, sdif))

		if (INET_MATCH(net, sk, acookie, ports, dif, sdif))

			return sk;

		/* Only check first socket in chain */

		break;