Commit 0a8afd9f authored by Pavel Begunkov's avatar Pavel Begunkov Committed by David S. Miller
Browse files

sock: optimise sock_def_write_space barriers 



Now we have a separate path for sock_def_write_space() and can go one
step further. When it's called from sock_wfree() we know that there is a
preceding atomic for putting down ->sk_wmem_alloc. We can use it to
replace to replace smb_mb() with a less expensive
smp_mb__after_atomic(). It also removes an extra RCU read lock/unlock as
a small bonus.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 052ada09

net/core/sock.c

+25 −1
Original line number Diff line number Diff line
@@ -146,6 +146,7 @@ 
static DEFINE_MUTEX(proto_list_mutex);

static LIST_HEAD(proto_list);



static void sock_def_write_space_wfree(struct sock *sk);

static void sock_def_write_space(struct sock *sk);



/**
@@ -2333,7 +2334,7 @@ void sock_wfree(struct sk_buff *skb) 
		    sk->sk_write_space == sock_def_write_space) {

			rcu_read_lock();

			free = refcount_sub_and_test(len, &sk->sk_wmem_alloc);

			sock_def_write_space(sk);

			sock_def_write_space_wfree(sk);

			rcu_read_unlock();

			if (unlikely(free))

				__sk_free(sk);
@@ -3218,6 +3219,29 @@ static void sock_def_write_space(struct sock *sk) 
	rcu_read_unlock();

}



/* An optimised version of sock_def_write_space(), should only be called

 * for SOCK_RCU_FREE sockets under RCU read section and after putting

 * ->sk_wmem_alloc.

 */

static void sock_def_write_space_wfree(struct sock *sk)

{

	/* Do not wake up a writer until he can make "significant"

	 * progress.  --DaveM

	 */

	if (sock_writeable(sk)) {

		struct socket_wq *wq = rcu_dereference(sk->sk_wq);



		/* rely on refcount_sub from sock_wfree() */

		smp_mb__after_atomic();

		if (wq && waitqueue_active(&wq->wait))

			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |

						EPOLLWRNORM | EPOLLWRBAND);



		/* Should agree with poll, otherwise some programs break */

		sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);

	}

}



static void sock_def_destruct(struct sock *sk)

{

}