Commit 1485348d authored by Ben Hutchings's avatar Ben Hutchings Committed by David S. Miller
Browse files

tcp: Apply device TSO segment limit earlier 



Cache the device gso_max_segs in sock::sk_gso_max_segs and use it to
limit the size of TSO skbs.  This avoids the need to fall back to
software GSO for local TCP senders.

Signed-off-by: default avatarBen Hutchings <bhutchings@solarflare.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7e6d06f0

include/net/sock.h

+2 −0
Original line number Diff line number Diff line
@@ -218,6 +218,7 @@ struct cg_proto; 
  *	@sk_route_nocaps: forbidden route capabilities (e.g NETIF_F_GSO_MASK)

  *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)

  *	@sk_gso_max_size: Maximum GSO segment size to build

  *	@sk_gso_max_segs: Maximum number of GSO segments

  *	@sk_lingertime: %SO_LINGER l_linger setting

  *	@sk_backlog: always used with the per-socket spinlock held

  *	@sk_callback_lock: used with the callbacks in the end of this struct
@@ -338,6 +339,7 @@ struct sock { 
	netdev_features_t	sk_route_nocaps;

	int			sk_gso_type;

	unsigned int		sk_gso_max_size;

	u16			sk_gso_max_segs;

	int			sk_rcvlowat;

	unsigned long	        sk_lingertime;

	struct sk_buff_head	sk_error_queue;

net/core/sock.c

+1 −0
Original line number Diff line number Diff line
@@ -1458,6 +1458,7 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 
		} else {

			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;

			sk->sk_gso_max_size = dst->dev->gso_max_size;

			sk->sk_gso_max_segs = dst->dev->gso_max_segs;

		}

	}

}

net/ipv4/tcp.c

+3 −1
Original line number Diff line number Diff line
@@ -811,7 +811,9 @@ static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 
			   old_size_goal + mss_now > xmit_size_goal)) {

			xmit_size_goal = old_size_goal;

		} else {

			tp->xmit_size_goal_segs = xmit_size_goal / mss_now;

			tp->xmit_size_goal_segs =

				min_t(u16, xmit_size_goal / mss_now,

				      sk->sk_gso_max_segs);

			xmit_size_goal = tp->xmit_size_goal_segs * mss_now;

		}

	}

net/ipv4/tcp_cong.c

+2 −1
Original line number Diff line number Diff line
@@ -291,7 +291,8 @@ bool tcp_is_cwnd_limited(const struct sock *sk, u32 in_flight) 
	left = tp->snd_cwnd - in_flight;

	if (sk_can_gso(sk) &&

	    left * sysctl_tcp_tso_win_divisor < tp->snd_cwnd &&

	    left * tp->mss_cache < sk->sk_gso_max_size)

	    left * tp->mss_cache < sk->sk_gso_max_size &&

	    left < sk->sk_gso_max_segs)

		return true;

	return left <= tcp_max_tso_deferred_mss(tp);

}

net/ipv4/tcp_output.c

+12 −9
Original line number Diff line number Diff line
@@ -1522,21 +1522,21 @@ static void tcp_cwnd_validate(struct sock *sk) 
 * when we would be allowed to send the split-due-to-Nagle skb fully.

 */

static unsigned int tcp_mss_split_point(const struct sock *sk, const struct sk_buff *skb,

					unsigned int mss_now, unsigned int cwnd)

					unsigned int mss_now, unsigned int max_segs)

{

	const struct tcp_sock *tp = tcp_sk(sk);

	u32 needed, window, cwnd_len;

	u32 needed, window, max_len;



	window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;

	cwnd_len = mss_now * cwnd;

	max_len = mss_now * max_segs;



	if (likely(cwnd_len <= window && skb != tcp_write_queue_tail(sk)))

		return cwnd_len;

	if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))

		return max_len;



	needed = min(skb->len, window);



	if (cwnd_len <= needed)

		return cwnd_len;

	if (max_len <= needed)

		return max_len;



	return needed - needed % mss_now;

}
@@ -1765,7 +1765,8 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb) 
	limit = min(send_win, cong_win);



	/* If a full-sized TSO skb can be sent, do it. */

	if (limit >= sk->sk_gso_max_size)

	if (limit >= min_t(unsigned int, sk->sk_gso_max_size,

			   sk->sk_gso_max_segs * tp->mss_cache))

		goto send_now;



	/* Middle in queue won't get any more data, full sendable already? */
@@ -1999,7 +2000,9 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 
		limit = mss_now;

		if (tso_segs > 1 && !tcp_urg_mode(tp))

			limit = tcp_mss_split_point(sk, skb, mss_now,

						    cwnd_quota);

						    min_t(unsigned int,

							  cwnd_quota,

							  sk->sk_gso_max_segs));



		if (skb->len > limit &&

		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))