Commit 9efb4b5b authored by Paolo Abeni's avatar Paolo Abeni Committed by David S. Miller
Browse files

net: optimize GRO for the common case. 



After the previous patches, at GRO time, skb->slow_gro is
usually 0, unless the packets comes from some H/W offload
slowpath or tunnel.

We can optimize the GRO code assuming !skb->slow_gro is likely.
This remove multiple conditionals in the most common path, at the
price of an additional one when we hit the above "slow-paths".

Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent b0999f38

net/core/dev.c

+22 −8
Original line number Diff line number Diff line
@@ -6014,7 +6014,6 @@ static void gro_list_prepare(const struct list_head *head, 
		diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);

		if (skb_vlan_tag_present(p))

			diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);

		diffs |= skb_metadata_dst_cmp(p, skb);

		diffs |= skb_metadata_differs(p, skb);

		if (maclen == ETH_HLEN)

			diffs |= compare_ether_header(skb_mac_header(p),
@@ -6024,17 +6023,29 @@ static void gro_list_prepare(const struct list_head *head, 
				       skb_mac_header(skb),

				       maclen);



		/* in most common scenarions _state is 0

		 * otherwise we are already on some slower paths

		 * either skip all the infrequent tests altogether or

		 * avoid trying too hard to skip each of them individually

		 */

		if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {

#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)

			struct tc_skb_ext *skb_ext;

			struct tc_skb_ext *p_ext;

#endif



			diffs |= skb_metadata_dst_cmp(p, skb);

			diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);



#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)

		if (!diffs) {

			struct tc_skb_ext *skb_ext = skb_ext_find(skb, TC_SKB_EXT);

			struct tc_skb_ext *p_ext = skb_ext_find(p, TC_SKB_EXT);

			skb_ext = skb_ext_find(skb, TC_SKB_EXT);

			p_ext = skb_ext_find(p, TC_SKB_EXT);



			diffs |= (!!p_ext) ^ (!!skb_ext);

			if (!diffs && unlikely(skb_ext))

				diffs |= p_ext->chain ^ skb_ext->chain;

		}

#endif

		}



		NAPI_GRO_CB(p)->same_flow = !diffs;

	}
@@ -6299,8 +6310,11 @@ static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 
	skb->encapsulation = 0;

	skb_shinfo(skb)->gso_type = 0;

	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));

	if (unlikely(skb->slow_gro)) {

		skb_ext_reset(skb);

		nf_reset_ct(skb);

		skb->slow_gro = 0;

	}



	napi->skb = skb;

}

net/core/skbuff.c

+6 −3
Original line number Diff line number Diff line
@@ -954,9 +954,12 @@ void __kfree_skb_defer(struct sk_buff *skb) 


void napi_skb_free_stolen_head(struct sk_buff *skb)

{

	if (unlikely(skb->slow_gro)) {

		nf_reset_ct(skb);

		skb_dst_drop(skb);

		skb_ext_put(skb);

		skb->slow_gro = 0;

	}

	napi_skb_cache_put(skb);

}