Commit 05cb8b39 authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'net-use-kmem_cache_free_bulk-in-kfree_skb_list' 

Jesper Dangaard Brouer says:

====================
net: use kmem_cache_free_bulk in kfree_skb_list

The kfree_skb_list function walks SKB (via skb->next) and frees them
individually to the SLUB/SLAB allocator (kmem_cache). It is more
efficient to bulk free them via the kmem_cache_free_bulk API.

Netstack NAPI fastpath already uses kmem_cache bulk alloc and free
APIs for SKBs.

The kfree_skb_list call got an interesting optimization in commit
520ac30f ("net_sched: drop packets after root qdisc lock is
released") that can create a list of SKBs "to_free" e.g. when qdisc
enqueue fails or deliberately chooses to drop . It isn't a normal data
fastpath, but the situation will likely occur when system/qdisc are
under heavy workloads, thus it makes sense to use a faster API for
freeing the SKBs.

E.g. the (often distro default) qdisc fq_codel will drop batches of
packets from fattest elephant flow, default capped at 64 packets (but
adjustable via tc argument drop_batch).

Performance measurements done in [1]:
 [1] https://github.com/xdp-project/xdp-project/blob/master/areas/mem/kfree_skb_list01.org
====================

Link: https://lore.kernel.org/r/167361788585.531803.686364041841425360.stgit@firesoul


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 501543b4 eedade12

net/core/skbuff.c

+57 −11
Original line number Diff line number Diff line
@@ -930,6 +930,21 @@ void __kfree_skb(struct sk_buff *skb) 
}

EXPORT_SYMBOL(__kfree_skb);



static __always_inline

bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)

{

	if (unlikely(!skb_unref(skb)))

		return false;



	DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX);



	if (reason == SKB_CONSUMED)

		trace_consume_skb(skb);

	else

		trace_kfree_skb(skb, __builtin_return_address(0), reason);

	return true;

}



/**

 *	kfree_skb_reason - free an sk_buff with special reason

 *	@skb: buffer to free
@@ -942,28 +957,59 @@ EXPORT_SYMBOL(__kfree_skb); 
void __fix_address

kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)

{

	if (unlikely(!skb_unref(skb)))

		return;



	DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX);



	if (reason == SKB_CONSUMED)

		trace_consume_skb(skb);

	else

		trace_kfree_skb(skb, __builtin_return_address(0), reason);

	if (__kfree_skb_reason(skb, reason))

		__kfree_skb(skb);

}

EXPORT_SYMBOL(kfree_skb_reason);



void kfree_skb_list_reason(struct sk_buff *segs,

#define KFREE_SKB_BULK_SIZE	16



struct skb_free_array {

	unsigned int skb_count;

	void *skb_array[KFREE_SKB_BULK_SIZE];

};



static void kfree_skb_add_bulk(struct sk_buff *skb,

			       struct skb_free_array *sa,

			       enum skb_drop_reason reason)

{

	/* if SKB is a clone, don't handle this case */

	if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {

		__kfree_skb(skb);

		return;

	}



	skb_release_all(skb, reason);

	sa->skb_array[sa->skb_count++] = skb;



	if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {

		kmem_cache_free_bulk(skbuff_head_cache, KFREE_SKB_BULK_SIZE,

				     sa->skb_array);

		sa->skb_count = 0;

	}

}



void __fix_address

kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)

{

	struct skb_free_array sa;



	sa.skb_count = 0;



	while (segs) {

		struct sk_buff *next = segs->next;



		kfree_skb_reason(segs, reason);

		skb_mark_not_on_list(segs);



		if (__kfree_skb_reason(segs, reason))

			kfree_skb_add_bulk(segs, &sa, reason);



		segs = next;

	}



	if (sa.skb_count)

		kmem_cache_free_bulk(skbuff_head_cache, sa.skb_count,

				     sa.skb_array);

}

EXPORT_SYMBOL(kfree_skb_list_reason);