Commit 07df3fc9 authored by Jason Baron's avatar Jason Baron Committed by Pablo Neira Ayuso
Browse files

netfilter: x_tables: improve limit_mt scalability 



We've seen this spin_lock show up high in profiles. Let's introduce a
lockless version. I've tested this using pktgen_sample01_simple.sh.

Signed-off-by: default avatarJason Baron <jbaron@akamai.com>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent 06f02993

net/netfilter/xt_limit.c

+26 −20
Original line number Diff line number Diff line
@@ -8,16 +8,14 @@ 
#include <linux/slab.h>

#include <linux/module.h>

#include <linux/skbuff.h>

#include <linux/spinlock.h>

#include <linux/interrupt.h>



#include <linux/netfilter/x_tables.h>

#include <linux/netfilter/xt_limit.h>



struct xt_limit_priv {

	spinlock_t lock;

	unsigned long prev;

	uint32_t credit;

	u32 credit;

};



MODULE_LICENSE("GPL");
@@ -66,22 +64,31 @@ limit_mt(const struct sk_buff *skb, struct xt_action_param *par) 
{

	const struct xt_rateinfo *r = par->matchinfo;

	struct xt_limit_priv *priv = r->master;

	unsigned long now = jiffies;



	spin_lock_bh(&priv->lock);

	priv->credit += (now - xchg(&priv->prev, now)) * CREDITS_PER_JIFFY;

	if (priv->credit > r->credit_cap)

		priv->credit = r->credit_cap;



	if (priv->credit >= r->cost) {

		/* We're not limited. */

		priv->credit -= r->cost;

		spin_unlock_bh(&priv->lock);

		return true;

	}

	unsigned long now;

	u32 old_credit, new_credit, credit_increase = 0;

	bool ret;



	spin_unlock_bh(&priv->lock);

	/* fastpath if there is nothing to update */

	if ((READ_ONCE(priv->credit) < r->cost) && (READ_ONCE(priv->prev) == jiffies))

		return false;



	do {

		now = jiffies;

		credit_increase += (now - xchg(&priv->prev, now)) * CREDITS_PER_JIFFY;

		old_credit = READ_ONCE(priv->credit);

		new_credit = old_credit;

		new_credit += credit_increase;

		if (new_credit > r->credit_cap)

			new_credit = r->credit_cap;

		if (new_credit >= r->cost) {

			ret = true;

			new_credit -= r->cost;

		} else {

			ret = false;

		}

	} while (cmpxchg(&priv->credit, old_credit, new_credit) != old_credit);



	return ret;

}



/* Precision saver. */
@@ -122,7 +129,6 @@ static int limit_mt_check(const struct xt_mtchk_param *par) 
		r->credit_cap = priv->credit; /* Credits full. */

		r->cost = user2credits(r->avg);

	}

	spin_lock_init(&priv->lock);



	return 0;

}