Commit d9e78914 authored by Florian Westphal's avatar Florian Westphal
Browse files

netfilter: nf_tables: avoid retpoline overhead for some ct expression calls 



nft_ct expression cannot be made builtin to nf_tables without also
forcing the conntrack itself to be builtin.

However, this can be avoided by splitting retrieval of a few
selector keys that only need to access the nf_conn structure,
i.e. no function calls to nf_conntrack code.

Many rulesets start with something like
"ct status established,related accept"

With this change, this no longer requires an indirect call, which
gives about 1.8% more throughput with a simple conntrack-enabled
forwarding test (retpoline thunk used).

Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
parent 2032e907

include/net/netfilter/nf_tables_core.h

+12 −0
Original line number Diff line number Diff line
@@ -61,6 +61,16 @@ struct nft_immediate_expr { 
extern const struct nft_expr_ops nft_cmp_fast_ops;

extern const struct nft_expr_ops nft_cmp16_fast_ops;



struct nft_ct {

	enum nft_ct_keys	key:8;

	enum ip_conntrack_dir	dir:8;

	u8			len;

	union {

		u8		dreg;

		u8		sreg;

	};

};



struct nft_payload {

	enum nft_payload_bases	base:8;

	u8			offset;
@@ -140,6 +150,8 @@ void nft_rt_get_eval(const struct nft_expr *expr, 
		     struct nft_regs *regs, const struct nft_pktinfo *pkt);

void nft_counter_eval(const struct nft_expr *expr, struct nft_regs *regs,

                      const struct nft_pktinfo *pkt);

void nft_ct_get_fast_eval(const struct nft_expr *expr,

			  struct nft_regs *regs, const struct nft_pktinfo *pkt);



enum {

	NFT_PAYLOAD_CTX_INNER_TUN	= (1 << 0),

net/netfilter/Makefile

+6 −0
Original line number Diff line number Diff line
@@ -98,6 +98,12 @@ nf_tables-objs += nft_set_pipapo_avx2.o 
endif

endif



ifdef CONFIG_NFT_CT

ifdef CONFIG_RETPOLINE

nf_tables-objs += nft_ct_fast.o

endif

endif



obj-$(CONFIG_NF_TABLES)		+= nf_tables.o

obj-$(CONFIG_NFT_COMPAT)	+= nft_compat.o

obj-$(CONFIG_NFT_CONNLIMIT)	+= nft_connlimit.o

net/netfilter/nf_tables_core.c

+3 −0
Original line number Diff line number Diff line
@@ -228,6 +228,9 @@ static void expr_call_ops_eval(const struct nft_expr *expr, 
	X(e, nft_counter_eval);

	X(e, nft_meta_get_eval);

	X(e, nft_lookup_eval);

#if IS_ENABLED(CONFIG_NFT_CT)

	X(e, nft_ct_get_fast_eval);

#endif

	X(e, nft_range_eval);

	X(e, nft_immediate_eval);

	X(e, nft_byteorder_eval);

net/netfilter/nft_ct.c

+27 −12
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
#include <linux/netlink.h>

#include <linux/netfilter.h>

#include <linux/netfilter/nf_tables.h>

#include <net/netfilter/nf_tables.h>

#include <net/netfilter/nf_tables_core.h>

#include <net/netfilter/nf_conntrack.h>

#include <net/netfilter/nf_conntrack_acct.h>

#include <net/netfilter/nf_conntrack_tuple.h>
@@ -23,16 +23,6 @@ 
#include <net/netfilter/nf_conntrack_l4proto.h>

#include <net/netfilter/nf_conntrack_expect.h>



struct nft_ct {

	enum nft_ct_keys	key:8;

	enum ip_conntrack_dir	dir:8;

	u8			len;

	union {

		u8		dreg;

		u8		sreg;

	};

};



struct nft_ct_helper_obj  {

	struct nf_conntrack_helper *helper4;

	struct nf_conntrack_helper *helper6;
@@ -759,6 +749,18 @@ static bool nft_ct_set_reduce(struct nft_regs_track *track, 
	return false;

}



#ifdef CONFIG_RETPOLINE

static const struct nft_expr_ops nft_ct_get_fast_ops = {

	.type		= &nft_ct_type,

	.size		= NFT_EXPR_SIZE(sizeof(struct nft_ct)),

	.eval		= nft_ct_get_fast_eval,

	.init		= nft_ct_get_init,

	.destroy	= nft_ct_get_destroy,

	.dump		= nft_ct_get_dump,

	.reduce		= nft_ct_set_reduce,

};

#endif



static const struct nft_expr_ops nft_ct_set_ops = {

	.type		= &nft_ct_type,

	.size		= NFT_EXPR_SIZE(sizeof(struct nft_ct)),
@@ -791,8 +793,21 @@ nft_ct_select_ops(const struct nft_ctx *ctx, 
	if (tb[NFTA_CT_DREG] && tb[NFTA_CT_SREG])

		return ERR_PTR(-EINVAL);



	if (tb[NFTA_CT_DREG])

	if (tb[NFTA_CT_DREG]) {

#ifdef CONFIG_RETPOLINE

		u32 k = ntohl(nla_get_be32(tb[NFTA_CT_KEY]));



		switch (k) {

		case NFT_CT_STATE:

		case NFT_CT_DIRECTION:

		case NFT_CT_STATUS:

		case NFT_CT_MARK:

		case NFT_CT_SECMARK:

			return &nft_ct_get_fast_ops;

		}

#endif

		return &nft_ct_get_ops;

	}



	if (tb[NFTA_CT_SREG]) {

#ifdef CONFIG_NF_CONNTRACK_ZONES

net/netfilter/nft_ct_fast.c

0 → 100644
+56 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

#if IS_ENABLED(CONFIG_NFT_CT)

#include <linux/netfilter/nf_tables.h>

#include <net/netfilter/nf_tables_core.h>

#include <net/netfilter/nf_conntrack.h>



void nft_ct_get_fast_eval(const struct nft_expr *expr,

			  struct nft_regs *regs,

			  const struct nft_pktinfo *pkt)

{

	const struct nft_ct *priv = nft_expr_priv(expr);

	u32 *dest = &regs->data[priv->dreg];

	enum ip_conntrack_info ctinfo;

	const struct nf_conn *ct;

	unsigned int state;



	ct = nf_ct_get(pkt->skb, &ctinfo);

	if (!ct) {

		regs->verdict.code = NFT_BREAK;

		return;

	}



	switch (priv->key) {

	case NFT_CT_STATE:

		if (ct)

			state = NF_CT_STATE_BIT(ctinfo);

		else if (ctinfo == IP_CT_UNTRACKED)

			state = NF_CT_STATE_UNTRACKED_BIT;

		else

			state = NF_CT_STATE_INVALID_BIT;

		*dest = state;

		return;

	case NFT_CT_DIRECTION:

		nft_reg_store8(dest, CTINFO2DIR(ctinfo));

		return;

	case NFT_CT_STATUS:

		*dest = ct->status;

		return;

#ifdef CONFIG_NF_CONNTRACK_MARK

	case NFT_CT_MARK:

		*dest = ct->mark;

		return;

#endif

#ifdef CONFIG_NF_CONNTRACK_SECMARK

	case NFT_CT_SECMARK:

		*dest = ct->secmark;

		return;

#endif

	default:

		WARN_ON_ONCE(1);

		regs->verdict.code = NFT_BREAK;

		break;

	}

}

EXPORT_SYMBOL_GPL(nft_ct_get_fast_eval);

#endif