Commit f227925e authored by Florian Westphal's avatar Florian Westphal Committed by Pablo Neira Ayuso
Browse files

netfilter: nf_tables: prefer direct calls for set lookups 



Extend nft_set_do_lookup() to use direct calls when retpoline feature
is enabled.

Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent 0974cff3

include/net/netfilter/nf_tables_core.h

+24 −0
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#define _NET_NF_TABLES_CORE_H



#include <net/netfilter/nf_tables.h>

#include <linux/indirect_call_wrapper.h>



extern struct nft_expr_type nft_imm_type;

extern struct nft_expr_type nft_cmp_type;
@@ -88,12 +89,35 @@ extern const struct nft_set_type nft_set_bitmap_type; 
extern const struct nft_set_type nft_set_pipapo_type;

extern const struct nft_set_type nft_set_pipapo_avx2_type;



#ifdef CONFIG_RETPOLINE

bool nft_rhash_lookup(const struct net *net, const struct nft_set *set,

		      const u32 *key, const struct nft_set_ext **ext);

bool nft_rbtree_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext);

bool nft_bitmap_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext);

bool nft_hash_lookup_fast(const struct net *net,

			  const struct nft_set *set,

			  const u32 *key, const struct nft_set_ext **ext);

bool nft_hash_lookup(const struct net *net, const struct nft_set *set,

		     const u32 *key, const struct nft_set_ext **ext);

bool nft_set_do_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext);

#else

static inline bool

nft_set_do_lookup(const struct net *net, const struct nft_set *set,

		  const u32 *key, const struct nft_set_ext **ext)

{

	return set->ops->lookup(net, set, key, ext);

}

#endif



/* called from nft_pipapo_avx2.c */

bool nft_pipapo_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext);

/* called from nft_set_pipapo.c */

bool nft_pipapo_avx2_lookup(const struct net *net, const struct nft_set *set,

			    const u32 *key, const struct nft_set_ext **ext);



struct nft_expr;

struct nft_regs;

net/netfilter/nft_lookup.c

+31 −0
Original line number Diff line number Diff line
@@ -23,6 +23,37 @@ struct nft_lookup { 
	struct nft_set_binding		binding;

};



#ifdef CONFIG_RETPOLINE

bool nft_set_do_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext)

{

	if (set->ops == &nft_set_hash_fast_type.ops)

		return nft_hash_lookup_fast(net, set, key, ext);

	if (set->ops == &nft_set_hash_type.ops)

		return nft_hash_lookup(net, set, key, ext);



	if (set->ops == &nft_set_rhash_type.ops)

		return nft_rhash_lookup(net, set, key, ext);



	if (set->ops == &nft_set_bitmap_type.ops)

		return nft_bitmap_lookup(net, set, key, ext);



	if (set->ops == &nft_set_pipapo_type.ops)

		return nft_pipapo_lookup(net, set, key, ext);

#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)

	if (set->ops == &nft_set_pipapo_avx2_type.ops)

		return nft_pipapo_avx2_lookup(net, set, key, ext);

#endif



	if (set->ops == &nft_set_rbtree_type.ops)

		return nft_rbtree_lookup(net, set, key, ext);



	WARN_ON_ONCE(1);

	return set->ops->lookup(net, set, key, ext);

}

EXPORT_SYMBOL_GPL(nft_set_do_lookup);

#endif



void nft_lookup_eval(const struct nft_expr *expr,

		     struct nft_regs *regs,

		     const struct nft_pktinfo *pkt)

net/netfilter/nft_set_bitmap.c

+3 −2
Original line number Diff line number Diff line
@@ -73,7 +73,8 @@ nft_bitmap_active(const u8 *bitmap, u32 idx, u32 off, u8 genmask) 
	return (bitmap[idx] & (0x3 << off)) & (genmask << off);

}



static bool nft_bitmap_lookup(const struct net *net, const struct nft_set *set,

INDIRECT_CALLABLE_SCOPE

bool nft_bitmap_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext)

{

	const struct nft_bitmap *priv = nft_set_priv(set);

net/netfilter/nft_set_hash.c

+10 −7
Original line number Diff line number Diff line
@@ -74,7 +74,8 @@ static const struct rhashtable_params nft_rhash_params = { 
	.automatic_shrinking	= true,

};



static bool nft_rhash_lookup(const struct net *net, const struct nft_set *set,

INDIRECT_CALLABLE_SCOPE

bool nft_rhash_lookup(const struct net *net, const struct nft_set *set,

		      const u32 *key, const struct nft_set_ext **ext)

{

	struct nft_rhash *priv = nft_set_priv(set);
@@ -446,7 +447,8 @@ struct nft_hash_elem { 
	struct nft_set_ext		ext;

};



static bool nft_hash_lookup(const struct net *net, const struct nft_set *set,

INDIRECT_CALLABLE_SCOPE

bool nft_hash_lookup(const struct net *net, const struct nft_set *set,

		     const u32 *key, const struct nft_set_ext **ext)

{

	struct nft_hash *priv = nft_set_priv(set);
@@ -484,7 +486,8 @@ static void *nft_hash_get(const struct net *net, const struct nft_set *set, 
	return ERR_PTR(-ENOENT);

}



static bool nft_hash_lookup_fast(const struct net *net,

INDIRECT_CALLABLE_SCOPE

bool nft_hash_lookup_fast(const struct net *net,

			  const struct nft_set *set,

			  const u32 *key, const struct nft_set_ext **ext)

{

net/netfilter/nft_set_pipapo.h

+0 −2
Original line number Diff line number Diff line
@@ -178,8 +178,6 @@ struct nft_pipapo_elem { 


int pipapo_refill(unsigned long *map, int len, int rules, unsigned long *dst,

		  union nft_pipapo_map_bucket *mt, bool match_only);

bool nft_pipapo_lookup(const struct net *net, const struct nft_set *set,

		       const u32 *key, const struct nft_set_ext **ext);



/**

 * pipapo_and_field_buckets_4bit() - Intersect 4-bit buckets