Commit 09eed119 authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

neighbour: switch to standard rcu, instead of rcu_bh 



rcu_bh is no longer a win, especially for objects freed
with standard call_rcu().

Switch neighbour code to no longer disable BH when not necessary.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 4c5c496a

include/net/arp.h

+4 −4
Original line number Diff line number Diff line
@@ -38,11 +38,11 @@ static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32 
{

	struct neighbour *n;



	rcu_read_lock_bh();

	rcu_read_lock();

	n = __ipv4_neigh_lookup_noref(dev, key);

	if (n && !refcount_inc_not_zero(&n->refcnt))

		n = NULL;

	rcu_read_unlock_bh();

	rcu_read_unlock();



	return n;

}
@@ -51,10 +51,10 @@ static inline void __ipv4_confirm_neigh(struct net_device *dev, u32 key) 
{

	struct neighbour *n;



	rcu_read_lock_bh();

	rcu_read_lock();

	n = __ipv4_neigh_lookup_noref(dev, key);

	neigh_confirm(n);

	rcu_read_unlock_bh();

	rcu_read_unlock();

}



void arp_init(void);

include/net/ndisc.h

+6 −6
Original line number Diff line number Diff line
@@ -395,11 +395,11 @@ static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, cons 
{

	struct neighbour *n;



	rcu_read_lock_bh();

	rcu_read_lock();

	n = __ipv6_neigh_lookup_noref(dev, pkey);

	if (n && !refcount_inc_not_zero(&n->refcnt))

		n = NULL;

	rcu_read_unlock_bh();

	rcu_read_unlock();



	return n;

}
@@ -409,10 +409,10 @@ static inline void __ipv6_confirm_neigh(struct net_device *dev, 
{

	struct neighbour *n;



	rcu_read_lock_bh();

	rcu_read_lock();

	n = __ipv6_neigh_lookup_noref(dev, pkey);

	neigh_confirm(n);

	rcu_read_unlock_bh();

	rcu_read_unlock();

}



static inline void __ipv6_confirm_neigh_stub(struct net_device *dev,
@@ -420,10 +420,10 @@ static inline void __ipv6_confirm_neigh_stub(struct net_device *dev, 
{

	struct neighbour *n;



	rcu_read_lock_bh();

	rcu_read_lock();

	n = __ipv6_neigh_lookup_noref_stub(dev, pkey);

	neigh_confirm(n);

	rcu_read_unlock_bh();

	rcu_read_unlock();

}



/* uses ipv6_stub and is meant for use outside of IPv6 core */

include/net/neighbour.h

+3 −3
Original line number Diff line number Diff line
@@ -299,14 +299,14 @@ static inline struct neighbour *___neigh_lookup_noref( 
	const void *pkey,

	struct net_device *dev)

{

	struct neigh_hash_table *nht = rcu_dereference_bh(tbl->nht);

	struct neigh_hash_table *nht = rcu_dereference(tbl->nht);

	struct neighbour *n;

	u32 hash_val;



	hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);

	for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);

	for (n = rcu_dereference(nht->hash_buckets[hash_val]);

	     n != NULL;

	     n = rcu_dereference_bh(n->next)) {

	     n = rcu_dereference(n->next)) {

		if (n->dev == dev && key_eq(n, pkey))

			return n;

	}

include/net/nexthop.h

+3 −3
Original line number Diff line number Diff line
@@ -498,7 +498,7 @@ static inline struct fib6_nh *nexthop_fib6_nh(struct nexthop *nh) 
}



/* Variant of nexthop_fib6_nh().

 * Caller should either hold rcu_read_lock_bh(), or RTNL.

 * Caller should either hold rcu_read_lock(), or RTNL.

 */

static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh)

{
@@ -507,13 +507,13 @@ static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh) 
	if (nh->is_group) {

		struct nh_group *nh_grp;



		nh_grp = rcu_dereference_bh_rtnl(nh->nh_grp);

		nh_grp = rcu_dereference_rtnl(nh->nh_grp);

		nh = nexthop_mpath_select(nh_grp, 0);

		if (!nh)

			return NULL;

	}



	nhi = rcu_dereference_bh_rtnl(nh->nh_info);

	nhi = rcu_dereference_rtnl(nh->nh_info);

	if (nhi->family == AF_INET6)

		return &nhi->fib6_nh;

net/core/filter.c

+10 −6
Original line number Diff line number Diff line
@@ -2204,7 +2204,7 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb, 
			return -ENOMEM;

	}



	rcu_read_lock_bh();

	rcu_read_lock();

	if (!nh) {

		dst = skb_dst(skb);

		nexthop = rt6_nexthop(container_of(dst, struct rt6_info, dst),
@@ -2217,10 +2217,12 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb, 
		int ret;



		sock_confirm_neigh(skb, neigh);

		local_bh_disable();

		dev_xmit_recursion_inc();

		ret = neigh_output(neigh, skb, false);

		dev_xmit_recursion_dec();

		rcu_read_unlock_bh();

		local_bh_enable();

		rcu_read_unlock();

		return ret;

	}

	rcu_read_unlock_bh();
@@ -2302,7 +2304,7 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb, 
			return -ENOMEM;

	}



	rcu_read_lock_bh();

	rcu_read_lock();

	if (!nh) {

		struct dst_entry *dst = skb_dst(skb);

		struct rtable *rt = container_of(dst, struct rtable, dst);
@@ -2314,7 +2316,7 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb, 
	} else if (nh->nh_family == AF_INET) {

		neigh = ip_neigh_gw4(dev, nh->ipv4_nh);

	} else {

		rcu_read_unlock_bh();

		rcu_read_unlock();

		goto out_drop;

	}
@@ -2322,13 +2324,15 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb, 
		int ret;



		sock_confirm_neigh(skb, neigh);

		local_bh_disable();

		dev_xmit_recursion_inc();

		ret = neigh_output(neigh, skb, is_v6gw);

		dev_xmit_recursion_dec();

		rcu_read_unlock_bh();

		local_bh_enable();

		rcu_read_unlock();

		return ret;

	}

	rcu_read_unlock_bh();

	rcu_read_unlock();

out_drop:

	kfree_skb(skb);

	return -ENETDOWN;