Commit a263b309 authored by David S. Miller's avatar David S. Miller
Browse files

ipv4: Make neigh lookups directly in output packet path. 



Do not use the dst cached neigh, we'll be getting rid of that.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 11604721

include/net/arp.h

+19 −9
Original line number Diff line number Diff line
@@ -15,24 +15,34 @@ static inline u32 arp_hashfn(u32 key, const struct net_device *dev, u32 hash_rnd 
	return val * hash_rnd;

}



static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32 key)

static inline struct neighbour *__ipv4_neigh_lookup_noref(struct net_device *dev, u32 key)

{

	struct neigh_hash_table *nht;

	struct neigh_hash_table *nht = rcu_dereference_bh(arp_tbl.nht);

	struct neighbour *n;

	u32 hash_val;



	rcu_read_lock_bh();

	nht = rcu_dereference_bh(arp_tbl.nht);

	if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))

		key = 0;



	hash_val = arp_hashfn(key, dev, nht->hash_rnd[0]) >> (32 - nht->hash_shift);

	for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);

	     n != NULL;

	     n = rcu_dereference_bh(n->next)) {

		if (n->dev == dev && *(u32 *)n->primary_key == key) {

			if (!atomic_inc_not_zero(&n->refcnt))

				n = NULL;

			break;

		if (n->dev == dev && *(u32 *)n->primary_key == key)

			return n;

	}



	return NULL;

}



static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32 key)

{

	struct neighbour *n;



	rcu_read_lock_bh();

	n = __ipv4_neigh_lookup_noref(dev, key);

	if (n && !atomic_inc_not_zero(&n->refcnt))

		n = NULL;

	rcu_read_unlock_bh();



	return n;

include/net/neighbour.h

+9 −2
Original line number Diff line number Diff line
@@ -202,9 +202,16 @@ extern struct neighbour * neigh_lookup(struct neigh_table *tbl, 
extern struct neighbour *	neigh_lookup_nodev(struct neigh_table *tbl,

						   struct net *net,

						   const void *pkey);

extern struct neighbour *	neigh_create(struct neigh_table *tbl,

extern struct neighbour *	__neigh_create(struct neigh_table *tbl,

					       const void *pkey,

					     struct net_device *dev);

					       struct net_device *dev,

					       bool want_ref);

static inline struct neighbour *neigh_create(struct neigh_table *tbl,

					     const void *pkey,

					     struct net_device *dev)

{

	return __neigh_create(tbl, pkey, dev, true);

}

extern void			neigh_destroy(struct neighbour *neigh);

extern int			__neigh_event_send(struct neighbour *neigh, struct sk_buff *skb);

extern int			neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,

net/core/neighbour.c

+7 −5
Original line number Diff line number Diff line
@@ -474,8 +474,8 @@ struct neighbour *neigh_lookup_nodev(struct neigh_table *tbl, struct net *net, 
}

EXPORT_SYMBOL(neigh_lookup_nodev);



struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey,

			       struct net_device *dev)

struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,

				 struct net_device *dev, bool want_ref)

{

	u32 hash_val;

	int key_len = tbl->key_len;
@@ -535,6 +535,7 @@ struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey, 
	     n1 = rcu_dereference_protected(n1->next,

			lockdep_is_held(&tbl->lock))) {

		if (dev == n1->dev && !memcmp(n1->primary_key, pkey, key_len)) {

			if (want_ref)

				neigh_hold(n1);

			rc = n1;

			goto out_tbl_unlock;
@@ -542,6 +543,7 @@ struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey, 
	}



	n->dead = 0;

	if (want_ref)

		neigh_hold(n);

	rcu_assign_pointer(n->next,

			   rcu_dereference_protected(nht->hash_buckets[hash_val],
@@ -558,7 +560,7 @@ struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey, 
	neigh_release(n);

	goto out;

}

EXPORT_SYMBOL(neigh_create);

EXPORT_SYMBOL(__neigh_create);



static u32 pneigh_hash(const void *pkey, int key_len)

{

net/ipv4/ip_output.c

+8 −4
Original line number Diff line number Diff line
@@ -170,6 +170,7 @@ static inline int ip_finish_output2(struct sk_buff *skb) 
	struct net_device *dev = dst->dev;

	unsigned int hh_len = LL_RESERVED_SPACE(dev);

	struct neighbour *neigh;

	u32 nexthop;



	if (rt->rt_type == RTN_MULTICAST) {

		IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
@@ -191,15 +192,18 @@ static inline int ip_finish_output2(struct sk_buff *skb) 
		skb = skb2;

	}



	rcu_read_lock();

	neigh = dst_get_neighbour_noref(dst);

	rcu_read_lock_bh();

	nexthop = rt->rt_gateway ? rt->rt_gateway : ip_hdr(skb)->daddr;

	neigh = __ipv4_neigh_lookup_noref(dev, nexthop);

	if (unlikely(!neigh))

		neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);

	if (neigh) {

		int res = neigh_output(neigh, skb);



		rcu_read_unlock();

		rcu_read_unlock_bh();

		return res;

	}

	rcu_read_unlock();

	rcu_read_unlock_bh();



	net_dbg_ratelimited("%s: No header cache and no neighbour!\n",

			    __func__);

net/ipv4/route.c

+1 −5
Original line number Diff line number Diff line
@@ -1098,17 +1098,13 @@ static int slow_chain_length(const struct rtable *head) 


static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, const void *daddr)

{

	static const __be32 inaddr_any = 0;

	struct net_device *dev = dst->dev;

	const __be32 *pkey = daddr;

	const struct rtable *rt;

	struct neighbour *n;



	rt = (const struct rtable *) dst;



	if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))

		pkey = &inaddr_any;

	else if (rt->rt_gateway)

	if (rt->rt_gateway)

		pkey = (const __be32 *) &rt->rt_gateway;



	n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);