Commit 0e884c78 authored by Peter Nørlund's avatar Peter Nørlund Committed by David S. Miller
Browse files

ipv4: L3 hash-based multipath 



Replaces the per-packet multipath with a hash-based multipath using
source and destination address.

Signed-off-by: default avatarPeter Nørlund <pch@ordbogen.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 2472186f

include/net/ip_fib.h

+11 −3
Original line number Diff line number Diff line
@@ -79,7 +79,7 @@ struct fib_nh { 
	unsigned char		nh_scope;

#ifdef CONFIG_IP_ROUTE_MULTIPATH

	int			nh_weight;

	int			nh_power;

	atomic_t		nh_upper_bound;

#endif

#ifdef CONFIG_IP_ROUTE_CLASSID

	__u32			nh_tclassid;
@@ -118,7 +118,7 @@ struct fib_info { 
#define fib_advmss fib_metrics[RTAX_ADVMSS-1]

	int			fib_nhs;

#ifdef CONFIG_IP_ROUTE_MULTIPATH

	int			fib_power;

	int			fib_weight;

#endif

	struct rcu_head		rcu;

	struct fib_nh		fib_nh[0];
@@ -320,7 +320,15 @@ int ip_fib_check_default(__be32 gw, struct net_device *dev); 
int fib_sync_down_dev(struct net_device *dev, unsigned long event);

int fib_sync_down_addr(struct net *net, __be32 local);

int fib_sync_up(struct net_device *dev, unsigned int nh_flags);

void fib_select_multipath(struct fib_result *res);



extern u32 fib_multipath_secret __read_mostly;



static inline int fib_multipath_hash(__be32 saddr, __be32 daddr)

{

	return jhash_2words(saddr, daddr, fib_multipath_secret) >> 1;

}



void fib_select_multipath(struct fib_result *res, int hash);



/* Exported by fib_trie.c */

void fib_trie_init(void);

net/ipv4/fib_semantics.c

+75 −65
Original line number Diff line number Diff line
@@ -57,8 +57,7 @@ static unsigned int fib_info_cnt; 
static struct hlist_head fib_info_devhash[DEVINDEX_HASHSIZE];



#ifdef CONFIG_IP_ROUTE_MULTIPATH



static DEFINE_SPINLOCK(fib_multipath_lock);

u32 fib_multipath_secret __read_mostly;



#define for_nexthops(fi) {						\

	int nhsel; const struct fib_nh *nh;				\
@@ -532,7 +531,67 @@ static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh, 
	return ret;

}



#endif

static void fib_rebalance(struct fib_info *fi)

{

	int total;

	int w;

	struct in_device *in_dev;



	if (fi->fib_nhs < 2)

		return;



	total = 0;

	for_nexthops(fi) {

		if (nh->nh_flags & RTNH_F_DEAD)

			continue;



		in_dev = __in_dev_get_rcu(nh->nh_dev);



		if (in_dev &&

		    IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&

		    nh->nh_flags & RTNH_F_LINKDOWN)

			continue;



		total += nh->nh_weight;

	} endfor_nexthops(fi);



	w = 0;

	change_nexthops(fi) {

		int upper_bound;



		in_dev = __in_dev_get_rcu(nexthop_nh->nh_dev);



		if (nexthop_nh->nh_flags & RTNH_F_DEAD) {

			upper_bound = -1;

		} else if (in_dev &&

			   IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&

			   nexthop_nh->nh_flags & RTNH_F_LINKDOWN) {

			upper_bound = -1;

		} else {

			w += nexthop_nh->nh_weight;

			upper_bound = DIV_ROUND_CLOSEST(2147483648LL * w,

							total) - 1;

		}



		atomic_set(&nexthop_nh->nh_upper_bound, upper_bound);

	} endfor_nexthops(fi);



	net_get_random_once(&fib_multipath_secret,

			    sizeof(fib_multipath_secret));

}



static inline void fib_add_weight(struct fib_info *fi,

				  const struct fib_nh *nh)

{

	fi->fib_weight += nh->nh_weight;

}



#else /* CONFIG_IP_ROUTE_MULTIPATH */



#define fib_rebalance(fi) do { } while (0)

#define fib_add_weight(fi, nh) do { } while (0)



#endif /* CONFIG_IP_ROUTE_MULTIPATH */



static int fib_encap_match(struct net *net, u16 encap_type,

			   struct nlattr *encap,
@@ -1094,8 +1153,11 @@ struct fib_info *fib_create_info(struct fib_config *cfg) 


	change_nexthops(fi) {

		fib_info_update_nh_saddr(net, nexthop_nh);

		fib_add_weight(fi, nexthop_nh);

	} endfor_nexthops(fi)



	fib_rebalance(fi);



link_it:

	ofi = fib_find_info(fi);

	if (ofi) {
@@ -1317,12 +1379,6 @@ int fib_sync_down_dev(struct net_device *dev, unsigned long event) 
					nexthop_nh->nh_flags |= RTNH_F_LINKDOWN;

					break;

				}

#ifdef CONFIG_IP_ROUTE_MULTIPATH

				spin_lock_bh(&fib_multipath_lock);

				fi->fib_power -= nexthop_nh->nh_power;

				nexthop_nh->nh_power = 0;

				spin_unlock_bh(&fib_multipath_lock);

#endif

				dead++;

			}

#ifdef CONFIG_IP_ROUTE_MULTIPATH
@@ -1345,6 +1401,8 @@ int fib_sync_down_dev(struct net_device *dev, unsigned long event) 
			}

			ret++;

		}



		fib_rebalance(fi);

	}



	return ret;
@@ -1467,20 +1525,15 @@ int fib_sync_up(struct net_device *dev, unsigned int nh_flags) 
			    !__in_dev_get_rtnl(dev))

				continue;

			alive++;

#ifdef CONFIG_IP_ROUTE_MULTIPATH

			spin_lock_bh(&fib_multipath_lock);

			nexthop_nh->nh_power = 0;

			nexthop_nh->nh_flags &= ~nh_flags;

			spin_unlock_bh(&fib_multipath_lock);

#else

			nexthop_nh->nh_flags &= ~nh_flags;

#endif

		} endfor_nexthops(fi)



		if (alive > 0) {

			fi->fib_flags &= ~nh_flags;

			ret++;

		}



		fib_rebalance(fi);

	}



	return ret;
@@ -1488,62 +1541,19 @@ int fib_sync_up(struct net_device *dev, unsigned int nh_flags) 


#ifdef CONFIG_IP_ROUTE_MULTIPATH



/*

 * The algorithm is suboptimal, but it provides really

 * fair weighted route distribution.

 */

void fib_select_multipath(struct fib_result *res)

void fib_select_multipath(struct fib_result *res, int hash)

{

	struct fib_info *fi = res->fi;

	struct in_device *in_dev;

	int w;



	spin_lock_bh(&fib_multipath_lock);

	if (fi->fib_power <= 0) {

		int power = 0;

		change_nexthops(fi) {

			in_dev = __in_dev_get_rcu(nexthop_nh->nh_dev);

			if (nexthop_nh->nh_flags & RTNH_F_DEAD)

				continue;

			if (in_dev &&

			    IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&

			    nexthop_nh->nh_flags & RTNH_F_LINKDOWN)

	for_nexthops(fi) {

		if (hash > atomic_read(&nh->nh_upper_bound))

			continue;

			power += nexthop_nh->nh_weight;

			nexthop_nh->nh_power = nexthop_nh->nh_weight;

		} endfor_nexthops(fi);

		fi->fib_power = power;

		if (power <= 0) {

			spin_unlock_bh(&fib_multipath_lock);

			/* Race condition: route has just become dead. */

			res->nh_sel = 0;

			return;

		}

	}





	/* w should be random number [0..fi->fib_power-1],

	 * it is pretty bad approximation.

	 */



	w = jiffies % fi->fib_power;



	change_nexthops(fi) {

		if (!(nexthop_nh->nh_flags & RTNH_F_DEAD) &&

		    nexthop_nh->nh_power) {

			w -= nexthop_nh->nh_power;

			if (w <= 0) {

				nexthop_nh->nh_power--;

				fi->fib_power--;

		res->nh_sel = nhsel;

				spin_unlock_bh(&fib_multipath_lock);

		return;

			}

		}

	} endfor_nexthops(fi);



	/* Race condition: route has just become dead. */

	res->nh_sel = 0;

	spin_unlock_bh(&fib_multipath_lock);

}

#endif

net/ipv4/route.c

+12 −4
Original line number Diff line number Diff line
@@ -1658,8 +1658,12 @@ static int ip_mkroute_input(struct sk_buff *skb, 
			    __be32 daddr, __be32 saddr, u32 tos)

{

#ifdef CONFIG_IP_ROUTE_MULTIPATH

	if (res->fi && res->fi->fib_nhs > 1)

		fib_select_multipath(res);

	if (res->fi && res->fi->fib_nhs > 1) {

		int h;



		h = fib_multipath_hash(saddr, daddr);

		fib_select_multipath(res, h);

	}

#endif



	/* create a routing cache entry */
@@ -2189,8 +2193,12 @@ struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *fl4) 
	}



#ifdef CONFIG_IP_ROUTE_MULTIPATH

	if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)

		fib_select_multipath(&res);

	if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0) {

		int h;



		h = fib_multipath_hash(fl4->saddr, fl4->daddr);

		fib_select_multipath(&res, h);

	}

	else

#endif

	if (!res.prefixlen &&