Commit 4b200e39 authored by Taehee Yoo's avatar Taehee Yoo Committed by David S. Miller
Browse files

mld: convert ip6_sf_list to RCU 



The ip6_sf_list has been protected by mca_lock(spin_lock) so that the
critical section is atomic context. In order to switch this context,
changing locking is needed. The ip6_sf_list actually already protected
by RTNL So if it's converted to use RCU, its control path context can
be switched to sleepable.
But It doesn't remove mca_lock yet because ifmcaddr6 isn't converted
to RCU yet. So, It's not fully converted to the sleepable context.

Suggested-by: default avatarCong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: default avatarTaehee Yoo <ap420073@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 882ba1f7

include/net/if_inet6.h

+4 −3
Original line number Diff line number Diff line
@@ -97,12 +97,13 @@ struct ipv6_mc_socklist { 
};



struct ip6_sf_list {

	struct ip6_sf_list	*sf_next;

	struct ip6_sf_list __rcu *sf_next;

	struct in6_addr		sf_addr;

	unsigned long		sf_count[2];	/* include/exclude counts */

	unsigned char		sf_gsresp;	/* include in g & s response? */

	unsigned char		sf_oldin;	/* change state */

	unsigned char		sf_crcount;	/* retrans. left to send */

	struct rcu_head		rcu;

};



#define MAF_TIMER_RUNNING	0x01
@@ -115,8 +116,8 @@ struct ifmcaddr6 { 
	struct in6_addr		mca_addr;

	struct inet6_dev	*idev;

	struct ifmcaddr6	*next;

	struct ip6_sf_list	*mca_sources;

	struct ip6_sf_list	*mca_tomb;

	struct ip6_sf_list	__rcu *mca_sources;

	struct ip6_sf_list	__rcu *mca_tomb;

	unsigned int		mca_sfmode;

	unsigned char		mca_crcount;

	unsigned long		mca_sfcount[2];

net/ipv6/mcast.c

+126 −74
Original line number Diff line number Diff line
@@ -113,10 +113,25 @@ int sysctl_mld_qrv __read_mostly = MLD_QRV_DEFAULT; 
 */



#define for_each_pmc_rcu(np, pmc)				\

	for (pmc = rcu_dereference(np->ipv6_mc_list);		\

	     pmc != NULL;					\

	for (pmc = rcu_dereference((np)->ipv6_mc_list);		\

	     pmc;						\

	     pmc = rcu_dereference(pmc->next))



#define for_each_psf_rtnl(mc, psf)				\

	for (psf = rtnl_dereference((mc)->mca_sources);		\

	     psf;						\

	     psf = rtnl_dereference(psf->sf_next))



#define for_each_psf_rcu(mc, psf)				\

	for (psf = rcu_dereference((mc)->mca_sources);		\

	     psf;						\

	     psf = rcu_dereference(psf->sf_next))



#define for_each_psf_tomb(mc, psf)				\

	for (psf = rtnl_dereference((mc)->mca_tomb);		\

	     psf;						\

	     psf = rtnl_dereference(psf->sf_next))



static int unsolicited_report_interval(struct inet6_dev *idev)

{

	int iv;
@@ -734,10 +749,14 @@ static void mld_add_delrec(struct inet6_dev *idev, struct ifmcaddr6 *im) 
	if (pmc->mca_sfmode == MCAST_INCLUDE) {

		struct ip6_sf_list *psf;



		pmc->mca_tomb = im->mca_tomb;

		pmc->mca_sources = im->mca_sources;

		im->mca_tomb = im->mca_sources = NULL;

		for (psf = pmc->mca_sources; psf; psf = psf->sf_next)

		rcu_assign_pointer(pmc->mca_tomb,

				   rtnl_dereference(im->mca_tomb));

		rcu_assign_pointer(pmc->mca_sources,

				   rtnl_dereference(im->mca_sources));

		RCU_INIT_POINTER(im->mca_tomb, NULL);

		RCU_INIT_POINTER(im->mca_sources, NULL);



		for_each_psf_rtnl(pmc, psf)

			psf->sf_crcount = pmc->mca_crcount;

	}

	spin_unlock_bh(&im->mca_lock);
@@ -748,9 +767,9 @@ static void mld_add_delrec(struct inet6_dev *idev, struct ifmcaddr6 *im) 


static void mld_del_delrec(struct inet6_dev *idev, struct ifmcaddr6 *im)

{

	struct ifmcaddr6 *pmc, *pmc_prev;

	struct ip6_sf_list *psf;

	struct ip6_sf_list *psf, *sources, *tomb;

	struct in6_addr *pmca = &im->mca_addr;

	struct ifmcaddr6 *pmc, *pmc_prev;



	pmc_prev = NULL;

	for (pmc = idev->mc_tomb; pmc; pmc = pmc->next) {
@@ -769,9 +788,16 @@ static void mld_del_delrec(struct inet6_dev *idev, struct ifmcaddr6 *im) 
	if (pmc) {

		im->idev = pmc->idev;

		if (im->mca_sfmode == MCAST_INCLUDE) {

			swap(im->mca_tomb, pmc->mca_tomb);

			swap(im->mca_sources, pmc->mca_sources);

			for (psf = im->mca_sources; psf; psf = psf->sf_next)

			tomb = rcu_replace_pointer(im->mca_tomb,

						   rtnl_dereference(pmc->mca_tomb),

						   lockdep_rtnl_is_held());

			rcu_assign_pointer(pmc->mca_tomb, tomb);



			sources = rcu_replace_pointer(im->mca_sources,

						      rtnl_dereference(pmc->mca_sources),

						      lockdep_rtnl_is_held());

			rcu_assign_pointer(pmc->mca_sources, sources);

			for_each_psf_rtnl(im, psf)

				psf->sf_crcount = idev->mc_qrv;

		} else {

			im->mca_crcount = idev->mc_qrv;
@@ -803,12 +829,12 @@ static void mld_clear_delrec(struct inet6_dev *idev) 
		struct ip6_sf_list *psf, *psf_next;



		spin_lock_bh(&pmc->mca_lock);

		psf = pmc->mca_tomb;

		pmc->mca_tomb = NULL;

		psf = rtnl_dereference(pmc->mca_tomb);

		RCU_INIT_POINTER(pmc->mca_tomb, NULL);

		spin_unlock_bh(&pmc->mca_lock);

		for (; psf; psf = psf_next) {

			psf_next = psf->sf_next;

			kfree(psf);

			psf_next = rtnl_dereference(psf->sf_next);

			kfree_rcu(psf, rcu);

		}

	}

	read_unlock_bh(&idev->lock);
@@ -990,7 +1016,7 @@ bool ipv6_chk_mcast_addr(struct net_device *dev, const struct in6_addr *group, 
				struct ip6_sf_list *psf;



				spin_lock_bh(&mc->mca_lock);

				for (psf = mc->mca_sources; psf; psf = psf->sf_next) {

				for_each_psf_rcu(mc, psf) {

					if (ipv6_addr_equal(&psf->sf_addr, src_addr))

						break;

				}
@@ -1089,7 +1115,7 @@ static bool mld_xmarksources(struct ifmcaddr6 *pmc, int nsrcs, 
	int i, scount;



	scount = 0;

	for (psf = pmc->mca_sources; psf; psf = psf->sf_next) {

	for_each_psf_rcu(pmc, psf) {

		if (scount == nsrcs)

			break;

		for (i = 0; i < nsrcs; i++) {
@@ -1122,7 +1148,7 @@ static bool mld_marksources(struct ifmcaddr6 *pmc, int nsrcs, 
	/* mark INCLUDE-mode sources */



	scount = 0;

	for (psf = pmc->mca_sources; psf; psf = psf->sf_next) {

	for_each_psf_rcu(pmc, psf) {

		if (scount == nsrcs)

			break;

		for (i = 0; i < nsrcs; i++) {
@@ -1532,7 +1558,7 @@ mld_scount(struct ifmcaddr6 *pmc, int type, int gdeleted, int sdeleted) 
	struct ip6_sf_list *psf;

	int scount = 0;



	for (psf = pmc->mca_sources; psf; psf = psf->sf_next) {

	for_each_psf_rtnl(pmc, psf) {

		if (!is_in(pmc, psf, type, gdeleted, sdeleted))

			continue;

		scount++;
@@ -1707,14 +1733,16 @@ static struct sk_buff *add_grhead(struct sk_buff *skb, struct ifmcaddr6 *pmc, 
#define AVAILABLE(skb)	((skb) ? skb_availroom(skb) : 0)



static struct sk_buff *add_grec(struct sk_buff *skb, struct ifmcaddr6 *pmc,

	int type, int gdeleted, int sdeleted, int crsend)

				int type, int gdeleted, int sdeleted,

				int crsend)

{

	struct ip6_sf_list *psf, *psf_prev, *psf_next;

	int scount, stotal, first, isquery, truncate;

	struct ip6_sf_list __rcu **psf_list;

	struct inet6_dev *idev = pmc->idev;

	struct net_device *dev = idev->dev;

	struct mld2_report *pmr;

	struct mld2_grec *pgr = NULL;

	struct ip6_sf_list *psf, *psf_next, *psf_prev, **psf_list;

	int scount, stotal, first, isquery, truncate;

	struct mld2_report *pmr;

	unsigned int mtu;



	if (pmc->mca_flags & MAF_NOREPORT)
@@ -1733,7 +1761,7 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ifmcaddr6 *pmc, 


	psf_list = sdeleted ? &pmc->mca_tomb : &pmc->mca_sources;



	if (!*psf_list)

	if (!rcu_access_pointer(*psf_list))

		goto empty_source;



	pmr = skb ? (struct mld2_report *)skb_transport_header(skb) : NULL;
@@ -1749,10 +1777,12 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ifmcaddr6 *pmc, 
	}

	first = 1;

	psf_prev = NULL;

	for (psf = *psf_list; psf; psf = psf_next) {

	for (psf = rtnl_dereference(*psf_list);

	     psf;

	     psf = psf_next) {

		struct in6_addr *psrc;



		psf_next = psf->sf_next;

		psf_next = rtnl_dereference(psf->sf_next);



		if (!is_in(pmc, psf, type, gdeleted, sdeleted) && !crsend) {

			psf_prev = psf;
@@ -1799,10 +1829,12 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ifmcaddr6 *pmc, 
			psf->sf_crcount--;

			if ((sdeleted || gdeleted) && psf->sf_crcount == 0) {

				if (psf_prev)

					psf_prev->sf_next = psf->sf_next;

					rcu_assign_pointer(psf_prev->sf_next,

							   rtnl_dereference(psf->sf_next));

				else

					*psf_list = psf->sf_next;

				kfree(psf);

					rcu_assign_pointer(*psf_list,

							   rtnl_dereference(psf->sf_next));

				kfree_rcu(psf, rcu);

				continue;

			}

		}
@@ -1866,23 +1898,28 @@ static void mld_send_report(struct inet6_dev *idev, struct ifmcaddr6 *pmc) 
/*

 * remove zero-count source records from a source filter list

 */

static void mld_clear_zeros(struct ip6_sf_list **ppsf)

static void mld_clear_zeros(struct ip6_sf_list __rcu **ppsf)

{

	struct ip6_sf_list *psf_prev, *psf_next, *psf;



	psf_prev = NULL;

	for (psf = *ppsf; psf; psf = psf_next) {

		psf_next = psf->sf_next;

	for (psf = rtnl_dereference(*ppsf);

	     psf;

	     psf = psf_next) {

		psf_next = rtnl_dereference(psf->sf_next);

		if (psf->sf_crcount == 0) {

			if (psf_prev)

				psf_prev->sf_next = psf->sf_next;

				rcu_assign_pointer(psf_prev->sf_next,

						   rtnl_dereference(psf->sf_next));

			else

				*ppsf = psf->sf_next;

			kfree(psf);

		} else

				rcu_assign_pointer(*ppsf,

						   rtnl_dereference(psf->sf_next));

			kfree_rcu(psf, rcu);

		} else {

			psf_prev = psf;

		}

	}

}



static void mld_send_cr(struct inet6_dev *idev)

{
@@ -1913,8 +1950,9 @@ static void mld_send_cr(struct inet6_dev *idev) 
				mld_clear_zeros(&pmc->mca_sources);

			}

		}

		if (pmc->mca_crcount == 0 && !pmc->mca_tomb &&

		    !pmc->mca_sources) {

		if (pmc->mca_crcount == 0 &&

		    !rcu_access_pointer(pmc->mca_tomb) &&

		    !rcu_access_pointer(pmc->mca_sources)) {

			if (pmc_prev)

				pmc_prev->next = pmc_next;

			else
@@ -2111,7 +2149,7 @@ static int ip6_mc_del1_src(struct ifmcaddr6 *pmc, int sfmode, 
	int rv = 0;



	psf_prev = NULL;

	for (psf = pmc->mca_sources; psf; psf = psf->sf_next) {

	for_each_psf_rtnl(pmc, psf) {

		if (ipv6_addr_equal(&psf->sf_addr, psfsrc))

			break;

		psf_prev = psf;
@@ -2126,17 +2164,22 @@ static int ip6_mc_del1_src(struct ifmcaddr6 *pmc, int sfmode, 


		/* no more filters for this source */

		if (psf_prev)

			psf_prev->sf_next = psf->sf_next;

			rcu_assign_pointer(psf_prev->sf_next,

					   rtnl_dereference(psf->sf_next));

		else

			pmc->mca_sources = psf->sf_next;

			rcu_assign_pointer(pmc->mca_sources,

					   rtnl_dereference(psf->sf_next));



		if (psf->sf_oldin && !(pmc->mca_flags & MAF_NOREPORT) &&

		    !mld_in_v1_mode(idev)) {

			psf->sf_crcount = idev->mc_qrv;

			psf->sf_next = pmc->mca_tomb;

			pmc->mca_tomb = psf;

			rcu_assign_pointer(psf->sf_next,

					   rtnl_dereference(pmc->mca_tomb));

			rcu_assign_pointer(pmc->mca_tomb, psf);

			rv = 1;

		} else

			kfree(psf);

		} else {

			kfree_rcu(psf, rcu);

		}

	}

	return rv;

}
@@ -2188,7 +2231,7 @@ static int ip6_mc_del_src(struct inet6_dev *idev, const struct in6_addr *pmca, 
		pmc->mca_sfmode = MCAST_INCLUDE;

		pmc->mca_crcount = idev->mc_qrv;

		idev->mc_ifc_count = pmc->mca_crcount;

		for (psf = pmc->mca_sources; psf; psf = psf->sf_next)

		for_each_psf_rtnl(pmc, psf)

			psf->sf_crcount = 0;

		mld_ifc_event(pmc->idev);

	} else if (sf_setstate(pmc) || changerec)
@@ -2207,7 +2250,7 @@ static int ip6_mc_add1_src(struct ifmcaddr6 *pmc, int sfmode, 
	struct ip6_sf_list *psf, *psf_prev;



	psf_prev = NULL;

	for (psf = pmc->mca_sources; psf; psf = psf->sf_next) {

	for_each_psf_rtnl(pmc, psf) {

		if (ipv6_addr_equal(&psf->sf_addr, psfsrc))

			break;

		psf_prev = psf;
@@ -2219,9 +2262,10 @@ static int ip6_mc_add1_src(struct ifmcaddr6 *pmc, int sfmode, 


		psf->sf_addr = *psfsrc;

		if (psf_prev) {

			psf_prev->sf_next = psf;

		} else

			pmc->mca_sources = psf;

			rcu_assign_pointer(psf_prev->sf_next, psf);

		} else {

			rcu_assign_pointer(pmc->mca_sources, psf);

		}

	}

	psf->sf_count[sfmode]++;

	return 0;
@@ -2232,14 +2276,16 @@ static void sf_markstate(struct ifmcaddr6 *pmc) 
	struct ip6_sf_list *psf;

	int mca_xcount = pmc->mca_sfcount[MCAST_EXCLUDE];



	for (psf = pmc->mca_sources; psf; psf = psf->sf_next)

	for_each_psf_rtnl(pmc, psf) {

		if (pmc->mca_sfcount[MCAST_EXCLUDE]) {

			psf->sf_oldin = mca_xcount ==

				psf->sf_count[MCAST_EXCLUDE] &&

				!psf->sf_count[MCAST_INCLUDE];

		} else

		} else {

			psf->sf_oldin = psf->sf_count[MCAST_INCLUDE] != 0;

		}

	}

}



static int sf_setstate(struct ifmcaddr6 *pmc)

{
@@ -2249,7 +2295,7 @@ static int sf_setstate(struct ifmcaddr6 *pmc) 
	int new_in, rv;



	rv = 0;

	for (psf = pmc->mca_sources; psf; psf = psf->sf_next) {

	for_each_psf_rtnl(pmc, psf) {

		if (pmc->mca_sfcount[MCAST_EXCLUDE]) {

			new_in = mca_xcount == psf->sf_count[MCAST_EXCLUDE] &&

				!psf->sf_count[MCAST_INCLUDE];
@@ -2259,8 +2305,7 @@ static int sf_setstate(struct ifmcaddr6 *pmc) 
			if (!psf->sf_oldin) {

				struct ip6_sf_list *prev = NULL;



				for (dpsf = pmc->mca_tomb; dpsf;

				     dpsf = dpsf->sf_next) {

				for_each_psf_tomb(pmc, dpsf) {

					if (ipv6_addr_equal(&dpsf->sf_addr,

					    &psf->sf_addr))

						break;
@@ -2268,10 +2313,12 @@ static int sf_setstate(struct ifmcaddr6 *pmc) 
				}

				if (dpsf) {

					if (prev)

						prev->sf_next = dpsf->sf_next;

						rcu_assign_pointer(prev->sf_next,

								   rtnl_dereference(dpsf->sf_next));

					else

						pmc->mca_tomb = dpsf->sf_next;

					kfree(dpsf);

						rcu_assign_pointer(pmc->mca_tomb,

								   rtnl_dereference(dpsf->sf_next));

					kfree_rcu(dpsf, rcu);

				}

				psf->sf_crcount = qrv;

				rv++;
@@ -2282,7 +2329,8 @@ static int sf_setstate(struct ifmcaddr6 *pmc) 
			 * add or update "delete" records if an active filter

			 * is now inactive

			 */

			for (dpsf = pmc->mca_tomb; dpsf; dpsf = dpsf->sf_next)



			for_each_psf_tomb(pmc, dpsf)

				if (ipv6_addr_equal(&dpsf->sf_addr,

				    &psf->sf_addr))

					break;
@@ -2291,9 +2339,9 @@ static int sf_setstate(struct ifmcaddr6 *pmc) 
				if (!dpsf)

					continue;

				*dpsf = *psf;

				/* pmc->mca_lock held by callers */

				dpsf->sf_next = pmc->mca_tomb;

				pmc->mca_tomb = dpsf;

				rcu_assign_pointer(dpsf->sf_next,

						   rtnl_dereference(pmc->mca_tomb));

				rcu_assign_pointer(pmc->mca_tomb, dpsf);

			}

			dpsf->sf_crcount = qrv;

			rv++;
@@ -2356,7 +2404,7 @@ static int ip6_mc_add_src(struct inet6_dev *idev, const struct in6_addr *pmca, 


		pmc->mca_crcount = idev->mc_qrv;

		idev->mc_ifc_count = pmc->mca_crcount;

		for (psf = pmc->mca_sources; psf; psf = psf->sf_next)

		for_each_psf_rtnl(pmc, psf)

			psf->sf_crcount = 0;

		mld_ifc_event(idev);

	} else if (sf_setstate(pmc))
@@ -2370,16 +2418,20 @@ static void ip6_mc_clear_src(struct ifmcaddr6 *pmc) 
{

	struct ip6_sf_list *psf, *nextpsf;



	for (psf = pmc->mca_tomb; psf; psf = nextpsf) {

		nextpsf = psf->sf_next;

		kfree(psf);

	for (psf = rtnl_dereference(pmc->mca_tomb);

	     psf;

	     psf = nextpsf) {

		nextpsf = rtnl_dereference(psf->sf_next);

		kfree_rcu(psf, rcu);

	}

	pmc->mca_tomb = NULL;

	for (psf = pmc->mca_sources; psf; psf = nextpsf) {

		nextpsf = psf->sf_next;

		kfree(psf);

	RCU_INIT_POINTER(pmc->mca_tomb, NULL);

	for (psf = rtnl_dereference(pmc->mca_sources);

	     psf;

	     psf = nextpsf) {

		nextpsf = rtnl_dereference(psf->sf_next);

		kfree_rcu(psf, rcu);

	}

	pmc->mca_sources = NULL;

	RCU_INIT_POINTER(pmc->mca_sources, NULL);

	pmc->mca_sfmode = MCAST_EXCLUDE;

	pmc->mca_sfcount[MCAST_INCLUDE] = 0;

	pmc->mca_sfcount[MCAST_EXCLUDE] = 1;
@@ -2789,7 +2841,7 @@ static inline struct ip6_sf_list *igmp6_mcf_get_first(struct seq_file *seq) 
		im = idev->mc_list;

		if (likely(im)) {

			spin_lock_bh(&im->mca_lock);

			psf = im->mca_sources;

			psf = rcu_dereference(im->mca_sources);

			if (likely(psf)) {

				state->im = im;

				state->idev = idev;
@@ -2806,7 +2858,7 @@ static struct ip6_sf_list *igmp6_mcf_get_next(struct seq_file *seq, struct ip6_s 
{

	struct igmp6_mcf_iter_state *state = igmp6_mcf_seq_private(seq);



	psf = psf->sf_next;

	psf = rcu_dereference(psf->sf_next);

	while (!psf) {

		spin_unlock_bh(&state->im->mca_lock);

		state->im = state->im->next;
@@ -2828,7 +2880,7 @@ static struct ip6_sf_list *igmp6_mcf_get_next(struct seq_file *seq, struct ip6_s 
		if (!state->im)

			break;

		spin_lock_bh(&state->im->mca_lock);

		psf = state->im->mca_sources;

		psf = rcu_dereference(state->im->mca_sources);

	}

out:

	return psf;