Commit 29d2bb94 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Provide internal interface to start a Tree SRCU grace period 

There is a need for a polling interface for SRCU grace periods.
This polling needs to initiate an SRCU grace period without having
to queue (and manage) a callback.  This commit therefore splits the
Tree SRCU __call_srcu() function into callback-initialization and
queuing/start-grace-period portions, with the latter in a new function
named srcu_gp_start_if_needed().  This function may be passed a NULL
callback pointer, in which case it will refrain from queuing anything.

Why have the new function mess with queuing?  Locking considerations,
of course!

Link: https://lore.kernel.org/rcu/20201112201547.GF3365678@moria.home.lan/


Reported-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
Reviewed-by: default avatarNeeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 1a893c71

kernel/rcu/srcutree.c

+37 −29
Original line number Diff line number Diff line
@@ -807,6 +807,42 @@ static void srcu_leak_callback(struct rcu_head *rhp) 
{

}



/*

 * Start an SRCU grace period, and also queue the callback if non-NULL.

 */

static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rhp, bool do_norm)

{

	unsigned long flags;

	int idx;

	bool needexp = false;

	bool needgp = false;

	unsigned long s;

	struct srcu_data *sdp;



	idx = srcu_read_lock(ssp);

	sdp = raw_cpu_ptr(ssp->sda);

	spin_lock_irqsave_rcu_node(sdp, flags);

	rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);

	rcu_segcblist_advance(&sdp->srcu_cblist,

			      rcu_seq_current(&ssp->srcu_gp_seq));

	s = rcu_seq_snap(&ssp->srcu_gp_seq);

	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);

	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {

		sdp->srcu_gp_seq_needed = s;

		needgp = true;

	}

	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {

		sdp->srcu_gp_seq_needed_exp = s;

		needexp = true;

	}

	spin_unlock_irqrestore_rcu_node(sdp, flags);

	if (needgp)

		srcu_funnel_gp_start(ssp, sdp, s, do_norm);

	else if (needexp)

		srcu_funnel_exp_start(ssp, sdp->mynode, s);

	srcu_read_unlock(ssp, idx);

}



/*

 * Enqueue an SRCU callback on the srcu_data structure associated with

 * the current CPU and the specified srcu_struct structure, initiating
@@ -838,13 +874,6 @@ static void srcu_leak_callback(struct rcu_head *rhp) 
static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,

			rcu_callback_t func, bool do_norm)

{

	unsigned long flags;

	int idx;

	bool needexp = false;

	bool needgp = false;

	unsigned long s;

	struct srcu_data *sdp;



	check_init_srcu_struct(ssp);

	if (debug_rcu_head_queue(rhp)) {

		/* Probable double call_srcu(), so leak the callback. */
@@ -853,28 +882,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, 
		return;

	}

	rhp->func = func;

	idx = srcu_read_lock(ssp);

	sdp = raw_cpu_ptr(ssp->sda);

	spin_lock_irqsave_rcu_node(sdp, flags);

	rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);

	rcu_segcblist_advance(&sdp->srcu_cblist,

			      rcu_seq_current(&ssp->srcu_gp_seq));

	s = rcu_seq_snap(&ssp->srcu_gp_seq);

	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);

	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {

		sdp->srcu_gp_seq_needed = s;

		needgp = true;

	}

	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {

		sdp->srcu_gp_seq_needed_exp = s;

		needexp = true;

	}

	spin_unlock_irqrestore_rcu_node(sdp, flags);

	if (needgp)

		srcu_funnel_gp_start(ssp, sdp, s, do_norm);

	else if (needexp)

		srcu_funnel_exp_start(ssp, sdp->mynode, s);

	srcu_read_unlock(ssp, idx);

	srcu_gp_start_if_needed(ssp, rhp, do_norm);

}



/**