Commit 5358c9fa authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Provide polling interfaces for Tree SRCU grace periods 

There is a need for a polling interface for SRCU grace
periods, so this commit supplies get_state_synchronize_srcu(),
start_poll_synchronize_srcu(), and poll_state_synchronize_srcu() for this
purpose.  The first can be used if future grace periods are inevitable
(perhaps due to a later call_srcu() invocation), the second if future
grace periods might not otherwise happen, and the third to check if a
grace period has elapsed since the corresponding call to either of the
first two.

As with get_state_synchronize_rcu() and cond_synchronize_rcu(),
the return value from either get_state_synchronize_srcu() or
start_poll_synchronize_srcu() must be passed in to a later call to
poll_state_synchronize_srcu().

Link: https://lore.kernel.org/rcu/20201112201547.GF3365678@moria.home.lan/


Reported-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
[ paulmck: Add EXPORT_SYMBOL_GPL() per kernel test robot feedback. ]
[ paulmck: Apply feedback from Neeraj Upadhyay. ]
Link: https://lore.kernel.org/lkml/20201117004017.GA7444@paulmck-ThinkPad-P72/


Reviewed-by: default avatarNeeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 8b5bd67c

kernel/rcu/srcutree.c

+63 −4
Original line number Diff line number Diff line
@@ -810,7 +810,8 @@ static void srcu_leak_callback(struct rcu_head *rhp) 
/*

 * Start an SRCU grace period, and also queue the callback if non-NULL.

 */

static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rhp, bool do_norm)

static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,

					     struct rcu_head *rhp, bool do_norm)

{

	unsigned long flags;

	int idx;
@@ -819,9 +820,11 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rh 
	unsigned long s;

	struct srcu_data *sdp;



	check_init_srcu_struct(ssp);

	idx = srcu_read_lock(ssp);

	sdp = raw_cpu_ptr(ssp->sda);

	spin_lock_irqsave_rcu_node(sdp, flags);

	if (rhp)

		rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);

	rcu_segcblist_advance(&sdp->srcu_cblist,

			      rcu_seq_current(&ssp->srcu_gp_seq));
@@ -841,6 +844,7 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rh 
	else if (needexp)

		srcu_funnel_exp_start(ssp, sdp->mynode, s);

	srcu_read_unlock(ssp, idx);

	return s;

}



/*
@@ -874,7 +878,6 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rh 
static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,

			rcu_callback_t func, bool do_norm)

{

	check_init_srcu_struct(ssp);

	if (debug_rcu_head_queue(rhp)) {

		/* Probable double call_srcu(), so leak the callback. */

		WRITE_ONCE(rhp->func, srcu_leak_callback);
@@ -882,7 +885,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, 
		return;

	}

	rhp->func = func;

	srcu_gp_start_if_needed(ssp, rhp, do_norm);

	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);

}



/**
@@ -1011,6 +1014,62 @@ void synchronize_srcu(struct srcu_struct *ssp) 
}

EXPORT_SYMBOL_GPL(synchronize_srcu);



/**

 * get_state_synchronize_srcu - Provide an end-of-grace-period cookie

 * @ssp: srcu_struct to provide cookie for.

 *

 * This function returns a cookie that can be passed to

 * poll_state_synchronize_srcu(), which will return true if a full grace

 * period has elapsed in the meantime.  It is the caller's responsibility

 * to make sure that grace period happens, for example, by invoking

 * call_srcu() after return from get_state_synchronize_srcu().

 */

unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)

{

	// Any prior manipulation of SRCU-protected data must happen

	// before the load from ->srcu_gp_seq.

	smp_mb();

	return rcu_seq_snap(&ssp->srcu_gp_seq);

}

EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);



/**

 * start_poll_synchronize_srcu - Provide cookie and start grace period

 * @ssp: srcu_struct to provide cookie for.

 *

 * This function returns a cookie that can be passed to

 * poll_state_synchronize_srcu(), which will return true if a full grace

 * period has elapsed in the meantime.  Unlike get_state_synchronize_srcu(),

 * this function also ensures that any needed SRCU grace period will be

 * started.  This convenience does come at a cost in terms of CPU overhead.

 */

unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)

{

	return srcu_gp_start_if_needed(ssp, NULL, true);

}

EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);



/**

 * poll_state_synchronize_srcu - Has cookie's grace period ended?

 * @ssp: srcu_struct to provide cookie for.

 * @cookie: Return value from get_state_synchronize_srcu() or start_poll_synchronize_srcu().

 *

 * This function takes the cookie that was returned from either

 * get_state_synchronize_srcu() or start_poll_synchronize_srcu(), and

 * returns @true if an SRCU grace period elapsed since the time that the

 * cookie was created.

 */

bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)

{

	if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie))

		return false;

	// Ensure that the end of the SRCU grace period happens before

	// any subsequent code that the caller might execute.

	smp_mb(); // ^^^

	return true;

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);



/*

 * Callback function for srcu_barrier() use.

 */