Commit 7abb18bd authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Provide polling interfaces for Tree RCU grace periods 



There is a need for a non-blocking polling interface for RCU grace
periods, so this commit supplies start_poll_synchronize_rcu() and
poll_state_synchronize_rcu() for this purpose.  Note that the existing
get_state_synchronize_rcu() may be used if future grace periods are
inevitable (perhaps due to a later call_rcu() invocation).  The new
start_poll_synchronize_rcu() is to be used if future grace periods
might not otherwise happen.  Finally, poll_state_synchronize_rcu()
provides a lockless check for a grace period having elapsed since
the corresponding call to either of the get_state_synchronize_rcu()
or start_poll_synchronize_rcu().

As with get_state_synchronize_rcu(), the return value from either
get_state_synchronize_rcu() or start_poll_synchronize_rcu() is passed in
to a later call to either poll_state_synchronize_rcu() or the existing
(might_sleep) cond_synchronize_rcu().

[ paulmck: Remove redundant smp_mb() per Frederic Weisbecker feedback. ]
[ Update poll_state_synchronize_rcu() docbook per Frederic Weisbecker feedback. ]
Reviewed-by: default avatarFrederic Weisbecker <frederic@kernel.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent a38fd874

include/linux/rcutree.h

+2 −0
Original line number Diff line number Diff line
@@ -41,6 +41,8 @@ void rcu_momentary_dyntick_idle(void); 
void kfree_rcu_scheduler_running(void);

bool rcu_gp_might_be_stalled(void);

unsigned long get_state_synchronize_rcu(void);

unsigned long start_poll_synchronize_rcu(void);

bool poll_state_synchronize_rcu(unsigned long oldstate);

void cond_synchronize_rcu(unsigned long oldstate);



void rcu_idle_enter(void);

kernel/rcu/tree.c

+68 −7
Original line number Diff line number Diff line
@@ -3774,8 +3774,8 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); 
 * get_state_synchronize_rcu - Snapshot current RCU state

 *

 * Returns a cookie that is used by a later call to cond_synchronize_rcu()

 * to determine whether or not a full grace period has elapsed in the

 * meantime.

 * or poll_state_synchronize_rcu() to determine whether or not a full

 * grace period has elapsed in the meantime.

 */

unsigned long get_state_synchronize_rcu(void)

{
@@ -3788,14 +3788,77 @@ unsigned long get_state_synchronize_rcu(void) 
}

EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);



/**

 * start_poll_synchronize_rcu - Snapshot and start RCU grace period

 *

 * Returns a cookie that is used by a later call to cond_synchronize_rcu()

 * or poll_state_synchronize_rcu() to determine whether or not a full

 * grace period has elapsed in the meantime.  If the needed grace period

 * is not already slated to start, notifies RCU core of the need for that

 * grace period.

 *

 * Interrupts must be enabled for the case where it is necessary to awaken

 * the grace-period kthread.

 */

unsigned long start_poll_synchronize_rcu(void)

{

	unsigned long flags;

	unsigned long gp_seq = get_state_synchronize_rcu();

	bool needwake;

	struct rcu_data *rdp;

	struct rcu_node *rnp;



	lockdep_assert_irqs_enabled();

	local_irq_save(flags);

	rdp = this_cpu_ptr(&rcu_data);

	rnp = rdp->mynode;

	raw_spin_lock_rcu_node(rnp); // irqs already disabled.

	needwake = rcu_start_this_gp(rnp, rdp, gp_seq);

	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

	if (needwake)

		rcu_gp_kthread_wake();

	return gp_seq;

}

EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);



/**

 * poll_state_synchronize_rcu - Conditionally wait for an RCU grace period

 *

 * @oldstate: return from call to get_state_synchronize_rcu() or start_poll_synchronize_rcu()

 *

 * If a full RCU grace period has elapsed since the earlier call from

 * which oldstate was obtained, return @true, otherwise return @false.

 * If @false is returned, it is the caller's responsibilty to invoke this

 * function later on until it does return @true.  Alternatively, the caller

 * can explicitly wait for a grace period, for example, by passing @oldstate

 * to cond_synchronize_rcu() or by directly invoking synchronize_rcu().

 *

 * Yes, this function does not take counter wrap into account.

 * But counter wrap is harmless.  If the counter wraps, we have waited for

 * more than 2 billion grace periods (and way more on a 64-bit system!).

 * Those needing to keep oldstate values for very long time periods

 * (many hours even on 32-bit systems) should check them occasionally

 * and either refresh them or set a flag indicating that the grace period

 * has completed.

 */

bool poll_state_synchronize_rcu(unsigned long oldstate)

{

	if (rcu_seq_done(&rcu_state.gp_seq, oldstate)) {

		smp_mb(); /* Ensure GP ends before subsequent accesses. */

		return true;

	}

	return false;

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu);



/**

 * cond_synchronize_rcu - Conditionally wait for an RCU grace period

 *

 * @oldstate: return value from earlier call to get_state_synchronize_rcu()

 *

 * If a full RCU grace period has elapsed since the earlier call to

 * get_state_synchronize_rcu(), just return.  Otherwise, invoke

 * synchronize_rcu() to wait for a full grace period.

 * get_state_synchronize_rcu() or start_poll_synchronize_rcu(), just return.

 * Otherwise, invoke synchronize_rcu() to wait for a full grace period.

 *

 * Yes, this function does not take counter wrap into account.  But

 * counter wrap is harmless.  If the counter wraps, we have waited for
@@ -3804,10 +3867,8 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); 
 */

void cond_synchronize_rcu(unsigned long oldstate)

{

	if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))

	if (!poll_state_synchronize_rcu(oldstate))

		synchronize_rcu();

	else

		smp_mb(); /* Ensure GP ends before subsequent accesses. */

}

EXPORT_SYMBOL_GPL(cond_synchronize_rcu);