Commit 8b5bd67c authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Provide polling interfaces for Tiny SRCU grace periods 

There is a need for a polling interface for SRCU grace
periods, so this commit supplies get_state_synchronize_srcu(),
start_poll_synchronize_srcu(), and poll_state_synchronize_srcu() for this
purpose.  The first can be used if future grace periods are inevitable
(perhaps due to a later call_srcu() invocation), the second if future
grace periods might not otherwise happen, and the third to check if a
grace period has elapsed since the corresponding call to either of the
first two.

As with get_state_synchronize_rcu() and cond_synchronize_rcu(),
the return value from either get_state_synchronize_srcu() or
start_poll_synchronize_srcu() must be passed in to a later call to
poll_state_synchronize_srcu().

Link: https://lore.kernel.org/rcu/20201112201547.GF3365678@moria.home.lan/


Reported-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
[ paulmck: Add EXPORT_SYMBOL_GPL() per kernel test robot feedback. ]
[ paulmck: Apply feedback from Neeraj Upadhyay. ]
Link: https://lore.kernel.org/lkml/20201117004017.GA7444@paulmck-ThinkPad-P72/


Reviewed-by: default avatarNeeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 29d2bb94

include/linux/rcupdate.h

+2 −0
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ 
#define ULONG_CMP_GE(a, b)	(ULONG_MAX / 2 >= (a) - (b))

#define ULONG_CMP_LT(a, b)	(ULONG_MAX / 2 < (a) - (b))

#define ulong2long(a)		(*(long *)(&(a)))

#define USHORT_CMP_GE(a, b)	(USHRT_MAX / 2 >= (unsigned short)((a) - (b)))

#define USHORT_CMP_LT(a, b)	(USHRT_MAX / 2 < (unsigned short)((a) - (b)))



/* Exported common interfaces */

void call_rcu(struct rcu_head *head, rcu_callback_t func);

include/linux/srcu.h

+3 −0
Original line number Diff line number Diff line
@@ -60,6 +60,9 @@ void cleanup_srcu_struct(struct srcu_struct *ssp); 
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);

void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);

void synchronize_srcu(struct srcu_struct *ssp);

unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);

unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);

bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);



#ifdef CONFIG_DEBUG_LOCK_ALLOC

include/linux/srcutiny.h

+1 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
struct srcu_struct {

	short srcu_lock_nesting[2];	/* srcu_read_lock() nesting depth. */

	unsigned short srcu_idx;	/* Current reader array element in bit 0x2. */

	unsigned short srcu_idx_max;	/* Furthest future srcu_idx request. */

	u8 srcu_gp_running;		/* GP workqueue running? */

	u8 srcu_gp_waiting;		/* GP waiting for readers? */

	struct swait_queue_head srcu_wq;

kernel/rcu/srcutiny.c

+53 −2
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) 
	ssp->srcu_gp_running = false;

	ssp->srcu_gp_waiting = false;

	ssp->srcu_idx = 0;

	ssp->srcu_idx_max = 0;

	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);

	INIT_LIST_HEAD(&ssp->srcu_work.entry);

	return 0;
@@ -84,6 +85,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) 
	WARN_ON(ssp->srcu_gp_waiting);

	WARN_ON(ssp->srcu_cb_head);

	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);

	WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);

	WARN_ON(ssp->srcu_idx & 0x1);

}

EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
@@ -114,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) 
	struct srcu_struct *ssp;



	ssp = container_of(wp, struct srcu_struct, srcu_work);

	if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head))

	if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))

		return; /* Already running or nothing to do. */



	/* Remove recently arrived callbacks and wait for readers. */
@@ -147,13 +150,19 @@ void srcu_drive_gp(struct work_struct *wp) 
	 * straighten that out.

	 */

	WRITE_ONCE(ssp->srcu_gp_running, false);

	if (READ_ONCE(ssp->srcu_cb_head))

	if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))

		schedule_work(&ssp->srcu_work);

}

EXPORT_SYMBOL_GPL(srcu_drive_gp);



static void srcu_gp_start_if_needed(struct srcu_struct *ssp)

{

	unsigned short cookie;



	cookie = get_state_synchronize_srcu(ssp);

	if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))

		return;

	WRITE_ONCE(ssp->srcu_idx_max, cookie);

	if (!READ_ONCE(ssp->srcu_gp_running)) {

		if (likely(srcu_init_done))

			schedule_work(&ssp->srcu_work);
@@ -196,6 +205,48 @@ void synchronize_srcu(struct srcu_struct *ssp) 
}

EXPORT_SYMBOL_GPL(synchronize_srcu);



/*

 * get_state_synchronize_srcu - Provide an end-of-grace-period cookie

 */

unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)

{

	unsigned long ret;



	barrier();

	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;

	barrier();

	return ret & USHRT_MAX;

}

EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);



/*

 * start_poll_synchronize_srcu - Provide cookie and start grace period

 *

 * The difference between this and get_state_synchronize_srcu() is that

 * this function ensures that the poll_state_synchronize_srcu() will

 * eventually return the value true.

 */

unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)

{

	unsigned long ret = get_state_synchronize_srcu(ssp);



	srcu_gp_start_if_needed(ssp);

	return ret;

}

EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);



/*

 * poll_state_synchronize_srcu - Has cookie's grace period ended?

 */

bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)

{

	bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie);



	barrier();

	return ret;

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);



/* Lockdep diagnostics.  */

void __init rcu_scheduler_starting(void)

{