Commit 74612a07 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Make Tiny SRCU use multi-bit grace-period counter 

There is a need for a polling interface for SRCU grace periods.  This
polling needs to distinguish between an SRCU instance being idle on the
one hand or in the middle of a grace period on the other.  This commit
therefore converts the Tiny SRCU srcu_struct structure's srcu_idx from
a defacto boolean to a free-running counter, using the bottom bit to
indicate that a grace period is in progress.  The second-from-bottom
bit is thus used as the index returned by srcu_read_lock().

Link: https://lore.kernel.org/rcu/20201112201547.GF3365678@moria.home.lan/


Reported-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
[ paulmck: Fix ->srcu_lock_nesting[] indexing per Neeraj Upadhyay. ]
Reviewed-by: default avatarNeeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 5c8fe583

include/linux/srcutiny.h

+3 −3
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 


struct srcu_struct {

	short srcu_lock_nesting[2];	/* srcu_read_lock() nesting depth. */

	short srcu_idx;			/* Current reader array element. */

	unsigned short srcu_idx;	/* Current reader array element in bit 0x2. */

	u8 srcu_gp_running;		/* GP workqueue running? */

	u8 srcu_gp_waiting;		/* GP waiting for readers? */

	struct swait_queue_head srcu_wq;
@@ -59,7 +59,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp) 
{

	int idx;



	idx = READ_ONCE(ssp->srcu_idx);

	idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1;

	WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1);

	return idx;

}
@@ -80,7 +80,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, 
{

	int idx;



	idx = READ_ONCE(ssp->srcu_idx) & 0x1;

	idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1;

	pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",

		 tt, tf, idx,

		 READ_ONCE(ssp->srcu_lock_nesting[!idx]),

kernel/rcu/srcutiny.c

+3 −2
Original line number Diff line number Diff line
@@ -124,11 +124,12 @@ void srcu_drive_gp(struct work_struct *wp) 
	ssp->srcu_cb_head = NULL;

	ssp->srcu_cb_tail = &ssp->srcu_cb_head;

	local_irq_enable();

	idx = ssp->srcu_idx;

	WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx);

	idx = (ssp->srcu_idx & 0x2) / 2;

	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);

	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */

	swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));

	WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */

	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);



	/* Invoke the callbacks we removed above. */

	while (lh) {