Commit 5fe89191 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Make Tiny SRCU use full-sized grace-period counters 



This commit makes Tiny SRCU use full-sized grace-period counters to
further avoid counter-wrap issues when using polled grace-period APIs.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent de3f2671

include/linux/srcutiny.h

+3 −3
Original line number Diff line number Diff line
@@ -15,10 +15,10 @@ 


struct srcu_struct {

	short srcu_lock_nesting[2];	/* srcu_read_lock() nesting depth. */

	unsigned short srcu_idx;	/* Current reader array element in bit 0x2. */

	unsigned short srcu_idx_max;	/* Furthest future srcu_idx request. */

	u8 srcu_gp_running;		/* GP workqueue running? */

	u8 srcu_gp_waiting;		/* GP waiting for readers? */

	unsigned long srcu_idx;		/* Current reader array element in bit 0x2. */

	unsigned long srcu_idx_max;	/* Furthest future srcu_idx request. */

	struct swait_queue_head srcu_wq;

					/* Last srcu_read_unlock() wakes GP. */

	struct rcu_head *srcu_cb_head;	/* Pending callbacks: Head. */
@@ -82,7 +82,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, 
	int idx;



	idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1;

	pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %hu->%hu\n",

	pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n",

		 tt, tf, idx,

		 data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])),

		 data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])),

kernel/rcu/srcutiny.c

+7 −7
Original line number Diff line number Diff line
@@ -117,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) 
	struct srcu_struct *ssp;



	ssp = container_of(wp, struct srcu_struct, srcu_work);

	if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))

	if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))

		return; /* Already running or nothing to do. */



	/* Remove recently arrived callbacks and wait for readers. */
@@ -150,17 +150,17 @@ void srcu_drive_gp(struct work_struct *wp) 
	 * straighten that out.

	 */

	WRITE_ONCE(ssp->srcu_gp_running, false);

	if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))

	if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))

		schedule_work(&ssp->srcu_work);

}

EXPORT_SYMBOL_GPL(srcu_drive_gp);



static void srcu_gp_start_if_needed(struct srcu_struct *ssp)

{

	unsigned short cookie;

	unsigned long cookie;



	cookie = get_state_synchronize_srcu(ssp);

	if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))

	if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))

		return;

	WRITE_ONCE(ssp->srcu_idx_max, cookie);

	if (!READ_ONCE(ssp->srcu_gp_running)) {
@@ -215,7 +215,7 @@ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) 
	barrier();

	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;

	barrier();

	return ret & USHRT_MAX;

	return ret;

}

EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
@@ -240,10 +240,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); 
 */

bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)

{

	unsigned short cur_s = READ_ONCE(ssp->srcu_idx);

	unsigned long cur_s = READ_ONCE(ssp->srcu_idx);



	barrier();

	return USHORT_CMP_GE(cur_s, cookie) || USHORT_CMP_LT(cur_s, cookie - 3);

	return ULONG_CMP_GE(cur_s, cookie) || ULONG_CMP_LT(cur_s, cookie - 3);

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);