Commit 6c502b14 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Add full-sized polling for start_poll_expedited() 



The start_poll_synchronize_rcu_expedited() API compresses the combined
expedited and normal grace-period states into a single unsigned long,
which conserves storage, but can miss grace periods in certain cases
involving overlapping normal and expedited grace periods.  Missing the
occasional grace period is usually not a problem, but there are use
cases that care about each and every grace period.

This commit therefore adds yet another member of the
full-state RCU grace-period polling API, which is the
start_poll_synchronize_rcu_expedited_full() function.  This uses up to
three times the storage (rcu_gp_oldstate structure instead of unsigned
long), but is guaranteed not to miss grace periods.

[ paulmck: Apply feedback from kernel test robot and Julia Lawall. ]

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 76ea3641

include/linux/rcutiny.h

+5 −0
Original line number Diff line number Diff line
@@ -49,6 +49,11 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void) 
	return start_poll_synchronize_rcu();

}



static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)

{

	rgosp->rgos_norm = start_poll_synchronize_rcu_expedited();

}



static inline void cond_synchronize_rcu_expedited(unsigned long oldstate)

{

	cond_synchronize_rcu(oldstate);

include/linux/rcutree.h

+1 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ struct rcu_gp_oldstate { 
};



unsigned long start_poll_synchronize_rcu_expedited(void);

void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);

void cond_synchronize_rcu_expedited(unsigned long oldstate);

unsigned long get_state_synchronize_rcu(void);

void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

kernel/rcu/rcutorture.c

+41 −10
Original line number Diff line number Diff line
@@ -89,6 +89,7 @@ torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primit 
torture_param(bool, gp_poll, false, "Use polling GP wait primitives");

torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");

torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives");

torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives");

torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");

torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");

torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
@@ -201,12 +202,14 @@ static int rcu_torture_writer_state; 
#define RTWS_POLL_GET		9

#define RTWS_POLL_GET_FULL	10

#define RTWS_POLL_GET_EXP	11

#define RTWS_POLL_WAIT		12

#define RTWS_POLL_WAIT_FULL	13

#define RTWS_POLL_WAIT_EXP	14

#define RTWS_SYNC		15

#define RTWS_STUTTER		16

#define RTWS_STOPPING		17

#define RTWS_POLL_GET_EXP_FULL	12

#define RTWS_POLL_WAIT		13

#define RTWS_POLL_WAIT_FULL	14

#define RTWS_POLL_WAIT_EXP	15

#define RTWS_POLL_WAIT_EXP_FULL	16

#define RTWS_SYNC		17

#define RTWS_STUTTER		18

#define RTWS_STOPPING		19

static const char * const rcu_torture_writer_state_names[] = {

	"RTWS_FIXED_DELAY",

	"RTWS_DELAY",
@@ -220,9 +223,11 @@ static const char * const rcu_torture_writer_state_names[] = { 
	"RTWS_POLL_GET",

	"RTWS_POLL_GET_FULL",

	"RTWS_POLL_GET_EXP",

	"RTWS_POLL_GET_EXP_FULL",

	"RTWS_POLL_WAIT",

	"RTWS_POLL_WAIT_FULL",

	"RTWS_POLL_WAIT_EXP",

	"RTWS_POLL_WAIT_EXP_FULL",

	"RTWS_SYNC",

	"RTWS_STUTTER",

	"RTWS_STOPPING",
@@ -337,6 +342,7 @@ struct rcu_torture_ops { 
	void (*exp_sync)(void);

	unsigned long (*get_gp_state_exp)(void);

	unsigned long (*start_gp_poll_exp)(void);

	void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp);

	bool (*poll_gp_state_exp)(unsigned long oldstate);

	void (*cond_sync_exp)(unsigned long oldstate);

	unsigned long (*get_gp_state)(void);
@@ -528,6 +534,7 @@ static struct rcu_torture_ops rcu_ops = { 
	.cond_sync		= cond_synchronize_rcu,

	.get_gp_state_exp	= get_state_synchronize_rcu,

	.start_gp_poll_exp	= start_poll_synchronize_rcu_expedited,

	.start_gp_poll_exp_full	= start_poll_synchronize_rcu_expedited_full,

	.poll_gp_state_exp	= poll_state_synchronize_rcu,

	.cond_sync_exp		= cond_synchronize_rcu_expedited,

	.call			= call_rcu,
@@ -1169,13 +1176,14 @@ static int nsynctypes; 
static void rcu_torture_write_types(void)

{

	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp;

	bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll;

	bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;

	bool gp_poll_exp1 = gp_poll_exp, gp_poll_exp_full1 = gp_poll_exp_full;

	bool gp_normal1 = gp_normal, gp_poll1 = gp_poll, gp_poll_full1 = gp_poll_full;

	bool gp_sync1 = gp_sync;



	/* Initialize synctype[] array.  If none set, take default. */

	if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp &&

	if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp && !gp_poll_exp_full1 &&

	    !gp_normal1 && !gp_poll1 && !gp_poll_full1 && !gp_sync1)

		gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 =

		gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 = gp_poll_exp_full1 =

			   gp_normal1 = gp_poll1 = gp_poll_full1 = gp_sync1 = true;

	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {

		synctype[nsynctypes++] = RTWS_COND_GET;
@@ -1219,6 +1227,13 @@ static void rcu_torture_write_types(void) 
	} else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {

		pr_alert("%s: gp_poll_exp without primitives.\n", __func__);

	}

	if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {

		synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL;

		pr_info("%s: Testing polling full-state expedited GPs.\n", __func__);

	} else if (gp_poll_exp_full &&

		   (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {

		pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__);

	}

	if (gp_sync1 && cur_ops->sync) {

		synctype[nsynctypes++] = RTWS_SYNC;

		pr_info("%s: Testing normal GPs.\n", __func__);
@@ -1408,6 +1423,15 @@ rcu_torture_writer(void *arg) 
								  &rand);

				rcu_torture_pipe_update(old_rp);

				break;

			case RTWS_POLL_GET_EXP_FULL:

				rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL;

				cur_ops->start_gp_poll_exp_full(&gp_snap_full);

				rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL;

				while (!cur_ops->poll_gp_state_full(&gp_snap_full))

					torture_hrtimeout_jiffies(torture_random(&rand) % 16,

								  &rand);

				rcu_torture_pipe_update(old_rp);

				break;

			case RTWS_SYNC:

				rcu_torture_writer_state = RTWS_SYNC;

				do_rtws_sync(&rand, cur_ops->sync);
@@ -1537,6 +1561,13 @@ rcu_torture_fakewriter(void *arg) 
								  &rand);

				}

				break;

			case RTWS_POLL_GET_EXP_FULL:

				cur_ops->start_gp_poll_exp_full(&gp_snap_full);

				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {

					torture_hrtimeout_jiffies(torture_random(&rand) % 16,

								  &rand);

				}

				break;

			case RTWS_SYNC:

				cur_ops->sync();

				break;

kernel/rcu/tree_exp.h

+18 −0
Original line number Diff line number Diff line
@@ -1027,6 +1027,24 @@ unsigned long start_poll_synchronize_rcu_expedited(void) 
}

EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited);



/**

 * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period

 * @rgosp: Place to put snapshot of grace-period state

 *

 * Places the normal and expedited grace-period states in rgosp.  This

 * state value can be passed to a later call to cond_synchronize_rcu_full()

 * or poll_state_synchronize_rcu_full() to determine whether or not a

 * grace period (whether normal or expedited) has elapsed in the meantime.

 * If the needed expedited grace period is not already slated to start,

 * initiates that grace period.

 */

void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)

{

	get_state_synchronize_rcu_full(rgosp);

	(void)start_poll_synchronize_rcu_expedited();

}

EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full);



/**

 * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period

 *