Commit 76ea3641 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Add full-sized polling for start_poll() 



The start_poll_synchronize_rcu() API compresses the combined expedited and
normal grace-period states into a single unsigned long, which conserves
storage, but can miss grace periods in certain cases involving overlapping
normal and expedited grace periods.  Missing the occasional grace period
is usually not a problem, but there are use cases that care about each
and every grace period.

This commit therefore adds the next member of the full-state RCU
grace-period polling API, namely the start_poll_synchronize_rcu_full()
function.  This uses up to three times the storage (rcu_gp_oldstate
structure instead of unsigned long), but is guaranteed not to miss
grace periods.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent f4754ad2

include/linux/rcutiny.h

+6 −0
Original line number Diff line number Diff line
@@ -26,6 +26,12 @@ static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) 
}



unsigned long start_poll_synchronize_rcu(void);



static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

{

	rgosp->rgos_norm = start_poll_synchronize_rcu();

}



bool poll_state_synchronize_rcu(unsigned long oldstate);



static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

include/linux/rcutree.h

+1 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate); 
unsigned long get_state_synchronize_rcu(void);

void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

unsigned long start_poll_synchronize_rcu(void);

void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

bool poll_state_synchronize_rcu(unsigned long oldstate);

bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

void cond_synchronize_rcu(unsigned long oldstate);

kernel/rcu/rcutorture.c

+40 −9
Original line number Diff line number Diff line
@@ -88,6 +88,7 @@ torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 
torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");

torture_param(bool, gp_poll, false, "Use polling GP wait primitives");

torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");

torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives");

torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");

torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");

torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
@@ -198,12 +199,14 @@ static int rcu_torture_writer_state; 
#define RTWS_COND_SYNC		7

#define RTWS_COND_SYNC_EXP	8

#define RTWS_POLL_GET		9

#define RTWS_POLL_GET_EXP	10

#define RTWS_POLL_WAIT		11

#define RTWS_POLL_WAIT_EXP	12

#define RTWS_SYNC		13

#define RTWS_STUTTER		14

#define RTWS_STOPPING		15

#define RTWS_POLL_GET_FULL	10

#define RTWS_POLL_GET_EXP	11

#define RTWS_POLL_WAIT		12

#define RTWS_POLL_WAIT_FULL	13

#define RTWS_POLL_WAIT_EXP	14

#define RTWS_SYNC		15

#define RTWS_STUTTER		16

#define RTWS_STOPPING		17

static const char * const rcu_torture_writer_state_names[] = {

	"RTWS_FIXED_DELAY",

	"RTWS_DELAY",
@@ -215,8 +218,10 @@ static const char * const rcu_torture_writer_state_names[] = { 
	"RTWS_COND_SYNC",

	"RTWS_COND_SYNC_EXP",

	"RTWS_POLL_GET",

	"RTWS_POLL_GET_FULL",

	"RTWS_POLL_GET_EXP",

	"RTWS_POLL_WAIT",

	"RTWS_POLL_WAIT_FULL",

	"RTWS_POLL_WAIT_EXP",

	"RTWS_SYNC",

	"RTWS_STUTTER",
@@ -339,6 +344,7 @@ struct rcu_torture_ops { 
	unsigned long (*get_gp_completed)(void);

	void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);

	unsigned long (*start_gp_poll)(void);

	void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp);

	bool (*poll_gp_state)(unsigned long oldstate);

	bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);

	bool (*poll_need_2gp)(bool poll, bool poll_full);
@@ -515,6 +521,7 @@ static struct rcu_torture_ops rcu_ops = { 
	.get_gp_completed	= get_completed_synchronize_rcu,

	.get_gp_completed_full	= get_completed_synchronize_rcu_full,

	.start_gp_poll		= start_poll_synchronize_rcu,

	.start_gp_poll_full	= start_poll_synchronize_rcu_full,

	.poll_gp_state		= poll_state_synchronize_rcu,

	.poll_gp_state_full	= poll_state_synchronize_rcu_full,

	.poll_need_2gp		= rcu_poll_need_2gp,
@@ -1163,13 +1170,13 @@ static void rcu_torture_write_types(void) 
{

	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp;

	bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll;

	bool gp_sync1 = gp_sync;

	bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;



	/* Initialize synctype[] array.  If none set, take default. */

	if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp &&

	    !gp_normal1 && !gp_poll1 && !gp_sync1)

	    !gp_normal1 && !gp_poll1 && !gp_poll_full1 && !gp_sync1)

		gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 =

			   gp_normal1 = gp_poll1 = gp_sync1 = true;

			   gp_normal1 = gp_poll1 = gp_poll_full1 = gp_sync1 = true;

	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {

		synctype[nsynctypes++] = RTWS_COND_GET;

		pr_info("%s: Testing conditional GPs.\n", __func__);
@@ -1200,6 +1207,12 @@ static void rcu_torture_write_types(void) 
	} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {

		pr_alert("%s: gp_poll without primitives.\n", __func__);

	}

	if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {

		synctype[nsynctypes++] = RTWS_POLL_GET_FULL;

		pr_info("%s: Testing polling full-state GPs.\n", __func__);

	} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {

		pr_alert("%s: gp_poll_full without primitives.\n", __func__);

	}

	if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {

		synctype[nsynctypes++] = RTWS_POLL_GET_EXP;

		pr_info("%s: Testing polling expedited GPs.\n", __func__);
@@ -1262,6 +1275,7 @@ rcu_torture_writer(void *arg) 
	struct rcu_gp_oldstate cookie_full;

	int expediting = 0;

	unsigned long gp_snap;

	struct rcu_gp_oldstate gp_snap_full;

	int i;

	int idx;

	int oldnice = task_nice(current);
@@ -1376,6 +1390,15 @@ rcu_torture_writer(void *arg) 
								  &rand);

				rcu_torture_pipe_update(old_rp);

				break;

			case RTWS_POLL_GET_FULL:

				rcu_torture_writer_state = RTWS_POLL_GET_FULL;

				cur_ops->start_gp_poll_full(&gp_snap_full);

				rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;

				while (!cur_ops->poll_gp_state_full(&gp_snap_full))

					torture_hrtimeout_jiffies(torture_random(&rand) % 16,

								  &rand);

				rcu_torture_pipe_update(old_rp);

				break;

			case RTWS_POLL_GET_EXP:

				rcu_torture_writer_state = RTWS_POLL_GET_EXP;

				gp_snap = cur_ops->start_gp_poll_exp();
@@ -1454,6 +1477,7 @@ static int 
rcu_torture_fakewriter(void *arg)

{

	unsigned long gp_snap;

	struct rcu_gp_oldstate gp_snap_full;

	DEFINE_TORTURE_RANDOM(rand);



	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
@@ -1499,6 +1523,13 @@ rcu_torture_fakewriter(void *arg) 
								  &rand);

				}

				break;

			case RTWS_POLL_GET_FULL:

				cur_ops->start_gp_poll_full(&gp_snap_full);

				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {

					torture_hrtimeout_jiffies(torture_random(&rand) % 16,

								  &rand);

				}

				break;

			case RTWS_POLL_GET_EXP:

				gp_snap = cur_ops->start_gp_poll_exp();

				while (!cur_ops->poll_gp_state_exp(gp_snap)) {

kernel/rcu/tree.c

+45 −13
Original line number Diff line number Diff line
@@ -3589,22 +3589,13 @@ void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) 
}

EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full);



/**

 * start_poll_synchronize_rcu - Snapshot and start RCU grace period

 *

 * Returns a cookie that is used by a later call to cond_synchronize_rcu()

 * or poll_state_synchronize_rcu() to determine whether or not a full

 * grace period has elapsed in the meantime.  If the needed grace period

 * is not already slated to start, notifies RCU core of the need for that

 * grace period.

 *

 * Interrupts must be enabled for the case where it is necessary to awaken

 * the grace-period kthread.

/*

 * Helper function for start_poll_synchronize_rcu() and

 * start_poll_synchronize_rcu_full().

 */

unsigned long start_poll_synchronize_rcu(void)

static void start_poll_synchronize_rcu_common(void)

{

	unsigned long flags;

	unsigned long gp_seq = get_state_synchronize_rcu();

	bool needwake;

	struct rcu_data *rdp;

	struct rcu_node *rnp;
@@ -3624,10 +3615,51 @@ unsigned long start_poll_synchronize_rcu(void) 
	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

	if (needwake)

		rcu_gp_kthread_wake();

}



/**

 * start_poll_synchronize_rcu - Snapshot and start RCU grace period

 *

 * Returns a cookie that is used by a later call to cond_synchronize_rcu()

 * or poll_state_synchronize_rcu() to determine whether or not a full

 * grace period has elapsed in the meantime.  If the needed grace period

 * is not already slated to start, notifies RCU core of the need for that

 * grace period.

 *

 * Interrupts must be enabled for the case where it is necessary to awaken

 * the grace-period kthread.

 */

unsigned long start_poll_synchronize_rcu(void)

{

	unsigned long gp_seq = get_state_synchronize_rcu();



	start_poll_synchronize_rcu_common();

	return gp_seq;

}

EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);



/**

 * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period

 * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full()

 *

 * Places the normal and expedited grace-period states in *@rgos.  This

 * state value can be passed to a later call to cond_synchronize_rcu_full()

 * or poll_state_synchronize_rcu_full() to determine whether or not a

 * grace period (whether normal or expedited) has elapsed in the meantime.

 * If the needed grace period is not already slated to start, notifies

 * RCU core of the need for that grace period.

 *

 * Interrupts must be enabled for the case where it is necessary to awaken

 * the grace-period kthread.

 */

void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

{

	get_state_synchronize_rcu_full(rgosp);



	start_poll_synchronize_rcu_common();

}

EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full);



/**

 * poll_state_synchronize_rcu - Has the specified RCU grace period completed?

 *