Commit 8df13f01 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Add full-sized polling for cond_sync_exp_full() 



The cond_synchronize_rcu_expedited() API compresses the combined expedited and
normal grace-period states into a single unsigned long, which conserves
storage, but can miss grace periods in certain cases involving overlapping
normal and expedited grace periods.  Missing the occasional grace period
is usually not a problem, but there are use cases that care about each
and every grace period.

This commit therefore adds yet another member of the full-state RCU
grace-period polling API, which is the cond_synchronize_rcu_exp_full()
function.  This uses up to three times the storage (rcu_gp_oldstate
structure instead of unsigned long), but is guaranteed not to miss
grace periods.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent b6fe4917

include/linux/rcutiny.h

+5 −0
Original line number Diff line number Diff line
@@ -64,6 +64,11 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) 
	cond_synchronize_rcu(oldstate);

}



static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)

{

	cond_synchronize_rcu_expedited(rgosp->rgos_norm);

}



extern void rcu_barrier(void);



static inline void synchronize_rcu_expedited(void)

include/linux/rcutree.h

+1 −0
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ struct rcu_gp_oldstate { 
unsigned long start_poll_synchronize_rcu_expedited(void);

void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);

void cond_synchronize_rcu_expedited(unsigned long oldstate);

void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);

unsigned long get_state_synchronize_rcu(void);

void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

unsigned long start_poll_synchronize_rcu(void);

kernel/rcu/rcutorture.c

+50 −22
Original line number Diff line number Diff line
@@ -85,6 +85,8 @@ torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind ne 
torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");

torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives");

torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives");

torture_param(bool, gp_cond_exp_full, false,

		    "Use conditional/async full-stateexpedited GP wait primitives");

torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");

torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");

torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
@@ -199,20 +201,22 @@ static int rcu_torture_writer_state; 
#define RTWS_COND_GET		5

#define RTWS_COND_GET_FULL	6

#define RTWS_COND_GET_EXP	7

#define RTWS_COND_SYNC		8

#define RTWS_COND_SYNC_FULL	9

#define RTWS_COND_SYNC_EXP	10

#define RTWS_POLL_GET		11

#define RTWS_POLL_GET_FULL	12

#define RTWS_POLL_GET_EXP	13

#define RTWS_POLL_GET_EXP_FULL	14

#define RTWS_POLL_WAIT		15

#define RTWS_POLL_WAIT_FULL	16

#define RTWS_POLL_WAIT_EXP	17

#define RTWS_POLL_WAIT_EXP_FULL	18

#define RTWS_SYNC		19

#define RTWS_STUTTER		20

#define RTWS_STOPPING		21

#define RTWS_COND_GET_EXP_FULL	8

#define RTWS_COND_SYNC		9

#define RTWS_COND_SYNC_FULL	10

#define RTWS_COND_SYNC_EXP	11

#define RTWS_COND_SYNC_EXP_FULL	12

#define RTWS_POLL_GET		13

#define RTWS_POLL_GET_FULL	14

#define RTWS_POLL_GET_EXP	15

#define RTWS_POLL_GET_EXP_FULL	16

#define RTWS_POLL_WAIT		17

#define RTWS_POLL_WAIT_FULL	18

#define RTWS_POLL_WAIT_EXP	19

#define RTWS_POLL_WAIT_EXP_FULL	20

#define RTWS_SYNC		21

#define RTWS_STUTTER		22

#define RTWS_STOPPING		23

static const char * const rcu_torture_writer_state_names[] = {

	"RTWS_FIXED_DELAY",

	"RTWS_DELAY",
@@ -222,9 +226,11 @@ static const char * const rcu_torture_writer_state_names[] = { 
	"RTWS_COND_GET",

	"RTWS_COND_GET_FULL",

	"RTWS_COND_GET_EXP",

	"RTWS_COND_GET_EXP_FULL",

	"RTWS_COND_SYNC",

	"RTWS_COND_SYNC_FULL",

	"RTWS_COND_SYNC_EXP",

	"RTWS_COND_SYNC_EXP_FULL",

	"RTWS_POLL_GET",

	"RTWS_POLL_GET_FULL",

	"RTWS_POLL_GET_EXP",
@@ -350,6 +356,7 @@ struct rcu_torture_ops { 
	void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp);

	bool (*poll_gp_state_exp)(unsigned long oldstate);

	void (*cond_sync_exp)(unsigned long oldstate);

	void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp);

	unsigned long (*get_gp_state)(void);

	void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);

	unsigned long (*get_gp_completed)(void);
@@ -1183,16 +1190,17 @@ static int nsynctypes; 
static void rcu_torture_write_types(void)

{

	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full;

	bool gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp, gp_poll_exp_full1 = gp_poll_exp_full;

	bool gp_normal1 = gp_normal, gp_poll1 = gp_poll, gp_poll_full1 = gp_poll_full;

	bool gp_sync1 = gp_sync;

	bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp;

	bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll;

	bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;



	/* Initialize synctype[] array.  If none set, take default. */

	if (!gp_cond1 && !gp_cond_exp1 && !gp_cond_full1 && !gp_exp1 && !gp_poll_exp &&

	    !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 && !gp_sync1)

		gp_cond1 = gp_cond_exp1 = gp_cond_full1 = gp_exp1 = gp_poll_exp1 =

			   gp_poll_exp_full1 = gp_normal1 = gp_poll1 = gp_poll_full1 =

			   gp_sync1 = true;

	if (!gp_cond1 && !gp_cond_exp1 && !gp_cond_full1 && !gp_cond_exp_full1 && !gp_exp1 &&

	    !gp_poll_exp && !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 &&

	    !gp_sync1)

		gp_cond1 = gp_cond_exp1 = gp_cond_full1 = gp_cond_exp_full1 = gp_exp1 =

			   gp_poll_exp1 = gp_poll_exp_full1 = gp_normal1 = gp_poll1 =

			   gp_poll_full1 = gp_sync1 = true;

	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {

		synctype[nsynctypes++] = RTWS_COND_GET;

		pr_info("%s: Testing conditional GPs.\n", __func__);
@@ -1211,6 +1219,13 @@ static void rcu_torture_write_types(void) 
	} else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {

		pr_alert("%s: gp_cond_full without primitives.\n", __func__);

	}

	if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {

		synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL;

		pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__);

	} else if (gp_cond_exp_full &&

		   (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {

		pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__);

	}

	if (gp_exp1 && cur_ops->exp_sync) {

		synctype[nsynctypes++] = RTWS_EXP_SYNC;

		pr_info("%s: Testing expedited GPs.\n", __func__);
@@ -1418,6 +1433,14 @@ rcu_torture_writer(void *arg) 
				cur_ops->cond_sync_full(&gp_snap_full);

				rcu_torture_pipe_update(old_rp);

				break;

			case RTWS_COND_GET_EXP_FULL:

				rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL;

				cur_ops->get_gp_state_full(&gp_snap_full);

				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);

				rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL;

				cur_ops->cond_sync_exp_full(&gp_snap_full);

				rcu_torture_pipe_update(old_rp);

				break;

			case RTWS_POLL_GET:

				rcu_torture_writer_state = RTWS_POLL_GET;

				gp_snap = cur_ops->start_gp_poll();
@@ -1567,6 +1590,11 @@ rcu_torture_fakewriter(void *arg) 
				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);

				cur_ops->cond_sync_full(&gp_snap_full);

				break;

			case RTWS_COND_GET_EXP_FULL:

				cur_ops->get_gp_state_full(&gp_snap_full);

				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);

				cur_ops->cond_sync_exp_full(&gp_snap_full);

				break;

			case RTWS_POLL_GET:

				gp_snap = cur_ops->start_gp_poll();

				while (!cur_ops->poll_gp_state(gp_snap)) {

kernel/rcu/tree_exp.h

+27 −0
Original line number Diff line number Diff line
@@ -1071,3 +1071,30 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate) 
		synchronize_rcu_expedited();

}

EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited);



/**

 * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period

 * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full()

 *

 * If a full RCU grace period has elapsed since the call to

 * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),

 * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was

 * obtained, just return.  Otherwise, invoke synchronize_rcu_expedited()

 * to wait for a full grace period.

 *

 * Yes, this function does not take counter wrap into account.

 * But counter wrap is harmless.  If the counter wraps, we have waited for

 * more than 2 billion grace periods (and way more on a 64-bit system!),

 * so waiting for a couple of additional grace periods should be just fine.

 *

 * This function provides the same memory-ordering guarantees that

 * would be provided by a synchronize_rcu() that was invoked at the call

 * to the function that provided @rgosp and that returned at the end of

 * this function.

 */

void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)

{

	if (!poll_state_synchronize_rcu_full(rgosp))

		synchronize_rcu_expedited();

}

EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full);