Commit 95433f72 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Begin offloading srcu_struct fields to srcu_update 

The current srcu_struct structure is on the order of 200 bytes in size
(depending on architecture and .config), which is much better than the
old-style 26K bytes, but still all too inconvenient when one is trying
to achieve good cache locality on a fastpath involving SRCU readers.

However, only a few fields in srcu_struct are used by SRCU readers.
The remaining fields could be offloaded to a new srcu_update
structure, thus shrinking the srcu_struct structure down to a few
tens of bytes.  This commit begins this noble quest, a quest that is
complicated by open-coded initialization of the srcu_struct within the
srcu_notifier_head structure.  This complication is addressed by updating
the srcu_notifier_head structure's open coding, given that there does
not appear to be a straightforward way of abstracting that initialization.

This commit moves only the ->node pointer to srcu_update.  Later commits
will move additional fields.

[ paulmck: Fold in qiang1.zhang@intel.com's memory-leak fix. ]

Link: https://lore.kernel.org/all/20230320055751.4120251-1-qiang1.zhang@intel.com/


Suggested-by: default avatarChristoph Hellwig <hch@lst.de>
Cc: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Cc: "Michał Mirosław" <mirq-linux@rere.qmqm.pl>
Cc: Dmitry Osipenko <dmitry.osipenko@collabora.com>
Tested-by: default avatarSachin Sant <sachinp@linux.ibm.com>
Tested-by: default avatar"Zhang, Qiang1" <qiang1.zhang@intel.com>
Acked-by: default avatarRafael J. Wysocki <rafael.j.wysocki@intel.com>
Tested-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent f4d01a25

include/linux/notifier.h

+4 −1
Original line number Diff line number Diff line
@@ -73,6 +73,9 @@ struct raw_notifier_head { 


struct srcu_notifier_head {

	struct mutex mutex;

#ifdef CONFIG_TREE_SRCU

	struct srcu_usage srcuu;

#endif

	struct srcu_struct srcu;

	struct notifier_block __rcu *head;

};
@@ -107,7 +110,7 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); 
	{							\

		.mutex = __MUTEX_INITIALIZER(name.mutex),	\

		.head = NULL,					\

		.srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu),	\

		.srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \

	}



#define ATOMIC_NOTIFIER_HEAD(name)				\

include/linux/srcutiny.h

+3 −3
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ struct srcu_struct { 


void srcu_drive_gp(struct work_struct *wp);



#define __SRCU_STRUCT_INIT(name, __ignored)				\

#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored)			\

{									\

	.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),	\

	.srcu_cb_tail = &name.srcu_cb_head,				\
@@ -44,9 +44,9 @@ void srcu_drive_gp(struct work_struct *wp); 
 * Tree SRCU, which needs some per-CPU data.

 */

#define DEFINE_SRCU(name) \

	struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)

	struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)

#define DEFINE_STATIC_SRCU(name) \

	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)

	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)



void synchronize_srcu(struct srcu_struct *ssp);

include/linux/srcutree.h

+18 −9
Original line number Diff line number Diff line
@@ -57,11 +57,17 @@ struct srcu_node { 
	int grphi;				/* Biggest CPU for node. */

};



/*

 * Per-SRCU-domain structure, update-side data linked from srcu_struct.

 */

struct srcu_usage {

	struct srcu_node *node;			/* Combining tree. */

};



/*

 * Per-SRCU-domain structure, similar in function to rcu_state.

 */

struct srcu_struct {

	struct srcu_node *node;			/* Combining tree. */

	struct srcu_node *level[RCU_NUM_LVLS + 1];

						/* First node at each level. */

	int srcu_size_state;			/* Small-to-big transition state. */
@@ -90,6 +96,7 @@ struct srcu_struct { 
	unsigned long reschedule_count;

	struct delayed_work work;

	struct lockdep_map dep_map;

	struct srcu_usage *srcu_sup;		/* Update-side data. */

};



/* Values for size state variable (->srcu_size_state). */
@@ -108,24 +115,24 @@ struct srcu_struct { 
#define SRCU_STATE_SCAN1	1

#define SRCU_STATE_SCAN2	2



#define __SRCU_STRUCT_INIT_COMMON(name)								\

#define __SRCU_STRUCT_INIT_COMMON(name, usage_name)						\

	.lock = __SPIN_LOCK_UNLOCKED(name.lock),						\

	.srcu_gp_seq_needed = -1UL,								\

	.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),					\

	.srcu_sup = &usage_name,								\

	__SRCU_DEP_MAP_INIT(name)



#define __SRCU_STRUCT_INIT_MODULE(name)								\

#define __SRCU_STRUCT_INIT_MODULE(name, usage_name)						\

{												\

	__SRCU_STRUCT_INIT_COMMON(name)								\

	__SRCU_STRUCT_INIT_COMMON(name, usage_name)						\

}



#define __SRCU_STRUCT_INIT(name, pcpu_name)							\

#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name)						\

{												\

	.sda = &pcpu_name,									\

	__SRCU_STRUCT_INIT_COMMON(name)								\

	__SRCU_STRUCT_INIT_COMMON(name, usage_name)						\

}





/*

 * Define and initialize a srcu struct at build time.

 * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.
@@ -147,15 +154,17 @@ struct srcu_struct { 
 */

#ifdef MODULE

# define __DEFINE_SRCU(name, is_static)								\

	is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name);			\

	static struct srcu_usage name##_srcu_usage;						\

	is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage);	\

	extern struct srcu_struct * const __srcu_struct_##name;					\

	struct srcu_struct * const __srcu_struct_##name						\

		__section("___srcu_struct_ptrs") = &name

#else

# define __DEFINE_SRCU(name, is_static)								\

	static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data);				\

	static struct srcu_usage name##_srcu_usage;						\

	is_static struct srcu_struct name =							\

		__SRCU_STRUCT_INIT(name, name##_srcu_data)

		__SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data)

#endif

#define DEFINE_SRCU(name)		__DEFINE_SRCU(name, /* not static */)

#define DEFINE_STATIC_SRCU(name)	__DEFINE_SRCU(name, static)

kernel/rcu/rcu.h

+4 −2
Original line number Diff line number Diff line
@@ -341,11 +341,13 @@ extern void rcu_init_geometry(void); 
 * specified state structure (for SRCU) or the only rcu_state structure

 * (for RCU).

 */

#define srcu_for_each_node_breadth_first(sp, rnp) \

#define _rcu_for_each_node_breadth_first(sp, rnp) \

	for ((rnp) = &(sp)->node[0]; \

	     (rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++)

#define rcu_for_each_node_breadth_first(rnp) \

	srcu_for_each_node_breadth_first(&rcu_state, rnp)

	_rcu_for_each_node_breadth_first(&rcu_state, rnp)

#define srcu_for_each_node_breadth_first(ssp, rnp) \

	_rcu_for_each_node_breadth_first(ssp->srcu_sup, rnp)



/*

 * Scan the leaves of the rcu_node hierarchy for the rcu_state structure.

kernel/rcu/srcutree.c

+19 −9
Original line number Diff line number Diff line
@@ -173,12 +173,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) 


	/* Initialize geometry if it has not already been initialized. */

	rcu_init_geometry();

	ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags);

	if (!ssp->node)

	ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);

	if (!ssp->srcu_sup->node)

		return false;



	/* Work out the overall tree geometry. */

	ssp->level[0] = &ssp->node[0];

	ssp->level[0] = &ssp->srcu_sup->node[0];

	for (i = 1; i < rcu_num_lvls; i++)

		ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1];

	rcu_init_levelspread(levelspread, num_rcu_lvl);
@@ -195,7 +195,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) 
		snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ;

		snp->grplo = -1;

		snp->grphi = -1;

		if (snp == &ssp->node[0]) {

		if (snp == &ssp->srcu_sup->node[0]) {

			/* Root node, special case. */

			snp->srcu_parent = NULL;

			continue;
@@ -236,8 +236,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) 
 */

static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)

{

	if (!is_static)

		ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);

	if (!ssp->srcu_sup)

		return -ENOMEM;

	ssp->srcu_size_state = SRCU_SIZE_SMALL;

	ssp->node = NULL;

	ssp->srcu_sup->node = NULL;

	mutex_init(&ssp->srcu_cb_mutex);

	mutex_init(&ssp->srcu_gp_mutex);

	ssp->srcu_idx = 0;
@@ -249,8 +253,11 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
	ssp->sda_is_static = is_static;

	if (!is_static)

		ssp->sda = alloc_percpu(struct srcu_data);

	if (!ssp->sda)

	if (!ssp->sda) {

		if (!is_static)

			kfree(ssp->srcu_sup);

		return -ENOMEM;

	}

	init_srcu_struct_data(ssp);

	ssp->srcu_gp_seq_needed_exp = 0;

	ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
@@ -259,6 +266,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
			if (!ssp->sda_is_static) {

				free_percpu(ssp->sda);

				ssp->sda = NULL;

				kfree(ssp->srcu_sup);

				return -ENOMEM;

			}

		} else {
@@ -656,13 +664,15 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) 
			rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed);

		return; /* Caller forgot to stop doing call_srcu()? */

	}

	kfree(ssp->srcu_sup->node);

	ssp->srcu_sup->node = NULL;

	ssp->srcu_size_state = SRCU_SIZE_SMALL;

	if (!ssp->sda_is_static) {

		free_percpu(ssp->sda);

		ssp->sda = NULL;

		kfree(ssp->srcu_sup);

		ssp->srcu_sup = NULL;

	}

	kfree(ssp->node);

	ssp->node = NULL;

	ssp->srcu_size_state = SRCU_SIZE_SMALL;

}

EXPORT_SYMBOL_GPL(cleanup_srcu_struct);