Commit d8be8173 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Create a tiny SRCU 



In response to automated complaints about modifications to SRCU
increasing its size, this commit creates a tiny SRCU that is
used in SMP=n && PREEMPT=n builds.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent dde8da6c

include/linux/srcu.h

+8 −61
Original line number Diff line number Diff line
@@ -34,28 +34,7 @@ 
#include <linux/workqueue.h>

#include <linux/rcu_segcblist.h>



struct srcu_array {

	unsigned long lock_count[2];

	unsigned long unlock_count[2];

};



struct srcu_struct {

	unsigned long completed;

	unsigned long srcu_gp_seq;

	atomic_t srcu_exp_cnt;

	struct srcu_array __percpu *per_cpu_ref;

	spinlock_t queue_lock; /* protect ->srcu_cblist */

	struct rcu_segcblist srcu_cblist;

	struct delayed_work work;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map dep_map;

#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

};



/* Values for -> state variable. */

#define SRCU_STATE_IDLE		0

#define SRCU_STATE_SCAN1	1

#define SRCU_STATE_SCAN2	2

struct srcu_struct;



#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -77,42 +56,13 @@ int init_srcu_struct(struct srcu_struct *sp); 
#define __SRCU_DEP_MAP_INIT(srcu_name)

#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */



void process_srcu(struct work_struct *work);



#define __SRCU_STRUCT_INIT(name)					\

	{								\

		.completed = -300,					\

		.per_cpu_ref = &name##_srcu_array,			\

		.queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock),	\

		.srcu_cblist = RCU_SEGCBLIST_INITIALIZER(name.srcu_cblist),\

		.work = __DELAYED_WORK_INITIALIZER(name.work, process_srcu, 0),\

		__SRCU_DEP_MAP_INIT(name)				\

	}



/*

 * Define and initialize a srcu struct at build time.

 * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.

 *

 * Note that although DEFINE_STATIC_SRCU() hides the name from other

 * files, the per-CPU variable rules nevertheless require that the

 * chosen name be globally unique.  These rules also prohibit use of

 * DEFINE_STATIC_SRCU() within a function.  If these rules are too

 * restrictive, declare the srcu_struct manually.  For example, in

 * each file:

 *

 *	static struct srcu_struct my_srcu;

 *

 * Then, before the first use of each my_srcu, manually initialize it:

 *

 *	init_srcu_struct(&my_srcu);

 *

 * See include/linux/percpu-defs.h for the rules on per-CPU variables.

 */

#define __DEFINE_SRCU(name, is_static)					\

	static DEFINE_PER_CPU(struct srcu_array, name##_srcu_array);\

	is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)

#define DEFINE_SRCU(name)		__DEFINE_SRCU(name, /* not static */)

#define DEFINE_STATIC_SRCU(name)	__DEFINE_SRCU(name, static)

#ifdef CONFIG_TINY_SRCU

#include <linux/srcutiny.h>

#elif defined(CONFIG_TREE_SRCU)

#include <linux/srcutree.h>

#else

#error "Unknown SRCU implementation specified to kernel configuration"

#endif



/**

 * call_srcu() - Queue a callback for invocation after an SRCU grace period
@@ -138,9 +88,6 @@ void cleanup_srcu_struct(struct srcu_struct *sp); 
int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);

void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);

void synchronize_srcu(struct srcu_struct *sp);

void synchronize_srcu_expedited(struct srcu_struct *sp);

unsigned long srcu_batches_completed(struct srcu_struct *sp);

void srcu_barrier(struct srcu_struct *sp);



#ifdef CONFIG_DEBUG_LOCK_ALLOC

include/linux/srcutiny.h

0 → 100644
+81 −0
Original line number Diff line number Diff line 
/*

 * Sleepable Read-Copy Update mechanism for mutual exclusion,

 *	tiny variant.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, you can access it online at

 * http://www.gnu.org/licenses/gpl-2.0.html.

 *

 * Copyright (C) IBM Corporation, 2017

 *

 * Author: Paul McKenney <paulmck@us.ibm.com>

 */



#ifndef _LINUX_SRCU_TINY_H

#define _LINUX_SRCU_TINY_H



#include <linux/swait.h>



struct srcu_struct {

	int srcu_lock_nesting[2];	/* srcu_read_lock() nesting depth. */

	struct swait_queue_head srcu_wq;

					/* Last srcu_read_unlock() wakes GP. */

	unsigned long srcu_gp_seq;	/* GP seq # for callback tagging. */

	struct rcu_segcblist srcu_cblist;

					/* Pending SRCU callbacks. */

	int srcu_idx;			/* Current reader array element. */

	bool srcu_gp_running;		/* GP workqueue running? */

	bool srcu_gp_waiting;		/* GP waiting for readers? */

	struct work_struct srcu_work;	/* For driving grace periods. */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map dep_map;

#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

};



void srcu_drive_gp(struct work_struct *wp);



#define __SRCU_STRUCT_INIT(name)					\

{									\

	.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),	\

	.srcu_cblist = RCU_SEGCBLIST_INITIALIZER(name.srcu_cblist),	\

	.srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp),	\

	__SRCU_DEP_MAP_INIT(name)					\

}



/*

 * This odd _STATIC_ arrangement is needed for API compatibility with

 * Tree SRCU, which needs some per-CPU data.

 */

#define DEFINE_SRCU(name) \

	struct srcu_struct name = __SRCU_STRUCT_INIT(name)

#define DEFINE_STATIC_SRCU(name) \

	static struct srcu_struct name = __SRCU_STRUCT_INIT(name)



void synchronize_srcu(struct srcu_struct *sp);



static inline void synchronize_srcu_expedited(struct srcu_struct *sp)

{

	synchronize_srcu(sp);

}



static inline void srcu_barrier(struct srcu_struct *sp)

{

	synchronize_srcu(sp);

}



static inline unsigned long srcu_batches_completed(struct srcu_struct *sp)

{

	return 0;

}



#endif

include/linux/srcutree.h

0 → 100644
+91 −0
Original line number Diff line number Diff line 
/*

 * Sleepable Read-Copy Update mechanism for mutual exclusion,

 *	tree variant.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, you can access it online at

 * http://www.gnu.org/licenses/gpl-2.0.html.

 *

 * Copyright (C) IBM Corporation, 2017

 *

 * Author: Paul McKenney <paulmck@us.ibm.com>

 */



#ifndef _LINUX_SRCU_TREE_H

#define _LINUX_SRCU_TREE_H



struct srcu_array {

	unsigned long lock_count[2];

	unsigned long unlock_count[2];

};



struct srcu_struct {

	unsigned long completed;

	unsigned long srcu_gp_seq;

	atomic_t srcu_exp_cnt;

	struct srcu_array __percpu *per_cpu_ref;

	spinlock_t queue_lock; /* protect ->srcu_cblist */

	struct rcu_segcblist srcu_cblist;

	struct delayed_work work;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map dep_map;

#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

};



/* Values for -> state variable. */

#define SRCU_STATE_IDLE		0

#define SRCU_STATE_SCAN1	1

#define SRCU_STATE_SCAN2	2



void process_srcu(struct work_struct *work);



#define __SRCU_STRUCT_INIT(name)					\

	{								\

		.completed = -300,					\

		.per_cpu_ref = &name##_srcu_array,			\

		.queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock),	\

		.srcu_cblist = RCU_SEGCBLIST_INITIALIZER(name.srcu_cblist),\

		.work = __DELAYED_WORK_INITIALIZER(name.work, process_srcu, 0),\

		__SRCU_DEP_MAP_INIT(name)				\

	}



/*

 * Define and initialize a srcu struct at build time.

 * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.

 *

 * Note that although DEFINE_STATIC_SRCU() hides the name from other

 * files, the per-CPU variable rules nevertheless require that the

 * chosen name be globally unique.  These rules also prohibit use of

 * DEFINE_STATIC_SRCU() within a function.  If these rules are too

 * restrictive, declare the srcu_struct manually.  For example, in

 * each file:

 *

 *	static struct srcu_struct my_srcu;

 *

 * Then, before the first use of each my_srcu, manually initialize it:

 *

 *	init_srcu_struct(&my_srcu);

 *

 * See include/linux/percpu-defs.h for the rules on per-CPU variables.

 */

#define __DEFINE_SRCU(name, is_static)					\

	static DEFINE_PER_CPU(struct srcu_array, name##_srcu_array);\

	is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)

#define DEFINE_SRCU(name)		__DEFINE_SRCU(name, /* not static */)

#define DEFINE_STATIC_SRCU(name)	__DEFINE_SRCU(name, static)



void synchronize_srcu_expedited(struct srcu_struct *sp);

void srcu_barrier(struct srcu_struct *sp);

unsigned long srcu_batches_completed(struct srcu_struct *sp);



#endif

init/Kconfig

+12 −0
Original line number Diff line number Diff line
@@ -526,6 +526,18 @@ config SRCU 
	  permits arbitrary sleeping or blocking within RCU read-side critical

	  sections.



config TINY_SRCU

	bool

	default y if TINY_RCU

	help

	  This option selects the single-CPU non-preemptible version of SRCU.



config TREE_SRCU

	bool

	default y if !TINY_RCU

	help

	  This option selects the full-fledged version of SRCU.



config TASKS_RCU

	bool

	default n

kernel/rcu/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -3,7 +3,8 @@ 
KCOV_INSTRUMENT := n



obj-y += update.o sync.o

obj-$(CONFIG_SRCU) += srcu.o

obj-$(CONFIG_TREE_SRCU) += srcu.o

obj-$(CONFIG_TINY_SRCU) += srcutiny.o

obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o

obj-$(CONFIG_RCU_PERF_TEST) += rcuperf.o

obj-$(CONFIG_TREE_RCU) += tree.o