Commit 8282947f authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Ingo Molnar
Browse files

locking/rwlock: Provide RT variant 



Similar to rw_semaphores, on RT the rwlock substitution is not writer fair,
because it's not feasible to have a writer inherit its priority to
multiple readers. Readers blocked on a writer follow the normal rules of
priority inheritance. Like RT spinlocks, RT rwlocks are state preserving
across the slow lock operations (contended case).

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210815211303.882793524@linutronix.de
parent 0f383b6d

include/linux/rwlock_rt.h

0 → 100644
+140 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

#ifndef __LINUX_RWLOCK_RT_H

#define __LINUX_RWLOCK_RT_H



#ifndef __LINUX_SPINLOCK_RT_H

#error Do not #include directly. Use <linux/spinlock.h>.

#endif



#ifdef CONFIG_DEBUG_LOCK_ALLOC

extern void __rt_rwlock_init(rwlock_t *rwlock, const char *name,

			     struct lock_class_key *key);

#else

static inline void __rt_rwlock_init(rwlock_t *rwlock, char *name,

				    struct lock_class_key *key)

{

}

#endif



#define rwlock_init(rwl)				\

do {							\

	static struct lock_class_key __key;		\

							\

	init_rwbase_rt(&(rwl)->rwbase);			\

	__rt_rwlock_init(rwl, #rwl, &__key);		\

} while (0)



extern void rt_read_lock(rwlock_t *rwlock);

extern int rt_read_trylock(rwlock_t *rwlock);

extern void rt_read_unlock(rwlock_t *rwlock);

extern void rt_write_lock(rwlock_t *rwlock);

extern int rt_write_trylock(rwlock_t *rwlock);

extern void rt_write_unlock(rwlock_t *rwlock);



static __always_inline void read_lock(rwlock_t *rwlock)

{

	rt_read_lock(rwlock);

}



static __always_inline void read_lock_bh(rwlock_t *rwlock)

{

	local_bh_disable();

	rt_read_lock(rwlock);

}



static __always_inline void read_lock_irq(rwlock_t *rwlock)

{

	rt_read_lock(rwlock);

}



#define read_lock_irqsave(lock, flags)			\

	do {						\

		typecheck(unsigned long, flags);	\

		rt_read_lock(lock);			\

		flags = 0;				\

	} while (0)



#define read_trylock(lock)	__cond_lock(lock, rt_read_trylock(lock))



static __always_inline void read_unlock(rwlock_t *rwlock)

{

	rt_read_unlock(rwlock);

}



static __always_inline void read_unlock_bh(rwlock_t *rwlock)

{

	rt_read_unlock(rwlock);

	local_bh_enable();

}



static __always_inline void read_unlock_irq(rwlock_t *rwlock)

{

	rt_read_unlock(rwlock);

}



static __always_inline void read_unlock_irqrestore(rwlock_t *rwlock,

						   unsigned long flags)

{

	rt_read_unlock(rwlock);

}



static __always_inline void write_lock(rwlock_t *rwlock)

{

	rt_write_lock(rwlock);

}



static __always_inline void write_lock_bh(rwlock_t *rwlock)

{

	local_bh_disable();

	rt_write_lock(rwlock);

}



static __always_inline void write_lock_irq(rwlock_t *rwlock)

{

	rt_write_lock(rwlock);

}



#define write_lock_irqsave(lock, flags)			\

	do {						\

		typecheck(unsigned long, flags);	\

		rt_write_lock(lock);			\

		flags = 0;				\

	} while (0)



#define write_trylock(lock)	__cond_lock(lock, rt_write_trylock(lock))



#define write_trylock_irqsave(lock, flags)		\

({							\

	int __locked;					\

							\

	typecheck(unsigned long, flags);		\

	flags = 0;					\

	__locked = write_trylock(lock);			\

	__locked;					\

})



static __always_inline void write_unlock(rwlock_t *rwlock)

{

	rt_write_unlock(rwlock);

}



static __always_inline void write_unlock_bh(rwlock_t *rwlock)

{

	rt_write_unlock(rwlock);

	local_bh_enable();

}



static __always_inline void write_unlock_irq(rwlock_t *rwlock)

{

	rt_write_unlock(rwlock);

}



static __always_inline void write_unlock_irqrestore(rwlock_t *rwlock,

						    unsigned long flags)

{

	rt_write_unlock(rwlock);

}



#define rwlock_is_contended(lock)		(((void)(lock), 0))



#endif /* __LINUX_RWLOCK_RT_H */

include/linux/rwlock_types.h

+37 −12
Original line number Diff line number Diff line
@@ -5,9 +5,19 @@ 
# error "Do not include directly, include spinlock_types.h"

#endif



#ifdef CONFIG_DEBUG_LOCK_ALLOC

# define RW_DEP_MAP_INIT(lockname)					\

	.dep_map = {							\

		.name = #lockname,					\

		.wait_type_inner = LD_WAIT_CONFIG,			\

	}

#else

# define RW_DEP_MAP_INIT(lockname)

#endif



#ifndef CONFIG_PREEMPT_RT

/*

 * include/linux/rwlock_types.h - generic rwlock type definitions

 *				  and initializers

 * generic rwlock type definitions and initializers

 *

 * portions Copyright 2005, Red Hat, Inc., Ingo Molnar

 * Released under the General Public License (GPL).
@@ -25,16 +35,6 @@ typedef struct { 


#define RWLOCK_MAGIC		0xdeaf1eed



#ifdef CONFIG_DEBUG_LOCK_ALLOC

# define RW_DEP_MAP_INIT(lockname)					\

	.dep_map = {							\

		.name = #lockname,					\

		.wait_type_inner = LD_WAIT_CONFIG,			\

	}

#else

# define RW_DEP_MAP_INIT(lockname)

#endif



#ifdef CONFIG_DEBUG_SPINLOCK

#define __RW_LOCK_UNLOCKED(lockname)					\

	(rwlock_t)	{	.raw_lock = __ARCH_RW_LOCK_UNLOCKED,	\
@@ -50,4 +50,29 @@ typedef struct { 


#define DEFINE_RWLOCK(x)	rwlock_t x = __RW_LOCK_UNLOCKED(x)



#else /* !CONFIG_PREEMPT_RT */



#include <linux/rwbase_rt.h>



typedef struct {

	struct rwbase_rt	rwbase;

	atomic_t		readers;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map	dep_map;

#endif

} rwlock_t;



#define __RWLOCK_RT_INITIALIZER(name)					\

{									\

	.rwbase = __RWBASE_INITIALIZER(name),				\

	RW_DEP_MAP_INIT(name)						\

}



#define __RW_LOCK_UNLOCKED(name) __RWLOCK_RT_INITIALIZER(name)



#define DEFINE_RWLOCK(name)						\

	rwlock_t name = __RW_LOCK_UNLOCKED(name)



#endif /* CONFIG_PREEMPT_RT */



#endif /* __LINUX_RWLOCK_TYPES_H */

include/linux/spinlock_rt.h

+2 −0
Original line number Diff line number Diff line
@@ -146,4 +146,6 @@ static inline int spin_is_locked(spinlock_t *lock) 


#define assert_spin_locked(lock) BUG_ON(!spin_is_locked(lock))



#include <linux/rwlock_rt.h>



#endif

kernel/Kconfig.locks

+1 −1
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ config ARCH_USE_QUEUED_RWLOCKS 


config QUEUED_RWLOCKS

	def_bool y if ARCH_USE_QUEUED_RWLOCKS

	depends on SMP

	depends on SMP && !PREEMPT_RT



config ARCH_HAS_MMIOWB

	bool

kernel/locking/spinlock.c

+7 −0
Original line number Diff line number Diff line
@@ -124,8 +124,11 @@ void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 
 *         __[spin|read|write]_lock_bh()

 */

BUILD_LOCK_OPS(spin, raw_spinlock);



#ifndef CONFIG_PREEMPT_RT

BUILD_LOCK_OPS(read, rwlock);

BUILD_LOCK_OPS(write, rwlock);

#endif



#endif
@@ -209,6 +212,8 @@ void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) 
EXPORT_SYMBOL(_raw_spin_unlock_bh);

#endif



#ifndef CONFIG_PREEMPT_RT



#ifndef CONFIG_INLINE_READ_TRYLOCK

int __lockfunc _raw_read_trylock(rwlock_t *lock)

{
@@ -353,6 +358,8 @@ void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) 
EXPORT_SYMBOL(_raw_write_unlock_bh);

#endif



#endif /* !CONFIG_PREEMPT_RT */



#ifdef CONFIG_DEBUG_LOCK_ALLOC



void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)