Commit 456cfbc6 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Ingo Molnar
Browse files

locking/rtmutex: Prepare RT rt_mutex_wake_q for RT locks 



Add an rtlock_task pointer to rt_mutex_wake_q, which allows to handle the RT
specific wakeup for spin/rwlock waiters. The pointer is just consuming 4/8
bytes on the stack so it is provided unconditionaly to avoid #ifdeffery all
over the place.

This cannot use a regular wake_q, because a task can have concurrent wakeups which
would make it miss either lock or the regular wakeups, depending on what gets
queued first, unless task struct gains a separate wake_q_node for this, which
would be overkill, because there can only be a single task which gets woken
up in the spin/rw_lock unlock path.

No functional change for non-RT enabled kernels.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210815211303.253614678@linutronix.de
parent 7980aa39

kernel/locking/rtmutex.c

+16 −2
Original line number Diff line number Diff line
@@ -351,11 +351,25 @@ static __always_inline void rt_mutex_adjust_prio(struct task_struct *p) 
static __always_inline void rt_mutex_wake_q_add(struct rt_wake_q_head *wqh,

						struct rt_mutex_waiter *w)

{

	if (IS_ENABLED(CONFIG_PREEMPT_RT) && w->wake_state != TASK_NORMAL) {

		if (IS_ENABLED(CONFIG_PROVE_LOCKING))

			WARN_ON_ONCE(wqh->rtlock_task);

		get_task_struct(w->task);

		wqh->rtlock_task = w->task;

	} else {

		wake_q_add(&wqh->head, w->task);

	}

}



static __always_inline void rt_mutex_wake_up_q(struct rt_wake_q_head *wqh)

{

	if (IS_ENABLED(CONFIG_PREEMPT_RT) && wqh->rtlock_task) {

		wake_up_state(wqh->rtlock_task, TASK_RTLOCK_WAIT);

		put_task_struct(wqh->rtlock_task);

		wqh->rtlock_task = NULL;

	}



	if (!wake_q_empty(&wqh->head))

		wake_up_q(&wqh->head);



	/* Pairs with preempt_disable() in mark_wakeup_next_waiter() */

kernel/locking/rtmutex_common.h

+4 −1
Original line number Diff line number Diff line
@@ -43,14 +43,17 @@ struct rt_mutex_waiter { 
 * rt_wake_q_head - Wrapper around regular wake_q_head to support

 *		    "sleeping" spinlocks on RT

 * @head:		The regular wake_q_head for sleeping lock variants

 * @rtlock_task:	Task pointer for RT lock (spin/rwlock) wakeups

 */

struct rt_wake_q_head {

	struct wake_q_head	head;

	struct task_struct	*rtlock_task;

};



#define DEFINE_RT_WAKE_Q(name)						\

	struct rt_wake_q_head name = {					\

		.head		= WAKE_Q_HEAD_INITIALIZER(name.head),	\

		.rtlock_task	= NULL,					\

	}



/*