Unverified Commit f3d58fc0 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!6795 rtmutex: Add acquire semantics for rtmutex lock acquisition slow path 

Merge Pull Request from: @ci-robot 
 
PR sync from: Yang Yingliang <yangyingliang@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/YWEGUQHQRYFUW5OCQZEFGXZHGKLV6KFV/ 
 
https://gitee.com/openeuler/kernel/issues/I9JNPZ 
 
Link:https://gitee.com/openeuler/kernel/pulls/6795

 

Reviewed-by: default avatarXie XiuQi <xiexiuqi@huawei.com>
Reviewed-by: default avatarLiu YongQiang <liuyongqiang13@huawei.com>
Signed-off-by: default avatarZhang Changzhong <zhangchangzhong@huawei.com>
parents bb8542f3 08759148

kernel/locking/rtmutex.c

+47 −11
Original line number Diff line number Diff line
@@ -48,15 +48,31 @@ 
 * set this bit before looking at the lock.

 */



static void

rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)

static struct task_struct *

rt_mutex_owner_encode(struct rt_mutex *lock, struct task_struct *owner)

{

	unsigned long val = (unsigned long)owner;



	if (rt_mutex_has_waiters(lock))

		val |= RT_MUTEX_HAS_WAITERS;



	lock->owner = (struct task_struct *)val;

	return (struct task_struct *)val;

}



static void

rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)

{

	/*

	 * lock->wait_lock is held but explicit acquire semantics are needed

	 * for a new lock owner so WRITE_ONCE is insufficient.

	 */

	xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner));

}



static void rt_mutex_clear_owner(struct rt_mutex *lock)

{

	/* lock->wait_lock is held so the unlock provides release semantics. */

	WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));

}



static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
@@ -65,7 +81,7 @@ static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) 
			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);

}



static void fixup_rt_mutex_waiters(struct rt_mutex *lock)

static void fixup_rt_mutex_waiters(struct rt_mutex *lock, bool acquire_lock)

{

	unsigned long owner, *p = (unsigned long *) &lock->owner;
@@ -131,9 +147,22 @@ static void fixup_rt_mutex_waiters(struct rt_mutex *lock) 
	 * still set.

	 */

	owner = READ_ONCE(*p);

	if (owner & RT_MUTEX_HAS_WAITERS)

	if (owner & RT_MUTEX_HAS_WAITERS) {

		/*

		 * See rt_mutex_set_owner() and rt_mutex_clear_owner() on

		 * why xchg_acquire() is used for updating owner for

		 * locking and WRITE_ONCE() for unlocking.

		 *

		 * WRITE_ONCE() would work for the acquire case too, but

		 * in case that the lock acquisition failed it might

		 * force other lockers into the slow path unnecessarily.

		 */

		if (acquire_lock)

			xchg_acquire(p, owner & ~RT_MUTEX_HAS_WAITERS);

		else

			WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);

	}

}



/*

 * We can speed up the acquire/release, if there's no debugging state to be
@@ -157,6 +186,13 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) 
		owner = *p;

	} while (cmpxchg_relaxed(p, owner,

				 owner | RT_MUTEX_HAS_WAITERS) != owner);



	/*

	 * The cmpxchg loop above is relaxed to avoid back-to-back ACQUIRE

	 * operations in the event of contention. Ensure the successful

	 * cmpxchg is visible.

	 */

	smp_mb__after_atomic();

}



/*
@@ -1276,7 +1312,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, 
	 * try_to_take_rt_mutex() sets the waiter bit

	 * unconditionally. We might have to fix that up.

	 */

	fixup_rt_mutex_waiters(lock);

	fixup_rt_mutex_waiters(lock, true);



	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
@@ -1297,7 +1333,7 @@ static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock) 
	 * try_to_take_rt_mutex() sets the lock waiters bit

	 * unconditionally. Clean this up.

	 */

	fixup_rt_mutex_waiters(lock);

	fixup_rt_mutex_waiters(lock, true);



	return ret;

}
@@ -1722,7 +1758,7 @@ void rt_mutex_init_proxy_locked(struct rt_mutex *lock, 
void rt_mutex_proxy_unlock(struct rt_mutex *lock)

{

	debug_rt_mutex_proxy_unlock(lock);

	rt_mutex_set_owner(lock, NULL);

	rt_mutex_clear_owner(lock);

}



/**
@@ -1859,7 +1895,7 @@ int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, 
	 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might

	 * have to fix that up.

	 */

	fixup_rt_mutex_waiters(lock);

	fixup_rt_mutex_waiters(lock, true);

	raw_spin_unlock_irq(&lock->wait_lock);



	return ret;
@@ -1915,7 +1951,7 @@ bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock, 
	 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might

	 * have to fix that up.

	 */

	fixup_rt_mutex_waiters(lock);

	fixup_rt_mutex_waiters(lock, false);



	raw_spin_unlock_irq(&lock->wait_lock);