Commit c8fe8b05 authored by Waiman Long's avatar Waiman Long Committed by Peter Zijlstra
Browse files

locking/rwsem: Pass the current atomic count to rwsem_down_read_slowpath() 



The atomic count value right after reader count increment can be useful
to determine the rwsem state at trylock time. So the count value is
passed down to rwsem_down_read_slowpath() to be used when appropriate.

Signed-off-by: default avatarWaiman Long <longman@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarDavidlohr Bueso <dbueso@suse.de>
Link: https://lkml.kernel.org/r/20201121041416.12285-2-longman@redhat.com
parent c995e638

kernel/locking/rwsem.c

+10 −8
Original line number Diff line number Diff line
@@ -270,14 +270,14 @@ static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) 
					  owner | RWSEM_NONSPINNABLE));

}



static inline bool rwsem_read_trylock(struct rw_semaphore *sem)

static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp)

{

	long cnt = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count);

	*cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count);



	if (WARN_ON_ONCE(cnt < 0))

	if (WARN_ON_ONCE(*cntp < 0))

		rwsem_set_nonspinnable(sem);



	if (!(cnt & RWSEM_READ_FAILED_MASK)) {

	if (!(*cntp & RWSEM_READ_FAILED_MASK)) {

		rwsem_set_reader_owned(sem);

		return true;

	}
@@ -1008,9 +1008,9 @@ rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable) 
 * Wait for the read lock to be granted

 */

static struct rw_semaphore __sched *

rwsem_down_read_slowpath(struct rw_semaphore *sem, int state)

rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, int state)

{

	long count, adjustment = -RWSEM_READER_BIAS;

	long adjustment = -RWSEM_READER_BIAS;

	struct rwsem_waiter waiter;

	DEFINE_WAKE_Q(wake_q);

	bool wake = false;
@@ -1356,8 +1356,10 @@ static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) 
 */

static inline int __down_read_common(struct rw_semaphore *sem, int state)

{

	if (!rwsem_read_trylock(sem)) {

		if (IS_ERR(rwsem_down_read_slowpath(sem, state)))

	long count;



	if (!rwsem_read_trylock(sem, &count)) {

		if (IS_ERR(rwsem_down_read_slowpath(sem, count, state)))

			return -EINTR;

		DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);

	}