locking: Apply contention tracepoints in the slow path (ee042be1) · Commits · EulixOS / Software / Kernel

kernel/locking/mutex.c

+3 −0

Original line number	Diff line number	Diff line
		@@ -644,6 +644,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
		}

		set_current_state(state);
		trace_contention_begin(lock, 0);
		for (;;) {
		bool first;

		@@ -710,6 +711,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
		skip_wait:
		/* got the lock - cleanup and rejoice! */
		lock_acquired(&lock->dep_map, ip);
		trace_contention_end(lock, 0);

		if (ww_ctx)
		ww_mutex_lock_acquired(ww, ww_ctx);
		@@ -721,6 +723,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
		err:
		__set_current_state(TASK_RUNNING);
		__mutex_remove_waiter(lock, &waiter);
		trace_contention_end(lock, ret);
		err_early_kill:
		raw_spin_unlock(&lock->wait_lock);
		debug_mutex_free_waiter(&waiter);

+5 −0

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@
		#include <linux/sched/task.h>
		#include <linux/sched/debug.h>
		#include <linux/errno.h>
		#include <trace/events/lock.h>

		int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
		const char name, struct lock_class_key key)
		@@ -171,9 +172,11 @@ bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
		if (try)
		return false;

		trace_contention_begin(sem, LCB_F_PERCPU \| LCB_F_READ);
		preempt_enable();
		percpu_rwsem_wait(sem, /* .reader = */ true);
		preempt_disable();
		trace_contention_end(sem, 0);

		return true;
		}
		@@ -216,6 +219,7 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
		{
		might_sleep();
		rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
		trace_contention_begin(sem, LCB_F_PERCPU \| LCB_F_WRITE);

		/* Notify readers to take the slow path. */
		rcu_sync_enter(&sem->rss);
		@@ -237,6 +241,7 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem)

		/* Wait for all active readers to complete. */
		rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
		trace_contention_end(sem, 0);
		}
		EXPORT_SYMBOL_GPL(percpu_down_write);

+9 −0

Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@
		#include <linux/percpu.h>
		#include <linux/hardirq.h>
		#include <linux/spinlock.h>
		#include <trace/events/lock.h>

		/**
		* queued_read_lock_slowpath - acquire read lock of a queue rwlock
		@@ -34,6 +35,8 @@ void queued_read_lock_slowpath(struct qrwlock *lock)
		}
		atomic_sub(_QR_BIAS, &lock->cnts);

		trace_contention_begin(lock, LCB_F_SPIN \| LCB_F_READ);

		/*
		* Put the reader into the wait queue
		*/
		@@ -51,6 +54,8 @@ void queued_read_lock_slowpath(struct qrwlock *lock)
		* Signal the next one in queue to become queue head
		*/
		arch_spin_unlock(&lock->wait_lock);

		trace_contention_end(lock, 0);
		}
		EXPORT_SYMBOL(queued_read_lock_slowpath);

		@@ -62,6 +67,8 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
		{
		int cnts;

		trace_contention_begin(lock, LCB_F_SPIN \| LCB_F_WRITE);

		/* Put the writer into the wait queue */
		arch_spin_lock(&lock->wait_lock);

		@@ -79,5 +86,7 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
		} while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
		unlock:
		arch_spin_unlock(&lock->wait_lock);

		trace_contention_end(lock, 0);
		}
		EXPORT_SYMBOL(queued_write_lock_slowpath);

+5 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@
		#include <linux/prefetch.h>
		#include <asm/byteorder.h>
		#include <asm/qspinlock.h>
		#include <trace/events/lock.h>

		/*
		* Include queued spinlock statistics code
		@@ -401,6 +402,8 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
		idx = node->count++;
		tail = encode_tail(smp_processor_id(), idx);

		trace_contention_begin(lock, LCB_F_SPIN);

		/*
		* 4 nodes are allocated based on the assumption that there will
		* not be nested NMIs taking spinlocks. That may not be true in
		@@ -554,6 +557,8 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
		pv_kick_node(lock, next);

		release:
		trace_contention_end(lock, 0);

		/*
		* release the node
		*/

+11 −0

Original line number	Diff line number	Diff line
		@@ -24,6 +24,8 @@
		#include <linux/sched/wake_q.h>
		#include <linux/ww_mutex.h>

		#include <trace/events/lock.h>

		#include "rtmutex_common.h"

		#ifndef WW_RT
		@@ -1579,6 +1581,8 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,

		set_current_state(state);

		trace_contention_begin(lock, LCB_F_RT);

		ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);
		if (likely(!ret))
		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);
		@@ -1601,6 +1605,9 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
		* unconditionally. We might have to fix that up.
		*/
		fixup_rt_mutex_waiters(lock);

		trace_contention_end(lock, ret);

		return ret;
		}

		@@ -1683,6 +1690,8 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
		/* Save current state and set state to TASK_RTLOCK_WAIT */
		current_save_and_set_rtlock_wait_state();

		trace_contention_begin(lock, LCB_F_RT);

		task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);

		for (;;) {
		@@ -1712,6 +1721,8 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
		*/
		fixup_rt_mutex_waiters(lock);
		debug_rt_mutex_free_waiter(&waiter);

		trace_contention_end(lock, 0);
		}

		static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)