Commit 949fa3f1 authored by Leonardo Bras's avatar Leonardo Bras Committed by Peter Zijlstra
Browse files

trace,smp: Add tracepoints around remotelly called functions 



The recently added ipi_send_{cpu,cpumask} tracepoints allow finding sources
of IPIs targeting CPUs running latency-sensitive applications.

For NOHZ_FULL CPUs, all IPIs are interference, and those tracepoints are
sufficient to find them and work on getting rid of them. In some setups
however, not *all* IPIs are to be suppressed, but long-running IPI
callbacks can still be problematic.

Add a pair of tracepoints to mark the start and end of processing a CSD IPI
callback, similar to what exists for softirq, workqueue or timer callbacks.

Signed-off-by: default avatarLeonardo Bras <leobras@redhat.com>
Tested-and-reviewed-by: default avatarValentin Schneider <vschneid@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20230615065944.188876-5-leobras@redhat.com
parent 60be49bd

include/trace/events/csd.h

0 → 100644
+45 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#undef TRACE_SYSTEM

#define TRACE_SYSTEM csd



#if !defined(_TRACE_CSD_H) || defined(TRACE_HEADER_MULTI_READ)

#define _TRACE_CSD_H



#include <linux/tracepoint.h>



/*

 * Tracepoints for a function which is called as an effect of smp_call_function.*

 */

DECLARE_EVENT_CLASS(csd_function,



	TP_PROTO(smp_call_func_t func, struct __call_single_data *csd),



	TP_ARGS(func, csd),



	TP_STRUCT__entry(

		__field(void *,	func)

		__field(void *,	csd)

	),



	TP_fast_assign(

		__entry->func	= func;

		__entry->csd	= csd;

	),



	TP_printk("func=%ps, csd=%p", __entry->func, __entry->csd)

);



DEFINE_EVENT(csd_function, csd_function_entry,

	TP_PROTO(smp_call_func_t func, struct __call_single_data *csd),

	TP_ARGS(func, csd)

);



DEFINE_EVENT(csd_function, csd_function_exit,

	TP_PROTO(smp_call_func_t func, struct __call_single_data *csd),

	TP_ARGS(func, csd)

);



#endif /* _TRACE_CSD_H */



/* This part must be outside protection */

#include <trace/define_trace.h>

kernel/smp.c

+19 −6
Original line number Diff line number Diff line
@@ -27,6 +27,9 @@ 
#include <linux/jump_label.h>



#include <trace/events/ipi.h>

#define CREATE_TRACE_POINTS

#include <trace/events/csd.h>

#undef CREATE_TRACE_POINTS



#include "smpboot.h"

#include "sched/smp.h"
@@ -121,6 +124,14 @@ send_call_function_ipi_mask(struct cpumask *mask) 
	arch_send_call_function_ipi_mask(mask);

}



static __always_inline void

csd_do_func(smp_call_func_t func, void *info, struct __call_single_data *csd)

{

	trace_csd_function_entry(func, csd);

	func(info);

	trace_csd_function_exit(func, csd);

}



#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG



static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
@@ -375,7 +386,7 @@ static int generic_exec_single(int cpu, struct __call_single_data *csd) 
		csd_lock_record(csd);

		csd_unlock(csd);

		local_irq_save(flags);

		func(info);

		csd_do_func(func, info, NULL);

		csd_lock_record(NULL);

		local_irq_restore(flags);

		return 0;
@@ -477,7 +488,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) 
			}



			csd_lock_record(csd);

			func(info);

			csd_do_func(func, info, csd);

			csd_unlock(csd);

			csd_lock_record(NULL);

		} else {
@@ -508,7 +519,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) 


				csd_lock_record(csd);

				csd_unlock(csd);

				func(info);

				csd_do_func(func, info, csd);

				csd_lock_record(NULL);

			} else if (type == CSD_TYPE_IRQ_WORK) {

				irq_work_single(csd);
@@ -522,8 +533,10 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) 
	/*

	 * Third; only CSD_TYPE_TTWU is left, issue those.

	 */

	if (entry)

		sched_ttwu_pending(entry);

	if (entry) {

		csd = llist_entry(entry, typeof(*csd), node.llist);

		csd_do_func(sched_ttwu_pending, entry, csd);

	}

}
@@ -816,7 +829,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask, 
		unsigned long flags;



		local_irq_save(flags);

		func(info);

		csd_do_func(func, info, NULL);

		local_irq_restore(flags);

	}