Commit 6ba5aa54 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/pseries: Move dtl scanning and steal time accounting to pseries platform 



dtl is the PAPR Dispatch Trace Log, which is entirely a pseries feature.
The pseries platform alrady has a file dealing with the dtl, so move
scanning for stolen time accounting there from kernel/time.c.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20220902085316.2071519-5-npiggin@gmail.com
parent 02382aff

arch/powerpc/include/asm/cputime.h

+1 −1
Original line number Diff line number Diff line
@@ -95,7 +95,7 @@ static notrace inline void account_stolen_time(void) 
		struct lppaca *lp = local_paca->lppaca_ptr;



		if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx)))

			accumulate_stolen_time();

			pseries_accumulate_stolen_time();

	}

#endif

}

arch/powerpc/include/asm/dtl.h

+0 −8
Original line number Diff line number Diff line
@@ -37,14 +37,6 @@ struct dtl_entry { 
extern struct kmem_cache *dtl_cache;

extern rwlock_t dtl_access_lock;



/*

 * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls

 * reading from the dispatch trace log.  If other code wants to consume

 * DTL entries, it can set this pointer to a function that will get

 * called once for each DTL entry that gets processed.

 */

extern void (*dtl_consumer)(struct dtl_entry *entry, u64 index);



extern void register_dtl_buffer(int cpu);

extern void alloc_dtl_buffers(unsigned long *time_limit);

extern long hcall_vphn(unsigned long cpu, u64 flags, __be32 *associativity);

arch/powerpc/include/asm/time.h

+3 −2
Original line number Diff line number Diff line
@@ -116,8 +116,9 @@ unsigned long long tb_to_ns(unsigned long long tb_ticks); 


void timer_broadcast_interrupt(void);



/* SPLPAR */

void accumulate_stolen_time(void);

/* SPLPAR and VIRT_CPU_ACCOUNTING_NATIVE */

void pseries_accumulate_stolen_time(void);

u64 pseries_calculate_stolen_time(u64 stop_tb);



#endif /* __KERNEL__ */

#endif /* __POWERPC_TIME_H */

arch/powerpc/kernel/time.c

+5 −87
Original line number Diff line number Diff line
@@ -178,92 +178,6 @@ static inline unsigned long read_spurr(unsigned long tb) 
	return tb;

}



#ifdef CONFIG_PPC_SPLPAR



#include <asm/dtl.h>



void (*dtl_consumer)(struct dtl_entry *, u64);



/*

 * Scan the dispatch trace log and count up the stolen time.

 * Should be called with interrupts disabled.

 */

static u64 scan_dispatch_log(u64 stop_tb)

{

	u64 i = local_paca->dtl_ridx;

	struct dtl_entry *dtl = local_paca->dtl_curr;

	struct dtl_entry *dtl_end = local_paca->dispatch_log_end;

	struct lppaca *vpa = local_paca->lppaca_ptr;

	u64 tb_delta;

	u64 stolen = 0;

	u64 dtb;



	if (!dtl)

		return 0;



	if (i == be64_to_cpu(vpa->dtl_idx))

		return 0;

	while (i < be64_to_cpu(vpa->dtl_idx)) {

		dtb = be64_to_cpu(dtl->timebase);

		tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) +

			be32_to_cpu(dtl->ready_to_enqueue_time);

		barrier();

		if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {

			/* buffer has overflowed */

			i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;

			dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);

			continue;

		}

		if (dtb > stop_tb)

			break;

		if (dtl_consumer)

			dtl_consumer(dtl, i);

		stolen += tb_delta;

		++i;

		++dtl;

		if (dtl == dtl_end)

			dtl = local_paca->dispatch_log;

	}

	local_paca->dtl_ridx = i;

	local_paca->dtl_curr = dtl;

	return stolen;

}



/*

 * Accumulate stolen time by scanning the dispatch trace log.

 * Called on entry from user mode.

 */

void notrace accumulate_stolen_time(void)

{

	u64 sst, ust;

	struct cpu_accounting_data *acct = &local_paca->accounting;



	sst = scan_dispatch_log(acct->starttime_user);

	ust = scan_dispatch_log(acct->starttime);

	acct->stime -= sst;

	acct->utime -= ust;

	acct->steal_time += ust + sst;

}



static inline u64 calculate_stolen_time(u64 stop_tb)

{

	if (!firmware_has_feature(FW_FEATURE_SPLPAR))

		return 0;



	if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx))

		return scan_dispatch_log(stop_tb);



	return 0;

}



#else /* CONFIG_PPC_SPLPAR */

static inline u64 calculate_stolen_time(u64 stop_tb)

{

	return 0;

}



#endif /* CONFIG_PPC_SPLPAR */



/*

 * Account time for a transition between system, hard irq

 * or soft irq state.
@@ -322,7 +236,11 @@ static unsigned long vtime_delta(struct cpu_accounting_data *acct, 


	*stime_scaled = vtime_delta_scaled(acct, now, stime);



	*steal_time = calculate_stolen_time(now);

	if (IS_ENABLED(CONFIG_PPC_SPLPAR) &&

			firmware_has_feature(FW_FEATURE_SPLPAR))

		*steal_time = pseries_calculate_stolen_time(now);

	else

		*steal_time = 0;



	return stime;

}

arch/powerpc/platforms/pseries/dtl.c

+81 −0
Original line number Diff line number Diff line
@@ -37,6 +37,15 @@ static u8 dtl_event_mask = DTL_LOG_ALL; 
static int dtl_buf_entries = N_DISPATCH_LOG;



#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE



/*

 * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls

 * reading from the dispatch trace log.  If other code wants to consume

 * DTL entries, it can set this pointer to a function that will get

 * called once for each DTL entry that gets processed.

 */

static void (*dtl_consumer)(struct dtl_entry *entry, u64 index);



struct dtl_ring {

	u64	write_index;

	struct dtl_entry *write_ptr;
@@ -48,6 +57,78 @@ static DEFINE_PER_CPU(struct dtl_ring, dtl_rings); 


static atomic_t dtl_count;



/*

 * Scan the dispatch trace log and count up the stolen time.

 * Should be called with interrupts disabled.

 */

static notrace u64 scan_dispatch_log(u64 stop_tb)

{

	u64 i = local_paca->dtl_ridx;

	struct dtl_entry *dtl = local_paca->dtl_curr;

	struct dtl_entry *dtl_end = local_paca->dispatch_log_end;

	struct lppaca *vpa = local_paca->lppaca_ptr;

	u64 tb_delta;

	u64 stolen = 0;

	u64 dtb;



	if (!dtl)

		return 0;



	if (i == be64_to_cpu(vpa->dtl_idx))

		return 0;

	while (i < be64_to_cpu(vpa->dtl_idx)) {

		dtb = be64_to_cpu(dtl->timebase);

		tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) +

			be32_to_cpu(dtl->ready_to_enqueue_time);

		barrier();

		if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {

			/* buffer has overflowed */

			i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;

			dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);

			continue;

		}

		if (dtb > stop_tb)

			break;

		if (dtl_consumer)

			dtl_consumer(dtl, i);

		stolen += tb_delta;

		++i;

		++dtl;

		if (dtl == dtl_end)

			dtl = local_paca->dispatch_log;

	}

	local_paca->dtl_ridx = i;

	local_paca->dtl_curr = dtl;

	return stolen;

}



/*

 * Accumulate stolen time by scanning the dispatch trace log.

 * Called on entry from user mode.

 */

void notrace pseries_accumulate_stolen_time(void)

{

	u64 sst, ust;

	struct cpu_accounting_data *acct = &local_paca->accounting;



	sst = scan_dispatch_log(acct->starttime_user);

	ust = scan_dispatch_log(acct->starttime);

	acct->stime -= sst;

	acct->utime -= ust;

	acct->steal_time += ust + sst;

}



u64 pseries_calculate_stolen_time(u64 stop_tb)

{

	if (!firmware_has_feature(FW_FEATURE_SPLPAR))

		return 0;



	if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx))

		return scan_dispatch_log(stop_tb);



	return 0;

}



/*

 * The cpu accounting code controls the DTL ring buffer, and we get

 * given entries as they are processed.