Commit 1a7c6115 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'perf-core-2023-08-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull perf event updates from Ingo Molnar:

 - AMD IBS improvements

 - Intel PMU driver updates

 - Extend core perf facilities & the ARM PMU driver to better handle ARM big.LITTLE events

 - Micro-optimize software events and the ring-buffer code

 - Misc cleanups & fixes

* tag 'perf-core-2023-08-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  perf/x86/uncore: Remove unnecessary ?: operator around pcibios_err_to_errno() call
  perf/x86/intel: Add Crestmont PMU
  x86/cpu: Update Hybrids
  x86/cpu: Fix Crestmont uarch
  x86/cpu: Fix Gracemont uarch
  perf: Remove unused extern declaration arch_perf_get_page_size()
  perf: Remove unused PERF_PMU_CAP_HETEROGENEOUS_CPUS capability
  arm_pmu: Remove unused PERF_PMU_CAP_HETEROGENEOUS_CPUS capability
  perf/x86: Remove unused PERF_PMU_CAP_HETEROGENEOUS_CPUS capability
  arm_pmu: Add PERF_PMU_CAP_EXTENDED_HW_TYPE capability
  perf/x86/ibs: Set mem_lvl_num, mem_remote and mem_hops for data_src
  perf/mem: Add PERF_MEM_LVLNUM_NA to PERF_MEM_NA
  perf/mem: Introduce PERF_MEM_LVLNUM_UNC
  perf/ring_buffer: Use local_try_cmpxchg in __perf_output_begin
  locking/arch: Avoid variable shadowing in local_try_cmpxchg()
  perf/core: Use local64_try_cmpxchg in perf_swevent_set_period
  perf/x86: Use local64_try_cmpxchg
  perf/amd: Prevent grouping of IBS events
parents d637fce0 2c65477f

arch/loongarch/include/asm/local.h

+2 −2
Original line number Diff line number Diff line
@@ -63,8 +63,8 @@ static inline long local_cmpxchg(local_t *l, long old, long new) 


static inline bool local_try_cmpxchg(local_t *l, long *old, long new)

{

	typeof(l->a.counter) *__old = (typeof(l->a.counter) *) old;

	return try_cmpxchg_local(&l->a.counter, __old, new);

	return try_cmpxchg_local(&l->a.counter,

				 (typeof(l->a.counter) *) old, new);

}



#define local_xchg(l, n) (atomic_long_xchg((&(l)->a), (n)))

arch/mips/include/asm/local.h

+2 −2
Original line number Diff line number Diff line
@@ -101,8 +101,8 @@ static __inline__ long local_cmpxchg(local_t *l, long old, long new) 


static __inline__ bool local_try_cmpxchg(local_t *l, long *old, long new)

{

	typeof(l->a.counter) *__old = (typeof(l->a.counter) *) old;

	return try_cmpxchg_local(&l->a.counter, __old, new);

	return try_cmpxchg_local(&l->a.counter,

				 (typeof(l->a.counter) *) old, new);

}



#define local_xchg(l, n) (atomic_long_xchg((&(l)->a), (n)))

arch/x86/events/amd/ibs.c

+96 −90
Original line number Diff line number Diff line
@@ -156,8 +156,8 @@ perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) 
	 * count to the generic event atomically:

	 */

	prev_raw_count = local64_read(&hwc->prev_count);

	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,

					new_raw_count) != prev_raw_count)

	if (!local64_try_cmpxchg(&hwc->prev_count,

				 &prev_raw_count, new_raw_count))

		return 0;



	/*
@@ -247,11 +247,33 @@ int forward_event_to_ibs(struct perf_event *event) 
	return -ENOENT;

}



/*

 * Grouping of IBS events is not possible since IBS can have only

 * one event active at any point in time.

 */

static int validate_group(struct perf_event *event)

{

	struct perf_event *sibling;



	if (event->group_leader == event)

		return 0;



	if (event->group_leader->pmu == event->pmu)

		return -EINVAL;



	for_each_sibling_event(sibling, event->group_leader) {

		if (sibling->pmu == event->pmu)

			return -EINVAL;

	}

	return 0;

}



static int perf_ibs_init(struct perf_event *event)

{

	struct hw_perf_event *hwc = &event->hw;

	struct perf_ibs *perf_ibs;

	u64 max_cnt, config;

	int ret;



	perf_ibs = get_ibs_pmu(event->attr.type);

	if (!perf_ibs)
@@ -265,6 +287,10 @@ static int perf_ibs_init(struct perf_event *event) 
	if (config & ~perf_ibs->config_mask)

		return -EINVAL;



	ret = validate_group(event);

	if (ret)

		return ret;



	if (hwc->sample_period) {

		if (config & perf_ibs->cnt_mask)

			/* raw max_cnt may not be set */
@@ -702,7 +728,37 @@ static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2) 
	return op_data2->data_src_lo;

}



static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2,

#define	L(x)		(PERF_MEM_S(LVL, x) | PERF_MEM_S(LVL, HIT))

#define	LN(x)		PERF_MEM_S(LVLNUM, x)

#define	REM		PERF_MEM_S(REMOTE, REMOTE)

#define	HOPS(x)		PERF_MEM_S(HOPS, x)



static u64 g_data_src[8] = {

	[IBS_DATA_SRC_LOC_CACHE]	  = L(L3) | L(REM_CCE1) | LN(ANY_CACHE) | HOPS(0),

	[IBS_DATA_SRC_DRAM]		  = L(LOC_RAM) | LN(RAM),

	[IBS_DATA_SRC_REM_CACHE]	  = L(REM_CCE2) | LN(ANY_CACHE) | REM | HOPS(1),

	[IBS_DATA_SRC_IO]		  = L(IO) | LN(IO),

};



#define RMT_NODE_BITS			(1 << IBS_DATA_SRC_DRAM)

#define RMT_NODE_APPLICABLE(x)		(RMT_NODE_BITS & (1 << x))



static u64 g_zen4_data_src[32] = {

	[IBS_DATA_SRC_EXT_LOC_CACHE]	  = L(L3) | LN(L3),

	[IBS_DATA_SRC_EXT_NEAR_CCX_CACHE] = L(REM_CCE1) | LN(ANY_CACHE) | REM | HOPS(0),

	[IBS_DATA_SRC_EXT_DRAM]		  = L(LOC_RAM) | LN(RAM),

	[IBS_DATA_SRC_EXT_FAR_CCX_CACHE]  = L(REM_CCE2) | LN(ANY_CACHE) | REM | HOPS(1),

	[IBS_DATA_SRC_EXT_PMEM]		  = LN(PMEM),

	[IBS_DATA_SRC_EXT_IO]		  = L(IO) | LN(IO),

	[IBS_DATA_SRC_EXT_EXT_MEM]	  = LN(CXL),

};



#define ZEN4_RMT_NODE_BITS		((1 << IBS_DATA_SRC_EXT_DRAM) | \

					 (1 << IBS_DATA_SRC_EXT_PMEM) | \

					 (1 << IBS_DATA_SRC_EXT_EXT_MEM))

#define ZEN4_RMT_NODE_APPLICABLE(x)	(ZEN4_RMT_NODE_BITS & (1 << x))



static __u64 perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2,

				  union ibs_op_data3 *op_data3,

				  struct perf_sample_data *data)

{
@@ -710,30 +766,25 @@ static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, 
	u8 ibs_data_src = perf_ibs_data_src(op_data2);



	data_src->mem_lvl = 0;

	data_src->mem_lvl_num = 0;



	/*

	 * DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached

	 * memory accesses. So, check DcUcMemAcc bit early.

	 */

	if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO) {

		data_src->mem_lvl = PERF_MEM_LVL_UNC | PERF_MEM_LVL_HIT;

		return;

	}

	if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO)

		return L(UNC) | LN(UNC);



	/* L1 Hit */

	if (op_data3->dc_miss == 0) {

		data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;

		return;

	}

	if (op_data3->dc_miss == 0)

		return L(L1) | LN(L1);



	/* L2 Hit */

	if (op_data3->l2_miss == 0) {

		/* Erratum #1293 */

		if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF ||

		    !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) {

			data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;

			return;

		}

		    !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc))

			return L(L2) | LN(L2);

	}



	/*
@@ -743,82 +794,36 @@ static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, 
	if (data_src->mem_op != PERF_MEM_OP_LOAD)

		goto check_mab;



	/* L3 Hit */

	if (ibs_caps & IBS_CAPS_ZEN4) {

		if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE) {

			data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;

			return;

		}

	} else {

		if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) {

			data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_REM_CCE1 |

					    PERF_MEM_LVL_HIT;

			return;

		}

	}



	/* A peer cache in a near CCX */

	if (ibs_caps & IBS_CAPS_ZEN4 &&

	    ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE) {

		data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT;

		return;

	}

		u64 val = g_zen4_data_src[ibs_data_src];



	/* A peer cache in a far CCX */

	if (ibs_caps & IBS_CAPS_ZEN4) {

		if (ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) {

			data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT;

			return;

		}

	} else {

		if (ibs_data_src == IBS_DATA_SRC_REM_CACHE) {

			data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT;

			return;

		}

	}

		if (!val)

			goto check_mab;



	/* DRAM */

	if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM) {

		if (op_data2->rmt_node == 0)

			data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;

		/* HOPS_1 because IBS doesn't provide remote socket detail */

		if (op_data2->rmt_node && ZEN4_RMT_NODE_APPLICABLE(ibs_data_src)) {

			if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM)

				val = L(REM_RAM1) | LN(RAM) | REM | HOPS(1);

			else

			data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT;

		return;

				val |= REM | HOPS(1);

		}



	/* PMEM */

	if (ibs_caps & IBS_CAPS_ZEN4 && ibs_data_src == IBS_DATA_SRC_EXT_PMEM) {

		data_src->mem_lvl_num = PERF_MEM_LVLNUM_PMEM;

		if (op_data2->rmt_node) {

			data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;

			/* IBS doesn't provide Remote socket detail */

			data_src->mem_hops = PERF_MEM_HOPS_1;

		}

		return;

	}

		return val;

	} else {

		u64 val = g_data_src[ibs_data_src];



	/* Extension Memory */

	if (ibs_caps & IBS_CAPS_ZEN4 &&

	    ibs_data_src == IBS_DATA_SRC_EXT_EXT_MEM) {

		data_src->mem_lvl_num = PERF_MEM_LVLNUM_CXL;

		if (op_data2->rmt_node) {

			data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;

			/* IBS doesn't provide Remote socket detail */

			data_src->mem_hops = PERF_MEM_HOPS_1;

		}

		return;

	}

		if (!val)

			goto check_mab;



	/* IO */

	if (ibs_data_src == IBS_DATA_SRC_EXT_IO) {

		data_src->mem_lvl = PERF_MEM_LVL_IO;

		data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO;

		if (op_data2->rmt_node) {

			data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;

			/* IBS doesn't provide Remote socket detail */

			data_src->mem_hops = PERF_MEM_HOPS_1;

		/* HOPS_1 because IBS doesn't provide remote socket detail */

		if (op_data2->rmt_node && RMT_NODE_APPLICABLE(ibs_data_src)) {

			if (ibs_data_src == IBS_DATA_SRC_DRAM)

				val = L(REM_RAM1) | LN(RAM) | REM | HOPS(1);

			else

				val |= REM | HOPS(1);

		}

		return;



		return val;

	}



check_mab:
@@ -829,12 +834,11 @@ static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, 
	 * DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set

	 * MAB only when IBS fails to provide DataSrc.

	 */

	if (op_data3->dc_miss_no_mab_alloc) {

		data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT;

		return;

	}

	if (op_data3->dc_miss_no_mab_alloc)

		return L(LFB) | LN(LFB);



	data_src->mem_lvl = PERF_MEM_LVL_NA;

	/* Don't set HIT with NA */

	return PERF_MEM_S(LVL, NA) | LN(NA);

}



static bool perf_ibs_cache_hit_st_valid(void)
@@ -924,7 +928,9 @@ static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, 
				  union ibs_op_data2 *op_data2,

				  union ibs_op_data3 *op_data3)

{

	perf_ibs_get_mem_lvl(op_data2, op_data3, data);

	union perf_mem_data_src *data_src = &data->data_src;



	data_src->val |= perf_ibs_get_mem_lvl(op_data2, op_data3, data);

	perf_ibs_get_mem_snoop(op_data2, data);

	perf_ibs_get_tlb_lvl(op_data3, data);

	perf_ibs_get_mem_lock(op_data3, data);

arch/x86/events/core.c

+4 −7
Original line number Diff line number Diff line
@@ -129,13 +129,11 @@ u64 x86_perf_event_update(struct perf_event *event) 
	 * exchange a new raw count - then add that new-prev delta

	 * count to the generic event atomically:

	 */

again:

	prev_raw_count = local64_read(&hwc->prev_count);

	do {

		rdpmcl(hwc->event_base_rdpmc, new_raw_count);



	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,

					new_raw_count) != prev_raw_count)

		goto again;

	} while (!local64_try_cmpxchg(&hwc->prev_count,

				      &prev_raw_count, new_raw_count));



	/*

	 * Now we have the new raw value and have updated the prev
@@ -2168,7 +2166,6 @@ static int __init init_hw_perf_events(void) 
			hybrid_pmu->pmu = pmu;

			hybrid_pmu->pmu.type = -1;

			hybrid_pmu->pmu.attr_update = x86_pmu.attr_update;

			hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_HETEROGENEOUS_CPUS;

			hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE;



			err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name,

arch/x86/events/intel/core.c

+52 −2
Original line number Diff line number Diff line
@@ -2129,6 +2129,17 @@ static struct extra_reg intel_grt_extra_regs[] __read_mostly = { 
	EVENT_EXTRA_END

};



EVENT_ATTR_STR(topdown-retiring,       td_retiring_cmt,        "event=0x72,umask=0x0");

EVENT_ATTR_STR(topdown-bad-spec,       td_bad_spec_cmt,        "event=0x73,umask=0x0");



static struct attribute *cmt_events_attrs[] = {

	EVENT_PTR(td_fe_bound_tnt),

	EVENT_PTR(td_retiring_cmt),

	EVENT_PTR(td_bad_spec_cmt),

	EVENT_PTR(td_be_bound_tnt),

	NULL

};



static struct extra_reg intel_cmt_extra_regs[] __read_mostly = {

	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */

	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x800ff3ffffffffffull, RSP_0),
@@ -4847,6 +4858,8 @@ PMU_FORMAT_ATTR(ldlat, "config1:0-15"); 


PMU_FORMAT_ATTR(frontend, "config1:0-23");



PMU_FORMAT_ATTR(snoop_rsp, "config1:0-63");



static struct attribute *intel_arch3_formats_attr[] = {

	&format_attr_event.attr,

	&format_attr_umask.attr,
@@ -4877,6 +4890,13 @@ static struct attribute *slm_format_attr[] = { 
	NULL

};



static struct attribute *cmt_format_attr[] = {

	&format_attr_offcore_rsp.attr,

	&format_attr_ldlat.attr,

	&format_attr_snoop_rsp.attr,

	NULL

};



static struct attribute *skl_format_attr[] = {

	&format_attr_frontend.attr,

	NULL,
@@ -5656,7 +5676,6 @@ static struct attribute *adl_hybrid_extra_attr[] = { 
	NULL

};



PMU_FORMAT_ATTR_SHOW(snoop_rsp, "config1:0-63");

FORMAT_ATTR_HYBRID(snoop_rsp,	hybrid_small);



static struct attribute *mtl_hybrid_extra_attr_rtm[] = {
@@ -6174,7 +6193,7 @@ __init int intel_pmu_init(void) 
		name = "Tremont";

		break;



	case INTEL_FAM6_ALDERLAKE_N:

	case INTEL_FAM6_ATOM_GRACEMONT:

		x86_pmu.mid_ack = true;

		memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,

		       sizeof(hw_cache_event_ids));
@@ -6204,6 +6223,37 @@ __init int intel_pmu_init(void) 
		name = "gracemont";

		break;



	case INTEL_FAM6_ATOM_CRESTMONT:

	case INTEL_FAM6_ATOM_CRESTMONT_X:

		x86_pmu.mid_ack = true;

		memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,

		       sizeof(hw_cache_event_ids));

		memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs,

		       sizeof(hw_cache_extra_regs));

		hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;



		x86_pmu.event_constraints = intel_slm_event_constraints;

		x86_pmu.pebs_constraints = intel_grt_pebs_event_constraints;

		x86_pmu.extra_regs = intel_cmt_extra_regs;



		x86_pmu.pebs_aliases = NULL;

		x86_pmu.pebs_prec_dist = true;

		x86_pmu.lbr_pt_coexist = true;

		x86_pmu.pebs_block = true;

		x86_pmu.flags |= PMU_FL_HAS_RSP_1;

		x86_pmu.flags |= PMU_FL_INSTR_LATENCY;



		intel_pmu_pebs_data_source_cmt();

		x86_pmu.pebs_latency_data = mtl_latency_data_small;

		x86_pmu.get_event_constraints = cmt_get_event_constraints;

		x86_pmu.limit_period = spr_limit_period;

		td_attr = cmt_events_attrs;

		mem_attr = grt_mem_attrs;

		extra_attr = cmt_format_attr;

		pr_cont("Crestmont events, ");

		name = "crestmont";

		break;



	case INTEL_FAM6_WESTMERE:

	case INTEL_FAM6_WESTMERE_EP:

	case INTEL_FAM6_WESTMERE_EX: