Commit 2690e65a authored by Anshuman Khandual's avatar Anshuman Khandual Committed by Junhao He
Browse files

drivers: perf: arm_pmuv3: Enable branch stack sampling via FEAT_BRBE 

maillist inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8EC9K
CVE: NA

Reference: https://lore.kernel.org/lkml/20230711082455.215983-7-anshuman.khandual@arm.com/



--------------------------------

This extends recently added branch stack sampling framework in ARMV8 PMU to
enable such events via new architecture feature called Branch Record Buffer
Extension aka BRBE. This implements all the armv8pmu_branch_xxx() callbacks
as expected at ARMV8 PMU level required to drive perf branch stack sampling
events. This adds a new config option CONFIG_ARM64_BRBE to encapsulate this
BRBE based implementation, available only on ARM64 platforms.

BRBE hardware captures a branch record via three distinct system registers
representing branch source address, branch target address, and other branch
information. A BRBE buffer implementation is organized as multiple banks of
32 branch records each, which is a collection of BRBSRC_EL1, BRBTGT_EL1 and
BRBINF_EL1 registers. Though total BRBE record entries i.e BRBE_MAX_ENTRIES
cannot exceed MAX_BRANCH_RECORDS as defined for ARM PMU.

BRBE hardware attributes get captured in a new reg_brbidr element in struct
arm_pmu during armv8pmu_branch_probe() which is called from broader probing
function __armv8pmu_probe_pmu(). Attributes such as number of branch record
entries implemented in the hardware can be derived from armpmu->reg_brbidr.

BRBE gets enabled via armv8pmu_branch_enable() where it also derives branch
filter, and additional requirements from event's 'attr.branch_sample_type'
and configures them via BRBFCR_EL1 and BRBCR_EL1 registers.

PMU event overflow triggers IRQ, where current branch records get captured,
stitched along with older records available in 'task_ctx', before getting
processed for core perf ring buffer. Task context switch outs incrementally
save current branch records in event's 'pmu_ctx->task_ctx_data' to optimize
workload's branch record samples.

In case multiple events with different branch sample type requests converge
on the same PMU, BRBE gets enabled for branch filters for the last event's
branch sample type. No branch records will be captured and processed for an
event if BRBE hardware config does not match its branch sample type, while
handling the PMU IRQ.

Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarAnshuman Khandual <anshuman.khandual@arm.com>
Signed-off-by: default avatarJunhao He <hejunhao3@huawei.com>
parent 49a0ee0e

arch/arm64/include/asm/perf_event.h

+13 −0
Original line number Diff line number Diff line
@@ -250,6 +250,18 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs); 
#define perf_misc_flags(regs)	perf_misc_flags(regs)

#define perf_arch_bpf_user_pt_regs(regs) &regs->user_regs



#ifdef CONFIG_ARM64_BRBE

void armv8pmu_branch_reset(void);

void armv8pmu_branch_probe(struct arm_pmu *arm_pmu);

bool armv8pmu_branch_attr_valid(struct perf_event *event);

void armv8pmu_branch_enable(struct perf_event *event);

void armv8pmu_branch_disable(struct perf_event *event);

void armv8pmu_branch_read(struct pmu_hw_events *cpuc,

			  struct perf_event *event);

void armv8pmu_branch_save(struct arm_pmu *arm_pmu, void *ctx);

int armv8pmu_task_ctx_cache_alloc(struct arm_pmu *arm_pmu);

void armv8pmu_task_ctx_cache_free(struct arm_pmu *arm_pmu);

#else  /* !CONFIG_ARM64_BRBE */

static inline void armv8pmu_branch_reset(void)

{

}
@@ -288,6 +300,7 @@ static inline int armv8pmu_task_ctx_cache_alloc(struct arm_pmu *arm_pmu) 
static inline void armv8pmu_task_ctx_cache_free(struct arm_pmu *arm_pmu)

{

}

#endif /* CONFIG_ARM64_BRBE */

#endif



#define perf_arch_fetch_caller_regs(regs, __ip) { \

arch/arm64/include/asm/sysreg.h

+8 −0
Original line number Diff line number Diff line
@@ -110,6 +110,14 @@ 
#define SYS_DC_ISW			sys_insn(1, 0, 7, 6, 2)

#define SYS_DC_CSW			sys_insn(1, 0, 7, 10, 2)

#define SYS_DC_CISW			sys_insn(1, 0, 7, 14, 2)

#define SYS_BRB_IALL			sys_insn(1, 1, 7, 2, 4)

#define SYS_BRB_INJ			sys_insn(1, 1, 7, 2, 5)



/*

 * BRBE Instructions

 */

#define BRB_IALL_INSN			__emit_inst(0xd5000000 | SYS_BRB_IALL | (0x1f))

#define BRB_INJ_INSN			__emit_inst(0xd5000000 | SYS_BRB_INJ  | (0x1f))



/*

 * System registers, organised loosely by encoding but grouped together

arch/arm64/kernel/head.S

+48 −0
Original line number Diff line number Diff line
@@ -486,6 +486,51 @@ EXPORT_SYMBOL(kimage_vaddr) 
 */

	.section ".idmap.text","awx"



#ifdef CONFIG_ARM64_BRBE

/*

 * Enable BRBE cycle count

 *

 * BRBE requires both BRBCR_EL1.CC and BRBCR_EL2.CC fields, be set

 * for the cycle counts to be available in BRBINF<N>_EL1.CC during

 * branch record processing after a PMU interrupt. This enables CC

 * field on both these registers while still executing inside EL2.

 *

 * BRBE driver would still be able to toggle branch records cycle

 * count support via BRBCR_EL1.CC field regardless of whether the

 * kernel ends up executing in EL1 or EL2.

 */

.macro __init_el2_brbe

	mrs	x1, id_aa64dfr0_el1

	ubfx	x1, x1, #ID_AA64DFR0_EL1_BRBE_SHIFT, #4

	cbz	x1, .Lskip_brbe_cc_\@



	mrs_s	x0, SYS_BRBCR_EL2

	orr	x0, x0, BRBCR_ELx_CC

	msr_s	SYS_BRBCR_EL2, x0



	/*

	 * Accessing BRBCR_EL1 register here does not require

	 * BRBCR_EL12 addressing mode as HCR_EL2.E2H is still

	 * clear. Regardless, check for HCR_E2H and be on the

	 * safer side.

	 */

	mrs	x1, hcr_el2

	and	x1, x1, #HCR_E2H

	cbz	x1, .Lset_brbe_el1_direct_\@



	mrs_s	x0, SYS_BRBCR_EL12

	orr	x0, x0, BRBCR_ELx_CC

	msr_s	SYS_BRBCR_EL12, x0

	b	.Lskip_brbe_cc_\@



.Lset_brbe_el1_direct_\@:

	mrs_s	x0, SYS_BRBCR_EL1

	orr	x0, x0, BRBCR_ELx_CC

	msr_s	SYS_BRBCR_EL1, x0

.Lskip_brbe_cc_\@:

.endm



#endif

/*

 * If we're fortunate enough to boot at EL2, ensure that the world is

 * sane before dropping to EL1.
@@ -601,6 +646,9 @@ set_hcr: 
7:

	msr	mdcr_el2, x3			// Configure debug traps



#ifdef CONFIG_ARM64_BRBE

	__init_el2_brbe

#endif

	/* LORegions */

	mrs	x1, id_aa64mmfr1_el1

	ubfx	x0, x1, #ID_AA64MMFR1_LOR_SHIFT, 4

drivers/perf/Kconfig

+11 −0
Original line number Diff line number Diff line
@@ -130,6 +130,17 @@ config ARM_SPE_PMU 
	  Extension, which provides periodic sampling of operations in

	  the CPU pipeline and reports this via the perf AUX interface.



config ARM64_BRBE

	bool "Enable support for Branch Record Buffer Extension (BRBE)"

	depends on PERF_EVENTS && ARM64 && ARM_PMU

	default y

	help

	  Enable perf support for Branch Record Buffer Extension (BRBE) which

	  records all branches taken in an execution path. This supports some

	  branch types and privilege based filtering. It captures additional

	  relevant information such as cycle count, misprediction and branch

	  type, branch privilege level etc.



source "drivers/perf/hisilicon/Kconfig"



endmenu

drivers/perf/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -13,3 +13,4 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o 
obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o

obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o

obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o

obj-$(CONFIG_ARM64_BRBE) += arm_brbe.o