Commit 0408b5bc authored by Lorenzo Pieralisi's avatar Lorenzo Pieralisi Committed by Jie Liu
Browse files

irqchip/gic-v3: Implement FEAT_GICv3_NMI support 

kunpeng inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I90N2C
CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/broonie/misc.git/commit/?h=arm64-nmi&id=a9952323528691e1dd56a01e507ec53fc1053f86



----------------------------------------------------------------------

The FEAT_GICv3_NMI GIC feature coupled with the CPU FEAT_NMI enables
handling NMI interrupts in HW on aarch64, by adding a superpriority
interrupt to the existing GIC priority scheme.

Implement GIC driver support for the FEAT_GICv3_NMI feature.

Rename gic_supports_nmi() helper function to gic_supports_pseudo_nmis()
to make the pseudo NMIs code path clearer and more explicit.

Check, through the ARM64 capabilitity infrastructure, if support
for FEAT_NMI was detected on the core and the system has not overridden
the detection and forced pseudo-NMIs enablement.

If FEAT_NMI is detected, it was not overridden (check embedded in the
system_uses_nmi() call) and the GIC supports the FEAT_GICv3_NMI feature,
install an NMI handler and initialize NMIs related HW GIC registers.

Signed-off-by: default avatarLorenzo Pieralisi <lpieralisi@kernel.org>
Signed-off-by: default avatarMark Brown <broonie@kernel.org>
Signed-off-by: default avatarJie Liu <liujie375@h-partners.com>
parent 77ca8065

drivers/irqchip/irq-gic-v3.c

+126 −27
Original line number Diff line number Diff line
@@ -61,6 +61,7 @@ struct gic_chip_data { 
	u32			nr_redist_regions;

	u64			flags;

	bool			has_rss;

	bool			has_nmi;

	unsigned int		ppi_nr;

	struct partition_desc	**ppi_descs;

};
@@ -149,6 +150,20 @@ enum gic_intid_range { 
	__INVALID_RANGE__

};



#ifdef CONFIG_ARM64

#include <asm/cpufeature.h>



static inline bool has_v3_3_nmi(void)

{

	return gic_data.has_nmi && system_uses_nmi();

}

#else

static inline bool has_v3_3_nmi(void)

{

	return false;

}

#endif



static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)

{

	switch (hwirq) {
@@ -387,6 +402,42 @@ static int gic_peek_irq(struct irq_data *d, u32 offset) 
	return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);

}



static DEFINE_RAW_SPINLOCK(irq_controller_lock);



static void gic_irq_configure_nmi(struct irq_data *d, bool enable)

{

	void __iomem *base, *addr;

	u32 offset, index, mask, val;



	offset = convert_offset_index(d, GICD_INMIR, &index);

	mask = 1 << (index % 32);



	if (gic_irq_in_rdist(d))

		base = gic_data_rdist_sgi_base();

	else

		base = gic_data.dist_base;



	addr = base + offset + (index / 32) * 4;



	raw_spin_lock(&irq_controller_lock);



	val = readl_relaxed(addr);

	val = enable ? (val | mask) : (val & ~mask);

	writel_relaxed(val, addr);



	raw_spin_unlock(&irq_controller_lock);

}



static void gic_irq_enable_nmi(struct irq_data *d)

{

	gic_irq_configure_nmi(d, true);

}



static void gic_irq_disable_nmi(struct irq_data *d)

{

	gic_irq_configure_nmi(d, false);

}



static void gic_poke_irq(struct irq_data *d, u32 offset)

{

	void __iomem *base;
@@ -432,7 +483,7 @@ static void gic_unmask_irq(struct irq_data *d) 
	gic_poke_irq(d, GICD_ISENABLER);

}



static inline bool gic_supports_nmi(void)

static inline bool gic_supports_pseudo_nmis(void)

{

	return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&

	       static_branch_likely(&supports_pseudo_nmis);
@@ -529,7 +580,7 @@ static int gic_irq_nmi_setup(struct irq_data *d) 
	struct irq_desc *desc = irq_to_desc(d->irq);

	u32 idx;



	if (!gic_supports_nmi())

	if (!gic_supports_pseudo_nmis() && !has_v3_3_nmi())

		return -EINVAL;



	if (gic_peek_irq(d, GICD_ISENABLER)) {
@@ -563,6 +614,9 @@ static int gic_irq_nmi_setup(struct irq_data *d) 
		break;

	}



	if (has_v3_3_nmi())

		gic_irq_enable_nmi(d);

	else

		gic_irq_set_prio(d, GICD_INT_NMI_PRI);



	return 0;
@@ -573,7 +627,7 @@ static void gic_irq_nmi_teardown(struct irq_data *d) 
	struct irq_desc *desc = irq_to_desc(d->irq);

	u32 idx;



	if (WARN_ON(!gic_supports_nmi()))

	if (WARN_ON(!gic_supports_pseudo_nmis() && !has_v3_3_nmi()))

		return;



	if (gic_peek_irq(d, GICD_ISENABLER)) {
@@ -605,6 +659,9 @@ static void gic_irq_nmi_teardown(struct irq_data *d) 
		break;

	}



	if (has_v3_3_nmi())

		gic_irq_disable_nmi(d);

	else

		gic_irq_set_prio(d, GICD_INT_DEF_PRI);

}
@@ -764,7 +821,7 @@ static inline void gic_complete_ack(u32 irqnr) 


static bool gic_rpr_is_nmi_prio(void)

{

	if (!gic_supports_nmi())

	if (!gic_supports_pseudo_nmis())

		return false;



	return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI));
@@ -796,7 +853,8 @@ static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 
	gic_complete_ack(irqnr);



	if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {

		WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);

		WARN_ONCE(true, "Unexpected %sNMI (irqnr %u)\n",

			  gic_supports_pseudo_nmis() ? "pseudo-" : "", irqnr);

		gic_deactivate_unhandled(irqnr);

	}

}
@@ -872,9 +930,37 @@ static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 
	__gic_handle_nmi(irqnr, regs);

}



#ifdef CONFIG_ARM64

static inline u64 gic_read_nmiar(void)

{

	u64 irqstat;



	irqstat = read_sysreg_s(SYS_ICC_NMIAR1_EL1);



	dsb(sy);



	return irqstat;

}



static asmlinkage void __exception_irq_entry gic_handle_nmi_irq(struct pt_regs *regs)

{

	u32 irqnr = gic_read_nmiar();



	__gic_handle_nmi(irqnr, regs);

}



static inline void gic_setup_nmi_handler(void)

{

	if (has_v3_3_nmi())

		set_handle_nmi_irq(gic_handle_nmi_irq);

}

#else

static inline void gic_setup_nmi_handler(void) { }

#endif



static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)

{

	if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))

	if (unlikely(gic_supports_pseudo_nmis() && !interrupts_enabled(regs)))

		__gic_handle_irq_from_irqsoff(regs);

	else

		__gic_handle_irq_from_irqson(regs);
@@ -1164,7 +1250,7 @@ static void gic_cpu_sys_reg_init(void) 
	/* Set priority mask register */

	if (!gic_prio_masking_enabled()) {

		write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);

	} else if (gic_supports_nmi()) {

	} else if (gic_supports_pseudo_nmis()) {

		/*

		 * Mismatch configuration with boot CPU, the system is likely

		 * to die as interrupt masking will not work properly on all
@@ -1954,25 +2040,8 @@ static const struct gic_quirk gic_quirks[] = { 
	}

};



static void gic_enable_nmi_support(void)

static void gic_enable_pseudo_nmis(void)

{

	int i;



	if (!gic_prio_masking_enabled())

		return;



	if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) {

		pr_warn("Skipping NMI enable due to firmware issues\n");

		return;

	}



	ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);

	if (!ppi_nmi_refs)

		return;



	for (i = 0; i < gic_data.ppi_nr; i++)

		refcount_set(&ppi_nmi_refs[i], 0);



	pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",

		gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");
@@ -2007,6 +2076,33 @@ static void gic_enable_nmi_support(void) 
		static_branch_enable(&gic_nonsecure_priorities);



	static_branch_enable(&supports_pseudo_nmis);

}



static void gic_enable_nmi_support(void)

{

	int i;



	if (!gic_prio_masking_enabled() && !has_v3_3_nmi())

		return;



	if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) {

		pr_warn("Skipping NMI enable due to firmware issues\n");

		return;

	}



	ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);

	if (!ppi_nmi_refs)

		return;



	for (i = 0; i < gic_data.ppi_nr; i++)

		refcount_set(&ppi_nmi_refs[i], 0);



	/*

	 * Initialize pseudo-NMIs only if GIC driver cannot take advantage

	 * of core (FEAT_NMI) and GIC (FEAT_GICv3_NMI) in HW

	 */

	if (!has_v3_3_nmi())

		gic_enable_pseudo_nmis();



	if (static_branch_likely(&supports_deactivate_key))

		gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
@@ -2075,6 +2171,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, 
	irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);



	gic_data.has_rss = !!(typer & GICD_TYPER_RSS);

	gic_data.has_nmi = !!(typer & GICD_TYPER_NMI);



	if (typer & GICD_TYPER_MBIS) {

		err = mbi_init(handle, gic_data.domain);
@@ -2084,6 +2181,8 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, 


	set_handle_irq(gic_handle_irq);



	gic_setup_nmi_handler();



	gic_update_rdist_properties();



	gic_dist_init();

include/linux/irqchip/arm-gic-v3.h

+4 −0
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ 
#define GICD_ICFGR			0x0C00

#define GICD_IGRPMODR			0x0D00

#define GICD_NSACR			0x0E00

#define GICD_INMIR			0x0F80

#define GICD_IGROUPRnE			0x1000

#define GICD_ISENABLERnE		0x1200

#define GICD_ICENABLERnE		0x1400
@@ -39,6 +40,7 @@ 
#define GICD_ICACTIVERnE		0x1C00

#define GICD_IPRIORITYRnE		0x2000

#define GICD_ICFGRnE			0x3000

#define GICD_INMIRnE			0x3B00

#define GICD_IROUTER			0x6000

#define GICD_IROUTERnE			0x8000

#define GICD_IDREGS			0xFFD0
@@ -83,6 +85,7 @@ 
#define GICD_TYPER_LPIS			(1U << 17)

#define GICD_TYPER_MBIS			(1U << 16)

#define GICD_TYPER_ESPI			(1U << 8)

#define GICD_TYPER_NMI			(1U << 9)



#define GICD_TYPER_ID_BITS(typer)	((((typer) >> 19) & 0x1f) + 1)

#define GICD_TYPER_NUM_LPIS(typer)	((((typer) >> 11) & 0x1f) + 1)
@@ -238,6 +241,7 @@ 
#define GICR_ICFGR0			GICD_ICFGR

#define GICR_IGRPMODR0			GICD_IGRPMODR

#define GICR_NSACR			GICD_NSACR

#define GICR_INMIR0			GICD_INMIR



#define GICR_TYPER_PLPIS		(1U << 0)

#define GICR_TYPER_VLPIS		(1U << 1)