Commit 091c0edb authored by Yipeng Zou's avatar Yipeng Zou Committed by Jinjie Ruan
Browse files

arm64: entry: Fix some warnings when enable nohz full CPU 

hulk inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/release-management/issues/IB6JLE



--------------------------------

Fix following warnings when enable nohz full CPU, by not skipping
context tracking and irq tick idle inform.

Also remove el1 xint, as less savings for both assembly and C.

	ct_state() == CONTEXT_USER
	WARNING: CPU: 66 PID: 10214 at kernel/sched/core.c:6012 __schedule+0x738/0xcf8
	CPU: 66 PID: 10214 Comm: sh Kdump: loaded Not tainted 6.6.0xcall_xint_lcxint6.6_test+ #26
	Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022
	pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
	pc : __schedule+0x738/0xcf8
	lr : __schedule+0x738/0xcf8
	sp : ffff8000d193bd90
	pmr_save: 00000060
	x29: ffff8000d193bd90 x28: ffff20203e95d100 x27: ffff20203e95d100
	x26: 0000000000000000 x25: ffff800080e3fbb8 x24: ffff800081a2a008
	x23: ffff800081e15370 x22: 0000000000000000 x21: ffff800081a40d80
	x20: ffff20203e95d100 x19: ffff2027dfa1ad80 x18: 0000000000000000
	x17: ffffa0275dfda000 x16: ffff800083008000 x15: 0000000000000000
	x14: 0000000000000000 x13: 524553555f545845 x12: 544e4f43203d3d20
	x11: 00000000ffff7fff x10: ffff80008210b368 x9 : ffff800080157fa8
	x8 : 00000000000bffe8 x7 : c0000000ffff7fff x6 : 00000000002bffa8
	x5 : ffff2027dfa07ac8 x4 : ffff8000d193bbf0 x3 : ffffa0275dfda000
	x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff20203e95d100
	Call trace:
	 __schedule+0x738/0xcf8
	 schedule+0x58/0xf0
	 do_notify_resume+0x50/0x350
	 el0_xint.constprop.0+0xa0/0xe0
	 el0t_64_xint_handler+0x18/0x28
	 el0t_64_irq+0x170/0x3a0
	---[ end trace 0000000000000000 ]---
	------------[ cut here ]------------
	WARNING: CPU: 66 PID: 0 at kernel/context_tracking.c:128 ct_kernel_exit.constprop.0+0xa0/0xa8
	CPU: 66 PID: 0 Comm: swapper/66 Kdump: loaded Tainted: G        W          6.6.0xcall_xint_lcxint6.6_test+ #26
	Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022
	pstate: 80400009 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
	pc : ct_kernel_exit.constprop.0+0xa0/0xa8
	lr : ct_idle_enter+0x14/0x20
	sp : ffff8000839abdb0
	pmr_save: 00000060
	x29: ffff8000839abdb0 x28: 0000000000000000 x27: 0000000000000000
	x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000
	x23: 0000000000000000 x22: ffff2020001f6540 x21: ffff2020001f6540
	x20: 0000000000000005 x19: ffff2027dfa16760 x18: 0000000000000000
	x17: ffffa0275dfda000 x16: ffff800083008000 x15: 0000000000000000
	x14: 0000000000000000 x13: 524553555f545845 x12: 0000000000000000
	x11: 0000000000000000 x10: 0000000000000f90 x9 : ffff8000801bb9b8
	x8 : ffff2020001f7530 x7 : 00000000ffff9d08 x6 : 0000000748890425
	x5 : 0000000000000000 x4 : ffffa0275dfda000 x3 : ffff8000839abdb0
	x2 : ffff800081a3c760 x1 : ffff800081a3c760 x0 : 4000000000000000
	Call trace:
	 ct_kernel_exit.constprop.0+0xa0/0xa8
	 ct_idle_enter+0x14/0x20
	 default_idle_call+0x38/0x120
	 cpuidle_idle_call+0x164/0x1a8
	 do_idle+0xac/0x118
	 cpu_startup_entry+0x3c/0x50
	 secondary_start_kernel+0x14c/0x1e8
	 __secondary_switched+0xb8/0xc0
	---[ end trace 0000000000000000 ]---
	------------[ cut here ]------------
	WARNING: CPU: 0 PID: 10214 at arch/arm64/kernel/entry-common.c:105 el0_da+0x98/0x1a8
	CPU: 0 PID: 10214 Comm: sh Kdump: loaded Tainted: G        W          6.6.0xcall_xint_lcxint6.6_test+ #26
	Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022
	pstate: 804003c9 (Nzcv DAIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
	pc : el0_da+0x98/0x1a8
	lr : el0t_64_sync_handler+0xb8/0x130
	sp : ffff8000d193be70
	pmr_save: 000000f0
	x29: ffff8000d193be70 x28: ffff20203e95d100 x27: 0000000000000000
	x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000
	x23: 0000000060000000 x22: 0000ffffbb7a9028 x21: 0000aaaafa8a1c18
	x20: 000000009200004f x19: ffff8000d193beb0 x18: 0000000000000000
	x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
	x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000
	x11: 0000000000000000 x10: 0000000000000000 x9 : 0000000000000000
	x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000
	x5 : 0000000000000000 x4 : ffff8000d193beb0 x3 : 0000000000000000
	x2 : ffff800080e36fac x1 : ffff80275df96000 x0 : 0000000000000000
	Call trace:
	 el0_da+0x98/0x1a8
	 el0t_64_sync_handler+0xb8/0x130
	 el0t_64_sync+0x3b8/0x3c0
	---[ end trace 0000000000000000 ]---
	------------[ cut here ]------------
	WARNING: CPU: 0 PID: 15840 at arch/arm64/kernel/entry-common.c:105 el0_ia+0xc0/0x200
	CPU: 0 PID: 15840 Comm: sh Kdump: loaded Tainted: G        W          6.6.0xcall_xint_lcxint6.6_test+ #26
	Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022
	pstate: 804003c9 (Nzcv DAIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
	pc : el0_ia+0xc0/0x200
	lr : el0t_64_sync_handler+0xd0/0x130
	sp : ffff8000dc90be70
	pmr_save: 000000f0
	x29: ffff8000dc90be70 x28: ffff20204f112880 x27: 0000000000000000
	x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000
	x23: 0000000020000000 x22: 0000ffffbb801080 x21: 0000ffffbb801080
	x20: 0000000082000007 x19: ffff8000dc90beb0 x18: 0000000000000000
	x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
	x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000
	x11: 0000000000000000 x10: 0000000000000000 x9 : 0000000000000000
	x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000
	x5 : 0000000000000000 x4 : ffff8000dc90beb0 x3 : 0000000000000000
	x2 : 0000000000000000 x1 : ffff80275df96000 x0 : 0000000000000000
	Call trace:
	 el0_ia+0xc0/0x200
	 el0t_64_sync_handler+0xd0/0x130
	 el0t_64_sync+0x3b8/0x3c0
	---[ end trace 0000000000000000 ]---

Fixes: cf5f2d9f ("arm64: Faster SVC exception handler with xcall")
Fixes: 3053668e ("arm64: Introduce Xint software solution")
Signed-off-by: default avatarYipeng Zou <zouyipeng@huawei.com>
Signed-off-by: default avatarJinjie Ruan <ruanjinjie@huawei.com>
parent 429a1987

arch/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -1573,6 +1573,7 @@ config FAST_IRQ 
config DEBUG_FEATURE_BYPASS

	bool "Bypass debug feature in fast syscall"

	depends on FAST_SYSCALL || FAST_IRQ

	depends on !LOCKDEP

	default y

	help

	  This to bypass debug feature in fast syscall.

arch/arm64/kernel/entry-common.c

+60 −101
Original line number Diff line number Diff line
@@ -151,6 +151,62 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 
	exit_to_user_mode(regs);

}



#if defined(CONFIG_FAST_SYSCALL) || defined(CONFIG_FAST_IRQ)

/*

 * Copy from exit_to_user_mode_prepare

 */

static __always_inline void fast_exit_to_user_mode_prepare(struct pt_regs *regs)

{

	unsigned long flags;



	local_daif_mask();



	flags = read_thread_flags();

	if (unlikely(flags & _TIF_WORK_MASK))

		do_notify_resume(regs, flags);



#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	lockdep_sys_exit();

#endif

}



/* Copy from __exit_to_user_mode */

static __always_inline void __fast_exit_to_user_mode(void)

{

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare();

#endif

	user_enter_irqoff();

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	lockdep_hardirqs_on(CALLER_ADDR0);

#endif

}



static __always_inline void fast_exit_to_user_mode(struct pt_regs *regs)

{

	fast_exit_to_user_mode_prepare(regs);

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	mte_check_tfsr_exit();

#endif

	__fast_exit_to_user_mode();

}



/* Copy from __enter_from_user_mode */

static __always_inline void fast_enter_from_user_mode(struct pt_regs *regs)

{

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	lockdep_hardirqs_off(CALLER_ADDR0);

#endif

	CT_WARN_ON(ct_state() != CONTEXT_USER);

	user_exit_irqoff();

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	trace_hardirqs_off_finish();

	mte_disable_tco_entry(current);

#endif

}

#endif



/*

 * Handle IRQ/context state management when entering an NMI from user/kernel

 * mode. Before this function is called it is not safe to call regular kernel
@@ -513,80 +569,12 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, 
}



#ifdef CONFIG_FAST_IRQ

static __always_inline void __el1_xint(struct pt_regs *regs,

				       void (*handler)(struct pt_regs *))

{

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	enter_from_kernel_mode(regs);

#endif



	xint_enter_rcu();

	do_interrupt_handler(regs, handler);

	xint_exit_rcu();



	arm64_preempt_schedule_irq();



#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	exit_to_kernel_mode(regs);

#endif

}



static void noinstr el1_xint(struct pt_regs *regs, u64 nmi_flag,

			     void (*handler)(struct pt_regs *),

			     void (*nmi_handler)(struct pt_regs *))

{

	/* Is there a NMI to handle? */

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	if (system_uses_nmi() && (read_sysreg(isr_el1) & nmi_flag)) {

		__el1_nmi(regs, nmi_handler);

		return;

	}

#endif



	write_sysreg(DAIF_PROCCTX_NOIRQ, daif);



	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))

		__el1_pnmi(regs, handler);

	else

		__el1_xint(regs, handler);

}



asmlinkage void noinstr el1h_64_xint_handler(struct pt_regs *regs)

{

	el1_xint(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq);

}



static __always_inline void xint_exit_to_user_mode_prepare(struct pt_regs *regs)

{

	unsigned long flags;



	local_daif_mask();



	flags = read_thread_flags();

	if (unlikely(flags & _TIF_WORK_MASK))

		do_notify_resume(regs, flags);



#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	lockdep_sys_exit();

#endif

}



static __always_inline void xint_exit_to_user_mode(struct pt_regs *regs)

{

	xint_exit_to_user_mode_prepare(regs);

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	mte_check_tfsr_exit();

	__exit_to_user_mode();

#endif

}



static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag,

			     void (*handler)(struct pt_regs *),

			     void (*nmi_handler)(struct pt_regs *))

{

	fast_enter_from_user_mode(regs);

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	enter_from_user_mode(regs);



	/* Is there a NMI to handle? */

	if (system_uses_nmi() && (read_sysreg(isr_el1) & nmi_flag)) {

		/*
@@ -615,7 +603,7 @@ static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, 
	do_interrupt_handler(regs, handler);

	xint_exit_rcu();



	xint_exit_to_user_mode(regs);

	fast_exit_to_user_mode(regs);

}
@@ -828,46 +816,17 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 
}



#ifdef CONFIG_FAST_SYSCALL

/*

 * Copy from exit_to_user_mode_prepare

 */

static __always_inline void xcall_exit_to_user_mode_prepare(struct pt_regs *regs)

{

	unsigned long flags;



	local_daif_mask();



	flags = read_thread_flags();

	if (unlikely(flags & _TIF_WORK_MASK))

		do_notify_resume(regs, flags);



#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	lockdep_sys_exit();

#endif

}



static __always_inline void xcall_exit_to_user_mode(struct pt_regs *regs)

{

	xcall_exit_to_user_mode_prepare(regs);

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	mte_check_tfsr_exit();

	__exit_to_user_mode();

#endif

}



/* Copy from el0_sync */

static void noinstr el0_xcall(struct pt_regs *regs)

{

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	enter_from_user_mode(regs);

#endif

	fast_enter_from_user_mode(regs);

#ifndef CONFIG_SECURITY_FEATURE_BYPASS

	cortex_a76_erratum_1463225_svc_handler();

#endif

	fp_user_discard();

	local_daif_restore(DAIF_PROCCTX);

	do_el0_svc(regs);

	xcall_exit_to_user_mode(regs);

	fast_exit_to_user_mode(regs);

}



asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs)

arch/arm64/kernel/entry.S

+7 −11
Original line number Diff line number Diff line
@@ -660,7 +660,7 @@ SYM_CODE_END(__bad_stack) 
#endif



#ifdef CONFIG_FAST_IRQ

.macro check_xint_pre_kernel_entry el:req, ht:req

.macro check_xint_pre_kernel_entry

	stp	x0, x1, [sp, #0]

	stp	x2, x3, [sp, #16]
@@ -691,20 +691,16 @@ SYM_CODE_END(__bad_stack) 
	ldp	x0, x1, [sp, #0]

	ldp	x2, x3, [sp, #16]

#ifdef CONFIG_SECURITY_FEATURE_BYPASS

	kernel_entry \el, 64, xint

	kernel_entry 0, 64, xint

#else

	kernel_entry \el, 64

	kernel_entry 0, 64

#endif

	mov	x0, sp

	bl	el\el\ht\()_64_xint_handler

	bl	el0t_64_xint_handler

#ifdef CONFIG_SECURITY_FEATURE_BYPASS

	kernel_exit \el, xint

	kernel_exit 0, xint

#else

	.if	\el == 0

	b	ret_to_user

	.else

	b	ret_to_kernel

	.endif

#endif



.Lskip_xint\@:
@@ -726,11 +722,11 @@ SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) 
	.endif

#endif

#ifdef CONFIG_FAST_IRQ

	.if \regsize == 64 && \label == irq && (( \el == 0 && \ht == t) || (\el == 1 && \ht == h))

	.if \regsize == 64 && \label == irq && \el == 0 && \ht == t

	alternative_if_not ARM64_HAS_XINT

		b .Lskip_check_xint\@

	alternative_else_nop_endif

		check_xint_pre_kernel_entry \el, \ht

		check_xint_pre_kernel_entry

.Lskip_check_xint\@:

	.endif

#endif

kernel/softirq.c

+2 −2
Original line number Diff line number Diff line
@@ -669,11 +669,13 @@ void xint_enter_rcu(void) 
	preempt_count_add(HARDIRQ_OFFSET);

#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	lockdep_hardirq_enter();

#endif



	if (tick_nohz_full_cpu(smp_processor_id()) ||

	    (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET)))

		tick_irq_enter();



#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	account_hardirq_enter(current);

#endif

}
@@ -695,9 +697,7 @@ static inline void __xint_exit_rcu(void) 
	if (!in_interrupt() && local_softirq_pending())

		invoke_softirq();



#ifndef CONFIG_DEBUG_FEATURE_BYPASS

	tick_irq_exit();

#endif

}



void xint_exit_rcu(void)