Commit d05e98f5 authored by Chen Ridong's avatar Chen Ridong
Browse files

memcg: fix soft lockup in the OOM process 

maillist inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IBE42N

Reference: https://lore.kernel.org/linux-kernel/8cf29751-7c71-52ff-5492-0019ca7b0e02@google.com/



----------------------------------------

A soft lockup issue was found in the product with about 56,000 tasks were
in the OOM cgroup, it was traversing them when the soft lockup was
triggered.

watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [VM Thread:1503066]
CPU: 2 PID: 1503066 Comm: VM Thread Kdump: loaded Tainted: G
Hardware name: Huawei Cloud OpenStack Nova, BIOS
RIP: 0010:console_unlock+0x343/0x540
RSP: 0000:ffffb751447db9a0 EFLAGS: 00000247 ORIG_RAX: ffffffffffffff13
RAX: 0000000000000001 RBX: 0000000000000000 RCX: 00000000ffffffff
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000247
RBP: ffffffffafc71f90 R08: 0000000000000000 R09: 0000000000000040
R10: 0000000000000080 R11: 0000000000000000 R12: ffffffffafc74bd0
R13: ffffffffaf60a220 R14: 0000000000000247 R15: 0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2fe6ad91f0 CR3: 00000004b2076003 CR4: 0000000000360ee0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
 vprintk_emit+0x193/0x280
 printk+0x52/0x6e
 dump_task+0x114/0x130
 mem_cgroup_scan_tasks+0x76/0x100
 dump_header+0x1fe/0x210
 oom_kill_process+0xd1/0x100
 out_of_memory+0x125/0x570
 mem_cgroup_out_of_memory+0xb5/0xd0
 try_charge+0x720/0x770
 mem_cgroup_try_charge+0x86/0x180
 mem_cgroup_try_charge_delay+0x1c/0x40
 do_anonymous_page+0xb5/0x390
 handle_mm_fault+0xc4/0x1f0

This is because thousands of processes are in the OOM cgroup, it takes a
long time to traverse all of them. As a result, this lead to soft lockup
in the OOM process.

To fix this issue, call 'cond_resched' in the 'mem_cgroup_scan_tasks'
function per 1000 iterations. For global OOM, call
'touch_softlockup_watchdog' per 1000 iterations to avoid this issue.

Fixes: 9cbb78bb ("mm, memcg: introduce own oom handler to iterate only over its own threads")
Signed-off-by: default avatarChen Ridong <chenridong@huawei.com>
parent efbe143e

mm/memcontrol.c

+6 −1
Original line number Diff line number Diff line
@@ -1322,6 +1322,7 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 
{

	struct mem_cgroup *iter;

	int ret = 0;

	int i = 0;



	BUG_ON(mem_cgroup_is_root(memcg));
@@ -1330,8 +1331,12 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 
		struct task_struct *task;



		css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);

		while (!ret && (task = css_task_iter_next(&it)))

		while (!ret && (task = css_task_iter_next(&it))) {

			/* Avoid potential softlockup warning */

			if ((++i & 1023) == 0)

				cond_resched();

			ret = fn(task, arg);

		}

		css_task_iter_end(&it);

		if (ret) {

			mem_cgroup_iter_break(memcg, iter);

mm/oom_kill.c

+7 −1
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ 
#include <linux/kthread.h>

#include <linux/init.h>

#include <linux/mmu_notifier.h>

#include <linux/nmi.h>



#include <asm/tlb.h>

#include "internal.h"
@@ -482,10 +483,15 @@ static void dump_tasks(struct oom_control *oc) 
		memcg_print_bad_task(oc);

	} else {

		struct task_struct *p;

		int i = 0;



		rcu_read_lock();

		for_each_process(p)

		for_each_process(p) {

			/* Avoid potential softlockup warning */

			if ((++i & 1023) == 0)

				touch_softlockup_watchdog();

			dump_task(p, oc);

		}

		rcu_read_unlock();

	}

}