Commit 0cd4d02c authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Andrew Morton
Browse files

sched: use maple tree iterator to walk VMAs 

The linked list is slower than walking the VMAs using the maple tree.  We
can't use the VMA iterator here because it doesn't support moving to an
earlier position.

Link: https://lkml.kernel.org/r/20220906194824.2110408-49-Liam.Howlett@oracle.com


Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: default avatarLiam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Tested-by: default avatarYu Zhao <yuzhao@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent fcb72a58

kernel/sched/fair.c

+7 −3
Original line number Diff line number Diff line
@@ -2930,6 +2930,7 @@ static void task_numa_work(struct callback_head *work) 
	struct task_struct *p = current;

	struct mm_struct *mm = p->mm;

	u64 runtime = p->se.sum_exec_runtime;

	MA_STATE(mas, &mm->mm_mt, 0, 0);

	struct vm_area_struct *vma;

	unsigned long start, end;

	unsigned long nr_pte_updates = 0;
@@ -2986,13 +2987,16 @@ static void task_numa_work(struct callback_head *work) 


	if (!mmap_read_trylock(mm))

		return;

	vma = find_vma(mm, start);

	mas_set(&mas, start);

	vma = mas_find(&mas, ULONG_MAX);

	if (!vma) {

		reset_ptenuma_scan(p);

		start = 0;

		vma = mm->mmap;

		mas_set(&mas, start);

		vma = mas_find(&mas, ULONG_MAX);

	}

	for (; vma; vma = vma->vm_next) {



	for (; vma; vma = mas_find(&mas, ULONG_MAX)) {

		if (!vma_migratable(vma) || !vma_policy_mof(vma) ||

			is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {

			continue;