Commit 33108b05 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Andrew Morton
Browse files

mm/mlock: use vma iterator and maple state instead of vma linked list 

Handle overflow checking in count_mm_mlocked_page_nr() differently.

Link: https://lkml.kernel.org/r/20220906194824.2110408-58-Liam.Howlett@oracle.com


Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: default avatarLiam R. Howlett <Liam.Howlett@Oracle.com>
Tested-by: default avatarYu Zhao <yuzhao@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 66850be5

mm/mlock.c

+19 −16
Original line number Diff line number Diff line
@@ -471,6 +471,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, 
	unsigned long nstart, end, tmp;

	struct vm_area_struct *vma, *prev;

	int error;

	MA_STATE(mas, &current->mm->mm_mt, start, start);



	VM_BUG_ON(offset_in_page(start));

	VM_BUG_ON(len != PAGE_ALIGN(len));
@@ -479,13 +480,14 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, 
		return -EINVAL;

	if (end == start)

		return 0;

	vma = find_vma(current->mm, start);

	if (!vma || vma->vm_start > start)

	vma = mas_walk(&mas);

	if (!vma)

		return -ENOMEM;



	prev = vma->vm_prev;

	if (start > vma->vm_start)

		prev = vma;

	else

		prev = mas_prev(&mas, 0);



	for (nstart = start ; ; ) {

		vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
@@ -505,7 +507,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, 
		if (nstart >= end)

			break;



		vma = prev->vm_next;

		vma = find_vma(prev->vm_mm, prev->vm_end);

		if (!vma || vma->vm_start != nstart) {

			error = -ENOMEM;

			break;
@@ -526,24 +528,23 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, 
{

	struct vm_area_struct *vma;

	unsigned long count = 0;

	unsigned long end;

	VMA_ITERATOR(vmi, mm, start);



	if (mm == NULL)

		mm = current->mm;



	vma = find_vma(mm, start);

	if (vma == NULL)

		return 0;



	for (; vma ; vma = vma->vm_next) {

		if (start >= vma->vm_end)

			continue;

		if (start + len <=  vma->vm_start)

			break;

	/* Don't overflow past ULONG_MAX */

	if (unlikely(ULONG_MAX - len < start))

		end = ULONG_MAX;

	else

		end = start + len;

	for_each_vma_range(vmi, vma, end) {

		if (vma->vm_flags & VM_LOCKED) {

			if (start > vma->vm_start)

				count -= (start - vma->vm_start);

			if (start + len < vma->vm_end) {

				count += start + len - vma->vm_start;

			if (end < vma->vm_end) {

				count += end - vma->vm_start;

				break;

			}

			count += vma->vm_end - vma->vm_start;
@@ -659,6 +660,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) 
 */

static int apply_mlockall_flags(int flags)

{

	MA_STATE(mas, &current->mm->mm_mt, 0, 0);

	struct vm_area_struct *vma, *prev = NULL;

	vm_flags_t to_add = 0;
@@ -679,7 +681,7 @@ static int apply_mlockall_flags(int flags) 
			to_add |= VM_LOCKONFAULT;

	}



	for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {

	mas_for_each(&mas, vma, ULONG_MAX) {

		vm_flags_t newflags;



		newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
@@ -687,6 +689,7 @@ static int apply_mlockall_flags(int flags) 


		/* Ignore errors */

		mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);

		mas_pause(&mas);

		cond_resched();

	}

out: