Commit 0d2ebf9c authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

mm/mmap: free vm_area_struct without call_rcu in exit_mmap 

call_rcu() can take a long time when callback offloading is enabled.  Its
use in the vm_area_free can cause regressions in the exit path when
multiple VMAs are being freed.

Because exit_mmap() is called only after the last mm user drops its
refcount, the page fault handlers can't be racing with it.  Any other
possible user like oom-reaper or process_mrelease are already synchronized
using mmap_lock.  Therefore exit_mmap() can free VMAs directly, without
the use of call_rcu().

Expose __vm_area_free() and use it from exit_mmap() to avoid possible
call_rcu() floods and performance regressions caused by it.

Link: https://lkml.kernel.org/r/20230227173632.3292573-33-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 70d4cbc8

include/linux/mm.h

+2 −0
Original line number Diff line number Diff line
@@ -257,6 +257,8 @@ void setup_initial_init_mm(void *start_code, void *end_code, 
struct vm_area_struct *vm_area_alloc(struct mm_struct *);

struct vm_area_struct *vm_area_dup(struct vm_area_struct *);

void vm_area_free(struct vm_area_struct *);

/* Use only if VMA has no other users */

void __vm_area_free(struct vm_area_struct *vma);



#ifndef CONFIG_MMU

extern struct rb_root nommu_region_tree;

kernel/fork.c

+1 −1
Original line number Diff line number Diff line
@@ -480,7 +480,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) 
	return new;

}



static void __vm_area_free(struct vm_area_struct *vma)

void __vm_area_free(struct vm_area_struct *vma)

{

	free_anon_vma_name(vma);

	kmem_cache_free(vm_area_cachep, vma);

mm/mmap.c

+7 −4
Original line number Diff line number Diff line
@@ -133,7 +133,7 @@ void unlink_file_vma(struct vm_area_struct *vma) 
/*

 * Close a vm structure and free it.

 */

static void remove_vma(struct vm_area_struct *vma)

static void remove_vma(struct vm_area_struct *vma, bool unreachable)

{

	might_sleep();

	if (vma->vm_ops && vma->vm_ops->close)
@@ -141,6 +141,9 @@ static void remove_vma(struct vm_area_struct *vma) 
	if (vma->vm_file)

		fput(vma->vm_file);

	mpol_put(vma_policy(vma));

	if (unreachable)

		__vm_area_free(vma);

	else

		vm_area_free(vma);

}
@@ -2145,7 +2148,7 @@ static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) 
		if (vma->vm_flags & VM_ACCOUNT)

			nr_accounted += nrpages;

		vm_stat_account(mm, vma->vm_flags, -nrpages);

		remove_vma(vma);

		remove_vma(vma, false);

	}

	vm_unacct_memory(nr_accounted);

	validate_mm(mm);
@@ -3078,7 +3081,7 @@ void exit_mmap(struct mm_struct *mm) 
	do {

		if (vma->vm_flags & VM_ACCOUNT)

			nr_accounted += vma_pages(vma);

		remove_vma(vma);

		remove_vma(vma, true);

		count++;

		cond_resched();

	} while ((vma = mas_find(&mas, ULONG_MAX)) != NULL);