Commit 440703e0 authored by Liam R. Howlett's avatar Liam R. Howlett Committed by Andrew Morton
Browse files

mm/mmap: refactor locking out of __vma_adjust() 

Move the locking into vma_prepare() and vma_complete() for use elsewhere

Link: https://lkml.kernel.org/r/20230120162650.984577-41-Liam.Howlett@oracle.com


Signed-off-by: default avatarLiam R. Howlett <Liam.Howlett@oracle.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent e3d73f84

mm/internal.h

+14 −0
Original line number Diff line number Diff line
@@ -941,4 +941,18 @@ static inline int vma_iter_store_gfp(struct vma_iterator *vmi, 


	return 0;

}



/*

 * VMA lock generalization

 */

struct vma_prepare {

	struct vm_area_struct *vma;

	struct vm_area_struct *adj_next;

	struct file *file;

	struct address_space *mapping;

	struct anon_vma *anon_vma;

	struct vm_area_struct *insert;

	struct vm_area_struct *remove;

	struct vm_area_struct *remove2;

};

#endif	/* __MM_INTERNAL_H */

mm/mmap.c

+136 −95
Original line number Diff line number Diff line
@@ -573,6 +573,127 @@ inline int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 
	return -ENOMEM;

}



/*

 * vma_prepare() - Helper function for handling locking VMAs prior to altering

 * @vp: The initialized vma_prepare struct

 */

static inline void vma_prepare(struct vma_prepare *vp)

{

	if (vp->file) {

		uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);



		if (vp->adj_next)

			uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,

				      vp->adj_next->vm_end);



		i_mmap_lock_write(vp->mapping);

		if (vp->insert && vp->insert->vm_file) {

			/*

			 * Put into interval tree now, so instantiated pages

			 * are visible to arm/parisc __flush_dcache_page

			 * throughout; but we cannot insert into address

			 * space until vma start or end is updated.

			 */

			__vma_link_file(vp->insert,

					vp->insert->vm_file->f_mapping);

		}

	}



	if (vp->anon_vma) {

		anon_vma_lock_write(vp->anon_vma);

		anon_vma_interval_tree_pre_update_vma(vp->vma);

		if (vp->adj_next)

			anon_vma_interval_tree_pre_update_vma(vp->adj_next);

	}



	if (vp->file) {

		flush_dcache_mmap_lock(vp->mapping);

		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);

		if (vp->adj_next)

			vma_interval_tree_remove(vp->adj_next,

						 &vp->mapping->i_mmap);

	}



}



/*

 * vma_complete- Helper function for handling the unlocking after altering VMAs,

 * or for inserting a VMA.

 *

 * @vp: The vma_prepare struct

 * @vmi: The vma iterator

 * @mm: The mm_struct

 */

static inline void vma_complete(struct vma_prepare *vp,

				struct vma_iterator *vmi, struct mm_struct *mm)

{

	if (vp->file) {

		if (vp->adj_next)

			vma_interval_tree_insert(vp->adj_next,

						 &vp->mapping->i_mmap);

		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);

		flush_dcache_mmap_unlock(vp->mapping);

	}



	if (vp->remove && vp->file) {

		__remove_shared_vm_struct(vp->remove, vp->file, vp->mapping);

		if (vp->remove2)

			__remove_shared_vm_struct(vp->remove2, vp->file,

						  vp->mapping);

	} else if (vp->insert) {

		/*

		 * split_vma has split insert from vma, and needs

		 * us to insert it before dropping the locks

		 * (it may either follow vma or precede it).

		 */

		vma_iter_store(vmi, vp->insert);

		mm->map_count++;

	}



	if (vp->anon_vma) {

		anon_vma_interval_tree_post_update_vma(vp->vma);

		if (vp->adj_next)

			anon_vma_interval_tree_post_update_vma(vp->adj_next);

		anon_vma_unlock_write(vp->anon_vma);

	}



	if (vp->file) {

		i_mmap_unlock_write(vp->mapping);

		uprobe_mmap(vp->vma);



		if (vp->adj_next)

			uprobe_mmap(vp->adj_next);

	}



	if (vp->remove) {

again:

		if (vp->file) {

			uprobe_munmap(vp->remove, vp->remove->vm_start,

				      vp->remove->vm_end);

			fput(vp->file);

		}

		if (vp->remove->anon_vma)

			anon_vma_merge(vp->vma, vp->remove);

		mm->map_count--;

		mpol_put(vma_policy(vp->remove));

		if (!vp->remove2)

			WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);

		vm_area_free(vp->remove);



		/*

		 * In mprotect's case 6 (see comments on vma_merge),

		 * we must remove next_next too.

		 */

		if (vp->remove2) {

			vp->remove = vp->remove2;

			vp->remove2 = NULL;

			goto again;

		}

	}

	if (vp->insert && vp->file)

		uprobe_mmap(vp->insert);

}



/*

 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that

 * is already present in an i_mmap tree without adjusting the tree.
@@ -588,14 +709,13 @@ int __vma_adjust(struct vma_iterator *vmi, struct vm_area_struct *vma, 
	struct vm_area_struct *next_next = NULL;	/* uninit var warning */

	struct vm_area_struct *next = find_vma(mm, vma->vm_end);

	struct vm_area_struct *orig_vma = vma;

	struct address_space *mapping = NULL;

	struct rb_root_cached *root = NULL;

	struct anon_vma *anon_vma = NULL;

	struct file *file = vma->vm_file;

	bool vma_changed = false;

	long adjust_next = 0;

	int remove_next = 0;

	struct vm_area_struct *exporter = NULL, *importer = NULL;

	struct vma_prepare vma_prep;



	if (next && !insert) {

		if (end >= next->vm_end) {
@@ -691,39 +811,22 @@ int __vma_adjust(struct vma_iterator *vmi, struct vm_area_struct *vma, 
			   anon_vma != next->anon_vma);



	vma_adjust_trans_huge(orig_vma, start, end, adjust_next);

	if (file) {

		mapping = file->f_mapping;

		root = &mapping->i_mmap;

		uprobe_munmap(vma, vma->vm_start, vma->vm_end);



	memset(&vma_prep, 0, sizeof(vma_prep));

	vma_prep.vma = vma;

	vma_prep.anon_vma = anon_vma;

	vma_prep.file = file;

	if (adjust_next)

			uprobe_munmap(next, next->vm_start, next->vm_end);



		i_mmap_lock_write(mapping);

		if (insert && insert->vm_file) {

			/*

			 * Put into interval tree now, so instantiated pages

			 * are visible to arm/parisc __flush_dcache_page

			 * throughout; but we cannot insert into address

			 * space until vma start or end is updated.

			 */

			__vma_link_file(insert, insert->vm_file->f_mapping);

		}

	}



	if (anon_vma) {

		anon_vma_lock_write(anon_vma);

		anon_vma_interval_tree_pre_update_vma(vma);

		if (adjust_next)

			anon_vma_interval_tree_pre_update_vma(next);

		vma_prep.adj_next = next;

	if (file)

		vma_prep.mapping = file->f_mapping;

	vma_prep.insert = insert;

	if (remove_next) {

		vma_prep.remove = next;

		vma_prep.remove2 = next_next;

	}



	if (file) {

		flush_dcache_mmap_lock(mapping);

		vma_interval_tree_remove(vma, root);

		if (adjust_next)

			vma_interval_tree_remove(next, root);

	}

	vma_prepare(&vma_prep);



	if (start != vma->vm_start) {

		if (vma->vm_start < start) {
@@ -761,69 +864,7 @@ int __vma_adjust(struct vma_iterator *vmi, struct vm_area_struct *vma, 
		vma_iter_store(vmi, next);

	}



	if (file) {

		if (adjust_next)

			vma_interval_tree_insert(next, root);

		vma_interval_tree_insert(vma, root);

		flush_dcache_mmap_unlock(mapping);

	}



	if (remove_next && file) {

		__remove_shared_vm_struct(next, file, mapping);

		if (remove_next == 2)

			__remove_shared_vm_struct(next_next, file, mapping);

	} else if (insert) {

		/*

		 * split_vma has split insert from vma, and needs

		 * us to insert it before dropping the locks

		 * (it may either follow vma or precede it).

		 */

		vma_iter_store(vmi, insert);

		mm->map_count++;

	}



	if (anon_vma) {

		anon_vma_interval_tree_post_update_vma(vma);

		if (adjust_next)

			anon_vma_interval_tree_post_update_vma(next);

		anon_vma_unlock_write(anon_vma);

	}



	if (file) {

		i_mmap_unlock_write(mapping);

		uprobe_mmap(vma);



		if (adjust_next)

			uprobe_mmap(next);

	}



	if (remove_next) {

again:

		if (file) {

			uprobe_munmap(next, next->vm_start, next->vm_end);

			fput(file);

		}

		if (next->anon_vma)

			anon_vma_merge(vma, next);

		mm->map_count--;

		mpol_put(vma_policy(next));

		if (remove_next != 2)

			BUG_ON(vma->vm_end < next->vm_end);

		vm_area_free(next);



		/*

		 * In mprotect's case 6 (see comments on vma_merge),

		 * we must remove next_next too.

		 */

		if (remove_next == 2) {

			remove_next = 1;

			next = next_next;

			goto again;

		}

	}

	if (insert && file)

		uprobe_mmap(insert);



	vma_complete(&vma_prep, vmi, mm);

	vma_iter_free(vmi);

	validate_mm(mm);