Commit ec8832d0 authored by Alistair Popple's avatar Alistair Popple Committed by Andrew Morton
Browse files

mmu_notifiers: don't invalidate secondary TLBs as part of mmu_notifier_invalidate_range_end() 

Secondary TLBs are now invalidated from the architecture specific TLB
invalidation functions.  Therefore there is no need to explicitly notify
or invalidate as part of the range end functions.  This means we can
remove mmu_notifier_invalidate_range_end_only() and some of the
ptep_*_notify() functions.

Link: https://lkml.kernel.org/r/90d749d03cbab256ca0edeb5287069599566d783.1690292440.git-series.apopple@nvidia.com


Signed-off-by: default avatarAlistair Popple <apopple@nvidia.com>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Cc: Andrew Donnellan <ajd@linux.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chaitanya Kumar Borah <chaitanya.kumar.borah@intel.com>
Cc: Frederic Barrat <fbarrat@linux.ibm.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Kevin Tian <kevin.tian@intel.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Nicolin Chen <nicolinc@nvidia.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Will Deacon <will@kernel.org>
Cc: Zhi Wang <zhi.wang.linux@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 6bbd42e2

include/linux/mmu_notifier.h

+2 −54
Original line number Diff line number Diff line
@@ -395,8 +395,7 @@ extern int __mmu_notifier_test_young(struct mm_struct *mm, 
extern void __mmu_notifier_change_pte(struct mm_struct *mm,

				      unsigned long address, pte_t pte);

extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);

extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,

				  bool only_end);

extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r);

extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,

				  unsigned long start, unsigned long end);

extern bool
@@ -481,14 +480,7 @@ mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range) 
		might_sleep();



	if (mm_has_notifiers(range->mm))

		__mmu_notifier_invalidate_range_end(range, false);

}



static inline void

mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)

{

	if (mm_has_notifiers(range->mm))

		__mmu_notifier_invalidate_range_end(range, true);

		__mmu_notifier_invalidate_range_end(range);

}



static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
@@ -582,45 +574,6 @@ static inline void mmu_notifier_range_init_owner( 
	__young;							\

})



#define	ptep_clear_flush_notify(__vma, __address, __ptep)		\

({									\

	unsigned long ___addr = __address & PAGE_MASK;			\

	struct mm_struct *___mm = (__vma)->vm_mm;			\

	pte_t ___pte;							\

									\

	___pte = ptep_clear_flush(__vma, __address, __ptep);		\

	mmu_notifier_invalidate_range(___mm, ___addr,			\

					___addr + PAGE_SIZE);		\

									\

	___pte;								\

})



#define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd)		\

({									\

	unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;		\

	struct mm_struct *___mm = (__vma)->vm_mm;			\

	pmd_t ___pmd;							\

									\

	___pmd = pmdp_huge_clear_flush(__vma, __haddr, __pmd);		\

	mmu_notifier_invalidate_range(___mm, ___haddr,			\

				      ___haddr + HPAGE_PMD_SIZE);	\

									\

	___pmd;								\

})



#define pudp_huge_clear_flush_notify(__vma, __haddr, __pud)		\

({									\

	unsigned long ___haddr = __haddr & HPAGE_PUD_MASK;		\

	struct mm_struct *___mm = (__vma)->vm_mm;			\

	pud_t ___pud;							\

									\

	___pud = pudp_huge_clear_flush(__vma, __haddr, __pud);		\

	mmu_notifier_invalidate_range(___mm, ___haddr,			\

				      ___haddr + HPAGE_PUD_SIZE);	\

									\

	___pud;								\

})



/*

 * set_pte_at_notify() sets the pte _after_ running the notifier.

 * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -711,11 +664,6 @@ void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range) 
{

}



static inline void

mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)

{

}



static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,

				  unsigned long start, unsigned long end)

{

kernel/events/uprobes.c

+1 −1
Original line number Diff line number Diff line
@@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 
	}



	flush_cache_page(vma, addr, pte_pfn(ptep_get(pvmw.pte)));

	ptep_clear_flush_notify(vma, addr, pvmw.pte);

	ptep_clear_flush(vma, addr, pvmw.pte);

	if (new_page)

		set_pte_at_notify(mm, addr, pvmw.pte,

				  mk_pte(new_page, vma->vm_page_prot));

mm/huge_memory.c

+4 −21
Original line number Diff line number Diff line
@@ -2003,7 +2003,7 @@ static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 


	count_vm_event(THP_SPLIT_PUD);



	pudp_huge_clear_flush_notify(vma, haddr, pud);

	pudp_huge_clear_flush(vma, haddr, pud);

}



void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
@@ -2023,11 +2023,7 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 


out:

	spin_unlock(ptl);

	/*

	 * No need to double call mmu_notifier->invalidate_range() callback as

	 * the above pudp_huge_clear_flush_notify() did already call it.

	 */

	mmu_notifier_invalidate_range_only_end(&range);

	mmu_notifier_invalidate_range_end(&range);

}

#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
@@ -2094,7 +2090,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 
	count_vm_event(THP_SPLIT_PMD);



	if (!vma_is_anonymous(vma)) {

		old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);

		old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);

		/*

		 * We are going to unmap this huge page. So

		 * just go ahead and zap it
@@ -2304,20 +2300,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 


out:

	spin_unlock(ptl);

	/*

	 * No need to double call mmu_notifier->invalidate_range() callback.

	 * They are 3 cases to consider inside __split_huge_pmd_locked():

	 *  1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious

	 *  2) __split_huge_zero_page_pmd() read only zero page and any write

	 *    fault will trigger a flush_notify before pointing to a new page

	 *    (it is fine if the secondary mmu keeps pointing to the old zero

	 *    page in the meantime)

	 *  3) Split a huge pmd into pte pointing to the same page. No need

	 *     to invalidate secondary tlb entry they are all still valid.

	 *     any further changes to individual pte will notify. So no need

	 *     to call mmu_notifier->invalidate_range()

	 */

	mmu_notifier_invalidate_range_only_end(&range);

	mmu_notifier_invalidate_range_end(&range);

}



void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,

mm/hugetlb.c

+0 −1
Original line number Diff line number Diff line
@@ -5688,7 +5688,6 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, 


		/* Break COW or unshare */

		huge_ptep_clear_flush(vma, haddr, ptep);

		mmu_notifier_invalidate_range(mm, range.start, range.end);

		page_remove_rmap(&old_folio->page, vma, true);

		hugepage_add_new_anon_rmap(new_folio, vma, haddr);

		if (huge_pte_uffd_wp(pte))

mm/memory.c

+2 −6
Original line number Diff line number Diff line
@@ -3155,7 +3155,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) 
		 * that left a window where the new PTE could be loaded into

		 * some TLBs while the old PTE remains in others.

		 */

		ptep_clear_flush_notify(vma, vmf->address, vmf->pte);

		ptep_clear_flush(vma, vmf->address, vmf->pte);

		folio_add_new_anon_rmap(new_folio, vma, vmf->address);

		folio_add_lru_vma(new_folio, vma);

		/*
@@ -3201,11 +3201,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) 
		pte_unmap_unlock(vmf->pte, vmf->ptl);

	}



	/*

	 * No need to double call mmu_notifier->invalidate_range() callback as

	 * the above ptep_clear_flush_notify() did already call it.

	 */

	mmu_notifier_invalidate_range_only_end(&range);

	mmu_notifier_invalidate_range_end(&range);



	if (new_folio)

		folio_put(new_folio);