Commit c11d34fa authored by Ryan Roberts's avatar Ryan Roberts Committed by Andrew Morton
Browse files

mm/damon/ops-common: atomically test and clear young on ptes and pmds 

It is racy to non-atomically read a pte, then clear the young bit, then
write it back as this could discard dirty information.  Further, it is bad
practice to directly set a pte entry within a table.  Instead clearing
young must go through the arch-provided helper,
ptep_test_and_clear_young() to ensure it is modified atomically and to
give the arch code visibility and allow it to check (and potentially
modify) the operation.

Link: https://lkml.kernel.org/r/20230602092949.545577-3-ryan.roberts@arm.com


Fixes: 3f49584b ("mm/damon: implement primitives for the virtual memory address spaces").
Signed-off-by: default avatarRyan Roberts <ryan.roberts@arm.com>
Reviewed-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarSeongJae Park <sj@kernel.org>
Reviewed-by: default avatarMike Rapoport (IBM) <rppt@kernel.org>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent b3f78e74

mm/damon/ops-common.c

+6 −10
Original line number Diff line number Diff line
@@ -37,7 +37,7 @@ struct folio *damon_get_folio(unsigned long pfn) 
	return folio;

}



void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)

void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)

{

	bool referenced = false;

	struct folio *folio = damon_get_folio(pte_pfn(*pte));
@@ -45,13 +45,11 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr) 
	if (!folio)

		return;



	if (pte_young(*pte)) {

	if (ptep_test_and_clear_young(vma, addr, pte))

		referenced = true;

		*pte = pte_mkold(*pte);

	}



#ifdef CONFIG_MMU_NOTIFIER

	if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))

	if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE))

		referenced = true;

#endif /* CONFIG_MMU_NOTIFIER */
@@ -62,7 +60,7 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr) 
	folio_put(folio);

}



void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)

void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr)

{

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

	bool referenced = false;
@@ -71,13 +69,11 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) 
	if (!folio)

		return;



	if (pmd_young(*pmd)) {

	if (pmdp_test_and_clear_young(vma, addr, pmd))

		referenced = true;

		*pmd = pmd_mkold(*pmd);

	}



#ifdef CONFIG_MMU_NOTIFIER

	if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE))

	if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE))

		referenced = true;

#endif /* CONFIG_MMU_NOTIFIER */

mm/damon/ops-common.h

+2 −2
Original line number Diff line number Diff line
@@ -9,8 +9,8 @@ 


struct folio *damon_get_folio(unsigned long pfn);



void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr);

void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr);

void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr);

void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr);



int damon_cold_score(struct damon_ctx *c, struct damon_region *r,

			struct damos *s);

mm/damon/paddr.c

+2 −2
Original line number Diff line number Diff line
@@ -24,9 +24,9 @@ static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma, 
	while (page_vma_mapped_walk(&pvmw)) {

		addr = pvmw.address;

		if (pvmw.pte)

			damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr);

			damon_ptep_mkold(pvmw.pte, vma, addr);

		else

			damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr);

			damon_pmdp_mkold(pvmw.pmd, vma, addr);

	}

	return true;

}

mm/damon/vaddr.c

+2 −2
Original line number Diff line number Diff line
@@ -311,7 +311,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, 
		}



		if (pmd_trans_huge(*pmd)) {

			damon_pmdp_mkold(pmd, walk->mm, addr);

			damon_pmdp_mkold(pmd, walk->vma, addr);

			spin_unlock(ptl);

			return 0;

		}
@@ -323,7 +323,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, 
	pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);

	if (!pte_present(*pte))

		goto out;

	damon_ptep_mkold(pte, walk->mm, addr);

	damon_ptep_mkold(pte, walk->vma, addr);

out:

	pte_unmap_unlock(pte, ptl);

	return 0;