Commit 3218f871 authored by Alex Sierra's avatar Alex Sierra Committed by Andrew Morton
Browse files

mm: handling Non-LRU pages returned by vm_normal_pages 

With DEVICE_COHERENT, we'll soon have vm_normal_pages() return
device-managed anonymous pages that are not LRU pages.  Although they
behave like normal pages for purposes of mapping in CPU page, and for COW.
They do not support LRU lists, NUMA migration or THP.

Callers to follow_page() currently don't expect ZONE_DEVICE pages,
however, with DEVICE_COHERENT we might now return ZONE_DEVICE.  Check for
ZONE_DEVICE pages in applicable users of follow_page() as well.

Link: https://lkml.kernel.org/r/20220715150521.18165-5-alex.sierra@amd.com


Signed-off-by: default avatarAlex Sierra <alex.sierra@amd.com>
Acked-by: Felix Kuehling <Felix.Kuehling@amd.com>	[v2]
Reviewed-by: Alistair Popple <apopple@nvidia.com>	[v6]
Cc: Christoph Hellwig <hch@lst.de>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent f25cbb7a

fs/proc/task_mmu.c

+1 −1
Original line number Diff line number Diff line
@@ -1795,7 +1795,7 @@ static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, 
		return NULL;



	page = vm_normal_page(vma, addr, pte);

	if (!page)

	if (!page || is_zone_device_page(page))

		return NULL;



	if (PageReserved(page))

mm/huge_memory.c

+1 −1
Original line number Diff line number Diff line
@@ -2910,7 +2910,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 


		if (IS_ERR(page))

			continue;

		if (!page)

		if (!page || is_zone_device_page(page))

			continue;



		if (!is_transparent_hugepage(page))

mm/khugepaged.c

+6 −3
Original line number Diff line number Diff line
@@ -611,7 +611,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 
			goto out;

		}

		page = vm_normal_page(vma, address, pteval);

		if (unlikely(!page)) {

		if (unlikely(!page) || unlikely(is_zone_device_page(page))) {

			result = SCAN_PAGE_NULL;

			goto out;

		}
@@ -1261,7 +1261,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, 
			writable = true;



		page = vm_normal_page(vma, _address, pteval);

		if (unlikely(!page)) {

		if (unlikely(!page) || unlikely(is_zone_device_page(page))) {

			result = SCAN_PAGE_NULL;

			goto out_unmap;

		}
@@ -1472,7 +1472,8 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) 
			goto abort;



		page = vm_normal_page(vma, addr, *pte);



		if (WARN_ON_ONCE(page && is_zone_device_page(page)))

			page = NULL;

		/*

		 * Note that uprobe, debugger, or MAP_PRIVATE may change the

		 * page table, but the new page will not be a subpage of hpage.
@@ -1490,6 +1491,8 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) 
		if (pte_none(*pte))

			continue;

		page = vm_normal_page(vma, addr, *pte);

		if (WARN_ON_ONCE(page && is_zone_device_page(page)))

			goto abort;

		page_remove_rmap(page, vma, false);

	}

mm/ksm.c

+3 −3
Original line number Diff line number Diff line
@@ -475,7 +475,7 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) 
		cond_resched();

		page = follow_page(vma, addr,

				FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE);

		if (IS_ERR_OR_NULL(page))

		if (IS_ERR_OR_NULL(page) || is_zone_device_page(page))

			break;

		if (PageKsm(page))

			ret = handle_mm_fault(vma, addr,
@@ -560,7 +560,7 @@ static struct page *get_mergeable_page(struct rmap_item *rmap_item) 
		goto out;



	page = follow_page(vma, addr, FOLL_GET);

	if (IS_ERR_OR_NULL(page))

	if (IS_ERR_OR_NULL(page) || is_zone_device_page(page))

		goto out;

	if (PageAnon(page)) {

		flush_anon_page(vma, page, addr);
@@ -2308,7 +2308,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) 
			if (ksm_test_exit(mm))

				break;

			*page = follow_page(vma, ksm_scan.address, FOLL_GET);

			if (IS_ERR_OR_NULL(*page)) {

			if (IS_ERR_OR_NULL(*page) || is_zone_device_page(*page)) {

				ksm_scan.address += PAGE_SIZE;

				cond_resched();

				continue;

mm/madvise.c

+2 −2
Original line number Diff line number Diff line
@@ -421,7 +421,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, 
			continue;



		page = vm_normal_page(vma, addr, ptent);

		if (!page)

		if (!page || is_zone_device_page(page))

			continue;



		/*
@@ -639,7 +639,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, 
		}



		page = vm_normal_page(vma, addr, ptent);

		if (!page)

		if (!page || is_zone_device_page(page))

			continue;



		/*