Commit 589d9973 authored by Mel Gorman's avatar Mel Gorman Committed by Andrew Morton
Browse files

mm/page_alloc: split out buddy removal code from rmqueue into separate helper 

This is a preparation page to allow the buddy removal code to be reused in
a later patch.

No functional change.

Link: https://lkml.kernel.org/r/20220624125423.6126-4-mgorman@techsingularity.net


Signed-off-by: default avatarMel Gorman <mgorman@techsingularity.net>
Tested-by: default avatarMinchan Kim <minchan@kernel.org>
Acked-by: default avatarMinchan Kim <minchan@kernel.org>
Reviewed-by: default avatarNicolas Saenz Julienne <nsaenzju@redhat.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Tested-by: default avatarYu Zhao <yuzhao@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 5d0a661d

mm/page_alloc.c

+47 −34
Original line number Diff line number Diff line
@@ -3641,6 +3641,43 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 
#endif

}



static __always_inline

struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,

			   unsigned int order, unsigned int alloc_flags,

			   int migratetype)

{

	struct page *page;

	unsigned long flags;



	do {

		page = NULL;

		spin_lock_irqsave(&zone->lock, flags);

		/*

		 * order-0 request can reach here when the pcplist is skipped

		 * due to non-CMA allocation context. HIGHATOMIC area is

		 * reserved for high-order atomic allocation, so order-0

		 * request should skip it.

		 */

		if (order > 0 && alloc_flags & ALLOC_HARDER)

			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);

		if (!page) {

			page = __rmqueue(zone, order, migratetype, alloc_flags);

			if (!page) {

				spin_unlock_irqrestore(&zone->lock, flags);

				return NULL;

			}

		}

		__mod_zone_freepage_state(zone, -(1 << order),

					  get_pcppage_migratetype(page));

		spin_unlock_irqrestore(&zone->lock, flags);

	} while (check_new_pages(page, order));



	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);

	zone_statistics(preferred_zone, zone, 1);



	return page;

}



/* Remove page from the per-cpu list, caller must protect the list */

static inline

struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
@@ -3721,9 +3758,14 @@ struct page *rmqueue(struct zone *preferred_zone, 
			gfp_t gfp_flags, unsigned int alloc_flags,

			int migratetype)

{

	unsigned long flags;

	struct page *page;



	/*

	 * We most definitely don't want callers attempting to

	 * allocate greater than order-1 page units with __GFP_NOFAIL.

	 */

	WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));



	if (likely(pcp_allowed_order(order))) {

		/*

		 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
@@ -3737,35 +3779,10 @@ struct page *rmqueue(struct zone *preferred_zone, 
		}

	}



	/*

	 * We most definitely don't want callers attempting to

	 * allocate greater than order-1 page units with __GFP_NOFAIL.

	 */

	WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));



	do {

		page = NULL;

		spin_lock_irqsave(&zone->lock, flags);

		/*

		 * order-0 request can reach here when the pcplist is skipped

		 * due to non-CMA allocation context. HIGHATOMIC area is

		 * reserved for high-order atomic allocation, so order-0

		 * request should skip it.

		 */

		if (order > 0 && alloc_flags & ALLOC_HARDER)

			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);

		if (!page) {

			page = __rmqueue(zone, order, migratetype, alloc_flags);

			if (!page)

				goto failed;

		}

		__mod_zone_freepage_state(zone, -(1 << order),

					  get_pcppage_migratetype(page));

		spin_unlock_irqrestore(&zone->lock, flags);

	} while (check_new_pages(page, order));



	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);

	zone_statistics(preferred_zone, zone, 1);

	page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags,

							migratetype);

	if (unlikely(!page))

		return NULL;



out:

	/* Separate test+clear to avoid unnecessary atomics */
@@ -3776,10 +3793,6 @@ struct page *rmqueue(struct zone *preferred_zone, 


	VM_BUG_ON_PAGE(page && bad_range(zone, page), page);

	return page;



failed:

	spin_unlock_irqrestore(&zone->lock, flags);

	return NULL;

}



#ifdef CONFIG_FAIL_PAGE_ALLOC