Commit 1ebbb218 authored by Mel Gorman's avatar Mel Gorman Committed by Andrew Morton
Browse files

mm/page_alloc: explicitly define how __GFP_HIGH non-blocking allocations accesses reserves 

GFP_ATOMIC allocations get flagged ALLOC_HARDER which is a vague
description.  In preparation for the removal of GFP_ATOMIC redefine
__GFP_ATOMIC to simply mean non-blocking and renaming ALLOC_HARDER to
ALLOC_NON_BLOCK accordingly.  __GFP_HIGH is required for access to
reserves but non-blocking is granted more access.  For example, GFP_NOWAIT
is non-blocking but has no special access to reserves.  A __GFP_NOFAIL
blocking allocation is granted access similar to __GFP_HIGH if the only
alternative is an OOM kill.

Link: https://lkml.kernel.org/r/20230113111217.14134-6-mgorman@techsingularity.net


Signed-off-by: default avatarMel Gorman <mgorman@techsingularity.net>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: NeilBrown <neilb@suse.de>
Cc: Thierry Reding <thierry.reding@gmail.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent ab350885

mm/internal.h

+5 −2
Original line number Diff line number Diff line
@@ -778,7 +778,10 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, 
#define ALLOC_OOM		ALLOC_NO_WATERMARKS

#endif



#define ALLOC_HARDER		 0x10 /* try to alloc harder */

#define ALLOC_NON_BLOCK		 0x10 /* Caller cannot block. Allow access

				       * to 25% of the min watermark or

				       * 62.5% if __GFP_HIGH is set.

				       */

#define ALLOC_MIN_RESERVE	 0x20 /* __GFP_HIGH set. Allow access to 50%

				       * of the min watermark.

				       */
@@ -793,7 +796,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, 
#define ALLOC_KSWAPD		0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */



/* Flags that allow allocations below the min watermark. */

#define ALLOC_RESERVES (ALLOC_HARDER|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)

#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)



enum ttu_flags;

struct tlbflush_unmap_batch;

mm/page_alloc.c

+24 −20
Original line number Diff line number Diff line
@@ -4007,18 +4007,19 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 
		 * __GFP_HIGH allows access to 50% of the min reserve as well

		 * as OOM.

		 */

		if (alloc_flags & ALLOC_MIN_RESERVE)

		if (alloc_flags & ALLOC_MIN_RESERVE) {

			min -= min / 2;



			/*

		 * Non-blocking allocations can access some of the reserve

		 * with more access if also __GFP_HIGH. The reasoning is that

		 * a non-blocking caller may incur a more severe penalty

		 * if it cannot get memory quickly, particularly if it's

		 * also __GFP_HIGH.

			 * Non-blocking allocations (e.g. GFP_ATOMIC) can

			 * access more reserves than just __GFP_HIGH. Other

			 * non-blocking allocations requests such as GFP_NOWAIT

			 * or (GFP_KERNEL & ~__GFP_DIRECT_RECLAIM) do not get

			 * access to the min reserve.

			 */

		if (alloc_flags & ALLOC_HARDER)

			if (alloc_flags & ALLOC_NON_BLOCK)

				min -= min / 4;

		}



		/*

		 * OOM victims can try even harder than the normal reserve
@@ -4869,27 +4870,29 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) 
	 * The caller may dip into page reserves a bit more if the caller

	 * cannot run direct reclaim, or if the caller has realtime scheduling

	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will

	 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_MIN_RESERVE(__GFP_HIGH).

	 * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH).

	 */

	alloc_flags |= (__force int)

		(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));



	if (gfp_mask & __GFP_ATOMIC) {

	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {

		/*

		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even

		 * if it can't schedule.

		 */

		if (!(gfp_mask & __GFP_NOMEMALLOC)) {

			alloc_flags |= ALLOC_HARDER;

			alloc_flags |= ALLOC_NON_BLOCK;



			if (order > 0)

				alloc_flags |= ALLOC_HIGHATOMIC;

		}



		/*

		 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the

		 * comment for __cpuset_node_allowed().

		 * Ignore cpuset mems for non-blocking __GFP_HIGH (probably

		 * GFP_ATOMIC) rather than fail, see the comment for

		 * __cpuset_node_allowed().

		 */

		if (alloc_flags & ALLOC_MIN_RESERVE)

			alloc_flags &= ~ALLOC_CPUSET;

	} else if (unlikely(rt_task(current)) && in_task())

		alloc_flags |= ALLOC_MIN_RESERVE;
@@ -5321,12 +5324,13 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 
		WARN_ON_ONCE_GFP(costly_order, gfp_mask);



		/*

		 * Help non-failing allocations by giving them access to memory

		 * reserves but do not use ALLOC_NO_WATERMARKS because this

		 * Help non-failing allocations by giving some access to memory

		 * reserves normally used for high priority non-blocking

		 * allocations but do not use ALLOC_NO_WATERMARKS because this

		 * could deplete whole memory reserves which would just make

		 * the situation worse

		 * the situation worse.

		 */

		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);

		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac);

		if (page)

			goto got_pg;