!5688 v3 Optimize compaction

#ifdef CONFIG_COMPACTION

TRACE_EVENT(mm_compaction_migratepages,

	TP_PROTO(struct compact_control *cc,

	TP_PROTO(unsigned int nr_migratepages,

		unsigned int nr_succeeded),

	TP_ARGS(cc, nr_succeeded),

	TP_ARGS(nr_migratepages, nr_succeeded),

	TP_STRUCT__entry(

		__field(unsigned long, nr_migrated)

	TP_fast_assign(

		__entry->nr_migrated = nr_succeeded;

		__entry->nr_failed = cc->nr_migratepages - nr_succeeded;

		__entry->nr_failed = nr_migratepages - nr_succeeded;

	),

	TP_printk("nr_migrated=%lu nr_failed=%lu",

{

	count_vm_events(item, delta);

}

/*

 * order == -1 is expected when compacting proactively via

 * 1. /proc/sys/vm/compact_memory

 * 2. /sys/devices/system/node/nodex/compact

 * 3. /proc/sys/vm/compaction_proactiveness

 */

static inline bool is_via_compact_memory(int order)

{

	return order == -1;

}

#else

#define count_compact_event(item) do { } while (0)

#define count_compact_events(item, delta) do { } while (0)

static inline bool is_via_compact_memory(int order) { return false; }

#endif

#if defined CONFIG_COMPACTION || defined CONFIG_CMA

#define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)

#endif

static unsigned long release_freepages(struct list_head *freelist)

{

	struct page *page, *next;

	unsigned long high_pfn = 0;

	list_for_each_entry_safe(page, next, freelist, lru) {

		unsigned long pfn = page_to_pfn(page);

		list_del(&page->lru);

		__free_page(page);

		if (pfn > high_pfn)

			high_pfn = pfn;

	}

	return high_pfn;

}

static void split_map_pages(struct list_head *list)

static void split_map_pages(struct list_head *freepages)

{

	unsigned int i, order, nr_pages;

	unsigned int i, order;

	struct page *page, *next;

	LIST_HEAD(tmp_list);

	list_for_each_entry_safe(page, next, list, lru) {

	for (order = 0; order < NR_PAGE_ORDERS; order++) {

		list_for_each_entry_safe(page, next, &freepages[order], lru) {

			unsigned int nr_pages;

			list_del(&page->lru);

		order = page_private(page);

			nr_pages = 1 << order;

			post_alloc_hook(page, order, __GFP_MOVABLE);

				page++;

			}

		}

		list_splice_init(&tmp_list, &freepages[0]);

	}

}

static unsigned long release_free_list(struct list_head *freepages)

{

	int order;

	unsigned long high_pfn = 0;

	for (order = 0; order < NR_PAGE_ORDERS; order++) {

		struct page *page, *next;

		list_for_each_entry_safe(page, next, &freepages[order], lru) {

			unsigned long pfn = page_to_pfn(page);

	list_splice(&tmp_list, list);

			list_del(&page->lru);

			/*

			 * Convert free pages into post allocation pages, so

			 * that we can free them via __free_page.

			 */

			post_alloc_hook(page, order, __GFP_MOVABLE);

			__free_pages(page, order);

			if (pfn > high_pfn)

				high_pfn = pfn;

		}

	}

	return high_pfn;

}

#ifdef CONFIG_COMPACTION

	bool source_set = false;

	bool free_set = false;

	/* Only flush if a full compaction finished recently */

	if (!zone->compact_blockskip_flush)

		return;

		if (!populated_zone(zone))

			continue;

		/* Only flush if a full compaction finished recently */

		if (zone->compact_blockskip_flush)

		__reset_isolation_suitable(zone);

	}

}

		if (PageCompound(page)) {

			const unsigned int order = compound_order(page);

			if (likely(order <= MAX_ORDER)) {

			if (blockpfn + (1UL << order) <= end_pfn) {

				blockpfn += (1UL << order) - 1;

				page += (1UL << order) - 1;

				nr_scanned += (1UL << order) - 1;

			}

			goto isolate_fail;

		}

		nr_scanned += isolated - 1;

		total_isolated += isolated;

		cc->nr_freepages += isolated;

		list_add_tail(&page->lru, freelist);

		list_add_tail(&page->lru, &freelist[order]);

		if (!strict && cc->nr_migratepages <= cc->nr_freepages) {

			blockpfn += isolated;

		spin_unlock_irqrestore(&cc->zone->lock, flags);

	/*

	 * There is a tiny chance that we have read bogus compound_order(),

	 * so be careful to not go outside of the pageblock.

	 * Be careful to not go outside of the pageblock.

	 */

	if (unlikely(blockpfn > end_pfn))

		blockpfn = end_pfn;

			unsigned long start_pfn, unsigned long end_pfn)

{

	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;

	LIST_HEAD(freelist);

	int order;

	struct list_head tmp_freepages[NR_PAGE_ORDERS];

	for (order = 0; order < NR_PAGE_ORDERS; order++)

		INIT_LIST_HEAD(&tmp_freepages[order]);

	pfn = start_pfn;

	block_start_pfn = pageblock_start_pfn(pfn);

			break;

		isolated = isolate_freepages_block(cc, &isolate_start_pfn,

					block_end_pfn, &freelist, 0, true);

					block_end_pfn, tmp_freepages, 0, true);

		/*

		 * In strict mode, isolate_freepages_block() returns 0 if

		 */

	}

	/* __isolate_free_page() does not map the pages */

	split_map_pages(&freelist);

	if (pfn < end_pfn) {

		/* Loop terminated early, cleanup. */

		release_freepages(&freelist);

		release_free_list(tmp_freepages);

		return 0;

	}

	/* __isolate_free_page() does not map the pages */

	split_map_pages(tmp_freepages);

	/* We don't use freelists for anything. */

	return pfn;

}

	return too_many;

}

/**

 * skip_isolation_on_order() - determine when to skip folio isolation based on

 *			       folio order and compaction target order

 * @order:		to-be-isolated folio order

 * @target_order:	compaction target order

 *

 * This avoids unnecessary folio isolations during compaction.

 */

static bool skip_isolation_on_order(int order, int target_order)

{

	/*

	 * Unless we are performing global compaction (i.e.,

	 * is_via_compact_memory), skip any folios that are larger than the

	 * target order: we wouldn't be here if we'd have a free folio with

	 * the desired target_order, so migrating this folio would likely fail

	 * later.

	 */

	if (!is_via_compact_memory(target_order) && order >= target_order)

		return true;

	/*

	 * We limit memory compaction to pageblocks and won't try

	 * creating free blocks of memory that are larger than that.

	 */

	return order >= pageblock_order;

}

/**

 * isolate_migratepages_block() - isolate all migrate-able pages within

 *				  a single pageblock

			valid_page = page;

		}

		if (PageHuge(page) && cc->alloc_contig) {

		if (PageHuge(page)) {

			/*

			 * skip hugetlbfs if we are not compacting for pages

			 * bigger than its order. THPs and other compound pages

			 * are handled below.

			 */

			if (!cc->alloc_contig) {

				const unsigned int order = compound_order(page);

				if (order <= MAX_PAGE_ORDER) {

					low_pfn += (1UL << order) - 1;

					nr_scanned += (1UL << order) - 1;

				}

				goto isolate_fail;

			}

			/* for alloc_contig case */

			if (locked) {

				unlock_page_lruvec_irqrestore(locked, flags);

				locked = NULL;

			 * a valid page order. Consider only values in the

			 * valid order range to prevent low_pfn overflow.

			 */

			if (freepage_order > 0 && freepage_order <= MAX_ORDER) {

			if (freepage_order > 0 && freepage_order <= MAX_PAGE_ORDER) {

				low_pfn += (1UL << freepage_order) - 1;

				nr_scanned += (1UL << freepage_order) - 1;

			}

		}

		/*

		 * Regardless of being on LRU, compound pages such as THP and

		 * hugetlbfs are not to be compacted unless we are attempting

		 * an allocation much larger than the huge page size (eg CMA).

		 * We can potentially save a lot of iterations if we skip them

		 * at once. The check is racy, but we can consider only valid

		 * values and the only danger is skipping too much.

		 * Regardless of being on LRU, compound pages such as THP

		 * (hugetlbfs is handled above) are not to be compacted unless

		 * we are attempting an allocation larger than the compound

		 * page size. We can potentially save a lot of iterations if we

		 * skip them at once. The check is racy, but we can consider

		 * only valid values and the only danger is skipping too much.

		 */

		if (PageCompound(page) && !cc->alloc_contig) {

			const unsigned int order = compound_order(page);

			if (likely(order <= MAX_ORDER)) {

			/* Skip based on page order and compaction target order. */

			if (skip_isolation_on_order(order, cc->order)) {

				if (order <= MAX_PAGE_ORDER) {

					low_pfn += (1UL << order) - 1;

					nr_scanned += (1UL << order) - 1;

				}

				goto isolate_fail;

			}

		}

		/*

		 * Check may be lockless but that's ok as we recheck later.

			}

			/*

			 * folio become large since the non-locked check,

			 * and it's on LRU.

			 * Check LRU folio order under the lock

			 */

			if (unlikely(folio_test_large(folio) && !cc->alloc_contig)) {

			if (unlikely(skip_isolation_on_order(folio_order(folio),

							     cc->order) &&

				     !cc->alloc_contig)) {

				low_pfn += folio_nr_pages(folio) - 1;

				nr_scanned += folio_nr_pages(folio) - 1;

				folio_set_lru(folio);

{

	/* If the page is a large free page, then disallow migration */

	if (PageBuddy(page)) {

		int order = cc->order > 0 ? cc->order : pageblock_order;

		/*

		 * We are checking page_order without zone->lock taken. But

		 * the only small danger is that we skip a potentially suitable

		 * pageblock, so it's not worth to check order for valid range.

		 */

		if (buddy_order_unsafe(page) >= pageblock_order)

		if (buddy_order_unsafe(page) >= order)

			return false;

	}

{

	LIST_HEAD(sublist);

	if (!list_is_last(freelist, &freepage->lru)) {

		list_cut_before(&sublist, freelist, &freepage->lru);

	if (!list_is_first(&freepage->buddy_list, freelist)) {

		list_cut_before(&sublist, freelist, &freepage->buddy_list);

		list_splice_tail(&sublist, freelist);

	}

}

{

	LIST_HEAD(sublist);

	if (!list_is_first(freelist, &freepage->lru)) {

		list_cut_position(&sublist, freelist, &freepage->lru);

	if (!list_is_last(&freepage->buddy_list, freelist)) {

		list_cut_position(&sublist, freelist, &freepage->buddy_list);

		list_splice_tail(&sublist, freelist);

	}

}

	if (!page)

		return;

	isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);

	isolate_freepages_block(cc, &start_pfn, end_pfn, cc->freepages, 1, false);

	/* Skip this pageblock in the future as it's full or nearly full */

	if (start_pfn == end_pfn && !cc->no_set_skip_hint)

				nr_scanned += nr_isolated - 1;

				total_isolated += nr_isolated;

				cc->nr_freepages += nr_isolated;

				list_add_tail(&page->lru, &cc->freepages);

				list_add_tail(&page->lru, &cc->freepages[order]);

				count_compact_events(COMPACTISOLATED, nr_isolated);

			} else {

				/* If isolation fails, abort the search */

						min(pageblock_end_pfn(min_pfn),

						    zone_end_pfn(cc->zone)),

						cc->zone);

					if (page && !suitable_migration_target(cc, page))

						page = NULL;

					cc->free_pfn = min_pfn;

				}

			}

	unsigned long isolate_start_pfn; /* exact pfn we start at */

	unsigned long block_end_pfn;	/* end of current pageblock */

	unsigned long low_pfn;	     /* lowest pfn scanner is able to scan */

	struct list_head *freelist = &cc->freepages;

	unsigned int stride;

	/* Try a small search of the free lists for a candidate */

	fast_isolate_freepages(cc);

	if (cc->nr_freepages)

		goto splitmap;

		return;

	/*

	 * Initialise the free scanner. The starting point is where we last

		/* Found a block suitable for isolating free pages from. */

		nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,

					block_end_pfn, freelist, stride, false);

					block_end_pfn, cc->freepages, stride, false);

		/* Update the skip hint if the full pageblock was scanned */

		if (isolate_start_pfn == block_end_pfn)

	 * and the loop terminated due to isolate_start_pfn < low_pfn

	 */

	cc->free_pfn = isolate_start_pfn;

splitmap:

	/* __isolate_free_page() does not map the pages */

	split_map_pages(freelist);

}

/*

{

	struct compact_control *cc = (struct compact_control *)data;

	struct folio *dst;

	int order = folio_order(src);

	bool has_isolated_pages = false;

	int start_order;

	struct page *freepage;

	unsigned long size;

	if (list_empty(&cc->freepages)) {

		isolate_freepages(cc);

again:

	for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++)

		if (!list_empty(&cc->freepages[start_order]))

			break;

		if (list_empty(&cc->freepages))

	/* no free pages in the list */

	if (start_order == NR_PAGE_ORDERS) {

		if (has_isolated_pages)

			return NULL;

		isolate_freepages(cc);

		has_isolated_pages = true;

		goto again;

	}

	dst = list_entry(cc->freepages.next, struct folio, lru);

	list_del(&dst->lru);

	cc->nr_freepages--;

	freepage = list_first_entry(&cc->freepages[start_order], struct page,

				lru);

	size = 1 << start_order;

	list_del(&freepage->lru);

	while (start_order > order) {

		start_order--;

		size >>= 1;

	return dst;

		list_add(&freepage[size].lru, &cc->freepages[start_order]);

		set_page_private(&freepage[size], start_order);

	}

	dst = (struct folio *)freepage;

	post_alloc_hook(&dst->page, order, __GFP_MOVABLE);

	if (order)

		prep_compound_page(&dst->page, order);

	cc->nr_freepages -= 1 << order;

	cc->nr_migratepages -= 1 << order;

	return page_rmappable_folio(&dst->page);

}

/*

static void compaction_free(struct folio *dst, unsigned long data)

{

	struct compact_control *cc = (struct compact_control *)data;

	int order = folio_order(dst);

	struct page *page = &dst->page;

	list_add(&dst->lru, &cc->freepages);

	cc->nr_freepages++;

	if (folio_put_testzero(dst)) {

		free_pages_prepare(page, order);

		list_add(&dst->lru, &cc->freepages[order]);

		cc->nr_freepages += 1 << order;

	}

	cc->nr_migratepages += 1 << order;

	/*

	 * someone else has referenced the page, we cannot take it back to our

	 * free list.

	 */

}

/* possible outcome of isolate_migratepages */

	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;

}

/*

 * order == -1 is expected when compacting via

 * /proc/sys/vm/compact_memory

 */

static inline bool is_via_compact_memory(int order)

{

	return order == -1;

}

/*

 * Determine whether kswapd is (or recently was!) running on this node.

 *

	/* Direct compactor: Is a suitable page free? */

	ret = COMPACT_NO_SUITABLE_PAGE;

	for (order = cc->order; order <= MAX_ORDER; order++) {

	for (order = cc->order; order < NR_PAGE_ORDERS; order++) {

		struct free_area *area = &cc->zone->free_area[order];

		bool can_steal;

	return false;

}

/*

 * Should we do compaction for target allocation order.

 * Return COMPACT_SUCCESS if allocation for target order can be already

 * satisfied

 * Return COMPACT_SKIPPED if compaction for target order is likely to fail

 * Return COMPACT_CONTINUE if compaction for target order should be ran

 */

static enum compact_result

compaction_suit_allocation_order(struct zone *zone, unsigned int order,

				 int highest_zoneidx, unsigned int alloc_flags)

{

	unsigned long watermark;

	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);

	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,

			      alloc_flags))

		return COMPACT_SUCCESS;

	if (!compaction_suitable(zone, order, highest_zoneidx))

		return COMPACT_SKIPPED;

	return COMPACT_CONTINUE;

}

static enum compact_result

compact_zone(struct compact_control *cc, struct capture_control *capc)

{

	unsigned long last_migrated_pfn;

	const bool sync = cc->mode != MIGRATE_ASYNC;

	bool update_cached;

	unsigned int nr_succeeded = 0;

	unsigned int nr_succeeded = 0, nr_migratepages;

	int order;

	/*

	 * These counters track activities during zone compaction.  Initialize

	cc->total_free_scanned = 0;

	cc->nr_migratepages = 0;

	cc->nr_freepages = 0;

	INIT_LIST_HEAD(&cc->freepages);

	for (order = 0; order < NR_PAGE_ORDERS; order++)

		INIT_LIST_HEAD(&cc->freepages[order]);

	INIT_LIST_HEAD(&cc->migratepages);

	cc->migratetype = gfp_migratetype(cc->gfp_mask);

	if (!is_via_compact_memory(cc->order)) {

		unsigned long watermark;

		/* Allocation can already succeed, nothing to do */

		watermark = wmark_pages(cc->zone,

					cc->alloc_flags & ALLOC_WMARK_MASK);

		if (zone_watermark_ok(cc->zone, cc->order, watermark,

				      cc->highest_zoneidx, cc->alloc_flags))

			return COMPACT_SUCCESS;

		/* Compaction is likely to fail */

		if (!compaction_suitable(cc->zone, cc->order,

					 cc->highest_zoneidx))

			return COMPACT_SKIPPED;

		ret = compaction_suit_allocation_order(cc->zone, cc->order,

						       cc->highest_zoneidx,

						       cc->alloc_flags);

		if (ret != COMPACT_CONTINUE)

			return ret;

	}

	/*

				pageblock_start_pfn(cc->migrate_pfn - 1));

		}

		/*

		 * Record the number of pages to migrate since the

		 * compaction_alloc/free() will update cc->nr_migratepages

		 * properly.

		 */

		nr_migratepages = cc->nr_migratepages;

		err = migrate_pages(&cc->migratepages, compaction_alloc,

				compaction_free, (unsigned long)cc, cc->mode,

				MR_COMPACTION, &nr_succeeded);

		trace_mm_compaction_migratepages(cc, nr_succeeded);

		trace_mm_compaction_migratepages(nr_migratepages, nr_succeeded);

		/* All pages were either migrated or will be released */

		cc->nr_migratepages = 0;

	 * so we don't leave any returned pages behind in the next attempt.

	 */

	if (cc->nr_freepages > 0) {

		unsigned long free_pfn = release_freepages(&cc->freepages);

		unsigned long free_pfn = release_free_list(cc->freepages);

		cc->nr_freepages = 0;

		VM_BUG_ON(free_pfn == 0);

	trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret);

	VM_BUG_ON(!list_empty(&cc->freepages));

	VM_BUG_ON(!list_empty(&cc->migratepages));

	return ret;

	int zoneid;

	struct zone *zone;

	enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;

	enum compact_result ret;

	for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {

		zone = &pgdat->node_zones[zoneid];

		if (!populated_zone(zone))

			continue;