!15159 mm/thp: fix deferred split unqueue naming and locking (e835a628) · Commits · EulixOS / Software / Kernel

mm/huge_memory.c

+26 −9

Original line number	Diff line number	Diff line
		@@ -3620,18 +3620,38 @@ int split_huge_page_to_list_to_order(struct page page, struct list_head list,
		return ret;
		}

		void __folio_undo_large_rmappable(struct folio *folio)
		/*
		* __folio_unqueue_deferred_split() is not to be called directly:
		* the folio_unqueue_deferred_split() inline wrapper in mm/internal.h
		* limits its calls to those folios which may have a _deferred_list for
		* queueing THP splits, and that list is (racily observed to be) non-empty.
		*
		* It is unsafe to call folio_unqueue_deferred_split() until folio refcount is
		* zero: because even when split_queue_lock is held, a non-empty _deferred_list
		* might be in use on deferred_split_scan()'s unlocked on-stack list.
		*
		* If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is
		* therefore important to unqueue deferred split before changing folio memcg.
		*/
		bool __folio_unqueue_deferred_split(struct folio *folio)
		{
		struct deferred_split *ds_queue;
		unsigned long flags;
		bool unqueued = false;

		WARN_ON_ONCE(folio_ref_count(folio));
		WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio));

		ds_queue = get_deferred_split_queue(folio);
		spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
		if (!list_empty(&folio->_deferred_list)) {
		ds_queue->split_queue_len--;
		list_del_init(&folio->_deferred_list);
		unqueued = true;
		}
		spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);

		return unqueued; /* useful for debug warnings */
		}

		void deferred_split_folio(struct folio *folio)
		@@ -3650,14 +3670,11 @@ void deferred_split_folio(struct folio *folio)
		return;

		/*
		* The try_to_unmap() in page reclaim path might reach here too,
		* this may cause a race condition to corrupt deferred split queue.
		* And, if page reclaim is already handling the same folio, it is
		* unnecessary to handle it again in shrinker.
		*
		* Check the swapcache flag to determine if the folio is being
		* handled by page reclaim since THP swap would add the folio into
		* swap cache before calling try_to_unmap().
		* Exclude swapcache: originally to avoid a corrupt deferred split
		* queue. Nowadays that is fully prevented by mem_cgroup_swapout();
		* but if page reclaim is already handling the same folio, it is
		* unnecessary to handle it again in the shrinker, so excluding
		* swapcache here may still be a useful optimization.
		*/
		if (folio_test_swapcache(folio))
		return;

mm/internal.h

+5 −5

Original line number	Diff line number	Diff line
		@@ -636,11 +636,11 @@ static inline void folio_set_order(struct folio *folio, unsigned int order)
		#endif
		}

		void __folio_undo_large_rmappable(struct folio *folio);
		static inline void folio_undo_large_rmappable(struct folio *folio)
		bool __folio_unqueue_deferred_split(struct folio *folio);
		static inline bool folio_unqueue_deferred_split(struct folio *folio)
		{
		if (folio_order(folio) <= 1 \|\| !folio_test_large_rmappable(folio))
		return;
		return false;

		/*
		* At this point, there is no one trying to add the folio to
		@@ -648,9 +648,9 @@ static inline void folio_undo_large_rmappable(struct folio *folio)
		* to check without acquiring the split_queue_lock.
		*/
		if (data_race(list_empty(&folio->_deferred_list)))
		return;
		return false;

		__folio_undo_large_rmappable(folio);
		return __folio_unqueue_deferred_split(folio);
		}

		static inline struct folio page_rmappable_folio(struct page page)

mm/memcontrol.c

+32 −3

Original line number	Diff line number	Diff line
		@@ -7115,6 +7115,8 @@ static int mem_cgroup_move_account(struct page *page,
		css_get(&to->css);
		css_put(&from->css);

		/* Warning should never happen, so don't worry about refcount non-0 */
		WARN_ON_ONCE(folio_unqueue_deferred_split(folio));
		folio->memcg_data = (unsigned long)to;

		__folio_memcg_unlock(from);
		@@ -7483,7 +7485,10 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
		enum mc_target_type target_type;
		union mc_target target;
		struct page *page;
		struct folio *folio;
		bool tried_split_before = false;

		retry_pmd:
		ptl = pmd_trans_huge_lock(pmd, vma);
		if (ptl) {
		if (mc.precharge < HPAGE_PMD_NR) {
		@@ -7493,6 +7498,28 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
		page = target.page;
		folio = page_folio(page);
		/*
		* Deferred split queue locking depends on memcg,
		* and unqueue is unsafe unless folio refcount is 0:
		* split or skip if on the queue? first try to split.
		*/
		if (!list_empty(&folio->_deferred_list)) {
		spin_unlock(ptl);
		if (!tried_split_before)
		split_folio(folio);
		folio_unlock(folio);
		folio_put(folio);
		if (tried_split_before)
		return 0;
		tried_split_before = true;
		goto retry_pmd;
		}
		/*
		* So long as that pmd lock is held, the folio cannot
		* be racily added to the _deferred_list, because
		* page_remove_rmap() will find it still pmdmapped.
		*/
		if (isolate_lru_page(page)) {
		if (!mem_cgroup_move_account(page, true,
		mc.from, mc.to)) {
		@@ -8546,9 +8573,6 @@ static void uncharge_folio(struct folio folio, struct uncharge_gather ug)
		struct obj_cgroup *objcg;

		VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
		VM_BUG_ON_FOLIO(folio_order(folio) > 1 &&
		!folio_test_hugetlb(folio) &&
		!list_empty(&folio->_deferred_list), folio);

		/*
		* Nobody should be changing or seriously looking at
		@@ -8595,6 +8619,7 @@ static void uncharge_folio(struct folio folio, struct uncharge_gather ug)
		ug->nr_memory += nr_pages;
		ug->pgpgout++;

		WARN_ON_ONCE(folio_unqueue_deferred_split(folio));
		folio->memcg_data = 0;
		}

		@@ -8710,6 +8735,9 @@ void mem_cgroup_migrate(struct folio old, struct folio new)

		/* Transfer the charge and the css ref */
		commit_charge(new, memcg);

		/* Warning should never happen, so don't worry about refcount non-0 */
		WARN_ON_ONCE(folio_unqueue_deferred_split(old));
		old->memcg_data = 0;
		}

		@@ -8970,6 +8998,7 @@ void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
		VM_BUG_ON_FOLIO(oldid, folio);
		mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);

		folio_unqueue_deferred_split(folio);
		folio->memcg_data = 0;

		if (!mem_cgroup_is_root(memcg))

mm/migrate.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -413,7 +413,7 @@ static int __folio_migrate_mapping(struct address_space *mapping,
		folio_test_large_rmappable(folio)) {
		if (!folio_ref_freeze(folio, expected_count))
		return -EAGAIN;
		folio_undo_large_rmappable(folio);
		folio_unqueue_deferred_split(folio);
		folio_ref_unfreeze(folio, expected_count);
		}

		@@ -438,7 +438,7 @@ static int __folio_migrate_mapping(struct address_space *mapping,
		}

		/* Take off deferred split queue while frozen and memcg set */
		folio_undo_large_rmappable(folio);
		folio_unqueue_deferred_split(folio);

		/*
		* Now we know that no one else is looking at the folio:

mm/page_alloc.c

+0 −1

Original line number	Diff line number	Diff line
		@@ -2591,7 +2591,6 @@ void free_unref_folios(struct folio_batch *folios)
		continue;
		}

		folio_undo_large_rmappable(folio);
		if (!free_unref_page_prepare(&folio->page, pfn, order))
		continue;