Unverified Commit 0293ebbc authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!6626 Backport page fault and fork optimization 

Merge Pull Request from: @ci-robot 
 
PR sync from: Peng Zhang <zhangpeng362@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/KS5TC4GQCBOUZJ3BKBIOBHCFTBP55W5R/ 
From: ZhangPeng <zhangpeng362@huawei.com>

Backport page fault and fork optimization, including optimization for
folio add and split, remove uffd_wp and anon/cow/shared fault
optimization.

We can get a 3% performance improvment for lmbench fork_proc and 1.5%
for lmbench page_fault.

Kairui Song (5):
  mm/filemap: return early if failed to allocate memory for split
  mm/filemap: clean up hugetlb exclusion code
  lib/xarray: introduce a new helper xas_get_order
  lib/xarray: introduce a new helper xas_get_order
  mm/filemap: optimize filemap folio adding

Kefeng Wang (2):
  mm: memory: check userfaultfd_wp() in vmf_orig_pte_uffd_wp()
  mm: swapfile: check usable swap device in __folio_throttle_swaprate()


-- 
2.25.1
 
https://gitee.com/openeuler/kernel/issues/I9JAY9 
 
Link:https://gitee.com/openeuler/kernel/pulls/6626

 

Reviewed-by: default avatarXu Kuohai <xukuohai@huawei.com>
Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: default avatarWeilong Chen <chenweilong@huawei.com>
Signed-off-by: default avatarXie XiuQi <xiexiuqi@huawei.com>
parents c48dd34e e8a2c76e

include/linux/xarray.h

+6 −0
Original line number Diff line number Diff line
@@ -1551,6 +1551,7 @@ void xas_create_range(struct xa_state *); 


#ifdef CONFIG_XARRAY_MULTI

int xa_get_order(struct xarray *, unsigned long index);

int xas_get_order(struct xa_state *xas);

void xas_split(struct xa_state *, void *entry, unsigned int order);

void xas_split_alloc(struct xa_state *, void *entry, unsigned int order, gfp_t);

#else
@@ -1559,6 +1560,11 @@ static inline int xa_get_order(struct xarray *xa, unsigned long index) 
	return 0;

}



static inline int xas_get_order(struct xa_state *xas)

{

	return 0;

}



static inline void xas_split(struct xa_state *xas, void *entry,

		unsigned int order)

{

lib/test_xarray.c

+93 −0
Original line number Diff line number Diff line
@@ -1756,6 +1756,97 @@ static noinline void check_get_order(struct xarray *xa) 
	}

}



static noinline void check_xas_get_order(struct xarray *xa)

{

	XA_STATE(xas, xa, 0);



	unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1;

	unsigned int order;

	unsigned long i, j;



	for (order = 0; order < max_order; order++) {

		for (i = 0; i < 10; i++) {

			xas_set_order(&xas, i << order, order);

			do {

				xas_lock(&xas);

				xas_store(&xas, xa_mk_value(i));

				xas_unlock(&xas);

			} while (xas_nomem(&xas, GFP_KERNEL));



			for (j = i << order; j < (i + 1) << order; j++) {

				xas_set_order(&xas, j, 0);

				rcu_read_lock();

				xas_load(&xas);

				XA_BUG_ON(xa, xas_get_order(&xas) != order);

				rcu_read_unlock();

			}



			xas_lock(&xas);

			xas_set_order(&xas, i << order, order);

			xas_store(&xas, NULL);

			xas_unlock(&xas);

		}

	}

}



static noinline void check_xas_conflict_get_order(struct xarray *xa)

{

	XA_STATE(xas, xa, 0);



	void *entry;

	int only_once;

	unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1;

	unsigned int order;

	unsigned long i, j, k;



	for (order = 0; order < max_order; order++) {

		for (i = 0; i < 10; i++) {

			xas_set_order(&xas, i << order, order);

			do {

				xas_lock(&xas);

				xas_store(&xas, xa_mk_value(i));

				xas_unlock(&xas);

			} while (xas_nomem(&xas, GFP_KERNEL));



			/*

			 * Ensure xas_get_order works with xas_for_each_conflict.

			 */

			j = i << order;

			for (k = 0; k < order; k++) {

				only_once = 0;

				xas_set_order(&xas, j + (1 << k), k);

				xas_lock(&xas);

				xas_for_each_conflict(&xas, entry) {

					XA_BUG_ON(xa, entry != xa_mk_value(i));

					XA_BUG_ON(xa, xas_get_order(&xas) != order);

					only_once++;

				}

				XA_BUG_ON(xa, only_once != 1);

				xas_unlock(&xas);

			}



			if (order < max_order - 1) {

				only_once = 0;

				xas_set_order(&xas, (i & ~1UL) << order, order + 1);

				xas_lock(&xas);

				xas_for_each_conflict(&xas, entry) {

					XA_BUG_ON(xa, entry != xa_mk_value(i));

					XA_BUG_ON(xa, xas_get_order(&xas) != order);

					only_once++;

				}

				XA_BUG_ON(xa, only_once != 1);

				xas_unlock(&xas);

			}



			xas_set_order(&xas, i << order, order);

			xas_lock(&xas);

			xas_store(&xas, NULL);

			xas_unlock(&xas);

		}

	}

}





static noinline void check_destroy(struct xarray *xa)

{

	unsigned long index;
@@ -1805,6 +1896,8 @@ static int xarray_checks(void) 
	check_reserve(&xa0);

	check_multi_store(&array);

	check_get_order(&array);

	check_xas_get_order(&array);

	check_xas_conflict_get_order(&array);

	check_xa_alloc();

	check_find(&array);

	check_find_entry(&array);

lib/xarray.c

+31 −18
Original line number Diff line number Diff line
@@ -1751,39 +1751,52 @@ void *xa_store_range(struct xarray *xa, unsigned long first, 
EXPORT_SYMBOL(xa_store_range);



/**

 * xa_get_order() - Get the order of an entry.

 * @xa: XArray.

 * @index: Index of the entry.

 * xas_get_order() - Get the order of an entry.

 * @xas: XArray operation state.

 *

 * Called after xas_load, the xas should not be in an error state.

 *

 * Return: A number between 0 and 63 indicating the order of the entry.

 */

int xa_get_order(struct xarray *xa, unsigned long index)

int xas_get_order(struct xa_state *xas)

{

	XA_STATE(xas, xa, index);

	void *entry;

	int order = 0;



	rcu_read_lock();

	entry = xas_load(&xas);



	if (!entry)

		goto unlock;



	if (!xas.xa_node)

		goto unlock;

	if (!xas->xa_node)

		return 0;



	for (;;) {

		unsigned int slot = xas.xa_offset + (1 << order);

		unsigned int slot = xas->xa_offset + (1 << order);



		if (slot >= XA_CHUNK_SIZE)

			break;

		if (!xa_is_sibling(xas.xa_node->slots[slot]))

		if (!xa_is_sibling(xa_entry(xas->xa, xas->xa_node, slot)))

			break;

		order++;

	}



	order += xas.xa_node->shift;

unlock:

	order += xas->xa_node->shift;

	return order;

}

EXPORT_SYMBOL_GPL(xas_get_order);



/**

 * xa_get_order() - Get the order of an entry.

 * @xa: XArray.

 * @index: Index of the entry.

 *

 * Return: A number between 0 and 63 indicating the order of the entry.

 */

int xa_get_order(struct xarray *xa, unsigned long index)

{

	XA_STATE(xas, xa, index);

	int order = 0;

	void *entry;



	rcu_read_lock();

	entry = xas_load(&xas);

	if (entry)

		order = xas_get_order(&xas);

	rcu_read_unlock();



	return order;

mm/filemap.c

+49 −25
Original line number Diff line number Diff line
@@ -848,23 +848,18 @@ noinline int __filemap_add_folio(struct address_space *mapping, 
		struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp)

{

	XA_STATE(xas, &mapping->i_pages, index);

	int huge = folio_test_hugetlb(folio);

	bool charged = false;

	long nr = 1;

	void *alloced_shadow = NULL;

	int alloced_order = 0;

	bool huge;

	long nr;



	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);

	VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio);

	mapping_set_update(&xas, mapping);



	if (!huge) {

		int error = mem_cgroup_charge(folio, NULL, gfp);

		if (error)

			return error;

		charged = true;

	}



	VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);

	xas_set_order(&xas, index, folio_order(folio));

	huge = folio_test_hugetlb(folio);

	nr = folio_nr_pages(folio);



	gfp &= GFP_RECLAIM_MASK;
@@ -872,13 +867,10 @@ noinline int __filemap_add_folio(struct address_space *mapping, 
	folio->mapping = mapping;

	folio->index = xas.xa_index;



	do {

		unsigned int order = xa_get_order(xas.xa, xas.xa_index);

	for (;;) {

		int order = -1, split_order = 0;

		void *entry, *old = NULL;



		if (order > folio_order(folio))

			xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index),

					order, gfp);

		xas_lock_irq(&xas);

		xas_for_each_conflict(&xas, entry) {

			old = entry;
@@ -886,19 +878,33 @@ noinline int __filemap_add_folio(struct address_space *mapping, 
				xas_set_err(&xas, -EEXIST);

				goto unlock;

			}

			/*

			 * If a larger entry exists,

			 * it will be the first and only entry iterated.

			 */

			if (order == -1)

				order = xas_get_order(&xas);

		}



		/* entry may have changed before we re-acquire the lock */

		if (alloced_order && (old != alloced_shadow || order != alloced_order)) {

			xas_destroy(&xas);

			alloced_order = 0;

		}



		if (old) {

			if (shadowp)

				*shadowp = old;

			/* entry may have been split before we acquired lock */

			order = xa_get_order(xas.xa, xas.xa_index);

			if (order > folio_order(folio)) {

			if (order > 0 && order > folio_order(folio)) {

				/* How to handle large swap entries? */

				BUG_ON(shmem_mapping(mapping));

				if (!alloced_order) {

					split_order = order;

					goto unlock;

				}

				xas_split(&xas, old, order);

				xas_reset(&xas);

			}

			if (shadowp)

				*shadowp = old;

		}



		xas_store(&xas, folio);
@@ -914,9 +920,24 @@ noinline int __filemap_add_folio(struct address_space *mapping, 
				__lruvec_stat_mod_folio(folio,

						NR_FILE_THPS, nr);

		}



unlock:

		xas_unlock_irq(&xas);

	} while (xas_nomem(&xas, gfp));



		/* split needed, alloc here and retry. */

		if (split_order) {

			xas_split_alloc(&xas, old, split_order, gfp);

			if (xas_error(&xas))

				goto error;

			alloced_shadow = old;

			alloced_order = split_order;

			xas_reset(&xas);

			continue;

		}



		if (!xas_nomem(&xas, gfp))

			break;

	}



	if (xas_error(&xas))

		goto error;
@@ -924,8 +945,6 @@ noinline int __filemap_add_folio(struct address_space *mapping, 
	trace_mm_filemap_add_to_page_cache(folio);

	return 0;

error:

	if (charged)

		mem_cgroup_uncharge(folio);

	folio->mapping = NULL;

	/* Leave page->index set: truncation relies upon it */

	folio_put_refs(folio, nr);
@@ -939,11 +958,16 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio, 
	void *shadow = NULL;

	int ret;



	ret = mem_cgroup_charge(folio, NULL, gfp);

	if (ret)

		return ret;



	__folio_set_locked(folio);

	ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow);

	if (unlikely(ret))

	if (unlikely(ret)) {

		mem_cgroup_uncharge(folio);

		__folio_clear_locked(folio);

	else {

	} else {

		/*

		 * The folio might have been evicted from cache only

		 * recently, in which case it should be activated like

mm/memory.c

+5 −5
Original line number Diff line number Diff line
@@ -113,8 +113,10 @@ static bool vmf_pte_changed(struct vm_fault *vmf); 
 * Return true if the original pte was a uffd-wp pte marker (so the pte was

 * wr-protected).

 */

static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)

static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)

{

	if (!userfaultfd_wp(vmf->vma))

		return false;

	if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))

		return false;
@@ -4366,7 +4368,6 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) 
 */

static vm_fault_t do_anonymous_page(struct vm_fault *vmf)

{

	bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);

	struct vm_area_struct *vma = vmf->vma;

	unsigned long addr = vmf->address;

	struct folio *folio;
@@ -4464,7 +4465,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) 
	folio_add_new_anon_rmap(folio, vma, addr);

	folio_add_lru_vma(folio, vma);

setpte:

	if (uffd_wp)

	if (vmf_orig_pte_uffd_wp(vmf))

		entry = pte_mkuffd_wp(entry);

	set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages);
@@ -4640,7 +4641,6 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, 
		struct page *page, unsigned int nr, unsigned long addr)

{

	struct vm_area_struct *vma = vmf->vma;

	bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);

	bool write = vmf->flags & FAULT_FLAG_WRITE;

	bool prefault = in_range(vmf->address, addr, nr * PAGE_SIZE);

	pte_t entry;
@@ -4655,7 +4655,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, 


	if (write)

		entry = maybe_mkwrite(pte_mkdirty(entry), vma);

	if (unlikely(uffd_wp))

	if (unlikely(vmf_orig_pte_uffd_wp(vmf)))

		entry = pte_mkuffd_wp(entry);

	/* copy-on-write page */

	add_reliable_folio_counter(folio, vma->vm_mm, nr);