Unverified Commit 398d8373 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!5656 v3 mm: backport fork/unmap/zap optimize 

Merge Pull Request from: @ci-robot 
 
PR sync from: Kefeng Wang <wangkefeng.wang@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/44237F77Y7OHOA7X4ST5JJPZVEEBBAA3/ 
This is to optimize fork/unmap/zap() with PTE-mapped THP.

Catalin Marinas (1):
  arm64: Mark the 'addr' argument to set_ptes() and __set_pte_at() as
    unused

David Hildenbrand (24):
  arm/pgtable: define PFN_PTE_SHIFT
  nios2/pgtable: define PFN_PTE_SHIFT
  powerpc/pgtable: define PFN_PTE_SHIFT
  riscv/pgtable: define PFN_PTE_SHIFT
  s390/pgtable: define PFN_PTE_SHIFT
  sparc/pgtable: define PFN_PTE_SHIFT
  mm/pgtable: make pte_next_pfn() independent of set_ptes()
  arm/mm: use pte_next_pfn() in set_ptes()
  powerpc/mm: use pte_next_pfn() in set_ptes()
  mm/memory: factor out copying the actual PTE in copy_present_pte()
  mm/memory: pass PTE to copy_present_pte()
  mm/memory: optimize fork() with PTE-mapped THP
  mm/memory: ignore dirty/accessed/soft-dirty bits in folio_pte_batch()
  mm/memory: ignore writable bit in folio_pte_batch()
  mm/memory: factor out zapping of present pte into zap_present_pte()
  mm/memory: handle !page case in zap_present_pte() separately
  mm/memory: further separate anon and pagecache folio handling in
    zap_present_pte()
  mm/memory: factor out zapping folio pte into zap_present_folio_pte()
  mm/mmu_gather: pass "delay_rmap" instead of encoded page to
    __tlb_remove_page_size()
  mm/mmu_gather: define ENCODED_PAGE_FLAG_DELAY_RMAP
  mm/mmu_gather: add tlb_remove_tlb_entries()
  mm/mmu_gather: add __tlb_remove_folio_pages()
  mm/mmu_gather: improve cond_resched() handling with large folios and
    expensive page freeing
  mm/memory: optimize unmap/zap with PTE-mapped THP

Kefeng Wang (7):
  s390: use pfn_swap_entry_folio() in ptep_zap_swap_entry()
  mm: use pfn_swap_entry_folio() in __split_huge_pmd_locked()
  mm: use pfn_swap_entry_to_folio() in zap_huge_pmd()
  mm: use pfn_swap_entry_folio() in copy_nonpresent_pte()
  mm: convert to should_zap_page() to should_zap_folio()
  mm: convert mm_counter() to take a folio
  mm: convert mm_counter_file() to take a folio

Matthew Wilcox (Oracle) (2):
  mm: add pfn_swap_entry_folio()
  mprotect: use pfn_swap_entry_folio

Peter Xu (1):
  mm/memory: fix missing pte marker for !page on pte zaps

Ryan Roberts (2):
  arm64/mm: Hoist synchronization out of set_ptes() loop
  arm64/mm: make set_ptes() robust when OAs cross 48-bit boundary


-- 
2.27.0
 
https://gitee.com/openeuler/kernel/issues/I9CHB4 
 
Link:https://gitee.com/openeuler/kernel/pulls/5656

 

Reviewed-by: default avatarXu Kuohai <xukuohai@huawei.com>
Signed-off-by: default avatarZheng Zengkai <zhengzengkai@huawei.com>
Acked-by: default avatarXie XiuQi <xiexiuqi@huawei.com>
parents 37c14791 c3b31ea7

arch/arm/include/asm/pgtable.h

+2 −0
Original line number Diff line number Diff line
@@ -208,6 +208,8 @@ static inline void __sync_icache_dcache(pte_t pteval) 
extern void __sync_icache_dcache(pte_t pteval);

#endif



#define PFN_PTE_SHIFT		PAGE_SHIFT



void set_ptes(struct mm_struct *mm, unsigned long addr,

		      pte_t *ptep, pte_t pteval, unsigned int nr);

#define set_ptes set_ptes

arch/arm/mm/mmu.c

+1 −1
Original line number Diff line number Diff line
@@ -1814,6 +1814,6 @@ void set_ptes(struct mm_struct *mm, unsigned long addr, 
		if (--nr == 0)

			break;

		ptep++;

		pte_val(pteval) += PAGE_SIZE;

		pteval = pte_next_pfn(pteval);

	}

}

arch/arm64/include/asm/mte.h

+2 −2
Original line number Diff line number Diff line
@@ -90,7 +90,7 @@ static inline bool try_page_mte_tagging(struct page *page) 
}



void mte_zero_clear_page_tags(void *addr);

void mte_sync_tags(pte_t pte);

void mte_sync_tags(pte_t pte, unsigned int nr_pages);

void mte_copy_page_tags(void *kto, const void *kfrom);

int mte_copy_mc_page_tags(void *kto, const void *kfrom);

void mte_thread_init_user(void);
@@ -123,7 +123,7 @@ static inline bool try_page_mte_tagging(struct page *page) 
static inline void mte_zero_clear_page_tags(void *addr)

{

}

static inline void mte_sync_tags(pte_t pte)

static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages)

{

}

static inline void mte_copy_page_tags(void *kto, const void *kfrom)

arch/arm64/include/asm/pgtable.h

+36 −22
Original line number Diff line number Diff line
@@ -325,8 +325,7 @@ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, 
		     __func__, pte_val(old_pte), pte_val(pte));

}



static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, pte_t pte)

static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages)

{

	if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))

		__sync_icache_dcache(pte);
@@ -339,25 +338,39 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, 
	 */

	if (system_supports_mte() && pte_access_permitted(pte, false) &&

	    !pte_special(pte) && pte_tagged(pte))

		mte_sync_tags(pte);

		mte_sync_tags(pte, nr_pages);

}



	__check_safe_pte_update(mm, ptep, pte);

/*

 * Select all bits except the pfn

 */

static inline pgprot_t pte_pgprot(pte_t pte)

{

	unsigned long pfn = pte_pfn(pte);



	set_pte(ptep, pte);

	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));

}



#define pte_next_pfn pte_next_pfn

static inline pte_t pte_next_pfn(pte_t pte)

{

	return pfn_pte(pte_pfn(pte) + 1, pte_pgprot(pte));

}



static inline void set_ptes(struct mm_struct *mm, unsigned long addr,

static inline void set_ptes(struct mm_struct *mm,

			    unsigned long __always_unused addr,

			    pte_t *ptep, pte_t pte, unsigned int nr)

{

	page_table_check_ptes_set(mm, ptep, pte, nr);

	__sync_cache_and_tags(pte, nr);



	for (;;) {

		__set_pte_at(mm, addr, ptep, pte);

		__check_safe_pte_update(mm, ptep, pte);

		set_pte(ptep, pte);

		if (--nr == 0)

			break;

		ptep++;

		addr += PAGE_SIZE;

		pte_val(pte) += PAGE_SIZE;

		pte = pte_next_pfn(pte);

	}

}

#define set_ptes set_ptes
@@ -436,16 +449,6 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) 
	return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));

}



/*

 * Select all bits except the pfn

 */

static inline pgprot_t pte_pgprot(pte_t pte)

{

	unsigned long pfn = pte_pfn(pte);



	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));

}



#ifdef CONFIG_NUMA_BALANCING

/*

 * See the comment in include/linux/pgtable.h
@@ -531,18 +534,29 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd) 
#define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)

#define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))



static inline void __set_pte_at(struct mm_struct *mm,

				unsigned long __always_unused addr,

				pte_t *ptep, pte_t pte, unsigned int nr)

{

	__sync_cache_and_tags(pte, nr);

	__check_safe_pte_update(mm, ptep, pte);

	set_pte(ptep, pte);

}



static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,

			      pmd_t *pmdp, pmd_t pmd)

{

	page_table_check_pmd_set(mm, pmdp, pmd);

	return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd));

	return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd),

						PMD_SIZE >> PAGE_SHIFT);

}



static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,

			      pud_t *pudp, pud_t pud)

{

	page_table_check_pud_set(mm, pudp, pud);

	return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud));

	return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud),

						PUD_SIZE >> PAGE_SHIFT);

}



#define __p4d_to_phys(p4d)	__pte_to_phys(p4d_pte(p4d))

arch/arm64/kernel/mte.c

+2 −2
Original line number Diff line number Diff line
@@ -35,10 +35,10 @@ DEFINE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); 
EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);

#endif



void mte_sync_tags(pte_t pte)

void mte_sync_tags(pte_t pte, unsigned int nr_pages)

{

	struct page *page = pte_page(pte);

	long i, nr_pages = compound_nr(page);

	unsigned int i;



	/* if PG_mte_tagged is set, tags have already been initialised */

	for (i = 0; i < nr_pages; i++, page++) {