Unverified Commit 99b7d010 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!5663 arm64: transparent contiguous PTEs for user mappings 

Merge Pull Request from: @ci-robot 
 
PR sync from: Kefeng Wang <wangkefeng.wang@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/SFL63ZGEY4RMBNBLZ4OYROJWKCJEHL7F/ 
This is to support Transparent Contiguous PTEs for User Mappings on arm64.

Barry Song (1):
  mm: make folio_pte_batch available outside of mm/memory.c

Kefeng Wang (5):
  mm: memory: rename page_copy_prealloc() to folio_prealloc()
  mm: memory: use a folio in do_cow_fault()
  mm: memory: use folio_prealloc() in wp_page_copy()
  mm: memory: move mem_cgroup_charge() into alloc_anon_folio()
  arm64: configs: enable ARM64_CONTPTE

Ryan Roberts (19):
  mm: clarify the spec for set_ptes()
  mm: thp: batch-collapse PMD with set_ptes()
  mm: introduce pte_advance_pfn() and use for pte_next_pfn()
  arm64/mm: convert pte_next_pfn() to pte_advance_pfn()
  x86/mm: convert pte_next_pfn() to pte_advance_pfn()
  mm: tidy up pte_next_pfn() definition
  arm64/mm: convert READ_ONCE(*ptep) to ptep_get(ptep)
  arm64/mm: convert set_pte_at() to set_ptes(..., 1)
  arm64/mm: convert ptep_clear() to ptep_get_and_clear()
  arm64/mm: new ptep layer to manage contig bit
  arm64/mm: dplit __flush_tlb_range() to elide trailing DSB
  arm64/mm: wire up PTE_CONT for user mappings
  arm64/mm: implement new wrprotect_ptes() batch API
  arm64/mm: implement new [get_and_]clear_full_ptes() batch APIs
  mm: add pte_batch_hint() to reduce scanning in folio_pte_batch()
  arm64/mm: implement pte_batch_hint()
  arm64/mm: __always_inline to improve fork() perf
  arm64/mm: automatically fold contpte mappings
  tools/mm: add thpmaps script to dump THP usage info


-- 
2.27.0
 
https://gitee.com/openeuler/kernel/issues/I9CUEQ 
 
Link:https://gitee.com/openeuler/kernel/pulls/5663

 

Reviewed-by: default avatarZhang Jianhua <chris.zjh@huawei.com>
Signed-off-by: default avatarZheng Zengkai <zhengzengkai@huawei.com>
Acked-by: default avatarXie XiuQi <xiexiuqi@huawei.com>
parents 9b83b962 88b57612

arch/arm64/Kconfig

+9 −0
Original line number Diff line number Diff line
@@ -2441,6 +2441,15 @@ config UNWIND_PATCH_PAC_INTO_SCS 
	select UNWIND_TABLES

	select DYNAMIC_SCS



config ARM64_CONTPTE

	bool "Contiguous PTE mappings for user memory" if EXPERT

	depends on TRANSPARENT_HUGEPAGE

	default y

	help

	  When enabled, user mappings are configured using the PTE contiguous

	  bit, for any mappings that meet the size and alignment requirements.

	  This reduces TLB pressure and improves performance.



config CLEAR_USER_WORKAROUND

	bool "Enable clear user workaround"

	depends on ARCH_HISI

arch/arm64/configs/openeuler_defconfig

+1 −0
Original line number Diff line number Diff line
@@ -569,6 +569,7 @@ CONFIG_KASLR_SKIP_MEM_RANGE=y 
CONFIG_RANDOMIZE_MODULE_REGION_FULL=y

CONFIG_CC_HAVE_STACKPROTECTOR_SYSREG=y

CONFIG_STACKPROTECTOR_PER_TASK=y

CONFIG_ARM64_CONTPTE=y

CONFIG_CLEAR_USER_WORKAROUND=y

# end of Kernel Features

arch/arm64/include/asm/pgtable.h

+367 −44
Original line number Diff line number Diff line
@@ -93,7 +93,8 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) 
	__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))



#define pte_none(pte)		(!pte_val(pte))

#define pte_clear(mm,addr,ptep)	set_pte(ptep, __pte(0))

#define __pte_clear(mm, addr, ptep) \

				__set_pte(ptep, __pte(0))

#define pte_page(pte)		(pfn_to_page(pte_pfn(pte)))



/*
@@ -132,12 +133,16 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) 
 */

#define pte_valid_not_user(pte) \

	((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))

/*

 * Returns true if the pte is valid and has the contiguous bit set.

 */

#define pte_valid_cont(pte)	(pte_valid(pte) && pte_cont(pte))

/*

 * Could the pte be present in the TLB? We must check mm_tlb_flush_pending

 * so that we don't erroneously return false for pages that have been

 * remapped as PROT_NONE but are yet to be flushed from the TLB.

 * Note that we can't make any assumptions based on the state of the access

 * flag, since ptep_clear_flush_young() elides a DSB when invalidating the

 * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the

 * TLB.

 */

#define pte_accessible(mm, pte)	\
@@ -261,7 +266,7 @@ static inline pte_t pte_mkdevmap(pte_t pte) 
	return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));

}



static inline void set_pte(pte_t *ptep, pte_t pte)

static inline void __set_pte(pte_t *ptep, pte_t pte)

{

	WRITE_ONCE(*ptep, pte);
@@ -275,6 +280,11 @@ static inline void set_pte(pte_t *ptep, pte_t pte) 
	}

}



static inline pte_t __ptep_get(pte_t *ptep)

{

	return READ_ONCE(*ptep);

}



extern void __sync_icache_dcache(pte_t pteval);

bool pgattr_change_is_safe(u64 old, u64 new);
@@ -302,7 +312,7 @@ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, 
	if (!IS_ENABLED(CONFIG_DEBUG_VM))

		return;



	old_pte = READ_ONCE(*ptep);

	old_pte = __ptep_get(ptep);



	if (!pte_valid(old_pte) || !pte_valid(pte))

		return;
@@ -311,7 +321,7 @@ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, 


	/*

	 * Check for potential race with hardware updates of the pte

	 * (ptep_set_access_flags safely changes valid ptes without going

	 * (__ptep_set_access_flags safely changes valid ptes without going

	 * through an invalid entry).

	 */

	VM_WARN_ONCE(!pte_young(pte),
@@ -351,13 +361,13 @@ static inline pgprot_t pte_pgprot(pte_t pte) 
	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));

}



#define pte_next_pfn pte_next_pfn

static inline pte_t pte_next_pfn(pte_t pte)

#define pte_advance_pfn pte_advance_pfn

static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)

{

	return pfn_pte(pte_pfn(pte) + 1, pte_pgprot(pte));

	return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));

}



static inline void set_ptes(struct mm_struct *mm,

static inline void __set_ptes(struct mm_struct *mm,

			      unsigned long __always_unused addr,

			      pte_t *ptep, pte_t pte, unsigned int nr)

{
@@ -366,14 +376,13 @@ static inline void set_ptes(struct mm_struct *mm, 


	for (;;) {

		__check_safe_pte_update(mm, ptep, pte);

		set_pte(ptep, pte);

		__set_pte(ptep, pte);

		if (--nr == 0)

			break;

		ptep++;

		pte = pte_next_pfn(pte);

		pte = pte_advance_pfn(pte, 1);

	}

}

#define set_ptes set_ptes



/*

 * Huge pte definitions.
@@ -540,7 +549,7 @@ static inline void __set_pte_at(struct mm_struct *mm, 
{

	__sync_cache_and_tags(pte, nr);

	__check_safe_pte_update(mm, ptep, pte);

	set_pte(ptep, pte);

	__set_pte(ptep, pte);

}



static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
@@ -854,8 +863,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) 
	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));

}



#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS

extern int ptep_set_access_flags(struct vm_area_struct *vma,

extern int __ptep_set_access_flags(struct vm_area_struct *vma,

				 unsigned long address, pte_t *ptep,

				 pte_t entry, int dirty);
@@ -865,7 +873,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 
					unsigned long address, pmd_t *pmdp,

					pmd_t entry, int dirty)

{

	return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);

	return __ptep_set_access_flags(vma, address, (pte_t *)pmdp,

							pmd_pte(entry), dirty);

}



static inline int pud_devmap(pud_t pud)
@@ -899,12 +908,13 @@ static inline bool pud_user_accessible_page(pud_t pud) 
/*

 * Atomic pte/pmd modifications.

 */

#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG

static inline int __ptep_test_and_clear_young(pte_t *ptep)

static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma,

					      unsigned long address,

					      pte_t *ptep)

{

	pte_t old_pte, pte;



	pte = READ_ONCE(*ptep);

	pte = __ptep_get(ptep);

	do {

		old_pte = pte;

		pte = pte_mkold(pte);
@@ -915,18 +925,10 @@ static inline int __ptep_test_and_clear_young(pte_t *ptep) 
	return pte_young(pte);

}



static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,

					    unsigned long address,

					    pte_t *ptep)

{

	return __ptep_test_and_clear_young(ptep);

}



#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH

static inline int ptep_clear_flush_young(struct vm_area_struct *vma,

static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,

					 unsigned long address, pte_t *ptep)

{

	int young = ptep_test_and_clear_young(vma, address, ptep);

	int young = __ptep_test_and_clear_young(vma, address, ptep);



	if (young) {

		/*
@@ -949,12 +951,11 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 
					    unsigned long address,

					    pmd_t *pmdp)

{

	return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);

	return __ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



#define __HAVE_ARCH_PTEP_GET_AND_CLEAR

static inline pte_t ptep_get_and_clear(struct mm_struct *mm,

static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,

				       unsigned long address, pte_t *ptep)

{

	pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
@@ -964,6 +965,37 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 
	return pte;

}



static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, unsigned int nr, int full)

{

	for (;;) {

		__ptep_get_and_clear(mm, addr, ptep);

		if (--nr == 0)

			break;

		ptep++;

		addr += PAGE_SIZE;

	}

}



static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep,

				unsigned int nr, int full)

{

	pte_t pte, tmp_pte;



	pte = __ptep_get_and_clear(mm, addr, ptep);

	while (--nr) {

		ptep++;

		addr += PAGE_SIZE;

		tmp_pte = __ptep_get_and_clear(mm, addr, ptep);

		if (pte_dirty(tmp_pte))

			pte = pte_mkdirty(pte);

		if (pte_young(tmp_pte))

			pte = pte_mkyoung(pte);

	}

	return pte;

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR

static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
@@ -977,16 +1009,12 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 
}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



/*

 * ptep_set_wrprotect - mark read-only while trasferring potential hardware

 * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.

 */

#define __HAVE_ARCH_PTEP_SET_WRPROTECT

static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)

static inline void ___ptep_set_wrprotect(struct mm_struct *mm,

					unsigned long address, pte_t *ptep,

					pte_t pte)

{

	pte_t old_pte, pte;

	pte_t old_pte;



	pte = READ_ONCE(*ptep);

	do {

		old_pte = pte;

		pte = pte_wrprotect(pte);
@@ -995,12 +1023,31 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres 
	} while (pte_val(pte) != pte_val(old_pte));

}



/*

 * __ptep_set_wrprotect - mark read-only while trasferring potential hardware

 * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.

 */

static inline void __ptep_set_wrprotect(struct mm_struct *mm,

					unsigned long address, pte_t *ptep)

{

	___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep));

}



static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address,

				pte_t *ptep, unsigned int nr)

{

	unsigned int i;



	for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++)

		__ptep_set_wrprotect(mm, address, ptep);

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define __HAVE_ARCH_PMDP_SET_WRPROTECT

static inline void pmdp_set_wrprotect(struct mm_struct *mm,

				      unsigned long address, pmd_t *pmdp)

{

	ptep_set_wrprotect(mm, address, (pte_t *)pmdp);

	__ptep_set_wrprotect(mm, address, (pte_t *)pmdp);

}



#define pmdp_establish pmdp_establish
@@ -1078,7 +1125,7 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) 
#endif /* CONFIG_ARM64_MTE */



/*

 * On AArch64, the cache coherency is handled via the set_pte_at() function.

 * On AArch64, the cache coherency is handled via the __set_ptes() function.

 */

static inline void update_mmu_cache_range(struct vm_fault *vmf,

		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
@@ -1130,6 +1177,282 @@ extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma, 
extern void ptep_modify_prot_commit(struct vm_area_struct *vma,

				    unsigned long addr, pte_t *ptep,

				    pte_t old_pte, pte_t new_pte);



#ifdef CONFIG_ARM64_CONTPTE



/*

 * The contpte APIs are used to transparently manage the contiguous bit in ptes

 * where it is possible and makes sense to do so. The PTE_CONT bit is considered

 * a private implementation detail of the public ptep API (see below).

 */

extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, pte_t pte);

extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, pte_t pte);

extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte);

extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep);

extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, pte_t pte, unsigned int nr);

extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, unsigned int nr, int full);

extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep,

				unsigned int nr, int full);

extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma,

				unsigned long addr, pte_t *ptep);

extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma,

				unsigned long addr, pte_t *ptep);

extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, unsigned int nr);

extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma,

				unsigned long addr, pte_t *ptep,

				pte_t entry, int dirty);



static __always_inline void contpte_try_fold(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep, pte_t pte)

{

	/*

	 * Only bother trying if both the virtual and physical addresses are

	 * aligned and correspond to the last entry in a contig range. The core

	 * code mostly modifies ranges from low to high, so this is the likely

	 * the last modification in the contig range, so a good time to fold.

	 * We can't fold special mappings, because there is no associated folio.

	 */



	const unsigned long contmask = CONT_PTES - 1;

	bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask;



	if (unlikely(valign)) {

		bool palign = (pte_pfn(pte) & contmask) == contmask;



		if (unlikely(palign &&

		    pte_valid(pte) && !pte_cont(pte) && !pte_special(pte)))

			__contpte_try_fold(mm, addr, ptep, pte);

	}

}



static __always_inline void contpte_try_unfold(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep, pte_t pte)

{

	if (unlikely(pte_valid_cont(pte)))

		__contpte_try_unfold(mm, addr, ptep, pte);

}



#define pte_batch_hint pte_batch_hint

static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)

{

	if (!pte_valid_cont(pte))

		return 1;



	return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1));

}



/*

 * The below functions constitute the public API that arm64 presents to the

 * core-mm to manipulate PTE entries within their page tables (or at least this

 * is the subset of the API that arm64 needs to implement). These public

 * versions will automatically and transparently apply the contiguous bit where

 * it makes sense to do so. Therefore any users that are contig-aware (e.g.

 * hugetlb, kernel mapper) should NOT use these APIs, but instead use the

 * private versions, which are prefixed with double underscore. All of these

 * APIs except for ptep_get_lockless() are expected to be called with the PTL

 * held. Although the contiguous bit is considered private to the

 * implementation, it is deliberately allowed to leak through the getters (e.g.

 * ptep_get()), back to core code. This is required so that pte_leaf_size() can

 * provide an accurate size for perf_get_pgtable_size(). But this leakage means

 * its possible a pte will be passed to a setter with the contiguous bit set, so

 * we explicitly clear the contiguous bit in those cases to prevent accidentally

 * setting it in the pgtable.

 */



#define ptep_get ptep_get

static inline pte_t ptep_get(pte_t *ptep)

{

	pte_t pte = __ptep_get(ptep);



	if (likely(!pte_valid_cont(pte)))

		return pte;



	return contpte_ptep_get(ptep, pte);

}



#define ptep_get_lockless ptep_get_lockless

static inline pte_t ptep_get_lockless(pte_t *ptep)

{

	pte_t pte = __ptep_get(ptep);



	if (likely(!pte_valid_cont(pte)))

		return pte;



	return contpte_ptep_get_lockless(ptep);

}



static inline void set_pte(pte_t *ptep, pte_t pte)

{

	/*

	 * We don't have the mm or vaddr so cannot unfold contig entries (since

	 * it requires tlb maintenance). set_pte() is not used in core code, so

	 * this should never even be called. Regardless do our best to service

	 * any call and emit a warning if there is any attempt to set a pte on

	 * top of an existing contig range.

	 */

	pte_t orig_pte = __ptep_get(ptep);



	WARN_ON_ONCE(pte_valid_cont(orig_pte));

	__set_pte(ptep, pte_mknoncont(pte));

}



#define set_ptes set_ptes

static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, pte_t pte, unsigned int nr)

{

	pte = pte_mknoncont(pte);



	if (likely(nr == 1)) {

		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));

		__set_ptes(mm, addr, ptep, pte, 1);

		contpte_try_fold(mm, addr, ptep, pte);

	} else {

		contpte_set_ptes(mm, addr, ptep, pte, nr);

	}

}



static inline void pte_clear(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep)

{

	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));

	__pte_clear(mm, addr, ptep);

}



#define clear_full_ptes clear_full_ptes

static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, unsigned int nr, int full)

{

	if (likely(nr == 1)) {

		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));

		__clear_full_ptes(mm, addr, ptep, nr, full);

	} else {

		contpte_clear_full_ptes(mm, addr, ptep, nr, full);

	}

}



#define get_and_clear_full_ptes get_and_clear_full_ptes

static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep,

				unsigned int nr, int full)

{

	pte_t pte;



	if (likely(nr == 1)) {

		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));

		pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);

	} else {

		pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);

	}



	return pte;

}



#define __HAVE_ARCH_PTEP_GET_AND_CLEAR

static inline pte_t ptep_get_and_clear(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep)

{

	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));

	return __ptep_get_and_clear(mm, addr, ptep);

}



#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG

static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,

				unsigned long addr, pte_t *ptep)

{

	pte_t orig_pte = __ptep_get(ptep);



	if (likely(!pte_valid_cont(orig_pte)))

		return __ptep_test_and_clear_young(vma, addr, ptep);



	return contpte_ptep_test_and_clear_young(vma, addr, ptep);

}



#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH

static inline int ptep_clear_flush_young(struct vm_area_struct *vma,

				unsigned long addr, pte_t *ptep)

{

	pte_t orig_pte = __ptep_get(ptep);



	if (likely(!pte_valid_cont(orig_pte)))

		return __ptep_clear_flush_young(vma, addr, ptep);



	return contpte_ptep_clear_flush_young(vma, addr, ptep);

}



#define wrprotect_ptes wrprotect_ptes

static __always_inline void wrprotect_ptes(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep, unsigned int nr)

{

	if (likely(nr == 1)) {

		/*

		 * Optimization: wrprotect_ptes() can only be called for present

		 * ptes so we only need to check contig bit as condition for

		 * unfold, and we can remove the contig bit from the pte we read

		 * to avoid re-reading. This speeds up fork() which is sensitive

		 * for order-0 folios. Equivalent to contpte_try_unfold().

		 */

		pte_t orig_pte = __ptep_get(ptep);



		if (unlikely(pte_cont(orig_pte))) {

			__contpte_try_unfold(mm, addr, ptep, orig_pte);

			orig_pte = pte_mknoncont(orig_pte);

		}

		___ptep_set_wrprotect(mm, addr, ptep, orig_pte);

	} else {

		contpte_wrprotect_ptes(mm, addr, ptep, nr);

	}

}



#define __HAVE_ARCH_PTEP_SET_WRPROTECT

static inline void ptep_set_wrprotect(struct mm_struct *mm,

				unsigned long addr, pte_t *ptep)

{

	wrprotect_ptes(mm, addr, ptep, 1);

}



#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS

static inline int ptep_set_access_flags(struct vm_area_struct *vma,

				unsigned long addr, pte_t *ptep,

				pte_t entry, int dirty)

{

	pte_t orig_pte = __ptep_get(ptep);



	entry = pte_mknoncont(entry);



	if (likely(!pte_valid_cont(orig_pte)))

		return __ptep_set_access_flags(vma, addr, ptep, entry, dirty);



	return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty);

}



#else /* CONFIG_ARM64_CONTPTE */



#define ptep_get				__ptep_get

#define set_pte					__set_pte

#define set_ptes				__set_ptes

#define pte_clear				__pte_clear

#define clear_full_ptes				__clear_full_ptes

#define get_and_clear_full_ptes			__get_and_clear_full_ptes

#define __HAVE_ARCH_PTEP_GET_AND_CLEAR

#define ptep_get_and_clear			__ptep_get_and_clear

#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG

#define ptep_test_and_clear_young		__ptep_test_and_clear_young

#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH

#define ptep_clear_flush_young			__ptep_clear_flush_young

#define __HAVE_ARCH_PTEP_SET_WRPROTECT

#define ptep_set_wrprotect			__ptep_set_wrprotect

#define wrprotect_ptes				__wrprotect_ptes

#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS

#define ptep_set_access_flags			__ptep_set_access_flags



#endif /* CONFIG_ARM64_CONTPTE */



#endif /* !__ASSEMBLY__ */



#endif /* __ASM_PGTABLE_H */

arch/arm64/include/asm/tlbflush.h

+11 −2
Original line number Diff line number Diff line
@@ -400,7 +400,7 @@ do { \ 
#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \

	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false)



static inline void __flush_tlb_range(struct vm_area_struct *vma,

static inline void __flush_tlb_range_nosync(struct vm_area_struct *vma,

				     unsigned long start, unsigned long end,

				     unsigned long stride, bool last_level,

				     int tlb_level)
@@ -432,10 +432,19 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma, 
	else

		__flush_tlb_range_op(vae1is, start, pages, stride, asid, tlb_level, true);



	dsb(ish);

	mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, start, end);

}



static inline void __flush_tlb_range(struct vm_area_struct *vma,

				     unsigned long start, unsigned long end,

				     unsigned long stride, bool last_level,

				     int tlb_level)

{

	__flush_tlb_range_nosync(vma, start, end, stride,

				 last_level, tlb_level);

	dsb(ish);

}



static inline void flush_tlb_range(struct vm_area_struct *vma,

				   unsigned long start, unsigned long end)

{

arch/arm64/kernel/efi.c

+2 −2
Original line number Diff line number Diff line
@@ -107,7 +107,7 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data) 
{

	struct set_perm_data *spd = data;

	const efi_memory_desc_t *md = spd->md;

	pte_t pte = READ_ONCE(*ptep);

	pte_t pte = __ptep_get(ptep);



	if (md->attribute & EFI_MEMORY_RO)

		pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
@@ -116,7 +116,7 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data) 
	else if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) &&

		 system_supports_bti() && spd->has_bti)

		pte = set_pte_bit(pte, __pgprot(PTE_GP));

	set_pte(ptep, pte);

	__set_pte(ptep, pte);

	return 0;

}