Commit 9e7e3687 authored by Tong Tiangen's avatar Tong Tiangen
Browse files

arm64: support copy_mc_[user]_highpage() 

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8M74H


CVE: NA

-------------------------------

Currently, many scenarios that can tolerate memory errors when copying page
have been supported in the kernel[1][2][3], all of which are implemented by
copy_mc_[user]_highpage(). arm64 should also support this mechanism.

Due to mte, arm64 needs to have its own copy_mc_[user]_highpage()
architecture implementation, macros __HAVE_ARCH_COPY_MC_HIGHPAGE and
__HAVE_ARCH_COPY_MC_USER_HIGHPAGE have been added to control it.

Add new helper copy_mc_page() which provide a page copy implementation with
machine check safe. The copy_mc_page() in copy_mc_page.S is largely borrows
from copy_page() in copy_page.S and the main difference is copy_mc_page()
add extable entry to every load/store insn to support machine check safe.

Add new extable type EX_TYPE_COPY_MC_PAGE_ERR_ZERO which used in
copy_mc_page().

[1]a873dfe1 ("mm, hwpoison: try to recover from copy-on write faults")
[2]5f2500b93cc9 ("mm/khugepaged: recover from poisoned anonymous memory")
[3]6b970599 ("mm: hwpoison: support recovery from ksm_might_need_to_copy()")

Signed-off-by: default avatarTong Tiangen <tongtiangen@huawei.com>
parent 681be068

arch/arm64/include/asm/asm-extable.h

+15 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#define EX_TYPE_UACCESS_ERR_ZERO	2

#define EX_TYPE_KACCESS_ERR_ZERO	3

#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD	4

#define EX_TYPE_COPY_MC_PAGE_ERR_ZERO	5



/* Data fields for EX_TYPE_UACCESS_ERR_ZERO */

#define EX_DATA_REG_ERR_SHIFT	0
@@ -51,6 +52,16 @@ 
#define _ASM_EXTABLE_UACCESS(insn, fixup)				\

	_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, wzr, wzr)



#define _ASM_EXTABLE_COPY_MC_PAGE_ERR_ZERO(insn, fixup, err, zero)	\

	__ASM_EXTABLE_RAW(insn, fixup, 					\

			  EX_TYPE_COPY_MC_PAGE_ERR_ZERO,		\

			  (						\

			    EX_DATA_REG(ERR, err) |			\

			    EX_DATA_REG(ZERO, zero)			\

			  ))



#define _ASM_EXTABLE_COPY_MC_PAGE(insn, fixup)				\

	_ASM_EXTABLE_COPY_MC_PAGE_ERR_ZERO(insn, fixup, wzr, wzr)

/*

 * Create an exception table entry for uaccess `insn`, which will branch to `fixup`

 * when an unhandled fault is taken.
@@ -59,6 +70,10 @@ 
	_ASM_EXTABLE_UACCESS(\insn, \fixup)

	.endm



	.macro          _asm_extable_copy_mc_page, insn, fixup

	_ASM_EXTABLE_COPY_MC_PAGE(\insn, \fixup)

	.endm



/*

 * Create an exception table entry for `insn` if `fixup` is provided. Otherwise

 * do nothing.

arch/arm64/include/asm/assembler.h

+4 −0
Original line number Diff line number Diff line
@@ -154,6 +154,10 @@ lr .req x30 // link register 
#define CPU_LE(code...) code

#endif



#define CPY_MC(l, x...)		\

9999:   x;			\

	_asm_extable_copy_mc_page    9999b, l



/*

 * Define a macro that constructs a 64-bit value by concatenating two

 * 32-bit registers. Note that on big endian systems the order of the

arch/arm64/include/asm/mte.h

+5 −0
Original line number Diff line number Diff line
@@ -92,6 +92,7 @@ static inline bool try_page_mte_tagging(struct page *page) 
void mte_zero_clear_page_tags(void *addr);

void mte_sync_tags(pte_t pte);

void mte_copy_page_tags(void *kto, const void *kfrom);

int mte_copy_mc_page_tags(void *kto, const void *kfrom);

void mte_thread_init_user(void);

void mte_thread_switch(struct task_struct *next);

void mte_cpu_setup(void);
@@ -128,6 +129,10 @@ static inline void mte_sync_tags(pte_t pte) 
static inline void mte_copy_page_tags(void *kto, const void *kfrom)

{

}

static inline int mte_copy_mc_page_tags(void *kto, const void *kfrom)

{

	return 0;

}

static inline void mte_thread_init_user(void)

{

}

arch/arm64/include/asm/page.h

+10 −0
Original line number Diff line number Diff line
@@ -29,6 +29,16 @@ void copy_user_highpage(struct page *to, struct page *from, 
void copy_highpage(struct page *to, struct page *from);

#define __HAVE_ARCH_COPY_HIGHPAGE



#ifdef CONFIG_ARCH_HAS_COPY_MC

int copy_mc_page(void *to, const void *from);

int copy_mc_highpage(struct page *to, struct page *from);

#define __HAVE_ARCH_COPY_MC_HIGHPAGE



int copy_mc_user_highpage(struct page *to, struct page *from,

		unsigned long vaddr, struct vm_area_struct *vma);

#define __HAVE_ARCH_COPY_MC_USER_HIGHPAGE

#endif



struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,

						unsigned long vaddr);

#define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio

arch/arm64/lib/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ endif 


lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o



lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o



obj-$(CONFIG_CRC32) += crc32.o



obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o