Unverified Commit 9d952771 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!14005 v2 Optimize functions like get_user and put_user 

Merge Pull Request from: @ci-robot 
 
PR sync from: Ze Zuo <zuoze1@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/BKPSMARG5K6JUQME6ZUH3NP2Y3I22BUP/ 
Due to the performance degradation of the lsmem command on the ARM
architecture, unsafe_copy_to_user, unsafe_put_user, and unsafe_get_user
have been introduced to bypass the usercopy checks. This optimizes the
data exchange between user space and kernel space, thereby improving
performance.

v1->v2:
-- Some commit message cleanup on patch 5.

Linus Torvalds (2):
  arm64: start using 'asm goto' for get_user() when available
  arm64: start using 'asm goto' for put_user()

Mark Rutland (2):
  init/Kconfig: remove CONFIG_GCC_ASM_GOTO_OUTPUT_WORKAROUND
  arm64: uaccess: correct thinko in __get_mem_asm()

Ze Zuo (1):
  openeuler_defconfig: remove GCC_ASM_GOTO_OUTPUT_WORKAROUND for x86 and
    arm64


-- 
2.25.1
 
https://gitee.com/openeuler/kernel/issues/IB81V6 
 
Link:https://gitee.com/openeuler/kernel/pulls/14005

 

Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: default avatarLiu Chao <liuchao173@huawei.com>
Signed-off-by: default avatarZhang Peng <zhangpeng362@huawei.com>
parents cec470c9 ab0055f9

arch/arm64/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -1758,6 +1758,10 @@ config RODATA_FULL_DEFAULT_ENABLED 


config ARM64_SW_TTBR0_PAN

	bool "Emulate Privileged Access Never using TTBR0_EL1 switching"

	  # KCSAN and SW_TTBR0_PAN were probably always incompatible, but

	  # it became obvious only when implementing the unsafe user access

	  # functions. So don't use it together with KCSAN.

	  # depends on !KCSAN

	help

	  Enabling this option prevents the kernel from accessing

	  user-space memory directly by pointing TTBR0_EL1 to a reserved

arch/arm64/configs/openeuler_defconfig

+0 −1
Original line number Diff line number Diff line
@@ -2,7 +2,6 @@ 
# Automatically generated file; DO NOT EDIT.

# Linux/arm64 6.6.0 Kernel Configuration

#

CONFIG_GCC_ASM_GOTO_OUTPUT_WORKAROUND=y

CONFIG_IRQ_WORK=y

CONFIG_BUILDTIME_TABLE_SORT=y

CONFIG_THREAD_INFO_IN_TASK=y

arch/arm64/include/asm/asm-extable.h

+3 −0
Original line number Diff line number Diff line
@@ -133,6 +133,9 @@ 
#define _ASM_EXTABLE_KACCESS_ERR(insn, fixup, err)			\

	_ASM_EXTABLE_KACCESS_ERR_ZERO(insn, fixup, err, wzr)



#define _ASM_EXTABLE_KACCESS(insn, fixup)				\

	_ASM_EXTABLE_KACCESS_ERR_ZERO(insn, fixup, wzr, wzr)



#define _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(insn, fixup, data, addr)		\

	__DEFINE_ASM_GPR_NUMS							\

	__ASM_EXTABLE_RAW(#insn, #fixup,					\

arch/arm64/include/asm/uaccess.h

+123 −46
Original line number Diff line number Diff line
@@ -184,29 +184,40 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr) 
 * The "__xxx_error" versions set the third argument to -EFAULT if an error

 * occurs, and leave it unchanged on success.

 */

#define __get_mem_asm(load, reg, x, addr, err, type)			\

#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT

#define __get_mem_asm(load, reg, x, addr, label, type)			\

	asm_goto_output(						\

	"1:	" load "	" reg "0, [%1]\n"			\

	_ASM_EXTABLE_##type##ACCESS(1b, %l2)				\

	: "=r" (x)							\

	: "r" (addr) : : label)

#else

#define __get_mem_asm(load, reg, x, addr, label, type) do {		\

	int __gma_err = 0;						\

	asm volatile(							\

	"1:	" load "	" reg "1, [%2]\n"			\

	"2:\n"								\

	_ASM_EXTABLE_##type##ACCESS_ERR_ZERO(1b, 2b, %w0, %w1)		\

	: "+r" (err), "=r" (x)						\

	: "r" (addr))

	: "+r" (__gma_err), "=r" (x)					\

	: "r" (addr));							\

	if (__gma_err) goto label; } while (0)

#endif



#define __raw_get_mem(ldr, x, ptr, err, type)					\

#define __raw_get_mem(ldr, x, ptr, label, type)					\

do {										\

	unsigned long __gu_val;							\

	switch (sizeof(*(ptr))) {						\

	case 1:									\

		__get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err), type);	\

		__get_mem_asm(ldr "b", "%w", __gu_val, (ptr), label, type);	\

		break;								\

	case 2:									\

		__get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err), type);	\

		__get_mem_asm(ldr "h", "%w", __gu_val, (ptr), label, type);	\

		break;								\

	case 4:									\

		__get_mem_asm(ldr, "%w", __gu_val, (ptr), (err), type);		\

		__get_mem_asm(ldr, "%w", __gu_val, (ptr), label, type);		\

		break;								\

	case 8:									\

		__get_mem_asm(ldr, "%x",  __gu_val, (ptr), (err), type);	\

		__get_mem_asm(ldr, "%x",  __gu_val, (ptr), label, type);	\

		break;								\

	default:								\

		BUILD_BUG();							\
@@ -219,27 +230,34 @@ do { \ 
 * uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions,

 * we must evaluate these outside of the critical section.

 */

#define __raw_get_user(x, ptr, err)					\

#define __raw_get_user(x, ptr, label)					\

do {									\

	__typeof__(*(ptr)) __user *__rgu_ptr = (ptr);			\

	__typeof__(x) __rgu_val;					\

	__chk_user_ptr(ptr);						\

									\

	do {								\

		__label__ __rgu_failed;					\

		uaccess_ttbr0_enable();					\

	__raw_get_mem("ldtr", __rgu_val, __rgu_ptr, err, U);		\

		__raw_get_mem("ldtr", __rgu_val, __rgu_ptr, __rgu_failed, U);	\

		uaccess_ttbr0_disable();				\

									\

		(x) = __rgu_val;					\

		break;							\

	__rgu_failed:							\

		uaccess_ttbr0_disable();				\

		goto label;						\

	} while (0);							\

} while (0)



#define __get_user_error(x, ptr, err)					\

do {									\

	__label__ __gu_failed;						\

	__typeof__(*(ptr)) __user *__p = (ptr);				\

	might_fault();							\

	if (access_ok(__p, sizeof(*__p))) {				\

		__p = uaccess_mask_ptr(__p);				\

		__raw_get_user((x), __p, (err));			\

		__raw_get_user((x), __p, __gu_failed);			\

	} else {							\

	__gu_failed:							\

		(x) = (__force __typeof__(x))0; (err) = -EFAULT;	\

	}								\

} while (0)
@@ -262,40 +280,42 @@ do { \ 
do {									\

	__typeof__(dst) __gkn_dst = (dst);				\

	__typeof__(src) __gkn_src = (src);				\

	int __gkn_err = 0;						\

	do { 								\

		__label__ __gkn_label;					\

									\

		__mte_enable_tco_async();				\

		__raw_get_mem("ldr", *((type *)(__gkn_dst)),		\

		      (__force type *)(__gkn_src), __gkn_err, K);	\

		      (__force type *)(__gkn_src), __gkn_label, K);	\

		__mte_disable_tco_async();				\

		break;							\

	__gkn_label:							\

		__mte_disable_tco_async();				\

									\

	if (unlikely(__gkn_err))					\

		goto err_label;						\

	} while (0);							\

} while (0)



#define __put_mem_asm(store, reg, x, addr, err, type)			\

	asm volatile(							\

	"1:	" store "	" reg "1, [%2]\n"			\

#define __put_mem_asm(store, reg, x, addr, label, type)			\

	asm goto(							\

	"1:	" store "	" reg "0, [%1]\n"			\

	"2:\n"								\

	_ASM_EXTABLE_##type##ACCESS_ERR(1b, 2b, %w0)			\

	: "+r" (err)							\

	: "rZ" (x), "r" (addr))

	_ASM_EXTABLE_##type##ACCESS(1b, %l2)				\

	: : "rZ" (x), "r" (addr) : : label)



#define __raw_put_mem(str, x, ptr, err, type)					\

#define __raw_put_mem(str, x, ptr, label, type)					\

do {										\

	__typeof__(*(ptr)) __pu_val = (x);					\

	switch (sizeof(*(ptr))) {						\

	case 1:									\

		__put_mem_asm(str "b", "%w", __pu_val, (ptr), (err), type);	\

		__put_mem_asm(str "b", "%w", __pu_val, (ptr), label, type);	\

		break;								\

	case 2:									\

		__put_mem_asm(str "h", "%w", __pu_val, (ptr), (err), type);	\

		__put_mem_asm(str "h", "%w", __pu_val, (ptr), label, type);	\

		break;								\

	case 4:									\

		__put_mem_asm(str, "%w", __pu_val, (ptr), (err), type);		\

		__put_mem_asm(str, "%w", __pu_val, (ptr), label, type);		\

		break;								\

	case 8:									\

		__put_mem_asm(str, "%x", __pu_val, (ptr), (err), type);		\

		__put_mem_asm(str, "%x", __pu_val, (ptr), label, type);		\

		break;								\

	default:								\

		BUILD_BUG();							\
@@ -307,25 +327,34 @@ do { \ 
 * uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions,

 * we must evaluate these outside of the critical section.

 */

#define __raw_put_user(x, ptr, err)					\

#define __raw_put_user(x, ptr, label)					\

do {									\

	__label__ __rpu_failed;						\

	__typeof__(*(ptr)) __user *__rpu_ptr = (ptr);			\

	__typeof__(*(ptr)) __rpu_val = (x);				\

	__chk_user_ptr(__rpu_ptr);					\

									\

	do {								\

		uaccess_ttbr0_enable();					\

	__raw_put_mem("sttr", __rpu_val, __rpu_ptr, err, U);		\

		__raw_put_mem("sttr", __rpu_val, __rpu_ptr, __rpu_failed, U);	\

		uaccess_ttbr0_disable();				\

		break;							\

	__rpu_failed:							\

		uaccess_ttbr0_disable();				\

		goto label;						\

	} while (0);							\

} while (0)



#define __put_user_error(x, ptr, err)					\

do {									\

	__label__ __pu_failed;						\

	__typeof__(*(ptr)) __user *__p = (ptr);				\

	might_fault();							\

	if (access_ok(__p, sizeof(*__p))) {				\

		__p = uaccess_mask_ptr(__p);				\

		__raw_put_user((x), __p, (err));			\

		__raw_put_user((x), __p, __pu_failed);			\

	} else	{							\

	__pu_failed:							\

		(err) = -EFAULT;					\

	}								\

} while (0)
@@ -348,15 +377,18 @@ do { \ 
do {									\

	__typeof__(dst) __pkn_dst = (dst);				\

	__typeof__(src) __pkn_src = (src);				\

	int __pkn_err = 0;						\

									\

	do {								\

		__label__ __pkn_err;					\

		__mte_enable_tco_async();				\

		__raw_put_mem("str", *((type *)(__pkn_src)),		\

			      (__force type *)(__pkn_dst), __pkn_err, K);	\

		__mte_disable_tco_async();				\

									\

	if (unlikely(__pkn_err))					\

		break;							\

	__pkn_err:							\

		__mte_disable_tco_async();				\

		goto err_label;						\

	} while (0);							\

} while(0)



extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
@@ -381,6 +413,51 @@ extern unsigned long __must_check __arch_copy_to_user(void __user *to, const voi 
	__actu_ret;							\

})



static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)

{

	if (unlikely(!access_ok(ptr,len)))

		return 0;

	uaccess_ttbr0_enable();

	return 1;

}

#define user_access_begin(a,b)	user_access_begin(a,b)

#define user_access_end()	uaccess_ttbr0_disable()

#define unsafe_put_user(x, ptr, label) \

	__raw_put_mem("sttr", x, uaccess_mask_ptr(ptr), label, U)

#define unsafe_get_user(x, ptr, label) \

	__raw_get_mem("ldtr", x, uaccess_mask_ptr(ptr), label, U)



/*

 * KCSAN uses these to save and restore ttbr state.

 * We do not support KCSAN with ARM64_SW_TTBR0_PAN, so

 * they are no-ops.

 */

static inline unsigned long user_access_save(void) { return 0; }

static inline void user_access_restore(unsigned long enabled) { }



/*

 * We want the unsafe accessors to always be inlined and use

 * the error labels - thus the macro games.

 */

#define unsafe_copy_loop(dst, src, len, type, label)				\

	while (len >= sizeof(type)) {						\

		unsafe_put_user(*(type *)(src),(type __user *)(dst),label);	\

		dst += sizeof(type);						\

		src += sizeof(type);						\

		len -= sizeof(type);						\

	}



#define unsafe_copy_to_user(_dst,_src,_len,label)			\

do {									\

	char __user *__ucu_dst = (_dst);				\

	const char *__ucu_src = (_src);					\

	size_t __ucu_len = (_len);					\

	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u64, label);	\

	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u32, label);	\

	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u16, label);	\

	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label);	\

} while (0)



#define INLINE_COPY_TO_USER

#define INLINE_COPY_FROM_USER

arch/arm64/kernel/mte.c

+5 −7
Original line number Diff line number Diff line
@@ -582,12 +582,9 @@ subsys_initcall(register_mte_tcf_preferred_sysctl); 
size_t mte_probe_user_range(const char __user *uaddr, size_t size)

{

	const char __user *end = uaddr + size;

	int err = 0;

	char val;



	__raw_get_user(val, uaddr, err);

	if (err)

		return size;

	__raw_get_user(val, uaddr, efault);



	uaddr = PTR_ALIGN(uaddr, MTE_GRANULE_SIZE);

	while (uaddr < end) {
@@ -595,12 +592,13 @@ size_t mte_probe_user_range(const char __user *uaddr, size_t size) 
		 * A read is sufficient for mte, the caller should have probed

		 * for the pte write permission if required.

		 */

		__raw_get_user(val, uaddr, err);

		if (err)

			return end - uaddr;

		__raw_get_user(val, uaddr, efault);

		uaddr += MTE_GRANULE_SIZE;

	}

	(void)val;



	return 0;



efault:

	return end - uaddr;

}