Commit d5d797dc authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

x86/usercopy: Remove .fixup usage 



Typically usercopy does whole word copies followed by a number of byte
copies to finish the tail. This means that on exception it needs to
compute the remaining length as: words*sizeof(long) + bytes.

Create a new extable handler to do just this.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20211110101326.081701085@infradead.org
parent 13e4bf1b

arch/x86/include/asm/extable_fixup_types.h

+5 −0
Original line number Diff line number Diff line
@@ -50,4 +50,9 @@ 


#define	EX_TYPE_FAULT_SGX		18



#define	EX_TYPE_UCOPY_LEN		19 /* cx := reg + imm*cx */

#define	EX_TYPE_UCOPY_LEN1		(EX_TYPE_UCOPY_LEN | EX_DATA_IMM(1))

#define	EX_TYPE_UCOPY_LEN4		(EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4))

#define	EX_TYPE_UCOPY_LEN8		(EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8))



#endif

arch/x86/lib/usercopy_32.c

+5 −23
Original line number Diff line number Diff line
@@ -42,11 +42,7 @@ do { \ 
		"	movl %2,%0\n"					\

		"1:	rep; stosb\n"					\

		"2: " ASM_CLAC "\n"					\

		".section .fixup,\"ax\"\n"				\

		"3:	lea 0(%2,%0,4),%0\n"				\

		"	jmp 2b\n"					\

		".previous\n"						\

		_ASM_EXTABLE_UA(0b, 3b)					\

		_ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2)	\

		_ASM_EXTABLE_UA(1b, 2b)					\

		: "=&c"(size), "=&D" (__d0)				\

		: "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));	\
@@ -148,10 +144,6 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size) 
		       "36:    movl %%eax, %0\n"

		       "37:    rep; movsb\n"

		       "100:\n"

		       ".section .fixup,\"ax\"\n"

		       "101:   lea 0(%%eax,%0,4),%0\n"

		       "       jmp 100b\n"

		       ".previous\n"

		       _ASM_EXTABLE_UA(1b, 100b)

		       _ASM_EXTABLE_UA(2b, 100b)

		       _ASM_EXTABLE_UA(3b, 100b)
@@ -189,7 +181,7 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size) 
		       _ASM_EXTABLE_UA(35b, 100b)

		       _ASM_EXTABLE_UA(36b, 100b)

		       _ASM_EXTABLE_UA(37b, 100b)

		       _ASM_EXTABLE_UA(99b, 101b)

		       _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UCOPY_LEN4, %%eax)

		       : "=&c"(size), "=&D" (d0), "=&S" (d1)

		       :  "1"(to), "2"(from), "0"(size)

		       : "eax", "edx", "memory");
@@ -254,10 +246,6 @@ static unsigned long __copy_user_intel_nocache(void *to, 
	       "        movl %%eax,%0\n"

	       "7:      rep; movsb\n"

	       "8:\n"

	       ".section .fixup,\"ax\"\n"

	       "9:      lea 0(%%eax,%0,4),%0\n"

	       "        jmp 8b\n"

	       ".previous\n"

	       _ASM_EXTABLE_UA(0b, 8b)

	       _ASM_EXTABLE_UA(1b, 8b)

	       _ASM_EXTABLE_UA(2b, 8b)
@@ -276,7 +264,7 @@ static unsigned long __copy_user_intel_nocache(void *to, 
	       _ASM_EXTABLE_UA(81b, 8b)

	       _ASM_EXTABLE_UA(14b, 8b)

	       _ASM_EXTABLE_UA(91b, 8b)

	       _ASM_EXTABLE_UA(6b, 9b)

	       _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UCOPY_LEN4, %%eax)

	       _ASM_EXTABLE_UA(7b, 8b)

	       : "=&c"(size), "=&D" (d0), "=&S" (d1)

	       :  "1"(to), "2"(from), "0"(size)
@@ -314,14 +302,8 @@ do { \ 
		"	movl %3,%0\n"					\

		"1:	rep; movsb\n"					\

		"2:\n"							\

		".section .fixup,\"ax\"\n"				\

		"5:	addl %3,%0\n"					\

		"	jmp 2b\n"					\

		"3:	lea 0(%3,%0,4),%0\n"				\

		"	jmp 2b\n"					\

		".previous\n"						\

		_ASM_EXTABLE_UA(4b, 5b)					\

		_ASM_EXTABLE_UA(0b, 3b)					\

		_ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3)	\

		_ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3)	\

		_ASM_EXTABLE_UA(1b, 2b)					\

		: "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)	\

		: "3"(size), "0"(size), "1"(to), "2"(from)		\

arch/x86/lib/usercopy_64.c

+3 −5
Original line number Diff line number Diff line
@@ -35,12 +35,10 @@ unsigned long __clear_user(void __user *addr, unsigned long size) 
		"	incq   %[dst]\n"

		"	decl %%ecx ; jnz  1b\n"

		"2:\n"

		".section .fixup,\"ax\"\n"

		"3:	lea 0(%[size1],%[size8],8),%[size8]\n"

		"	jmp 2b\n"

		".previous\n"

		_ASM_EXTABLE_UA(0b, 3b)



		_ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1])

		_ASM_EXTABLE_UA(1b, 2b)



		: [size8] "=&c"(size), [dst] "=&D" (__d0)

		: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));

	clac();

arch/x86/mm/extable.c

+9 −0
Original line number Diff line number Diff line
@@ -145,6 +145,13 @@ static bool ex_handler_imm_reg(const struct exception_table_entry *fixup, 
	return ex_handler_default(fixup, regs);

}



static bool ex_handler_ucopy_len(const struct exception_table_entry *fixup,

				  struct pt_regs *regs, int trapnr, int reg, int imm)

{

	regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg);

	return ex_handler_uaccess(fixup, regs, trapnr);

}



int ex_get_fixup_type(unsigned long ip)

{

	const struct exception_table_entry *e = search_exception_tables(ip);
@@ -217,6 +224,8 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, 
		return ex_handler_imm_reg(e, regs, reg, imm);

	case EX_TYPE_FAULT_SGX:

		return ex_handler_sgx(e, regs, trapnr);

	case EX_TYPE_UCOPY_LEN:

		return ex_handler_ucopy_len(e, regs, trapnr, reg, imm);

	}

	BUG();

}