Commit 03b2cd72 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'objtool-core-2021-04-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull objtool updates from Ingo Molnar:

 - Standardize the crypto asm code so that it looks like compiler-
   generated code to objtool - so that it can understand it. This
   enables unwinding from crypto asm code - and also fixes the last
   known remaining objtool warnings for LTO and more.

 - x86 decoder fixes: clean up and fix the decoder, and also extend it a
   bit

 - Misc fixes and cleanups

* tag 'objtool-core-2021-04-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (24 commits)
  x86/crypto: Enable objtool in crypto code
  x86/crypto/sha512-ssse3: Standardize stack alignment prologue
  x86/crypto/sha512-avx2: Standardize stack alignment prologue
  x86/crypto/sha512-avx: Standardize stack alignment prologue
  x86/crypto/sha256-avx2: Standardize stack alignment prologue
  x86/crypto/sha1_avx2: Standardize stack alignment prologue
  x86/crypto/sha_ni: Standardize stack alignment prologue
  x86/crypto/crc32c-pcl-intel: Standardize jump table
  x86/crypto/camellia-aesni-avx2: Unconditionally allocate stack buffer
  x86/crypto/aesni-intel_avx: Standardize stack alignment prologue
  x86/crypto/aesni-intel_avx: Fix register usage comments
  x86/crypto/aesni-intel_avx: Remove unused macros
  objtool: Support asm jump tables
  objtool: Parse options from OBJTOOL_ARGS
  objtool: Collate parse_options() users
  objtool: Add --backup
  objtool,x86: More ModRM sugar
  objtool,x86: Rewrite ADD/SUB/AND
  objtool,x86: Support %riz encodings
  objtool,x86: Simplify register decode
  ...
parents 0ff0edb5 7d3d10e0

arch/x86/crypto/Makefile

+0 −2
Original line number Diff line number Diff line
@@ -2,8 +2,6 @@ 
#

# x86 crypto algorithms



OBJECT_FILES_NON_STANDARD := y



obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o

twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o

obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o

arch/x86/crypto/aesni-intel_avx-x86_64.S

+9 −19
Original line number Diff line number Diff line
@@ -212,10 +212,6 @@ HashKey_8_k = 16*21 # store XOR of HashKey^8 <<1 mod poly here (for Karatsu 
#define arg4 %rcx

#define arg5 %r8

#define arg6 %r9

#define arg7 STACK_OFFSET+8*1(%r14)

#define arg8 STACK_OFFSET+8*2(%r14)

#define arg9 STACK_OFFSET+8*3(%r14)

#define arg10 STACK_OFFSET+8*4(%r14)

#define keysize 2*15*16(arg1)



i = 0
@@ -237,9 +233,6 @@ define_reg j %j 
.noaltmacro

.endm



# need to push 4 registers into stack to maintain

STACK_OFFSET = 8*4



TMP1 =   16*0    # Temporary storage for AAD

TMP2 =   16*1    # Temporary storage for AES State 2 (State 1 is stored in an XMM register)

TMP3 =   16*2    # Temporary storage for AES State 3
@@ -256,25 +249,22 @@ VARIABLE_OFFSET = 16*8 
################################



.macro FUNC_SAVE

        #the number of pushes must equal STACK_OFFSET

        push    %r12

        push    %r13

        push    %r14

        push    %r15



        mov     %rsp, %r14





	push	%rbp

	mov	%rsp, %rbp



        sub     $VARIABLE_OFFSET, %rsp

        and     $~63, %rsp                    # align rsp to 64 bytes

.endm



.macro FUNC_RESTORE

        mov     %r14, %rsp

        mov     %rbp, %rsp

	pop	%rbp



        pop     %r15

        pop     %r14

        pop     %r13

        pop     %r12

.endm
@@ -294,7 +284,7 @@ VARIABLE_OFFSET = 16*8 


# combined for GCM encrypt and decrypt functions

# clobbering all xmm registers

# clobbering r10, r11, r12, r13, r14, r15

# clobbering r10, r11, r12, r13, r15, rax

.macro  GCM_ENC_DEC INITIAL_BLOCKS GHASH_8_ENCRYPT_8_PARALLEL GHASH_LAST_8 GHASH_MUL ENC_DEC REP

        vmovdqu AadHash(arg2), %xmm8

        vmovdqu  HashKey(arg2), %xmm13      # xmm13 = HashKey
@@ -996,7 +986,7 @@ _partial_block_done_\@: 
## num_initial_blocks = b mod 4#

## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext

## r10, r11, r12, rax are clobbered

## arg1, arg3, arg4, r14 are used as a pointer only, not modified

## arg1, arg2, arg3, arg4 are used as pointers only, not modified



.macro INITIAL_BLOCKS_AVX REP num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC

	i = (8-\num_initial_blocks)
@@ -1231,7 +1221,7 @@ _initial_blocks_done\@: 


# encrypt 8 blocks at a time

# ghash the 8 previously encrypted ciphertext blocks

# arg1, arg3, arg4 are used as pointers only, not modified

# arg1, arg2, arg3, arg4 are used as pointers only, not modified

# r11 is the data offset value

.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX REP T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
@@ -1944,7 +1934,7 @@ SYM_FUNC_END(aesni_gcm_finalize_avx_gen2) 
## num_initial_blocks = b mod 4#

## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext

## r10, r11, r12, rax are clobbered

## arg1, arg3, arg4, r14 are used as a pointer only, not modified

## arg1, arg2, arg3, arg4 are used as pointers only, not modified



.macro INITIAL_BLOCKS_AVX2 REP num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER

	i = (8-\num_initial_blocks)
@@ -2186,7 +2176,7 @@ _initial_blocks_done\@: 


# encrypt 8 blocks at a time

# ghash the 8 previously encrypted ciphertext blocks

# arg1, arg3, arg4 are used as pointers only, not modified

# arg1, arg2, arg3, arg4 are used as pointers only, not modified

# r11 is the data offset value

.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX2 REP T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC

arch/x86/crypto/camellia-aesni-avx2-asm_64.S

+2 −3
Original line number Diff line number Diff line
@@ -990,6 +990,7 @@ SYM_FUNC_START(camellia_cbc_dec_32way) 
	 *	%rdx: src (32 blocks)

	 */

	FRAME_BEGIN

	subq $(16 * 32), %rsp;



	vzeroupper;
@@ -1002,7 +1003,6 @@ SYM_FUNC_START(camellia_cbc_dec_32way) 
		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,

		     %ymm15, %rdx, (key_table)(CTX, %r8, 8));



	movq %rsp, %r10;

	cmpq %rsi, %rdx;

	je .Lcbc_dec_use_stack;
@@ -1015,7 +1015,6 @@ SYM_FUNC_START(camellia_cbc_dec_32way) 
	 * dst still in-use (because dst == src), so use stack for temporary

	 * storage.

	 */

	subq $(16 * 32), %rsp;

	movq %rsp, %rax;



.Lcbc_dec_continue:
@@ -1025,7 +1024,6 @@ SYM_FUNC_START(camellia_cbc_dec_32way) 
	vpxor %ymm7, %ymm7, %ymm7;

	vinserti128 $1, (%rdx), %ymm7, %ymm7;

	vpxor (%rax), %ymm7, %ymm7;

	movq %r10, %rsp;

	vpxor (0 * 32 + 16)(%rdx), %ymm6, %ymm6;

	vpxor (1 * 32 + 16)(%rdx), %ymm5, %ymm5;

	vpxor (2 * 32 + 16)(%rdx), %ymm4, %ymm4;
@@ -1047,6 +1045,7 @@ SYM_FUNC_START(camellia_cbc_dec_32way) 


	vzeroupper;



	addq $(16 * 32), %rsp;

	FRAME_END

	ret;

SYM_FUNC_END(camellia_cbc_dec_32way)

arch/x86/crypto/crc32c-pcl-intel-asm_64.S

+2 −5
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ 
.endm



.macro JMPTBL_ENTRY i

.word crc_\i - crc_array

.quad crc_\i

.endm



.macro JNC_LESS_THAN j
@@ -168,10 +168,7 @@ continue_block: 
	xor     crc2, crc2



	## branch into array

	lea	jump_table(%rip), %bufp

	movzwq  (%bufp, %rax, 2), len

	lea	crc_array(%rip), %bufp

	lea     (%bufp, len, 1), %bufp

	mov	jump_table(,%rax,8), %bufp

	JMP_NOSPEC bufp



	################################################################

arch/x86/crypto/sha1_avx2_x86_64_asm.S

+4 −4
Original line number Diff line number Diff line
@@ -645,9 +645,9 @@ _loop3: 
	RESERVE_STACK  = (W_SIZE*4 + 8+24)



	/* Align stack */

	mov	%rsp, %rbx

	push	%rbp

	mov	%rsp, %rbp

	and	$~(0x20-1), %rsp

	push	%rbx

	sub	$RESERVE_STACK, %rsp



	avx2_zeroupper
@@ -665,8 +665,8 @@ _loop3: 


	avx2_zeroupper



	add	$RESERVE_STACK, %rsp

	pop	%rsp

	mov	%rbp, %rsp

	pop	%rbp



	pop	%r15

	pop	%r14