Commit 09ef057b authored by Danny Tsen's avatar Danny Tsen Committed by Herbert Xu
Browse files

crypto: poly1305-p10 - An optimized Poly1305 implementation with 4-way unrolling for ppc64le 



Improve overall performance of Poly1305 for Power10 or later CPU.

Signed-off-by: default avatarDanny Tsen <dtsen@linux.ibm.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent a09450e5

arch/powerpc/crypto/poly1305-p10le_64.S

0 → 100644
+1075 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-or-later */

#

# Accelerated poly1305 implementation for ppc64le.

#

# Copyright 2023- IBM Corp. All rights reserved

#

#===================================================================================

# Written by Danny Tsen <dtsen@us.ibm.com>

#

# Poly1305 - this version mainly using vector/VSX/Scalar

#  - 26 bits limbs

#  - Handle multiple 64 byte blcok.

#

# Block size 16 bytes

# key = (r, s)

# clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF

# p = 2^130 - 5

# a += m

# a = (r + a) % p

# a += s

#

# Improve performance by breaking down polynominal to the sum of products with

#     h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r

#

#  07/22/21 - this revison based on the above sum of products.  Setup r^4, r^3, r^2, r and s3, s2, s1, s0

#             to 9 vectors for multiplications.

#

# setup r^4, r^3, r^2, r vectors

#    vs    [r^1, r^3, r^2, r^4]

#    vs0 = [r0,.....]

#    vs1 = [r1,.....]

#    vs2 = [r2,.....]

#    vs3 = [r3,.....]

#    vs4 = [r4,.....]

#    vs5 = [r1*5,...]

#    vs6 = [r2*5,...]

#    vs7 = [r2*5,...]

#    vs8 = [r4*5,...]

#

#  Each word in a vector consists a member of a "r/s" in [a * r/s].

#

# r0, r4*5, r3*5, r2*5, r1*5;

# r1, r0,   r4*5, r3*5, r2*5;

# r2, r1,   r0,   r4*5, r3*5;

# r3, r2,   r1,   r0,   r4*5;

# r4, r3,   r2,   r1,   r0  ;

#

#

# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)

#  k = 32 bytes key

#  r3 = k (r, s)

#  r4 = mlen

#  r5 = m

#

#include <asm/ppc_asm.h>

#include <asm/asm-offsets.h>

#include <asm/asm-compat.h>

#include <linux/linkage.h>



.machine "any"



.text



.macro	SAVE_GPR GPR OFFSET FRAME

	std	\GPR,\OFFSET(\FRAME)

.endm



.macro	SAVE_VRS VRS OFFSET FRAME

	li	16, \OFFSET

	stvx	\VRS, 16, \FRAME

.endm



.macro	SAVE_VSX VSX OFFSET FRAME

	li	16, \OFFSET

	stxvx	\VSX, 16, \FRAME

.endm



.macro	RESTORE_GPR GPR OFFSET FRAME

	ld	\GPR,\OFFSET(\FRAME)

.endm



.macro	RESTORE_VRS VRS OFFSET FRAME

	li	16, \OFFSET

	lvx	\VRS, 16, \FRAME

.endm



.macro	RESTORE_VSX VSX OFFSET FRAME

	li	16, \OFFSET

	lxvx	\VSX, 16, \FRAME

.endm



.macro SAVE_REGS

	mflr 0

	std 0, 16(1)

	stdu 1,-752(1)



	SAVE_GPR 14, 112, 1

	SAVE_GPR 15, 120, 1

	SAVE_GPR 16, 128, 1

	SAVE_GPR 17, 136, 1

	SAVE_GPR 18, 144, 1

	SAVE_GPR 19, 152, 1

	SAVE_GPR 20, 160, 1

	SAVE_GPR 21, 168, 1

	SAVE_GPR 22, 176, 1

	SAVE_GPR 23, 184, 1

	SAVE_GPR 24, 192, 1

	SAVE_GPR 25, 200, 1

	SAVE_GPR 26, 208, 1

	SAVE_GPR 27, 216, 1

	SAVE_GPR 28, 224, 1

	SAVE_GPR 29, 232, 1

	SAVE_GPR 30, 240, 1

	SAVE_GPR 31, 248, 1



	addi	9, 1, 256

	SAVE_VRS 20, 0, 9

	SAVE_VRS 21, 16, 9

	SAVE_VRS 22, 32, 9

	SAVE_VRS 23, 48, 9

	SAVE_VRS 24, 64, 9

	SAVE_VRS 25, 80, 9

	SAVE_VRS 26, 96, 9

	SAVE_VRS 27, 112, 9

	SAVE_VRS 28, 128, 9

	SAVE_VRS 29, 144, 9

	SAVE_VRS 30, 160, 9

	SAVE_VRS 31, 176, 9



	SAVE_VSX 14, 192, 9

	SAVE_VSX 15, 208, 9

	SAVE_VSX 16, 224, 9

	SAVE_VSX 17, 240, 9

	SAVE_VSX 18, 256, 9

	SAVE_VSX 19, 272, 9

	SAVE_VSX 20, 288, 9

	SAVE_VSX 21, 304, 9

	SAVE_VSX 22, 320, 9

	SAVE_VSX 23, 336, 9

	SAVE_VSX 24, 352, 9

	SAVE_VSX 25, 368, 9

	SAVE_VSX 26, 384, 9

	SAVE_VSX 27, 400, 9

	SAVE_VSX 28, 416, 9

	SAVE_VSX 29, 432, 9

	SAVE_VSX 30, 448, 9

	SAVE_VSX 31, 464, 9

.endm # SAVE_REGS



.macro RESTORE_REGS

	addi	9, 1, 256

	RESTORE_VRS 20, 0, 9

	RESTORE_VRS 21, 16, 9

	RESTORE_VRS 22, 32, 9

	RESTORE_VRS 23, 48, 9

	RESTORE_VRS 24, 64, 9

	RESTORE_VRS 25, 80, 9

	RESTORE_VRS 26, 96, 9

	RESTORE_VRS 27, 112, 9

	RESTORE_VRS 28, 128, 9

	RESTORE_VRS 29, 144, 9

	RESTORE_VRS 30, 160, 9

	RESTORE_VRS 31, 176, 9



	RESTORE_VSX 14, 192, 9

	RESTORE_VSX 15, 208, 9

	RESTORE_VSX 16, 224, 9

	RESTORE_VSX 17, 240, 9

	RESTORE_VSX 18, 256, 9

	RESTORE_VSX 19, 272, 9

	RESTORE_VSX 20, 288, 9

	RESTORE_VSX 21, 304, 9

	RESTORE_VSX 22, 320, 9

	RESTORE_VSX 23, 336, 9

	RESTORE_VSX 24, 352, 9

	RESTORE_VSX 25, 368, 9

	RESTORE_VSX 26, 384, 9

	RESTORE_VSX 27, 400, 9

	RESTORE_VSX 28, 416, 9

	RESTORE_VSX 29, 432, 9

	RESTORE_VSX 30, 448, 9

	RESTORE_VSX 31, 464, 9



	RESTORE_GPR 14, 112, 1

	RESTORE_GPR 15, 120, 1

	RESTORE_GPR 16, 128, 1

	RESTORE_GPR 17, 136, 1

	RESTORE_GPR 18, 144, 1

	RESTORE_GPR 19, 152, 1

	RESTORE_GPR 20, 160, 1

	RESTORE_GPR 21, 168, 1

	RESTORE_GPR 22, 176, 1

	RESTORE_GPR 23, 184, 1

	RESTORE_GPR 24, 192, 1

	RESTORE_GPR 25, 200, 1

	RESTORE_GPR 26, 208, 1

	RESTORE_GPR 27, 216, 1

	RESTORE_GPR 28, 224, 1

	RESTORE_GPR 29, 232, 1

	RESTORE_GPR 30, 240, 1

	RESTORE_GPR 31, 248, 1



	addi    1, 1, 752

	ld 0, 16(1)

	mtlr 0

.endm # RESTORE_REGS



#

# p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5;

# p[1] = a0*r1 + a1*r0   + a2*r4*5 + a3*r3*5 + a4*r2*5;

# p[2] = a0*r2 + a1*r1   + a2*r0   + a3*r4*5 + a4*r3*5;

# p[3] = a0*r3 + a1*r2   + a2*r1   + a3*r0   + a4*r4*5;

# p[4] = a0*r4 + a1*r3   + a2*r2   + a3*r1   + a4*r0  ;

#

#    [r^2, r^3, r^1, r^4]

#    [m3,  m2,  m4,  m1]

#

# multiply odd and even words

.macro mul_odd

	vmulouw	14, 4, 26

	vmulouw	10, 5, 3

	vmulouw	11, 6, 2

	vmulouw	12, 7, 1

	vmulouw	13, 8, 0

	vmulouw	15, 4, 27

	vaddudm	14, 14, 10

	vaddudm	14, 14, 11

	vmulouw	10, 5, 26

	vmulouw	11, 6, 3

	vaddudm	14, 14, 12

	vaddudm	14, 14, 13	# x0

	vaddudm	15, 15, 10

	vaddudm	15, 15, 11

	vmulouw	12, 7, 2

	vmulouw	13, 8, 1

	vaddudm	15, 15, 12

	vaddudm	15, 15, 13	# x1

	vmulouw	16, 4, 28

	vmulouw	10, 5, 27

	vmulouw	11, 6, 26

	vaddudm	16, 16, 10

	vaddudm	16, 16, 11

	vmulouw	12, 7, 3

	vmulouw	13, 8, 2

	vaddudm	16, 16, 12

	vaddudm	16, 16, 13	# x2

	vmulouw	17, 4, 29

	vmulouw	10, 5, 28

	vmulouw	11, 6, 27

	vaddudm	17, 17, 10

	vaddudm	17, 17, 11

	vmulouw	12, 7, 26

	vmulouw	13, 8, 3

	vaddudm	17, 17, 12

	vaddudm	17, 17, 13	# x3

	vmulouw	18, 4, 30

	vmulouw	10, 5, 29

	vmulouw	11, 6, 28

	vaddudm	18, 18, 10

	vaddudm	18, 18, 11

	vmulouw	12, 7, 27

	vmulouw	13, 8, 26

	vaddudm	18, 18, 12

	vaddudm	18, 18, 13	# x4

.endm



.macro mul_even

	vmuleuw	9, 4, 26

	vmuleuw	10, 5, 3

	vmuleuw	11, 6, 2

	vmuleuw	12, 7, 1

	vmuleuw	13, 8, 0

	vaddudm	14, 14, 9

	vaddudm	14, 14, 10

	vaddudm	14, 14, 11

	vaddudm	14, 14, 12

	vaddudm	14, 14, 13	# x0



	vmuleuw	9, 4, 27

	vmuleuw	10, 5, 26

	vmuleuw	11, 6, 3

	vmuleuw	12, 7, 2

	vmuleuw	13, 8, 1

	vaddudm	15, 15, 9

	vaddudm	15, 15, 10

	vaddudm	15, 15, 11

	vaddudm	15, 15, 12

	vaddudm	15, 15, 13	# x1



	vmuleuw	9, 4, 28

	vmuleuw	10, 5, 27

	vmuleuw	11, 6, 26

	vmuleuw	12, 7, 3

	vmuleuw	13, 8, 2

	vaddudm	16, 16, 9

	vaddudm	16, 16, 10

	vaddudm	16, 16, 11

	vaddudm	16, 16, 12

	vaddudm	16, 16, 13	# x2



	vmuleuw	9, 4, 29

	vmuleuw	10, 5, 28

	vmuleuw	11, 6, 27

	vmuleuw	12, 7, 26

	vmuleuw	13, 8, 3

	vaddudm	17, 17, 9

	vaddudm	17, 17, 10

	vaddudm	17, 17, 11

	vaddudm	17, 17, 12

	vaddudm	17, 17, 13	# x3



	vmuleuw	9, 4, 30

	vmuleuw	10, 5, 29

	vmuleuw	11, 6, 28

	vmuleuw	12, 7, 27

	vmuleuw	13, 8, 26

	vaddudm	18, 18, 9

	vaddudm	18, 18, 10

	vaddudm	18, 18, 11

	vaddudm	18, 18, 12

	vaddudm	18, 18, 13	# x4

.endm



#

# poly1305_setup_r

#

# setup r^4, r^3, r^2, r vectors

#    [r, r^3, r^2, r^4]

#    vs0 = [r0,...]

#    vs1 = [r1,...]

#    vs2 = [r2,...]

#    vs3 = [r3,...]

#    vs4 = [r4,...]

#    vs5 = [r4*5,...]

#    vs6 = [r3*5,...]

#    vs7 = [r2*5,...]

#    vs8 = [r1*5,...]

#

# r0, r4*5, r3*5, r2*5, r1*5;

# r1, r0,   r4*5, r3*5, r2*5;

# r2, r1,   r0,   r4*5, r3*5;

# r3, r2,   r1,   r0,   r4*5;

# r4, r3,   r2,   r1,   r0  ;

#

.macro poly1305_setup_r



	# save r

	xxlor	26, 58, 58

	xxlor	27, 59, 59

	xxlor	28, 60, 60

	xxlor	29, 61, 61

	xxlor	30, 62, 62



	xxlxor	31, 31, 31



#    [r, r^3, r^2, r^4]

	# compute r^2

	vmr	4, 26

	vmr	5, 27

	vmr	6, 28

	vmr	7, 29

	vmr	8, 30

	bl	do_mul		# r^2 r^1

	xxpermdi 58, 58, 36, 0x3		# r0

	xxpermdi 59, 59, 37, 0x3		# r1

	xxpermdi 60, 60, 38, 0x3		# r2

	xxpermdi 61, 61, 39, 0x3		# r3

	xxpermdi 62, 62, 40, 0x3		# r4

	xxpermdi 36, 36, 36, 0x3

	xxpermdi 37, 37, 37, 0x3

	xxpermdi 38, 38, 38, 0x3

	xxpermdi 39, 39, 39, 0x3

	xxpermdi 40, 40, 40, 0x3

	vspltisb 13, 2

	vsld	9, 27, 13

	vsld	10, 28, 13

	vsld	11, 29, 13

	vsld	12, 30, 13

	vaddudm	0, 9, 27

	vaddudm	1, 10, 28

	vaddudm	2, 11, 29

	vaddudm	3, 12, 30



	bl	do_mul		# r^4 r^3

	vmrgow	26, 26, 4

	vmrgow	27, 27, 5

	vmrgow	28, 28, 6

	vmrgow	29, 29, 7

	vmrgow	30, 30, 8

	vspltisb 13, 2

	vsld	9, 27, 13

	vsld	10, 28, 13

	vsld	11, 29, 13

	vsld	12, 30, 13

	vaddudm	0, 9, 27

	vaddudm	1, 10, 28

	vaddudm	2, 11, 29

	vaddudm	3, 12, 30



	# r^2 r^4

	xxlor	0, 58, 58

	xxlor	1, 59, 59

	xxlor	2, 60, 60

	xxlor	3, 61, 61

	xxlor	4, 62, 62

	xxlor	5, 32, 32

	xxlor	6, 33, 33

	xxlor	7, 34, 34

	xxlor	8, 35, 35



	vspltw	9, 26, 3

	vspltw	10, 26, 2

	vmrgow	26, 10, 9

	vspltw	9, 27, 3

	vspltw	10, 27, 2

	vmrgow	27, 10, 9

	vspltw	9, 28, 3

	vspltw	10, 28, 2

	vmrgow	28, 10, 9

	vspltw	9, 29, 3

	vspltw	10, 29, 2

	vmrgow	29, 10, 9

	vspltw	9, 30, 3

	vspltw	10, 30, 2

	vmrgow	30, 10, 9



	vsld	9, 27, 13

	vsld	10, 28, 13

	vsld	11, 29, 13

	vsld	12, 30, 13

	vaddudm	0, 9, 27

	vaddudm	1, 10, 28

	vaddudm	2, 11, 29

	vaddudm	3, 12, 30

.endm



SYM_FUNC_START_LOCAL(do_mul)

	mul_odd



	# do reduction ( h %= p )

	# carry reduction

	vspltisb 9, 2

	vsrd	10, 14, 31

	vsrd	11, 17, 31

	vand	7, 17, 25

	vand	4, 14, 25

	vaddudm	18, 18, 11

	vsrd	12, 18, 31

	vaddudm	15, 15, 10



	vsrd	11, 15, 31

	vand	8, 18, 25

	vand	5, 15, 25

	vaddudm	4, 4, 12

	vsld	10, 12, 9

	vaddudm	6, 16, 11



	vsrd	13, 6, 31

	vand	6, 6, 25

	vaddudm	4, 4, 10

	vsrd	10, 4, 31

	vaddudm	7, 7, 13



	vsrd	11, 7, 31

	vand	7, 7, 25

	vand	4, 4, 25

	vaddudm	5, 5, 10

	vaddudm	8, 8, 11

	blr

SYM_FUNC_END(do_mul)



#

# init key

#

.macro do_poly1305_init

	addis	10, 2, rmask@toc@ha

	addi	10, 10, rmask@toc@l



	ld	11, 0(10)

	ld	12, 8(10)



	li	14, 16

	li	15, 32

	addis	10, 2, cnum@toc@ha

	addi	10, 10, cnum@toc@l

	lvx	25, 0, 10	# v25 - mask

	lvx	31, 14, 10	# v31 = 1a

	lvx	19, 15, 10	# v19 = 1 << 24

	lxv	24, 48(10)	# vs24

	lxv	25, 64(10)	# vs25



	# initialize

	# load key from r3 to vectors

	ld	9, 24(3)

	ld	10, 32(3)

	and.	9, 9, 11

	and.	10, 10, 12



	# break 26 bits

	extrdi	14, 9, 26, 38

	extrdi	15, 9, 26, 12

	extrdi	16, 9, 12, 0

	mtvsrdd	58, 0, 14

	insrdi	16, 10, 14, 38

	mtvsrdd	59, 0, 15

	extrdi	17, 10, 26, 24

	mtvsrdd	60, 0, 16

	extrdi	18, 10, 24, 0

	mtvsrdd	61, 0, 17

	mtvsrdd	62, 0, 18



	# r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5

	li	9, 5

	mtvsrdd	36, 0, 9

	vmulouw	0, 27, 4		# v0 = rr0

	vmulouw	1, 28, 4		# v1 = rr1

	vmulouw	2, 29, 4		# v2 = rr2

	vmulouw	3, 30, 4		# v3 = rr3

.endm



#

# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)

#  k = 32 bytes key

#  r3 = k (r, s)

#  r4 = mlen

#  r5 = m

#

SYM_FUNC_START(poly1305_p10le_4blocks)

.align 5

	cmpdi	5, 64

	blt	Out_no_poly1305



	SAVE_REGS



	do_poly1305_init



	li	21, 0	# counter to message



	poly1305_setup_r



	# load previous H state

	# break/convert r6 to 26 bits

	ld	9, 0(3)

	ld	10, 8(3)

	ld	19, 16(3)

	sldi	19, 19, 24

	mtvsrdd	41, 0, 19

	extrdi	14, 9, 26, 38

	extrdi	15, 9, 26, 12

	extrdi	16, 9, 12, 0

	mtvsrdd	36, 0, 14

	insrdi	16, 10, 14, 38

	mtvsrdd	37, 0, 15

	extrdi	17, 10, 26, 24

	mtvsrdd	38, 0, 16

	extrdi	18, 10, 24, 0

	mtvsrdd	39, 0, 17

	mtvsrdd	40, 0, 18

	vor	8, 8, 9



	# input m1 m2

	add	20, 4, 21

	xxlor	49, 24, 24

	xxlor	50, 25, 25

	lxvw4x	43, 0, 20

	addi	17, 20, 16

	lxvw4x	44, 0, 17

	vperm	14, 11, 12, 17

	vperm	15, 11, 12, 18

	vand	9, 14, 25	# a0

	vsrd	10, 14, 31	# >> 26

	vsrd	11, 10, 31	# 12 bits left

	vand	10, 10, 25	# a1

	vspltisb 13, 12

	vand	16, 15, 25

	vsld	12, 16, 13

	vor	11, 11, 12

	vand	11, 11, 25	# a2

	vspltisb 13, 14

	vsrd	12, 15, 13	# >> 14

	vsrd	13, 12, 31	# >> 26, a4

	vand	12, 12, 25	# a3



	vaddudm	20, 4, 9

	vaddudm	21, 5, 10

	vaddudm	22, 6, 11

	vaddudm	23, 7, 12

	vaddudm	24, 8, 13



	# m3 m4

	addi	17, 17, 16

	lxvw4x	43, 0, 17

	addi	17, 17, 16

	lxvw4x	44, 0, 17

	vperm	14, 11, 12, 17

	vperm	15, 11, 12, 18

	vand	9, 14, 25	# a0

	vsrd	10, 14, 31	# >> 26

	vsrd	11, 10, 31	# 12 bits left

	vand	10, 10, 25	# a1

	vspltisb 13, 12

	vand	16, 15, 25

	vsld	12, 16, 13

	vspltisb 13, 14

	vor	11, 11, 12

	vand	11, 11, 25	# a2

	vsrd	12, 15, 13	# >> 14

	vsrd	13, 12, 31	# >> 26, a4

	vand	12, 12, 25	# a3



	# Smash 4 message blocks into 5 vectors of [m4,  m2,  m3,  m1]

	vmrgow	4, 9, 20

	vmrgow	5, 10, 21

	vmrgow	6, 11, 22

	vmrgow	7, 12, 23

	vmrgow	8, 13, 24

	vaddudm	8, 8, 19



	addi	5, 5, -64	# len -= 64

	addi	21, 21, 64	# offset += 64



	li      9, 64

	divdu   31, 5, 9



	cmpdi	31, 0

	ble	Skip_block_loop



	mtctr	31



# h4 =   m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r

# Rewrite the polynominal sum of product as follows,

# h1 = (h0 + m1) * r^2,	h2 = (h0 + m2) * r^2

# h3 = (h1 + m3) * r^2,	h4 = (h2 + m4) * r^2  --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2

#  .... Repeat

# h5 = (h3 + m5) * r^2,	h6 = (h4 + m6) * r^2  -->

# h7 = (h5 + m7) * r^2,	h8 = (h6 + m8) * r^1  --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r

#

loop_4blocks:



	# Multiply odd words and even words

	mul_odd

	mul_even

	# carry reduction

	vspltisb 9, 2

	vsrd	10, 14, 31

	vsrd	11, 17, 31

	vand	7, 17, 25

	vand	4, 14, 25

	vaddudm	18, 18, 11

	vsrd	12, 18, 31

	vaddudm	15, 15, 10



	vsrd	11, 15, 31

	vand	8, 18, 25

	vand	5, 15, 25

	vaddudm	4, 4, 12

	vsld	10, 12, 9

	vaddudm	6, 16, 11



	vsrd	13, 6, 31

	vand	6, 6, 25

	vaddudm	4, 4, 10

	vsrd	10, 4, 31

	vaddudm	7, 7, 13



	vsrd	11, 7, 31

	vand	7, 7, 25

	vand	4, 4, 25

	vaddudm	5, 5, 10

	vaddudm	8, 8, 11



	# input m1  m2  m3  m4

	add	20, 4, 21

	xxlor	49, 24, 24

	xxlor	50, 25, 25

	lxvw4x	43, 0, 20

	addi	17, 20, 16

	lxvw4x	44, 0, 17

	vperm	14, 11, 12, 17

	vperm	15, 11, 12, 18

	addi	17, 17, 16

	lxvw4x	43, 0, 17

	addi	17, 17, 16

	lxvw4x	44, 0, 17

	vperm	17, 11, 12, 17

	vperm	18, 11, 12, 18



	vand	20, 14, 25	# a0

	vand	9, 17, 25	# a0

	vsrd	21, 14, 31	# >> 26

	vsrd	22, 21, 31	# 12 bits left

	vsrd	10, 17, 31	# >> 26

	vsrd	11, 10, 31	# 12 bits left



	vand	21, 21, 25	# a1

	vand	10, 10, 25	# a1



	vspltisb 13, 12

	vand	16, 15, 25

	vsld	23, 16, 13

	vor	22, 22, 23

	vand	22, 22, 25	# a2

	vand	16, 18, 25

	vsld	12, 16, 13

	vor	11, 11, 12

	vand	11, 11, 25	# a2

	vspltisb 13, 14

	vsrd	23, 15, 13	# >> 14

	vsrd	24, 23, 31	# >> 26, a4

	vand	23, 23, 25	# a3

	vsrd	12, 18, 13	# >> 14

	vsrd	13, 12, 31	# >> 26, a4

	vand	12, 12, 25	# a3



	vaddudm	4, 4, 20

	vaddudm	5, 5, 21

	vaddudm	6, 6, 22

	vaddudm	7, 7, 23

	vaddudm	8, 8, 24



	# Smash 4 message blocks into 5 vectors of [m4,  m2,  m3,  m1]

	vmrgow	4, 9, 4

	vmrgow	5, 10, 5

	vmrgow	6, 11, 6

	vmrgow	7, 12, 7

	vmrgow	8, 13, 8

	vaddudm	8, 8, 19



	addi	5, 5, -64	# len -= 64

	addi	21, 21, 64	# offset += 64



	bdnz	loop_4blocks



Skip_block_loop:

	xxlor	58, 0, 0

	xxlor	59, 1, 1

	xxlor	60, 2, 2

	xxlor	61, 3, 3

	xxlor	62, 4, 4

	xxlor	32, 5, 5

	xxlor	33, 6, 6

	xxlor	34, 7, 7

	xxlor	35, 8, 8



	# Multiply odd words and even words

	mul_odd

	mul_even



	# Sum the products.

	xxpermdi 41, 31, 46, 0

	xxpermdi 42, 31, 47, 0

	vaddudm	4, 14, 9

	xxpermdi 36, 31, 36, 3

	vaddudm	5, 15, 10

	xxpermdi 37, 31, 37, 3

	xxpermdi 43, 31, 48, 0

	vaddudm	6, 16, 11

	xxpermdi 38, 31, 38, 3

	xxpermdi 44, 31, 49, 0

	vaddudm	7, 17, 12

	xxpermdi 39, 31, 39, 3

	xxpermdi 45, 31, 50, 0

	vaddudm	8, 18, 13

	xxpermdi 40, 31, 40, 3



	# carry reduction

	vspltisb 9, 2

	vsrd	10, 4, 31

	vsrd	11, 7, 31

	vand	7, 7, 25

	vand	4, 4, 25

	vaddudm	8, 8, 11

	vsrd	12, 8, 31

	vaddudm	5, 5, 10



	vsrd	11, 5, 31

	vand	8, 8, 25

	vand	5, 5, 25

	vaddudm	4, 4, 12

	vsld	10, 12, 9

	vaddudm	6, 6, 11



	vsrd	13, 6, 31

	vand	6, 6, 25

	vaddudm	4, 4, 10

	vsrd	10, 4, 31

	vaddudm	7, 7, 13



	vsrd	11, 7, 31

	vand	7, 7, 25

	vand	4, 4, 25

	vaddudm	5, 5, 10

	vsrd	10, 5, 31

	vand	5, 5, 25

	vaddudm	6, 6, 10

	vaddudm	8, 8, 11



	b	do_final_update



do_final_update:

	# combine 26 bit limbs

	# v4, v5, v6, v7 and v8 are 26 bit vectors

	vsld	5, 5, 31

	vor	20, 4, 5

	vspltisb 11, 12

	vsrd	12, 6, 11

	vsld	6, 6, 31

	vsld	6, 6, 31

	vor	20, 20, 6

	vspltisb 11, 14

	vsld	7, 7, 11

	vor	21, 7, 12

	mfvsrld	16, 40		# save last 2 bytes

	vsld	8, 8, 11

	vsld	8, 8, 31

	vor	21, 21, 8

	mfvsrld	17, 52

	mfvsrld	19, 53

	srdi	16, 16, 24



	std	17, 0(3)

	std	19, 8(3)

	stw	16, 16(3)



Out_loop:

	li	3, 0



	RESTORE_REGS



	blr



Out_no_poly1305:

	li	3, 0

	blr

SYM_FUNC_END(poly1305_p10le_4blocks)



#

# =======================================================================

# The following functions implement 64 x 64 bits multiplication poly1305.

#

SYM_FUNC_START_LOCAL(Poly1305_init_64)

	#  mask 0x0FFFFFFC0FFFFFFC

	#  mask 0x0FFFFFFC0FFFFFFF

	addis	10, 2, rmask@toc@ha

	addi	10, 10, rmask@toc@l

	ld	11, 0(10)

	ld	12, 8(10)



	# initialize

	# load key from r3

	ld	9, 24(3)

	ld	10, 32(3)

	and.	9, 9, 11	# cramp mask r0

	and.	10, 10, 12	# cramp mask r1



        srdi    21, 10, 2

        add     19, 21, 10      # s1: r19 - (r1 >> 2) *5



        # setup r and s

        li      25, 0

	mtvsrdd 32+0, 9, 19	# r0, s1

	mtvsrdd 32+1, 10, 9	# r1, r0

	mtvsrdd 32+2, 19, 25	# s1

	mtvsrdd 32+3, 9, 25	# r0



	blr

SYM_FUNC_END(Poly1305_init_64)



# Poly1305_mult

# v6 = (h0, h1), v8 = h2

# v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0

#

# Output: v7, v10, v11

#

SYM_FUNC_START_LOCAL(Poly1305_mult)

	#

	#	d0 = h0 * r0 + h1 * s1

	vmsumudm	7, 6, 0, 9		# h0 * r0, h1 * s1



	#	d1 = h0 * r1 + h1 * r0 + h2 * s1

	vmsumudm	11, 6, 1, 9		# h0 * r1, h1 * r0

	vmsumudm	10, 8, 2, 11		# d1 += h2 * s1



	#       d2 = r0

	vmsumudm	11, 8, 3, 9		# d2 = h2 * r0

	blr

SYM_FUNC_END(Poly1305_mult)



#

# carry reduction

# h %=p

#

# Input: v7, v10, v11

# Output: r27, r28, r29

#

SYM_FUNC_START_LOCAL(Carry_reduction)

	mfvsrld	27, 32+7

	mfvsrld	28, 32+10

	mfvsrld	29, 32+11

	mfvsrd	20, 32+7	# h0.h

	mfvsrd	21, 32+10	# h1.h



	addc	28, 28, 20

	adde	29, 29, 21

	srdi	22, 29, 0x2

	sldi	23, 22, 0x2

	add	23, 23, 22	# (h2 & 3) * 5

	addc	27, 27, 23	# h0

	addze	28, 28		# h1

	andi.	29, 29, 0x3	# h2

	blr

SYM_FUNC_END(Carry_reduction)



#

# poly1305 multiplication

# h *= r, h %= p

#	d0 = h0 * r0 + h1 * s1

#	d1 = h0 * r1 + h1 * r0 + h2 * s1

#       d2 = h0 * r0

#

#

# unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit)

#   - no highbit if final leftover block (highbit = 0)

#

SYM_FUNC_START(poly1305_64s)

	cmpdi	5, 0

	ble	Out_no_poly1305_64



	mflr 0

	std 0, 16(1)

	stdu 1,-400(1)



	SAVE_GPR 14, 112, 1

	SAVE_GPR 15, 120, 1

	SAVE_GPR 16, 128, 1

	SAVE_GPR 17, 136, 1

	SAVE_GPR 18, 144, 1

	SAVE_GPR 19, 152, 1

	SAVE_GPR 20, 160, 1

	SAVE_GPR 21, 168, 1

	SAVE_GPR 22, 176, 1

	SAVE_GPR 23, 184, 1

	SAVE_GPR 24, 192, 1

	SAVE_GPR 25, 200, 1

	SAVE_GPR 26, 208, 1

	SAVE_GPR 27, 216, 1

	SAVE_GPR 28, 224, 1

	SAVE_GPR 29, 232, 1

	SAVE_GPR 30, 240, 1

	SAVE_GPR 31, 248, 1



	# Init poly1305

	bl Poly1305_init_64



	li 25, 0			# offset to inp and outp



	add 11, 25, 4



	# load h

	# h0, h1, h2?

        ld	27, 0(3)

        ld	28, 8(3)

        lwz	29, 16(3)



        li      30, 16

        divdu   31, 5, 30



        mtctr   31



        mr      24, 6		# highbit



Loop_block_64:

	vxor	9, 9, 9



	ld	20, 0(11)

	ld	21, 8(11)

	addi	11, 11, 16



	addc	27, 27, 20

	adde	28, 28, 21

	adde	29, 29, 24



	li	22, 0

	mtvsrdd	32+6, 27, 28	# h0, h1

	mtvsrdd	32+8, 29, 22	# h2



	bl	Poly1305_mult



	bl	Carry_reduction



	bdnz	Loop_block_64



	std	27, 0(3)

	std	28, 8(3)

	stw	29, 16(3)



	li	3, 0



	RESTORE_GPR 14, 112, 1

	RESTORE_GPR 15, 120, 1

	RESTORE_GPR 16, 128, 1

	RESTORE_GPR 17, 136, 1

	RESTORE_GPR 18, 144, 1

	RESTORE_GPR 19, 152, 1

	RESTORE_GPR 20, 160, 1

	RESTORE_GPR 21, 168, 1

	RESTORE_GPR 22, 176, 1

	RESTORE_GPR 23, 184, 1

	RESTORE_GPR 24, 192, 1

	RESTORE_GPR 25, 200, 1

	RESTORE_GPR 26, 208, 1

	RESTORE_GPR 27, 216, 1

	RESTORE_GPR 28, 224, 1

	RESTORE_GPR 29, 232, 1

	RESTORE_GPR 30, 240, 1

	RESTORE_GPR 31, 248, 1



	addi    1, 1, 400

	ld 0, 16(1)

	mtlr 0



	blr



Out_no_poly1305_64:

	li	3, 0

	blr

SYM_FUNC_END(poly1305_64s)



#

# Input: r3 = h, r4 = s, r5 = mac

# mac = h + s

#

SYM_FUNC_START(poly1305_emit_64)

	ld	10, 0(3)

	ld	11, 8(3)

	ld	12, 16(3)



	# compare modulus

	# h + 5 + (-p)

	mr	6, 10

	mr	7, 11

	mr	8, 12

	addic.	6, 6, 5

	addze	7, 7

	addze	8, 8

	srdi	9, 8, 2		# overflow?

	cmpdi	9, 0

	beq	Skip_h64

	mr	10, 6

	mr	11, 7

	mr	12, 8



Skip_h64:

	ld	6, 0(4)

	ld	7, 8(4)

	addc	10, 10, 6

	adde	11, 11, 7

	addze	12, 12



	std	10, 0(5)

	std	11, 8(5)

	blr

SYM_FUNC_END(poly1305_emit_64)



SYM_DATA_START_LOCAL(RMASK)

.align 5

rmask:

.byte	0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f

cnum:

.long	0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000

.long	0x1a, 0x00, 0x1a, 0x00

.long	0x01000000, 0x01000000, 0x01000000, 0x01000000

.long	0x00010203, 0x04050607, 0x10111213, 0x14151617

.long	0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f

SYM_DATA_END(RMASK)