mirror of https://gitee.com/openkylin/linux.git
836 lines
18 KiB
Plaintext
836 lines
18 KiB
Plaintext
|
#ifndef __KERNEL__
|
||
|
|
# include "arm_arch.h"
|
||
|
|
.extern OPENSSL_armcap_P
|
||
|
|
#endif
|
||
|
|
|
||
|
|
.text
|
||
|
|
|
||
|
|
// forward "declarations" are required for Apple
|
||
|
|
.globl poly1305_blocks
|
||
|
|
.globl poly1305_emit
|
||
|
|
|
||
|
|
.globl poly1305_init
|
||
|
|
.type poly1305_init,%function
|
||
|
|
.align 5
|
||
|
|
poly1305_init:
|
||
|
|
cmp x1,xzr
|
||
|
|
stp xzr,xzr,[x0] // zero hash value
|
||
|
|
stp xzr,xzr,[x0,#16] // [along with is_base2_26]
|
||
|
|
|
||
|
|
csel x0,xzr,x0,eq
|
||
|
|
b.eq .Lno_key
|
||
|
|
|
||
|
|
#ifndef __KERNEL__
|
||
|
|
adrp x17,OPENSSL_armcap_P
|
||
|
|
ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
|
||
|
|
#endif
|
||
|
|
|
||
|
|
ldp x7,x8,[x1] // load key
|
||
|
|
mov x9,#0xfffffffc0fffffff
|
||
|
|
movk x9,#0x0fff,lsl#48
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x7,x7 // flip bytes
|
||
|
|
rev x8,x8
|
||
|
|
#endif
|
||
|
|
and x7,x7,x9 // &=0ffffffc0fffffff
|
||
|
|
and x9,x9,#-4
|
||
|
|
and x8,x8,x9 // &=0ffffffc0ffffffc
|
||
|
|
mov w9,#-1
|
||
|
|
stp x7,x8,[x0,#32] // save key value
|
||
|
|
str w9,[x0,#48] // impossible key power value
|
||
|
|
|
||
|
|
#ifndef __KERNEL__
|
||
|
|
tst w17,#ARMV7_NEON
|
||
|
|
|
||
|
|
adr x12,.Lpoly1305_blocks
|
||
|
|
adr x7,.Lpoly1305_blocks_neon
|
||
|
|
adr x13,.Lpoly1305_emit
|
||
|
|
|
||
|
|
csel x12,x12,x7,eq
|
||
|
|
|
||
|
|
# ifdef __ILP32__
|
||
|
|
stp w12,w13,[x2]
|
||
|
|
# else
|
||
|
|
stp x12,x13,[x2]
|
||
|
|
# endif
|
||
|
|
#endif
|
||
|
|
mov x0,#1
|
||
|
|
.Lno_key:
|
||
|
|
ret
|
||
|
|
.size poly1305_init,.-poly1305_init
|
||
|
|
|
||
|
|
.type poly1305_blocks,%function
|
||
|
|
.align 5
|
||
|
|
poly1305_blocks:
|
||
|
|
.Lpoly1305_blocks:
|
||
|
|
ands x2,x2,#-16
|
||
|
|
b.eq .Lno_data
|
||
|
|
|
||
|
|
ldp x4,x5,[x0] // load hash value
|
||
|
|
ldp x6,x17,[x0,#16] // [along with is_base2_26]
|
||
|
|
ldp x7,x8,[x0,#32] // load key value
|
||
|
|
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
lsr x12,x4,#32
|
||
|
|
mov w13,w4
|
||
|
|
lsr x14,x5,#32
|
||
|
|
mov w15,w5
|
||
|
|
lsr x16,x6,#32
|
||
|
|
#else
|
||
|
|
mov w12,w4
|
||
|
|
lsr x13,x4,#32
|
||
|
|
mov w14,w5
|
||
|
|
lsr x15,x5,#32
|
||
|
|
mov w16,w6
|
||
|
|
#endif
|
||
|
|
|
||
|
|
add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64
|
||
|
|
lsr x13,x14,#12
|
||
|
|
adds x12,x12,x14,lsl#52
|
||
|
|
add x13,x13,x15,lsl#14
|
||
|
|
adc x13,x13,xzr
|
||
|
|
lsr x14,x16,#24
|
||
|
|
adds x13,x13,x16,lsl#40
|
||
|
|
adc x14,x14,xzr
|
||
|
|
|
||
|
|
cmp x17,#0 // is_base2_26?
|
||
|
|
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
|
||
|
|
csel x4,x4,x12,eq // choose between radixes
|
||
|
|
csel x5,x5,x13,eq
|
||
|
|
csel x6,x6,x14,eq
|
||
|
|
|
||
|
|
.Loop:
|
||
|
|
ldp x10,x11,[x1],#16 // load input
|
||
|
|
sub x2,x2,#16
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x10,x10
|
||
|
|
rev x11,x11
|
||
|
|
#endif
|
||
|
|
adds x4,x4,x10 // accumulate input
|
||
|
|
adcs x5,x5,x11
|
||
|
|
|
||
|
|
mul x12,x4,x7 // h0*r0
|
||
|
|
adc x6,x6,x3
|
||
|
|
umulh x13,x4,x7
|
||
|
|
|
||
|
|
mul x10,x5,x9 // h1*5*r1
|
||
|
|
umulh x11,x5,x9
|
||
|
|
|
||
|
|
adds x12,x12,x10
|
||
|
|
mul x10,x4,x8 // h0*r1
|
||
|
|
adc x13,x13,x11
|
||
|
|
umulh x14,x4,x8
|
||
|
|
|
||
|
|
adds x13,x13,x10
|
||
|
|
mul x10,x5,x7 // h1*r0
|
||
|
|
adc x14,x14,xzr
|
||
|
|
umulh x11,x5,x7
|
||
|
|
|
||
|
|
adds x13,x13,x10
|
||
|
|
mul x10,x6,x9 // h2*5*r1
|
||
|
|
adc x14,x14,x11
|
||
|
|
mul x11,x6,x7 // h2*r0
|
||
|
|
|
||
|
|
adds x13,x13,x10
|
||
|
|
adc x14,x14,x11
|
||
|
|
|
||
|
|
and x10,x14,#-4 // final reduction
|
||
|
|
and x6,x14,#3
|
||
|
|
add x10,x10,x14,lsr#2
|
||
|
|
adds x4,x12,x10
|
||
|
|
adcs x5,x13,xzr
|
||
|
|
adc x6,x6,xzr
|
||
|
|
|
||
|
|
cbnz x2,.Loop
|
||
|
|
|
||
|
|
stp x4,x5,[x0] // store hash value
|
||
|
|
stp x6,xzr,[x0,#16] // [and clear is_base2_26]
|
||
|
|
|
||
|
|
.Lno_data:
|
||
|
|
ret
|
||
|
|
.size poly1305_blocks,.-poly1305_blocks
|
||
|
|
|
||
|
|
.type poly1305_emit,%function
|
||
|
|
.align 5
|
||
|
|
poly1305_emit:
|
||
|
|
.Lpoly1305_emit:
|
||
|
|
ldp x4,x5,[x0] // load hash base 2^64
|
||
|
|
ldp x6,x7,[x0,#16] // [along with is_base2_26]
|
||
|
|
ldp x10,x11,[x2] // load nonce
|
||
|
|
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
lsr x12,x4,#32
|
||
|
|
mov w13,w4
|
||
|
|
lsr x14,x5,#32
|
||
|
|
mov w15,w5
|
||
|
|
lsr x16,x6,#32
|
||
|
|
#else
|
||
|
|
mov w12,w4
|
||
|
|
lsr x13,x4,#32
|
||
|
|
mov w14,w5
|
||
|
|
lsr x15,x5,#32
|
||
|
|
mov w16,w6
|
||
|
|
#endif
|
||
|
|
|
||
|
|
add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64
|
||
|
|
lsr x13,x14,#12
|
||
|
|
adds x12,x12,x14,lsl#52
|
||
|
|
add x13,x13,x15,lsl#14
|
||
|
|
adc x13,x13,xzr
|
||
|
|
lsr x14,x16,#24
|
||
|
|
adds x13,x13,x16,lsl#40
|
||
|
|
adc x14,x14,xzr
|
||
|
|
|
||
|
|
cmp x7,#0 // is_base2_26?
|
||
|
|
csel x4,x4,x12,eq // choose between radixes
|
||
|
|
csel x5,x5,x13,eq
|
||
|
|
csel x6,x6,x14,eq
|
||
|
|
|
||
|
|
adds x12,x4,#5 // compare to modulus
|
||
|
|
adcs x13,x5,xzr
|
||
|
|
adc x14,x6,xzr
|
||
|
|
|
||
|
|
tst x14,#-4 // see if it's carried/borrowed
|
||
|
|
|
||
|
|
csel x4,x4,x12,eq
|
||
|
|
csel x5,x5,x13,eq
|
||
|
|
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
ror x10,x10,#32 // flip nonce words
|
||
|
|
ror x11,x11,#32
|
||
|
|
#endif
|
||
|
|
adds x4,x4,x10 // accumulate nonce
|
||
|
|
adc x5,x5,x11
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x4,x4 // flip output bytes
|
||
|
|
rev x5,x5
|
||
|
|
#endif
|
||
|
|
stp x4,x5,[x1] // write result
|
||
|
|
|
||
|
|
ret
|
||
|
|
.size poly1305_emit,.-poly1305_emit
|
||
|
|
.type poly1305_mult,%function
|
||
|
|
.align 5
|
||
|
|
poly1305_mult:
|
||
|
|
mul x12,x4,x7 // h0*r0
|
||
|
|
umulh x13,x4,x7
|
||
|
|
|
||
|
|
mul x10,x5,x9 // h1*5*r1
|
||
|
|
umulh x11,x5,x9
|
||
|
|
|
||
|
|
adds x12,x12,x10
|
||
|
|
mul x10,x4,x8 // h0*r1
|
||
|
|
adc x13,x13,x11
|
||
|
|
umulh x14,x4,x8
|
||
|
|
|
||
|
|
adds x13,x13,x10
|
||
|
|
mul x10,x5,x7 // h1*r0
|
||
|
|
adc x14,x14,xzr
|
||
|
|
umulh x11,x5,x7
|
||
|
|
|
||
|
|
adds x13,x13,x10
|
||
|
|
mul x10,x6,x9 // h2*5*r1
|
||
|
|
adc x14,x14,x11
|
||
|
|
mul x11,x6,x7 // h2*r0
|
||
|
|
|
||
|
|
adds x13,x13,x10
|
||
|
|
adc x14,x14,x11
|
||
|
|
|
||
|
|
and x10,x14,#-4 // final reduction
|
||
|
|
and x6,x14,#3
|
||
|
|
add x10,x10,x14,lsr#2
|
||
|
|
adds x4,x12,x10
|
||
|
|
adcs x5,x13,xzr
|
||
|
|
adc x6,x6,xzr
|
||
|
|
|
||
|
|
ret
|
||
|
|
.size poly1305_mult,.-poly1305_mult
|
||
|
|
|
||
|
|
.type poly1305_splat,%function
|
||
|
|
.align 4
|
||
|
|
poly1305_splat:
|
||
|
|
and x12,x4,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
ubfx x13,x4,#26,#26
|
||
|
|
extr x14,x5,x4,#52
|
||
|
|
and x14,x14,#0x03ffffff
|
||
|
|
ubfx x15,x5,#14,#26
|
||
|
|
extr x16,x6,x5,#40
|
||
|
|
|
||
|
|
str w12,[x0,#16*0] // r0
|
||
|
|
add w12,w13,w13,lsl#2 // r1*5
|
||
|
|
str w13,[x0,#16*1] // r1
|
||
|
|
add w13,w14,w14,lsl#2 // r2*5
|
||
|
|
str w12,[x0,#16*2] // s1
|
||
|
|
str w14,[x0,#16*3] // r2
|
||
|
|
add w14,w15,w15,lsl#2 // r3*5
|
||
|
|
str w13,[x0,#16*4] // s2
|
||
|
|
str w15,[x0,#16*5] // r3
|
||
|
|
add w15,w16,w16,lsl#2 // r4*5
|
||
|
|
str w14,[x0,#16*6] // s3
|
||
|
|
str w16,[x0,#16*7] // r4
|
||
|
|
str w15,[x0,#16*8] // s4
|
||
|
|
|
||
|
|
ret
|
||
|
|
.size poly1305_splat,.-poly1305_splat
|
||
|
|
|
||
|
|
#ifdef __KERNEL__
|
||
|
|
.globl poly1305_blocks_neon
|
||
|
|
#endif
|
||
|
|
.type poly1305_blocks_neon,%function
|
||
|
|
.align 5
|
||
|
|
poly1305_blocks_neon:
|
||
|
|
.Lpoly1305_blocks_neon:
|
||
|
|
ldr x17,[x0,#24]
|
||
|
|
cmp x2,#128
|
||
|
|
b.lo .Lpoly1305_blocks
|
||
|
|
|
||
|
|
.inst 0xd503233f // paciasp
|
||
|
|
stp x29,x30,[sp,#-80]!
|
||
|
|
add x29,sp,#0
|
||
|
|
|
||
|
|
stp d8,d9,[sp,#16] // meet ABI requirements
|
||
|
|
stp d10,d11,[sp,#32]
|
||
|
|
stp d12,d13,[sp,#48]
|
||
|
|
stp d14,d15,[sp,#64]
|
||
|
|
|
||
|
|
cbz x17,.Lbase2_64_neon
|
||
|
|
|
||
|
|
ldp w10,w11,[x0] // load hash value base 2^26
|
||
|
|
ldp w12,w13,[x0,#8]
|
||
|
|
ldr w14,[x0,#16]
|
||
|
|
|
||
|
|
tst x2,#31
|
||
|
|
b.eq .Leven_neon
|
||
|
|
|
||
|
|
ldp x7,x8,[x0,#32] // load key value
|
||
|
|
|
||
|
|
add x4,x10,x11,lsl#26 // base 2^26 -> base 2^64
|
||
|
|
lsr x5,x12,#12
|
||
|
|
adds x4,x4,x12,lsl#52
|
||
|
|
add x5,x5,x13,lsl#14
|
||
|
|
adc x5,x5,xzr
|
||
|
|
lsr x6,x14,#24
|
||
|
|
adds x5,x5,x14,lsl#40
|
||
|
|
adc x14,x6,xzr // can be partially reduced...
|
||
|
|
|
||
|
|
ldp x12,x13,[x1],#16 // load input
|
||
|
|
sub x2,x2,#16
|
||
|
|
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
|
||
|
|
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x12,x12
|
||
|
|
rev x13,x13
|
||
|
|
#endif
|
||
|
|
adds x4,x4,x12 // accumulate input
|
||
|
|
adcs x5,x5,x13
|
||
|
|
adc x6,x6,x3
|
||
|
|
|
||
|
|
bl poly1305_mult
|
||
|
|
|
||
|
|
and x10,x4,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
ubfx x11,x4,#26,#26
|
||
|
|
extr x12,x5,x4,#52
|
||
|
|
and x12,x12,#0x03ffffff
|
||
|
|
ubfx x13,x5,#14,#26
|
||
|
|
extr x14,x6,x5,#40
|
||
|
|
|
||
|
|
b .Leven_neon
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Lbase2_64_neon:
|
||
|
|
ldp x7,x8,[x0,#32] // load key value
|
||
|
|
|
||
|
|
ldp x4,x5,[x0] // load hash value base 2^64
|
||
|
|
ldr x6,[x0,#16]
|
||
|
|
|
||
|
|
tst x2,#31
|
||
|
|
b.eq .Linit_neon
|
||
|
|
|
||
|
|
ldp x12,x13,[x1],#16 // load input
|
||
|
|
sub x2,x2,#16
|
||
|
|
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x12,x12
|
||
|
|
rev x13,x13
|
||
|
|
#endif
|
||
|
|
adds x4,x4,x12 // accumulate input
|
||
|
|
adcs x5,x5,x13
|
||
|
|
adc x6,x6,x3
|
||
|
|
|
||
|
|
bl poly1305_mult
|
||
|
|
|
||
|
|
.Linit_neon:
|
||
|
|
ldr w17,[x0,#48] // first table element
|
||
|
|
and x10,x4,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
ubfx x11,x4,#26,#26
|
||
|
|
extr x12,x5,x4,#52
|
||
|
|
and x12,x12,#0x03ffffff
|
||
|
|
ubfx x13,x5,#14,#26
|
||
|
|
extr x14,x6,x5,#40
|
||
|
|
|
||
|
|
cmp w17,#-1 // is value impossible?
|
||
|
|
b.ne .Leven_neon
|
||
|
|
|
||
|
|
fmov d24,x10
|
||
|
|
fmov d25,x11
|
||
|
|
fmov d26,x12
|
||
|
|
fmov d27,x13
|
||
|
|
fmov d28,x14
|
||
|
|
|
||
|
|
////////////////////////////////// initialize r^n table
|
||
|
|
mov x4,x7 // r^1
|
||
|
|
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
|
||
|
|
mov x5,x8
|
||
|
|
mov x6,xzr
|
||
|
|
add x0,x0,#48+12
|
||
|
|
bl poly1305_splat
|
||
|
|
|
||
|
|
bl poly1305_mult // r^2
|
||
|
|
sub x0,x0,#4
|
||
|
|
bl poly1305_splat
|
||
|
|
|
||
|
|
bl poly1305_mult // r^3
|
||
|
|
sub x0,x0,#4
|
||
|
|
bl poly1305_splat
|
||
|
|
|
||
|
|
bl poly1305_mult // r^4
|
||
|
|
sub x0,x0,#4
|
||
|
|
bl poly1305_splat
|
||
|
|
sub x0,x0,#48 // restore original x0
|
||
|
|
b .Ldo_neon
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Leven_neon:
|
||
|
|
fmov d24,x10
|
||
|
|
fmov d25,x11
|
||
|
|
fmov d26,x12
|
||
|
|
fmov d27,x13
|
||
|
|
fmov d28,x14
|
||
|
|
|
||
|
|
.Ldo_neon:
|
||
|
|
ldp x8,x12,[x1,#32] // inp[2:3]
|
||
|
|
subs x2,x2,#64
|
||
|
|
ldp x9,x13,[x1,#48]
|
||
|
|
add x16,x1,#96
|
||
|
|
adr x17,.Lzeros
|
||
|
|
|
||
|
|
lsl x3,x3,#24
|
||
|
|
add x15,x0,#48
|
||
|
|
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x8,x8
|
||
|
|
rev x12,x12
|
||
|
|
rev x9,x9
|
||
|
|
rev x13,x13
|
||
|
|
#endif
|
||
|
|
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
and x5,x9,#0x03ffffff
|
||
|
|
ubfx x6,x8,#26,#26
|
||
|
|
ubfx x7,x9,#26,#26
|
||
|
|
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
|
||
|
|
extr x8,x12,x8,#52
|
||
|
|
extr x9,x13,x9,#52
|
||
|
|
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
|
||
|
|
fmov d14,x4
|
||
|
|
and x8,x8,#0x03ffffff
|
||
|
|
and x9,x9,#0x03ffffff
|
||
|
|
ubfx x10,x12,#14,#26
|
||
|
|
ubfx x11,x13,#14,#26
|
||
|
|
add x12,x3,x12,lsr#40
|
||
|
|
add x13,x3,x13,lsr#40
|
||
|
|
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
|
||
|
|
fmov d15,x6
|
||
|
|
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
|
||
|
|
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
|
||
|
|
fmov d16,x8
|
||
|
|
fmov d17,x10
|
||
|
|
fmov d18,x12
|
||
|
|
|
||
|
|
ldp x8,x12,[x1],#16 // inp[0:1]
|
||
|
|
ldp x9,x13,[x1],#48
|
||
|
|
|
||
|
|
ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x15],#64
|
||
|
|
ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x15],#64
|
||
|
|
ld1 {v8.4s},[x15]
|
||
|
|
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x8,x8
|
||
|
|
rev x12,x12
|
||
|
|
rev x9,x9
|
||
|
|
rev x13,x13
|
||
|
|
#endif
|
||
|
|
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
and x5,x9,#0x03ffffff
|
||
|
|
ubfx x6,x8,#26,#26
|
||
|
|
ubfx x7,x9,#26,#26
|
||
|
|
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
|
||
|
|
extr x8,x12,x8,#52
|
||
|
|
extr x9,x13,x9,#52
|
||
|
|
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
|
||
|
|
fmov d9,x4
|
||
|
|
and x8,x8,#0x03ffffff
|
||
|
|
and x9,x9,#0x03ffffff
|
||
|
|
ubfx x10,x12,#14,#26
|
||
|
|
ubfx x11,x13,#14,#26
|
||
|
|
add x12,x3,x12,lsr#40
|
||
|
|
add x13,x3,x13,lsr#40
|
||
|
|
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
|
||
|
|
fmov d10,x6
|
||
|
|
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
|
||
|
|
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
|
||
|
|
movi v31.2d,#-1
|
||
|
|
fmov d11,x8
|
||
|
|
fmov d12,x10
|
||
|
|
fmov d13,x12
|
||
|
|
ushr v31.2d,v31.2d,#38
|
||
|
|
|
||
|
|
b.ls .Lskip_loop
|
||
|
|
|
||
|
|
.align 4
|
||
|
|
.Loop_neon:
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
|
||
|
|
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
|
||
|
|
// ___________________/
|
||
|
|
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
|
||
|
|
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
|
||
|
|
// ___________________/ ____________________/
|
||
|
|
//
|
||
|
|
// Note that we start with inp[2:3]*r^2. This is because it
|
||
|
|
// doesn't depend on reduction in previous iteration.
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
|
||
|
|
// d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
|
||
|
|
// d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
|
||
|
|
// d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
|
||
|
|
// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
|
||
|
|
|
||
|
|
subs x2,x2,#64
|
||
|
|
umull v23.2d,v14.2s,v7.s[2]
|
||
|
|
csel x16,x17,x16,lo
|
||
|
|
umull v22.2d,v14.2s,v5.s[2]
|
||
|
|
umull v21.2d,v14.2s,v3.s[2]
|
||
|
|
ldp x8,x12,[x16],#16 // inp[2:3] (or zero)
|
||
|
|
umull v20.2d,v14.2s,v1.s[2]
|
||
|
|
ldp x9,x13,[x16],#48
|
||
|
|
umull v19.2d,v14.2s,v0.s[2]
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x8,x8
|
||
|
|
rev x12,x12
|
||
|
|
rev x9,x9
|
||
|
|
rev x13,x13
|
||
|
|
#endif
|
||
|
|
|
||
|
|
umlal v23.2d,v15.2s,v5.s[2]
|
||
|
|
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
umlal v22.2d,v15.2s,v3.s[2]
|
||
|
|
and x5,x9,#0x03ffffff
|
||
|
|
umlal v21.2d,v15.2s,v1.s[2]
|
||
|
|
ubfx x6,x8,#26,#26
|
||
|
|
umlal v20.2d,v15.2s,v0.s[2]
|
||
|
|
ubfx x7,x9,#26,#26
|
||
|
|
umlal v19.2d,v15.2s,v8.s[2]
|
||
|
|
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
|
||
|
|
|
||
|
|
umlal v23.2d,v16.2s,v3.s[2]
|
||
|
|
extr x8,x12,x8,#52
|
||
|
|
umlal v22.2d,v16.2s,v1.s[2]
|
||
|
|
extr x9,x13,x9,#52
|
||
|
|
umlal v21.2d,v16.2s,v0.s[2]
|
||
|
|
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
|
||
|
|
umlal v20.2d,v16.2s,v8.s[2]
|
||
|
|
fmov d14,x4
|
||
|
|
umlal v19.2d,v16.2s,v6.s[2]
|
||
|
|
and x8,x8,#0x03ffffff
|
||
|
|
|
||
|
|
umlal v23.2d,v17.2s,v1.s[2]
|
||
|
|
and x9,x9,#0x03ffffff
|
||
|
|
umlal v22.2d,v17.2s,v0.s[2]
|
||
|
|
ubfx x10,x12,#14,#26
|
||
|
|
umlal v21.2d,v17.2s,v8.s[2]
|
||
|
|
ubfx x11,x13,#14,#26
|
||
|
|
umlal v20.2d,v17.2s,v6.s[2]
|
||
|
|
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
|
||
|
|
umlal v19.2d,v17.2s,v4.s[2]
|
||
|
|
fmov d15,x6
|
||
|
|
|
||
|
|
add v11.2s,v11.2s,v26.2s
|
||
|
|
add x12,x3,x12,lsr#40
|
||
|
|
umlal v23.2d,v18.2s,v0.s[2]
|
||
|
|
add x13,x3,x13,lsr#40
|
||
|
|
umlal v22.2d,v18.2s,v8.s[2]
|
||
|
|
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
|
||
|
|
umlal v21.2d,v18.2s,v6.s[2]
|
||
|
|
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
|
||
|
|
umlal v20.2d,v18.2s,v4.s[2]
|
||
|
|
fmov d16,x8
|
||
|
|
umlal v19.2d,v18.2s,v2.s[2]
|
||
|
|
fmov d17,x10
|
||
|
|
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// (hash+inp[0:1])*r^4 and accumulate
|
||
|
|
|
||
|
|
add v9.2s,v9.2s,v24.2s
|
||
|
|
fmov d18,x12
|
||
|
|
umlal v22.2d,v11.2s,v1.s[0]
|
||
|
|
ldp x8,x12,[x1],#16 // inp[0:1]
|
||
|
|
umlal v19.2d,v11.2s,v6.s[0]
|
||
|
|
ldp x9,x13,[x1],#48
|
||
|
|
umlal v23.2d,v11.2s,v3.s[0]
|
||
|
|
umlal v20.2d,v11.2s,v8.s[0]
|
||
|
|
umlal v21.2d,v11.2s,v0.s[0]
|
||
|
|
#ifdef __AARCH64EB__
|
||
|
|
rev x8,x8
|
||
|
|
rev x12,x12
|
||
|
|
rev x9,x9
|
||
|
|
rev x13,x13
|
||
|
|
#endif
|
||
|
|
|
||
|
|
add v10.2s,v10.2s,v25.2s
|
||
|
|
umlal v22.2d,v9.2s,v5.s[0]
|
||
|
|
umlal v23.2d,v9.2s,v7.s[0]
|
||
|
|
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
|
||
|
|
umlal v21.2d,v9.2s,v3.s[0]
|
||
|
|
and x5,x9,#0x03ffffff
|
||
|
|
umlal v19.2d,v9.2s,v0.s[0]
|
||
|
|
ubfx x6,x8,#26,#26
|
||
|
|
umlal v20.2d,v9.2s,v1.s[0]
|
||
|
|
ubfx x7,x9,#26,#26
|
||
|
|
|
||
|
|
add v12.2s,v12.2s,v27.2s
|
||
|
|
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
|
||
|
|
umlal v22.2d,v10.2s,v3.s[0]
|
||
|
|
extr x8,x12,x8,#52
|
||
|
|
umlal v23.2d,v10.2s,v5.s[0]
|
||
|
|
extr x9,x13,x9,#52
|
||
|
|
umlal v19.2d,v10.2s,v8.s[0]
|
||
|
|
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
|
||
|
|
umlal v21.2d,v10.2s,v1.s[0]
|
||
|
|
fmov d9,x4
|
||
|
|
umlal v20.2d,v10.2s,v0.s[0]
|
||
|
|
and x8,x8,#0x03ffffff
|
||
|
|
|
||
|
|
add v13.2s,v13.2s,v28.2s
|
||
|
|
and x9,x9,#0x03ffffff
|
||
|
|
umlal v22.2d,v12.2s,v0.s[0]
|
||
|
|
ubfx x10,x12,#14,#26
|
||
|
|
umlal v19.2d,v12.2s,v4.s[0]
|
||
|
|
ubfx x11,x13,#14,#26
|
||
|
|
umlal v23.2d,v12.2s,v1.s[0]
|
||
|
|
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
|
||
|
|
umlal v20.2d,v12.2s,v6.s[0]
|
||
|
|
fmov d10,x6
|
||
|
|
umlal v21.2d,v12.2s,v8.s[0]
|
||
|
|
add x12,x3,x12,lsr#40
|
||
|
|
|
||
|
|
umlal v22.2d,v13.2s,v8.s[0]
|
||
|
|
add x13,x3,x13,lsr#40
|
||
|
|
umlal v19.2d,v13.2s,v2.s[0]
|
||
|
|
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
|
||
|
|
umlal v23.2d,v13.2s,v0.s[0]
|
||
|
|
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
|
||
|
|
umlal v20.2d,v13.2s,v4.s[0]
|
||
|
|
fmov d11,x8
|
||
|
|
umlal v21.2d,v13.2s,v6.s[0]
|
||
|
|
fmov d12,x10
|
||
|
|
fmov d13,x12
|
||
|
|
|
||
|
|
/////////////////////////////////////////////////////////////////
|
||
|
|
// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
|
||
|
|
// and P. Schwabe
|
||
|
|
//
|
||
|
|
// [see discussion in poly1305-armv4 module]
|
||
|
|
|
||
|
|
ushr v29.2d,v22.2d,#26
|
||
|
|
xtn v27.2s,v22.2d
|
||
|
|
ushr v30.2d,v19.2d,#26
|
||
|
|
and v19.16b,v19.16b,v31.16b
|
||
|
|
add v23.2d,v23.2d,v29.2d // h3 -> h4
|
||
|
|
bic v27.2s,#0xfc,lsl#24 // &=0x03ffffff
|
||
|
|
add v20.2d,v20.2d,v30.2d // h0 -> h1
|
||
|
|
|
||
|
|
ushr v29.2d,v23.2d,#26
|
||
|
|
xtn v28.2s,v23.2d
|
||
|
|
ushr v30.2d,v20.2d,#26
|
||
|
|
xtn v25.2s,v20.2d
|
||
|
|
bic v28.2s,#0xfc,lsl#24
|
||
|
|
add v21.2d,v21.2d,v30.2d // h1 -> h2
|
||
|
|
|
||
|
|
add v19.2d,v19.2d,v29.2d
|
||
|
|
shl v29.2d,v29.2d,#2
|
||
|
|
shrn v30.2s,v21.2d,#26
|
||
|
|
xtn v26.2s,v21.2d
|
||
|
|
add v19.2d,v19.2d,v29.2d // h4 -> h0
|
||
|
|
bic v25.2s,#0xfc,lsl#24
|
||
|
|
add v27.2s,v27.2s,v30.2s // h2 -> h3
|
||
|
|
bic v26.2s,#0xfc,lsl#24
|
||
|
|
|
||
|
|
shrn v29.2s,v19.2d,#26
|
||
|
|
xtn v24.2s,v19.2d
|
||
|
|
ushr v30.2s,v27.2s,#26
|
||
|
|
bic v27.2s,#0xfc,lsl#24
|
||
|
|
bic v24.2s,#0xfc,lsl#24
|
||
|
|
add v25.2s,v25.2s,v29.2s // h0 -> h1
|
||
|
|
add v28.2s,v28.2s,v30.2s // h3 -> h4
|
||
|
|
|
||
|
|
b.hi .Loop_neon
|
||
|
|
|
||
|
|
.Lskip_loop:
|
||
|
|
dup v16.2d,v16.d[0]
|
||
|
|
add v11.2s,v11.2s,v26.2s
|
||
|
|
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
|
||
|
|
|
||
|
|
adds x2,x2,#32
|
||
|
|
b.ne .Long_tail
|
||
|
|
|
||
|
|
dup v16.2d,v11.d[0]
|
||
|
|
add v14.2s,v9.2s,v24.2s
|
||
|
|
add v17.2s,v12.2s,v27.2s
|
||
|
|
add v15.2s,v10.2s,v25.2s
|
||
|
|
add v18.2s,v13.2s,v28.2s
|
||
|
|
|
||
|
|
.Long_tail:
|
||
|
|
dup v14.2d,v14.d[0]
|
||
|
|
umull2 v19.2d,v16.4s,v6.4s
|
||
|
|
umull2 v22.2d,v16.4s,v1.4s
|
||
|
|
umull2 v23.2d,v16.4s,v3.4s
|
||
|
|
umull2 v21.2d,v16.4s,v0.4s
|
||
|
|
umull2 v20.2d,v16.4s,v8.4s
|
||
|
|
|
||
|
|
dup v15.2d,v15.d[0]
|
||
|
|
umlal2 v19.2d,v14.4s,v0.4s
|
||
|
|
umlal2 v21.2d,v14.4s,v3.4s
|
||
|
|
umlal2 v22.2d,v14.4s,v5.4s
|
||
|
|
umlal2 v23.2d,v14.4s,v7.4s
|
||
|
|
umlal2 v20.2d,v14.4s,v1.4s
|
||
|
|
|
||
|
|
dup v17.2d,v17.d[0]
|
||
|
|
umlal2 v19.2d,v15.4s,v8.4s
|
||
|
|
umlal2 v22.2d,v15.4s,v3.4s
|
||
|
|
umlal2 v21.2d,v15.4s,v1.4s
|
||
|
|
umlal2 v23.2d,v15.4s,v5.4s
|
||
|
|
umlal2 v20.2d,v15.4s,v0.4s
|
||
|
|
|
||
|
|
dup v18.2d,v18.d[0]
|
||
|
|
umlal2 v22.2d,v17.4s,v0.4s
|
||
|
|
umlal2 v23.2d,v17.4s,v1.4s
|
||
|
|
umlal2 v19.2d,v17.4s,v4.4s
|
||
|
|
umlal2 v20.2d,v17.4s,v6.4s
|
||
|
|
umlal2 v21.2d,v17.4s,v8.4s
|
||
|
|
|
||
|
|
umlal2 v22.2d,v18.4s,v8.4s
|
||
|
|
umlal2 v19.2d,v18.4s,v2.4s
|
||
|
|
umlal2 v23.2d,v18.4s,v0.4s
|
||
|
|
umlal2 v20.2d,v18.4s,v4.4s
|
||
|
|
umlal2 v21.2d,v18.4s,v6.4s
|
||
|
|
|
||
|
|
b.eq .Lshort_tail
|
||
|
|
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// (hash+inp[0:1])*r^4:r^3 and accumulate
|
||
|
|
|
||
|
|
add v9.2s,v9.2s,v24.2s
|
||
|
|
umlal v22.2d,v11.2s,v1.2s
|
||
|
|
umlal v19.2d,v11.2s,v6.2s
|
||
|
|
umlal v23.2d,v11.2s,v3.2s
|
||
|
|
umlal v20.2d,v11.2s,v8.2s
|
||
|
|
umlal v21.2d,v11.2s,v0.2s
|
||
|
|
|
||
|
|
add v10.2s,v10.2s,v25.2s
|
||
|
|
umlal v22.2d,v9.2s,v5.2s
|
||
|
|
umlal v19.2d,v9.2s,v0.2s
|
||
|
|
umlal v23.2d,v9.2s,v7.2s
|
||
|
|
umlal v20.2d,v9.2s,v1.2s
|
||
|
|
umlal v21.2d,v9.2s,v3.2s
|
||
|
|
|
||
|
|
add v12.2s,v12.2s,v27.2s
|
||
|
|
umlal v22.2d,v10.2s,v3.2s
|
||
|
|
umlal v19.2d,v10.2s,v8.2s
|
||
|
|
umlal v23.2d,v10.2s,v5.2s
|
||
|
|
umlal v20.2d,v10.2s,v0.2s
|
||
|
|
umlal v21.2d,v10.2s,v1.2s
|
||
|
|
|
||
|
|
add v13.2s,v13.2s,v28.2s
|
||
|
|
umlal v22.2d,v12.2s,v0.2s
|
||
|
|
umlal v19.2d,v12.2s,v4.2s
|
||
|
|
umlal v23.2d,v12.2s,v1.2s
|
||
|
|
umlal v20.2d,v12.2s,v6.2s
|
||
|
|
umlal v21.2d,v12.2s,v8.2s
|
||
|
|
|
||
|
|
umlal v22.2d,v13.2s,v8.2s
|
||
|
|
umlal v19.2d,v13.2s,v2.2s
|
||
|
|
umlal v23.2d,v13.2s,v0.2s
|
||
|
|
umlal v20.2d,v13.2s,v4.2s
|
||
|
|
umlal v21.2d,v13.2s,v6.2s
|
||
|
|
|
||
|
|
.Lshort_tail:
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// horizontal add
|
||
|
|
|
||
|
|
addp v22.2d,v22.2d,v22.2d
|
||
|
|
ldp d8,d9,[sp,#16] // meet ABI requirements
|
||
|
|
addp v19.2d,v19.2d,v19.2d
|
||
|
|
ldp d10,d11,[sp,#32]
|
||
|
|
addp v23.2d,v23.2d,v23.2d
|
||
|
|
ldp d12,d13,[sp,#48]
|
||
|
|
addp v20.2d,v20.2d,v20.2d
|
||
|
|
ldp d14,d15,[sp,#64]
|
||
|
|
addp v21.2d,v21.2d,v21.2d
|
||
|
|
ldr x30,[sp,#8]
|
||
|
|
.inst 0xd50323bf // autiasp
|
||
|
|
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// lazy reduction, but without narrowing
|
||
|
|
|
||
|
|
ushr v29.2d,v22.2d,#26
|
||
|
|
and v22.16b,v22.16b,v31.16b
|
||
|
|
ushr v30.2d,v19.2d,#26
|
||
|
|
and v19.16b,v19.16b,v31.16b
|
||
|
|
|
||
|
|
add v23.2d,v23.2d,v29.2d // h3 -> h4
|
||
|
|
add v20.2d,v20.2d,v30.2d // h0 -> h1
|
||
|
|
|
||
|
|
ushr v29.2d,v23.2d,#26
|
||
|
|
and v23.16b,v23.16b,v31.16b
|
||
|
|
ushr v30.2d,v20.2d,#26
|
||
|
|
and v20.16b,v20.16b,v31.16b
|
||
|
|
add v21.2d,v21.2d,v30.2d // h1 -> h2
|
||
|
|
|
||
|
|
add v19.2d,v19.2d,v29.2d
|
||
|
|
shl v29.2d,v29.2d,#2
|
||
|
|
ushr v30.2d,v21.2d,#26
|
||
|
|
and v21.16b,v21.16b,v31.16b
|
||
|
|
add v19.2d,v19.2d,v29.2d // h4 -> h0
|
||
|
|
add v22.2d,v22.2d,v30.2d // h2 -> h3
|
||
|
|
|
||
|
|
ushr v29.2d,v19.2d,#26
|
||
|
|
and v19.16b,v19.16b,v31.16b
|
||
|
|
ushr v30.2d,v22.2d,#26
|
||
|
|
and v22.16b,v22.16b,v31.16b
|
||
|
|
add v20.2d,v20.2d,v29.2d // h0 -> h1
|
||
|
|
add v23.2d,v23.2d,v30.2d // h3 -> h4
|
||
|
|
|
||
|
|
////////////////////////////////////////////////////////////////
|
||
|
|
// write the result, can be partially reduced
|
||
|
|
|
||
|
|
st4 {v19.s,v20.s,v21.s,v22.s}[0],[x0],#16
|
||
|
|
mov x4,#1
|
||
|
|
st1 {v23.s}[0],[x0]
|
||
|
|
str x4,[x0,#8] // set is_base2_26
|
||
|
|
|
||
|
|
ldr x29,[sp],#80
|
||
|
|
ret
|
||
|
|
.size poly1305_blocks_neon,.-poly1305_blocks_neon
|
||
|
|
|
||
|
|
.align 5
|
||
|
|
.Lzeros:
|
||
|
|
.long 0,0,0,0,0,0,0,0
|
||
|
|
.asciz "Poly1305 for ARMv8, CRYPTOGAMS by @dot-asm"
|
||
|
|
.align 2
|
||
|
|
#if !defined(__KERNEL__) && !defined(_WIN64)
|
||
|
|
.comm OPENSSL_armcap_P,4,4
|
||
|
|
.hidden OPENSSL_armcap_P
|
||
|
|
#endif
|