third_party/boringssl/linux-arm/crypto/modes/ghashv8-armx.S

   1 #include "arm_arch.h"
   2
   3 .text
   4 .fpu    neon
   5 .code   32
   6 .global gcm_init_v8
   7 .type   gcm_init_v8,%function
   8 .align  4
   9 gcm_init_v8:
  10         vld1.64         {q9},[r1]               @ load H
  11         vmov.i8         q8,#0xe1
  12         vext.8          q3,q9,q9,#8
  13         vshl.i64        q8,q8,#57
  14         vshr.u64        q10,q8,#63
  15         vext.8          q8,q10,q8,#8            @ t0=0xc2....01
  16         vdup.32 q9,d18[1]
  17         vshr.u64        q11,q3,#63
  18         vshr.s32        q9,q9,#31               @ broadcast carry bit
  19         vand            q11,q11,q8
  20         vshl.i64        q3,q3,#1
  21         vext.8          q11,q11,q11,#8
  22         vand            q8,q8,q9
  23         vorr            q3,q3,q11               @ H<<<=1
  24         veor            q3,q3,q8                @ twisted H
  25         vst1.64         {q3},[r0]
  26
  27         bx      lr
  28 .size   gcm_init_v8,.-gcm_init_v8
  29
  30 .global gcm_gmult_v8
  31 .type   gcm_gmult_v8,%function
  32 .align  4
  33 gcm_gmult_v8:
  34         vld1.64         {q9},[r0]               @ load Xi
  35         vmov.i8         q11,#0xe1
  36         vld1.64         {q12},[r1]              @ load twisted H
  37         vshl.u64        q11,q11,#57
  38 #ifndef __ARMEB__
  39         vrev64.8        q9,q9
  40 #endif
  41         vext.8          q13,q12,q12,#8
  42         mov             r3,#0
  43         vext.8          q3,q9,q9,#8
  44         mov             r12,#0
  45         veor            q13,q13,q12             @ Karatsuba pre-processing
  46         mov             r2,r0
  47         b               .Lgmult_v8
  48 .size   gcm_gmult_v8,.-gcm_gmult_v8
  49
  50 .global gcm_ghash_v8
  51 .type   gcm_ghash_v8,%function
  52 .align  4
  53 gcm_ghash_v8:
  54         vld1.64         {q0},[r0]               @ load [rotated] Xi
  55         subs            r3,r3,#16
  56         vmov.i8         q11,#0xe1
  57         mov             r12,#16
  58         vld1.64         {q12},[r1]              @ load twisted H
  59         moveq   r12,#0
  60         vext.8          q0,q0,q0,#8
  61         vshl.u64        q11,q11,#57
  62         vld1.64         {q9},[r2],r12   @ load [rotated] inp
  63         vext.8          q13,q12,q12,#8
  64 #ifndef __ARMEB__
  65         vrev64.8        q0,q0
  66         vrev64.8        q9,q9
  67 #endif
  68         veor            q13,q13,q12             @ Karatsuba pre-processing
  69         vext.8          q3,q9,q9,#8
  70         b               .Loop_v8
  71
  72 .align  4
  73 .Loop_v8:
  74         vext.8          q10,q0,q0,#8
  75         veor            q3,q3,q0                @ inp^=Xi
  76         veor            q9,q9,q10               @ q9 is rotated inp^Xi
  77
  78 .Lgmult_v8:
  79         .byte   0x86,0x0e,0xa8,0xf2     @ pmull q0,q12,q3               @ H.lo·Xi.lo
  80         veor            q9,q9,q3                @ Karatsuba pre-processing
  81         .byte   0x87,0x4e,0xa9,0xf2     @ pmull2 q2,q12,q3              @ H.hi·Xi.hi
  82         subs            r3,r3,#16
  83         .byte   0xa2,0x2e,0xaa,0xf2     @ pmull q1,q13,q9               @ (H.lo+H.hi)·(Xi.lo+Xi.hi)
  84         moveq   r12,#0
  85
  86         vext.8          q9,q0,q2,#8             @ Karatsuba post-processing
  87         veor            q10,q0,q2
  88         veor            q1,q1,q9
  89          vld1.64        {q9},[r2],r12   @ load [rotated] inp
  90         veor            q1,q1,q10
  91         .byte   0x26,0x4e,0xe0,0xf2     @ pmull q10,q0,q11              @ 1st phase
  92
  93         vmov            d4,d3           @ Xh|Xm - 256-bit result
  94         vmov            d3,d0           @ Xm is rotated Xl
  95 #ifndef __ARMEB__
  96          vrev64.8       q9,q9
  97 #endif
  98         veor            q0,q1,q10
  99          vext.8         q3,q9,q9,#8
 100
 101         vext.8          q10,q0,q0,#8            @ 2nd phase
 102         .byte   0x26,0x0e,0xa0,0xf2     @ pmull q0,q0,q11
 103         veor            q10,q10,q2
 104         veor            q0,q0,q10
 105         bhs             .Loop_v8
 106
 107 #ifndef __ARMEB__
 108         vrev64.8        q0,q0
 109 #endif
 110         vext.8          q0,q0,q0,#8
 111         vst1.64         {q0},[r0]               @ write out Xi
 112
 113         bx      lr
 114 .size   gcm_ghash_v8,.-gcm_ghash_v8
 115 .asciz  "GHASH for ARMv8, CRYPTOGAMS by <appro@openssl.org>"
 116 .align  2