third_party/boringssl/win-x86_64/crypto/aes/vpaes-x86_64.asm

   1 default rel
   2 %define XMMWORD
   3 %define YMMWORD
   4 %define ZMMWORD
   5 section .text code align=64
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24 ALIGN   16
  25 _vpaes_encrypt_core:
  26         mov     r9,rdx
  27         mov     r11,16
  28         mov     eax,DWORD[240+rdx]
  29         movdqa  xmm1,xmm9
  30         movdqa  xmm2,XMMWORD[$L$k_ipt]
  31         pandn   xmm1,xmm0
  32         movdqu  xmm5,XMMWORD[r9]
  33         psrld   xmm1,4
  34         pand    xmm0,xmm9
  35 DB      102,15,56,0,208
  36         movdqa  xmm0,XMMWORD[(($L$k_ipt+16))]
  37 DB      102,15,56,0,193
  38         pxor    xmm2,xmm5
  39         add     r9,16
  40         pxor    xmm0,xmm2
  41         lea     r10,[$L$k_mc_backward]
  42         jmp     NEAR $L$enc_entry
  43
  44 ALIGN   16
  45 $L$enc_loop:
  46
  47         movdqa  xmm4,xmm13
  48         movdqa  xmm0,xmm12
  49 DB      102,15,56,0,226
  50 DB      102,15,56,0,195
  51         pxor    xmm4,xmm5
  52         movdqa  xmm5,xmm15
  53         pxor    xmm0,xmm4
  54         movdqa  xmm1,XMMWORD[((-64))+r10*1+r11]
  55 DB      102,15,56,0,234
  56         movdqa  xmm4,XMMWORD[r10*1+r11]
  57         movdqa  xmm2,xmm14
  58 DB      102,15,56,0,211
  59         movdqa  xmm3,xmm0
  60         pxor    xmm2,xmm5
  61 DB      102,15,56,0,193
  62         add     r9,16
  63         pxor    xmm0,xmm2
  64 DB      102,15,56,0,220
  65         add     r11,16
  66         pxor    xmm3,xmm0
  67 DB      102,15,56,0,193
  68         and     r11,0x30
  69         sub     rax,1
  70         pxor    xmm0,xmm3
  71
  72 $L$enc_entry:
  73
  74         movdqa  xmm1,xmm9
  75         movdqa  xmm5,xmm11
  76         pandn   xmm1,xmm0
  77         psrld   xmm1,4
  78         pand    xmm0,xmm9
  79 DB      102,15,56,0,232
  80         movdqa  xmm3,xmm10
  81         pxor    xmm0,xmm1
  82 DB      102,15,56,0,217
  83         movdqa  xmm4,xmm10
  84         pxor    xmm3,xmm5
  85 DB      102,15,56,0,224
  86         movdqa  xmm2,xmm10
  87         pxor    xmm4,xmm5
  88 DB      102,15,56,0,211
  89         movdqa  xmm3,xmm10
  90         pxor    xmm2,xmm0
  91 DB      102,15,56,0,220
  92         movdqu  xmm5,XMMWORD[r9]
  93         pxor    xmm3,xmm1
  94         jnz     NEAR $L$enc_loop
  95
  96
  97         movdqa  xmm4,XMMWORD[((-96))+r10]
  98         movdqa  xmm0,XMMWORD[((-80))+r10]
  99 DB      102,15,56,0,226
 100         pxor    xmm4,xmm5
 101 DB      102,15,56,0,195
 102         movdqa  xmm1,XMMWORD[64+r10*1+r11]
 103         pxor    xmm0,xmm4
 104 DB      102,15,56,0,193
 105         DB      0F3h,0C3h               ;repret
 106
 107
 108
 109
 110
 111
 112
 113
 114 ALIGN   16
 115 _vpaes_decrypt_core:
 116         mov     r9,rdx
 117         mov     eax,DWORD[240+rdx]
 118         movdqa  xmm1,xmm9
 119         movdqa  xmm2,XMMWORD[$L$k_dipt]
 120         pandn   xmm1,xmm0
 121         mov     r11,rax
 122         psrld   xmm1,4
 123         movdqu  xmm5,XMMWORD[r9]
 124         shl     r11,4
 125         pand    xmm0,xmm9
 126 DB      102,15,56,0,208
 127         movdqa  xmm0,XMMWORD[(($L$k_dipt+16))]
 128         xor     r11,0x30
 129         lea     r10,[$L$k_dsbd]
 130 DB      102,15,56,0,193
 131         and     r11,0x30
 132         pxor    xmm2,xmm5
 133         movdqa  xmm5,XMMWORD[(($L$k_mc_forward+48))]
 134         pxor    xmm0,xmm2
 135         add     r9,16
 136         add     r11,r10
 137         jmp     NEAR $L$dec_entry
 138
 139 ALIGN   16
 140 $L$dec_loop:
 141
 142
 143
 144         movdqa  xmm4,XMMWORD[((-32))+r10]
 145         movdqa  xmm1,XMMWORD[((-16))+r10]
 146 DB      102,15,56,0,226
 147 DB      102,15,56,0,203
 148         pxor    xmm0,xmm4
 149         movdqa  xmm4,XMMWORD[r10]
 150         pxor    xmm0,xmm1
 151         movdqa  xmm1,XMMWORD[16+r10]
 152
 153 DB      102,15,56,0,226
 154 DB      102,15,56,0,197
 155 DB      102,15,56,0,203
 156         pxor    xmm0,xmm4
 157         movdqa  xmm4,XMMWORD[32+r10]
 158         pxor    xmm0,xmm1
 159         movdqa  xmm1,XMMWORD[48+r10]
 160
 161 DB      102,15,56,0,226
 162 DB      102,15,56,0,197
 163 DB      102,15,56,0,203
 164         pxor    xmm0,xmm4
 165         movdqa  xmm4,XMMWORD[64+r10]
 166         pxor    xmm0,xmm1
 167         movdqa  xmm1,XMMWORD[80+r10]
 168
 169 DB      102,15,56,0,226
 170 DB      102,15,56,0,197
 171 DB      102,15,56,0,203
 172         pxor    xmm0,xmm4
 173         add     r9,16
 174 DB      102,15,58,15,237,12
 175         pxor    xmm0,xmm1
 176         sub     rax,1
 177
 178 $L$dec_entry:
 179
 180         movdqa  xmm1,xmm9
 181         pandn   xmm1,xmm0
 182         movdqa  xmm2,xmm11
 183         psrld   xmm1,4
 184         pand    xmm0,xmm9
 185 DB      102,15,56,0,208
 186         movdqa  xmm3,xmm10
 187         pxor    xmm0,xmm1
 188 DB      102,15,56,0,217
 189         movdqa  xmm4,xmm10
 190         pxor    xmm3,xmm2
 191 DB      102,15,56,0,224
 192         pxor    xmm4,xmm2
 193         movdqa  xmm2,xmm10
 194 DB      102,15,56,0,211
 195         movdqa  xmm3,xmm10
 196         pxor    xmm2,xmm0
 197 DB      102,15,56,0,220
 198         movdqu  xmm0,XMMWORD[r9]
 199         pxor    xmm3,xmm1
 200         jnz     NEAR $L$dec_loop
 201
 202
 203         movdqa  xmm4,XMMWORD[96+r10]
 204 DB      102,15,56,0,226
 205         pxor    xmm4,xmm0
 206         movdqa  xmm0,XMMWORD[112+r10]
 207         movdqa  xmm2,XMMWORD[((-352))+r11]
 208 DB      102,15,56,0,195
 209         pxor    xmm0,xmm4
 210 DB      102,15,56,0,194
 211         DB      0F3h,0C3h               ;repret
 212
 213
 214
 215
 216
 217
 218
 219
 220 ALIGN   16
 221 _vpaes_schedule_core:
 222
 223
 224
 225
 226
 227         call    _vpaes_preheat
 228         movdqa  xmm8,XMMWORD[$L$k_rcon]
 229         movdqu  xmm0,XMMWORD[rdi]
 230
 231
 232         movdqa  xmm3,xmm0
 233         lea     r11,[$L$k_ipt]
 234         call    _vpaes_schedule_transform
 235         movdqa  xmm7,xmm0
 236
 237         lea     r10,[$L$k_sr]
 238         test    rcx,rcx
 239         jnz     NEAR $L$schedule_am_decrypting
 240
 241
 242         movdqu  XMMWORD[rdx],xmm0
 243         jmp     NEAR $L$schedule_go
 244
 245 $L$schedule_am_decrypting:
 246
 247         movdqa  xmm1,XMMWORD[r10*1+r8]
 248 DB      102,15,56,0,217
 249         movdqu  XMMWORD[rdx],xmm3
 250         xor     r8,0x30
 251
 252 $L$schedule_go:
 253         cmp     esi,192
 254         ja      NEAR $L$schedule_256
 255         je      NEAR $L$schedule_192
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266 $L$schedule_128:
 267         mov     esi,10
 268
 269 $L$oop_schedule_128:
 270         call    _vpaes_schedule_round
 271         dec     rsi
 272         jz      NEAR $L$schedule_mangle_last
 273         call    _vpaes_schedule_mangle
 274         jmp     NEAR $L$oop_schedule_128
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291 ALIGN   16
 292 $L$schedule_192:
 293         movdqu  xmm0,XMMWORD[8+rdi]
 294         call    _vpaes_schedule_transform
 295         movdqa  xmm6,xmm0
 296         pxor    xmm4,xmm4
 297         movhlps xmm6,xmm4
 298         mov     esi,4
 299
 300 $L$oop_schedule_192:
 301         call    _vpaes_schedule_round
 302 DB      102,15,58,15,198,8
 303         call    _vpaes_schedule_mangle
 304         call    _vpaes_schedule_192_smear
 305         call    _vpaes_schedule_mangle
 306         call    _vpaes_schedule_round
 307         dec     rsi
 308         jz      NEAR $L$schedule_mangle_last
 309         call    _vpaes_schedule_mangle
 310         call    _vpaes_schedule_192_smear
 311         jmp     NEAR $L$oop_schedule_192
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323 ALIGN   16
 324 $L$schedule_256:
 325         movdqu  xmm0,XMMWORD[16+rdi]
 326         call    _vpaes_schedule_transform
 327         mov     esi,7
 328
 329 $L$oop_schedule_256:
 330         call    _vpaes_schedule_mangle
 331         movdqa  xmm6,xmm0
 332
 333
 334         call    _vpaes_schedule_round
 335         dec     rsi
 336         jz      NEAR $L$schedule_mangle_last
 337         call    _vpaes_schedule_mangle
 338
 339
 340         pshufd  xmm0,xmm0,0xFF
 341         movdqa  xmm5,xmm7
 342         movdqa  xmm7,xmm6
 343         call    _vpaes_schedule_low_round
 344         movdqa  xmm7,xmm5
 345
 346         jmp     NEAR $L$oop_schedule_256
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359 ALIGN   16
 360 $L$schedule_mangle_last:
 361
 362         lea     r11,[$L$k_deskew]
 363         test    rcx,rcx
 364         jnz     NEAR $L$schedule_mangle_last_dec
 365
 366
 367         movdqa  xmm1,XMMWORD[r10*1+r8]
 368 DB      102,15,56,0,193
 369         lea     r11,[$L$k_opt]
 370         add     rdx,32
 371
 372 $L$schedule_mangle_last_dec:
 373         add     rdx,-16
 374         pxor    xmm0,XMMWORD[$L$k_s63]
 375         call    _vpaes_schedule_transform
 376         movdqu  XMMWORD[rdx],xmm0
 377
 378
 379         pxor    xmm0,xmm0
 380         pxor    xmm1,xmm1
 381         pxor    xmm2,xmm2
 382         pxor    xmm3,xmm3
 383         pxor    xmm4,xmm4
 384         pxor    xmm5,xmm5
 385         pxor    xmm6,xmm6
 386         pxor    xmm7,xmm7
 387         DB      0F3h,0C3h               ;repret
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405 ALIGN   16
 406 _vpaes_schedule_192_smear:
 407         pshufd  xmm1,xmm6,0x80
 408         pshufd  xmm0,xmm7,0xFE
 409         pxor    xmm6,xmm1
 410         pxor    xmm1,xmm1
 411         pxor    xmm6,xmm0
 412         movdqa  xmm0,xmm6
 413         movhlps xmm6,xmm1
 414         DB      0F3h,0C3h               ;repret
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436 ALIGN   16
 437 _vpaes_schedule_round:
 438
 439         pxor    xmm1,xmm1
 440 DB      102,65,15,58,15,200,15
 441 DB      102,69,15,58,15,192,15
 442         pxor    xmm7,xmm1
 443
 444
 445         pshufd  xmm0,xmm0,0xFF
 446 DB      102,15,58,15,192,1
 447
 448
 449
 450
 451 _vpaes_schedule_low_round:
 452
 453         movdqa  xmm1,xmm7
 454         pslldq  xmm7,4
 455         pxor    xmm7,xmm1
 456         movdqa  xmm1,xmm7
 457         pslldq  xmm7,8
 458         pxor    xmm7,xmm1
 459         pxor    xmm7,XMMWORD[$L$k_s63]
 460
 461
 462         movdqa  xmm1,xmm9
 463         pandn   xmm1,xmm0
 464         psrld   xmm1,4
 465         pand    xmm0,xmm9
 466         movdqa  xmm2,xmm11
 467 DB      102,15,56,0,208
 468         pxor    xmm0,xmm1
 469         movdqa  xmm3,xmm10
 470 DB      102,15,56,0,217
 471         pxor    xmm3,xmm2
 472         movdqa  xmm4,xmm10
 473 DB      102,15,56,0,224
 474         pxor    xmm4,xmm2
 475         movdqa  xmm2,xmm10
 476 DB      102,15,56,0,211
 477         pxor    xmm2,xmm0
 478         movdqa  xmm3,xmm10
 479 DB      102,15,56,0,220
 480         pxor    xmm3,xmm1
 481         movdqa  xmm4,xmm13
 482 DB      102,15,56,0,226
 483         movdqa  xmm0,xmm12
 484 DB      102,15,56,0,195
 485         pxor    xmm0,xmm4
 486
 487
 488         pxor    xmm0,xmm7
 489         movdqa  xmm7,xmm0
 490         DB      0F3h,0C3h               ;repret
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503 ALIGN   16
 504 _vpaes_schedule_transform:
 505         movdqa  xmm1,xmm9
 506         pandn   xmm1,xmm0
 507         psrld   xmm1,4
 508         pand    xmm0,xmm9
 509         movdqa  xmm2,XMMWORD[r11]
 510 DB      102,15,56,0,208
 511         movdqa  xmm0,XMMWORD[16+r11]
 512 DB      102,15,56,0,193
 513         pxor    xmm0,xmm2
 514         DB      0F3h,0C3h               ;repret
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541 ALIGN   16
 542 _vpaes_schedule_mangle:
 543         movdqa  xmm4,xmm0
 544         movdqa  xmm5,XMMWORD[$L$k_mc_forward]
 545         test    rcx,rcx
 546         jnz     NEAR $L$schedule_mangle_dec
 547
 548
 549         add     rdx,16
 550         pxor    xmm4,XMMWORD[$L$k_s63]
 551 DB      102,15,56,0,229
 552         movdqa  xmm3,xmm4
 553 DB      102,15,56,0,229
 554         pxor    xmm3,xmm4
 555 DB      102,15,56,0,229
 556         pxor    xmm3,xmm4
 557
 558         jmp     NEAR $L$schedule_mangle_both
 559 ALIGN   16
 560 $L$schedule_mangle_dec:
 561
 562         lea     r11,[$L$k_dksd]
 563         movdqa  xmm1,xmm9
 564         pandn   xmm1,xmm4
 565         psrld   xmm1,4
 566         pand    xmm4,xmm9
 567
 568         movdqa  xmm2,XMMWORD[r11]
 569 DB      102,15,56,0,212
 570         movdqa  xmm3,XMMWORD[16+r11]
 571 DB      102,15,56,0,217
 572         pxor    xmm3,xmm2
 573 DB      102,15,56,0,221
 574
 575         movdqa  xmm2,XMMWORD[32+r11]
 576 DB      102,15,56,0,212
 577         pxor    xmm2,xmm3
 578         movdqa  xmm3,XMMWORD[48+r11]
 579 DB      102,15,56,0,217
 580         pxor    xmm3,xmm2
 581 DB      102,15,56,0,221
 582
 583         movdqa  xmm2,XMMWORD[64+r11]
 584 DB      102,15,56,0,212
 585         pxor    xmm2,xmm3
 586         movdqa  xmm3,XMMWORD[80+r11]
 587 DB      102,15,56,0,217
 588         pxor    xmm3,xmm2
 589 DB      102,15,56,0,221
 590
 591         movdqa  xmm2,XMMWORD[96+r11]
 592 DB      102,15,56,0,212
 593         pxor    xmm2,xmm3
 594         movdqa  xmm3,XMMWORD[112+r11]
 595 DB      102,15,56,0,217
 596         pxor    xmm3,xmm2
 597
 598         add     rdx,-16
 599
 600 $L$schedule_mangle_both:
 601         movdqa  xmm1,XMMWORD[r10*1+r8]
 602 DB      102,15,56,0,217
 603         add     r8,-16
 604         and     r8,0x30
 605         movdqu  XMMWORD[rdx],xmm3
 606         DB      0F3h,0C3h               ;repret
 607
 608
 609
 610
 611
 612 global  vpaes_set_encrypt_key
 613
 614 ALIGN   16
 615 vpaes_set_encrypt_key:
 616         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 617         mov     QWORD[16+rsp],rsi
 618         mov     rax,rsp
 619 $L$SEH_begin_vpaes_set_encrypt_key:
 620         mov     rdi,rcx
 621         mov     rsi,rdx
 622         mov     rdx,r8
 623
 624
 625         lea     rsp,[((-184))+rsp]
 626         movaps  XMMWORD[16+rsp],xmm6
 627         movaps  XMMWORD[32+rsp],xmm7
 628         movaps  XMMWORD[48+rsp],xmm8
 629         movaps  XMMWORD[64+rsp],xmm9
 630         movaps  XMMWORD[80+rsp],xmm10
 631         movaps  XMMWORD[96+rsp],xmm11
 632         movaps  XMMWORD[112+rsp],xmm12
 633         movaps  XMMWORD[128+rsp],xmm13
 634         movaps  XMMWORD[144+rsp],xmm14
 635         movaps  XMMWORD[160+rsp],xmm15
 636 $L$enc_key_body:
 637         mov     eax,esi
 638         shr     eax,5
 639         add     eax,5
 640         mov     DWORD[240+rdx],eax
 641
 642         mov     ecx,0
 643         mov     r8d,0x30
 644         call    _vpaes_schedule_core
 645         movaps  xmm6,XMMWORD[16+rsp]
 646         movaps  xmm7,XMMWORD[32+rsp]
 647         movaps  xmm8,XMMWORD[48+rsp]
 648         movaps  xmm9,XMMWORD[64+rsp]
 649         movaps  xmm10,XMMWORD[80+rsp]
 650         movaps  xmm11,XMMWORD[96+rsp]
 651         movaps  xmm12,XMMWORD[112+rsp]
 652         movaps  xmm13,XMMWORD[128+rsp]
 653         movaps  xmm14,XMMWORD[144+rsp]
 654         movaps  xmm15,XMMWORD[160+rsp]
 655         lea     rsp,[184+rsp]
 656 $L$enc_key_epilogue:
 657         xor     eax,eax
 658         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 659         mov     rsi,QWORD[16+rsp]
 660         DB      0F3h,0C3h               ;repret
 661 $L$SEH_end_vpaes_set_encrypt_key:
 662
 663 global  vpaes_set_decrypt_key
 664
 665 ALIGN   16
 666 vpaes_set_decrypt_key:
 667         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 668         mov     QWORD[16+rsp],rsi
 669         mov     rax,rsp
 670 $L$SEH_begin_vpaes_set_decrypt_key:
 671         mov     rdi,rcx
 672         mov     rsi,rdx
 673         mov     rdx,r8
 674
 675
 676         lea     rsp,[((-184))+rsp]
 677         movaps  XMMWORD[16+rsp],xmm6
 678         movaps  XMMWORD[32+rsp],xmm7
 679         movaps  XMMWORD[48+rsp],xmm8
 680         movaps  XMMWORD[64+rsp],xmm9
 681         movaps  XMMWORD[80+rsp],xmm10
 682         movaps  XMMWORD[96+rsp],xmm11
 683         movaps  XMMWORD[112+rsp],xmm12
 684         movaps  XMMWORD[128+rsp],xmm13
 685         movaps  XMMWORD[144+rsp],xmm14
 686         movaps  XMMWORD[160+rsp],xmm15
 687 $L$dec_key_body:
 688         mov     eax,esi
 689         shr     eax,5
 690         add     eax,5
 691         mov     DWORD[240+rdx],eax
 692         shl     eax,4
 693         lea     rdx,[16+rax*1+rdx]
 694
 695         mov     ecx,1
 696         mov     r8d,esi
 697         shr     r8d,1
 698         and     r8d,32
 699         xor     r8d,32
 700         call    _vpaes_schedule_core
 701         movaps  xmm6,XMMWORD[16+rsp]
 702         movaps  xmm7,XMMWORD[32+rsp]
 703         movaps  xmm8,XMMWORD[48+rsp]
 704         movaps  xmm9,XMMWORD[64+rsp]
 705         movaps  xmm10,XMMWORD[80+rsp]
 706         movaps  xmm11,XMMWORD[96+rsp]
 707         movaps  xmm12,XMMWORD[112+rsp]
 708         movaps  xmm13,XMMWORD[128+rsp]
 709         movaps  xmm14,XMMWORD[144+rsp]
 710         movaps  xmm15,XMMWORD[160+rsp]
 711         lea     rsp,[184+rsp]
 712 $L$dec_key_epilogue:
 713         xor     eax,eax
 714         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 715         mov     rsi,QWORD[16+rsp]
 716         DB      0F3h,0C3h               ;repret
 717 $L$SEH_end_vpaes_set_decrypt_key:
 718
 719 global  vpaes_encrypt
 720
 721 ALIGN   16
 722 vpaes_encrypt:
 723         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 724         mov     QWORD[16+rsp],rsi
 725         mov     rax,rsp
 726 $L$SEH_begin_vpaes_encrypt:
 727         mov     rdi,rcx
 728         mov     rsi,rdx
 729         mov     rdx,r8
 730
 731
 732         lea     rsp,[((-184))+rsp]
 733         movaps  XMMWORD[16+rsp],xmm6
 734         movaps  XMMWORD[32+rsp],xmm7
 735         movaps  XMMWORD[48+rsp],xmm8
 736         movaps  XMMWORD[64+rsp],xmm9
 737         movaps  XMMWORD[80+rsp],xmm10
 738         movaps  XMMWORD[96+rsp],xmm11
 739         movaps  XMMWORD[112+rsp],xmm12
 740         movaps  XMMWORD[128+rsp],xmm13
 741         movaps  XMMWORD[144+rsp],xmm14
 742         movaps  XMMWORD[160+rsp],xmm15
 743 $L$enc_body:
 744         movdqu  xmm0,XMMWORD[rdi]
 745         call    _vpaes_preheat
 746         call    _vpaes_encrypt_core
 747         movdqu  XMMWORD[rsi],xmm0
 748         movaps  xmm6,XMMWORD[16+rsp]
 749         movaps  xmm7,XMMWORD[32+rsp]
 750         movaps  xmm8,XMMWORD[48+rsp]
 751         movaps  xmm9,XMMWORD[64+rsp]
 752         movaps  xmm10,XMMWORD[80+rsp]
 753         movaps  xmm11,XMMWORD[96+rsp]
 754         movaps  xmm12,XMMWORD[112+rsp]
 755         movaps  xmm13,XMMWORD[128+rsp]
 756         movaps  xmm14,XMMWORD[144+rsp]
 757         movaps  xmm15,XMMWORD[160+rsp]
 758         lea     rsp,[184+rsp]
 759 $L$enc_epilogue:
 760         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 761         mov     rsi,QWORD[16+rsp]
 762         DB      0F3h,0C3h               ;repret
 763 $L$SEH_end_vpaes_encrypt:
 764
 765 global  vpaes_decrypt
 766
 767 ALIGN   16
 768 vpaes_decrypt:
 769         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 770         mov     QWORD[16+rsp],rsi
 771         mov     rax,rsp
 772 $L$SEH_begin_vpaes_decrypt:
 773         mov     rdi,rcx
 774         mov     rsi,rdx
 775         mov     rdx,r8
 776
 777
 778         lea     rsp,[((-184))+rsp]
 779         movaps  XMMWORD[16+rsp],xmm6
 780         movaps  XMMWORD[32+rsp],xmm7
 781         movaps  XMMWORD[48+rsp],xmm8
 782         movaps  XMMWORD[64+rsp],xmm9
 783         movaps  XMMWORD[80+rsp],xmm10
 784         movaps  XMMWORD[96+rsp],xmm11
 785         movaps  XMMWORD[112+rsp],xmm12
 786         movaps  XMMWORD[128+rsp],xmm13
 787         movaps  XMMWORD[144+rsp],xmm14
 788         movaps  XMMWORD[160+rsp],xmm15
 789 $L$dec_body:
 790         movdqu  xmm0,XMMWORD[rdi]
 791         call    _vpaes_preheat
 792         call    _vpaes_decrypt_core
 793         movdqu  XMMWORD[rsi],xmm0
 794         movaps  xmm6,XMMWORD[16+rsp]
 795         movaps  xmm7,XMMWORD[32+rsp]
 796         movaps  xmm8,XMMWORD[48+rsp]
 797         movaps  xmm9,XMMWORD[64+rsp]
 798         movaps  xmm10,XMMWORD[80+rsp]
 799         movaps  xmm11,XMMWORD[96+rsp]
 800         movaps  xmm12,XMMWORD[112+rsp]
 801         movaps  xmm13,XMMWORD[128+rsp]
 802         movaps  xmm14,XMMWORD[144+rsp]
 803         movaps  xmm15,XMMWORD[160+rsp]
 804         lea     rsp,[184+rsp]
 805 $L$dec_epilogue:
 806         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 807         mov     rsi,QWORD[16+rsp]
 808         DB      0F3h,0C3h               ;repret
 809 $L$SEH_end_vpaes_decrypt:
 810 global  vpaes_cbc_encrypt
 811
 812 ALIGN   16
 813 vpaes_cbc_encrypt:
 814         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 815         mov     QWORD[16+rsp],rsi
 816         mov     rax,rsp
 817 $L$SEH_begin_vpaes_cbc_encrypt:
 818         mov     rdi,rcx
 819         mov     rsi,rdx
 820         mov     rdx,r8
 821         mov     rcx,r9
 822         mov     r8,QWORD[40+rsp]
 823         mov     r9,QWORD[48+rsp]
 824
 825
 826         xchg    rdx,rcx
 827         sub     rcx,16
 828         jc      NEAR $L$cbc_abort
 829         lea     rsp,[((-184))+rsp]
 830         movaps  XMMWORD[16+rsp],xmm6
 831         movaps  XMMWORD[32+rsp],xmm7
 832         movaps  XMMWORD[48+rsp],xmm8
 833         movaps  XMMWORD[64+rsp],xmm9
 834         movaps  XMMWORD[80+rsp],xmm10
 835         movaps  XMMWORD[96+rsp],xmm11
 836         movaps  XMMWORD[112+rsp],xmm12
 837         movaps  XMMWORD[128+rsp],xmm13
 838         movaps  XMMWORD[144+rsp],xmm14
 839         movaps  XMMWORD[160+rsp],xmm15
 840 $L$cbc_body:
 841         movdqu  xmm6,XMMWORD[r8]
 842         sub     rsi,rdi
 843         call    _vpaes_preheat
 844         cmp     r9d,0
 845         je      NEAR $L$cbc_dec_loop
 846         jmp     NEAR $L$cbc_enc_loop
 847 ALIGN   16
 848 $L$cbc_enc_loop:
 849         movdqu  xmm0,XMMWORD[rdi]
 850         pxor    xmm0,xmm6
 851         call    _vpaes_encrypt_core
 852         movdqa  xmm6,xmm0
 853         movdqu  XMMWORD[rdi*1+rsi],xmm0
 854         lea     rdi,[16+rdi]
 855         sub     rcx,16
 856         jnc     NEAR $L$cbc_enc_loop
 857         jmp     NEAR $L$cbc_done
 858 ALIGN   16
 859 $L$cbc_dec_loop:
 860         movdqu  xmm0,XMMWORD[rdi]
 861         movdqa  xmm7,xmm0
 862         call    _vpaes_decrypt_core
 863         pxor    xmm0,xmm6
 864         movdqa  xmm6,xmm7
 865         movdqu  XMMWORD[rdi*1+rsi],xmm0
 866         lea     rdi,[16+rdi]
 867         sub     rcx,16
 868         jnc     NEAR $L$cbc_dec_loop
 869 $L$cbc_done:
 870         movdqu  XMMWORD[r8],xmm6
 871         movaps  xmm6,XMMWORD[16+rsp]
 872         movaps  xmm7,XMMWORD[32+rsp]
 873         movaps  xmm8,XMMWORD[48+rsp]
 874         movaps  xmm9,XMMWORD[64+rsp]
 875         movaps  xmm10,XMMWORD[80+rsp]
 876         movaps  xmm11,XMMWORD[96+rsp]
 877         movaps  xmm12,XMMWORD[112+rsp]
 878         movaps  xmm13,XMMWORD[128+rsp]
 879         movaps  xmm14,XMMWORD[144+rsp]
 880         movaps  xmm15,XMMWORD[160+rsp]
 881         lea     rsp,[184+rsp]
 882 $L$cbc_epilogue:
 883 $L$cbc_abort:
 884         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 885         mov     rsi,QWORD[16+rsp]
 886         DB      0F3h,0C3h               ;repret
 887 $L$SEH_end_vpaes_cbc_encrypt:
 888
 889
 890
 891
 892
 893
 894
 895 ALIGN   16
 896 _vpaes_preheat:
 897         lea     r10,[$L$k_s0F]
 898         movdqa  xmm10,XMMWORD[((-32))+r10]
 899         movdqa  xmm11,XMMWORD[((-16))+r10]
 900         movdqa  xmm9,XMMWORD[r10]
 901         movdqa  xmm13,XMMWORD[48+r10]
 902         movdqa  xmm12,XMMWORD[64+r10]
 903         movdqa  xmm15,XMMWORD[80+r10]
 904         movdqa  xmm14,XMMWORD[96+r10]
 905         DB      0F3h,0C3h               ;repret
 906
 907
 908
 909
 910
 911
 912
 913 ALIGN   64
 914 _vpaes_consts:
 915 $L$k_inv:
 916         DQ      0x0E05060F0D080180,0x040703090A0B0C02
 917         DQ      0x01040A060F0B0780,0x030D0E0C02050809
 918
 919 $L$k_s0F:
 920         DQ      0x0F0F0F0F0F0F0F0F,0x0F0F0F0F0F0F0F0F
 921
 922 $L$k_ipt:
 923         DQ      0xC2B2E8985A2A7000,0xCABAE09052227808
 924         DQ      0x4C01307D317C4D00,0xCD80B1FCB0FDCC81
 925
 926 $L$k_sb1:
 927         DQ      0xB19BE18FCB503E00,0xA5DF7A6E142AF544
 928         DQ      0x3618D415FAE22300,0x3BF7CCC10D2ED9EF
 929 $L$k_sb2:
 930         DQ      0xE27A93C60B712400,0x5EB7E955BC982FCD
 931         DQ      0x69EB88400AE12900,0xC2A163C8AB82234A
 932 $L$k_sbo:
 933         DQ      0xD0D26D176FBDC700,0x15AABF7AC502A878
 934         DQ      0xCFE474A55FBB6A00,0x8E1E90D1412B35FA
 935
 936 $L$k_mc_forward:
 937         DQ      0x0407060500030201,0x0C0F0E0D080B0A09
 938         DQ      0x080B0A0904070605,0x000302010C0F0E0D
 939         DQ      0x0C0F0E0D080B0A09,0x0407060500030201
 940         DQ      0x000302010C0F0E0D,0x080B0A0904070605
 941
 942 $L$k_mc_backward:
 943         DQ      0x0605040702010003,0x0E0D0C0F0A09080B
 944         DQ      0x020100030E0D0C0F,0x0A09080B06050407
 945         DQ      0x0E0D0C0F0A09080B,0x0605040702010003
 946         DQ      0x0A09080B06050407,0x020100030E0D0C0F
 947
 948 $L$k_sr:
 949         DQ      0x0706050403020100,0x0F0E0D0C0B0A0908
 950         DQ      0x030E09040F0A0500,0x0B06010C07020D08
 951         DQ      0x0F060D040B020900,0x070E050C030A0108
 952         DQ      0x0B0E0104070A0D00,0x0306090C0F020508
 953
 954 $L$k_rcon:
 955         DQ      0x1F8391B9AF9DEEB6,0x702A98084D7C7D81
 956
 957 $L$k_s63:
 958         DQ      0x5B5B5B5B5B5B5B5B,0x5B5B5B5B5B5B5B5B
 959
 960 $L$k_opt:
 961         DQ      0xFF9F4929D6B66000,0xF7974121DEBE6808
 962         DQ      0x01EDBD5150BCEC00,0xE10D5DB1B05C0CE0
 963
 964 $L$k_deskew:
 965         DQ      0x07E4A34047A4E300,0x1DFEB95A5DBEF91A
 966         DQ      0x5F36B5DC83EA6900,0x2841C2ABF49D1E77
 967
 968
 969
 970
 971
 972 $L$k_dksd:
 973         DQ      0xFEB91A5DA3E44700,0x0740E3A45A1DBEF9
 974         DQ      0x41C277F4B5368300,0x5FDC69EAAB289D1E
 975 $L$k_dksb:
 976         DQ      0x9A4FCA1F8550D500,0x03D653861CC94C99
 977         DQ      0x115BEDA7B6FC4A00,0xD993256F7E3482C8
 978 $L$k_dkse:
 979         DQ      0xD5031CCA1FC9D600,0x53859A4C994F5086
 980         DQ      0xA23196054FDC7BE8,0xCD5EF96A20B31487
 981 $L$k_dks9:
 982         DQ      0xB6116FC87ED9A700,0x4AED933482255BFC
 983         DQ      0x4576516227143300,0x8BB89FACE9DAFDCE
 984
 985
 986
 987
 988
 989 $L$k_dipt:
 990         DQ      0x0F505B040B545F00,0x154A411E114E451A
 991         DQ      0x86E383E660056500,0x12771772F491F194
 992
 993 $L$k_dsb9:
 994         DQ      0x851C03539A86D600,0xCAD51F504F994CC9
 995         DQ      0xC03B1789ECD74900,0x725E2C9EB2FBA565
 996 $L$k_dsbd:
 997         DQ      0x7D57CCDFE6B1A200,0xF56E9B13882A4439
 998         DQ      0x3CE2FAF724C6CB00,0x2931180D15DEEFD3
 999 $L$k_dsbb:
1000         DQ      0xD022649296B44200,0x602646F6B0F2D404
1001         DQ      0xC19498A6CD596700,0xF3FF0C3E3255AA6B
1002 $L$k_dsbe:
1003         DQ      0x46F2929626D4D000,0x2242600464B4F6B0
1004         DQ      0x0C55A6CDFFAAC100,0x9467F36B98593E32
1005 $L$k_dsbo:
1006         DQ      0x1387EA537EF94000,0xC7AA6DB9D4943E2D
1007         DQ      0x12D7560F93441D00,0xCA4B8159D8C58E9C
1008 DB      86,101,99,116,111,114,32,80,101,114,109,117,116,97,116,105
1009 DB      111,110,32,65,69,83,32,102,111,114,32,120,56,54,95,54
1010 DB      52,47,83,83,83,69,51,44,32,77,105,107,101,32,72,97
1011 DB      109,98,117,114,103,32,40,83,116,97,110,102,111,114,100,32
1012 DB      85,110,105,118,101,114,115,105,116,121,41,0
1013 ALIGN   64
1014
1015 EXTERN  __imp_RtlVirtualUnwind
1016
1017 ALIGN   16
1018 se_handler:
1019         push    rsi
1020         push    rdi
1021         push    rbx
1022         push    rbp
1023         push    r12
1024         push    r13
1025         push    r14
1026         push    r15
1027         pushfq
1028         sub     rsp,64
1029
1030         mov     rax,QWORD[120+r8]
1031         mov     rbx,QWORD[248+r8]
1032
1033         mov     rsi,QWORD[8+r9]
1034         mov     r11,QWORD[56+r9]
1035
1036         mov     r10d,DWORD[r11]
1037         lea     r10,[r10*1+rsi]
1038         cmp     rbx,r10
1039         jb      NEAR $L$in_prologue
1040
1041         mov     rax,QWORD[152+r8]
1042
1043         mov     r10d,DWORD[4+r11]
1044         lea     r10,[r10*1+rsi]
1045         cmp     rbx,r10
1046         jae     NEAR $L$in_prologue
1047
1048         lea     rsi,[16+rax]
1049         lea     rdi,[512+r8]
1050         mov     ecx,20
1051         DD      0xa548f3fc
1052         lea     rax,[184+rax]
1053
1054 $L$in_prologue:
1055         mov     rdi,QWORD[8+rax]
1056         mov     rsi,QWORD[16+rax]
1057         mov     QWORD[152+r8],rax
1058         mov     QWORD[168+r8],rsi
1059         mov     QWORD[176+r8],rdi
1060
1061         mov     rdi,QWORD[40+r9]
1062         mov     rsi,r8
1063         mov     ecx,154
1064         DD      0xa548f3fc
1065
1066         mov     rsi,r9
1067         xor     rcx,rcx
1068         mov     rdx,QWORD[8+rsi]
1069         mov     r8,QWORD[rsi]
1070         mov     r9,QWORD[16+rsi]
1071         mov     r10,QWORD[40+rsi]
1072         lea     r11,[56+rsi]
1073         lea     r12,[24+rsi]
1074         mov     QWORD[32+rsp],r10
1075         mov     QWORD[40+rsp],r11
1076         mov     QWORD[48+rsp],r12
1077         mov     QWORD[56+rsp],rcx
1078         call    QWORD[__imp_RtlVirtualUnwind]
1079
1080         mov     eax,1
1081         add     rsp,64
1082         popfq
1083         pop     r15
1084         pop     r14
1085         pop     r13
1086         pop     r12
1087         pop     rbp
1088         pop     rbx
1089         pop     rdi
1090         pop     rsi
1091         DB      0F3h,0C3h               ;repret
1092
1093
1094 section .pdata rdata align=4
1095 ALIGN   4
1096         DD      $L$SEH_begin_vpaes_set_encrypt_key wrt ..imagebase
1097         DD      $L$SEH_end_vpaes_set_encrypt_key wrt ..imagebase
1098         DD      $L$SEH_info_vpaes_set_encrypt_key wrt ..imagebase
1099
1100         DD      $L$SEH_begin_vpaes_set_decrypt_key wrt ..imagebase
1101         DD      $L$SEH_end_vpaes_set_decrypt_key wrt ..imagebase
1102         DD      $L$SEH_info_vpaes_set_decrypt_key wrt ..imagebase
1103
1104         DD      $L$SEH_begin_vpaes_encrypt wrt ..imagebase
1105         DD      $L$SEH_end_vpaes_encrypt wrt ..imagebase
1106         DD      $L$SEH_info_vpaes_encrypt wrt ..imagebase
1107
1108         DD      $L$SEH_begin_vpaes_decrypt wrt ..imagebase
1109         DD      $L$SEH_end_vpaes_decrypt wrt ..imagebase
1110         DD      $L$SEH_info_vpaes_decrypt wrt ..imagebase
1111
1112         DD      $L$SEH_begin_vpaes_cbc_encrypt wrt ..imagebase
1113         DD      $L$SEH_end_vpaes_cbc_encrypt wrt ..imagebase
1114         DD      $L$SEH_info_vpaes_cbc_encrypt wrt ..imagebase
1115
1116 section .xdata rdata align=8
1117 ALIGN   8
1118 $L$SEH_info_vpaes_set_encrypt_key:
1119 DB      9,0,0,0
1120         DD      se_handler wrt ..imagebase
1121         DD      $L$enc_key_body wrt ..imagebase,$L$enc_key_epilogue wrt ..imagebase
1122 $L$SEH_info_vpaes_set_decrypt_key:
1123 DB      9,0,0,0
1124         DD      se_handler wrt ..imagebase
1125         DD      $L$dec_key_body wrt ..imagebase,$L$dec_key_epilogue wrt ..imagebase
1126 $L$SEH_info_vpaes_encrypt:
1127 DB      9,0,0,0
1128         DD      se_handler wrt ..imagebase
1129         DD      $L$enc_body wrt ..imagebase,$L$enc_epilogue wrt ..imagebase
1130 $L$SEH_info_vpaes_decrypt:
1131 DB      9,0,0,0
1132         DD      se_handler wrt ..imagebase
1133         DD      $L$dec_body wrt ..imagebase,$L$dec_epilogue wrt ..imagebase
1134 $L$SEH_info_vpaes_cbc_encrypt:
1135 DB      9,0,0,0
1136         DD      se_handler wrt ..imagebase
1137         DD      $L$cbc_body wrt ..imagebase,$L$cbc_epilogue wrt ..imagebase