module/icp/asm-x86_64/sha2/sha512-x86_64.S

   1 /*
   2  * Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     https://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * Portions Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
  19  * - modified assembly to fit into OpenZFS
  20  */
  21
  22 #if defined(__x86_64)
  23
  24 #define _ASM
  25 #include <sys/asm_linkage.h>
  26
  27 SECTION_STATIC
  28
  29 .balign 64
  30 SET_OBJ(K512)
  31 K512:
  32 .quad   0x428a2f98d728ae22,0x7137449123ef65cd
  33 .quad   0x428a2f98d728ae22,0x7137449123ef65cd
  34 .quad   0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
  35 .quad   0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
  36 .quad   0x3956c25bf348b538,0x59f111f1b605d019
  37 .quad   0x3956c25bf348b538,0x59f111f1b605d019
  38 .quad   0x923f82a4af194f9b,0xab1c5ed5da6d8118
  39 .quad   0x923f82a4af194f9b,0xab1c5ed5da6d8118
  40 .quad   0xd807aa98a3030242,0x12835b0145706fbe
  41 .quad   0xd807aa98a3030242,0x12835b0145706fbe
  42 .quad   0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
  43 .quad   0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
  44 .quad   0x72be5d74f27b896f,0x80deb1fe3b1696b1
  45 .quad   0x72be5d74f27b896f,0x80deb1fe3b1696b1
  46 .quad   0x9bdc06a725c71235,0xc19bf174cf692694
  47 .quad   0x9bdc06a725c71235,0xc19bf174cf692694
  48 .quad   0xe49b69c19ef14ad2,0xefbe4786384f25e3
  49 .quad   0xe49b69c19ef14ad2,0xefbe4786384f25e3
  50 .quad   0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
  51 .quad   0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
  52 .quad   0x2de92c6f592b0275,0x4a7484aa6ea6e483
  53 .quad   0x2de92c6f592b0275,0x4a7484aa6ea6e483
  54 .quad   0x5cb0a9dcbd41fbd4,0x76f988da831153b5
  55 .quad   0x5cb0a9dcbd41fbd4,0x76f988da831153b5
  56 .quad   0x983e5152ee66dfab,0xa831c66d2db43210
  57 .quad   0x983e5152ee66dfab,0xa831c66d2db43210
  58 .quad   0xb00327c898fb213f,0xbf597fc7beef0ee4
  59 .quad   0xb00327c898fb213f,0xbf597fc7beef0ee4
  60 .quad   0xc6e00bf33da88fc2,0xd5a79147930aa725
  61 .quad   0xc6e00bf33da88fc2,0xd5a79147930aa725
  62 .quad   0x06ca6351e003826f,0x142929670a0e6e70
  63 .quad   0x06ca6351e003826f,0x142929670a0e6e70
  64 .quad   0x27b70a8546d22ffc,0x2e1b21385c26c926
  65 .quad   0x27b70a8546d22ffc,0x2e1b21385c26c926
  66 .quad   0x4d2c6dfc5ac42aed,0x53380d139d95b3df
  67 .quad   0x4d2c6dfc5ac42aed,0x53380d139d95b3df
  68 .quad   0x650a73548baf63de,0x766a0abb3c77b2a8
  69 .quad   0x650a73548baf63de,0x766a0abb3c77b2a8
  70 .quad   0x81c2c92e47edaee6,0x92722c851482353b
  71 .quad   0x81c2c92e47edaee6,0x92722c851482353b
  72 .quad   0xa2bfe8a14cf10364,0xa81a664bbc423001
  73 .quad   0xa2bfe8a14cf10364,0xa81a664bbc423001
  74 .quad   0xc24b8b70d0f89791,0xc76c51a30654be30
  75 .quad   0xc24b8b70d0f89791,0xc76c51a30654be30
  76 .quad   0xd192e819d6ef5218,0xd69906245565a910
  77 .quad   0xd192e819d6ef5218,0xd69906245565a910
  78 .quad   0xf40e35855771202a,0x106aa07032bbd1b8
  79 .quad   0xf40e35855771202a,0x106aa07032bbd1b8
  80 .quad   0x19a4c116b8d2d0c8,0x1e376c085141ab53
  81 .quad   0x19a4c116b8d2d0c8,0x1e376c085141ab53
  82 .quad   0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
  83 .quad   0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
  84 .quad   0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
  85 .quad   0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
  86 .quad   0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
  87 .quad   0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
  88 .quad   0x748f82ee5defb2fc,0x78a5636f43172f60
  89 .quad   0x748f82ee5defb2fc,0x78a5636f43172f60
  90 .quad   0x84c87814a1f0ab72,0x8cc702081a6439ec
  91 .quad   0x84c87814a1f0ab72,0x8cc702081a6439ec
  92 .quad   0x90befffa23631e28,0xa4506cebde82bde9
  93 .quad   0x90befffa23631e28,0xa4506cebde82bde9
  94 .quad   0xbef9a3f7b2c67915,0xc67178f2e372532b
  95 .quad   0xbef9a3f7b2c67915,0xc67178f2e372532b
  96 .quad   0xca273eceea26619c,0xd186b8c721c0c207
  97 .quad   0xca273eceea26619c,0xd186b8c721c0c207
  98 .quad   0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
  99 .quad   0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 100 .quad   0x06f067aa72176fba,0x0a637dc5a2c898a6
 101 .quad   0x06f067aa72176fba,0x0a637dc5a2c898a6
 102 .quad   0x113f9804bef90dae,0x1b710b35131c471b
 103 .quad   0x113f9804bef90dae,0x1b710b35131c471b
 104 .quad   0x28db77f523047d84,0x32caab7b40c72493
 105 .quad   0x28db77f523047d84,0x32caab7b40c72493
 106 .quad   0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 107 .quad   0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 108 .quad   0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 109 .quad   0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 110 .quad   0x5fcb6fab3ad6faec,0x6c44198c4a475817
 111 .quad   0x5fcb6fab3ad6faec,0x6c44198c4a475817
 112 .quad   0x0001020304050607,0x08090a0b0c0d0e0f
 113 .quad   0x0001020304050607,0x08090a0b0c0d0e0f
 114
 115 ENTRY_ALIGN(zfs_sha512_transform_x64, 16)
 116 .cfi_startproc
 117         ENDBR
 118         movq    %rsp,%rax
 119 .cfi_def_cfa_register   %rax
 120         pushq   %rbx
 121 .cfi_offset     %rbx,-16
 122         pushq   %rbp
 123 .cfi_offset     %rbp,-24
 124         pushq   %r12
 125 .cfi_offset     %r12,-32
 126         pushq   %r13
 127 .cfi_offset     %r13,-40
 128         pushq   %r14
 129 .cfi_offset     %r14,-48
 130         pushq   %r15
 131 .cfi_offset     %r15,-56
 132         shlq    $4,%rdx
 133         subq    $128+32,%rsp
 134         leaq    (%rsi,%rdx,8),%rdx
 135         andq    $-64,%rsp
 136         movq    %rdi,128+0(%rsp)
 137         movq    %rsi,128+8(%rsp)
 138         movq    %rdx,128+16(%rsp)
 139         movq    %rax,152(%rsp)
 140 .cfi_escape     0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
 141 .Lprologue:
 142         movq    0(%rdi),%rax
 143         movq    8(%rdi),%rbx
 144         movq    16(%rdi),%rcx
 145         movq    24(%rdi),%rdx
 146         movq    32(%rdi),%r8
 147         movq    40(%rdi),%r9
 148         movq    48(%rdi),%r10
 149         movq    56(%rdi),%r11
 150         jmp     .Lloop
 151 .balign 16
 152 .Lloop:
 153         movq    %rbx,%rdi
 154         leaq    K512(%rip),%rbp
 155         xorq    %rcx,%rdi
 156         movq    0(%rsi),%r12
 157         movq    %r8,%r13
 158         movq    %rax,%r14
 159         bswapq  %r12
 160         rorq    $23,%r13
 161         movq    %r9,%r15
 162         xorq    %r8,%r13
 163         rorq    $5,%r14
 164         xorq    %r10,%r15
 165         movq    %r12,0(%rsp)
 166         xorq    %rax,%r14
 167         andq    %r8,%r15
 168         rorq    $4,%r13
 169         addq    %r11,%r12
 170         xorq    %r10,%r15
 171         rorq    $6,%r14
 172         xorq    %r8,%r13
 173         addq    %r15,%r12
 174         movq    %rax,%r15
 175         addq    (%rbp),%r12
 176         xorq    %rax,%r14
 177         xorq    %rbx,%r15
 178         rorq    $14,%r13
 179         movq    %rbx,%r11
 180         andq    %r15,%rdi
 181         rorq    $28,%r14
 182         addq    %r13,%r12
 183         xorq    %rdi,%r11
 184         addq    %r12,%rdx
 185         addq    %r12,%r11
 186         leaq    8(%rbp),%rbp
 187         addq    %r14,%r11
 188         movq    8(%rsi),%r12
 189         movq    %rdx,%r13
 190         movq    %r11,%r14
 191         bswapq  %r12
 192         rorq    $23,%r13
 193         movq    %r8,%rdi
 194         xorq    %rdx,%r13
 195         rorq    $5,%r14
 196         xorq    %r9,%rdi
 197         movq    %r12,8(%rsp)
 198         xorq    %r11,%r14
 199         andq    %rdx,%rdi
 200         rorq    $4,%r13
 201         addq    %r10,%r12
 202         xorq    %r9,%rdi
 203         rorq    $6,%r14
 204         xorq    %rdx,%r13
 205         addq    %rdi,%r12
 206         movq    %r11,%rdi
 207         addq    (%rbp),%r12
 208         xorq    %r11,%r14
 209         xorq    %rax,%rdi
 210         rorq    $14,%r13
 211         movq    %rax,%r10
 212         andq    %rdi,%r15
 213         rorq    $28,%r14
 214         addq    %r13,%r12
 215         xorq    %r15,%r10
 216         addq    %r12,%rcx
 217         addq    %r12,%r10
 218         leaq    24(%rbp),%rbp
 219         addq    %r14,%r10
 220         movq    16(%rsi),%r12
 221         movq    %rcx,%r13
 222         movq    %r10,%r14
 223         bswapq  %r12
 224         rorq    $23,%r13
 225         movq    %rdx,%r15
 226         xorq    %rcx,%r13
 227         rorq    $5,%r14
 228         xorq    %r8,%r15
 229         movq    %r12,16(%rsp)
 230         xorq    %r10,%r14
 231         andq    %rcx,%r15
 232         rorq    $4,%r13
 233         addq    %r9,%r12
 234         xorq    %r8,%r15
 235         rorq    $6,%r14
 236         xorq    %rcx,%r13
 237         addq    %r15,%r12
 238         movq    %r10,%r15
 239         addq    (%rbp),%r12
 240         xorq    %r10,%r14
 241         xorq    %r11,%r15
 242         rorq    $14,%r13
 243         movq    %r11,%r9
 244         andq    %r15,%rdi
 245         rorq    $28,%r14
 246         addq    %r13,%r12
 247         xorq    %rdi,%r9
 248         addq    %r12,%rbx
 249         addq    %r12,%r9
 250         leaq    8(%rbp),%rbp
 251         addq    %r14,%r9
 252         movq    24(%rsi),%r12
 253         movq    %rbx,%r13
 254         movq    %r9,%r14
 255         bswapq  %r12
 256         rorq    $23,%r13
 257         movq    %rcx,%rdi
 258         xorq    %rbx,%r13
 259         rorq    $5,%r14
 260         xorq    %rdx,%rdi
 261         movq    %r12,24(%rsp)
 262         xorq    %r9,%r14
 263         andq    %rbx,%rdi
 264         rorq    $4,%r13
 265         addq    %r8,%r12
 266         xorq    %rdx,%rdi
 267         rorq    $6,%r14
 268         xorq    %rbx,%r13
 269         addq    %rdi,%r12
 270         movq    %r9,%rdi
 271         addq    (%rbp),%r12
 272         xorq    %r9,%r14
 273         xorq    %r10,%rdi
 274         rorq    $14,%r13
 275         movq    %r10,%r8
 276         andq    %rdi,%r15
 277         rorq    $28,%r14
 278         addq    %r13,%r12
 279         xorq    %r15,%r8
 280         addq    %r12,%rax
 281         addq    %r12,%r8
 282         leaq    24(%rbp),%rbp
 283         addq    %r14,%r8
 284         movq    32(%rsi),%r12
 285         movq    %rax,%r13
 286         movq    %r8,%r14
 287         bswapq  %r12
 288         rorq    $23,%r13
 289         movq    %rbx,%r15
 290         xorq    %rax,%r13
 291         rorq    $5,%r14
 292         xorq    %rcx,%r15
 293         movq    %r12,32(%rsp)
 294         xorq    %r8,%r14
 295         andq    %rax,%r15
 296         rorq    $4,%r13
 297         addq    %rdx,%r12
 298         xorq    %rcx,%r15
 299         rorq    $6,%r14
 300         xorq    %rax,%r13
 301         addq    %r15,%r12
 302         movq    %r8,%r15
 303         addq    (%rbp),%r12
 304         xorq    %r8,%r14
 305         xorq    %r9,%r15
 306         rorq    $14,%r13
 307         movq    %r9,%rdx
 308         andq    %r15,%rdi
 309         rorq    $28,%r14
 310         addq    %r13,%r12
 311         xorq    %rdi,%rdx
 312         addq    %r12,%r11
 313         addq    %r12,%rdx
 314         leaq    8(%rbp),%rbp
 315         addq    %r14,%rdx
 316         movq    40(%rsi),%r12
 317         movq    %r11,%r13
 318         movq    %rdx,%r14
 319         bswapq  %r12
 320         rorq    $23,%r13
 321         movq    %rax,%rdi
 322         xorq    %r11,%r13
 323         rorq    $5,%r14
 324         xorq    %rbx,%rdi
 325         movq    %r12,40(%rsp)
 326         xorq    %rdx,%r14
 327         andq    %r11,%rdi
 328         rorq    $4,%r13
 329         addq    %rcx,%r12
 330         xorq    %rbx,%rdi
 331         rorq    $6,%r14
 332         xorq    %r11,%r13
 333         addq    %rdi,%r12
 334         movq    %rdx,%rdi
 335         addq    (%rbp),%r12
 336         xorq    %rdx,%r14
 337         xorq    %r8,%rdi
 338         rorq    $14,%r13
 339         movq    %r8,%rcx
 340         andq    %rdi,%r15
 341         rorq    $28,%r14
 342         addq    %r13,%r12
 343         xorq    %r15,%rcx
 344         addq    %r12,%r10
 345         addq    %r12,%rcx
 346         leaq    24(%rbp),%rbp
 347         addq    %r14,%rcx
 348         movq    48(%rsi),%r12
 349         movq    %r10,%r13
 350         movq    %rcx,%r14
 351         bswapq  %r12
 352         rorq    $23,%r13
 353         movq    %r11,%r15
 354         xorq    %r10,%r13
 355         rorq    $5,%r14
 356         xorq    %rax,%r15
 357         movq    %r12,48(%rsp)
 358         xorq    %rcx,%r14
 359         andq    %r10,%r15
 360         rorq    $4,%r13
 361         addq    %rbx,%r12
 362         xorq    %rax,%r15
 363         rorq    $6,%r14
 364         xorq    %r10,%r13
 365         addq    %r15,%r12
 366         movq    %rcx,%r15
 367         addq    (%rbp),%r12
 368         xorq    %rcx,%r14
 369         xorq    %rdx,%r15
 370         rorq    $14,%r13
 371         movq    %rdx,%rbx
 372         andq    %r15,%rdi
 373         rorq    $28,%r14
 374         addq    %r13,%r12
 375         xorq    %rdi,%rbx
 376         addq    %r12,%r9
 377         addq    %r12,%rbx
 378         leaq    8(%rbp),%rbp
 379         addq    %r14,%rbx
 380         movq    56(%rsi),%r12
 381         movq    %r9,%r13
 382         movq    %rbx,%r14
 383         bswapq  %r12
 384         rorq    $23,%r13
 385         movq    %r10,%rdi
 386         xorq    %r9,%r13
 387         rorq    $5,%r14
 388         xorq    %r11,%rdi
 389         movq    %r12,56(%rsp)
 390         xorq    %rbx,%r14
 391         andq    %r9,%rdi
 392         rorq    $4,%r13
 393         addq    %rax,%r12
 394         xorq    %r11,%rdi
 395         rorq    $6,%r14
 396         xorq    %r9,%r13
 397         addq    %rdi,%r12
 398         movq    %rbx,%rdi
 399         addq    (%rbp),%r12
 400         xorq    %rbx,%r14
 401         xorq    %rcx,%rdi
 402         rorq    $14,%r13
 403         movq    %rcx,%rax
 404         andq    %rdi,%r15
 405         rorq    $28,%r14
 406         addq    %r13,%r12
 407         xorq    %r15,%rax
 408         addq    %r12,%r8
 409         addq    %r12,%rax
 410         leaq    24(%rbp),%rbp
 411         addq    %r14,%rax
 412         movq    64(%rsi),%r12
 413         movq    %r8,%r13
 414         movq    %rax,%r14
 415         bswapq  %r12
 416         rorq    $23,%r13
 417         movq    %r9,%r15
 418         xorq    %r8,%r13
 419         rorq    $5,%r14
 420         xorq    %r10,%r15
 421         movq    %r12,64(%rsp)
 422         xorq    %rax,%r14
 423         andq    %r8,%r15
 424         rorq    $4,%r13
 425         addq    %r11,%r12
 426         xorq    %r10,%r15
 427         rorq    $6,%r14
 428         xorq    %r8,%r13
 429         addq    %r15,%r12
 430         movq    %rax,%r15
 431         addq    (%rbp),%r12
 432         xorq    %rax,%r14
 433         xorq    %rbx,%r15
 434         rorq    $14,%r13
 435         movq    %rbx,%r11
 436         andq    %r15,%rdi
 437         rorq    $28,%r14
 438         addq    %r13,%r12
 439         xorq    %rdi,%r11
 440         addq    %r12,%rdx
 441         addq    %r12,%r11
 442         leaq    8(%rbp),%rbp
 443         addq    %r14,%r11
 444         movq    72(%rsi),%r12
 445         movq    %rdx,%r13
 446         movq    %r11,%r14
 447         bswapq  %r12
 448         rorq    $23,%r13
 449         movq    %r8,%rdi
 450         xorq    %rdx,%r13
 451         rorq    $5,%r14
 452         xorq    %r9,%rdi
 453         movq    %r12,72(%rsp)
 454         xorq    %r11,%r14
 455         andq    %rdx,%rdi
 456         rorq    $4,%r13
 457         addq    %r10,%r12
 458         xorq    %r9,%rdi
 459         rorq    $6,%r14
 460         xorq    %rdx,%r13
 461         addq    %rdi,%r12
 462         movq    %r11,%rdi
 463         addq    (%rbp),%r12
 464         xorq    %r11,%r14
 465         xorq    %rax,%rdi
 466         rorq    $14,%r13
 467         movq    %rax,%r10
 468         andq    %rdi,%r15
 469         rorq    $28,%r14
 470         addq    %r13,%r12
 471         xorq    %r15,%r10
 472         addq    %r12,%rcx
 473         addq    %r12,%r10
 474         leaq    24(%rbp),%rbp
 475         addq    %r14,%r10
 476         movq    80(%rsi),%r12
 477         movq    %rcx,%r13
 478         movq    %r10,%r14
 479         bswapq  %r12
 480         rorq    $23,%r13
 481         movq    %rdx,%r15
 482         xorq    %rcx,%r13
 483         rorq    $5,%r14
 484         xorq    %r8,%r15
 485         movq    %r12,80(%rsp)
 486         xorq    %r10,%r14
 487         andq    %rcx,%r15
 488         rorq    $4,%r13
 489         addq    %r9,%r12
 490         xorq    %r8,%r15
 491         rorq    $6,%r14
 492         xorq    %rcx,%r13
 493         addq    %r15,%r12
 494         movq    %r10,%r15
 495         addq    (%rbp),%r12
 496         xorq    %r10,%r14
 497         xorq    %r11,%r15
 498         rorq    $14,%r13
 499         movq    %r11,%r9
 500         andq    %r15,%rdi
 501         rorq    $28,%r14
 502         addq    %r13,%r12
 503         xorq    %rdi,%r9
 504         addq    %r12,%rbx
 505         addq    %r12,%r9
 506         leaq    8(%rbp),%rbp
 507         addq    %r14,%r9
 508         movq    88(%rsi),%r12
 509         movq    %rbx,%r13
 510         movq    %r9,%r14
 511         bswapq  %r12
 512         rorq    $23,%r13
 513         movq    %rcx,%rdi
 514         xorq    %rbx,%r13
 515         rorq    $5,%r14
 516         xorq    %rdx,%rdi
 517         movq    %r12,88(%rsp)
 518         xorq    %r9,%r14
 519         andq    %rbx,%rdi
 520         rorq    $4,%r13
 521         addq    %r8,%r12
 522         xorq    %rdx,%rdi
 523         rorq    $6,%r14
 524         xorq    %rbx,%r13
 525         addq    %rdi,%r12
 526         movq    %r9,%rdi
 527         addq    (%rbp),%r12
 528         xorq    %r9,%r14
 529         xorq    %r10,%rdi
 530         rorq    $14,%r13
 531         movq    %r10,%r8
 532         andq    %rdi,%r15
 533         rorq    $28,%r14
 534         addq    %r13,%r12
 535         xorq    %r15,%r8
 536         addq    %r12,%rax
 537         addq    %r12,%r8
 538         leaq    24(%rbp),%rbp
 539         addq    %r14,%r8
 540         movq    96(%rsi),%r12
 541         movq    %rax,%r13
 542         movq    %r8,%r14
 543         bswapq  %r12
 544         rorq    $23,%r13
 545         movq    %rbx,%r15
 546         xorq    %rax,%r13
 547         rorq    $5,%r14
 548         xorq    %rcx,%r15
 549         movq    %r12,96(%rsp)
 550         xorq    %r8,%r14
 551         andq    %rax,%r15
 552         rorq    $4,%r13
 553         addq    %rdx,%r12
 554         xorq    %rcx,%r15
 555         rorq    $6,%r14
 556         xorq    %rax,%r13
 557         addq    %r15,%r12
 558         movq    %r8,%r15
 559         addq    (%rbp),%r12
 560         xorq    %r8,%r14
 561         xorq    %r9,%r15
 562         rorq    $14,%r13
 563         movq    %r9,%rdx
 564         andq    %r15,%rdi
 565         rorq    $28,%r14
 566         addq    %r13,%r12
 567         xorq    %rdi,%rdx
 568         addq    %r12,%r11
 569         addq    %r12,%rdx
 570         leaq    8(%rbp),%rbp
 571         addq    %r14,%rdx
 572         movq    104(%rsi),%r12
 573         movq    %r11,%r13
 574         movq    %rdx,%r14
 575         bswapq  %r12
 576         rorq    $23,%r13
 577         movq    %rax,%rdi
 578         xorq    %r11,%r13
 579         rorq    $5,%r14
 580         xorq    %rbx,%rdi
 581         movq    %r12,104(%rsp)
 582         xorq    %rdx,%r14
 583         andq    %r11,%rdi
 584         rorq    $4,%r13
 585         addq    %rcx,%r12
 586         xorq    %rbx,%rdi
 587         rorq    $6,%r14
 588         xorq    %r11,%r13
 589         addq    %rdi,%r12
 590         movq    %rdx,%rdi
 591         addq    (%rbp),%r12
 592         xorq    %rdx,%r14
 593         xorq    %r8,%rdi
 594         rorq    $14,%r13
 595         movq    %r8,%rcx
 596         andq    %rdi,%r15
 597         rorq    $28,%r14
 598         addq    %r13,%r12
 599         xorq    %r15,%rcx
 600         addq    %r12,%r10
 601         addq    %r12,%rcx
 602         leaq    24(%rbp),%rbp
 603         addq    %r14,%rcx
 604         movq    112(%rsi),%r12
 605         movq    %r10,%r13
 606         movq    %rcx,%r14
 607         bswapq  %r12
 608         rorq    $23,%r13
 609         movq    %r11,%r15
 610         xorq    %r10,%r13
 611         rorq    $5,%r14
 612         xorq    %rax,%r15
 613         movq    %r12,112(%rsp)
 614         xorq    %rcx,%r14
 615         andq    %r10,%r15
 616         rorq    $4,%r13
 617         addq    %rbx,%r12
 618         xorq    %rax,%r15
 619         rorq    $6,%r14
 620         xorq    %r10,%r13
 621         addq    %r15,%r12
 622         movq    %rcx,%r15
 623         addq    (%rbp),%r12
 624         xorq    %rcx,%r14
 625         xorq    %rdx,%r15
 626         rorq    $14,%r13
 627         movq    %rdx,%rbx
 628         andq    %r15,%rdi
 629         rorq    $28,%r14
 630         addq    %r13,%r12
 631         xorq    %rdi,%rbx
 632         addq    %r12,%r9
 633         addq    %r12,%rbx
 634         leaq    8(%rbp),%rbp
 635         addq    %r14,%rbx
 636         movq    120(%rsi),%r12
 637         movq    %r9,%r13
 638         movq    %rbx,%r14
 639         bswapq  %r12
 640         rorq    $23,%r13
 641         movq    %r10,%rdi
 642         xorq    %r9,%r13
 643         rorq    $5,%r14
 644         xorq    %r11,%rdi
 645         movq    %r12,120(%rsp)
 646         xorq    %rbx,%r14
 647         andq    %r9,%rdi
 648         rorq    $4,%r13
 649         addq    %rax,%r12
 650         xorq    %r11,%rdi
 651         rorq    $6,%r14
 652         xorq    %r9,%r13
 653         addq    %rdi,%r12
 654         movq    %rbx,%rdi
 655         addq    (%rbp),%r12
 656         xorq    %rbx,%r14
 657         xorq    %rcx,%rdi
 658         rorq    $14,%r13
 659         movq    %rcx,%rax
 660         andq    %rdi,%r15
 661         rorq    $28,%r14
 662         addq    %r13,%r12
 663         xorq    %r15,%rax
 664         addq    %r12,%r8
 665         addq    %r12,%rax
 666         leaq    24(%rbp),%rbp
 667         jmp     .Lrounds_16_xx
 668 .balign 16
 669 .Lrounds_16_xx:
 670         movq    8(%rsp),%r13
 671         movq    112(%rsp),%r15
 672         movq    %r13,%r12
 673         rorq    $7,%r13
 674         addq    %r14,%rax
 675         movq    %r15,%r14
 676         rorq    $42,%r15
 677         xorq    %r12,%r13
 678         shrq    $7,%r12
 679         rorq    $1,%r13
 680         xorq    %r14,%r15
 681         shrq    $6,%r14
 682         rorq    $19,%r15
 683         xorq    %r13,%r12
 684         xorq    %r14,%r15
 685         addq    72(%rsp),%r12
 686         addq    0(%rsp),%r12
 687         movq    %r8,%r13
 688         addq    %r15,%r12
 689         movq    %rax,%r14
 690         rorq    $23,%r13
 691         movq    %r9,%r15
 692         xorq    %r8,%r13
 693         rorq    $5,%r14
 694         xorq    %r10,%r15
 695         movq    %r12,0(%rsp)
 696         xorq    %rax,%r14
 697         andq    %r8,%r15
 698         rorq    $4,%r13
 699         addq    %r11,%r12
 700         xorq    %r10,%r15
 701         rorq    $6,%r14
 702         xorq    %r8,%r13
 703         addq    %r15,%r12
 704         movq    %rax,%r15
 705         addq    (%rbp),%r12
 706         xorq    %rax,%r14
 707         xorq    %rbx,%r15
 708         rorq    $14,%r13
 709         movq    %rbx,%r11
 710         andq    %r15,%rdi
 711         rorq    $28,%r14
 712         addq    %r13,%r12
 713         xorq    %rdi,%r11
 714         addq    %r12,%rdx
 715         addq    %r12,%r11
 716         leaq    8(%rbp),%rbp
 717         movq    16(%rsp),%r13
 718         movq    120(%rsp),%rdi
 719         movq    %r13,%r12
 720         rorq    $7,%r13
 721         addq    %r14,%r11
 722         movq    %rdi,%r14
 723         rorq    $42,%rdi
 724         xorq    %r12,%r13
 725         shrq    $7,%r12
 726         rorq    $1,%r13
 727         xorq    %r14,%rdi
 728         shrq    $6,%r14
 729         rorq    $19,%rdi
 730         xorq    %r13,%r12
 731         xorq    %r14,%rdi
 732         addq    80(%rsp),%r12
 733         addq    8(%rsp),%r12
 734         movq    %rdx,%r13
 735         addq    %rdi,%r12
 736         movq    %r11,%r14
 737         rorq    $23,%r13
 738         movq    %r8,%rdi
 739         xorq    %rdx,%r13
 740         rorq    $5,%r14
 741         xorq    %r9,%rdi
 742         movq    %r12,8(%rsp)
 743         xorq    %r11,%r14
 744         andq    %rdx,%rdi
 745         rorq    $4,%r13
 746         addq    %r10,%r12
 747         xorq    %r9,%rdi
 748         rorq    $6,%r14
 749         xorq    %rdx,%r13
 750         addq    %rdi,%r12
 751         movq    %r11,%rdi
 752         addq    (%rbp),%r12
 753         xorq    %r11,%r14
 754         xorq    %rax,%rdi
 755         rorq    $14,%r13
 756         movq    %rax,%r10
 757         andq    %rdi,%r15
 758         rorq    $28,%r14
 759         addq    %r13,%r12
 760         xorq    %r15,%r10
 761         addq    %r12,%rcx
 762         addq    %r12,%r10
 763         leaq    24(%rbp),%rbp
 764         movq    24(%rsp),%r13
 765         movq    0(%rsp),%r15
 766         movq    %r13,%r12
 767         rorq    $7,%r13
 768         addq    %r14,%r10
 769         movq    %r15,%r14
 770         rorq    $42,%r15
 771         xorq    %r12,%r13
 772         shrq    $7,%r12
 773         rorq    $1,%r13
 774         xorq    %r14,%r15
 775         shrq    $6,%r14
 776         rorq    $19,%r15
 777         xorq    %r13,%r12
 778         xorq    %r14,%r15
 779         addq    88(%rsp),%r12
 780         addq    16(%rsp),%r12
 781         movq    %rcx,%r13
 782         addq    %r15,%r12
 783         movq    %r10,%r14
 784         rorq    $23,%r13
 785         movq    %rdx,%r15
 786         xorq    %rcx,%r13
 787         rorq    $5,%r14
 788         xorq    %r8,%r15
 789         movq    %r12,16(%rsp)
 790         xorq    %r10,%r14
 791         andq    %rcx,%r15
 792         rorq    $4,%r13
 793         addq    %r9,%r12
 794         xorq    %r8,%r15
 795         rorq    $6,%r14
 796         xorq    %rcx,%r13
 797         addq    %r15,%r12
 798         movq    %r10,%r15
 799         addq    (%rbp),%r12
 800         xorq    %r10,%r14
 801         xorq    %r11,%r15
 802         rorq    $14,%r13
 803         movq    %r11,%r9
 804         andq    %r15,%rdi
 805         rorq    $28,%r14
 806         addq    %r13,%r12
 807         xorq    %rdi,%r9
 808         addq    %r12,%rbx
 809         addq    %r12,%r9
 810         leaq    8(%rbp),%rbp
 811         movq    32(%rsp),%r13
 812         movq    8(%rsp),%rdi
 813         movq    %r13,%r12
 814         rorq    $7,%r13
 815         addq    %r14,%r9
 816         movq    %rdi,%r14
 817         rorq    $42,%rdi
 818         xorq    %r12,%r13
 819         shrq    $7,%r12
 820         rorq    $1,%r13
 821         xorq    %r14,%rdi
 822         shrq    $6,%r14
 823         rorq    $19,%rdi
 824         xorq    %r13,%r12
 825         xorq    %r14,%rdi
 826         addq    96(%rsp),%r12
 827         addq    24(%rsp),%r12
 828         movq    %rbx,%r13
 829         addq    %rdi,%r12
 830         movq    %r9,%r14
 831         rorq    $23,%r13
 832         movq    %rcx,%rdi
 833         xorq    %rbx,%r13
 834         rorq    $5,%r14
 835         xorq    %rdx,%rdi
 836         movq    %r12,24(%rsp)
 837         xorq    %r9,%r14
 838         andq    %rbx,%rdi
 839         rorq    $4,%r13
 840         addq    %r8,%r12
 841         xorq    %rdx,%rdi
 842         rorq    $6,%r14
 843         xorq    %rbx,%r13
 844         addq    %rdi,%r12
 845         movq    %r9,%rdi
 846         addq    (%rbp),%r12
 847         xorq    %r9,%r14
 848         xorq    %r10,%rdi
 849         rorq    $14,%r13
 850         movq    %r10,%r8
 851         andq    %rdi,%r15
 852         rorq    $28,%r14
 853         addq    %r13,%r12
 854         xorq    %r15,%r8
 855         addq    %r12,%rax
 856         addq    %r12,%r8
 857         leaq    24(%rbp),%rbp
 858         movq    40(%rsp),%r13
 859         movq    16(%rsp),%r15
 860         movq    %r13,%r12
 861         rorq    $7,%r13
 862         addq    %r14,%r8
 863         movq    %r15,%r14
 864         rorq    $42,%r15
 865         xorq    %r12,%r13
 866         shrq    $7,%r12
 867         rorq    $1,%r13
 868         xorq    %r14,%r15
 869         shrq    $6,%r14
 870         rorq    $19,%r15
 871         xorq    %r13,%r12
 872         xorq    %r14,%r15
 873         addq    104(%rsp),%r12
 874         addq    32(%rsp),%r12
 875         movq    %rax,%r13
 876         addq    %r15,%r12
 877         movq    %r8,%r14
 878         rorq    $23,%r13
 879         movq    %rbx,%r15
 880         xorq    %rax,%r13
 881         rorq    $5,%r14
 882         xorq    %rcx,%r15
 883         movq    %r12,32(%rsp)
 884         xorq    %r8,%r14
 885         andq    %rax,%r15
 886         rorq    $4,%r13
 887         addq    %rdx,%r12
 888         xorq    %rcx,%r15
 889         rorq    $6,%r14
 890         xorq    %rax,%r13
 891         addq    %r15,%r12
 892         movq    %r8,%r15
 893         addq    (%rbp),%r12
 894         xorq    %r8,%r14
 895         xorq    %r9,%r15
 896         rorq    $14,%r13
 897         movq    %r9,%rdx
 898         andq    %r15,%rdi
 899         rorq    $28,%r14
 900         addq    %r13,%r12
 901         xorq    %rdi,%rdx
 902         addq    %r12,%r11
 903         addq    %r12,%rdx
 904         leaq    8(%rbp),%rbp
 905         movq    48(%rsp),%r13
 906         movq    24(%rsp),%rdi
 907         movq    %r13,%r12
 908         rorq    $7,%r13
 909         addq    %r14,%rdx
 910         movq    %rdi,%r14
 911         rorq    $42,%rdi
 912         xorq    %r12,%r13
 913         shrq    $7,%r12
 914         rorq    $1,%r13
 915         xorq    %r14,%rdi
 916         shrq    $6,%r14
 917         rorq    $19,%rdi
 918         xorq    %r13,%r12
 919         xorq    %r14,%rdi
 920         addq    112(%rsp),%r12
 921         addq    40(%rsp),%r12
 922         movq    %r11,%r13
 923         addq    %rdi,%r12
 924         movq    %rdx,%r14
 925         rorq    $23,%r13
 926         movq    %rax,%rdi
 927         xorq    %r11,%r13
 928         rorq    $5,%r14
 929         xorq    %rbx,%rdi
 930         movq    %r12,40(%rsp)
 931         xorq    %rdx,%r14
 932         andq    %r11,%rdi
 933         rorq    $4,%r13
 934         addq    %rcx,%r12
 935         xorq    %rbx,%rdi
 936         rorq    $6,%r14
 937         xorq    %r11,%r13
 938         addq    %rdi,%r12
 939         movq    %rdx,%rdi
 940         addq    (%rbp),%r12
 941         xorq    %rdx,%r14
 942         xorq    %r8,%rdi
 943         rorq    $14,%r13
 944         movq    %r8,%rcx
 945         andq    %rdi,%r15
 946         rorq    $28,%r14
 947         addq    %r13,%r12
 948         xorq    %r15,%rcx
 949         addq    %r12,%r10
 950         addq    %r12,%rcx
 951         leaq    24(%rbp),%rbp
 952         movq    56(%rsp),%r13
 953         movq    32(%rsp),%r15
 954         movq    %r13,%r12
 955         rorq    $7,%r13
 956         addq    %r14,%rcx
 957         movq    %r15,%r14
 958         rorq    $42,%r15
 959         xorq    %r12,%r13
 960         shrq    $7,%r12
 961         rorq    $1,%r13
 962         xorq    %r14,%r15
 963         shrq    $6,%r14
 964         rorq    $19,%r15
 965         xorq    %r13,%r12
 966         xorq    %r14,%r15
 967         addq    120(%rsp),%r12
 968         addq    48(%rsp),%r12
 969         movq    %r10,%r13
 970         addq    %r15,%r12
 971         movq    %rcx,%r14
 972         rorq    $23,%r13
 973         movq    %r11,%r15
 974         xorq    %r10,%r13
 975         rorq    $5,%r14
 976         xorq    %rax,%r15
 977         movq    %r12,48(%rsp)
 978         xorq    %rcx,%r14
 979         andq    %r10,%r15
 980         rorq    $4,%r13
 981         addq    %rbx,%r12
 982         xorq    %rax,%r15
 983         rorq    $6,%r14
 984         xorq    %r10,%r13
 985         addq    %r15,%r12
 986         movq    %rcx,%r15
 987         addq    (%rbp),%r12
 988         xorq    %rcx,%r14
 989         xorq    %rdx,%r15
 990         rorq    $14,%r13
 991         movq    %rdx,%rbx
 992         andq    %r15,%rdi
 993         rorq    $28,%r14
 994         addq    %r13,%r12
 995         xorq    %rdi,%rbx
 996         addq    %r12,%r9
 997         addq    %r12,%rbx
 998         leaq    8(%rbp),%rbp
 999         movq    64(%rsp),%r13
1000         movq    40(%rsp),%rdi
1001         movq    %r13,%r12
1002         rorq    $7,%r13
1003         addq    %r14,%rbx
1004         movq    %rdi,%r14
1005         rorq    $42,%rdi
1006         xorq    %r12,%r13
1007         shrq    $7,%r12
1008         rorq    $1,%r13
1009         xorq    %r14,%rdi
1010         shrq    $6,%r14
1011         rorq    $19,%rdi
1012         xorq    %r13,%r12
1013         xorq    %r14,%rdi
1014         addq    0(%rsp),%r12
1015         addq    56(%rsp),%r12
1016         movq    %r9,%r13
1017         addq    %rdi,%r12
1018         movq    %rbx,%r14
1019         rorq    $23,%r13
1020         movq    %r10,%rdi
1021         xorq    %r9,%r13
1022         rorq    $5,%r14
1023         xorq    %r11,%rdi
1024         movq    %r12,56(%rsp)
1025         xorq    %rbx,%r14
1026         andq    %r9,%rdi
1027         rorq    $4,%r13
1028         addq    %rax,%r12
1029         xorq    %r11,%rdi
1030         rorq    $6,%r14
1031         xorq    %r9,%r13
1032         addq    %rdi,%r12
1033         movq    %rbx,%rdi
1034         addq    (%rbp),%r12
1035         xorq    %rbx,%r14
1036         xorq    %rcx,%rdi
1037         rorq    $14,%r13
1038         movq    %rcx,%rax
1039         andq    %rdi,%r15
1040         rorq    $28,%r14
1041         addq    %r13,%r12
1042         xorq    %r15,%rax
1043         addq    %r12,%r8
1044         addq    %r12,%rax
1045         leaq    24(%rbp),%rbp
1046         movq    72(%rsp),%r13
1047         movq    48(%rsp),%r15
1048         movq    %r13,%r12
1049         rorq    $7,%r13
1050         addq    %r14,%rax
1051         movq    %r15,%r14
1052         rorq    $42,%r15
1053         xorq    %r12,%r13
1054         shrq    $7,%r12
1055         rorq    $1,%r13
1056         xorq    %r14,%r15
1057         shrq    $6,%r14
1058         rorq    $19,%r15
1059         xorq    %r13,%r12
1060         xorq    %r14,%r15
1061         addq    8(%rsp),%r12
1062         addq    64(%rsp),%r12
1063         movq    %r8,%r13
1064         addq    %r15,%r12
1065         movq    %rax,%r14
1066         rorq    $23,%r13
1067         movq    %r9,%r15
1068         xorq    %r8,%r13
1069         rorq    $5,%r14
1070         xorq    %r10,%r15
1071         movq    %r12,64(%rsp)
1072         xorq    %rax,%r14
1073         andq    %r8,%r15
1074         rorq    $4,%r13
1075         addq    %r11,%r12
1076         xorq    %r10,%r15
1077         rorq    $6,%r14
1078         xorq    %r8,%r13
1079         addq    %r15,%r12
1080         movq    %rax,%r15
1081         addq    (%rbp),%r12
1082         xorq    %rax,%r14
1083         xorq    %rbx,%r15
1084         rorq    $14,%r13
1085         movq    %rbx,%r11
1086         andq    %r15,%rdi
1087         rorq    $28,%r14
1088         addq    %r13,%r12
1089         xorq    %rdi,%r11
1090         addq    %r12,%rdx
1091         addq    %r12,%r11
1092         leaq    8(%rbp),%rbp
1093         movq    80(%rsp),%r13
1094         movq    56(%rsp),%rdi
1095         movq    %r13,%r12
1096         rorq    $7,%r13
1097         addq    %r14,%r11
1098         movq    %rdi,%r14
1099         rorq    $42,%rdi
1100         xorq    %r12,%r13
1101         shrq    $7,%r12
1102         rorq    $1,%r13
1103         xorq    %r14,%rdi
1104         shrq    $6,%r14
1105         rorq    $19,%rdi
1106         xorq    %r13,%r12
1107         xorq    %r14,%rdi
1108         addq    16(%rsp),%r12
1109         addq    72(%rsp),%r12
1110         movq    %rdx,%r13
1111         addq    %rdi,%r12
1112         movq    %r11,%r14
1113         rorq    $23,%r13
1114         movq    %r8,%rdi
1115         xorq    %rdx,%r13
1116         rorq    $5,%r14
1117         xorq    %r9,%rdi
1118         movq    %r12,72(%rsp)
1119         xorq    %r11,%r14
1120         andq    %rdx,%rdi
1121         rorq    $4,%r13
1122         addq    %r10,%r12
1123         xorq    %r9,%rdi
1124         rorq    $6,%r14
1125         xorq    %rdx,%r13
1126         addq    %rdi,%r12
1127         movq    %r11,%rdi
1128         addq    (%rbp),%r12
1129         xorq    %r11,%r14
1130         xorq    %rax,%rdi
1131         rorq    $14,%r13
1132         movq    %rax,%r10
1133         andq    %rdi,%r15
1134         rorq    $28,%r14
1135         addq    %r13,%r12
1136         xorq    %r15,%r10
1137         addq    %r12,%rcx
1138         addq    %r12,%r10
1139         leaq    24(%rbp),%rbp
1140         movq    88(%rsp),%r13
1141         movq    64(%rsp),%r15
1142         movq    %r13,%r12
1143         rorq    $7,%r13
1144         addq    %r14,%r10
1145         movq    %r15,%r14
1146         rorq    $42,%r15
1147         xorq    %r12,%r13
1148         shrq    $7,%r12
1149         rorq    $1,%r13
1150         xorq    %r14,%r15
1151         shrq    $6,%r14
1152         rorq    $19,%r15
1153         xorq    %r13,%r12
1154         xorq    %r14,%r15
1155         addq    24(%rsp),%r12
1156         addq    80(%rsp),%r12
1157         movq    %rcx,%r13
1158         addq    %r15,%r12
1159         movq    %r10,%r14
1160         rorq    $23,%r13
1161         movq    %rdx,%r15
1162         xorq    %rcx,%r13
1163         rorq    $5,%r14
1164         xorq    %r8,%r15
1165         movq    %r12,80(%rsp)
1166         xorq    %r10,%r14
1167         andq    %rcx,%r15
1168         rorq    $4,%r13
1169         addq    %r9,%r12
1170         xorq    %r8,%r15
1171         rorq    $6,%r14
1172         xorq    %rcx,%r13
1173         addq    %r15,%r12
1174         movq    %r10,%r15
1175         addq    (%rbp),%r12
1176         xorq    %r10,%r14
1177         xorq    %r11,%r15
1178         rorq    $14,%r13
1179         movq    %r11,%r9
1180         andq    %r15,%rdi
1181         rorq    $28,%r14
1182         addq    %r13,%r12
1183         xorq    %rdi,%r9
1184         addq    %r12,%rbx
1185         addq    %r12,%r9
1186         leaq    8(%rbp),%rbp
1187         movq    96(%rsp),%r13
1188         movq    72(%rsp),%rdi
1189         movq    %r13,%r12
1190         rorq    $7,%r13
1191         addq    %r14,%r9
1192         movq    %rdi,%r14
1193         rorq    $42,%rdi
1194         xorq    %r12,%r13
1195         shrq    $7,%r12
1196         rorq    $1,%r13
1197         xorq    %r14,%rdi
1198         shrq    $6,%r14
1199         rorq    $19,%rdi
1200         xorq    %r13,%r12
1201         xorq    %r14,%rdi
1202         addq    32(%rsp),%r12
1203         addq    88(%rsp),%r12
1204         movq    %rbx,%r13
1205         addq    %rdi,%r12
1206         movq    %r9,%r14
1207         rorq    $23,%r13
1208         movq    %rcx,%rdi
1209         xorq    %rbx,%r13
1210         rorq    $5,%r14
1211         xorq    %rdx,%rdi
1212         movq    %r12,88(%rsp)
1213         xorq    %r9,%r14
1214         andq    %rbx,%rdi
1215         rorq    $4,%r13
1216         addq    %r8,%r12
1217         xorq    %rdx,%rdi
1218         rorq    $6,%r14
1219         xorq    %rbx,%r13
1220         addq    %rdi,%r12
1221         movq    %r9,%rdi
1222         addq    (%rbp),%r12
1223         xorq    %r9,%r14
1224         xorq    %r10,%rdi
1225         rorq    $14,%r13
1226         movq    %r10,%r8
1227         andq    %rdi,%r15
1228         rorq    $28,%r14
1229         addq    %r13,%r12
1230         xorq    %r15,%r8
1231         addq    %r12,%rax
1232         addq    %r12,%r8
1233         leaq    24(%rbp),%rbp
1234         movq    104(%rsp),%r13
1235         movq    80(%rsp),%r15
1236         movq    %r13,%r12
1237         rorq    $7,%r13
1238         addq    %r14,%r8
1239         movq    %r15,%r14
1240         rorq    $42,%r15
1241         xorq    %r12,%r13
1242         shrq    $7,%r12
1243         rorq    $1,%r13
1244         xorq    %r14,%r15
1245         shrq    $6,%r14
1246         rorq    $19,%r15
1247         xorq    %r13,%r12
1248         xorq    %r14,%r15
1249         addq    40(%rsp),%r12
1250         addq    96(%rsp),%r12
1251         movq    %rax,%r13
1252         addq    %r15,%r12
1253         movq    %r8,%r14
1254         rorq    $23,%r13
1255         movq    %rbx,%r15
1256         xorq    %rax,%r13
1257         rorq    $5,%r14
1258         xorq    %rcx,%r15
1259         movq    %r12,96(%rsp)
1260         xorq    %r8,%r14
1261         andq    %rax,%r15
1262         rorq    $4,%r13
1263         addq    %rdx,%r12
1264         xorq    %rcx,%r15
1265         rorq    $6,%r14
1266         xorq    %rax,%r13
1267         addq    %r15,%r12
1268         movq    %r8,%r15
1269         addq    (%rbp),%r12
1270         xorq    %r8,%r14
1271         xorq    %r9,%r15
1272         rorq    $14,%r13
1273         movq    %r9,%rdx
1274         andq    %r15,%rdi
1275         rorq    $28,%r14
1276         addq    %r13,%r12
1277         xorq    %rdi,%rdx
1278         addq    %r12,%r11
1279         addq    %r12,%rdx
1280         leaq    8(%rbp),%rbp
1281         movq    112(%rsp),%r13
1282         movq    88(%rsp),%rdi
1283         movq    %r13,%r12
1284         rorq    $7,%r13
1285         addq    %r14,%rdx
1286         movq    %rdi,%r14
1287         rorq    $42,%rdi
1288         xorq    %r12,%r13
1289         shrq    $7,%r12
1290         rorq    $1,%r13
1291         xorq    %r14,%rdi
1292         shrq    $6,%r14
1293         rorq    $19,%rdi
1294         xorq    %r13,%r12
1295         xorq    %r14,%rdi
1296         addq    48(%rsp),%r12
1297         addq    104(%rsp),%r12
1298         movq    %r11,%r13
1299         addq    %rdi,%r12
1300         movq    %rdx,%r14
1301         rorq    $23,%r13
1302         movq    %rax,%rdi
1303         xorq    %r11,%r13
1304         rorq    $5,%r14
1305         xorq    %rbx,%rdi
1306         movq    %r12,104(%rsp)
1307         xorq    %rdx,%r14
1308         andq    %r11,%rdi
1309         rorq    $4,%r13
1310         addq    %rcx,%r12
1311         xorq    %rbx,%rdi
1312         rorq    $6,%r14
1313         xorq    %r11,%r13
1314         addq    %rdi,%r12
1315         movq    %rdx,%rdi
1316         addq    (%rbp),%r12
1317         xorq    %rdx,%r14
1318         xorq    %r8,%rdi
1319         rorq    $14,%r13
1320         movq    %r8,%rcx
1321         andq    %rdi,%r15
1322         rorq    $28,%r14
1323         addq    %r13,%r12
1324         xorq    %r15,%rcx
1325         addq    %r12,%r10
1326         addq    %r12,%rcx
1327         leaq    24(%rbp),%rbp
1328         movq    120(%rsp),%r13
1329         movq    96(%rsp),%r15
1330         movq    %r13,%r12
1331         rorq    $7,%r13
1332         addq    %r14,%rcx
1333         movq    %r15,%r14
1334         rorq    $42,%r15
1335         xorq    %r12,%r13
1336         shrq    $7,%r12
1337         rorq    $1,%r13
1338         xorq    %r14,%r15
1339         shrq    $6,%r14
1340         rorq    $19,%r15
1341         xorq    %r13,%r12
1342         xorq    %r14,%r15
1343         addq    56(%rsp),%r12
1344         addq    112(%rsp),%r12
1345         movq    %r10,%r13
1346         addq    %r15,%r12
1347         movq    %rcx,%r14
1348         rorq    $23,%r13
1349         movq    %r11,%r15
1350         xorq    %r10,%r13
1351         rorq    $5,%r14
1352         xorq    %rax,%r15
1353         movq    %r12,112(%rsp)
1354         xorq    %rcx,%r14
1355         andq    %r10,%r15
1356         rorq    $4,%r13
1357         addq    %rbx,%r12
1358         xorq    %rax,%r15
1359         rorq    $6,%r14
1360         xorq    %r10,%r13
1361         addq    %r15,%r12
1362         movq    %rcx,%r15
1363         addq    (%rbp),%r12
1364         xorq    %rcx,%r14
1365         xorq    %rdx,%r15
1366         rorq    $14,%r13
1367         movq    %rdx,%rbx
1368         andq    %r15,%rdi
1369         rorq    $28,%r14
1370         addq    %r13,%r12
1371         xorq    %rdi,%rbx
1372         addq    %r12,%r9
1373         addq    %r12,%rbx
1374         leaq    8(%rbp),%rbp
1375         movq    0(%rsp),%r13
1376         movq    104(%rsp),%rdi
1377         movq    %r13,%r12
1378         rorq    $7,%r13
1379         addq    %r14,%rbx
1380         movq    %rdi,%r14
1381         rorq    $42,%rdi
1382         xorq    %r12,%r13
1383         shrq    $7,%r12
1384         rorq    $1,%r13
1385         xorq    %r14,%rdi
1386         shrq    $6,%r14
1387         rorq    $19,%rdi
1388         xorq    %r13,%r12
1389         xorq    %r14,%rdi
1390         addq    64(%rsp),%r12
1391         addq    120(%rsp),%r12
1392         movq    %r9,%r13
1393         addq    %rdi,%r12
1394         movq    %rbx,%r14
1395         rorq    $23,%r13
1396         movq    %r10,%rdi
1397         xorq    %r9,%r13
1398         rorq    $5,%r14
1399         xorq    %r11,%rdi
1400         movq    %r12,120(%rsp)
1401         xorq    %rbx,%r14
1402         andq    %r9,%rdi
1403         rorq    $4,%r13
1404         addq    %rax,%r12
1405         xorq    %r11,%rdi
1406         rorq    $6,%r14
1407         xorq    %r9,%r13
1408         addq    %rdi,%r12
1409         movq    %rbx,%rdi
1410         addq    (%rbp),%r12
1411         xorq    %rbx,%r14
1412         xorq    %rcx,%rdi
1413         rorq    $14,%r13
1414         movq    %rcx,%rax
1415         andq    %rdi,%r15
1416         rorq    $28,%r14
1417         addq    %r13,%r12
1418         xorq    %r15,%rax
1419         addq    %r12,%r8
1420         addq    %r12,%rax
1421         leaq    24(%rbp),%rbp
1422         cmpb    $0,7(%rbp)
1423         jnz     .Lrounds_16_xx
1424         movq    128+0(%rsp),%rdi
1425         addq    %r14,%rax
1426         leaq    128(%rsi),%rsi
1427         addq    0(%rdi),%rax
1428         addq    8(%rdi),%rbx
1429         addq    16(%rdi),%rcx
1430         addq    24(%rdi),%rdx
1431         addq    32(%rdi),%r8
1432         addq    40(%rdi),%r9
1433         addq    48(%rdi),%r10
1434         addq    56(%rdi),%r11
1435         cmpq    128+16(%rsp),%rsi
1436         movq    %rax,0(%rdi)
1437         movq    %rbx,8(%rdi)
1438         movq    %rcx,16(%rdi)
1439         movq    %rdx,24(%rdi)
1440         movq    %r8,32(%rdi)
1441         movq    %r9,40(%rdi)
1442         movq    %r10,48(%rdi)
1443         movq    %r11,56(%rdi)
1444         jb      .Lloop
1445         movq    152(%rsp),%rsi
1446 .cfi_def_cfa    %rsi,8
1447         movq    -48(%rsi),%r15
1448 .cfi_restore    %r15
1449         movq    -40(%rsi),%r14
1450 .cfi_restore    %r14
1451         movq    -32(%rsi),%r13
1452 .cfi_restore    %r13
1453         movq    -24(%rsi),%r12
1454 .cfi_restore    %r12
1455         movq    -16(%rsi),%rbp
1456 .cfi_restore    %rbp
1457         movq    -8(%rsi),%rbx
1458 .cfi_restore    %rbx
1459         leaq    (%rsi),%rsp
1460 .cfi_def_cfa_register   %rsp
1461 .Lepilogue:
1462         RET
1463 .cfi_endproc
1464 SET_SIZE(zfs_sha512_transform_x64)
1465
1466 ENTRY_ALIGN(zfs_sha512_transform_avx, 64)
1467 .cfi_startproc
1468         ENDBR
1469         movq    %rsp,%rax
1470 .cfi_def_cfa_register   %rax
1471         pushq   %rbx
1472 .cfi_offset     %rbx,-16
1473         pushq   %rbp
1474 .cfi_offset     %rbp,-24
1475         pushq   %r12
1476 .cfi_offset     %r12,-32
1477         pushq   %r13
1478 .cfi_offset     %r13,-40
1479         pushq   %r14
1480 .cfi_offset     %r14,-48
1481         pushq   %r15
1482 .cfi_offset     %r15,-56
1483         shlq    $4,%rdx
1484         subq    $160,%rsp
1485         leaq    (%rsi,%rdx,8),%rdx
1486         andq    $-64,%rsp
1487         movq    %rdi,128+0(%rsp)
1488         movq    %rsi,128+8(%rsp)
1489         movq    %rdx,128+16(%rsp)
1490         movq    %rax,152(%rsp)
1491 .cfi_escape     0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
1492 .Lprologue_avx:
1493
1494         vzeroupper
1495         movq    0(%rdi),%rax
1496         movq    8(%rdi),%rbx
1497         movq    16(%rdi),%rcx
1498         movq    24(%rdi),%rdx
1499         movq    32(%rdi),%r8
1500         movq    40(%rdi),%r9
1501         movq    48(%rdi),%r10
1502         movq    56(%rdi),%r11
1503         jmp     .Lloop_avx
1504 .balign 16
1505 .Lloop_avx:
1506         vmovdqa K512+1280(%rip),%xmm11
1507         vmovdqu 0(%rsi),%xmm0
1508         leaq    K512+128(%rip),%rbp
1509         vmovdqu 16(%rsi),%xmm1
1510         vmovdqu 32(%rsi),%xmm2
1511         vpshufb %xmm11,%xmm0,%xmm0
1512         vmovdqu 48(%rsi),%xmm3
1513         vpshufb %xmm11,%xmm1,%xmm1
1514         vmovdqu 64(%rsi),%xmm4
1515         vpshufb %xmm11,%xmm2,%xmm2
1516         vmovdqu 80(%rsi),%xmm5
1517         vpshufb %xmm11,%xmm3,%xmm3
1518         vmovdqu 96(%rsi),%xmm6
1519         vpshufb %xmm11,%xmm4,%xmm4
1520         vmovdqu 112(%rsi),%xmm7
1521         vpshufb %xmm11,%xmm5,%xmm5
1522         vpaddq  -128(%rbp),%xmm0,%xmm8
1523         vpshufb %xmm11,%xmm6,%xmm6
1524         vpaddq  -96(%rbp),%xmm1,%xmm9
1525         vpshufb %xmm11,%xmm7,%xmm7
1526         vpaddq  -64(%rbp),%xmm2,%xmm10
1527         vpaddq  -32(%rbp),%xmm3,%xmm11
1528         vmovdqa %xmm8,0(%rsp)
1529         vpaddq  0(%rbp),%xmm4,%xmm8
1530         vmovdqa %xmm9,16(%rsp)
1531         vpaddq  32(%rbp),%xmm5,%xmm9
1532         vmovdqa %xmm10,32(%rsp)
1533         vpaddq  64(%rbp),%xmm6,%xmm10
1534         vmovdqa %xmm11,48(%rsp)
1535         vpaddq  96(%rbp),%xmm7,%xmm11
1536         vmovdqa %xmm8,64(%rsp)
1537         movq    %rax,%r14
1538         vmovdqa %xmm9,80(%rsp)
1539         movq    %rbx,%rdi
1540         vmovdqa %xmm10,96(%rsp)
1541         xorq    %rcx,%rdi
1542         vmovdqa %xmm11,112(%rsp)
1543         movq    %r8,%r13
1544         jmp     .Lavx_00_47
1545
1546 .balign 16
1547 .Lavx_00_47:
1548         addq    $256,%rbp
1549         vpalignr        $8,%xmm0,%xmm1,%xmm8
1550         shrdq   $23,%r13,%r13
1551         movq    %r14,%rax
1552         vpalignr        $8,%xmm4,%xmm5,%xmm11
1553         movq    %r9,%r12
1554         shrdq   $5,%r14,%r14
1555         vpsrlq  $1,%xmm8,%xmm10
1556         xorq    %r8,%r13
1557         xorq    %r10,%r12
1558         vpaddq  %xmm11,%xmm0,%xmm0
1559         shrdq   $4,%r13,%r13
1560         xorq    %rax,%r14
1561         vpsrlq  $7,%xmm8,%xmm11
1562         andq    %r8,%r12
1563         xorq    %r8,%r13
1564         vpsllq  $56,%xmm8,%xmm9
1565         addq    0(%rsp),%r11
1566         movq    %rax,%r15
1567         vpxor   %xmm10,%xmm11,%xmm8
1568         xorq    %r10,%r12
1569         shrdq   $6,%r14,%r14
1570         vpsrlq  $7,%xmm10,%xmm10
1571         xorq    %rbx,%r15
1572         addq    %r12,%r11
1573         vpxor   %xmm9,%xmm8,%xmm8
1574         shrdq   $14,%r13,%r13
1575         andq    %r15,%rdi
1576         vpsllq  $7,%xmm9,%xmm9
1577         xorq    %rax,%r14
1578         addq    %r13,%r11
1579         vpxor   %xmm10,%xmm8,%xmm8
1580         xorq    %rbx,%rdi
1581         shrdq   $28,%r14,%r14
1582         vpsrlq  $6,%xmm7,%xmm11
1583         addq    %r11,%rdx
1584         addq    %rdi,%r11
1585         vpxor   %xmm9,%xmm8,%xmm8
1586         movq    %rdx,%r13
1587         addq    %r11,%r14
1588         vpsllq  $3,%xmm7,%xmm10
1589         shrdq   $23,%r13,%r13
1590         movq    %r14,%r11
1591         vpaddq  %xmm8,%xmm0,%xmm0
1592         movq    %r8,%r12
1593         shrdq   $5,%r14,%r14
1594         vpsrlq  $19,%xmm7,%xmm9
1595         xorq    %rdx,%r13
1596         xorq    %r9,%r12
1597         vpxor   %xmm10,%xmm11,%xmm11
1598         shrdq   $4,%r13,%r13
1599         xorq    %r11,%r14
1600         vpsllq  $42,%xmm10,%xmm10
1601         andq    %rdx,%r12
1602         xorq    %rdx,%r13
1603         vpxor   %xmm9,%xmm11,%xmm11
1604         addq    8(%rsp),%r10
1605         movq    %r11,%rdi
1606         vpsrlq  $42,%xmm9,%xmm9
1607         xorq    %r9,%r12
1608         shrdq   $6,%r14,%r14
1609         vpxor   %xmm10,%xmm11,%xmm11
1610         xorq    %rax,%rdi
1611         addq    %r12,%r10
1612         vpxor   %xmm9,%xmm11,%xmm11
1613         shrdq   $14,%r13,%r13
1614         andq    %rdi,%r15
1615         vpaddq  %xmm11,%xmm0,%xmm0
1616         xorq    %r11,%r14
1617         addq    %r13,%r10
1618         vpaddq  -128(%rbp),%xmm0,%xmm10
1619         xorq    %rax,%r15
1620         shrdq   $28,%r14,%r14
1621         addq    %r10,%rcx
1622         addq    %r15,%r10
1623         movq    %rcx,%r13
1624         addq    %r10,%r14
1625         vmovdqa %xmm10,0(%rsp)
1626         vpalignr        $8,%xmm1,%xmm2,%xmm8
1627         shrdq   $23,%r13,%r13
1628         movq    %r14,%r10
1629         vpalignr        $8,%xmm5,%xmm6,%xmm11
1630         movq    %rdx,%r12
1631         shrdq   $5,%r14,%r14
1632         vpsrlq  $1,%xmm8,%xmm10
1633         xorq    %rcx,%r13
1634         xorq    %r8,%r12
1635         vpaddq  %xmm11,%xmm1,%xmm1
1636         shrdq   $4,%r13,%r13
1637         xorq    %r10,%r14
1638         vpsrlq  $7,%xmm8,%xmm11
1639         andq    %rcx,%r12
1640         xorq    %rcx,%r13
1641         vpsllq  $56,%xmm8,%xmm9
1642         addq    16(%rsp),%r9
1643         movq    %r10,%r15
1644         vpxor   %xmm10,%xmm11,%xmm8
1645         xorq    %r8,%r12
1646         shrdq   $6,%r14,%r14
1647         vpsrlq  $7,%xmm10,%xmm10
1648         xorq    %r11,%r15
1649         addq    %r12,%r9
1650         vpxor   %xmm9,%xmm8,%xmm8
1651         shrdq   $14,%r13,%r13
1652         andq    %r15,%rdi
1653         vpsllq  $7,%xmm9,%xmm9
1654         xorq    %r10,%r14
1655         addq    %r13,%r9
1656         vpxor   %xmm10,%xmm8,%xmm8
1657         xorq    %r11,%rdi
1658         shrdq   $28,%r14,%r14
1659         vpsrlq  $6,%xmm0,%xmm11
1660         addq    %r9,%rbx
1661         addq    %rdi,%r9
1662         vpxor   %xmm9,%xmm8,%xmm8
1663         movq    %rbx,%r13
1664         addq    %r9,%r14
1665         vpsllq  $3,%xmm0,%xmm10
1666         shrdq   $23,%r13,%r13
1667         movq    %r14,%r9
1668         vpaddq  %xmm8,%xmm1,%xmm1
1669         movq    %rcx,%r12
1670         shrdq   $5,%r14,%r14
1671         vpsrlq  $19,%xmm0,%xmm9
1672         xorq    %rbx,%r13
1673         xorq    %rdx,%r12
1674         vpxor   %xmm10,%xmm11,%xmm11
1675         shrdq   $4,%r13,%r13
1676         xorq    %r9,%r14
1677         vpsllq  $42,%xmm10,%xmm10
1678         andq    %rbx,%r12
1679         xorq    %rbx,%r13
1680         vpxor   %xmm9,%xmm11,%xmm11
1681         addq    24(%rsp),%r8
1682         movq    %r9,%rdi
1683         vpsrlq  $42,%xmm9,%xmm9
1684         xorq    %rdx,%r12
1685         shrdq   $6,%r14,%r14
1686         vpxor   %xmm10,%xmm11,%xmm11
1687         xorq    %r10,%rdi
1688         addq    %r12,%r8
1689         vpxor   %xmm9,%xmm11,%xmm11
1690         shrdq   $14,%r13,%r13
1691         andq    %rdi,%r15
1692         vpaddq  %xmm11,%xmm1,%xmm1
1693         xorq    %r9,%r14
1694         addq    %r13,%r8
1695         vpaddq  -96(%rbp),%xmm1,%xmm10
1696         xorq    %r10,%r15
1697         shrdq   $28,%r14,%r14
1698         addq    %r8,%rax
1699         addq    %r15,%r8
1700         movq    %rax,%r13
1701         addq    %r8,%r14
1702         vmovdqa %xmm10,16(%rsp)
1703         vpalignr        $8,%xmm2,%xmm3,%xmm8
1704         shrdq   $23,%r13,%r13
1705         movq    %r14,%r8
1706         vpalignr        $8,%xmm6,%xmm7,%xmm11
1707         movq    %rbx,%r12
1708         shrdq   $5,%r14,%r14
1709         vpsrlq  $1,%xmm8,%xmm10
1710         xorq    %rax,%r13
1711         xorq    %rcx,%r12
1712         vpaddq  %xmm11,%xmm2,%xmm2
1713         shrdq   $4,%r13,%r13
1714         xorq    %r8,%r14
1715         vpsrlq  $7,%xmm8,%xmm11
1716         andq    %rax,%r12
1717         xorq    %rax,%r13
1718         vpsllq  $56,%xmm8,%xmm9
1719         addq    32(%rsp),%rdx
1720         movq    %r8,%r15
1721         vpxor   %xmm10,%xmm11,%xmm8
1722         xorq    %rcx,%r12
1723         shrdq   $6,%r14,%r14
1724         vpsrlq  $7,%xmm10,%xmm10
1725         xorq    %r9,%r15
1726         addq    %r12,%rdx
1727         vpxor   %xmm9,%xmm8,%xmm8
1728         shrdq   $14,%r13,%r13
1729         andq    %r15,%rdi
1730         vpsllq  $7,%xmm9,%xmm9
1731         xorq    %r8,%r14
1732         addq    %r13,%rdx
1733         vpxor   %xmm10,%xmm8,%xmm8
1734         xorq    %r9,%rdi
1735         shrdq   $28,%r14,%r14
1736         vpsrlq  $6,%xmm1,%xmm11
1737         addq    %rdx,%r11
1738         addq    %rdi,%rdx
1739         vpxor   %xmm9,%xmm8,%xmm8
1740         movq    %r11,%r13
1741         addq    %rdx,%r14
1742         vpsllq  $3,%xmm1,%xmm10
1743         shrdq   $23,%r13,%r13
1744         movq    %r14,%rdx
1745         vpaddq  %xmm8,%xmm2,%xmm2
1746         movq    %rax,%r12
1747         shrdq   $5,%r14,%r14
1748         vpsrlq  $19,%xmm1,%xmm9
1749         xorq    %r11,%r13
1750         xorq    %rbx,%r12
1751         vpxor   %xmm10,%xmm11,%xmm11
1752         shrdq   $4,%r13,%r13
1753         xorq    %rdx,%r14
1754         vpsllq  $42,%xmm10,%xmm10
1755         andq    %r11,%r12
1756         xorq    %r11,%r13
1757         vpxor   %xmm9,%xmm11,%xmm11
1758         addq    40(%rsp),%rcx
1759         movq    %rdx,%rdi
1760         vpsrlq  $42,%xmm9,%xmm9
1761         xorq    %rbx,%r12
1762         shrdq   $6,%r14,%r14
1763         vpxor   %xmm10,%xmm11,%xmm11
1764         xorq    %r8,%rdi
1765         addq    %r12,%rcx
1766         vpxor   %xmm9,%xmm11,%xmm11
1767         shrdq   $14,%r13,%r13
1768         andq    %rdi,%r15
1769         vpaddq  %xmm11,%xmm2,%xmm2
1770         xorq    %rdx,%r14
1771         addq    %r13,%rcx
1772         vpaddq  -64(%rbp),%xmm2,%xmm10
1773         xorq    %r8,%r15
1774         shrdq   $28,%r14,%r14
1775         addq    %rcx,%r10
1776         addq    %r15,%rcx
1777         movq    %r10,%r13
1778         addq    %rcx,%r14
1779         vmovdqa %xmm10,32(%rsp)
1780         vpalignr        $8,%xmm3,%xmm4,%xmm8
1781         shrdq   $23,%r13,%r13
1782         movq    %r14,%rcx
1783         vpalignr        $8,%xmm7,%xmm0,%xmm11
1784         movq    %r11,%r12
1785         shrdq   $5,%r14,%r14
1786         vpsrlq  $1,%xmm8,%xmm10
1787         xorq    %r10,%r13
1788         xorq    %rax,%r12
1789         vpaddq  %xmm11,%xmm3,%xmm3
1790         shrdq   $4,%r13,%r13
1791         xorq    %rcx,%r14
1792         vpsrlq  $7,%xmm8,%xmm11
1793         andq    %r10,%r12
1794         xorq    %r10,%r13
1795         vpsllq  $56,%xmm8,%xmm9
1796         addq    48(%rsp),%rbx
1797         movq    %rcx,%r15
1798         vpxor   %xmm10,%xmm11,%xmm8
1799         xorq    %rax,%r12
1800         shrdq   $6,%r14,%r14
1801         vpsrlq  $7,%xmm10,%xmm10
1802         xorq    %rdx,%r15
1803         addq    %r12,%rbx
1804         vpxor   %xmm9,%xmm8,%xmm8
1805         shrdq   $14,%r13,%r13
1806         andq    %r15,%rdi
1807         vpsllq  $7,%xmm9,%xmm9
1808         xorq    %rcx,%r14
1809         addq    %r13,%rbx
1810         vpxor   %xmm10,%xmm8,%xmm8
1811         xorq    %rdx,%rdi
1812         shrdq   $28,%r14,%r14
1813         vpsrlq  $6,%xmm2,%xmm11
1814         addq    %rbx,%r9
1815         addq    %rdi,%rbx
1816         vpxor   %xmm9,%xmm8,%xmm8
1817         movq    %r9,%r13
1818         addq    %rbx,%r14
1819         vpsllq  $3,%xmm2,%xmm10
1820         shrdq   $23,%r13,%r13
1821         movq    %r14,%rbx
1822         vpaddq  %xmm8,%xmm3,%xmm3
1823         movq    %r10,%r12
1824         shrdq   $5,%r14,%r14
1825         vpsrlq  $19,%xmm2,%xmm9
1826         xorq    %r9,%r13
1827         xorq    %r11,%r12
1828         vpxor   %xmm10,%xmm11,%xmm11
1829         shrdq   $4,%r13,%r13
1830         xorq    %rbx,%r14
1831         vpsllq  $42,%xmm10,%xmm10
1832         andq    %r9,%r12
1833         xorq    %r9,%r13
1834         vpxor   %xmm9,%xmm11,%xmm11
1835         addq    56(%rsp),%rax
1836         movq    %rbx,%rdi
1837         vpsrlq  $42,%xmm9,%xmm9
1838         xorq    %r11,%r12
1839         shrdq   $6,%r14,%r14
1840         vpxor   %xmm10,%xmm11,%xmm11
1841         xorq    %rcx,%rdi
1842         addq    %r12,%rax
1843         vpxor   %xmm9,%xmm11,%xmm11
1844         shrdq   $14,%r13,%r13
1845         andq    %rdi,%r15
1846         vpaddq  %xmm11,%xmm3,%xmm3
1847         xorq    %rbx,%r14
1848         addq    %r13,%rax
1849         vpaddq  -32(%rbp),%xmm3,%xmm10
1850         xorq    %rcx,%r15
1851         shrdq   $28,%r14,%r14
1852         addq    %rax,%r8
1853         addq    %r15,%rax
1854         movq    %r8,%r13
1855         addq    %rax,%r14
1856         vmovdqa %xmm10,48(%rsp)
1857         vpalignr        $8,%xmm4,%xmm5,%xmm8
1858         shrdq   $23,%r13,%r13
1859         movq    %r14,%rax
1860         vpalignr        $8,%xmm0,%xmm1,%xmm11
1861         movq    %r9,%r12
1862         shrdq   $5,%r14,%r14
1863         vpsrlq  $1,%xmm8,%xmm10
1864         xorq    %r8,%r13
1865         xorq    %r10,%r12
1866         vpaddq  %xmm11,%xmm4,%xmm4
1867         shrdq   $4,%r13,%r13
1868         xorq    %rax,%r14
1869         vpsrlq  $7,%xmm8,%xmm11
1870         andq    %r8,%r12
1871         xorq    %r8,%r13
1872         vpsllq  $56,%xmm8,%xmm9
1873         addq    64(%rsp),%r11
1874         movq    %rax,%r15
1875         vpxor   %xmm10,%xmm11,%xmm8
1876         xorq    %r10,%r12
1877         shrdq   $6,%r14,%r14
1878         vpsrlq  $7,%xmm10,%xmm10
1879         xorq    %rbx,%r15
1880         addq    %r12,%r11
1881         vpxor   %xmm9,%xmm8,%xmm8
1882         shrdq   $14,%r13,%r13
1883         andq    %r15,%rdi
1884         vpsllq  $7,%xmm9,%xmm9
1885         xorq    %rax,%r14
1886         addq    %r13,%r11
1887         vpxor   %xmm10,%xmm8,%xmm8
1888         xorq    %rbx,%rdi
1889         shrdq   $28,%r14,%r14
1890         vpsrlq  $6,%xmm3,%xmm11
1891         addq    %r11,%rdx
1892         addq    %rdi,%r11
1893         vpxor   %xmm9,%xmm8,%xmm8
1894         movq    %rdx,%r13
1895         addq    %r11,%r14
1896         vpsllq  $3,%xmm3,%xmm10
1897         shrdq   $23,%r13,%r13
1898         movq    %r14,%r11
1899         vpaddq  %xmm8,%xmm4,%xmm4
1900         movq    %r8,%r12
1901         shrdq   $5,%r14,%r14
1902         vpsrlq  $19,%xmm3,%xmm9
1903         xorq    %rdx,%r13
1904         xorq    %r9,%r12
1905         vpxor   %xmm10,%xmm11,%xmm11
1906         shrdq   $4,%r13,%r13
1907         xorq    %r11,%r14
1908         vpsllq  $42,%xmm10,%xmm10
1909         andq    %rdx,%r12
1910         xorq    %rdx,%r13
1911         vpxor   %xmm9,%xmm11,%xmm11
1912         addq    72(%rsp),%r10
1913         movq    %r11,%rdi
1914         vpsrlq  $42,%xmm9,%xmm9
1915         xorq    %r9,%r12
1916         shrdq   $6,%r14,%r14
1917         vpxor   %xmm10,%xmm11,%xmm11
1918         xorq    %rax,%rdi
1919         addq    %r12,%r10
1920         vpxor   %xmm9,%xmm11,%xmm11
1921         shrdq   $14,%r13,%r13
1922         andq    %rdi,%r15
1923         vpaddq  %xmm11,%xmm4,%xmm4
1924         xorq    %r11,%r14
1925         addq    %r13,%r10
1926         vpaddq  0(%rbp),%xmm4,%xmm10
1927         xorq    %rax,%r15
1928         shrdq   $28,%r14,%r14
1929         addq    %r10,%rcx
1930         addq    %r15,%r10
1931         movq    %rcx,%r13
1932         addq    %r10,%r14
1933         vmovdqa %xmm10,64(%rsp)
1934         vpalignr        $8,%xmm5,%xmm6,%xmm8
1935         shrdq   $23,%r13,%r13
1936         movq    %r14,%r10
1937         vpalignr        $8,%xmm1,%xmm2,%xmm11
1938         movq    %rdx,%r12
1939         shrdq   $5,%r14,%r14
1940         vpsrlq  $1,%xmm8,%xmm10
1941         xorq    %rcx,%r13
1942         xorq    %r8,%r12
1943         vpaddq  %xmm11,%xmm5,%xmm5
1944         shrdq   $4,%r13,%r13
1945         xorq    %r10,%r14
1946         vpsrlq  $7,%xmm8,%xmm11
1947         andq    %rcx,%r12
1948         xorq    %rcx,%r13
1949         vpsllq  $56,%xmm8,%xmm9
1950         addq    80(%rsp),%r9
1951         movq    %r10,%r15
1952         vpxor   %xmm10,%xmm11,%xmm8
1953         xorq    %r8,%r12
1954         shrdq   $6,%r14,%r14
1955         vpsrlq  $7,%xmm10,%xmm10
1956         xorq    %r11,%r15
1957         addq    %r12,%r9
1958         vpxor   %xmm9,%xmm8,%xmm8
1959         shrdq   $14,%r13,%r13
1960         andq    %r15,%rdi
1961         vpsllq  $7,%xmm9,%xmm9
1962         xorq    %r10,%r14
1963         addq    %r13,%r9
1964         vpxor   %xmm10,%xmm8,%xmm8
1965         xorq    %r11,%rdi
1966         shrdq   $28,%r14,%r14
1967         vpsrlq  $6,%xmm4,%xmm11
1968         addq    %r9,%rbx
1969         addq    %rdi,%r9
1970         vpxor   %xmm9,%xmm8,%xmm8
1971         movq    %rbx,%r13
1972         addq    %r9,%r14
1973         vpsllq  $3,%xmm4,%xmm10
1974         shrdq   $23,%r13,%r13
1975         movq    %r14,%r9
1976         vpaddq  %xmm8,%xmm5,%xmm5
1977         movq    %rcx,%r12
1978         shrdq   $5,%r14,%r14
1979         vpsrlq  $19,%xmm4,%xmm9
1980         xorq    %rbx,%r13
1981         xorq    %rdx,%r12
1982         vpxor   %xmm10,%xmm11,%xmm11
1983         shrdq   $4,%r13,%r13
1984         xorq    %r9,%r14
1985         vpsllq  $42,%xmm10,%xmm10
1986         andq    %rbx,%r12
1987         xorq    %rbx,%r13
1988         vpxor   %xmm9,%xmm11,%xmm11
1989         addq    88(%rsp),%r8
1990         movq    %r9,%rdi
1991         vpsrlq  $42,%xmm9,%xmm9
1992         xorq    %rdx,%r12
1993         shrdq   $6,%r14,%r14
1994         vpxor   %xmm10,%xmm11,%xmm11
1995         xorq    %r10,%rdi
1996         addq    %r12,%r8
1997         vpxor   %xmm9,%xmm11,%xmm11
1998         shrdq   $14,%r13,%r13
1999         andq    %rdi,%r15
2000         vpaddq  %xmm11,%xmm5,%xmm5
2001         xorq    %r9,%r14
2002         addq    %r13,%r8
2003         vpaddq  32(%rbp),%xmm5,%xmm10
2004         xorq    %r10,%r15
2005         shrdq   $28,%r14,%r14
2006         addq    %r8,%rax
2007         addq    %r15,%r8
2008         movq    %rax,%r13
2009         addq    %r8,%r14
2010         vmovdqa %xmm10,80(%rsp)
2011         vpalignr        $8,%xmm6,%xmm7,%xmm8
2012         shrdq   $23,%r13,%r13
2013         movq    %r14,%r8
2014         vpalignr        $8,%xmm2,%xmm3,%xmm11
2015         movq    %rbx,%r12
2016         shrdq   $5,%r14,%r14
2017         vpsrlq  $1,%xmm8,%xmm10
2018         xorq    %rax,%r13
2019         xorq    %rcx,%r12
2020         vpaddq  %xmm11,%xmm6,%xmm6
2021         shrdq   $4,%r13,%r13
2022         xorq    %r8,%r14
2023         vpsrlq  $7,%xmm8,%xmm11
2024         andq    %rax,%r12
2025         xorq    %rax,%r13
2026         vpsllq  $56,%xmm8,%xmm9
2027         addq    96(%rsp),%rdx
2028         movq    %r8,%r15
2029         vpxor   %xmm10,%xmm11,%xmm8
2030         xorq    %rcx,%r12
2031         shrdq   $6,%r14,%r14
2032         vpsrlq  $7,%xmm10,%xmm10
2033         xorq    %r9,%r15
2034         addq    %r12,%rdx
2035         vpxor   %xmm9,%xmm8,%xmm8
2036         shrdq   $14,%r13,%r13
2037         andq    %r15,%rdi
2038         vpsllq  $7,%xmm9,%xmm9
2039         xorq    %r8,%r14
2040         addq    %r13,%rdx
2041         vpxor   %xmm10,%xmm8,%xmm8
2042         xorq    %r9,%rdi
2043         shrdq   $28,%r14,%r14
2044         vpsrlq  $6,%xmm5,%xmm11
2045         addq    %rdx,%r11
2046         addq    %rdi,%rdx
2047         vpxor   %xmm9,%xmm8,%xmm8
2048         movq    %r11,%r13
2049         addq    %rdx,%r14
2050         vpsllq  $3,%xmm5,%xmm10
2051         shrdq   $23,%r13,%r13
2052         movq    %r14,%rdx
2053         vpaddq  %xmm8,%xmm6,%xmm6
2054         movq    %rax,%r12
2055         shrdq   $5,%r14,%r14
2056         vpsrlq  $19,%xmm5,%xmm9
2057         xorq    %r11,%r13
2058         xorq    %rbx,%r12
2059         vpxor   %xmm10,%xmm11,%xmm11
2060         shrdq   $4,%r13,%r13
2061         xorq    %rdx,%r14
2062         vpsllq  $42,%xmm10,%xmm10
2063         andq    %r11,%r12
2064         xorq    %r11,%r13
2065         vpxor   %xmm9,%xmm11,%xmm11
2066         addq    104(%rsp),%rcx
2067         movq    %rdx,%rdi
2068         vpsrlq  $42,%xmm9,%xmm9
2069         xorq    %rbx,%r12
2070         shrdq   $6,%r14,%r14
2071         vpxor   %xmm10,%xmm11,%xmm11
2072         xorq    %r8,%rdi
2073         addq    %r12,%rcx
2074         vpxor   %xmm9,%xmm11,%xmm11
2075         shrdq   $14,%r13,%r13
2076         andq    %rdi,%r15
2077         vpaddq  %xmm11,%xmm6,%xmm6
2078         xorq    %rdx,%r14
2079         addq    %r13,%rcx
2080         vpaddq  64(%rbp),%xmm6,%xmm10
2081         xorq    %r8,%r15
2082         shrdq   $28,%r14,%r14
2083         addq    %rcx,%r10
2084         addq    %r15,%rcx
2085         movq    %r10,%r13
2086         addq    %rcx,%r14
2087         vmovdqa %xmm10,96(%rsp)
2088         vpalignr        $8,%xmm7,%xmm0,%xmm8
2089         shrdq   $23,%r13,%r13
2090         movq    %r14,%rcx
2091         vpalignr        $8,%xmm3,%xmm4,%xmm11
2092         movq    %r11,%r12
2093         shrdq   $5,%r14,%r14
2094         vpsrlq  $1,%xmm8,%xmm10
2095         xorq    %r10,%r13
2096         xorq    %rax,%r12
2097         vpaddq  %xmm11,%xmm7,%xmm7
2098         shrdq   $4,%r13,%r13
2099         xorq    %rcx,%r14
2100         vpsrlq  $7,%xmm8,%xmm11
2101         andq    %r10,%r12
2102         xorq    %r10,%r13
2103         vpsllq  $56,%xmm8,%xmm9
2104         addq    112(%rsp),%rbx
2105         movq    %rcx,%r15
2106         vpxor   %xmm10,%xmm11,%xmm8
2107         xorq    %rax,%r12
2108         shrdq   $6,%r14,%r14
2109         vpsrlq  $7,%xmm10,%xmm10
2110         xorq    %rdx,%r15
2111         addq    %r12,%rbx
2112         vpxor   %xmm9,%xmm8,%xmm8
2113         shrdq   $14,%r13,%r13
2114         andq    %r15,%rdi
2115         vpsllq  $7,%xmm9,%xmm9
2116         xorq    %rcx,%r14
2117         addq    %r13,%rbx
2118         vpxor   %xmm10,%xmm8,%xmm8
2119         xorq    %rdx,%rdi
2120         shrdq   $28,%r14,%r14
2121         vpsrlq  $6,%xmm6,%xmm11
2122         addq    %rbx,%r9
2123         addq    %rdi,%rbx
2124         vpxor   %xmm9,%xmm8,%xmm8
2125         movq    %r9,%r13
2126         addq    %rbx,%r14
2127         vpsllq  $3,%xmm6,%xmm10
2128         shrdq   $23,%r13,%r13
2129         movq    %r14,%rbx
2130         vpaddq  %xmm8,%xmm7,%xmm7
2131         movq    %r10,%r12
2132         shrdq   $5,%r14,%r14
2133         vpsrlq  $19,%xmm6,%xmm9
2134         xorq    %r9,%r13
2135         xorq    %r11,%r12
2136         vpxor   %xmm10,%xmm11,%xmm11
2137         shrdq   $4,%r13,%r13
2138         xorq    %rbx,%r14
2139         vpsllq  $42,%xmm10,%xmm10
2140         andq    %r9,%r12
2141         xorq    %r9,%r13
2142         vpxor   %xmm9,%xmm11,%xmm11
2143         addq    120(%rsp),%rax
2144         movq    %rbx,%rdi
2145         vpsrlq  $42,%xmm9,%xmm9
2146         xorq    %r11,%r12
2147         shrdq   $6,%r14,%r14
2148         vpxor   %xmm10,%xmm11,%xmm11
2149         xorq    %rcx,%rdi
2150         addq    %r12,%rax
2151         vpxor   %xmm9,%xmm11,%xmm11
2152         shrdq   $14,%r13,%r13
2153         andq    %rdi,%r15
2154         vpaddq  %xmm11,%xmm7,%xmm7
2155         xorq    %rbx,%r14
2156         addq    %r13,%rax
2157         vpaddq  96(%rbp),%xmm7,%xmm10
2158         xorq    %rcx,%r15
2159         shrdq   $28,%r14,%r14
2160         addq    %rax,%r8
2161         addq    %r15,%rax
2162         movq    %r8,%r13
2163         addq    %rax,%r14
2164         vmovdqa %xmm10,112(%rsp)
2165         cmpb    $0,135(%rbp)
2166         jne     .Lavx_00_47
2167         shrdq   $23,%r13,%r13
2168         movq    %r14,%rax
2169         movq    %r9,%r12
2170         shrdq   $5,%r14,%r14
2171         xorq    %r8,%r13
2172         xorq    %r10,%r12
2173         shrdq   $4,%r13,%r13
2174         xorq    %rax,%r14
2175         andq    %r8,%r12
2176         xorq    %r8,%r13
2177         addq    0(%rsp),%r11
2178         movq    %rax,%r15
2179         xorq    %r10,%r12
2180         shrdq   $6,%r14,%r14
2181         xorq    %rbx,%r15
2182         addq    %r12,%r11
2183         shrdq   $14,%r13,%r13
2184         andq    %r15,%rdi
2185         xorq    %rax,%r14
2186         addq    %r13,%r11
2187         xorq    %rbx,%rdi
2188         shrdq   $28,%r14,%r14
2189         addq    %r11,%rdx
2190         addq    %rdi,%r11
2191         movq    %rdx,%r13
2192         addq    %r11,%r14
2193         shrdq   $23,%r13,%r13
2194         movq    %r14,%r11
2195         movq    %r8,%r12
2196         shrdq   $5,%r14,%r14
2197         xorq    %rdx,%r13
2198         xorq    %r9,%r12
2199         shrdq   $4,%r13,%r13
2200         xorq    %r11,%r14
2201         andq    %rdx,%r12
2202         xorq    %rdx,%r13
2203         addq    8(%rsp),%r10
2204         movq    %r11,%rdi
2205         xorq    %r9,%r12
2206         shrdq   $6,%r14,%r14
2207         xorq    %rax,%rdi
2208         addq    %r12,%r10
2209         shrdq   $14,%r13,%r13
2210         andq    %rdi,%r15
2211         xorq    %r11,%r14
2212         addq    %r13,%r10
2213         xorq    %rax,%r15
2214         shrdq   $28,%r14,%r14
2215         addq    %r10,%rcx
2216         addq    %r15,%r10
2217         movq    %rcx,%r13
2218         addq    %r10,%r14
2219         shrdq   $23,%r13,%r13
2220         movq    %r14,%r10
2221         movq    %rdx,%r12
2222         shrdq   $5,%r14,%r14
2223         xorq    %rcx,%r13
2224         xorq    %r8,%r12
2225         shrdq   $4,%r13,%r13
2226         xorq    %r10,%r14
2227         andq    %rcx,%r12
2228         xorq    %rcx,%r13
2229         addq    16(%rsp),%r9
2230         movq    %r10,%r15
2231         xorq    %r8,%r12
2232         shrdq   $6,%r14,%r14
2233         xorq    %r11,%r15
2234         addq    %r12,%r9
2235         shrdq   $14,%r13,%r13
2236         andq    %r15,%rdi
2237         xorq    %r10,%r14
2238         addq    %r13,%r9
2239         xorq    %r11,%rdi
2240         shrdq   $28,%r14,%r14
2241         addq    %r9,%rbx
2242         addq    %rdi,%r9
2243         movq    %rbx,%r13
2244         addq    %r9,%r14
2245         shrdq   $23,%r13,%r13
2246         movq    %r14,%r9
2247         movq    %rcx,%r12
2248         shrdq   $5,%r14,%r14
2249         xorq    %rbx,%r13
2250         xorq    %rdx,%r12
2251         shrdq   $4,%r13,%r13
2252         xorq    %r9,%r14
2253         andq    %rbx,%r12
2254         xorq    %rbx,%r13
2255         addq    24(%rsp),%r8
2256         movq    %r9,%rdi
2257         xorq    %rdx,%r12
2258         shrdq   $6,%r14,%r14
2259         xorq    %r10,%rdi
2260         addq    %r12,%r8
2261         shrdq   $14,%r13,%r13
2262         andq    %rdi,%r15
2263         xorq    %r9,%r14
2264         addq    %r13,%r8
2265         xorq    %r10,%r15
2266         shrdq   $28,%r14,%r14
2267         addq    %r8,%rax
2268         addq    %r15,%r8
2269         movq    %rax,%r13
2270         addq    %r8,%r14
2271         shrdq   $23,%r13,%r13
2272         movq    %r14,%r8
2273         movq    %rbx,%r12
2274         shrdq   $5,%r14,%r14
2275         xorq    %rax,%r13
2276         xorq    %rcx,%r12
2277         shrdq   $4,%r13,%r13
2278         xorq    %r8,%r14
2279         andq    %rax,%r12
2280         xorq    %rax,%r13
2281         addq    32(%rsp),%rdx
2282         movq    %r8,%r15
2283         xorq    %rcx,%r12
2284         shrdq   $6,%r14,%r14
2285         xorq    %r9,%r15
2286         addq    %r12,%rdx
2287         shrdq   $14,%r13,%r13
2288         andq    %r15,%rdi
2289         xorq    %r8,%r14
2290         addq    %r13,%rdx
2291         xorq    %r9,%rdi
2292         shrdq   $28,%r14,%r14
2293         addq    %rdx,%r11
2294         addq    %rdi,%rdx
2295         movq    %r11,%r13
2296         addq    %rdx,%r14
2297         shrdq   $23,%r13,%r13
2298         movq    %r14,%rdx
2299         movq    %rax,%r12
2300         shrdq   $5,%r14,%r14
2301         xorq    %r11,%r13
2302         xorq    %rbx,%r12
2303         shrdq   $4,%r13,%r13
2304         xorq    %rdx,%r14
2305         andq    %r11,%r12
2306         xorq    %r11,%r13
2307         addq    40(%rsp),%rcx
2308         movq    %rdx,%rdi
2309         xorq    %rbx,%r12
2310         shrdq   $6,%r14,%r14
2311         xorq    %r8,%rdi
2312         addq    %r12,%rcx
2313         shrdq   $14,%r13,%r13
2314         andq    %rdi,%r15
2315         xorq    %rdx,%r14
2316         addq    %r13,%rcx
2317         xorq    %r8,%r15
2318         shrdq   $28,%r14,%r14
2319         addq    %rcx,%r10
2320         addq    %r15,%rcx
2321         movq    %r10,%r13
2322         addq    %rcx,%r14
2323         shrdq   $23,%r13,%r13
2324         movq    %r14,%rcx
2325         movq    %r11,%r12
2326         shrdq   $5,%r14,%r14
2327         xorq    %r10,%r13
2328         xorq    %rax,%r12
2329         shrdq   $4,%r13,%r13
2330         xorq    %rcx,%r14
2331         andq    %r10,%r12
2332         xorq    %r10,%r13
2333         addq    48(%rsp),%rbx
2334         movq    %rcx,%r15
2335         xorq    %rax,%r12
2336         shrdq   $6,%r14,%r14
2337         xorq    %rdx,%r15
2338         addq    %r12,%rbx
2339         shrdq   $14,%r13,%r13
2340         andq    %r15,%rdi
2341         xorq    %rcx,%r14
2342         addq    %r13,%rbx
2343         xorq    %rdx,%rdi
2344         shrdq   $28,%r14,%r14
2345         addq    %rbx,%r9
2346         addq    %rdi,%rbx
2347         movq    %r9,%r13
2348         addq    %rbx,%r14
2349         shrdq   $23,%r13,%r13
2350         movq    %r14,%rbx
2351         movq    %r10,%r12
2352         shrdq   $5,%r14,%r14
2353         xorq    %r9,%r13
2354         xorq    %r11,%r12
2355         shrdq   $4,%r13,%r13
2356         xorq    %rbx,%r14
2357         andq    %r9,%r12
2358         xorq    %r9,%r13
2359         addq    56(%rsp),%rax
2360         movq    %rbx,%rdi
2361         xorq    %r11,%r12
2362         shrdq   $6,%r14,%r14
2363         xorq    %rcx,%rdi
2364         addq    %r12,%rax
2365         shrdq   $14,%r13,%r13
2366         andq    %rdi,%r15
2367         xorq    %rbx,%r14
2368         addq    %r13,%rax
2369         xorq    %rcx,%r15
2370         shrdq   $28,%r14,%r14
2371         addq    %rax,%r8
2372         addq    %r15,%rax
2373         movq    %r8,%r13
2374         addq    %rax,%r14
2375         shrdq   $23,%r13,%r13
2376         movq    %r14,%rax
2377         movq    %r9,%r12
2378         shrdq   $5,%r14,%r14
2379         xorq    %r8,%r13
2380         xorq    %r10,%r12
2381         shrdq   $4,%r13,%r13
2382         xorq    %rax,%r14
2383         andq    %r8,%r12
2384         xorq    %r8,%r13
2385         addq    64(%rsp),%r11
2386         movq    %rax,%r15
2387         xorq    %r10,%r12
2388         shrdq   $6,%r14,%r14
2389         xorq    %rbx,%r15
2390         addq    %r12,%r11
2391         shrdq   $14,%r13,%r13
2392         andq    %r15,%rdi
2393         xorq    %rax,%r14
2394         addq    %r13,%r11
2395         xorq    %rbx,%rdi
2396         shrdq   $28,%r14,%r14
2397         addq    %r11,%rdx
2398         addq    %rdi,%r11
2399         movq    %rdx,%r13
2400         addq    %r11,%r14
2401         shrdq   $23,%r13,%r13
2402         movq    %r14,%r11
2403         movq    %r8,%r12
2404         shrdq   $5,%r14,%r14
2405         xorq    %rdx,%r13
2406         xorq    %r9,%r12
2407         shrdq   $4,%r13,%r13
2408         xorq    %r11,%r14
2409         andq    %rdx,%r12
2410         xorq    %rdx,%r13
2411         addq    72(%rsp),%r10
2412         movq    %r11,%rdi
2413         xorq    %r9,%r12
2414         shrdq   $6,%r14,%r14
2415         xorq    %rax,%rdi
2416         addq    %r12,%r10
2417         shrdq   $14,%r13,%r13
2418         andq    %rdi,%r15
2419         xorq    %r11,%r14
2420         addq    %r13,%r10
2421         xorq    %rax,%r15
2422         shrdq   $28,%r14,%r14
2423         addq    %r10,%rcx
2424         addq    %r15,%r10
2425         movq    %rcx,%r13
2426         addq    %r10,%r14
2427         shrdq   $23,%r13,%r13
2428         movq    %r14,%r10
2429         movq    %rdx,%r12
2430         shrdq   $5,%r14,%r14
2431         xorq    %rcx,%r13
2432         xorq    %r8,%r12
2433         shrdq   $4,%r13,%r13
2434         xorq    %r10,%r14
2435         andq    %rcx,%r12
2436         xorq    %rcx,%r13
2437         addq    80(%rsp),%r9
2438         movq    %r10,%r15
2439         xorq    %r8,%r12
2440         shrdq   $6,%r14,%r14
2441         xorq    %r11,%r15
2442         addq    %r12,%r9
2443         shrdq   $14,%r13,%r13
2444         andq    %r15,%rdi
2445         xorq    %r10,%r14
2446         addq    %r13,%r9
2447         xorq    %r11,%rdi
2448         shrdq   $28,%r14,%r14
2449         addq    %r9,%rbx
2450         addq    %rdi,%r9
2451         movq    %rbx,%r13
2452         addq    %r9,%r14
2453         shrdq   $23,%r13,%r13
2454         movq    %r14,%r9
2455         movq    %rcx,%r12
2456         shrdq   $5,%r14,%r14
2457         xorq    %rbx,%r13
2458         xorq    %rdx,%r12
2459         shrdq   $4,%r13,%r13
2460         xorq    %r9,%r14
2461         andq    %rbx,%r12
2462         xorq    %rbx,%r13
2463         addq    88(%rsp),%r8
2464         movq    %r9,%rdi
2465         xorq    %rdx,%r12
2466         shrdq   $6,%r14,%r14
2467         xorq    %r10,%rdi
2468         addq    %r12,%r8
2469         shrdq   $14,%r13,%r13
2470         andq    %rdi,%r15
2471         xorq    %r9,%r14
2472         addq    %r13,%r8
2473         xorq    %r10,%r15
2474         shrdq   $28,%r14,%r14
2475         addq    %r8,%rax
2476         addq    %r15,%r8
2477         movq    %rax,%r13
2478         addq    %r8,%r14
2479         shrdq   $23,%r13,%r13
2480         movq    %r14,%r8
2481         movq    %rbx,%r12
2482         shrdq   $5,%r14,%r14
2483         xorq    %rax,%r13
2484         xorq    %rcx,%r12
2485         shrdq   $4,%r13,%r13
2486         xorq    %r8,%r14
2487         andq    %rax,%r12
2488         xorq    %rax,%r13
2489         addq    96(%rsp),%rdx
2490         movq    %r8,%r15
2491         xorq    %rcx,%r12
2492         shrdq   $6,%r14,%r14
2493         xorq    %r9,%r15
2494         addq    %r12,%rdx
2495         shrdq   $14,%r13,%r13
2496         andq    %r15,%rdi
2497         xorq    %r8,%r14
2498         addq    %r13,%rdx
2499         xorq    %r9,%rdi
2500         shrdq   $28,%r14,%r14
2501         addq    %rdx,%r11
2502         addq    %rdi,%rdx
2503         movq    %r11,%r13
2504         addq    %rdx,%r14
2505         shrdq   $23,%r13,%r13
2506         movq    %r14,%rdx
2507         movq    %rax,%r12
2508         shrdq   $5,%r14,%r14
2509         xorq    %r11,%r13
2510         xorq    %rbx,%r12
2511         shrdq   $4,%r13,%r13
2512         xorq    %rdx,%r14
2513         andq    %r11,%r12
2514         xorq    %r11,%r13
2515         addq    104(%rsp),%rcx
2516         movq    %rdx,%rdi
2517         xorq    %rbx,%r12
2518         shrdq   $6,%r14,%r14
2519         xorq    %r8,%rdi
2520         addq    %r12,%rcx
2521         shrdq   $14,%r13,%r13
2522         andq    %rdi,%r15
2523         xorq    %rdx,%r14
2524         addq    %r13,%rcx
2525         xorq    %r8,%r15
2526         shrdq   $28,%r14,%r14
2527         addq    %rcx,%r10
2528         addq    %r15,%rcx
2529         movq    %r10,%r13
2530         addq    %rcx,%r14
2531         shrdq   $23,%r13,%r13
2532         movq    %r14,%rcx
2533         movq    %r11,%r12
2534         shrdq   $5,%r14,%r14
2535         xorq    %r10,%r13
2536         xorq    %rax,%r12
2537         shrdq   $4,%r13,%r13
2538         xorq    %rcx,%r14
2539         andq    %r10,%r12
2540         xorq    %r10,%r13
2541         addq    112(%rsp),%rbx
2542         movq    %rcx,%r15
2543         xorq    %rax,%r12
2544         shrdq   $6,%r14,%r14
2545         xorq    %rdx,%r15
2546         addq    %r12,%rbx
2547         shrdq   $14,%r13,%r13
2548         andq    %r15,%rdi
2549         xorq    %rcx,%r14
2550         addq    %r13,%rbx
2551         xorq    %rdx,%rdi
2552         shrdq   $28,%r14,%r14
2553         addq    %rbx,%r9
2554         addq    %rdi,%rbx
2555         movq    %r9,%r13
2556         addq    %rbx,%r14
2557         shrdq   $23,%r13,%r13
2558         movq    %r14,%rbx
2559         movq    %r10,%r12
2560         shrdq   $5,%r14,%r14
2561         xorq    %r9,%r13
2562         xorq    %r11,%r12
2563         shrdq   $4,%r13,%r13
2564         xorq    %rbx,%r14
2565         andq    %r9,%r12
2566         xorq    %r9,%r13
2567         addq    120(%rsp),%rax
2568         movq    %rbx,%rdi
2569         xorq    %r11,%r12
2570         shrdq   $6,%r14,%r14
2571         xorq    %rcx,%rdi
2572         addq    %r12,%rax
2573         shrdq   $14,%r13,%r13
2574         andq    %rdi,%r15
2575         xorq    %rbx,%r14
2576         addq    %r13,%rax
2577         xorq    %rcx,%r15
2578         shrdq   $28,%r14,%r14
2579         addq    %rax,%r8
2580         addq    %r15,%rax
2581         movq    %r8,%r13
2582         addq    %rax,%r14
2583         movq    128+0(%rsp),%rdi
2584         movq    %r14,%rax
2585
2586         addq    0(%rdi),%rax
2587         leaq    128(%rsi),%rsi
2588         addq    8(%rdi),%rbx
2589         addq    16(%rdi),%rcx
2590         addq    24(%rdi),%rdx
2591         addq    32(%rdi),%r8
2592         addq    40(%rdi),%r9
2593         addq    48(%rdi),%r10
2594         addq    56(%rdi),%r11
2595
2596         cmpq    128+16(%rsp),%rsi
2597
2598         movq    %rax,0(%rdi)
2599         movq    %rbx,8(%rdi)
2600         movq    %rcx,16(%rdi)
2601         movq    %rdx,24(%rdi)
2602         movq    %r8,32(%rdi)
2603         movq    %r9,40(%rdi)
2604         movq    %r10,48(%rdi)
2605         movq    %r11,56(%rdi)
2606         jb      .Lloop_avx
2607
2608         movq    152(%rsp),%rsi
2609 .cfi_def_cfa    %rsi,8
2610         vzeroupper
2611         movq    -48(%rsi),%r15
2612 .cfi_restore    %r15
2613         movq    -40(%rsi),%r14
2614 .cfi_restore    %r14
2615         movq    -32(%rsi),%r13
2616 .cfi_restore    %r13
2617         movq    -24(%rsi),%r12
2618 .cfi_restore    %r12
2619         movq    -16(%rsi),%rbp
2620 .cfi_restore    %rbp
2621         movq    -8(%rsi),%rbx
2622 .cfi_restore    %rbx
2623         leaq    (%rsi),%rsp
2624 .cfi_def_cfa_register   %rsp
2625 .Lepilogue_avx:
2626         RET
2627 .cfi_endproc
2628 SET_SIZE(zfs_sha512_transform_avx)
2629
2630 ENTRY_ALIGN(zfs_sha512_transform_avx2, 64)
2631 .cfi_startproc
2632         ENDBR
2633         movq    %rsp,%rax
2634 .cfi_def_cfa_register   %rax
2635         pushq   %rbx
2636 .cfi_offset     %rbx,-16
2637         pushq   %rbp
2638 .cfi_offset     %rbp,-24
2639         pushq   %r12
2640 .cfi_offset     %r12,-32
2641         pushq   %r13
2642 .cfi_offset     %r13,-40
2643         pushq   %r14
2644 .cfi_offset     %r14,-48
2645         pushq   %r15
2646 .cfi_offset     %r15,-56
2647         subq    $1312,%rsp
2648         shlq    $4,%rdx
2649         andq    $-2048,%rsp
2650         leaq    (%rsi,%rdx,8),%rdx
2651         addq    $1152,%rsp
2652         movq    %rdi,128+0(%rsp)
2653         movq    %rsi,128+8(%rsp)
2654         movq    %rdx,128+16(%rsp)
2655         movq    %rax,152(%rsp)
2656 .cfi_escape     0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
2657 .Lprologue_avx2:
2658
2659         vzeroupper
2660         subq    $-128,%rsi
2661         movq    0(%rdi),%rax
2662         movq    %rsi,%r12
2663         movq    8(%rdi),%rbx
2664         cmpq    %rdx,%rsi
2665         movq    16(%rdi),%rcx
2666         cmoveq  %rsp,%r12
2667         movq    24(%rdi),%rdx
2668         movq    32(%rdi),%r8
2669         movq    40(%rdi),%r9
2670         movq    48(%rdi),%r10
2671         movq    56(%rdi),%r11
2672         jmp     .Loop_avx2
2673 .balign 16
2674 .Loop_avx2:
2675         vmovdqu -128(%rsi),%xmm0
2676         vmovdqu -128+16(%rsi),%xmm1
2677         vmovdqu -128+32(%rsi),%xmm2
2678         leaq    K512+128(%rip),%rbp
2679         vmovdqu -128+48(%rsi),%xmm3
2680         vmovdqu -128+64(%rsi),%xmm4
2681         vmovdqu -128+80(%rsi),%xmm5
2682         vmovdqu -128+96(%rsi),%xmm6
2683         vmovdqu -128+112(%rsi),%xmm7
2684
2685         vmovdqa 1152(%rbp),%ymm10
2686         vinserti128     $1,(%r12),%ymm0,%ymm0
2687         vinserti128     $1,16(%r12),%ymm1,%ymm1
2688         vpshufb %ymm10,%ymm0,%ymm0
2689         vinserti128     $1,32(%r12),%ymm2,%ymm2
2690         vpshufb %ymm10,%ymm1,%ymm1
2691         vinserti128     $1,48(%r12),%ymm3,%ymm3
2692         vpshufb %ymm10,%ymm2,%ymm2
2693         vinserti128     $1,64(%r12),%ymm4,%ymm4
2694         vpshufb %ymm10,%ymm3,%ymm3
2695         vinserti128     $1,80(%r12),%ymm5,%ymm5
2696         vpshufb %ymm10,%ymm4,%ymm4
2697         vinserti128     $1,96(%r12),%ymm6,%ymm6
2698         vpshufb %ymm10,%ymm5,%ymm5
2699         vinserti128     $1,112(%r12),%ymm7,%ymm7
2700
2701         vpaddq  -128(%rbp),%ymm0,%ymm8
2702         vpshufb %ymm10,%ymm6,%ymm6
2703         vpaddq  -96(%rbp),%ymm1,%ymm9
2704         vpshufb %ymm10,%ymm7,%ymm7
2705         vpaddq  -64(%rbp),%ymm2,%ymm10
2706         vpaddq  -32(%rbp),%ymm3,%ymm11
2707         vmovdqa %ymm8,0(%rsp)
2708         vpaddq  0(%rbp),%ymm4,%ymm8
2709         vmovdqa %ymm9,32(%rsp)
2710         vpaddq  32(%rbp),%ymm5,%ymm9
2711         vmovdqa %ymm10,64(%rsp)
2712         vpaddq  64(%rbp),%ymm6,%ymm10
2713         vmovdqa %ymm11,96(%rsp)
2714
2715         movq    152(%rsp),%rdi
2716 .cfi_def_cfa    %rdi,8
2717         leaq    -128(%rsp),%rsp
2718
2719
2720
2721         movq    %rdi,-8(%rsp)
2722 .cfi_escape     0x0f,0x05,0x77,0x78,0x06,0x23,0x08
2723         vpaddq  96(%rbp),%ymm7,%ymm11
2724         vmovdqa %ymm8,0(%rsp)
2725         xorq    %r14,%r14
2726         vmovdqa %ymm9,32(%rsp)
2727         movq    %rbx,%rdi
2728         vmovdqa %ymm10,64(%rsp)
2729         xorq    %rcx,%rdi
2730         vmovdqa %ymm11,96(%rsp)
2731         movq    %r9,%r12
2732         addq    $32*8,%rbp
2733         jmp     .Lavx2_00_47
2734
2735 .balign 16
2736 .Lavx2_00_47:
2737         leaq    -128(%rsp),%rsp
2738 .cfi_escape     0x0f,0x06,0x77,0xf8,0x00,0x06,0x23,0x08
2739
2740         pushq   128-8(%rsp)
2741 .cfi_escape     0x0f,0x05,0x77,0x00,0x06,0x23,0x08
2742         leaq    8(%rsp),%rsp
2743 .cfi_escape     0x0f,0x05,0x77,0x78,0x06,0x23,0x08
2744         vpalignr        $8,%ymm0,%ymm1,%ymm8
2745         addq    0+256(%rsp),%r11
2746         andq    %r8,%r12
2747         rorxq   $41,%r8,%r13
2748         vpalignr        $8,%ymm4,%ymm5,%ymm11
2749         rorxq   $18,%r8,%r15
2750         leaq    (%rax,%r14,1),%rax
2751         leaq    (%r11,%r12,1),%r11
2752         vpsrlq  $1,%ymm8,%ymm10
2753         andnq   %r10,%r8,%r12
2754         xorq    %r15,%r13
2755         rorxq   $14,%r8,%r14
2756         vpaddq  %ymm11,%ymm0,%ymm0
2757         vpsrlq  $7,%ymm8,%ymm11
2758         leaq    (%r11,%r12,1),%r11
2759         xorq    %r14,%r13
2760         movq    %rax,%r15
2761         vpsllq  $56,%ymm8,%ymm9
2762         vpxor   %ymm10,%ymm11,%ymm8
2763         rorxq   $39,%rax,%r12
2764         leaq    (%r11,%r13,1),%r11
2765         xorq    %rbx,%r15
2766         vpsrlq  $7,%ymm10,%ymm10
2767         vpxor   %ymm9,%ymm8,%ymm8
2768         rorxq   $34,%rax,%r14
2769         rorxq   $28,%rax,%r13
2770         leaq    (%rdx,%r11,1),%rdx
2771         vpsllq  $7,%ymm9,%ymm9
2772         vpxor   %ymm10,%ymm8,%ymm8
2773         andq    %r15,%rdi
2774         xorq    %r12,%r14
2775         xorq    %rbx,%rdi
2776         vpsrlq  $6,%ymm7,%ymm11
2777         vpxor   %ymm9,%ymm8,%ymm8
2778         xorq    %r13,%r14
2779         leaq    (%r11,%rdi,1),%r11
2780         movq    %r8,%r12
2781         vpsllq  $3,%ymm7,%ymm10
2782         vpaddq  %ymm8,%ymm0,%ymm0
2783         addq    8+256(%rsp),%r10
2784         andq    %rdx,%r12
2785         rorxq   $41,%rdx,%r13
2786         vpsrlq  $19,%ymm7,%ymm9
2787         vpxor   %ymm10,%ymm11,%ymm11
2788         rorxq   $18,%rdx,%rdi
2789         leaq    (%r11,%r14,1),%r11
2790         leaq    (%r10,%r12,1),%r10
2791         vpsllq  $42,%ymm10,%ymm10
2792         vpxor   %ymm9,%ymm11,%ymm11
2793         andnq   %r9,%rdx,%r12
2794         xorq    %rdi,%r13
2795         rorxq   $14,%rdx,%r14
2796         vpsrlq  $42,%ymm9,%ymm9
2797         vpxor   %ymm10,%ymm11,%ymm11
2798         leaq    (%r10,%r12,1),%r10
2799         xorq    %r14,%r13
2800         movq    %r11,%rdi
2801         vpxor   %ymm9,%ymm11,%ymm11
2802         rorxq   $39,%r11,%r12
2803         leaq    (%r10,%r13,1),%r10
2804         xorq    %rax,%rdi
2805         vpaddq  %ymm11,%ymm0,%ymm0
2806         rorxq   $34,%r11,%r14
2807         rorxq   $28,%r11,%r13
2808         leaq    (%rcx,%r10,1),%rcx
2809         vpaddq  -128(%rbp),%ymm0,%ymm10
2810         andq    %rdi,%r15
2811         xorq    %r12,%r14
2812         xorq    %rax,%r15
2813         xorq    %r13,%r14
2814         leaq    (%r10,%r15,1),%r10
2815         movq    %rdx,%r12
2816         vmovdqa %ymm10,0(%rsp)
2817         vpalignr        $8,%ymm1,%ymm2,%ymm8
2818         addq    32+256(%rsp),%r9
2819         andq    %rcx,%r12
2820         rorxq   $41,%rcx,%r13
2821         vpalignr        $8,%ymm5,%ymm6,%ymm11
2822         rorxq   $18,%rcx,%r15
2823         leaq    (%r10,%r14,1),%r10
2824         leaq    (%r9,%r12,1),%r9
2825         vpsrlq  $1,%ymm8,%ymm10
2826         andnq   %r8,%rcx,%r12
2827         xorq    %r15,%r13
2828         rorxq   $14,%rcx,%r14
2829         vpaddq  %ymm11,%ymm1,%ymm1
2830         vpsrlq  $7,%ymm8,%ymm11
2831         leaq    (%r9,%r12,1),%r9
2832         xorq    %r14,%r13
2833         movq    %r10,%r15
2834         vpsllq  $56,%ymm8,%ymm9
2835         vpxor   %ymm10,%ymm11,%ymm8
2836         rorxq   $39,%r10,%r12
2837         leaq    (%r9,%r13,1),%r9
2838         xorq    %r11,%r15
2839         vpsrlq  $7,%ymm10,%ymm10
2840         vpxor   %ymm9,%ymm8,%ymm8
2841         rorxq   $34,%r10,%r14
2842         rorxq   $28,%r10,%r13
2843         leaq    (%rbx,%r9,1),%rbx
2844         vpsllq  $7,%ymm9,%ymm9
2845         vpxor   %ymm10,%ymm8,%ymm8
2846         andq    %r15,%rdi
2847         xorq    %r12,%r14
2848         xorq    %r11,%rdi
2849         vpsrlq  $6,%ymm0,%ymm11
2850         vpxor   %ymm9,%ymm8,%ymm8
2851         xorq    %r13,%r14
2852         leaq    (%r9,%rdi,1),%r9
2853         movq    %rcx,%r12
2854         vpsllq  $3,%ymm0,%ymm10
2855         vpaddq  %ymm8,%ymm1,%ymm1
2856         addq    40+256(%rsp),%r8
2857         andq    %rbx,%r12
2858         rorxq   $41,%rbx,%r13
2859         vpsrlq  $19,%ymm0,%ymm9
2860         vpxor   %ymm10,%ymm11,%ymm11
2861         rorxq   $18,%rbx,%rdi
2862         leaq    (%r9,%r14,1),%r9
2863         leaq    (%r8,%r12,1),%r8
2864         vpsllq  $42,%ymm10,%ymm10
2865         vpxor   %ymm9,%ymm11,%ymm11
2866         andnq   %rdx,%rbx,%r12
2867         xorq    %rdi,%r13
2868         rorxq   $14,%rbx,%r14
2869         vpsrlq  $42,%ymm9,%ymm9
2870         vpxor   %ymm10,%ymm11,%ymm11
2871         leaq    (%r8,%r12,1),%r8
2872         xorq    %r14,%r13
2873         movq    %r9,%rdi
2874         vpxor   %ymm9,%ymm11,%ymm11
2875         rorxq   $39,%r9,%r12
2876         leaq    (%r8,%r13,1),%r8
2877         xorq    %r10,%rdi
2878         vpaddq  %ymm11,%ymm1,%ymm1
2879         rorxq   $34,%r9,%r14
2880         rorxq   $28,%r9,%r13
2881         leaq    (%rax,%r8,1),%rax
2882         vpaddq  -96(%rbp),%ymm1,%ymm10
2883         andq    %rdi,%r15
2884         xorq    %r12,%r14
2885         xorq    %r10,%r15
2886         xorq    %r13,%r14
2887         leaq    (%r8,%r15,1),%r8
2888         movq    %rbx,%r12
2889         vmovdqa %ymm10,32(%rsp)
2890         vpalignr        $8,%ymm2,%ymm3,%ymm8
2891         addq    64+256(%rsp),%rdx
2892         andq    %rax,%r12
2893         rorxq   $41,%rax,%r13
2894         vpalignr        $8,%ymm6,%ymm7,%ymm11
2895         rorxq   $18,%rax,%r15
2896         leaq    (%r8,%r14,1),%r8
2897         leaq    (%rdx,%r12,1),%rdx
2898         vpsrlq  $1,%ymm8,%ymm10
2899         andnq   %rcx,%rax,%r12
2900         xorq    %r15,%r13
2901         rorxq   $14,%rax,%r14
2902         vpaddq  %ymm11,%ymm2,%ymm2
2903         vpsrlq  $7,%ymm8,%ymm11
2904         leaq    (%rdx,%r12,1),%rdx
2905         xorq    %r14,%r13
2906         movq    %r8,%r15
2907         vpsllq  $56,%ymm8,%ymm9
2908         vpxor   %ymm10,%ymm11,%ymm8
2909         rorxq   $39,%r8,%r12
2910         leaq    (%rdx,%r13,1),%rdx
2911         xorq    %r9,%r15
2912         vpsrlq  $7,%ymm10,%ymm10
2913         vpxor   %ymm9,%ymm8,%ymm8
2914         rorxq   $34,%r8,%r14
2915         rorxq   $28,%r8,%r13
2916         leaq    (%r11,%rdx,1),%r11
2917         vpsllq  $7,%ymm9,%ymm9
2918         vpxor   %ymm10,%ymm8,%ymm8
2919         andq    %r15,%rdi
2920         xorq    %r12,%r14
2921         xorq    %r9,%rdi
2922         vpsrlq  $6,%ymm1,%ymm11
2923         vpxor   %ymm9,%ymm8,%ymm8
2924         xorq    %r13,%r14
2925         leaq    (%rdx,%rdi,1),%rdx
2926         movq    %rax,%r12
2927         vpsllq  $3,%ymm1,%ymm10
2928         vpaddq  %ymm8,%ymm2,%ymm2
2929         addq    72+256(%rsp),%rcx
2930         andq    %r11,%r12
2931         rorxq   $41,%r11,%r13
2932         vpsrlq  $19,%ymm1,%ymm9
2933         vpxor   %ymm10,%ymm11,%ymm11
2934         rorxq   $18,%r11,%rdi
2935         leaq    (%rdx,%r14,1),%rdx
2936         leaq    (%rcx,%r12,1),%rcx
2937         vpsllq  $42,%ymm10,%ymm10
2938         vpxor   %ymm9,%ymm11,%ymm11
2939         andnq   %rbx,%r11,%r12
2940         xorq    %rdi,%r13
2941         rorxq   $14,%r11,%r14
2942         vpsrlq  $42,%ymm9,%ymm9
2943         vpxor   %ymm10,%ymm11,%ymm11
2944         leaq    (%rcx,%r12,1),%rcx
2945         xorq    %r14,%r13
2946         movq    %rdx,%rdi
2947         vpxor   %ymm9,%ymm11,%ymm11
2948         rorxq   $39,%rdx,%r12
2949         leaq    (%rcx,%r13,1),%rcx
2950         xorq    %r8,%rdi
2951         vpaddq  %ymm11,%ymm2,%ymm2
2952         rorxq   $34,%rdx,%r14
2953         rorxq   $28,%rdx,%r13
2954         leaq    (%r10,%rcx,1),%r10
2955         vpaddq  -64(%rbp),%ymm2,%ymm10
2956         andq    %rdi,%r15
2957         xorq    %r12,%r14
2958         xorq    %r8,%r15
2959         xorq    %r13,%r14
2960         leaq    (%rcx,%r15,1),%rcx
2961         movq    %r11,%r12
2962         vmovdqa %ymm10,64(%rsp)
2963         vpalignr        $8,%ymm3,%ymm4,%ymm8
2964         addq    96+256(%rsp),%rbx
2965         andq    %r10,%r12
2966         rorxq   $41,%r10,%r13
2967         vpalignr        $8,%ymm7,%ymm0,%ymm11
2968         rorxq   $18,%r10,%r15
2969         leaq    (%rcx,%r14,1),%rcx
2970         leaq    (%rbx,%r12,1),%rbx
2971         vpsrlq  $1,%ymm8,%ymm10
2972         andnq   %rax,%r10,%r12
2973         xorq    %r15,%r13
2974         rorxq   $14,%r10,%r14
2975         vpaddq  %ymm11,%ymm3,%ymm3
2976         vpsrlq  $7,%ymm8,%ymm11
2977         leaq    (%rbx,%r12,1),%rbx
2978         xorq    %r14,%r13
2979         movq    %rcx,%r15
2980         vpsllq  $56,%ymm8,%ymm9
2981         vpxor   %ymm10,%ymm11,%ymm8
2982         rorxq   $39,%rcx,%r12
2983         leaq    (%rbx,%r13,1),%rbx
2984         xorq    %rdx,%r15
2985         vpsrlq  $7,%ymm10,%ymm10
2986         vpxor   %ymm9,%ymm8,%ymm8
2987         rorxq   $34,%rcx,%r14
2988         rorxq   $28,%rcx,%r13
2989         leaq    (%r9,%rbx,1),%r9
2990         vpsllq  $7,%ymm9,%ymm9
2991         vpxor   %ymm10,%ymm8,%ymm8
2992         andq    %r15,%rdi
2993         xorq    %r12,%r14
2994         xorq    %rdx,%rdi
2995         vpsrlq  $6,%ymm2,%ymm11
2996         vpxor   %ymm9,%ymm8,%ymm8
2997         xorq    %r13,%r14
2998         leaq    (%rbx,%rdi,1),%rbx
2999         movq    %r10,%r12
3000         vpsllq  $3,%ymm2,%ymm10
3001         vpaddq  %ymm8,%ymm3,%ymm3
3002         addq    104+256(%rsp),%rax
3003         andq    %r9,%r12
3004         rorxq   $41,%r9,%r13
3005         vpsrlq  $19,%ymm2,%ymm9
3006         vpxor   %ymm10,%ymm11,%ymm11
3007         rorxq   $18,%r9,%rdi
3008         leaq    (%rbx,%r14,1),%rbx
3009         leaq    (%rax,%r12,1),%rax
3010         vpsllq  $42,%ymm10,%ymm10
3011         vpxor   %ymm9,%ymm11,%ymm11
3012         andnq   %r11,%r9,%r12
3013         xorq    %rdi,%r13
3014         rorxq   $14,%r9,%r14
3015         vpsrlq  $42,%ymm9,%ymm9
3016         vpxor   %ymm10,%ymm11,%ymm11
3017         leaq    (%rax,%r12,1),%rax
3018         xorq    %r14,%r13
3019         movq    %rbx,%rdi
3020         vpxor   %ymm9,%ymm11,%ymm11
3021         rorxq   $39,%rbx,%r12
3022         leaq    (%rax,%r13,1),%rax
3023         xorq    %rcx,%rdi
3024         vpaddq  %ymm11,%ymm3,%ymm3
3025         rorxq   $34,%rbx,%r14
3026         rorxq   $28,%rbx,%r13
3027         leaq    (%r8,%rax,1),%r8
3028         vpaddq  -32(%rbp),%ymm3,%ymm10
3029         andq    %rdi,%r15
3030         xorq    %r12,%r14
3031         xorq    %rcx,%r15
3032         xorq    %r13,%r14
3033         leaq    (%rax,%r15,1),%rax
3034         movq    %r9,%r12
3035         vmovdqa %ymm10,96(%rsp)
3036         leaq    -128(%rsp),%rsp
3037 .cfi_escape     0x0f,0x06,0x77,0xf8,0x00,0x06,0x23,0x08
3038
3039         pushq   128-8(%rsp)
3040 .cfi_escape     0x0f,0x05,0x77,0x00,0x06,0x23,0x08
3041         leaq    8(%rsp),%rsp
3042 .cfi_escape     0x0f,0x05,0x77,0x78,0x06,0x23,0x08
3043         vpalignr        $8,%ymm4,%ymm5,%ymm8
3044         addq    0+256(%rsp),%r11
3045         andq    %r8,%r12
3046         rorxq   $41,%r8,%r13
3047         vpalignr        $8,%ymm0,%ymm1,%ymm11
3048         rorxq   $18,%r8,%r15
3049         leaq    (%rax,%r14,1),%rax
3050         leaq    (%r11,%r12,1),%r11
3051         vpsrlq  $1,%ymm8,%ymm10
3052         andnq   %r10,%r8,%r12
3053         xorq    %r15,%r13
3054         rorxq   $14,%r8,%r14
3055         vpaddq  %ymm11,%ymm4,%ymm4
3056         vpsrlq  $7,%ymm8,%ymm11
3057         leaq    (%r11,%r12,1),%r11
3058         xorq    %r14,%r13
3059         movq    %rax,%r15
3060         vpsllq  $56,%ymm8,%ymm9
3061         vpxor   %ymm10,%ymm11,%ymm8
3062         rorxq   $39,%rax,%r12
3063         leaq    (%r11,%r13,1),%r11
3064         xorq    %rbx,%r15
3065         vpsrlq  $7,%ymm10,%ymm10
3066         vpxor   %ymm9,%ymm8,%ymm8
3067         rorxq   $34,%rax,%r14
3068         rorxq   $28,%rax,%r13
3069         leaq    (%rdx,%r11,1),%rdx
3070         vpsllq  $7,%ymm9,%ymm9
3071         vpxor   %ymm10,%ymm8,%ymm8
3072         andq    %r15,%rdi
3073         xorq    %r12,%r14
3074         xorq    %rbx,%rdi
3075         vpsrlq  $6,%ymm3,%ymm11
3076         vpxor   %ymm9,%ymm8,%ymm8
3077         xorq    %r13,%r14
3078         leaq    (%r11,%rdi,1),%r11
3079         movq    %r8,%r12
3080         vpsllq  $3,%ymm3,%ymm10
3081         vpaddq  %ymm8,%ymm4,%ymm4
3082         addq    8+256(%rsp),%r10
3083         andq    %rdx,%r12
3084         rorxq   $41,%rdx,%r13
3085         vpsrlq  $19,%ymm3,%ymm9
3086         vpxor   %ymm10,%ymm11,%ymm11
3087         rorxq   $18,%rdx,%rdi
3088         leaq    (%r11,%r14,1),%r11
3089         leaq    (%r10,%r12,1),%r10
3090         vpsllq  $42,%ymm10,%ymm10
3091         vpxor   %ymm9,%ymm11,%ymm11
3092         andnq   %r9,%rdx,%r12
3093         xorq    %rdi,%r13
3094         rorxq   $14,%rdx,%r14
3095         vpsrlq  $42,%ymm9,%ymm9
3096         vpxor   %ymm10,%ymm11,%ymm11
3097         leaq    (%r10,%r12,1),%r10
3098         xorq    %r14,%r13
3099         movq    %r11,%rdi
3100         vpxor   %ymm9,%ymm11,%ymm11
3101         rorxq   $39,%r11,%r12
3102         leaq    (%r10,%r13,1),%r10
3103         xorq    %rax,%rdi
3104         vpaddq  %ymm11,%ymm4,%ymm4
3105         rorxq   $34,%r11,%r14
3106         rorxq   $28,%r11,%r13
3107         leaq    (%rcx,%r10,1),%rcx
3108         vpaddq  0(%rbp),%ymm4,%ymm10
3109         andq    %rdi,%r15
3110         xorq    %r12,%r14
3111         xorq    %rax,%r15
3112         xorq    %r13,%r14
3113         leaq    (%r10,%r15,1),%r10
3114         movq    %rdx,%r12
3115         vmovdqa %ymm10,0(%rsp)
3116         vpalignr        $8,%ymm5,%ymm6,%ymm8
3117         addq    32+256(%rsp),%r9
3118         andq    %rcx,%r12
3119         rorxq   $41,%rcx,%r13
3120         vpalignr        $8,%ymm1,%ymm2,%ymm11
3121         rorxq   $18,%rcx,%r15
3122         leaq    (%r10,%r14,1),%r10
3123         leaq    (%r9,%r12,1),%r9
3124         vpsrlq  $1,%ymm8,%ymm10
3125         andnq   %r8,%rcx,%r12
3126         xorq    %r15,%r13
3127         rorxq   $14,%rcx,%r14
3128         vpaddq  %ymm11,%ymm5,%ymm5
3129         vpsrlq  $7,%ymm8,%ymm11
3130         leaq    (%r9,%r12,1),%r9
3131         xorq    %r14,%r13
3132         movq    %r10,%r15
3133         vpsllq  $56,%ymm8,%ymm9
3134         vpxor   %ymm10,%ymm11,%ymm8
3135         rorxq   $39,%r10,%r12
3136         leaq    (%r9,%r13,1),%r9
3137         xorq    %r11,%r15
3138         vpsrlq  $7,%ymm10,%ymm10
3139         vpxor   %ymm9,%ymm8,%ymm8
3140         rorxq   $34,%r10,%r14
3141         rorxq   $28,%r10,%r13
3142         leaq    (%rbx,%r9,1),%rbx
3143         vpsllq  $7,%ymm9,%ymm9
3144         vpxor   %ymm10,%ymm8,%ymm8
3145         andq    %r15,%rdi
3146         xorq    %r12,%r14
3147         xorq    %r11,%rdi
3148         vpsrlq  $6,%ymm4,%ymm11
3149         vpxor   %ymm9,%ymm8,%ymm8
3150         xorq    %r13,%r14
3151         leaq    (%r9,%rdi,1),%r9
3152         movq    %rcx,%r12
3153         vpsllq  $3,%ymm4,%ymm10
3154         vpaddq  %ymm8,%ymm5,%ymm5
3155         addq    40+256(%rsp),%r8
3156         andq    %rbx,%r12
3157         rorxq   $41,%rbx,%r13
3158         vpsrlq  $19,%ymm4,%ymm9
3159         vpxor   %ymm10,%ymm11,%ymm11
3160         rorxq   $18,%rbx,%rdi
3161         leaq    (%r9,%r14,1),%r9
3162         leaq    (%r8,%r12,1),%r8
3163         vpsllq  $42,%ymm10,%ymm10
3164         vpxor   %ymm9,%ymm11,%ymm11
3165         andnq   %rdx,%rbx,%r12
3166         xorq    %rdi,%r13
3167         rorxq   $14,%rbx,%r14
3168         vpsrlq  $42,%ymm9,%ymm9
3169         vpxor   %ymm10,%ymm11,%ymm11
3170         leaq    (%r8,%r12,1),%r8
3171         xorq    %r14,%r13
3172         movq    %r9,%rdi
3173         vpxor   %ymm9,%ymm11,%ymm11
3174         rorxq   $39,%r9,%r12
3175         leaq    (%r8,%r13,1),%r8
3176         xorq    %r10,%rdi
3177         vpaddq  %ymm11,%ymm5,%ymm5
3178         rorxq   $34,%r9,%r14
3179         rorxq   $28,%r9,%r13
3180         leaq    (%rax,%r8,1),%rax
3181         vpaddq  32(%rbp),%ymm5,%ymm10
3182         andq    %rdi,%r15
3183         xorq    %r12,%r14
3184         xorq    %r10,%r15
3185         xorq    %r13,%r14
3186         leaq    (%r8,%r15,1),%r8
3187         movq    %rbx,%r12
3188         vmovdqa %ymm10,32(%rsp)
3189         vpalignr        $8,%ymm6,%ymm7,%ymm8
3190         addq    64+256(%rsp),%rdx
3191         andq    %rax,%r12
3192         rorxq   $41,%rax,%r13
3193         vpalignr        $8,%ymm2,%ymm3,%ymm11
3194         rorxq   $18,%rax,%r15
3195         leaq    (%r8,%r14,1),%r8
3196         leaq    (%rdx,%r12,1),%rdx
3197         vpsrlq  $1,%ymm8,%ymm10
3198         andnq   %rcx,%rax,%r12
3199         xorq    %r15,%r13
3200         rorxq   $14,%rax,%r14
3201         vpaddq  %ymm11,%ymm6,%ymm6
3202         vpsrlq  $7,%ymm8,%ymm11
3203         leaq    (%rdx,%r12,1),%rdx
3204         xorq    %r14,%r13
3205         movq    %r8,%r15
3206         vpsllq  $56,%ymm8,%ymm9
3207         vpxor   %ymm10,%ymm11,%ymm8
3208         rorxq   $39,%r8,%r12
3209         leaq    (%rdx,%r13,1),%rdx
3210         xorq    %r9,%r15
3211         vpsrlq  $7,%ymm10,%ymm10
3212         vpxor   %ymm9,%ymm8,%ymm8
3213         rorxq   $34,%r8,%r14
3214         rorxq   $28,%r8,%r13
3215         leaq    (%r11,%rdx,1),%r11
3216         vpsllq  $7,%ymm9,%ymm9
3217         vpxor   %ymm10,%ymm8,%ymm8
3218         andq    %r15,%rdi
3219         xorq    %r12,%r14
3220         xorq    %r9,%rdi
3221         vpsrlq  $6,%ymm5,%ymm11
3222         vpxor   %ymm9,%ymm8,%ymm8
3223         xorq    %r13,%r14
3224         leaq    (%rdx,%rdi,1),%rdx
3225         movq    %rax,%r12
3226         vpsllq  $3,%ymm5,%ymm10
3227         vpaddq  %ymm8,%ymm6,%ymm6
3228         addq    72+256(%rsp),%rcx
3229         andq    %r11,%r12
3230         rorxq   $41,%r11,%r13
3231         vpsrlq  $19,%ymm5,%ymm9
3232         vpxor   %ymm10,%ymm11,%ymm11
3233         rorxq   $18,%r11,%rdi
3234         leaq    (%rdx,%r14,1),%rdx
3235         leaq    (%rcx,%r12,1),%rcx
3236         vpsllq  $42,%ymm10,%ymm10
3237         vpxor   %ymm9,%ymm11,%ymm11
3238         andnq   %rbx,%r11,%r12
3239         xorq    %rdi,%r13
3240         rorxq   $14,%r11,%r14
3241         vpsrlq  $42,%ymm9,%ymm9
3242         vpxor   %ymm10,%ymm11,%ymm11
3243         leaq    (%rcx,%r12,1),%rcx
3244         xorq    %r14,%r13
3245         movq    %rdx,%rdi
3246         vpxor   %ymm9,%ymm11,%ymm11
3247         rorxq   $39,%rdx,%r12
3248         leaq    (%rcx,%r13,1),%rcx
3249         xorq    %r8,%rdi
3250         vpaddq  %ymm11,%ymm6,%ymm6
3251         rorxq   $34,%rdx,%r14
3252         rorxq   $28,%rdx,%r13
3253         leaq    (%r10,%rcx,1),%r10
3254         vpaddq  64(%rbp),%ymm6,%ymm10
3255         andq    %rdi,%r15
3256         xorq    %r12,%r14
3257         xorq    %r8,%r15
3258         xorq    %r13,%r14
3259         leaq    (%rcx,%r15,1),%rcx
3260         movq    %r11,%r12
3261         vmovdqa %ymm10,64(%rsp)
3262         vpalignr        $8,%ymm7,%ymm0,%ymm8
3263         addq    96+256(%rsp),%rbx
3264         andq    %r10,%r12
3265         rorxq   $41,%r10,%r13
3266         vpalignr        $8,%ymm3,%ymm4,%ymm11
3267         rorxq   $18,%r10,%r15
3268         leaq    (%rcx,%r14,1),%rcx
3269         leaq    (%rbx,%r12,1),%rbx
3270         vpsrlq  $1,%ymm8,%ymm10
3271         andnq   %rax,%r10,%r12
3272         xorq    %r15,%r13
3273         rorxq   $14,%r10,%r14
3274         vpaddq  %ymm11,%ymm7,%ymm7
3275         vpsrlq  $7,%ymm8,%ymm11
3276         leaq    (%rbx,%r12,1),%rbx
3277         xorq    %r14,%r13
3278         movq    %rcx,%r15
3279         vpsllq  $56,%ymm8,%ymm9
3280         vpxor   %ymm10,%ymm11,%ymm8
3281         rorxq   $39,%rcx,%r12
3282         leaq    (%rbx,%r13,1),%rbx
3283         xorq    %rdx,%r15
3284         vpsrlq  $7,%ymm10,%ymm10
3285         vpxor   %ymm9,%ymm8,%ymm8
3286         rorxq   $34,%rcx,%r14
3287         rorxq   $28,%rcx,%r13
3288         leaq    (%r9,%rbx,1),%r9
3289         vpsllq  $7,%ymm9,%ymm9
3290         vpxor   %ymm10,%ymm8,%ymm8
3291         andq    %r15,%rdi
3292         xorq    %r12,%r14
3293         xorq    %rdx,%rdi
3294         vpsrlq  $6,%ymm6,%ymm11
3295         vpxor   %ymm9,%ymm8,%ymm8
3296         xorq    %r13,%r14
3297         leaq    (%rbx,%rdi,1),%rbx
3298         movq    %r10,%r12
3299         vpsllq  $3,%ymm6,%ymm10
3300         vpaddq  %ymm8,%ymm7,%ymm7
3301         addq    104+256(%rsp),%rax
3302         andq    %r9,%r12
3303         rorxq   $41,%r9,%r13
3304         vpsrlq  $19,%ymm6,%ymm9
3305         vpxor   %ymm10,%ymm11,%ymm11
3306         rorxq   $18,%r9,%rdi
3307         leaq    (%rbx,%r14,1),%rbx
3308         leaq    (%rax,%r12,1),%rax
3309         vpsllq  $42,%ymm10,%ymm10
3310         vpxor   %ymm9,%ymm11,%ymm11
3311         andnq   %r11,%r9,%r12
3312         xorq    %rdi,%r13
3313         rorxq   $14,%r9,%r14
3314         vpsrlq  $42,%ymm9,%ymm9
3315         vpxor   %ymm10,%ymm11,%ymm11
3316         leaq    (%rax,%r12,1),%rax
3317         xorq    %r14,%r13
3318         movq    %rbx,%rdi
3319         vpxor   %ymm9,%ymm11,%ymm11
3320         rorxq   $39,%rbx,%r12
3321         leaq    (%rax,%r13,1),%rax
3322         xorq    %rcx,%rdi
3323         vpaddq  %ymm11,%ymm7,%ymm7
3324         rorxq   $34,%rbx,%r14
3325         rorxq   $28,%rbx,%r13
3326         leaq    (%r8,%rax,1),%r8
3327         vpaddq  96(%rbp),%ymm7,%ymm10
3328         andq    %rdi,%r15
3329         xorq    %r12,%r14
3330         xorq    %rcx,%r15
3331         xorq    %r13,%r14
3332         leaq    (%rax,%r15,1),%rax
3333         movq    %r9,%r12
3334         vmovdqa %ymm10,96(%rsp)
3335         leaq    256(%rbp),%rbp
3336         cmpb    $0,-121(%rbp)
3337         jne     .Lavx2_00_47
3338         addq    0+128(%rsp),%r11
3339         andq    %r8,%r12
3340         rorxq   $41,%r8,%r13
3341         rorxq   $18,%r8,%r15
3342         leaq    (%rax,%r14,1),%rax
3343         leaq    (%r11,%r12,1),%r11
3344         andnq   %r10,%r8,%r12
3345         xorq    %r15,%r13
3346         rorxq   $14,%r8,%r14
3347         leaq    (%r11,%r12,1),%r11
3348         xorq    %r14,%r13
3349         movq    %rax,%r15
3350         rorxq   $39,%rax,%r12
3351         leaq    (%r11,%r13,1),%r11
3352         xorq    %rbx,%r15
3353         rorxq   $34,%rax,%r14
3354         rorxq   $28,%rax,%r13
3355         leaq    (%rdx,%r11,1),%rdx
3356         andq    %r15,%rdi
3357         xorq    %r12,%r14
3358         xorq    %rbx,%rdi
3359         xorq    %r13,%r14
3360         leaq    (%r11,%rdi,1),%r11
3361         movq    %r8,%r12
3362         addq    8+128(%rsp),%r10
3363         andq    %rdx,%r12
3364         rorxq   $41,%rdx,%r13
3365         rorxq   $18,%rdx,%rdi
3366         leaq    (%r11,%r14,1),%r11
3367         leaq    (%r10,%r12,1),%r10
3368         andnq   %r9,%rdx,%r12
3369         xorq    %rdi,%r13
3370         rorxq   $14,%rdx,%r14
3371         leaq    (%r10,%r12,1),%r10
3372         xorq    %r14,%r13
3373         movq    %r11,%rdi
3374         rorxq   $39,%r11,%r12
3375         leaq    (%r10,%r13,1),%r10
3376         xorq    %rax,%rdi
3377         rorxq   $34,%r11,%r14
3378         rorxq   $28,%r11,%r13
3379         leaq    (%rcx,%r10,1),%rcx
3380         andq    %rdi,%r15
3381         xorq    %r12,%r14
3382         xorq    %rax,%r15
3383         xorq    %r13,%r14
3384         leaq    (%r10,%r15,1),%r10
3385         movq    %rdx,%r12
3386         addq    32+128(%rsp),%r9
3387         andq    %rcx,%r12
3388         rorxq   $41,%rcx,%r13
3389         rorxq   $18,%rcx,%r15
3390         leaq    (%r10,%r14,1),%r10
3391         leaq    (%r9,%r12,1),%r9
3392         andnq   %r8,%rcx,%r12
3393         xorq    %r15,%r13
3394         rorxq   $14,%rcx,%r14
3395         leaq    (%r9,%r12,1),%r9
3396         xorq    %r14,%r13
3397         movq    %r10,%r15
3398         rorxq   $39,%r10,%r12
3399         leaq    (%r9,%r13,1),%r9
3400         xorq    %r11,%r15
3401         rorxq   $34,%r10,%r14
3402         rorxq   $28,%r10,%r13
3403         leaq    (%rbx,%r9,1),%rbx
3404         andq    %r15,%rdi
3405         xorq    %r12,%r14
3406         xorq    %r11,%rdi
3407         xorq    %r13,%r14
3408         leaq    (%r9,%rdi,1),%r9
3409         movq    %rcx,%r12
3410         addq    40+128(%rsp),%r8
3411         andq    %rbx,%r12
3412         rorxq   $41,%rbx,%r13
3413         rorxq   $18,%rbx,%rdi
3414         leaq    (%r9,%r14,1),%r9
3415         leaq    (%r8,%r12,1),%r8
3416         andnq   %rdx,%rbx,%r12
3417         xorq    %rdi,%r13
3418         rorxq   $14,%rbx,%r14
3419         leaq    (%r8,%r12,1),%r8
3420         xorq    %r14,%r13
3421         movq    %r9,%rdi
3422         rorxq   $39,%r9,%r12
3423         leaq    (%r8,%r13,1),%r8
3424         xorq    %r10,%rdi
3425         rorxq   $34,%r9,%r14
3426         rorxq   $28,%r9,%r13
3427         leaq    (%rax,%r8,1),%rax
3428         andq    %rdi,%r15
3429         xorq    %r12,%r14
3430         xorq    %r10,%r15
3431         xorq    %r13,%r14
3432         leaq    (%r8,%r15,1),%r8
3433         movq    %rbx,%r12
3434         addq    64+128(%rsp),%rdx
3435         andq    %rax,%r12
3436         rorxq   $41,%rax,%r13
3437         rorxq   $18,%rax,%r15
3438         leaq    (%r8,%r14,1),%r8
3439         leaq    (%rdx,%r12,1),%rdx
3440         andnq   %rcx,%rax,%r12
3441         xorq    %r15,%r13
3442         rorxq   $14,%rax,%r14
3443         leaq    (%rdx,%r12,1),%rdx
3444         xorq    %r14,%r13
3445         movq    %r8,%r15
3446         rorxq   $39,%r8,%r12
3447         leaq    (%rdx,%r13,1),%rdx
3448         xorq    %r9,%r15
3449         rorxq   $34,%r8,%r14
3450         rorxq   $28,%r8,%r13
3451         leaq    (%r11,%rdx,1),%r11
3452         andq    %r15,%rdi
3453         xorq    %r12,%r14
3454         xorq    %r9,%rdi
3455         xorq    %r13,%r14
3456         leaq    (%rdx,%rdi,1),%rdx
3457         movq    %rax,%r12
3458         addq    72+128(%rsp),%rcx
3459         andq    %r11,%r12
3460         rorxq   $41,%r11,%r13
3461         rorxq   $18,%r11,%rdi
3462         leaq    (%rdx,%r14,1),%rdx
3463         leaq    (%rcx,%r12,1),%rcx
3464         andnq   %rbx,%r11,%r12
3465         xorq    %rdi,%r13
3466         rorxq   $14,%r11,%r14
3467         leaq    (%rcx,%r12,1),%rcx
3468         xorq    %r14,%r13
3469         movq    %rdx,%rdi
3470         rorxq   $39,%rdx,%r12
3471         leaq    (%rcx,%r13,1),%rcx
3472         xorq    %r8,%rdi
3473         rorxq   $34,%rdx,%r14
3474         rorxq   $28,%rdx,%r13
3475         leaq    (%r10,%rcx,1),%r10
3476         andq    %rdi,%r15
3477         xorq    %r12,%r14
3478         xorq    %r8,%r15
3479         xorq    %r13,%r14
3480         leaq    (%rcx,%r15,1),%rcx
3481         movq    %r11,%r12
3482         addq    96+128(%rsp),%rbx
3483         andq    %r10,%r12
3484         rorxq   $41,%r10,%r13
3485         rorxq   $18,%r10,%r15
3486         leaq    (%rcx,%r14,1),%rcx
3487         leaq    (%rbx,%r12,1),%rbx
3488         andnq   %rax,%r10,%r12
3489         xorq    %r15,%r13
3490         rorxq   $14,%r10,%r14
3491         leaq    (%rbx,%r12,1),%rbx
3492         xorq    %r14,%r13
3493         movq    %rcx,%r15
3494         rorxq   $39,%rcx,%r12
3495         leaq    (%rbx,%r13,1),%rbx
3496         xorq    %rdx,%r15
3497         rorxq   $34,%rcx,%r14
3498         rorxq   $28,%rcx,%r13
3499         leaq    (%r9,%rbx,1),%r9
3500         andq    %r15,%rdi
3501         xorq    %r12,%r14
3502         xorq    %rdx,%rdi
3503         xorq    %r13,%r14
3504         leaq    (%rbx,%rdi,1),%rbx
3505         movq    %r10,%r12
3506         addq    104+128(%rsp),%rax
3507         andq    %r9,%r12
3508         rorxq   $41,%r9,%r13
3509         rorxq   $18,%r9,%rdi
3510         leaq    (%rbx,%r14,1),%rbx
3511         leaq    (%rax,%r12,1),%rax
3512         andnq   %r11,%r9,%r12
3513         xorq    %rdi,%r13
3514         rorxq   $14,%r9,%r14
3515         leaq    (%rax,%r12,1),%rax
3516         xorq    %r14,%r13
3517         movq    %rbx,%rdi
3518         rorxq   $39,%rbx,%r12
3519         leaq    (%rax,%r13,1),%rax
3520         xorq    %rcx,%rdi
3521         rorxq   $34,%rbx,%r14
3522         rorxq   $28,%rbx,%r13
3523         leaq    (%r8,%rax,1),%r8
3524         andq    %rdi,%r15
3525         xorq    %r12,%r14
3526         xorq    %rcx,%r15
3527         xorq    %r13,%r14
3528         leaq    (%rax,%r15,1),%rax
3529         movq    %r9,%r12
3530         addq    0(%rsp),%r11
3531         andq    %r8,%r12
3532         rorxq   $41,%r8,%r13
3533         rorxq   $18,%r8,%r15
3534         leaq    (%rax,%r14,1),%rax
3535         leaq    (%r11,%r12,1),%r11
3536         andnq   %r10,%r8,%r12
3537         xorq    %r15,%r13
3538         rorxq   $14,%r8,%r14
3539         leaq    (%r11,%r12,1),%r11
3540         xorq    %r14,%r13
3541         movq    %rax,%r15
3542         rorxq   $39,%rax,%r12
3543         leaq    (%r11,%r13,1),%r11
3544         xorq    %rbx,%r15
3545         rorxq   $34,%rax,%r14
3546         rorxq   $28,%rax,%r13
3547         leaq    (%rdx,%r11,1),%rdx
3548         andq    %r15,%rdi
3549         xorq    %r12,%r14
3550         xorq    %rbx,%rdi
3551         xorq    %r13,%r14
3552         leaq    (%r11,%rdi,1),%r11
3553         movq    %r8,%r12
3554         addq    8(%rsp),%r10
3555         andq    %rdx,%r12
3556         rorxq   $41,%rdx,%r13
3557         rorxq   $18,%rdx,%rdi
3558         leaq    (%r11,%r14,1),%r11
3559         leaq    (%r10,%r12,1),%r10
3560         andnq   %r9,%rdx,%r12
3561         xorq    %rdi,%r13
3562         rorxq   $14,%rdx,%r14
3563         leaq    (%r10,%r12,1),%r10
3564         xorq    %r14,%r13
3565         movq    %r11,%rdi
3566         rorxq   $39,%r11,%r12
3567         leaq    (%r10,%r13,1),%r10
3568         xorq    %rax,%rdi
3569         rorxq   $34,%r11,%r14
3570         rorxq   $28,%r11,%r13
3571         leaq    (%rcx,%r10,1),%rcx
3572         andq    %rdi,%r15
3573         xorq    %r12,%r14
3574         xorq    %rax,%r15
3575         xorq    %r13,%r14
3576         leaq    (%r10,%r15,1),%r10
3577         movq    %rdx,%r12
3578         addq    32(%rsp),%r9
3579         andq    %rcx,%r12
3580         rorxq   $41,%rcx,%r13
3581         rorxq   $18,%rcx,%r15
3582         leaq    (%r10,%r14,1),%r10
3583         leaq    (%r9,%r12,1),%r9
3584         andnq   %r8,%rcx,%r12
3585         xorq    %r15,%r13
3586         rorxq   $14,%rcx,%r14
3587         leaq    (%r9,%r12,1),%r9
3588         xorq    %r14,%r13
3589         movq    %r10,%r15
3590         rorxq   $39,%r10,%r12
3591         leaq    (%r9,%r13,1),%r9
3592         xorq    %r11,%r15
3593         rorxq   $34,%r10,%r14
3594         rorxq   $28,%r10,%r13
3595         leaq    (%rbx,%r9,1),%rbx
3596         andq    %r15,%rdi
3597         xorq    %r12,%r14
3598         xorq    %r11,%rdi
3599         xorq    %r13,%r14
3600         leaq    (%r9,%rdi,1),%r9
3601         movq    %rcx,%r12
3602         addq    40(%rsp),%r8
3603         andq    %rbx,%r12
3604         rorxq   $41,%rbx,%r13
3605         rorxq   $18,%rbx,%rdi
3606         leaq    (%r9,%r14,1),%r9
3607         leaq    (%r8,%r12,1),%r8
3608         andnq   %rdx,%rbx,%r12
3609         xorq    %rdi,%r13
3610         rorxq   $14,%rbx,%r14
3611         leaq    (%r8,%r12,1),%r8
3612         xorq    %r14,%r13
3613         movq    %r9,%rdi
3614         rorxq   $39,%r9,%r12
3615         leaq    (%r8,%r13,1),%r8
3616         xorq    %r10,%rdi
3617         rorxq   $34,%r9,%r14
3618         rorxq   $28,%r9,%r13
3619         leaq    (%rax,%r8,1),%rax
3620         andq    %rdi,%r15
3621         xorq    %r12,%r14
3622         xorq    %r10,%r15
3623         xorq    %r13,%r14
3624         leaq    (%r8,%r15,1),%r8
3625         movq    %rbx,%r12
3626         addq    64(%rsp),%rdx
3627         andq    %rax,%r12
3628         rorxq   $41,%rax,%r13
3629         rorxq   $18,%rax,%r15
3630         leaq    (%r8,%r14,1),%r8
3631         leaq    (%rdx,%r12,1),%rdx
3632         andnq   %rcx,%rax,%r12
3633         xorq    %r15,%r13
3634         rorxq   $14,%rax,%r14
3635         leaq    (%rdx,%r12,1),%rdx
3636         xorq    %r14,%r13
3637         movq    %r8,%r15
3638         rorxq   $39,%r8,%r12
3639         leaq    (%rdx,%r13,1),%rdx
3640         xorq    %r9,%r15
3641         rorxq   $34,%r8,%r14
3642         rorxq   $28,%r8,%r13
3643         leaq    (%r11,%rdx,1),%r11
3644         andq    %r15,%rdi
3645         xorq    %r12,%r14
3646         xorq    %r9,%rdi
3647         xorq    %r13,%r14
3648         leaq    (%rdx,%rdi,1),%rdx
3649         movq    %rax,%r12
3650         addq    72(%rsp),%rcx
3651         andq    %r11,%r12
3652         rorxq   $41,%r11,%r13
3653         rorxq   $18,%r11,%rdi
3654         leaq    (%rdx,%r14,1),%rdx
3655         leaq    (%rcx,%r12,1),%rcx
3656         andnq   %rbx,%r11,%r12
3657         xorq    %rdi,%r13
3658         rorxq   $14,%r11,%r14
3659         leaq    (%rcx,%r12,1),%rcx
3660         xorq    %r14,%r13
3661         movq    %rdx,%rdi
3662         rorxq   $39,%rdx,%r12
3663         leaq    (%rcx,%r13,1),%rcx
3664         xorq    %r8,%rdi
3665         rorxq   $34,%rdx,%r14
3666         rorxq   $28,%rdx,%r13
3667         leaq    (%r10,%rcx,1),%r10
3668         andq    %rdi,%r15
3669         xorq    %r12,%r14
3670         xorq    %r8,%r15
3671         xorq    %r13,%r14
3672         leaq    (%rcx,%r15,1),%rcx
3673         movq    %r11,%r12
3674         addq    96(%rsp),%rbx
3675         andq    %r10,%r12
3676         rorxq   $41,%r10,%r13
3677         rorxq   $18,%r10,%r15
3678         leaq    (%rcx,%r14,1),%rcx
3679         leaq    (%rbx,%r12,1),%rbx
3680         andnq   %rax,%r10,%r12
3681         xorq    %r15,%r13
3682         rorxq   $14,%r10,%r14
3683         leaq    (%rbx,%r12,1),%rbx
3684         xorq    %r14,%r13
3685         movq    %rcx,%r15
3686         rorxq   $39,%rcx,%r12
3687         leaq    (%rbx,%r13,1),%rbx
3688         xorq    %rdx,%r15
3689         rorxq   $34,%rcx,%r14
3690         rorxq   $28,%rcx,%r13
3691         leaq    (%r9,%rbx,1),%r9
3692         andq    %r15,%rdi
3693         xorq    %r12,%r14
3694         xorq    %rdx,%rdi
3695         xorq    %r13,%r14
3696         leaq    (%rbx,%rdi,1),%rbx
3697         movq    %r10,%r12
3698         addq    104(%rsp),%rax
3699         andq    %r9,%r12
3700         rorxq   $41,%r9,%r13
3701         rorxq   $18,%r9,%rdi
3702         leaq    (%rbx,%r14,1),%rbx
3703         leaq    (%rax,%r12,1),%rax
3704         andnq   %r11,%r9,%r12
3705         xorq    %rdi,%r13
3706         rorxq   $14,%r9,%r14
3707         leaq    (%rax,%r12,1),%rax
3708         xorq    %r14,%r13
3709         movq    %rbx,%rdi
3710         rorxq   $39,%rbx,%r12
3711         leaq    (%rax,%r13,1),%rax
3712         xorq    %rcx,%rdi
3713         rorxq   $34,%rbx,%r14
3714         rorxq   $28,%rbx,%r13
3715         leaq    (%r8,%rax,1),%r8
3716         andq    %rdi,%r15
3717         xorq    %r12,%r14
3718         xorq    %rcx,%r15
3719         xorq    %r13,%r14
3720         leaq    (%rax,%r15,1),%rax
3721         movq    %r9,%r12
3722         movq    1280(%rsp),%rdi
3723         addq    %r14,%rax
3724
3725         leaq    1152(%rsp),%rbp
3726
3727         addq    0(%rdi),%rax
3728         addq    8(%rdi),%rbx
3729         addq    16(%rdi),%rcx
3730         addq    24(%rdi),%rdx
3731         addq    32(%rdi),%r8
3732         addq    40(%rdi),%r9
3733         addq    48(%rdi),%r10
3734         addq    56(%rdi),%r11
3735
3736         movq    %rax,0(%rdi)
3737         movq    %rbx,8(%rdi)
3738         movq    %rcx,16(%rdi)
3739         movq    %rdx,24(%rdi)
3740         movq    %r8,32(%rdi)
3741         movq    %r9,40(%rdi)
3742         movq    %r10,48(%rdi)
3743         movq    %r11,56(%rdi)
3744
3745         cmpq    144(%rbp),%rsi
3746         je      .Ldone_avx2
3747
3748         xorq    %r14,%r14
3749         movq    %rbx,%rdi
3750         xorq    %rcx,%rdi
3751         movq    %r9,%r12
3752         jmp     .Lower_avx2
3753 .balign 16
3754 .Lower_avx2:
3755         addq    0+16(%rbp),%r11
3756         andq    %r8,%r12
3757         rorxq   $41,%r8,%r13
3758         rorxq   $18,%r8,%r15
3759         leaq    (%rax,%r14,1),%rax
3760         leaq    (%r11,%r12,1),%r11
3761         andnq   %r10,%r8,%r12
3762         xorq    %r15,%r13
3763         rorxq   $14,%r8,%r14
3764         leaq    (%r11,%r12,1),%r11
3765         xorq    %r14,%r13
3766         movq    %rax,%r15
3767         rorxq   $39,%rax,%r12
3768         leaq    (%r11,%r13,1),%r11
3769         xorq    %rbx,%r15
3770         rorxq   $34,%rax,%r14
3771         rorxq   $28,%rax,%r13
3772         leaq    (%rdx,%r11,1),%rdx
3773         andq    %r15,%rdi
3774         xorq    %r12,%r14
3775         xorq    %rbx,%rdi
3776         xorq    %r13,%r14
3777         leaq    (%r11,%rdi,1),%r11
3778         movq    %r8,%r12
3779         addq    8+16(%rbp),%r10
3780         andq    %rdx,%r12
3781         rorxq   $41,%rdx,%r13
3782         rorxq   $18,%rdx,%rdi
3783         leaq    (%r11,%r14,1),%r11
3784         leaq    (%r10,%r12,1),%r10
3785         andnq   %r9,%rdx,%r12
3786         xorq    %rdi,%r13
3787         rorxq   $14,%rdx,%r14
3788         leaq    (%r10,%r12,1),%r10
3789         xorq    %r14,%r13
3790         movq    %r11,%rdi
3791         rorxq   $39,%r11,%r12
3792         leaq    (%r10,%r13,1),%r10
3793         xorq    %rax,%rdi
3794         rorxq   $34,%r11,%r14
3795         rorxq   $28,%r11,%r13
3796         leaq    (%rcx,%r10,1),%rcx
3797         andq    %rdi,%r15
3798         xorq    %r12,%r14
3799         xorq    %rax,%r15
3800         xorq    %r13,%r14
3801         leaq    (%r10,%r15,1),%r10
3802         movq    %rdx,%r12
3803         addq    32+16(%rbp),%r9
3804         andq    %rcx,%r12
3805         rorxq   $41,%rcx,%r13
3806         rorxq   $18,%rcx,%r15
3807         leaq    (%r10,%r14,1),%r10
3808         leaq    (%r9,%r12,1),%r9
3809         andnq   %r8,%rcx,%r12
3810         xorq    %r15,%r13
3811         rorxq   $14,%rcx,%r14
3812         leaq    (%r9,%r12,1),%r9
3813         xorq    %r14,%r13
3814         movq    %r10,%r15
3815         rorxq   $39,%r10,%r12
3816         leaq    (%r9,%r13,1),%r9
3817         xorq    %r11,%r15
3818         rorxq   $34,%r10,%r14
3819         rorxq   $28,%r10,%r13
3820         leaq    (%rbx,%r9,1),%rbx
3821         andq    %r15,%rdi
3822         xorq    %r12,%r14
3823         xorq    %r11,%rdi
3824         xorq    %r13,%r14
3825         leaq    (%r9,%rdi,1),%r9
3826         movq    %rcx,%r12
3827         addq    40+16(%rbp),%r8
3828         andq    %rbx,%r12
3829         rorxq   $41,%rbx,%r13
3830         rorxq   $18,%rbx,%rdi
3831         leaq    (%r9,%r14,1),%r9
3832         leaq    (%r8,%r12,1),%r8
3833         andnq   %rdx,%rbx,%r12
3834         xorq    %rdi,%r13
3835         rorxq   $14,%rbx,%r14
3836         leaq    (%r8,%r12,1),%r8
3837         xorq    %r14,%r13
3838         movq    %r9,%rdi
3839         rorxq   $39,%r9,%r12
3840         leaq    (%r8,%r13,1),%r8
3841         xorq    %r10,%rdi
3842         rorxq   $34,%r9,%r14
3843         rorxq   $28,%r9,%r13
3844         leaq    (%rax,%r8,1),%rax
3845         andq    %rdi,%r15
3846         xorq    %r12,%r14
3847         xorq    %r10,%r15
3848         xorq    %r13,%r14
3849         leaq    (%r8,%r15,1),%r8
3850         movq    %rbx,%r12
3851         addq    64+16(%rbp),%rdx
3852         andq    %rax,%r12
3853         rorxq   $41,%rax,%r13
3854         rorxq   $18,%rax,%r15
3855         leaq    (%r8,%r14,1),%r8
3856         leaq    (%rdx,%r12,1),%rdx
3857         andnq   %rcx,%rax,%r12
3858         xorq    %r15,%r13
3859         rorxq   $14,%rax,%r14
3860         leaq    (%rdx,%r12,1),%rdx
3861         xorq    %r14,%r13
3862         movq    %r8,%r15
3863         rorxq   $39,%r8,%r12
3864         leaq    (%rdx,%r13,1),%rdx
3865         xorq    %r9,%r15
3866         rorxq   $34,%r8,%r14
3867         rorxq   $28,%r8,%r13
3868         leaq    (%r11,%rdx,1),%r11
3869         andq    %r15,%rdi
3870         xorq    %r12,%r14
3871         xorq    %r9,%rdi
3872         xorq    %r13,%r14
3873         leaq    (%rdx,%rdi,1),%rdx
3874         movq    %rax,%r12
3875         addq    72+16(%rbp),%rcx
3876         andq    %r11,%r12
3877         rorxq   $41,%r11,%r13
3878         rorxq   $18,%r11,%rdi
3879         leaq    (%rdx,%r14,1),%rdx
3880         leaq    (%rcx,%r12,1),%rcx
3881         andnq   %rbx,%r11,%r12
3882         xorq    %rdi,%r13
3883         rorxq   $14,%r11,%r14
3884         leaq    (%rcx,%r12,1),%rcx
3885         xorq    %r14,%r13
3886         movq    %rdx,%rdi
3887         rorxq   $39,%rdx,%r12
3888         leaq    (%rcx,%r13,1),%rcx
3889         xorq    %r8,%rdi
3890         rorxq   $34,%rdx,%r14
3891         rorxq   $28,%rdx,%r13
3892         leaq    (%r10,%rcx,1),%r10
3893         andq    %rdi,%r15
3894         xorq    %r12,%r14
3895         xorq    %r8,%r15
3896         xorq    %r13,%r14
3897         leaq    (%rcx,%r15,1),%rcx
3898         movq    %r11,%r12
3899         addq    96+16(%rbp),%rbx
3900         andq    %r10,%r12
3901         rorxq   $41,%r10,%r13
3902         rorxq   $18,%r10,%r15
3903         leaq    (%rcx,%r14,1),%rcx
3904         leaq    (%rbx,%r12,1),%rbx
3905         andnq   %rax,%r10,%r12
3906         xorq    %r15,%r13
3907         rorxq   $14,%r10,%r14
3908         leaq    (%rbx,%r12,1),%rbx
3909         xorq    %r14,%r13
3910         movq    %rcx,%r15
3911         rorxq   $39,%rcx,%r12
3912         leaq    (%rbx,%r13,1),%rbx
3913         xorq    %rdx,%r15
3914         rorxq   $34,%rcx,%r14
3915         rorxq   $28,%rcx,%r13
3916         leaq    (%r9,%rbx,1),%r9
3917         andq    %r15,%rdi
3918         xorq    %r12,%r14
3919         xorq    %rdx,%rdi
3920         xorq    %r13,%r14
3921         leaq    (%rbx,%rdi,1),%rbx
3922         movq    %r10,%r12
3923         addq    104+16(%rbp),%rax
3924         andq    %r9,%r12
3925         rorxq   $41,%r9,%r13
3926         rorxq   $18,%r9,%rdi
3927         leaq    (%rbx,%r14,1),%rbx
3928         leaq    (%rax,%r12,1),%rax
3929         andnq   %r11,%r9,%r12
3930         xorq    %rdi,%r13
3931         rorxq   $14,%r9,%r14
3932         leaq    (%rax,%r12,1),%rax
3933         xorq    %r14,%r13
3934         movq    %rbx,%rdi
3935         rorxq   $39,%rbx,%r12
3936         leaq    (%rax,%r13,1),%rax
3937         xorq    %rcx,%rdi
3938         rorxq   $34,%rbx,%r14
3939         rorxq   $28,%rbx,%r13
3940         leaq    (%r8,%rax,1),%r8
3941         andq    %rdi,%r15
3942         xorq    %r12,%r14
3943         xorq    %rcx,%r15
3944         xorq    %r13,%r14
3945         leaq    (%rax,%r15,1),%rax
3946         movq    %r9,%r12
3947         leaq    -128(%rbp),%rbp
3948         cmpq    %rsp,%rbp
3949         jae     .Lower_avx2
3950
3951         movq    1280(%rsp),%rdi
3952         addq    %r14,%rax
3953
3954         leaq    1152(%rsp),%rsp
3955
3956 .cfi_escape     0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
3957
3958         addq    0(%rdi),%rax
3959         addq    8(%rdi),%rbx
3960         addq    16(%rdi),%rcx
3961         addq    24(%rdi),%rdx
3962         addq    32(%rdi),%r8
3963         addq    40(%rdi),%r9
3964         leaq    256(%rsi),%rsi
3965         addq    48(%rdi),%r10
3966         movq    %rsi,%r12
3967         addq    56(%rdi),%r11
3968         cmpq    128+16(%rsp),%rsi
3969
3970         movq    %rax,0(%rdi)
3971         cmoveq  %rsp,%r12
3972         movq    %rbx,8(%rdi)
3973         movq    %rcx,16(%rdi)
3974         movq    %rdx,24(%rdi)
3975         movq    %r8,32(%rdi)
3976         movq    %r9,40(%rdi)
3977         movq    %r10,48(%rdi)
3978         movq    %r11,56(%rdi)
3979
3980         jbe     .Loop_avx2
3981         leaq    (%rsp),%rbp
3982
3983 .cfi_escape     0x0f,0x06,0x76,0x98,0x01,0x06,0x23,0x08
3984
3985 .Ldone_avx2:
3986         movq    152(%rbp),%rsi
3987 .cfi_def_cfa    %rsi,8
3988         vzeroupper
3989         movq    -48(%rsi),%r15
3990 .cfi_restore    %r15
3991         movq    -40(%rsi),%r14
3992 .cfi_restore    %r14
3993         movq    -32(%rsi),%r13
3994 .cfi_restore    %r13
3995         movq    -24(%rsi),%r12
3996 .cfi_restore    %r12
3997         movq    -16(%rsi),%rbp
3998 .cfi_restore    %rbp
3999         movq    -8(%rsi),%rbx
4000 .cfi_restore    %rbx
4001         leaq    (%rsi),%rsp
4002 .cfi_def_cfa_register   %rsp
4003 .Lepilogue_avx2:
4004         RET
4005 .cfi_endproc
4006 SET_SIZE(zfs_sha512_transform_avx2)
4007
4008 #if defined(__ELF__)
4009         .section .note.GNU-stack,"",%progbits
4010 #endif
4011 #endif