arch/x86/crypto/curve25519-x86_64.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright (C) 2020 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  * Copyright (c) 2016-2020 INRIA, CMU and Microsoft Corporation
   5  */
   6
   7 #include <crypto/curve25519.h>
   8 #include <crypto/internal/kpp.h>
   9
  10 #include <linux/types.h>
  11 #include <linux/jump_label.h>
  12 #include <linux/kernel.h>
  13 #include <linux/module.h>
  14 #include <linux/scatterlist.h>
  15
  16 #include <asm/cpufeature.h>
  17 #include <asm/processor.h>
  18
  19 static __always_inline u64 eq_mask(u64 a, u64 b)
  20 {
  21         u64 x = a ^ b;
  22         u64 minus_x = ~x + (u64)1U;
  23         u64 x_or_minus_x = x | minus_x;
  24         u64 xnx = x_or_minus_x >> (u32)63U;
  25         return xnx - (u64)1U;
  26 }
  27
  28 static __always_inline u64 gte_mask(u64 a, u64 b)
  29 {
  30         u64 x = a;
  31         u64 y = b;
  32         u64 x_xor_y = x ^ y;
  33         u64 x_sub_y = x - y;
  34         u64 x_sub_y_xor_y = x_sub_y ^ y;
  35         u64 q = x_xor_y | x_sub_y_xor_y;
  36         u64 x_xor_q = x ^ q;
  37         u64 x_xor_q_ = x_xor_q >> (u32)63U;
  38         return x_xor_q_ - (u64)1U;
  39 }
  40
  41 /* Computes the addition of four-element f1 with value in f2
  42  * and returns the carry (if any) */
  43 static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2)
  44 {
  45         u64 carry_r;
  46
  47         asm volatile(
  48                 /* Clear registers to propagate the carry bit */
  49                 "  xor %%r8d, %%r8d;"
  50                 "  xor %%r9d, %%r9d;"
  51                 "  xor %%r10d, %%r10d;"
  52                 "  xor %%r11d, %%r11d;"
  53                 "  xor %k1, %k1;"
  54
  55                 /* Begin addition chain */
  56                 "  addq 0(%3), %0;"
  57                 "  movq %0, 0(%2);"
  58                 "  adcxq 8(%3), %%r8;"
  59                 "  movq %%r8, 8(%2);"
  60                 "  adcxq 16(%3), %%r9;"
  61                 "  movq %%r9, 16(%2);"
  62                 "  adcxq 24(%3), %%r10;"
  63                 "  movq %%r10, 24(%2);"
  64
  65                 /* Return the carry bit in a register */
  66                 "  adcx %%r11, %1;"
  67                 : "+&r"(f2), "=&r"(carry_r)
  68                 : "r"(out), "r"(f1)
  69                 : "%r8", "%r9", "%r10", "%r11", "memory", "cc");
  70
  71         return carry_r;
  72 }
  73
  74 /* Computes the field addition of two field elements */
  75 static inline void fadd(u64 *out, const u64 *f1, const u64 *f2)
  76 {
  77         asm volatile(
  78                 /* Compute the raw addition of f1 + f2 */
  79                 "  movq 0(%0), %%r8;"
  80                 "  addq 0(%2), %%r8;"
  81                 "  movq 8(%0), %%r9;"
  82                 "  adcxq 8(%2), %%r9;"
  83                 "  movq 16(%0), %%r10;"
  84                 "  adcxq 16(%2), %%r10;"
  85                 "  movq 24(%0), %%r11;"
  86                 "  adcxq 24(%2), %%r11;"
  87
  88                 /* Wrap the result back into the field */
  89
  90                 /* Step 1: Compute carry*38 */
  91                 "  mov $0, %%rax;"
  92                 "  mov $38, %0;"
  93                 "  cmovc %0, %%rax;"
  94
  95                 /* Step 2: Add carry*38 to the original sum */
  96                 "  xor %%ecx, %%ecx;"
  97                 "  add %%rax, %%r8;"
  98                 "  adcx %%rcx, %%r9;"
  99                 "  movq %%r9, 8(%1);"
 100                 "  adcx %%rcx, %%r10;"
 101                 "  movq %%r10, 16(%1);"
 102                 "  adcx %%rcx, %%r11;"
 103                 "  movq %%r11, 24(%1);"
 104
 105                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 106                 "  mov $0, %%rax;"
 107                 "  cmovc %0, %%rax;"
 108                 "  add %%rax, %%r8;"
 109                 "  movq %%r8, 0(%1);"
 110                 : "+&r"(f2)
 111                 : "r"(out), "r"(f1)
 112                 : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
 113 }
 114
 115 /* Computes the field subtraction of two field elements */
 116 static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
 117 {
 118         asm volatile(
 119                 /* Compute the raw subtraction of f1-f2 */
 120                 "  movq 0(%1), %%r8;"
 121                 "  subq 0(%2), %%r8;"
 122                 "  movq 8(%1), %%r9;"
 123                 "  sbbq 8(%2), %%r9;"
 124                 "  movq 16(%1), %%r10;"
 125                 "  sbbq 16(%2), %%r10;"
 126                 "  movq 24(%1), %%r11;"
 127                 "  sbbq 24(%2), %%r11;"
 128
 129                 /* Wrap the result back into the field */
 130
 131                 /* Step 1: Compute carry*38 */
 132                 "  mov $0, %%rax;"
 133                 "  mov $38, %%rcx;"
 134                 "  cmovc %%rcx, %%rax;"
 135
 136                 /* Step 2: Subtract carry*38 from the original difference */
 137                 "  sub %%rax, %%r8;"
 138                 "  sbb $0, %%r9;"
 139                 "  sbb $0, %%r10;"
 140                 "  sbb $0, %%r11;"
 141
 142                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 143                 "  mov $0, %%rax;"
 144                 "  cmovc %%rcx, %%rax;"
 145                 "  sub %%rax, %%r8;"
 146
 147                 /* Store the result */
 148                 "  movq %%r8, 0(%0);"
 149                 "  movq %%r9, 8(%0);"
 150                 "  movq %%r10, 16(%0);"
 151                 "  movq %%r11, 24(%0);"
 152                 :
 153                 : "r"(out), "r"(f1), "r"(f2)
 154                 : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
 155 }
 156
 157 /* Computes a field multiplication: out <- f1 * f2
 158  * Uses the 8-element buffer tmp for intermediate results */
 159 static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
 160 {
 161         asm volatile(
 162
 163                 /* Compute the raw multiplication: tmp <- src1 * src2 */
 164
 165                 /* Compute src1[0] * src2 */
 166                 "  movq 0(%0), %%rdx;"
 167                 "  mulxq 0(%1), %%r8, %%r9;"
 168                 "  xor %%r10d, %%r10d;"
 169                 "  movq %%r8, 0(%2);"
 170                 "  mulxq 8(%1), %%r10, %%r11;"
 171                 "  adox %%r9, %%r10;"
 172                 "  movq %%r10, 8(%2);"
 173                 "  mulxq 16(%1), %%rbx, %%r13;"
 174                 "  adox %%r11, %%rbx;"
 175                 "  mulxq 24(%1), %%r14, %%rdx;"
 176                 "  adox %%r13, %%r14;"
 177                 "  mov $0, %%rax;"
 178                 "  adox %%rdx, %%rax;"
 179
 180                 /* Compute src1[1] * src2 */
 181                 "  movq 8(%0), %%rdx;"
 182                 "  mulxq 0(%1), %%r8, %%r9;"
 183                 "  xor %%r10d, %%r10d;"
 184                 "  adcxq 8(%2), %%r8;"
 185                 "  movq %%r8, 8(%2);"
 186                 "  mulxq 8(%1), %%r10, %%r11;"
 187                 "  adox %%r9, %%r10;"
 188                 "  adcx %%rbx, %%r10;"
 189                 "  movq %%r10, 16(%2);"
 190                 "  mulxq 16(%1), %%rbx, %%r13;"
 191                 "  adox %%r11, %%rbx;"
 192                 "  adcx %%r14, %%rbx;"
 193                 "  mov $0, %%r8;"
 194                 "  mulxq 24(%1), %%r14, %%rdx;"
 195                 "  adox %%r13, %%r14;"
 196                 "  adcx %%rax, %%r14;"
 197                 "  mov $0, %%rax;"
 198                 "  adox %%rdx, %%rax;"
 199                 "  adcx %%r8, %%rax;"
 200
 201                 /* Compute src1[2] * src2 */
 202                 "  movq 16(%0), %%rdx;"
 203                 "  mulxq 0(%1), %%r8, %%r9;"
 204                 "  xor %%r10d, %%r10d;"
 205                 "  adcxq 16(%2), %%r8;"
 206                 "  movq %%r8, 16(%2);"
 207                 "  mulxq 8(%1), %%r10, %%r11;"
 208                 "  adox %%r9, %%r10;"
 209                 "  adcx %%rbx, %%r10;"
 210                 "  movq %%r10, 24(%2);"
 211                 "  mulxq 16(%1), %%rbx, %%r13;"
 212                 "  adox %%r11, %%rbx;"
 213                 "  adcx %%r14, %%rbx;"
 214                 "  mov $0, %%r8;"
 215                 "  mulxq 24(%1), %%r14, %%rdx;"
 216                 "  adox %%r13, %%r14;"
 217                 "  adcx %%rax, %%r14;"
 218                 "  mov $0, %%rax;"
 219                 "  adox %%rdx, %%rax;"
 220                 "  adcx %%r8, %%rax;"
 221
 222                 /* Compute src1[3] * src2 */
 223                 "  movq 24(%0), %%rdx;"
 224                 "  mulxq 0(%1), %%r8, %%r9;"
 225                 "  xor %%r10d, %%r10d;"
 226                 "  adcxq 24(%2), %%r8;"
 227                 "  movq %%r8, 24(%2);"
 228                 "  mulxq 8(%1), %%r10, %%r11;"
 229                 "  adox %%r9, %%r10;"
 230                 "  adcx %%rbx, %%r10;"
 231                 "  movq %%r10, 32(%2);"
 232                 "  mulxq 16(%1), %%rbx, %%r13;"
 233                 "  adox %%r11, %%rbx;"
 234                 "  adcx %%r14, %%rbx;"
 235                 "  movq %%rbx, 40(%2);"
 236                 "  mov $0, %%r8;"
 237                 "  mulxq 24(%1), %%r14, %%rdx;"
 238                 "  adox %%r13, %%r14;"
 239                 "  adcx %%rax, %%r14;"
 240                 "  movq %%r14, 48(%2);"
 241                 "  mov $0, %%rax;"
 242                 "  adox %%rdx, %%rax;"
 243                 "  adcx %%r8, %%rax;"
 244                 "  movq %%rax, 56(%2);"
 245
 246                 /* Line up pointers */
 247                 "  mov %2, %0;"
 248                 "  mov %3, %2;"
 249
 250                 /* Wrap the result back into the field */
 251
 252                 /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
 253                 "  mov $38, %%rdx;"
 254                 "  mulxq 32(%0), %%r8, %%r13;"
 255                 "  xor %k1, %k1;"
 256                 "  adoxq 0(%0), %%r8;"
 257                 "  mulxq 40(%0), %%r9, %%rbx;"
 258                 "  adcx %%r13, %%r9;"
 259                 "  adoxq 8(%0), %%r9;"
 260                 "  mulxq 48(%0), %%r10, %%r13;"
 261                 "  adcx %%rbx, %%r10;"
 262                 "  adoxq 16(%0), %%r10;"
 263                 "  mulxq 56(%0), %%r11, %%rax;"
 264                 "  adcx %%r13, %%r11;"
 265                 "  adoxq 24(%0), %%r11;"
 266                 "  adcx %1, %%rax;"
 267                 "  adox %1, %%rax;"
 268                 "  imul %%rdx, %%rax;"
 269
 270                 /* Step 2: Fold the carry back into dst */
 271                 "  add %%rax, %%r8;"
 272                 "  adcx %1, %%r9;"
 273                 "  movq %%r9, 8(%2);"
 274                 "  adcx %1, %%r10;"
 275                 "  movq %%r10, 16(%2);"
 276                 "  adcx %1, %%r11;"
 277                 "  movq %%r11, 24(%2);"
 278
 279                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 280                 "  mov $0, %%rax;"
 281                 "  cmovc %%rdx, %%rax;"
 282                 "  add %%rax, %%r8;"
 283                 "  movq %%r8, 0(%2);"
 284                 : "+&r"(f1), "+&r"(f2), "+&r"(tmp)
 285                 : "r"(out)
 286                 : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
 287                   "%r14", "memory", "cc");
 288 }
 289
 290 /* Computes two field multiplications:
 291  *   out[0] <- f1[0] * f2[0]
 292  *   out[1] <- f1[1] * f2[1]
 293  * Uses the 16-element buffer tmp for intermediate results: */
 294 static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
 295 {
 296         asm volatile(
 297
 298                 /* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */
 299
 300                 /* Compute src1[0] * src2 */
 301                 "  movq 0(%0), %%rdx;"
 302                 "  mulxq 0(%1), %%r8, %%r9;"
 303                 "  xor %%r10d, %%r10d;"
 304                 "  movq %%r8, 0(%2);"
 305                 "  mulxq 8(%1), %%r10, %%r11;"
 306                 "  adox %%r9, %%r10;"
 307                 "  movq %%r10, 8(%2);"
 308                 "  mulxq 16(%1), %%rbx, %%r13;"
 309                 "  adox %%r11, %%rbx;"
 310                 "  mulxq 24(%1), %%r14, %%rdx;"
 311                 "  adox %%r13, %%r14;"
 312                 "  mov $0, %%rax;"
 313                 "  adox %%rdx, %%rax;"
 314
 315                 /* Compute src1[1] * src2 */
 316                 "  movq 8(%0), %%rdx;"
 317                 "  mulxq 0(%1), %%r8, %%r9;"
 318                 "  xor %%r10d, %%r10d;"
 319                 "  adcxq 8(%2), %%r8;"
 320                 "  movq %%r8, 8(%2);"
 321                 "  mulxq 8(%1), %%r10, %%r11;"
 322                 "  adox %%r9, %%r10;"
 323                 "  adcx %%rbx, %%r10;"
 324                 "  movq %%r10, 16(%2);"
 325                 "  mulxq 16(%1), %%rbx, %%r13;"
 326                 "  adox %%r11, %%rbx;"
 327                 "  adcx %%r14, %%rbx;"
 328                 "  mov $0, %%r8;"
 329                 "  mulxq 24(%1), %%r14, %%rdx;"
 330                 "  adox %%r13, %%r14;"
 331                 "  adcx %%rax, %%r14;"
 332                 "  mov $0, %%rax;"
 333                 "  adox %%rdx, %%rax;"
 334                 "  adcx %%r8, %%rax;"
 335
 336                 /* Compute src1[2] * src2 */
 337                 "  movq 16(%0), %%rdx;"
 338                 "  mulxq 0(%1), %%r8, %%r9;"
 339                 "  xor %%r10d, %%r10d;"
 340                 "  adcxq 16(%2), %%r8;"
 341                 "  movq %%r8, 16(%2);"
 342                 "  mulxq 8(%1), %%r10, %%r11;"
 343                 "  adox %%r9, %%r10;"
 344                 "  adcx %%rbx, %%r10;"
 345                 "  movq %%r10, 24(%2);"
 346                 "  mulxq 16(%1), %%rbx, %%r13;"
 347                 "  adox %%r11, %%rbx;"
 348                 "  adcx %%r14, %%rbx;"
 349                 "  mov $0, %%r8;"
 350                 "  mulxq 24(%1), %%r14, %%rdx;"
 351                 "  adox %%r13, %%r14;"
 352                 "  adcx %%rax, %%r14;"
 353                 "  mov $0, %%rax;"
 354                 "  adox %%rdx, %%rax;"
 355                 "  adcx %%r8, %%rax;"
 356
 357                 /* Compute src1[3] * src2 */
 358                 "  movq 24(%0), %%rdx;"
 359                 "  mulxq 0(%1), %%r8, %%r9;"
 360                 "  xor %%r10d, %%r10d;"
 361                 "  adcxq 24(%2), %%r8;"
 362                 "  movq %%r8, 24(%2);"
 363                 "  mulxq 8(%1), %%r10, %%r11;"
 364                 "  adox %%r9, %%r10;"
 365                 "  adcx %%rbx, %%r10;"
 366                 "  movq %%r10, 32(%2);"
 367                 "  mulxq 16(%1), %%rbx, %%r13;"
 368                 "  adox %%r11, %%rbx;"
 369                 "  adcx %%r14, %%rbx;"
 370                 "  movq %%rbx, 40(%2);"
 371                 "  mov $0, %%r8;"
 372                 "  mulxq 24(%1), %%r14, %%rdx;"
 373                 "  adox %%r13, %%r14;"
 374                 "  adcx %%rax, %%r14;"
 375                 "  movq %%r14, 48(%2);"
 376                 "  mov $0, %%rax;"
 377                 "  adox %%rdx, %%rax;"
 378                 "  adcx %%r8, %%rax;"
 379                 "  movq %%rax, 56(%2);"
 380
 381                 /* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */
 382
 383                 /* Compute src1[0] * src2 */
 384                 "  movq 32(%0), %%rdx;"
 385                 "  mulxq 32(%1), %%r8, %%r9;"
 386                 "  xor %%r10d, %%r10d;"
 387                 "  movq %%r8, 64(%2);"
 388                 "  mulxq 40(%1), %%r10, %%r11;"
 389                 "  adox %%r9, %%r10;"
 390                 "  movq %%r10, 72(%2);"
 391                 "  mulxq 48(%1), %%rbx, %%r13;"
 392                 "  adox %%r11, %%rbx;"
 393                 "  mulxq 56(%1), %%r14, %%rdx;"
 394                 "  adox %%r13, %%r14;"
 395                 "  mov $0, %%rax;"
 396                 "  adox %%rdx, %%rax;"
 397
 398                 /* Compute src1[1] * src2 */
 399                 "  movq 40(%0), %%rdx;"
 400                 "  mulxq 32(%1), %%r8, %%r9;"
 401                 "  xor %%r10d, %%r10d;"
 402                 "  adcxq 72(%2), %%r8;"
 403                 "  movq %%r8, 72(%2);"
 404                 "  mulxq 40(%1), %%r10, %%r11;"
 405                 "  adox %%r9, %%r10;"
 406                 "  adcx %%rbx, %%r10;"
 407                 "  movq %%r10, 80(%2);"
 408                 "  mulxq 48(%1), %%rbx, %%r13;"
 409                 "  adox %%r11, %%rbx;"
 410                 "  adcx %%r14, %%rbx;"
 411                 "  mov $0, %%r8;"
 412                 "  mulxq 56(%1), %%r14, %%rdx;"
 413                 "  adox %%r13, %%r14;"
 414                 "  adcx %%rax, %%r14;"
 415                 "  mov $0, %%rax;"
 416                 "  adox %%rdx, %%rax;"
 417                 "  adcx %%r8, %%rax;"
 418
 419                 /* Compute src1[2] * src2 */
 420                 "  movq 48(%0), %%rdx;"
 421                 "  mulxq 32(%1), %%r8, %%r9;"
 422                 "  xor %%r10d, %%r10d;"
 423                 "  adcxq 80(%2), %%r8;"
 424                 "  movq %%r8, 80(%2);"
 425                 "  mulxq 40(%1), %%r10, %%r11;"
 426                 "  adox %%r9, %%r10;"
 427                 "  adcx %%rbx, %%r10;"
 428                 "  movq %%r10, 88(%2);"
 429                 "  mulxq 48(%1), %%rbx, %%r13;"
 430                 "  adox %%r11, %%rbx;"
 431                 "  adcx %%r14, %%rbx;"
 432                 "  mov $0, %%r8;"
 433                 "  mulxq 56(%1), %%r14, %%rdx;"
 434                 "  adox %%r13, %%r14;"
 435                 "  adcx %%rax, %%r14;"
 436                 "  mov $0, %%rax;"
 437                 "  adox %%rdx, %%rax;"
 438                 "  adcx %%r8, %%rax;"
 439
 440                 /* Compute src1[3] * src2 */
 441                 "  movq 56(%0), %%rdx;"
 442                 "  mulxq 32(%1), %%r8, %%r9;"
 443                 "  xor %%r10d, %%r10d;"
 444                 "  adcxq 88(%2), %%r8;"
 445                 "  movq %%r8, 88(%2);"
 446                 "  mulxq 40(%1), %%r10, %%r11;"
 447                 "  adox %%r9, %%r10;"
 448                 "  adcx %%rbx, %%r10;"
 449                 "  movq %%r10, 96(%2);"
 450                 "  mulxq 48(%1), %%rbx, %%r13;"
 451                 "  adox %%r11, %%rbx;"
 452                 "  adcx %%r14, %%rbx;"
 453                 "  movq %%rbx, 104(%2);"
 454                 "  mov $0, %%r8;"
 455                 "  mulxq 56(%1), %%r14, %%rdx;"
 456                 "  adox %%r13, %%r14;"
 457                 "  adcx %%rax, %%r14;"
 458                 "  movq %%r14, 112(%2);"
 459                 "  mov $0, %%rax;"
 460                 "  adox %%rdx, %%rax;"
 461                 "  adcx %%r8, %%rax;"
 462                 "  movq %%rax, 120(%2);"
 463
 464                 /* Line up pointers */
 465                 "  mov %2, %0;"
 466                 "  mov %3, %2;"
 467
 468                 /* Wrap the results back into the field */
 469
 470                 /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
 471                 "  mov $38, %%rdx;"
 472                 "  mulxq 32(%0), %%r8, %%r13;"
 473                 "  xor %k1, %k1;"
 474                 "  adoxq 0(%0), %%r8;"
 475                 "  mulxq 40(%0), %%r9, %%rbx;"
 476                 "  adcx %%r13, %%r9;"
 477                 "  adoxq 8(%0), %%r9;"
 478                 "  mulxq 48(%0), %%r10, %%r13;"
 479                 "  adcx %%rbx, %%r10;"
 480                 "  adoxq 16(%0), %%r10;"
 481                 "  mulxq 56(%0), %%r11, %%rax;"
 482                 "  adcx %%r13, %%r11;"
 483                 "  adoxq 24(%0), %%r11;"
 484                 "  adcx %1, %%rax;"
 485                 "  adox %1, %%rax;"
 486                 "  imul %%rdx, %%rax;"
 487
 488                 /* Step 2: Fold the carry back into dst */
 489                 "  add %%rax, %%r8;"
 490                 "  adcx %1, %%r9;"
 491                 "  movq %%r9, 8(%2);"
 492                 "  adcx %1, %%r10;"
 493                 "  movq %%r10, 16(%2);"
 494                 "  adcx %1, %%r11;"
 495                 "  movq %%r11, 24(%2);"
 496
 497                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 498                 "  mov $0, %%rax;"
 499                 "  cmovc %%rdx, %%rax;"
 500                 "  add %%rax, %%r8;"
 501                 "  movq %%r8, 0(%2);"
 502
 503                 /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
 504                 "  mov $38, %%rdx;"
 505                 "  mulxq 96(%0), %%r8, %%r13;"
 506                 "  xor %k1, %k1;"
 507                 "  adoxq 64(%0), %%r8;"
 508                 "  mulxq 104(%0), %%r9, %%rbx;"
 509                 "  adcx %%r13, %%r9;"
 510                 "  adoxq 72(%0), %%r9;"
 511                 "  mulxq 112(%0), %%r10, %%r13;"
 512                 "  adcx %%rbx, %%r10;"
 513                 "  adoxq 80(%0), %%r10;"
 514                 "  mulxq 120(%0), %%r11, %%rax;"
 515                 "  adcx %%r13, %%r11;"
 516                 "  adoxq 88(%0), %%r11;"
 517                 "  adcx %1, %%rax;"
 518                 "  adox %1, %%rax;"
 519                 "  imul %%rdx, %%rax;"
 520
 521                 /* Step 2: Fold the carry back into dst */
 522                 "  add %%rax, %%r8;"
 523                 "  adcx %1, %%r9;"
 524                 "  movq %%r9, 40(%2);"
 525                 "  adcx %1, %%r10;"
 526                 "  movq %%r10, 48(%2);"
 527                 "  adcx %1, %%r11;"
 528                 "  movq %%r11, 56(%2);"
 529
 530                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 531                 "  mov $0, %%rax;"
 532                 "  cmovc %%rdx, %%rax;"
 533                 "  add %%rax, %%r8;"
 534                 "  movq %%r8, 32(%2);"
 535                 : "+&r"(f1), "+&r"(f2), "+&r"(tmp)
 536                 : "r"(out)
 537                 : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
 538                   "%r14", "memory", "cc");
 539 }
 540
 541 /* Computes the field multiplication of four-element f1 with value in f2
 542  * Requires f2 to be smaller than 2^17 */
 543 static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
 544 {
 545         register u64 f2_r asm("rdx") = f2;
 546
 547         asm volatile(
 548                 /* Compute the raw multiplication of f1*f2 */
 549                 "  mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
 550                 "  mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
 551                 "  add %%rcx, %%r9;"
 552                 "  mov $0, %%rcx;"
 553                 "  mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
 554                 "  adcx %%rbx, %%r10;"
 555                 "  mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
 556                 "  adcx %%r13, %%r11;"
 557                 "  adcx %%rcx, %%rax;"
 558
 559                 /* Wrap the result back into the field */
 560
 561                 /* Step 1: Compute carry*38 */
 562                 "  mov $38, %%rdx;"
 563                 "  imul %%rdx, %%rax;"
 564
 565                 /* Step 2: Fold the carry back into dst */
 566                 "  add %%rax, %%r8;"
 567                 "  adcx %%rcx, %%r9;"
 568                 "  movq %%r9, 8(%1);"
 569                 "  adcx %%rcx, %%r10;"
 570                 "  movq %%r10, 16(%1);"
 571                 "  adcx %%rcx, %%r11;"
 572                 "  movq %%r11, 24(%1);"
 573
 574                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 575                 "  mov $0, %%rax;"
 576                 "  cmovc %%rdx, %%rax;"
 577                 "  add %%rax, %%r8;"
 578                 "  movq %%r8, 0(%1);"
 579                 : "+&r"(f2_r)
 580                 : "r"(out), "r"(f1)
 581                 : "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13",
 582                   "memory", "cc");
 583 }
 584
 585 /* Computes p1 <- bit ? p2 : p1 in constant time */
 586 static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
 587 {
 588         asm volatile(
 589                 /* Transfer bit into CF flag */
 590                 "  add $18446744073709551615, %0;"
 591
 592                 /* cswap p1[0], p2[0] */
 593                 "  movq 0(%1), %%r8;"
 594                 "  movq 0(%2), %%r9;"
 595                 "  mov %%r8, %%r10;"
 596                 "  cmovc %%r9, %%r8;"
 597                 "  cmovc %%r10, %%r9;"
 598                 "  movq %%r8, 0(%1);"
 599                 "  movq %%r9, 0(%2);"
 600
 601                 /* cswap p1[1], p2[1] */
 602                 "  movq 8(%1), %%r8;"
 603                 "  movq 8(%2), %%r9;"
 604                 "  mov %%r8, %%r10;"
 605                 "  cmovc %%r9, %%r8;"
 606                 "  cmovc %%r10, %%r9;"
 607                 "  movq %%r8, 8(%1);"
 608                 "  movq %%r9, 8(%2);"
 609
 610                 /* cswap p1[2], p2[2] */
 611                 "  movq 16(%1), %%r8;"
 612                 "  movq 16(%2), %%r9;"
 613                 "  mov %%r8, %%r10;"
 614                 "  cmovc %%r9, %%r8;"
 615                 "  cmovc %%r10, %%r9;"
 616                 "  movq %%r8, 16(%1);"
 617                 "  movq %%r9, 16(%2);"
 618
 619                 /* cswap p1[3], p2[3] */
 620                 "  movq 24(%1), %%r8;"
 621                 "  movq 24(%2), %%r9;"
 622                 "  mov %%r8, %%r10;"
 623                 "  cmovc %%r9, %%r8;"
 624                 "  cmovc %%r10, %%r9;"
 625                 "  movq %%r8, 24(%1);"
 626                 "  movq %%r9, 24(%2);"
 627
 628                 /* cswap p1[4], p2[4] */
 629                 "  movq 32(%1), %%r8;"
 630                 "  movq 32(%2), %%r9;"
 631                 "  mov %%r8, %%r10;"
 632                 "  cmovc %%r9, %%r8;"
 633                 "  cmovc %%r10, %%r9;"
 634                 "  movq %%r8, 32(%1);"
 635                 "  movq %%r9, 32(%2);"
 636
 637                 /* cswap p1[5], p2[5] */
 638                 "  movq 40(%1), %%r8;"
 639                 "  movq 40(%2), %%r9;"
 640                 "  mov %%r8, %%r10;"
 641                 "  cmovc %%r9, %%r8;"
 642                 "  cmovc %%r10, %%r9;"
 643                 "  movq %%r8, 40(%1);"
 644                 "  movq %%r9, 40(%2);"
 645
 646                 /* cswap p1[6], p2[6] */
 647                 "  movq 48(%1), %%r8;"
 648                 "  movq 48(%2), %%r9;"
 649                 "  mov %%r8, %%r10;"
 650                 "  cmovc %%r9, %%r8;"
 651                 "  cmovc %%r10, %%r9;"
 652                 "  movq %%r8, 48(%1);"
 653                 "  movq %%r9, 48(%2);"
 654
 655                 /* cswap p1[7], p2[7] */
 656                 "  movq 56(%1), %%r8;"
 657                 "  movq 56(%2), %%r9;"
 658                 "  mov %%r8, %%r10;"
 659                 "  cmovc %%r9, %%r8;"
 660                 "  cmovc %%r10, %%r9;"
 661                 "  movq %%r8, 56(%1);"
 662                 "  movq %%r9, 56(%2);"
 663                 : "+&r"(bit)
 664                 : "r"(p1), "r"(p2)
 665                 : "%r8", "%r9", "%r10", "memory", "cc");
 666 }
 667
 668 /* Computes the square of a field element: out <- f * f
 669  * Uses the 8-element buffer tmp for intermediate results */
 670 static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
 671 {
 672         asm volatile(
 673                 /* Compute the raw multiplication: tmp <- f * f */
 674
 675                 /* Step 1: Compute all partial products */
 676                 "  movq 0(%0), %%rdx;" /* f[0] */
 677                 "  mulxq 8(%0), %%r8, %%r14;"
 678                 "  xor %%r15d, %%r15d;" /* f[1]*f[0] */
 679                 "  mulxq 16(%0), %%r9, %%r10;"
 680                 "  adcx %%r14, %%r9;" /* f[2]*f[0] */
 681                 "  mulxq 24(%0), %%rax, %%rcx;"
 682                 "  adcx %%rax, %%r10;" /* f[3]*f[0] */
 683                 "  movq 24(%0), %%rdx;" /* f[3] */
 684                 "  mulxq 8(%0), %%r11, %%rbx;"
 685                 "  adcx %%rcx, %%r11;" /* f[1]*f[3] */
 686                 "  mulxq 16(%0), %%rax, %%r13;"
 687                 "  adcx %%rax, %%rbx;" /* f[2]*f[3] */
 688                 "  movq 8(%0), %%rdx;"
 689                 "  adcx %%r15, %%r13;" /* f1 */
 690                 "  mulxq 16(%0), %%rax, %%rcx;"
 691                 "  mov $0, %%r14;" /* f[2]*f[1] */
 692
 693                 /* Step 2: Compute two parallel carry chains */
 694                 "  xor %%r15d, %%r15d;"
 695                 "  adox %%rax, %%r10;"
 696                 "  adcx %%r8, %%r8;"
 697                 "  adox %%rcx, %%r11;"
 698                 "  adcx %%r9, %%r9;"
 699                 "  adox %%r15, %%rbx;"
 700                 "  adcx %%r10, %%r10;"
 701                 "  adox %%r15, %%r13;"
 702                 "  adcx %%r11, %%r11;"
 703                 "  adox %%r15, %%r14;"
 704                 "  adcx %%rbx, %%rbx;"
 705                 "  adcx %%r13, %%r13;"
 706                 "  adcx %%r14, %%r14;"
 707
 708                 /* Step 3: Compute intermediate squares */
 709                 "  movq 0(%0), %%rdx;"
 710                 "  mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
 711                 "  movq %%rax, 0(%1);"
 712                 "  add %%rcx, %%r8;"
 713                 "  movq %%r8, 8(%1);"
 714                 "  movq 8(%0), %%rdx;"
 715                 "  mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
 716                 "  adcx %%rax, %%r9;"
 717                 "  movq %%r9, 16(%1);"
 718                 "  adcx %%rcx, %%r10;"
 719                 "  movq %%r10, 24(%1);"
 720                 "  movq 16(%0), %%rdx;"
 721                 "  mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
 722                 "  adcx %%rax, %%r11;"
 723                 "  movq %%r11, 32(%1);"
 724                 "  adcx %%rcx, %%rbx;"
 725                 "  movq %%rbx, 40(%1);"
 726                 "  movq 24(%0), %%rdx;"
 727                 "  mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
 728                 "  adcx %%rax, %%r13;"
 729                 "  movq %%r13, 48(%1);"
 730                 "  adcx %%rcx, %%r14;"
 731                 "  movq %%r14, 56(%1);"
 732
 733                 /* Line up pointers */
 734                 "  mov %1, %0;"
 735                 "  mov %2, %1;"
 736
 737                 /* Wrap the result back into the field */
 738
 739                 /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
 740                 "  mov $38, %%rdx;"
 741                 "  mulxq 32(%0), %%r8, %%r13;"
 742                 "  xor %%ecx, %%ecx;"
 743                 "  adoxq 0(%0), %%r8;"
 744                 "  mulxq 40(%0), %%r9, %%rbx;"
 745                 "  adcx %%r13, %%r9;"
 746                 "  adoxq 8(%0), %%r9;"
 747                 "  mulxq 48(%0), %%r10, %%r13;"
 748                 "  adcx %%rbx, %%r10;"
 749                 "  adoxq 16(%0), %%r10;"
 750                 "  mulxq 56(%0), %%r11, %%rax;"
 751                 "  adcx %%r13, %%r11;"
 752                 "  adoxq 24(%0), %%r11;"
 753                 "  adcx %%rcx, %%rax;"
 754                 "  adox %%rcx, %%rax;"
 755                 "  imul %%rdx, %%rax;"
 756
 757                 /* Step 2: Fold the carry back into dst */
 758                 "  add %%rax, %%r8;"
 759                 "  adcx %%rcx, %%r9;"
 760                 "  movq %%r9, 8(%1);"
 761                 "  adcx %%rcx, %%r10;"
 762                 "  movq %%r10, 16(%1);"
 763                 "  adcx %%rcx, %%r11;"
 764                 "  movq %%r11, 24(%1);"
 765
 766                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 767                 "  mov $0, %%rax;"
 768                 "  cmovc %%rdx, %%rax;"
 769                 "  add %%rax, %%r8;"
 770                 "  movq %%r8, 0(%1);"
 771                 : "+&r"(f), "+&r"(tmp)
 772                 : "r"(out)
 773                 : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
 774                   "%r13", "%r14", "%r15", "memory", "cc");
 775 }
 776
 777 /* Computes two field squarings:
 778  *   out[0] <- f[0] * f[0]
 779  *   out[1] <- f[1] * f[1]
 780  * Uses the 16-element buffer tmp for intermediate results */
 781 static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
 782 {
 783         asm volatile(
 784                 /* Step 1: Compute all partial products */
 785                 "  movq 0(%0), %%rdx;" /* f[0] */
 786                 "  mulxq 8(%0), %%r8, %%r14;"
 787                 "  xor %%r15d, %%r15d;" /* f[1]*f[0] */
 788                 "  mulxq 16(%0), %%r9, %%r10;"
 789                 "  adcx %%r14, %%r9;" /* f[2]*f[0] */
 790                 "  mulxq 24(%0), %%rax, %%rcx;"
 791                 "  adcx %%rax, %%r10;" /* f[3]*f[0] */
 792                 "  movq 24(%0), %%rdx;" /* f[3] */
 793                 "  mulxq 8(%0), %%r11, %%rbx;"
 794                 "  adcx %%rcx, %%r11;" /* f[1]*f[3] */
 795                 "  mulxq 16(%0), %%rax, %%r13;"
 796                 "  adcx %%rax, %%rbx;" /* f[2]*f[3] */
 797                 "  movq 8(%0), %%rdx;"
 798                 "  adcx %%r15, %%r13;" /* f1 */
 799                 "  mulxq 16(%0), %%rax, %%rcx;"
 800                 "  mov $0, %%r14;" /* f[2]*f[1] */
 801
 802                 /* Step 2: Compute two parallel carry chains */
 803                 "  xor %%r15d, %%r15d;"
 804                 "  adox %%rax, %%r10;"
 805                 "  adcx %%r8, %%r8;"
 806                 "  adox %%rcx, %%r11;"
 807                 "  adcx %%r9, %%r9;"
 808                 "  adox %%r15, %%rbx;"
 809                 "  adcx %%r10, %%r10;"
 810                 "  adox %%r15, %%r13;"
 811                 "  adcx %%r11, %%r11;"
 812                 "  adox %%r15, %%r14;"
 813                 "  adcx %%rbx, %%rbx;"
 814                 "  adcx %%r13, %%r13;"
 815                 "  adcx %%r14, %%r14;"
 816
 817                 /* Step 3: Compute intermediate squares */
 818                 "  movq 0(%0), %%rdx;"
 819                 "  mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
 820                 "  movq %%rax, 0(%1);"
 821                 "  add %%rcx, %%r8;"
 822                 "  movq %%r8, 8(%1);"
 823                 "  movq 8(%0), %%rdx;"
 824                 "  mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
 825                 "  adcx %%rax, %%r9;"
 826                 "  movq %%r9, 16(%1);"
 827                 "  adcx %%rcx, %%r10;"
 828                 "  movq %%r10, 24(%1);"
 829                 "  movq 16(%0), %%rdx;"
 830                 "  mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
 831                 "  adcx %%rax, %%r11;"
 832                 "  movq %%r11, 32(%1);"
 833                 "  adcx %%rcx, %%rbx;"
 834                 "  movq %%rbx, 40(%1);"
 835                 "  movq 24(%0), %%rdx;"
 836                 "  mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
 837                 "  adcx %%rax, %%r13;"
 838                 "  movq %%r13, 48(%1);"
 839                 "  adcx %%rcx, %%r14;"
 840                 "  movq %%r14, 56(%1);"
 841
 842                 /* Step 1: Compute all partial products */
 843                 "  movq 32(%0), %%rdx;" /* f[0] */
 844                 "  mulxq 40(%0), %%r8, %%r14;"
 845                 "  xor %%r15d, %%r15d;" /* f[1]*f[0] */
 846                 "  mulxq 48(%0), %%r9, %%r10;"
 847                 "  adcx %%r14, %%r9;" /* f[2]*f[0] */
 848                 "  mulxq 56(%0), %%rax, %%rcx;"
 849                 "  adcx %%rax, %%r10;" /* f[3]*f[0] */
 850                 "  movq 56(%0), %%rdx;" /* f[3] */
 851                 "  mulxq 40(%0), %%r11, %%rbx;"
 852                 "  adcx %%rcx, %%r11;" /* f[1]*f[3] */
 853                 "  mulxq 48(%0), %%rax, %%r13;"
 854                 "  adcx %%rax, %%rbx;" /* f[2]*f[3] */
 855                 "  movq 40(%0), %%rdx;"
 856                 "  adcx %%r15, %%r13;" /* f1 */
 857                 "  mulxq 48(%0), %%rax, %%rcx;"
 858                 "  mov $0, %%r14;" /* f[2]*f[1] */
 859
 860                 /* Step 2: Compute two parallel carry chains */
 861                 "  xor %%r15d, %%r15d;"
 862                 "  adox %%rax, %%r10;"
 863                 "  adcx %%r8, %%r8;"
 864                 "  adox %%rcx, %%r11;"
 865                 "  adcx %%r9, %%r9;"
 866                 "  adox %%r15, %%rbx;"
 867                 "  adcx %%r10, %%r10;"
 868                 "  adox %%r15, %%r13;"
 869                 "  adcx %%r11, %%r11;"
 870                 "  adox %%r15, %%r14;"
 871                 "  adcx %%rbx, %%rbx;"
 872                 "  adcx %%r13, %%r13;"
 873                 "  adcx %%r14, %%r14;"
 874
 875                 /* Step 3: Compute intermediate squares */
 876                 "  movq 32(%0), %%rdx;"
 877                 "  mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
 878                 "  movq %%rax, 64(%1);"
 879                 "  add %%rcx, %%r8;"
 880                 "  movq %%r8, 72(%1);"
 881                 "  movq 40(%0), %%rdx;"
 882                 "  mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
 883                 "  adcx %%rax, %%r9;"
 884                 "  movq %%r9, 80(%1);"
 885                 "  adcx %%rcx, %%r10;"
 886                 "  movq %%r10, 88(%1);"
 887                 "  movq 48(%0), %%rdx;"
 888                 "  mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
 889                 "  adcx %%rax, %%r11;"
 890                 "  movq %%r11, 96(%1);"
 891                 "  adcx %%rcx, %%rbx;"
 892                 "  movq %%rbx, 104(%1);"
 893                 "  movq 56(%0), %%rdx;"
 894                 "  mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
 895                 "  adcx %%rax, %%r13;"
 896                 "  movq %%r13, 112(%1);"
 897                 "  adcx %%rcx, %%r14;"
 898                 "  movq %%r14, 120(%1);"
 899
 900                 /* Line up pointers */
 901                 "  mov %1, %0;"
 902                 "  mov %2, %1;"
 903
 904                 /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
 905                 "  mov $38, %%rdx;"
 906                 "  mulxq 32(%0), %%r8, %%r13;"
 907                 "  xor %%ecx, %%ecx;"
 908                 "  adoxq 0(%0), %%r8;"
 909                 "  mulxq 40(%0), %%r9, %%rbx;"
 910                 "  adcx %%r13, %%r9;"
 911                 "  adoxq 8(%0), %%r9;"
 912                 "  mulxq 48(%0), %%r10, %%r13;"
 913                 "  adcx %%rbx, %%r10;"
 914                 "  adoxq 16(%0), %%r10;"
 915                 "  mulxq 56(%0), %%r11, %%rax;"
 916                 "  adcx %%r13, %%r11;"
 917                 "  adoxq 24(%0), %%r11;"
 918                 "  adcx %%rcx, %%rax;"
 919                 "  adox %%rcx, %%rax;"
 920                 "  imul %%rdx, %%rax;"
 921
 922                 /* Step 2: Fold the carry back into dst */
 923                 "  add %%rax, %%r8;"
 924                 "  adcx %%rcx, %%r9;"
 925                 "  movq %%r9, 8(%1);"
 926                 "  adcx %%rcx, %%r10;"
 927                 "  movq %%r10, 16(%1);"
 928                 "  adcx %%rcx, %%r11;"
 929                 "  movq %%r11, 24(%1);"
 930
 931                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 932                 "  mov $0, %%rax;"
 933                 "  cmovc %%rdx, %%rax;"
 934                 "  add %%rax, %%r8;"
 935                 "  movq %%r8, 0(%1);"
 936
 937                 /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
 938                 "  mov $38, %%rdx;"
 939                 "  mulxq 96(%0), %%r8, %%r13;"
 940                 "  xor %%ecx, %%ecx;"
 941                 "  adoxq 64(%0), %%r8;"
 942                 "  mulxq 104(%0), %%r9, %%rbx;"
 943                 "  adcx %%r13, %%r9;"
 944                 "  adoxq 72(%0), %%r9;"
 945                 "  mulxq 112(%0), %%r10, %%r13;"
 946                 "  adcx %%rbx, %%r10;"
 947                 "  adoxq 80(%0), %%r10;"
 948                 "  mulxq 120(%0), %%r11, %%rax;"
 949                 "  adcx %%r13, %%r11;"
 950                 "  adoxq 88(%0), %%r11;"
 951                 "  adcx %%rcx, %%rax;"
 952                 "  adox %%rcx, %%rax;"
 953                 "  imul %%rdx, %%rax;"
 954
 955                 /* Step 2: Fold the carry back into dst */
 956                 "  add %%rax, %%r8;"
 957                 "  adcx %%rcx, %%r9;"
 958                 "  movq %%r9, 40(%1);"
 959                 "  adcx %%rcx, %%r10;"
 960                 "  movq %%r10, 48(%1);"
 961                 "  adcx %%rcx, %%r11;"
 962                 "  movq %%r11, 56(%1);"
 963
 964                 /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
 965                 "  mov $0, %%rax;"
 966                 "  cmovc %%rdx, %%rax;"
 967                 "  add %%rax, %%r8;"
 968                 "  movq %%r8, 32(%1);"
 969                 : "+&r"(f), "+&r"(tmp)
 970                 : "r"(out)
 971                 : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
 972                   "%r13", "%r14", "%r15", "memory", "cc");
 973 }
 974
 975 static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2)
 976 {
 977         u64 *nq = p01_tmp1;
 978         u64 *nq_p1 = p01_tmp1 + (u32)8U;
 979         u64 *tmp1 = p01_tmp1 + (u32)16U;
 980         u64 *x1 = q;
 981         u64 *x2 = nq;
 982         u64 *z2 = nq + (u32)4U;
 983         u64 *z3 = nq_p1 + (u32)4U;
 984         u64 *a = tmp1;
 985         u64 *b = tmp1 + (u32)4U;
 986         u64 *ab = tmp1;
 987         u64 *dc = tmp1 + (u32)8U;
 988         u64 *x3;
 989         u64 *z31;
 990         u64 *d0;
 991         u64 *c0;
 992         u64 *a1;
 993         u64 *b1;
 994         u64 *d;
 995         u64 *c;
 996         u64 *ab1;
 997         u64 *dc1;
 998         fadd(a, x2, z2);
 999         fsub(b, x2, z2);
1000         x3 = nq_p1;
1001         z31 = nq_p1 + (u32)4U;
1002         d0 = dc;
1003         c0 = dc + (u32)4U;
1004         fadd(c0, x3, z31);
1005         fsub(d0, x3, z31);
1006         fmul2(dc, dc, ab, tmp2);
1007         fadd(x3, d0, c0);
1008         fsub(z31, d0, c0);
1009         a1 = tmp1;
1010         b1 = tmp1 + (u32)4U;
1011         d = tmp1 + (u32)8U;
1012         c = tmp1 + (u32)12U;
1013         ab1 = tmp1;
1014         dc1 = tmp1 + (u32)8U;
1015         fsqr2(dc1, ab1, tmp2);
1016         fsqr2(nq_p1, nq_p1, tmp2);
1017         a1[0U] = c[0U];
1018         a1[1U] = c[1U];
1019         a1[2U] = c[2U];
1020         a1[3U] = c[3U];
1021         fsub(c, d, c);
1022         fmul_scalar(b1, c, (u64)121665U);
1023         fadd(b1, b1, d);
1024         fmul2(nq, dc1, ab1, tmp2);
1025         fmul(z3, z3, x1, tmp2);
1026 }
1027
1028 static void point_double(u64 *nq, u64 *tmp1, u64 *tmp2)
1029 {
1030         u64 *x2 = nq;
1031         u64 *z2 = nq + (u32)4U;
1032         u64 *a = tmp1;
1033         u64 *b = tmp1 + (u32)4U;
1034         u64 *d = tmp1 + (u32)8U;
1035         u64 *c = tmp1 + (u32)12U;
1036         u64 *ab = tmp1;
1037         u64 *dc = tmp1 + (u32)8U;
1038         fadd(a, x2, z2);
1039         fsub(b, x2, z2);
1040         fsqr2(dc, ab, tmp2);
1041         a[0U] = c[0U];
1042         a[1U] = c[1U];
1043         a[2U] = c[2U];
1044         a[3U] = c[3U];
1045         fsub(c, d, c);
1046         fmul_scalar(b, c, (u64)121665U);
1047         fadd(b, b, d);
1048         fmul2(nq, dc, ab, tmp2);
1049 }
1050
1051 static void montgomery_ladder(u64 *out, const u8 *key, u64 *init1)
1052 {
1053         u64 tmp2[16U] = { 0U };
1054         u64 p01_tmp1_swap[33U] = { 0U };
1055         u64 *p0 = p01_tmp1_swap;
1056         u64 *p01 = p01_tmp1_swap;
1057         u64 *p03 = p01;
1058         u64 *p11 = p01 + (u32)8U;
1059         u64 *x0;
1060         u64 *z0;
1061         u64 *p01_tmp1;
1062         u64 *p01_tmp11;
1063         u64 *nq10;
1064         u64 *nq_p11;
1065         u64 *swap1;
1066         u64 sw0;
1067         u64 *nq1;
1068         u64 *tmp1;
1069         memcpy(p11, init1, (u32)8U * sizeof(init1[0U]));
1070         x0 = p03;
1071         z0 = p03 + (u32)4U;
1072         x0[0U] = (u64)1U;
1073         x0[1U] = (u64)0U;
1074         x0[2U] = (u64)0U;
1075         x0[3U] = (u64)0U;
1076         z0[0U] = (u64)0U;
1077         z0[1U] = (u64)0U;
1078         z0[2U] = (u64)0U;
1079         z0[3U] = (u64)0U;
1080         p01_tmp1 = p01_tmp1_swap;
1081         p01_tmp11 = p01_tmp1_swap;
1082         nq10 = p01_tmp1_swap;
1083         nq_p11 = p01_tmp1_swap + (u32)8U;
1084         swap1 = p01_tmp1_swap + (u32)32U;
1085         cswap2((u64)1U, nq10, nq_p11);
1086         point_add_and_double(init1, p01_tmp11, tmp2);
1087         swap1[0U] = (u64)1U;
1088         {
1089                 u32 i;
1090                 for (i = (u32)0U; i < (u32)251U; i = i + (u32)1U) {
1091                         u64 *p01_tmp12 = p01_tmp1_swap;
1092                         u64 *swap2 = p01_tmp1_swap + (u32)32U;
1093                         u64 *nq2 = p01_tmp12;
1094                         u64 *nq_p12 = p01_tmp12 + (u32)8U;
1095                         u64 bit = (u64)(key[((u32)253U - i) / (u32)8U] >> ((u32)253U - i) % (u32)8U & (u8)1U);
1096                         u64 sw = swap2[0U] ^ bit;
1097                         cswap2(sw, nq2, nq_p12);
1098                         point_add_and_double(init1, p01_tmp12, tmp2);
1099                         swap2[0U] = bit;
1100                 }
1101         }
1102         sw0 = swap1[0U];
1103         cswap2(sw0, nq10, nq_p11);
1104         nq1 = p01_tmp1;
1105         tmp1 = p01_tmp1 + (u32)16U;
1106         point_double(nq1, tmp1, tmp2);
1107         point_double(nq1, tmp1, tmp2);
1108         point_double(nq1, tmp1, tmp2);
1109         memcpy(out, p0, (u32)8U * sizeof(p0[0U]));
1110
1111         memzero_explicit(tmp2, sizeof(tmp2));
1112         memzero_explicit(p01_tmp1_swap, sizeof(p01_tmp1_swap));
1113 }
1114
1115 static void fsquare_times(u64 *o, const u64 *inp, u64 *tmp, u32 n1)
1116 {
1117         u32 i;
1118         fsqr(o, inp, tmp);
1119         for (i = (u32)0U; i < n1 - (u32)1U; i = i + (u32)1U)
1120                 fsqr(o, o, tmp);
1121 }
1122
1123 static void finv(u64 *o, const u64 *i, u64 *tmp)
1124 {
1125         u64 t1[16U] = { 0U };
1126         u64 *a0 = t1;
1127         u64 *b = t1 + (u32)4U;
1128         u64 *c = t1 + (u32)8U;
1129         u64 *t00 = t1 + (u32)12U;
1130         u64 *tmp1 = tmp;
1131         u64 *a;
1132         u64 *t0;
1133         fsquare_times(a0, i, tmp1, (u32)1U);
1134         fsquare_times(t00, a0, tmp1, (u32)2U);
1135         fmul(b, t00, i, tmp);
1136         fmul(a0, b, a0, tmp);
1137         fsquare_times(t00, a0, tmp1, (u32)1U);
1138         fmul(b, t00, b, tmp);
1139         fsquare_times(t00, b, tmp1, (u32)5U);
1140         fmul(b, t00, b, tmp);
1141         fsquare_times(t00, b, tmp1, (u32)10U);
1142         fmul(c, t00, b, tmp);
1143         fsquare_times(t00, c, tmp1, (u32)20U);
1144         fmul(t00, t00, c, tmp);
1145         fsquare_times(t00, t00, tmp1, (u32)10U);
1146         fmul(b, t00, b, tmp);
1147         fsquare_times(t00, b, tmp1, (u32)50U);
1148         fmul(c, t00, b, tmp);
1149         fsquare_times(t00, c, tmp1, (u32)100U);
1150         fmul(t00, t00, c, tmp);
1151         fsquare_times(t00, t00, tmp1, (u32)50U);
1152         fmul(t00, t00, b, tmp);
1153         fsquare_times(t00, t00, tmp1, (u32)5U);
1154         a = t1;
1155         t0 = t1 + (u32)12U;
1156         fmul(o, t0, a, tmp);
1157 }
1158
1159 static void store_felem(u64 *b, u64 *f)
1160 {
1161         u64 f30 = f[3U];
1162         u64 top_bit0 = f30 >> (u32)63U;
1163         u64 f31;
1164         u64 top_bit;
1165         u64 f0;
1166         u64 f1;
1167         u64 f2;
1168         u64 f3;
1169         u64 m0;
1170         u64 m1;
1171         u64 m2;
1172         u64 m3;
1173         u64 mask;
1174         u64 f0_;
1175         u64 f1_;
1176         u64 f2_;
1177         u64 f3_;
1178         u64 o0;
1179         u64 o1;
1180         u64 o2;
1181         u64 o3;
1182         f[3U] = f30 & (u64)0x7fffffffffffffffU;
1183         add_scalar(f, f, (u64)19U * top_bit0);
1184         f31 = f[3U];
1185         top_bit = f31 >> (u32)63U;
1186         f[3U] = f31 & (u64)0x7fffffffffffffffU;
1187         add_scalar(f, f, (u64)19U * top_bit);
1188         f0 = f[0U];
1189         f1 = f[1U];
1190         f2 = f[2U];
1191         f3 = f[3U];
1192         m0 = gte_mask(f0, (u64)0xffffffffffffffedU);
1193         m1 = eq_mask(f1, (u64)0xffffffffffffffffU);
1194         m2 = eq_mask(f2, (u64)0xffffffffffffffffU);
1195         m3 = eq_mask(f3, (u64)0x7fffffffffffffffU);
1196         mask = ((m0 & m1) & m2) & m3;
1197         f0_ = f0 - (mask & (u64)0xffffffffffffffedU);
1198         f1_ = f1 - (mask & (u64)0xffffffffffffffffU);
1199         f2_ = f2 - (mask & (u64)0xffffffffffffffffU);
1200         f3_ = f3 - (mask & (u64)0x7fffffffffffffffU);
1201         o0 = f0_;
1202         o1 = f1_;
1203         o2 = f2_;
1204         o3 = f3_;
1205         b[0U] = o0;
1206         b[1U] = o1;
1207         b[2U] = o2;
1208         b[3U] = o3;
1209 }
1210
1211 static void encode_point(u8 *o, const u64 *i)
1212 {
1213         const u64 *x = i;
1214         const u64 *z = i + (u32)4U;
1215         u64 tmp[4U] = { 0U };
1216         u64 tmp_w[16U] = { 0U };
1217         finv(tmp, z, tmp_w);
1218         fmul(tmp, tmp, x, tmp_w);
1219         store_felem((u64 *)o, tmp);
1220 }
1221
1222 static void curve25519_ever64(u8 *out, const u8 *priv, const u8 *pub)
1223 {
1224         u64 init1[8U] = { 0U };
1225         u64 tmp[4U] = { 0U };
1226         u64 tmp3;
1227         u64 *x;
1228         u64 *z;
1229         {
1230                 u32 i;
1231                 for (i = (u32)0U; i < (u32)4U; i = i + (u32)1U) {
1232                         u64 *os = tmp;
1233                         const u8 *bj = pub + i * (u32)8U;
1234                         u64 u = *(u64 *)bj;
1235                         u64 r = u;
1236                         u64 x0 = r;
1237                         os[i] = x0;
1238                 }
1239         }
1240         tmp3 = tmp[3U];
1241         tmp[3U] = tmp3 & (u64)0x7fffffffffffffffU;
1242         x = init1;
1243         z = init1 + (u32)4U;
1244         z[0U] = (u64)1U;
1245         z[1U] = (u64)0U;
1246         z[2U] = (u64)0U;
1247         z[3U] = (u64)0U;
1248         x[0U] = tmp[0U];
1249         x[1U] = tmp[1U];
1250         x[2U] = tmp[2U];
1251         x[3U] = tmp[3U];
1252         montgomery_ladder(init1, priv, init1);
1253         encode_point(out, init1);
1254 }
1255
1256 /* The below constants were generated using this sage script:
1257  *
1258  * #!/usr/bin/env sage
1259  * import sys
1260  * from sage.all import *
1261  * def limbs(n):
1262  *      n = int(n)
1263  *      l = ((n >> 0) % 2^64, (n >> 64) % 2^64, (n >> 128) % 2^64, (n >> 192) % 2^64)
1264  *      return "0x%016xULL, 0x%016xULL, 0x%016xULL, 0x%016xULL" % l
1265  * ec = EllipticCurve(GF(2^255 - 19), [0, 486662, 0, 1, 0])
1266  * p_minus_s = (ec.lift_x(9) - ec.lift_x(1))[0]
1267  * print("static const u64 p_minus_s[] = { %s };\n" % limbs(p_minus_s))
1268  * print("static const u64 table_ladder[] = {")
1269  * p = ec.lift_x(9)
1270  * for i in range(252):
1271  *      l = (p[0] + p[2]) / (p[0] - p[2])
1272  *      print(("\t%s" + ("," if i != 251 else "")) % limbs(l))
1273  *      p = p * 2
1274  * print("};")
1275  *
1276  */
1277
1278 static const u64 p_minus_s[] = { 0x816b1e0137d48290ULL, 0x440f6a51eb4d1207ULL, 0x52385f46dca2b71dULL, 0x215132111d8354cbULL };
1279
1280 static const u64 table_ladder[] = {
1281         0xfffffffffffffff3ULL, 0xffffffffffffffffULL, 0xffffffffffffffffULL, 0x5fffffffffffffffULL,
1282         0x6b8220f416aafe96ULL, 0x82ebeb2b4f566a34ULL, 0xd5a9a5b075a5950fULL, 0x5142b2cf4b2488f4ULL,
1283         0x6aaebc750069680cULL, 0x89cf7820a0f99c41ULL, 0x2a58d9183b56d0f4ULL, 0x4b5aca80e36011a4ULL,
1284         0x329132348c29745dULL, 0xf4a2e616e1642fd7ULL, 0x1e45bb03ff67bc34ULL, 0x306912d0f42a9b4aULL,
1285         0xff886507e6af7154ULL, 0x04f50e13dfeec82fULL, 0xaa512fe82abab5ceULL, 0x174e251a68d5f222ULL,
1286         0xcf96700d82028898ULL, 0x1743e3370a2c02c5ULL, 0x379eec98b4e86eaaULL, 0x0c59888a51e0482eULL,
1287         0xfbcbf1d699b5d189ULL, 0xacaef0d58e9fdc84ULL, 0xc1c20d06231f7614ULL, 0x2938218da274f972ULL,
1288         0xf6af49beff1d7f18ULL, 0xcc541c22387ac9c2ULL, 0x96fcc9ef4015c56bULL, 0x69c1627c690913a9ULL,
1289         0x7a86fd2f4733db0eULL, 0xfdb8c4f29e087de9ULL, 0x095e4b1a8ea2a229ULL, 0x1ad7a7c829b37a79ULL,
1290         0x342d89cad17ea0c0ULL, 0x67bedda6cced2051ULL, 0x19ca31bf2bb42f74ULL, 0x3df7b4c84980acbbULL,
1291         0xa8c6444dc80ad883ULL, 0xb91e440366e3ab85ULL, 0xc215cda00164f6d8ULL, 0x3d867c6ef247e668ULL,
1292         0xc7dd582bcc3e658cULL, 0xfd2c4748ee0e5528ULL, 0xa0fd9b95cc9f4f71ULL, 0x7529d871b0675ddfULL,
1293         0xb8f568b42d3cbd78ULL, 0x1233011b91f3da82ULL, 0x2dce6ccd4a7c3b62ULL, 0x75e7fc8e9e498603ULL,
1294         0x2f4f13f1fcd0b6ecULL, 0xf1a8ca1f29ff7a45ULL, 0xc249c1a72981e29bULL, 0x6ebe0dbb8c83b56aULL,
1295         0x7114fa8d170bb222ULL, 0x65a2dcd5bf93935fULL, 0xbdc41f68b59c979aULL, 0x2f0eef79a2ce9289ULL,
1296         0x42ecbf0c083c37ceULL, 0x2930bc09ec496322ULL, 0xf294b0c19cfeac0dULL, 0x3780aa4bedfabb80ULL,
1297         0x56c17d3e7cead929ULL, 0xe7cb4beb2e5722c5ULL, 0x0ce931732dbfe15aULL, 0x41b883c7621052f8ULL,
1298         0xdbf75ca0c3d25350ULL, 0x2936be086eb1e351ULL, 0xc936e03cb4a9b212ULL, 0x1d45bf82322225aaULL,
1299         0xe81ab1036a024cc5ULL, 0xe212201c304c9a72ULL, 0xc5d73fba6832b1fcULL, 0x20ffdb5a4d839581ULL,
1300         0xa283d367be5d0fadULL, 0x6c2b25ca8b164475ULL, 0x9d4935467caaf22eULL, 0x5166408eee85ff49ULL,
1301         0x3c67baa2fab4e361ULL, 0xb3e433c67ef35cefULL, 0x5259729241159b1cULL, 0x6a621892d5b0ab33ULL,
1302         0x20b74a387555cdcbULL, 0x532aa10e1208923fULL, 0xeaa17b7762281dd1ULL, 0x61ab3443f05c44bfULL,
1303         0x257a6c422324def8ULL, 0x131c6c1017e3cf7fULL, 0x23758739f630a257ULL, 0x295a407a01a78580ULL,
1304         0xf8c443246d5da8d9ULL, 0x19d775450c52fa5dULL, 0x2afcfc92731bf83dULL, 0x7d10c8e81b2b4700ULL,
1305         0xc8e0271f70baa20bULL, 0x993748867ca63957ULL, 0x5412efb3cb7ed4bbULL, 0x3196d36173e62975ULL,
1306         0xde5bcad141c7dffcULL, 0x47cc8cd2b395c848ULL, 0xa34cd942e11af3cbULL, 0x0256dbf2d04ecec2ULL,
1307         0x875ab7e94b0e667fULL, 0xcad4dd83c0850d10ULL, 0x47f12e8f4e72c79fULL, 0x5f1a87bb8c85b19bULL,
1308         0x7ae9d0b6437f51b8ULL, 0x12c7ce5518879065ULL, 0x2ade09fe5cf77aeeULL, 0x23a05a2f7d2c5627ULL,
1309         0x5908e128f17c169aULL, 0xf77498dd8ad0852dULL, 0x74b4c4ceab102f64ULL, 0x183abadd10139845ULL,
1310         0xb165ba8daa92aaacULL, 0xd5c5ef9599386705ULL, 0xbe2f8f0cf8fc40d1ULL, 0x2701e635ee204514ULL,
1311         0x629fa80020156514ULL, 0xf223868764a8c1ceULL, 0x5b894fff0b3f060eULL, 0x60d9944cf708a3faULL,
1312         0xaeea001a1c7a201fULL, 0xebf16a633ee2ce63ULL, 0x6f7709594c7a07e1ULL, 0x79b958150d0208cbULL,
1313         0x24b55e5301d410e7ULL, 0xe3a34edff3fdc84dULL, 0xd88768e4904032d8ULL, 0x131384427b3aaeecULL,
1314         0x8405e51286234f14ULL, 0x14dc4739adb4c529ULL, 0xb8a2b5b250634ffdULL, 0x2fe2a94ad8a7ff93ULL,
1315         0xec5c57efe843faddULL, 0x2843ce40f0bb9918ULL, 0xa4b561d6cf3d6305ULL, 0x743629bde8fb777eULL,
1316         0x343edd46bbaf738fULL, 0xed981828b101a651ULL, 0xa401760b882c797aULL, 0x1fc223e28dc88730ULL,
1317         0x48604e91fc0fba0eULL, 0xb637f78f052c6fa4ULL, 0x91ccac3d09e9239cULL, 0x23f7eed4437a687cULL,
1318         0x5173b1118d9bd800ULL, 0x29d641b63189d4a7ULL, 0xfdbf177988bbc586ULL, 0x2959894fcad81df5ULL,
1319         0xaebc8ef3b4bbc899ULL, 0x4148995ab26992b9ULL, 0x24e20b0134f92cfbULL, 0x40d158894a05dee8ULL,
1320         0x46b00b1185af76f6ULL, 0x26bac77873187a79ULL, 0x3dc0bf95ab8fff5fULL, 0x2a608bd8945524d7ULL,
1321         0x26449588bd446302ULL, 0x7c4bc21c0388439cULL, 0x8e98a4f383bd11b2ULL, 0x26218d7bc9d876b9ULL,
1322         0xe3081542997c178aULL, 0x3c2d29a86fb6606fULL, 0x5c217736fa279374ULL, 0x7dde05734afeb1faULL,
1323         0x3bf10e3906d42babULL, 0xe4f7803e1980649cULL, 0xe6053bf89595bf7aULL, 0x394faf38da245530ULL,
1324         0x7a8efb58896928f4ULL, 0xfbc778e9cc6a113cULL, 0x72670ce330af596fULL, 0x48f222a81d3d6cf7ULL,
1325         0xf01fce410d72caa7ULL, 0x5a20ecc7213b5595ULL, 0x7bc21165c1fa1483ULL, 0x07f89ae31da8a741ULL,
1326         0x05d2c2b4c6830ff9ULL, 0xd43e330fc6316293ULL, 0xa5a5590a96d3a904ULL, 0x705edb91a65333b6ULL,
1327         0x048ee15e0bb9a5f7ULL, 0x3240cfca9e0aaf5dULL, 0x8f4b71ceedc4a40bULL, 0x621c0da3de544a6dULL,
1328         0x92872836a08c4091ULL, 0xce8375b010c91445ULL, 0x8a72eb524f276394ULL, 0x2667fcfa7ec83635ULL,
1329         0x7f4c173345e8752aULL, 0x061b47feee7079a5ULL, 0x25dd9afa9f86ff34ULL, 0x3780cef5425dc89cULL,
1330         0x1a46035a513bb4e9ULL, 0x3e1ef379ac575adaULL, 0xc78c5f1c5fa24b50ULL, 0x321a967634fd9f22ULL,
1331         0x946707b8826e27faULL, 0x3dca84d64c506fd0ULL, 0xc189218075e91436ULL, 0x6d9284169b3b8484ULL,
1332         0x3a67e840383f2ddfULL, 0x33eec9a30c4f9b75ULL, 0x3ec7c86fa783ef47ULL, 0x26ec449fbac9fbc4ULL,
1333         0x5c0f38cba09b9e7dULL, 0x81168cc762a3478cULL, 0x3e23b0d306fc121cULL, 0x5a238aa0a5efdcddULL,
1334         0x1ba26121c4ea43ffULL, 0x36f8c77f7c8832b5ULL, 0x88fbea0b0adcf99aULL, 0x5ca9938ec25bebf9ULL,
1335         0xd5436a5e51fccda0ULL, 0x1dbc4797c2cd893bULL, 0x19346a65d3224a08ULL, 0x0f5034e49b9af466ULL,
1336         0xf23c3967a1e0b96eULL, 0xe58b08fa867a4d88ULL, 0xfb2fabc6a7341679ULL, 0x2a75381eb6026946ULL,
1337         0xc80a3be4c19420acULL, 0x66b1f6c681f2b6dcULL, 0x7cf7036761e93388ULL, 0x25abbbd8a660a4c4ULL,
1338         0x91ea12ba14fd5198ULL, 0x684950fc4a3cffa9ULL, 0xf826842130f5ad28ULL, 0x3ea988f75301a441ULL,
1339         0xc978109a695f8c6fULL, 0x1746eb4a0530c3f3ULL, 0x444d6d77b4459995ULL, 0x75952b8c054e5cc7ULL,
1340         0xa3703f7915f4d6aaULL, 0x66c346202f2647d8ULL, 0xd01469df811d644bULL, 0x77fea47d81a5d71fULL,
1341         0xc5e9529ef57ca381ULL, 0x6eeeb4b9ce2f881aULL, 0xb6e91a28e8009bd6ULL, 0x4b80be3e9afc3fecULL,
1342         0x7e3773c526aed2c5ULL, 0x1b4afcb453c9a49dULL, 0xa920bdd7baffb24dULL, 0x7c54699f122d400eULL,
1343         0xef46c8e14fa94bc8ULL, 0xe0b074ce2952ed5eULL, 0xbea450e1dbd885d5ULL, 0x61b68649320f712cULL,
1344         0x8a485f7309ccbdd1ULL, 0xbd06320d7d4d1a2dULL, 0x25232973322dbef4ULL, 0x445dc4758c17f770ULL,
1345         0xdb0434177cc8933cULL, 0xed6fe82175ea059fULL, 0x1efebefdc053db34ULL, 0x4adbe867c65daf99ULL,
1346         0x3acd71a2a90609dfULL, 0xe5e991856dd04050ULL, 0x1ec69b688157c23cULL, 0x697427f6885cfe4dULL,
1347         0xd7be7b9b65e1a851ULL, 0xa03d28d522c536ddULL, 0x28399d658fd2b645ULL, 0x49e5b7e17c2641e1ULL,
1348         0x6f8c3a98700457a4ULL, 0x5078f0a25ebb6778ULL, 0xd13c3ccbc382960fULL, 0x2e003258a7df84b1ULL,
1349         0x8ad1f39be6296a1cULL, 0xc1eeaa652a5fbfb2ULL, 0x33ee0673fd26f3cbULL, 0x59256173a69d2cccULL,
1350         0x41ea07aa4e18fc41ULL, 0xd9fc19527c87a51eULL, 0xbdaacb805831ca6fULL, 0x445b652dc916694fULL,
1351         0xce92a3a7f2172315ULL, 0x1edc282de11b9964ULL, 0xa1823aafe04c314aULL, 0x790a2d94437cf586ULL,
1352         0x71c447fb93f6e009ULL, 0x8922a56722845276ULL, 0xbf70903b204f5169ULL, 0x2f7a89891ba319feULL,
1353         0x02a08eb577e2140cULL, 0xed9a4ed4427bdcf4ULL, 0x5253ec44e4323cd1ULL, 0x3e88363c14e9355bULL,
1354         0xaa66c14277110b8cULL, 0x1ae0391610a23390ULL, 0x2030bd12c93fc2a2ULL, 0x3ee141579555c7abULL,
1355         0x9214de3a6d6e7d41ULL, 0x3ccdd88607f17efeULL, 0x674f1288f8e11217ULL, 0x5682250f329f93d0ULL,
1356         0x6cf00b136d2e396eULL, 0x6e4cf86f1014debfULL, 0x5930b1b5bfcc4e83ULL, 0x047069b48aba16b6ULL,
1357         0x0d4ce4ab69b20793ULL, 0xb24db91a97d0fb9eULL, 0xcdfa50f54e00d01dULL, 0x221b1085368bddb5ULL,
1358         0xe7e59468b1e3d8d2ULL, 0x53c56563bd122f93ULL, 0xeee8a903e0663f09ULL, 0x61efa662cbbe3d42ULL,
1359         0x2cf8ddddde6eab2aULL, 0x9bf80ad51435f231ULL, 0x5deadacec9f04973ULL, 0x29275b5d41d29b27ULL,
1360         0xcfde0f0895ebf14fULL, 0xb9aab96b054905a7ULL, 0xcae80dd9a1c420fdULL, 0x0a63bf2f1673bbc7ULL,
1361         0x092f6e11958fbc8cULL, 0x672a81e804822fadULL, 0xcac8351560d52517ULL, 0x6f3f7722c8f192f8ULL,
1362         0xf8ba90ccc2e894b7ULL, 0x2c7557a438ff9f0dULL, 0x894d1d855ae52359ULL, 0x68e122157b743d69ULL,
1363         0xd87e5570cfb919f3ULL, 0x3f2cdecd95798db9ULL, 0x2121154710c0a2ceULL, 0x3c66a115246dc5b2ULL,
1364         0xcbedc562294ecb72ULL, 0xba7143c36a280b16ULL, 0x9610c2efd4078b67ULL, 0x6144735d946a4b1eULL,
1365         0x536f111ed75b3350ULL, 0x0211db8c2041d81bULL, 0xf93cb1000e10413cULL, 0x149dfd3c039e8876ULL,
1366         0xd479dde46b63155bULL, 0xb66e15e93c837976ULL, 0xdafde43b1f13e038ULL, 0x5fafda1a2e4b0b35ULL,
1367         0x3600bbdf17197581ULL, 0x3972050bbe3cd2c2ULL, 0x5938906dbdd5be86ULL, 0x34fce5e43f9b860fULL,
1368         0x75a8a4cd42d14d02ULL, 0x828dabc53441df65ULL, 0x33dcabedd2e131d3ULL, 0x3ebad76fb814d25fULL,
1369         0xd4906f566f70e10fULL, 0x5d12f7aa51690f5aULL, 0x45adb16e76cefcf2ULL, 0x01f768aead232999ULL,
1370         0x2b6cc77b6248febdULL, 0x3cd30628ec3aaffdULL, 0xce1c0b80d4ef486aULL, 0x4c3bff2ea6f66c23ULL,
1371         0x3f2ec4094aeaeb5fULL, 0x61b19b286e372ca7ULL, 0x5eefa966de2a701dULL, 0x23b20565de55e3efULL,
1372         0xe301ca5279d58557ULL, 0x07b2d4ce27c2874fULL, 0xa532cd8a9dcf1d67ULL, 0x2a52fee23f2bff56ULL,
1373         0x8624efb37cd8663dULL, 0xbbc7ac20ffbd7594ULL, 0x57b85e9c82d37445ULL, 0x7b3052cb86a6ec66ULL,
1374         0x3482f0ad2525e91eULL, 0x2cb68043d28edca0ULL, 0xaf4f6d052e1b003aULL, 0x185f8c2529781b0aULL,
1375         0xaa41de5bd80ce0d6ULL, 0x9407b2416853e9d6ULL, 0x563ec36e357f4c3aULL, 0x4cc4b8dd0e297bceULL,
1376         0xa2fc1a52ffb8730eULL, 0x1811f16e67058e37ULL, 0x10f9a366cddf4ee1ULL, 0x72f4a0c4a0b9f099ULL,
1377         0x8c16c06f663f4ea7ULL, 0x693b3af74e970fbaULL, 0x2102e7f1d69ec345ULL, 0x0ba53cbc968a8089ULL,
1378         0xca3d9dc7fea15537ULL, 0x4c6824bb51536493ULL, 0xb9886314844006b1ULL, 0x40d2a72ab454cc60ULL,
1379         0x5936a1b712570975ULL, 0x91b9d648debda657ULL, 0x3344094bb64330eaULL, 0x006ba10d12ee51d0ULL,
1380         0x19228468f5de5d58ULL, 0x0eb12f4c38cc05b0ULL, 0xa1039f9dd5601990ULL, 0x4502d4ce4fff0e0bULL,
1381         0xeb2054106837c189ULL, 0xd0f6544c6dd3b93cULL, 0x40727064c416d74fULL, 0x6e15c6114b502ef0ULL,
1382         0x4df2a398cfb1a76bULL, 0x11256c7419f2f6b1ULL, 0x4a497962066e6043ULL, 0x705b3aab41355b44ULL,
1383         0x365ef536d797b1d8ULL, 0x00076bd622ddf0dbULL, 0x3bbf33b0e0575a88ULL, 0x3777aa05c8e4ca4dULL,
1384         0x392745c85578db5fULL, 0x6fda4149dbae5ae2ULL, 0xb1f0b00b8adc9867ULL, 0x09963437d36f1da3ULL,
1385         0x7e824e90a5dc3853ULL, 0xccb5f6641f135cbdULL, 0x6736d86c87ce8fccULL, 0x625f3ce26604249fULL,
1386         0xaf8ac8059502f63fULL, 0x0c05e70a2e351469ULL, 0x35292e9c764b6305ULL, 0x1a394360c7e23ac3ULL,
1387         0xd5c6d53251183264ULL, 0x62065abd43c2b74fULL, 0xb5fbf5d03b973f9bULL, 0x13a3da3661206e5eULL,
1388         0xc6bd5837725d94e5ULL, 0x18e30912205016c5ULL, 0x2088ce1570033c68ULL, 0x7fba1f495c837987ULL,
1389         0x5a8c7423f2f9079dULL, 0x1735157b34023fc5ULL, 0xe4f9b49ad2fab351ULL, 0x6691ff72c878e33cULL,
1390         0x122c2adedc5eff3eULL, 0xf8dd4bf1d8956cf4ULL, 0xeb86205d9e9e5bdaULL, 0x049b92b9d975c743ULL,
1391         0xa5379730b0f6c05aULL, 0x72a0ffacc6f3a553ULL, 0xb0032c34b20dcd6dULL, 0x470e9dbc88d5164aULL,
1392         0xb19cf10ca237c047ULL, 0xb65466711f6c81a2ULL, 0xb3321bd16dd80b43ULL, 0x48c14f600c5fbe8eULL,
1393         0x66451c264aa6c803ULL, 0xb66e3904a4fa7da6ULL, 0xd45f19b0b3128395ULL, 0x31602627c3c9bc10ULL,
1394         0x3120dc4832e4e10dULL, 0xeb20c46756c717f7ULL, 0x00f52e3f67280294ULL, 0x566d4fc14730c509ULL,
1395         0x7e3a5d40fd837206ULL, 0xc1e926dc7159547aULL, 0x216730fba68d6095ULL, 0x22e8c3843f69cea7ULL,
1396         0x33d074e8930e4b2bULL, 0xb6e4350e84d15816ULL, 0x5534c26ad6ba2365ULL, 0x7773c12f89f1f3f3ULL,
1397         0x8cba404da57962aaULL, 0x5b9897a81999ce56ULL, 0x508e862f121692fcULL, 0x3a81907fa093c291ULL,
1398         0x0dded0ff4725a510ULL, 0x10d8cc10673fc503ULL, 0x5b9d151c9f1f4e89ULL, 0x32a5c1d5cb09a44cULL,
1399         0x1e0aa442b90541fbULL, 0x5f85eb7cc1b485dbULL, 0xbee595ce8a9df2e5ULL, 0x25e496c722422236ULL,
1400         0x5edf3c46cd0fe5b9ULL, 0x34e75a7ed2a43388ULL, 0xe488de11d761e352ULL, 0x0e878a01a085545cULL,
1401         0xba493c77e021bb04ULL, 0x2b4d1843c7df899aULL, 0x9ea37a487ae80d67ULL, 0x67a9958011e41794ULL,
1402         0x4b58051a6697b065ULL, 0x47e33f7d8d6ba6d4ULL, 0xbb4da8d483ca46c1ULL, 0x68becaa181c2db0dULL,
1403         0x8d8980e90b989aa5ULL, 0xf95eb14a2c93c99bULL, 0x51c6c7c4796e73a2ULL, 0x6e228363b5efb569ULL,
1404         0xc6bbc0b02dd624c8ULL, 0x777eb47dec8170eeULL, 0x3cde15a004cfafa9ULL, 0x1dc6bc087160bf9bULL,
1405         0x2e07e043eec34002ULL, 0x18e9fc677a68dc7fULL, 0xd8da03188bd15b9aULL, 0x48fbc3bb00568253ULL,
1406         0x57547d4cfb654ce1ULL, 0xd3565b82a058e2adULL, 0xf63eaf0bbf154478ULL, 0x47531ef114dfbb18ULL,
1407         0xe1ec630a4278c587ULL, 0x5507d546ca8e83f3ULL, 0x85e135c63adc0c2bULL, 0x0aa7efa85682844eULL,
1408         0x72691ba8b3e1f615ULL, 0x32b4e9701fbe3ffaULL, 0x97b6d92e39bb7868ULL, 0x2cfe53dea02e39e8ULL,
1409         0x687392cd85cd52b0ULL, 0x27ff66c910e29831ULL, 0x97134556a9832d06ULL, 0x269bb0360a84f8a0ULL,
1410         0x706e55457643f85cULL, 0x3734a48c9b597d1bULL, 0x7aee91e8c6efa472ULL, 0x5cd6abc198a9d9e0ULL,
1411         0x0e04de06cb3ce41aULL, 0xd8c6eb893402e138ULL, 0x904659bb686e3772ULL, 0x7215c371746ba8c8ULL,
1412         0xfd12a97eeae4a2d9ULL, 0x9514b7516394f2c5ULL, 0x266fd5809208f294ULL, 0x5c847085619a26b9ULL,
1413         0x52985410fed694eaULL, 0x3c905b934a2ed254ULL, 0x10bb47692d3be467ULL, 0x063b3d2d69e5e9e1ULL,
1414         0x472726eedda57debULL, 0xefb6c4ae10f41891ULL, 0x2b1641917b307614ULL, 0x117c554fc4f45b7cULL,
1415         0xc07cf3118f9d8812ULL, 0x01dbd82050017939ULL, 0xd7e803f4171b2827ULL, 0x1015e87487d225eaULL,
1416         0xc58de3fed23acc4dULL, 0x50db91c294a7be2dULL, 0x0b94d43d1c9cf457ULL, 0x6b1640fa6e37524aULL,
1417         0x692f346c5fda0d09ULL, 0x200b1c59fa4d3151ULL, 0xb8c46f760777a296ULL, 0x4b38395f3ffdfbcfULL,
1418         0x18d25e00be54d671ULL, 0x60d50582bec8aba6ULL, 0x87ad8f263b78b982ULL, 0x50fdf64e9cda0432ULL,
1419         0x90f567aac578dcf0ULL, 0xef1e9b0ef2a3133bULL, 0x0eebba9242d9de71ULL, 0x15473c9bf03101c7ULL,
1420         0x7c77e8ae56b78095ULL, 0xb678e7666e6f078eULL, 0x2da0b9615348ba1fULL, 0x7cf931c1ff733f0bULL,
1421         0x26b357f50a0a366cULL, 0xe9708cf42b87d732ULL, 0xc13aeea5f91cb2c0ULL, 0x35d90c991143bb4cULL,
1422         0x47c1c404a9a0d9dcULL, 0x659e58451972d251ULL, 0x3875a8c473b38c31ULL, 0x1fbd9ed379561f24ULL,
1423         0x11fabc6fd41ec28dULL, 0x7ef8dfe3cd2a2dcaULL, 0x72e73b5d8c404595ULL, 0x6135fa4954b72f27ULL,
1424         0xccfc32a2de24b69cULL, 0x3f55698c1f095d88ULL, 0xbe3350ed5ac3f929ULL, 0x5e9bf806ca477eebULL,
1425         0xe9ce8fb63c309f68ULL, 0x5376f63565e1f9f4ULL, 0xd1afcfb35a6393f1ULL, 0x6632a1ede5623506ULL,
1426         0x0b7d6c390c2ded4cULL, 0x56cb3281df04cb1fULL, 0x66305a1249ecc3c7ULL, 0x5d588b60a38ca72aULL,
1427         0xa6ecbf78e8e5f42dULL, 0x86eeb44b3c8a3eecULL, 0xec219c48fbd21604ULL, 0x1aaf1af517c36731ULL,
1428         0xc306a2836769bde7ULL, 0x208280622b1e2adbULL, 0x8027f51ffbff94a6ULL, 0x76cfa1ce1124f26bULL,
1429         0x18eb00562422abb6ULL, 0xf377c4d58f8c29c3ULL, 0x4dbbc207f531561aULL, 0x0253b7f082128a27ULL,
1430         0x3d1f091cb62c17e0ULL, 0x4860e1abd64628a9ULL, 0x52d17436309d4253ULL, 0x356f97e13efae576ULL,
1431         0xd351e11aa150535bULL, 0x3e6b45bb1dd878ccULL, 0x0c776128bed92c98ULL, 0x1d34ae93032885b8ULL,
1432         0x4ba0488ca85ba4c3ULL, 0x985348c33c9ce6ceULL, 0x66124c6f97bda770ULL, 0x0f81a0290654124aULL,
1433         0x9ed09ca6569b86fdULL, 0x811009fd18af9a2dULL, 0xff08d03f93d8c20aULL, 0x52a148199faef26bULL,
1434         0x3e03f9dc2d8d1b73ULL, 0x4205801873961a70ULL, 0xc0d987f041a35970ULL, 0x07aa1f15a1c0d549ULL,
1435         0xdfd46ce08cd27224ULL, 0x6d0a024f934e4239ULL, 0x808a7a6399897b59ULL, 0x0a4556e9e13d95a2ULL,
1436         0xd21a991fe9c13045ULL, 0x9b0e8548fe7751b8ULL, 0x5da643cb4bf30035ULL, 0x77db28d63940f721ULL,
1437         0xfc5eeb614adc9011ULL, 0x5229419ae8c411ebULL, 0x9ec3e7787d1dcf74ULL, 0x340d053e216e4cb5ULL,
1438         0xcac7af39b48df2b4ULL, 0xc0faec2871a10a94ULL, 0x140a69245ca575edULL, 0x0cf1c37134273a4cULL,
1439         0xc8ee306ac224b8a5ULL, 0x57eaee7ccb4930b0ULL, 0xa1e806bdaacbe74fULL, 0x7d9a62742eeb657dULL,
1440         0x9eb6b6ef546c4830ULL, 0x885cca1fddb36e2eULL, 0xe6b9f383ef0d7105ULL, 0x58654fef9d2e0412ULL,
1441         0xa905c4ffbe0e8e26ULL, 0x942de5df9b31816eULL, 0x497d723f802e88e1ULL, 0x30684dea602f408dULL,
1442         0x21e5a278a3e6cb34ULL, 0xaefb6e6f5b151dc4ULL, 0xb30b8e049d77ca15ULL, 0x28c3c9cf53b98981ULL,
1443         0x287fb721556cdd2aULL, 0x0d317ca897022274ULL, 0x7468c7423a543258ULL, 0x4a7f11464eb5642fULL,
1444         0xa237a4774d193aa6ULL, 0xd865986ea92129a1ULL, 0x24c515ecf87c1a88ULL, 0x604003575f39f5ebULL,
1445         0x47b9f189570a9b27ULL, 0x2b98cede465e4b78ULL, 0x026df551dbb85c20ULL, 0x74fcd91047e21901ULL,
1446         0x13e2a90a23c1bfa3ULL, 0x0cb0074e478519f6ULL, 0x5ff1cbbe3af6cf44ULL, 0x67fe5438be812dbeULL,
1447         0xd13cf64fa40f05b0ULL, 0x054dfb2f32283787ULL, 0x4173915b7f0d2aeaULL, 0x482f144f1f610d4eULL,
1448         0xf6210201b47f8234ULL, 0x5d0ae1929e70b990ULL, 0xdcd7f455b049567cULL, 0x7e93d0f1f0916f01ULL,
1449         0xdd79cbf18a7db4faULL, 0xbe8391bf6f74c62fULL, 0x027145d14b8291bdULL, 0x585a73ea2cbf1705ULL,
1450         0x485ca03e928a0db2ULL, 0x10fc01a5742857e7ULL, 0x2f482edbd6d551a7ULL, 0x0f0433b5048fdb8aULL,
1451         0x60da2e8dd7dc6247ULL, 0x88b4c9d38cd4819aULL, 0x13033ac001f66697ULL, 0x273b24fe3b367d75ULL,
1452         0xc6e8f66a31b3b9d4ULL, 0x281514a494df49d5ULL, 0xd1726fdfc8b23da7ULL, 0x4b3ae7d103dee548ULL,
1453         0xc6256e19ce4b9d7eULL, 0xff5c5cf186e3c61cULL, 0xacc63ca34b8ec145ULL, 0x74621888fee66574ULL,
1454         0x956f409645290a1eULL, 0xef0bf8e3263a962eULL, 0xed6a50eb5ec2647bULL, 0x0694283a9dca7502ULL,
1455         0x769b963643a2dcd1ULL, 0x42b7c8ea09fc5353ULL, 0x4f002aee13397eabULL, 0x63005e2c19b7d63aULL,
1456         0xca6736da63023beaULL, 0x966c7f6db12a99b7ULL, 0xace09390c537c5e1ULL, 0x0b696063a1aa89eeULL,
1457         0xebb03e97288c56e5ULL, 0x432a9f9f938c8be8ULL, 0xa6a5a93d5b717f71ULL, 0x1a5fb4c3e18f9d97ULL,
1458         0x1c94e7ad1c60cdceULL, 0xee202a43fc02c4a0ULL, 0x8dafe4d867c46a20ULL, 0x0a10263c8ac27b58ULL,
1459         0xd0dea9dfe4432a4aULL, 0x856af87bbe9277c5ULL, 0xce8472acc212c71aULL, 0x6f151b6d9bbb1e91ULL,
1460         0x26776c527ceed56aULL, 0x7d211cb7fbf8faecULL, 0x37ae66a6fd4609ccULL, 0x1f81b702d2770c42ULL,
1461         0x2fb0b057eac58392ULL, 0xe1dd89fe29744e9dULL, 0xc964f8eb17beb4f8ULL, 0x29571073c9a2d41eULL,
1462         0xa948a18981c0e254ULL, 0x2df6369b65b22830ULL, 0xa33eb2d75fcfd3c6ULL, 0x078cd6ec4199a01fULL,
1463         0x4a584a41ad900d2fULL, 0x32142b78e2c74c52ULL, 0x68c4e8338431c978ULL, 0x7f69ea9008689fc2ULL,
1464         0x52f2c81e46a38265ULL, 0xfd78072d04a832fdULL, 0x8cd7d5fa25359e94ULL, 0x4de71b7454cc29d2ULL,
1465         0x42eb60ad1eda6ac9ULL, 0x0aad37dfdbc09c3aULL, 0x81004b71e33cc191ULL, 0x44e6be345122803cULL,
1466         0x03fe8388ba1920dbULL, 0xf5d57c32150db008ULL, 0x49c8c4281af60c29ULL, 0x21edb518de701aeeULL,
1467         0x7fb63e418f06dc99ULL, 0xa4460d99c166d7b8ULL, 0x24dd5248ce520a83ULL, 0x5ec3ad712b928358ULL,
1468         0x15022a5fbd17930fULL, 0xa4f64a77d82570e3ULL, 0x12bc8d6915783712ULL, 0x498194c0fc620abbULL,
1469         0x38a2d9d255686c82ULL, 0x785c6bd9193e21f0ULL, 0xe4d5c81ab24a5484ULL, 0x56307860b2e20989ULL,
1470         0x429d55f78b4d74c4ULL, 0x22f1834643350131ULL, 0x1e60c24598c71fffULL, 0x59f2f014979983efULL,
1471         0x46a47d56eb494a44ULL, 0x3e22a854d636a18eULL, 0xb346e15274491c3bULL, 0x2ceafd4e5390cde7ULL,
1472         0xba8a8538be0d6675ULL, 0x4b9074bb50818e23ULL, 0xcbdab89085d304c3ULL, 0x61a24fe0e56192c4ULL,
1473         0xcb7615e6db525bcbULL, 0xdd7d8c35a567e4caULL, 0xe6b4153acafcdd69ULL, 0x2d668e097f3c9766ULL,
1474         0xa57e7e265ce55ef0ULL, 0x5d9f4e527cd4b967ULL, 0xfbc83606492fd1e5ULL, 0x090d52beb7c3f7aeULL,
1475         0x09b9515a1e7b4d7cULL, 0x1f266a2599da44c0ULL, 0xa1c49548e2c55504ULL, 0x7ef04287126f15ccULL,
1476         0xfed1659dbd30ef15ULL, 0x8b4ab9eec4e0277bULL, 0x884d6236a5df3291ULL, 0x1fd96ea6bf5cf788ULL,
1477         0x42a161981f190d9aULL, 0x61d849507e6052c1ULL, 0x9fe113bf285a2cd5ULL, 0x7c22d676dbad85d8ULL,
1478         0x82e770ed2bfbd27dULL, 0x4c05b2ece996f5a5ULL, 0xcd40a9c2b0900150ULL, 0x5895319213d9bf64ULL,
1479         0xe7cc5d703fea2e08ULL, 0xb50c491258e2188cULL, 0xcce30baa48205bf0ULL, 0x537c659ccfa32d62ULL,
1480         0x37b6623a98cfc088ULL, 0xfe9bed1fa4d6aca4ULL, 0x04d29b8e56a8d1b0ULL, 0x725f71c40b519575ULL,
1481         0x28c7f89cd0339ce6ULL, 0x8367b14469ddc18bULL, 0x883ada83a6a1652cULL, 0x585f1974034d6c17ULL,
1482         0x89cfb266f1b19188ULL, 0xe63b4863e7c35217ULL, 0xd88c9da6b4c0526aULL, 0x3e035c9df0954635ULL,
1483         0xdd9d5412fb45de9dULL, 0xdd684532e4cff40dULL, 0x4b5c999b151d671cULL, 0x2d8c2cc811e7f690ULL,
1484         0x7f54be1d90055d40ULL, 0xa464c5df464aaf40ULL, 0x33979624f0e917beULL, 0x2c018dc527356b30ULL,
1485         0xa5415024e330b3d4ULL, 0x73ff3d96691652d3ULL, 0x94ec42c4ef9b59f1ULL, 0x0747201618d08e5aULL,
1486         0x4d6ca48aca411c53ULL, 0x66415f2fcfa66119ULL, 0x9c4dd40051e227ffULL, 0x59810bc09a02f7ebULL,
1487         0x2a7eb171b3dc101dULL, 0x441c5ab99ffef68eULL, 0x32025c9b93b359eaULL, 0x5e8ce0a71e9d112fULL,
1488         0xbfcccb92429503fdULL, 0xd271ba752f095d55ULL, 0x345ead5e972d091eULL, 0x18c8df11a83103baULL,
1489         0x90cd949a9aed0f4cULL, 0xc5d1f4cb6660e37eULL, 0xb8cac52d56c52e0bULL, 0x6e42e400c5808e0dULL,
1490         0xa3b46966eeaefd23ULL, 0x0c4f1f0be39ecdcaULL, 0x189dc8c9d683a51dULL, 0x51f27f054c09351bULL,
1491         0x4c487ccd2a320682ULL, 0x587ea95bb3df1c96ULL, 0xc8ccf79e555cb8e8ULL, 0x547dc829a206d73dULL,
1492         0xb822a6cd80c39b06ULL, 0xe96d54732000d4c6ULL, 0x28535b6f91463b4dULL, 0x228f4660e2486e1dULL,
1493         0x98799538de8d3abfULL, 0x8cd8330045ebca6eULL, 0x79952a008221e738ULL, 0x4322e1a7535cd2bbULL,
1494         0xb114c11819d1801cULL, 0x2016e4d84f3f5ec7ULL, 0xdd0e2df409260f4cULL, 0x5ec362c0ae5f7266ULL,
1495         0xc0462b18b8b2b4eeULL, 0x7cc8d950274d1afbULL, 0xf25f7105436b02d2ULL, 0x43bbf8dcbff9ccd3ULL,
1496         0xb6ad1767a039e9dfULL, 0xb0714da8f69d3583ULL, 0x5e55fa18b42931f5ULL, 0x4ed5558f33c60961ULL,
1497         0x1fe37901c647a5ddULL, 0x593ddf1f8081d357ULL, 0x0249a4fd813fd7a6ULL, 0x69acca274e9caf61ULL,
1498         0x047ba3ea330721c9ULL, 0x83423fc20e7e1ea0ULL, 0x1df4c0af01314a60ULL, 0x09a62dab89289527ULL,
1499         0xa5b325a49cc6cb00ULL, 0xe94b5dc654b56cb6ULL, 0x3be28779adc994a0ULL, 0x4296e8f8ba3a4aadULL,
1500         0x328689761e451eabULL, 0x2e4d598bff59594aULL, 0x49b96853d7a7084aULL, 0x4980a319601420a8ULL,
1501         0x9565b9e12f552c42ULL, 0x8a5318db7100fe96ULL, 0x05c90b4d43add0d7ULL, 0x538b4cd66a5d4edaULL,
1502         0xf4e94fc3e89f039fULL, 0x592c9af26f618045ULL, 0x08a36eb5fd4b9550ULL, 0x25fffaf6c2ed1419ULL,
1503         0x34434459cc79d354ULL, 0xeeecbfb4b1d5476bULL, 0xddeb34a061615d99ULL, 0x5129cecceb64b773ULL,
1504         0xee43215894993520ULL, 0x772f9c7cf14c0b3bULL, 0xd2e2fce306bedad5ULL, 0x715f42b546f06a97ULL,
1505         0x434ecdceda5b5f1aULL, 0x0da17115a49741a9ULL, 0x680bd77c73edad2eULL, 0x487c02354edd9041ULL,
1506         0xb8efeff3a70ed9c4ULL, 0x56a32aa3e857e302ULL, 0xdf3a68bd48a2a5a0ULL, 0x07f650b73176c444ULL,
1507         0xe38b9b1626e0ccb1ULL, 0x79e053c18b09fb36ULL, 0x56d90319c9f94964ULL, 0x1ca941e7ac9ff5c4ULL,
1508         0x49c4df29162fa0bbULL, 0x8488cf3282b33305ULL, 0x95dfda14cabb437dULL, 0x3391f78264d5ad86ULL,
1509         0x729ae06ae2b5095dULL, 0xd58a58d73259a946ULL, 0xe9834262d13921edULL, 0x27fedafaa54bb592ULL,
1510         0xa99dc5b829ad48bbULL, 0x5f025742499ee260ULL, 0x802c8ecd5d7513fdULL, 0x78ceb3ef3f6dd938ULL,
1511         0xc342f44f8a135d94ULL, 0x7b9edb44828cdda3ULL, 0x9436d11a0537cfe7ULL, 0x5064b164ec1ab4c8ULL,
1512         0x7020eccfd37eb2fcULL, 0x1f31ea3ed90d25fcULL, 0x1b930d7bdfa1bb34ULL, 0x5344467a48113044ULL,
1513         0x70073170f25e6dfbULL, 0xe385dc1a50114cc8ULL, 0x2348698ac8fc4f00ULL, 0x2a77a55284dd40d8ULL,
1514         0xfe06afe0c98c6ce4ULL, 0xc235df96dddfd6e4ULL, 0x1428d01e33bf1ed3ULL, 0x785768ec9300bdafULL,
1515         0x9702e57a91deb63bULL, 0x61bdb8bfe5ce8b80ULL, 0x645b426f3d1d58acULL, 0x4804a82227a557bcULL,
1516         0x8e57048ab44d2601ULL, 0x68d6501a4b3a6935ULL, 0xc39c9ec3f9e1c293ULL, 0x4172f257d4de63e2ULL,
1517         0xd368b450330c6401ULL, 0x040d3017418f2391ULL, 0x2c34bb6090b7d90dULL, 0x16f649228fdfd51fULL,
1518         0xbea6818e2b928ef5ULL, 0xe28ccf91cdc11e72ULL, 0x594aaa68e77a36cdULL, 0x313034806c7ffd0fULL,
1519         0x8a9d27ac2249bd65ULL, 0x19a3b464018e9512ULL, 0xc26ccff352b37ec7ULL, 0x056f68341d797b21ULL,
1520         0x5e79d6757efd2327ULL, 0xfabdbcb6553afe15ULL, 0xd3e7222c6eaf5a60ULL, 0x7046c76d4dae743bULL,
1521         0x660be872b18d4a55ULL, 0x19992518574e1496ULL, 0xc103053a302bdcbbULL, 0x3ed8e9800b218e8eULL,
1522         0x7b0b9239fa75e03eULL, 0xefe9fb684633c083ULL, 0x98a35fbe391a7793ULL, 0x6065510fe2d0fe34ULL,
1523         0x55cb668548abad0cULL, 0xb4584548da87e527ULL, 0x2c43ecea0107c1ddULL, 0x526028809372de35ULL,
1524         0x3415c56af9213b1fULL, 0x5bee1a4d017e98dbULL, 0x13f6b105b5cf709bULL, 0x5ff20e3482b29ab6ULL,
1525         0x0aa29c75cc2e6c90ULL, 0xfc7d73ca3a70e206ULL, 0x899fc38fc4b5c515ULL, 0x250386b124ffc207ULL,
1526         0x54ea28d5ae3d2b56ULL, 0x9913149dd6de60ceULL, 0x16694fc58f06d6c1ULL, 0x46b23975eb018fc7ULL,
1527         0x470a6a0fb4b7b4e2ULL, 0x5d92475a8f7253deULL, 0xabeee5b52fbd3adbULL, 0x7fa20801a0806968ULL,
1528         0x76f3faf19f7714d2ULL, 0xb3e840c12f4660c3ULL, 0x0fb4cd8df212744eULL, 0x4b065a251d3a2dd2ULL,
1529         0x5cebde383d77cd4aULL, 0x6adf39df882c9cb1ULL, 0xa2dd242eb09af759ULL, 0x3147c0e50e5f6422ULL,
1530         0x164ca5101d1350dbULL, 0xf8d13479c33fc962ULL, 0xe640ce4d13e5da08ULL, 0x4bdee0c45061f8baULL,
1531         0xd7c46dc1a4edb1c9ULL, 0x5514d7b6437fd98aULL, 0x58942f6bb2a1c00bULL, 0x2dffb2ab1d70710eULL,
1532         0xccdfcf2fc18b6d68ULL, 0xa8ebcba8b7806167ULL, 0x980697f95e2937e3ULL, 0x02fbba1cd0126e8cULL
1533 };
1534
1535 static void curve25519_ever64_base(u8 *out, const u8 *priv)
1536 {
1537         u64 swap = 1;
1538         int i, j, k;
1539         u64 tmp[16 + 32 + 4];
1540         u64 *x1 = &tmp[0];
1541         u64 *z1 = &tmp[4];
1542         u64 *x2 = &tmp[8];
1543         u64 *z2 = &tmp[12];
1544         u64 *xz1 = &tmp[0];
1545         u64 *xz2 = &tmp[8];
1546         u64 *a = &tmp[0 + 16];
1547         u64 *b = &tmp[4 + 16];
1548         u64 *c = &tmp[8 + 16];
1549         u64 *ab = &tmp[0 + 16];
1550         u64 *abcd = &tmp[0 + 16];
1551         u64 *ef = &tmp[16 + 16];
1552         u64 *efgh = &tmp[16 + 16];
1553         u64 *key = &tmp[0 + 16 + 32];
1554
1555         memcpy(key, priv, 32);
1556         ((u8 *)key)[0] &= 248;
1557         ((u8 *)key)[31] = (((u8 *)key)[31] & 127) | 64;
1558
1559         x1[0] = 1, x1[1] = x1[2] = x1[3] = 0;
1560         z1[0] = 1, z1[1] = z1[2] = z1[3] = 0;
1561         z2[0] = 1, z2[1] = z2[2] = z2[3] = 0;
1562         memcpy(x2, p_minus_s, sizeof(p_minus_s));
1563
1564         j = 3;
1565         for (i = 0; i < 4; ++i) {
1566                 while (j < (const int[]){ 64, 64, 64, 63 }[i]) {
1567                         u64 bit = (key[i] >> j) & 1;
1568                         k = (64 * i + j - 3);
1569                         swap = swap ^ bit;
1570                         cswap2(swap, xz1, xz2);
1571                         swap = bit;
1572                         fsub(b, x1, z1);
1573                         fadd(a, x1, z1);
1574                         fmul(c, &table_ladder[4 * k], b, ef);
1575                         fsub(b, a, c);
1576                         fadd(a, a, c);
1577                         fsqr2(ab, ab, efgh);
1578                         fmul2(xz1, xz2, ab, efgh);
1579                         ++j;
1580                 }
1581                 j = 0;
1582         }
1583
1584         point_double(xz1, abcd, efgh);
1585         point_double(xz1, abcd, efgh);
1586         point_double(xz1, abcd, efgh);
1587         encode_point(out, xz1);
1588
1589         memzero_explicit(tmp, sizeof(tmp));
1590 }
1591
1592 static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_bmi2_adx);
1593
1594 void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
1595                      const u8 secret[CURVE25519_KEY_SIZE],
1596                      const u8 basepoint[CURVE25519_KEY_SIZE])
1597 {
1598         if (static_branch_likely(&curve25519_use_bmi2_adx))
1599                 curve25519_ever64(mypublic, secret, basepoint);
1600         else
1601                 curve25519_generic(mypublic, secret, basepoint);
1602 }
1603 EXPORT_SYMBOL(curve25519_arch);
1604
1605 void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
1606                           const u8 secret[CURVE25519_KEY_SIZE])
1607 {
1608         if (static_branch_likely(&curve25519_use_bmi2_adx))
1609                 curve25519_ever64_base(pub, secret);
1610         else
1611                 curve25519_generic(pub, secret, curve25519_base_point);
1612 }
1613 EXPORT_SYMBOL(curve25519_base_arch);
1614
1615 static int curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1616                                  unsigned int len)
1617 {
1618         u8 *secret = kpp_tfm_ctx(tfm);
1619
1620         if (!len)
1621                 curve25519_generate_secret(secret);
1622         else if (len == CURVE25519_KEY_SIZE &&
1623                  crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE))
1624                 memcpy(secret, buf, CURVE25519_KEY_SIZE);
1625         else
1626                 return -EINVAL;
1627         return 0;
1628 }
1629
1630 static int curve25519_generate_public_key(struct kpp_request *req)
1631 {
1632         struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1633         const u8 *secret = kpp_tfm_ctx(tfm);
1634         u8 buf[CURVE25519_KEY_SIZE];
1635         int copied, nbytes;
1636
1637         if (req->src)
1638                 return -EINVAL;
1639
1640         curve25519_base_arch(buf, secret);
1641
1642         /* might want less than we've got */
1643         nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
1644         copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
1645                                                                 nbytes),
1646                                      buf, nbytes);
1647         if (copied != nbytes)
1648                 return -EINVAL;
1649         return 0;
1650 }
1651
1652 static int curve25519_compute_shared_secret(struct kpp_request *req)
1653 {
1654         struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1655         const u8 *secret = kpp_tfm_ctx(tfm);
1656         u8 public_key[CURVE25519_KEY_SIZE];
1657         u8 buf[CURVE25519_KEY_SIZE];
1658         int copied, nbytes;
1659
1660         if (!req->src)
1661                 return -EINVAL;
1662
1663         copied = sg_copy_to_buffer(req->src,
1664                                    sg_nents_for_len(req->src,
1665                                                     CURVE25519_KEY_SIZE),
1666                                    public_key, CURVE25519_KEY_SIZE);
1667         if (copied != CURVE25519_KEY_SIZE)
1668                 return -EINVAL;
1669
1670         curve25519_arch(buf, secret, public_key);
1671
1672         /* might want less than we've got */
1673         nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len);
1674         copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst,
1675                                                                 nbytes),
1676                                      buf, nbytes);
1677         if (copied != nbytes)
1678                 return -EINVAL;
1679         return 0;
1680 }
1681
1682 static unsigned int curve25519_max_size(struct crypto_kpp *tfm)
1683 {
1684         return CURVE25519_KEY_SIZE;
1685 }
1686
1687 static struct kpp_alg curve25519_alg = {
1688         .base.cra_name          = "curve25519",
1689         .base.cra_driver_name   = "curve25519-x86",
1690         .base.cra_priority      = 200,
1691         .base.cra_module        = THIS_MODULE,
1692         .base.cra_ctxsize       = CURVE25519_KEY_SIZE,
1693
1694         .set_secret             = curve25519_set_secret,
1695         .generate_public_key    = curve25519_generate_public_key,
1696         .compute_shared_secret  = curve25519_compute_shared_secret,
1697         .max_size               = curve25519_max_size,
1698 };
1699
1700
1701 static int __init curve25519_mod_init(void)
1702 {
1703         if (boot_cpu_has(X86_FEATURE_BMI2) && boot_cpu_has(X86_FEATURE_ADX))
1704                 static_branch_enable(&curve25519_use_bmi2_adx);
1705         else
1706                 return 0;
1707         return IS_REACHABLE(CONFIG_CRYPTO_KPP) ?
1708                 crypto_register_kpp(&curve25519_alg) : 0;
1709 }
1710
1711 static void __exit curve25519_mod_exit(void)
1712 {
1713         if (IS_REACHABLE(CONFIG_CRYPTO_KPP) &&
1714             static_branch_likely(&curve25519_use_bmi2_adx))
1715                 crypto_unregister_kpp(&curve25519_alg);
1716 }
1717
1718 module_init(curve25519_mod_init);
1719 module_exit(curve25519_mod_exit);
1720
1721 MODULE_ALIAS_CRYPTO("curve25519");
1722 MODULE_ALIAS_CRYPTO("curve25519-x86");
1723 MODULE_DESCRIPTION("Curve25519 algorithm, ADX optimized");
1724 MODULE_LICENSE("GPL v2");
1725 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");