target/arm/tcg/crypto_helper.c

   1 /*
   2  * crypto_helper.c - emulate v8 Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  */
  11
  12 #include "qemu/osdep.h"
  13
  14 #include "cpu.h"
  15 #include "exec/helper-proto.h"
  16 #include "tcg/tcg-gvec-desc.h"
  17 #include "crypto/aes-round.h"
  18 #include "crypto/sm4.h"
  19 #include "vec_internal.h"
  20
  21 union CRYPTO_STATE {
  22     uint8_t    bytes[16];
  23     uint32_t   words[4];
  24     uint64_t   l[2];
  25 };
  26
  27 #if HOST_BIG_ENDIAN
  28 #define CR_ST_BYTE(state, i)   ((state).bytes[(15 - (i)) ^ 8])
  29 #define CR_ST_WORD(state, i)   ((state).words[(3 - (i)) ^ 2])
  30 #else
  31 #define CR_ST_BYTE(state, i)   ((state).bytes[i])
  32 #define CR_ST_WORD(state, i)   ((state).words[i])
  33 #endif
  34
  35 /*
  36  * The caller has not been converted to full gvec, and so only
  37  * modifies the low 16 bytes of the vector register.
  38  */
  39 static void clear_tail_16(void *vd, uint32_t desc)
  40 {
  41     int opr_sz = simd_oprsz(desc);
  42     int max_sz = simd_maxsz(desc);
  43
  44     assert(opr_sz == 16);
  45     clear_tail(vd, opr_sz, max_sz);
  46 }
  47
  48 static const AESState aes_zero = { };
  49
  50 void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
  51 {
  52     intptr_t i, opr_sz = simd_oprsz(desc);
  53
  54     for (i = 0; i < opr_sz; i += 16) {
  55         AESState *ad = (AESState *)(vd + i);
  56         AESState *st = (AESState *)(vn + i);
  57         AESState *rk = (AESState *)(vm + i);
  58         AESState t;
  59
  60         /*
  61          * Our uint64_t are in the wrong order for big-endian.
  62          * The Arm AddRoundKey comes first, while the API AddRoundKey
  63          * comes last: perform the xor here, and provide zero to API.
  64          */
  65         if (HOST_BIG_ENDIAN) {
  66             t.d[0] = st->d[1] ^ rk->d[1];
  67             t.d[1] = st->d[0] ^ rk->d[0];
  68             aesenc_SB_SR_AK(&t, &t, &aes_zero, false);
  69             ad->d[0] = t.d[1];
  70             ad->d[1] = t.d[0];
  71         } else {
  72             t.v = st->v ^ rk->v;
  73             aesenc_SB_SR_AK(ad, &t, &aes_zero, false);
  74         }
  75     }
  76     clear_tail(vd, opr_sz, simd_maxsz(desc));
  77 }
  78
  79 void HELPER(crypto_aesd)(void *vd, void *vn, void *vm, uint32_t desc)
  80 {
  81     intptr_t i, opr_sz = simd_oprsz(desc);
  82
  83     for (i = 0; i < opr_sz; i += 16) {
  84         AESState *ad = (AESState *)(vd + i);
  85         AESState *st = (AESState *)(vn + i);
  86         AESState *rk = (AESState *)(vm + i);
  87         AESState t;
  88
  89         /* Our uint64_t are in the wrong order for big-endian. */
  90         if (HOST_BIG_ENDIAN) {
  91             t.d[0] = st->d[1] ^ rk->d[1];
  92             t.d[1] = st->d[0] ^ rk->d[0];
  93             aesdec_ISB_ISR_AK(&t, &t, &aes_zero, false);
  94             ad->d[0] = t.d[1];
  95             ad->d[1] = t.d[0];
  96         } else {
  97             t.v = st->v ^ rk->v;
  98             aesdec_ISB_ISR_AK(ad, &t, &aes_zero, false);
  99         }
 100     }
 101     clear_tail(vd, opr_sz, simd_maxsz(desc));
 102 }
 103
 104 void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
 105 {
 106     intptr_t i, opr_sz = simd_oprsz(desc);
 107
 108     for (i = 0; i < opr_sz; i += 16) {
 109         AESState *ad = (AESState *)(vd + i);
 110         AESState *st = (AESState *)(vm + i);
 111         AESState t;
 112
 113         /* Our uint64_t are in the wrong order for big-endian. */
 114         if (HOST_BIG_ENDIAN) {
 115             t.d[0] = st->d[1];
 116             t.d[1] = st->d[0];
 117             aesenc_MC(&t, &t, false);
 118             ad->d[0] = t.d[1];
 119             ad->d[1] = t.d[0];
 120         } else {
 121             aesenc_MC(ad, st, false);
 122         }
 123     }
 124     clear_tail(vd, opr_sz, simd_maxsz(desc));
 125 }
 126
 127 void HELPER(crypto_aesimc)(void *vd, void *vm, uint32_t desc)
 128 {
 129     intptr_t i, opr_sz = simd_oprsz(desc);
 130
 131     for (i = 0; i < opr_sz; i += 16) {
 132         AESState *ad = (AESState *)(vd + i);
 133         AESState *st = (AESState *)(vm + i);
 134         AESState t;
 135
 136         /* Our uint64_t are in the wrong order for big-endian. */
 137         if (HOST_BIG_ENDIAN) {
 138             t.d[0] = st->d[1];
 139             t.d[1] = st->d[0];
 140             aesdec_IMC(&t, &t, false);
 141             ad->d[0] = t.d[1];
 142             ad->d[1] = t.d[0];
 143         } else {
 144             aesdec_IMC(ad, st, false);
 145         }
 146     }
 147     clear_tail(vd, opr_sz, simd_maxsz(desc));
 148 }
 149
 150 /*
 151  * SHA-1 logical functions
 152  */
 153
 154 static uint32_t cho(uint32_t x, uint32_t y, uint32_t z)
 155 {
 156     return (x & (y ^ z)) ^ z;
 157 }
 158
 159 static uint32_t par(uint32_t x, uint32_t y, uint32_t z)
 160 {
 161     return x ^ y ^ z;
 162 }
 163
 164 static uint32_t maj(uint32_t x, uint32_t y, uint32_t z)
 165 {
 166     return (x & y) | ((x | y) & z);
 167 }
 168
 169 void HELPER(crypto_sha1su0)(void *vd, void *vn, void *vm, uint32_t desc)
 170 {
 171     uint64_t *d = vd, *n = vn, *m = vm;
 172     uint64_t d0, d1;
 173
 174     d0 = d[1] ^ d[0] ^ m[0];
 175     d1 = n[0] ^ d[1] ^ m[1];
 176     d[0] = d0;
 177     d[1] = d1;
 178
 179     clear_tail_16(vd, desc);
 180 }
 181
 182 static inline void crypto_sha1_3reg(uint64_t *rd, uint64_t *rn,
 183                                     uint64_t *rm, uint32_t desc,
 184                                     uint32_t (*fn)(union CRYPTO_STATE *d))
 185 {
 186     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 187     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 188     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 189     int i;
 190
 191     for (i = 0; i < 4; i++) {
 192         uint32_t t = fn(&d);
 193
 194         t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0)
 195              + CR_ST_WORD(m, i);
 196
 197         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3);
 198         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 199         CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2);
 200         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 201         CR_ST_WORD(d, 0) = t;
 202     }
 203     rd[0] = d.l[0];
 204     rd[1] = d.l[1];
 205
 206     clear_tail_16(rd, desc);
 207 }
 208
 209 static uint32_t do_sha1c(union CRYPTO_STATE *d)
 210 {
 211     return cho(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
 212 }
 213
 214 void HELPER(crypto_sha1c)(void *vd, void *vn, void *vm, uint32_t desc)
 215 {
 216     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1c);
 217 }
 218
 219 static uint32_t do_sha1p(union CRYPTO_STATE *d)
 220 {
 221     return par(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
 222 }
 223
 224 void HELPER(crypto_sha1p)(void *vd, void *vn, void *vm, uint32_t desc)
 225 {
 226     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1p);
 227 }
 228
 229 static uint32_t do_sha1m(union CRYPTO_STATE *d)
 230 {
 231     return maj(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
 232 }
 233
 234 void HELPER(crypto_sha1m)(void *vd, void *vn, void *vm, uint32_t desc)
 235 {
 236     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1m);
 237 }
 238
 239 void HELPER(crypto_sha1h)(void *vd, void *vm, uint32_t desc)
 240 {
 241     uint64_t *rd = vd;
 242     uint64_t *rm = vm;
 243     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 244
 245     CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2);
 246     CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0;
 247
 248     rd[0] = m.l[0];
 249     rd[1] = m.l[1];
 250
 251     clear_tail_16(vd, desc);
 252 }
 253
 254 void HELPER(crypto_sha1su1)(void *vd, void *vm, uint32_t desc)
 255 {
 256     uint64_t *rd = vd;
 257     uint64_t *rm = vm;
 258     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 259     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 260
 261     CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1);
 262     CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1);
 263     CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1);
 264     CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1);
 265
 266     rd[0] = d.l[0];
 267     rd[1] = d.l[1];
 268
 269     clear_tail_16(vd, desc);
 270 }
 271
 272 /*
 273  * The SHA-256 logical functions, according to
 274  * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
 275  */
 276
 277 static uint32_t S0(uint32_t x)
 278 {
 279     return ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22);
 280 }
 281
 282 static uint32_t S1(uint32_t x)
 283 {
 284     return ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25);
 285 }
 286
 287 static uint32_t s0(uint32_t x)
 288 {
 289     return ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3);
 290 }
 291
 292 static uint32_t s1(uint32_t x)
 293 {
 294     return ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10);
 295 }
 296
 297 void HELPER(crypto_sha256h)(void *vd, void *vn, void *vm, uint32_t desc)
 298 {
 299     uint64_t *rd = vd;
 300     uint64_t *rn = vn;
 301     uint64_t *rm = vm;
 302     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 303     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 304     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 305     int i;
 306
 307     for (i = 0; i < 4; i++) {
 308         uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2))
 309                      + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0))
 310                      + CR_ST_WORD(m, i);
 311
 312         CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2);
 313         CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1);
 314         CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0);
 315         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t;
 316
 317         t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
 318              + S0(CR_ST_WORD(d, 0));
 319
 320         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 321         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
 322         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 323         CR_ST_WORD(d, 0) = t;
 324     }
 325
 326     rd[0] = d.l[0];
 327     rd[1] = d.l[1];
 328
 329     clear_tail_16(vd, desc);
 330 }
 331
 332 void HELPER(crypto_sha256h2)(void *vd, void *vn, void *vm, uint32_t desc)
 333 {
 334     uint64_t *rd = vd;
 335     uint64_t *rn = vn;
 336     uint64_t *rm = vm;
 337     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 338     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 339     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 340     int i;
 341
 342     for (i = 0; i < 4; i++) {
 343         uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
 344                      + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0))
 345                      + CR_ST_WORD(m, i);
 346
 347         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 348         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
 349         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 350         CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t;
 351     }
 352
 353     rd[0] = d.l[0];
 354     rd[1] = d.l[1];
 355
 356     clear_tail_16(vd, desc);
 357 }
 358
 359 void HELPER(crypto_sha256su0)(void *vd, void *vm, uint32_t desc)
 360 {
 361     uint64_t *rd = vd;
 362     uint64_t *rm = vm;
 363     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 364     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 365
 366     CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1));
 367     CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2));
 368     CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3));
 369     CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0));
 370
 371     rd[0] = d.l[0];
 372     rd[1] = d.l[1];
 373
 374     clear_tail_16(vd, desc);
 375 }
 376
 377 void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm, uint32_t desc)
 378 {
 379     uint64_t *rd = vd;
 380     uint64_t *rn = vn;
 381     uint64_t *rm = vm;
 382     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 383     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 384     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 385
 386     CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1);
 387     CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2);
 388     CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3);
 389     CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0);
 390
 391     rd[0] = d.l[0];
 392     rd[1] = d.l[1];
 393
 394     clear_tail_16(vd, desc);
 395 }
 396
 397 /*
 398  * The SHA-512 logical functions (same as above but using 64-bit operands)
 399  */
 400
 401 static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z)
 402 {
 403     return (x & (y ^ z)) ^ z;
 404 }
 405
 406 static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z)
 407 {
 408     return (x & y) | ((x | y) & z);
 409 }
 410
 411 static uint64_t S0_512(uint64_t x)
 412 {
 413     return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39);
 414 }
 415
 416 static uint64_t S1_512(uint64_t x)
 417 {
 418     return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41);
 419 }
 420
 421 static uint64_t s0_512(uint64_t x)
 422 {
 423     return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7);
 424 }
 425
 426 static uint64_t s1_512(uint64_t x)
 427 {
 428     return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6);
 429 }
 430
 431 void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm, uint32_t desc)
 432 {
 433     uint64_t *rd = vd;
 434     uint64_t *rn = vn;
 435     uint64_t *rm = vm;
 436     uint64_t d0 = rd[0];
 437     uint64_t d1 = rd[1];
 438
 439     d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]);
 440     d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]);
 441
 442     rd[0] = d0;
 443     rd[1] = d1;
 444
 445     clear_tail_16(vd, desc);
 446 }
 447
 448 void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm, uint32_t desc)
 449 {
 450     uint64_t *rd = vd;
 451     uint64_t *rn = vn;
 452     uint64_t *rm = vm;
 453     uint64_t d0 = rd[0];
 454     uint64_t d1 = rd[1];
 455
 456     d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]);
 457     d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]);
 458
 459     rd[0] = d0;
 460     rd[1] = d1;
 461
 462     clear_tail_16(vd, desc);
 463 }
 464
 465 void HELPER(crypto_sha512su0)(void *vd, void *vn, uint32_t desc)
 466 {
 467     uint64_t *rd = vd;
 468     uint64_t *rn = vn;
 469     uint64_t d0 = rd[0];
 470     uint64_t d1 = rd[1];
 471
 472     d0 += s0_512(rd[1]);
 473     d1 += s0_512(rn[0]);
 474
 475     rd[0] = d0;
 476     rd[1] = d1;
 477
 478     clear_tail_16(vd, desc);
 479 }
 480
 481 void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm, uint32_t desc)
 482 {
 483     uint64_t *rd = vd;
 484     uint64_t *rn = vn;
 485     uint64_t *rm = vm;
 486
 487     rd[0] += s1_512(rn[0]) + rm[0];
 488     rd[1] += s1_512(rn[1]) + rm[1];
 489
 490     clear_tail_16(vd, desc);
 491 }
 492
 493 void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm, uint32_t desc)
 494 {
 495     uint64_t *rd = vd;
 496     uint64_t *rn = vn;
 497     uint64_t *rm = vm;
 498     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 499     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 500     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 501     uint32_t t;
 502
 503     t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17);
 504     CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9);
 505
 506     t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17);
 507     CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9);
 508
 509     t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17);
 510     CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9);
 511
 512     t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17);
 513     CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9);
 514
 515     rd[0] = d.l[0];
 516     rd[1] = d.l[1];
 517
 518     clear_tail_16(vd, desc);
 519 }
 520
 521 void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm, uint32_t desc)
 522 {
 523     uint64_t *rd = vd;
 524     uint64_t *rn = vn;
 525     uint64_t *rm = vm;
 526     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 527     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 528     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 529     uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25);
 530
 531     CR_ST_WORD(d, 0) ^= t;
 532     CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25);
 533     CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25);
 534     CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^
 535                         ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26);
 536
 537     rd[0] = d.l[0];
 538     rd[1] = d.l[1];
 539
 540     clear_tail_16(vd, desc);
 541 }
 542
 543 static inline void QEMU_ALWAYS_INLINE
 544 crypto_sm3tt(uint64_t *rd, uint64_t *rn, uint64_t *rm,
 545              uint32_t desc, uint32_t opcode)
 546 {
 547     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 548     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 549     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 550     uint32_t imm2 = simd_data(desc);
 551     uint32_t t;
 552
 553     assert(imm2 < 4);
 554
 555     if (opcode == 0 || opcode == 2) {
 556         /* SM3TT1A, SM3TT2A */
 557         t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
 558     } else if (opcode == 1) {
 559         /* SM3TT1B */
 560         t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
 561     } else if (opcode == 3) {
 562         /* SM3TT2B */
 563         t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
 564     } else {
 565         qemu_build_not_reached();
 566     }
 567
 568     t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2);
 569
 570     CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1);
 571
 572     if (opcode < 2) {
 573         /* SM3TT1A, SM3TT1B */
 574         t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20);
 575
 576         CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23);
 577     } else {
 578         /* SM3TT2A, SM3TT2B */
 579         t += CR_ST_WORD(n, 3);
 580         t ^= rol32(t, 9) ^ rol32(t, 17);
 581
 582         CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13);
 583     }
 584
 585     CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3);
 586     CR_ST_WORD(d, 3) = t;
 587
 588     rd[0] = d.l[0];
 589     rd[1] = d.l[1];
 590
 591     clear_tail_16(rd, desc);
 592 }
 593
 594 #define DO_SM3TT(NAME, OPCODE) \
 595     void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
 596     { crypto_sm3tt(vd, vn, vm, desc, OPCODE); }
 597
 598 DO_SM3TT(crypto_sm3tt1a, 0)
 599 DO_SM3TT(crypto_sm3tt1b, 1)
 600 DO_SM3TT(crypto_sm3tt2a, 2)
 601 DO_SM3TT(crypto_sm3tt2b, 3)
 602
 603 #undef DO_SM3TT
 604
 605 static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
 606 {
 607     union CRYPTO_STATE d = { .l = { rn[0], rn[1] } };
 608     union CRYPTO_STATE n = { .l = { rm[0], rm[1] } };
 609     uint32_t t, i;
 610
 611     for (i = 0; i < 4; i++) {
 612         t = CR_ST_WORD(d, (i + 1) % 4) ^
 613             CR_ST_WORD(d, (i + 2) % 4) ^
 614             CR_ST_WORD(d, (i + 3) % 4) ^
 615             CR_ST_WORD(n, i);
 616
 617         t = sm4_subword(t);
 618
 619         CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
 620                             rol32(t, 24);
 621     }
 622
 623     rd[0] = d.l[0];
 624     rd[1] = d.l[1];
 625 }
 626
 627 void HELPER(crypto_sm4e)(void *vd, void *vn, void *vm, uint32_t desc)
 628 {
 629     intptr_t i, opr_sz = simd_oprsz(desc);
 630
 631     for (i = 0; i < opr_sz; i += 16) {
 632         do_crypto_sm4e(vd + i, vn + i, vm + i);
 633     }
 634     clear_tail(vd, opr_sz, simd_maxsz(desc));
 635 }
 636
 637 static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
 638 {
 639     union CRYPTO_STATE d;
 640     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 641     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 642     uint32_t t, i;
 643
 644     d = n;
 645     for (i = 0; i < 4; i++) {
 646         t = CR_ST_WORD(d, (i + 1) % 4) ^
 647             CR_ST_WORD(d, (i + 2) % 4) ^
 648             CR_ST_WORD(d, (i + 3) % 4) ^
 649             CR_ST_WORD(m, i);
 650
 651         t = sm4_subword(t);
 652
 653         CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
 654     }
 655
 656     rd[0] = d.l[0];
 657     rd[1] = d.l[1];
 658 }
 659
 660 void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm, uint32_t desc)
 661 {
 662     intptr_t i, opr_sz = simd_oprsz(desc);
 663
 664     for (i = 0; i < opr_sz; i += 16) {
 665         do_crypto_sm4ekey(vd + i, vn + i, vm + i);
 666     }
 667     clear_tail(vd, opr_sz, simd_maxsz(desc));
 668 }
 669
 670 void HELPER(crypto_rax1)(void *vd, void *vn, void *vm, uint32_t desc)
 671 {
 672     intptr_t i, opr_sz = simd_oprsz(desc);
 673     uint64_t *d = vd, *n = vn, *m = vm;
 674
 675     for (i = 0; i < opr_sz / 8; ++i) {
 676         d[i] = n[i] ^ rol64(m[i], 1);
 677     }
 678     clear_tail(vd, opr_sz, simd_maxsz(desc));
 679 }