target-arm/crypto_helper.c

   1 /*
   2  * crypto_helper.c - emulate v8 Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #include "qemu/osdep.h"
  13
  14 #include "cpu.h"
  15 #include "exec/exec-all.h"
  16 #include "exec/helper-proto.h"
  17 #include "crypto/aes.h"
  18
  19 union CRYPTO_STATE {
  20     uint8_t    bytes[16];
  21     uint32_t   words[4];
  22     uint64_t   l[2];
  23 };
  24
  25 #ifdef HOST_WORDS_BIGENDIAN
  26 #define CR_ST_BYTE(state, i)   (state.bytes[(15 - (i)) ^ 8])
  27 #define CR_ST_WORD(state, i)   (state.words[(3 - (i)) ^ 2])
  28 #else
  29 #define CR_ST_BYTE(state, i)   (state.bytes[i])
  30 #define CR_ST_WORD(state, i)   (state.words[i])
  31 #endif
  32
  33 void HELPER(crypto_aese)(CPUARMState *env, uint32_t rd, uint32_t rm,
  34                          uint32_t decrypt)
  35 {
  36     static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox };
  37     static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts };
  38
  39     union CRYPTO_STATE rk = { .l = {
  40         float64_val(env->vfp.regs[rm]),
  41         float64_val(env->vfp.regs[rm + 1])
  42     } };
  43     union CRYPTO_STATE st = { .l = {
  44         float64_val(env->vfp.regs[rd]),
  45         float64_val(env->vfp.regs[rd + 1])
  46     } };
  47     int i;
  48
  49     assert(decrypt < 2);
  50
  51     /* xor state vector with round key */
  52     rk.l[0] ^= st.l[0];
  53     rk.l[1] ^= st.l[1];
  54
  55     /* combine ShiftRows operation and sbox substitution */
  56     for (i = 0; i < 16; i++) {
  57         CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])];
  58     }
  59
  60     env->vfp.regs[rd] = make_float64(st.l[0]);
  61     env->vfp.regs[rd + 1] = make_float64(st.l[1]);
  62 }
  63
  64 void HELPER(crypto_aesmc)(CPUARMState *env, uint32_t rd, uint32_t rm,
  65                           uint32_t decrypt)
  66 {
  67     static uint32_t const mc[][256] = { {
  68         /* MixColumns lookup table */
  69         0x00000000, 0x03010102, 0x06020204, 0x05030306,
  70         0x0c040408, 0x0f05050a, 0x0a06060c, 0x0907070e,
  71         0x18080810, 0x1b090912, 0x1e0a0a14, 0x1d0b0b16,
  72         0x140c0c18, 0x170d0d1a, 0x120e0e1c, 0x110f0f1e,
  73         0x30101020, 0x33111122, 0x36121224, 0x35131326,
  74         0x3c141428, 0x3f15152a, 0x3a16162c, 0x3917172e,
  75         0x28181830, 0x2b191932, 0x2e1a1a34, 0x2d1b1b36,
  76         0x241c1c38, 0x271d1d3a, 0x221e1e3c, 0x211f1f3e,
  77         0x60202040, 0x63212142, 0x66222244, 0x65232346,
  78         0x6c242448, 0x6f25254a, 0x6a26264c, 0x6927274e,
  79         0x78282850, 0x7b292952, 0x7e2a2a54, 0x7d2b2b56,
  80         0x742c2c58, 0x772d2d5a, 0x722e2e5c, 0x712f2f5e,
  81         0x50303060, 0x53313162, 0x56323264, 0x55333366,
  82         0x5c343468, 0x5f35356a, 0x5a36366c, 0x5937376e,
  83         0x48383870, 0x4b393972, 0x4e3a3a74, 0x4d3b3b76,
  84         0x443c3c78, 0x473d3d7a, 0x423e3e7c, 0x413f3f7e,
  85         0xc0404080, 0xc3414182, 0xc6424284, 0xc5434386,
  86         0xcc444488, 0xcf45458a, 0xca46468c, 0xc947478e,
  87         0xd8484890, 0xdb494992, 0xde4a4a94, 0xdd4b4b96,
  88         0xd44c4c98, 0xd74d4d9a, 0xd24e4e9c, 0xd14f4f9e,
  89         0xf05050a0, 0xf35151a2, 0xf65252a4, 0xf55353a6,
  90         0xfc5454a8, 0xff5555aa, 0xfa5656ac, 0xf95757ae,
  91         0xe85858b0, 0xeb5959b2, 0xee5a5ab4, 0xed5b5bb6,
  92         0xe45c5cb8, 0xe75d5dba, 0xe25e5ebc, 0xe15f5fbe,
  93         0xa06060c0, 0xa36161c2, 0xa66262c4, 0xa56363c6,
  94         0xac6464c8, 0xaf6565ca, 0xaa6666cc, 0xa96767ce,
  95         0xb86868d0, 0xbb6969d2, 0xbe6a6ad4, 0xbd6b6bd6,
  96         0xb46c6cd8, 0xb76d6dda, 0xb26e6edc, 0xb16f6fde,
  97         0x907070e0, 0x937171e2, 0x967272e4, 0x957373e6,
  98         0x9c7474e8, 0x9f7575ea, 0x9a7676ec, 0x997777ee,
  99         0x887878f0, 0x8b7979f2, 0x8e7a7af4, 0x8d7b7bf6,
 100         0x847c7cf8, 0x877d7dfa, 0x827e7efc, 0x817f7ffe,
 101         0x9b80801b, 0x98818119, 0x9d82821f, 0x9e83831d,
 102         0x97848413, 0x94858511, 0x91868617, 0x92878715,
 103         0x8388880b, 0x80898909, 0x858a8a0f, 0x868b8b0d,
 104         0x8f8c8c03, 0x8c8d8d01, 0x898e8e07, 0x8a8f8f05,
 105         0xab90903b, 0xa8919139, 0xad92923f, 0xae93933d,
 106         0xa7949433, 0xa4959531, 0xa1969637, 0xa2979735,
 107         0xb398982b, 0xb0999929, 0xb59a9a2f, 0xb69b9b2d,
 108         0xbf9c9c23, 0xbc9d9d21, 0xb99e9e27, 0xba9f9f25,
 109         0xfba0a05b, 0xf8a1a159, 0xfda2a25f, 0xfea3a35d,
 110         0xf7a4a453, 0xf4a5a551, 0xf1a6a657, 0xf2a7a755,
 111         0xe3a8a84b, 0xe0a9a949, 0xe5aaaa4f, 0xe6abab4d,
 112         0xefacac43, 0xecadad41, 0xe9aeae47, 0xeaafaf45,
 113         0xcbb0b07b, 0xc8b1b179, 0xcdb2b27f, 0xceb3b37d,
 114         0xc7b4b473, 0xc4b5b571, 0xc1b6b677, 0xc2b7b775,
 115         0xd3b8b86b, 0xd0b9b969, 0xd5baba6f, 0xd6bbbb6d,
 116         0xdfbcbc63, 0xdcbdbd61, 0xd9bebe67, 0xdabfbf65,
 117         0x5bc0c09b, 0x58c1c199, 0x5dc2c29f, 0x5ec3c39d,
 118         0x57c4c493, 0x54c5c591, 0x51c6c697, 0x52c7c795,
 119         0x43c8c88b, 0x40c9c989, 0x45caca8f, 0x46cbcb8d,
 120         0x4fcccc83, 0x4ccdcd81, 0x49cece87, 0x4acfcf85,
 121         0x6bd0d0bb, 0x68d1d1b9, 0x6dd2d2bf, 0x6ed3d3bd,
 122         0x67d4d4b3, 0x64d5d5b1, 0x61d6d6b7, 0x62d7d7b5,
 123         0x73d8d8ab, 0x70d9d9a9, 0x75dadaaf, 0x76dbdbad,
 124         0x7fdcdca3, 0x7cdddda1, 0x79dedea7, 0x7adfdfa5,
 125         0x3be0e0db, 0x38e1e1d9, 0x3de2e2df, 0x3ee3e3dd,
 126         0x37e4e4d3, 0x34e5e5d1, 0x31e6e6d7, 0x32e7e7d5,
 127         0x23e8e8cb, 0x20e9e9c9, 0x25eaeacf, 0x26ebebcd,
 128         0x2fececc3, 0x2cededc1, 0x29eeeec7, 0x2aefefc5,
 129         0x0bf0f0fb, 0x08f1f1f9, 0x0df2f2ff, 0x0ef3f3fd,
 130         0x07f4f4f3, 0x04f5f5f1, 0x01f6f6f7, 0x02f7f7f5,
 131         0x13f8f8eb, 0x10f9f9e9, 0x15fafaef, 0x16fbfbed,
 132         0x1ffcfce3, 0x1cfdfde1, 0x19fefee7, 0x1affffe5,
 133     }, {
 134         /* Inverse MixColumns lookup table */
 135         0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12,
 136         0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a,
 137         0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362,
 138         0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a,
 139         0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2,
 140         0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca,
 141         0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382,
 142         0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba,
 143         0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9,
 144         0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1,
 145         0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9,
 146         0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81,
 147         0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029,
 148         0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411,
 149         0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859,
 150         0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61,
 151         0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf,
 152         0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987,
 153         0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf,
 154         0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7,
 155         0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f,
 156         0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967,
 157         0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f,
 158         0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117,
 159         0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664,
 160         0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c,
 161         0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14,
 162         0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c,
 163         0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684,
 164         0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc,
 165         0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4,
 166         0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc,
 167         0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753,
 168         0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b,
 169         0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23,
 170         0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b,
 171         0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3,
 172         0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b,
 173         0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3,
 174         0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb,
 175         0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88,
 176         0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0,
 177         0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8,
 178         0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0,
 179         0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68,
 180         0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850,
 181         0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418,
 182         0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020,
 183         0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe,
 184         0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6,
 185         0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e,
 186         0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6,
 187         0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e,
 188         0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526,
 189         0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e,
 190         0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56,
 191         0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25,
 192         0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d,
 193         0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255,
 194         0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d,
 195         0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5,
 196         0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd,
 197         0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5,
 198         0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d,
 199     } };
 200     union CRYPTO_STATE st = { .l = {
 201         float64_val(env->vfp.regs[rm]),
 202         float64_val(env->vfp.regs[rm + 1])
 203     } };
 204     int i;
 205
 206     assert(decrypt < 2);
 207
 208     for (i = 0; i < 16; i += 4) {
 209         CR_ST_WORD(st, i >> 2) =
 210             mc[decrypt][CR_ST_BYTE(st, i)] ^
 211             rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^
 212             rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^
 213             rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24);
 214     }
 215
 216     env->vfp.regs[rd] = make_float64(st.l[0]);
 217     env->vfp.regs[rd + 1] = make_float64(st.l[1]);
 218 }
 219
 220 /*
 221  * SHA-1 logical functions
 222  */
 223
 224 static uint32_t cho(uint32_t x, uint32_t y, uint32_t z)
 225 {
 226     return (x & (y ^ z)) ^ z;
 227 }
 228
 229 static uint32_t par(uint32_t x, uint32_t y, uint32_t z)
 230 {
 231     return x ^ y ^ z;
 232 }
 233
 234 static uint32_t maj(uint32_t x, uint32_t y, uint32_t z)
 235 {
 236     return (x & y) | ((x | y) & z);
 237 }
 238
 239 void HELPER(crypto_sha1_3reg)(CPUARMState *env, uint32_t rd, uint32_t rn,
 240                               uint32_t rm, uint32_t op)
 241 {
 242     union CRYPTO_STATE d = { .l = {
 243         float64_val(env->vfp.regs[rd]),
 244         float64_val(env->vfp.regs[rd + 1])
 245     } };
 246     union CRYPTO_STATE n = { .l = {
 247         float64_val(env->vfp.regs[rn]),
 248         float64_val(env->vfp.regs[rn + 1])
 249     } };
 250     union CRYPTO_STATE m = { .l = {
 251         float64_val(env->vfp.regs[rm]),
 252         float64_val(env->vfp.regs[rm + 1])
 253     } };
 254
 255     if (op == 3) { /* sha1su0 */
 256         d.l[0] ^= d.l[1] ^ m.l[0];
 257         d.l[1] ^= n.l[0] ^ m.l[1];
 258     } else {
 259         int i;
 260
 261         for (i = 0; i < 4; i++) {
 262             uint32_t t;
 263
 264             switch (op) {
 265             case 0: /* sha1c */
 266                 t = cho(CR_ST_WORD(d, 1), CR_ST_WORD(d, 2), CR_ST_WORD(d, 3));
 267                 break;
 268             case 1: /* sha1p */
 269                 t = par(CR_ST_WORD(d, 1), CR_ST_WORD(d, 2), CR_ST_WORD(d, 3));
 270                 break;
 271             case 2: /* sha1m */
 272                 t = maj(CR_ST_WORD(d, 1), CR_ST_WORD(d, 2), CR_ST_WORD(d, 3));
 273                 break;
 274             default:
 275                 g_assert_not_reached();
 276             }
 277             t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0)
 278                  + CR_ST_WORD(m, i);
 279
 280             CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3);
 281             CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 282             CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2);
 283             CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 284             CR_ST_WORD(d, 0) = t;
 285         }
 286     }
 287     env->vfp.regs[rd] = make_float64(d.l[0]);
 288     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
 289 }
 290
 291 void HELPER(crypto_sha1h)(CPUARMState *env, uint32_t rd, uint32_t rm)
 292 {
 293     union CRYPTO_STATE m = { .l = {
 294         float64_val(env->vfp.regs[rm]),
 295         float64_val(env->vfp.regs[rm + 1])
 296     } };
 297
 298     CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2);
 299     CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0;
 300
 301     env->vfp.regs[rd] = make_float64(m.l[0]);
 302     env->vfp.regs[rd + 1] = make_float64(m.l[1]);
 303 }
 304
 305 void HELPER(crypto_sha1su1)(CPUARMState *env, uint32_t rd, uint32_t rm)
 306 {
 307     union CRYPTO_STATE d = { .l = {
 308         float64_val(env->vfp.regs[rd]),
 309         float64_val(env->vfp.regs[rd + 1])
 310     } };
 311     union CRYPTO_STATE m = { .l = {
 312         float64_val(env->vfp.regs[rm]),
 313         float64_val(env->vfp.regs[rm + 1])
 314     } };
 315
 316     CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1);
 317     CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1);
 318     CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1);
 319     CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1);
 320
 321     env->vfp.regs[rd] = make_float64(d.l[0]);
 322     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
 323 }
 324
 325 /*
 326  * The SHA-256 logical functions, according to
 327  * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
 328  */
 329
 330 static uint32_t S0(uint32_t x)
 331 {
 332     return ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22);
 333 }
 334
 335 static uint32_t S1(uint32_t x)
 336 {
 337     return ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25);
 338 }
 339
 340 static uint32_t s0(uint32_t x)
 341 {
 342     return ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3);
 343 }
 344
 345 static uint32_t s1(uint32_t x)
 346 {
 347     return ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10);
 348 }
 349
 350 void HELPER(crypto_sha256h)(CPUARMState *env, uint32_t rd, uint32_t rn,
 351                             uint32_t rm)
 352 {
 353     union CRYPTO_STATE d = { .l = {
 354         float64_val(env->vfp.regs[rd]),
 355         float64_val(env->vfp.regs[rd + 1])
 356     } };
 357     union CRYPTO_STATE n = { .l = {
 358         float64_val(env->vfp.regs[rn]),
 359         float64_val(env->vfp.regs[rn + 1])
 360     } };
 361     union CRYPTO_STATE m = { .l = {
 362         float64_val(env->vfp.regs[rm]),
 363         float64_val(env->vfp.regs[rm + 1])
 364     } };
 365     int i;
 366
 367     for (i = 0; i < 4; i++) {
 368         uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2))
 369                      + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0))
 370                      + CR_ST_WORD(m, i);
 371
 372         CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2);
 373         CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1);
 374         CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0);
 375         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t;
 376
 377         t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
 378              + S0(CR_ST_WORD(d, 0));
 379
 380         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 381         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
 382         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 383         CR_ST_WORD(d, 0) = t;
 384     }
 385
 386     env->vfp.regs[rd] = make_float64(d.l[0]);
 387     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
 388 }
 389
 390 void HELPER(crypto_sha256h2)(CPUARMState *env, uint32_t rd, uint32_t rn,
 391                              uint32_t rm)
 392 {
 393     union CRYPTO_STATE d = { .l = {
 394         float64_val(env->vfp.regs[rd]),
 395         float64_val(env->vfp.regs[rd + 1])
 396     } };
 397     union CRYPTO_STATE n = { .l = {
 398         float64_val(env->vfp.regs[rn]),
 399         float64_val(env->vfp.regs[rn + 1])
 400     } };
 401     union CRYPTO_STATE m = { .l = {
 402         float64_val(env->vfp.regs[rm]),
 403         float64_val(env->vfp.regs[rm + 1])
 404     } };
 405     int i;
 406
 407     for (i = 0; i < 4; i++) {
 408         uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
 409                      + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0))
 410                      + CR_ST_WORD(m, i);
 411
 412         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 413         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
 414         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 415         CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t;
 416     }
 417
 418     env->vfp.regs[rd] = make_float64(d.l[0]);
 419     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
 420 }
 421
 422 void HELPER(crypto_sha256su0)(CPUARMState *env, uint32_t rd, uint32_t rm)
 423 {
 424     union CRYPTO_STATE d = { .l = {
 425         float64_val(env->vfp.regs[rd]),
 426         float64_val(env->vfp.regs[rd + 1])
 427     } };
 428     union CRYPTO_STATE m = { .l = {
 429         float64_val(env->vfp.regs[rm]),
 430         float64_val(env->vfp.regs[rm + 1])
 431     } };
 432
 433     CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1));
 434     CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2));
 435     CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3));
 436     CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0));
 437
 438     env->vfp.regs[rd] = make_float64(d.l[0]);
 439     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
 440 }
 441
 442 void HELPER(crypto_sha256su1)(CPUARMState *env, uint32_t rd, uint32_t rn,
 443                               uint32_t rm)
 444 {
 445     union CRYPTO_STATE d = { .l = {
 446         float64_val(env->vfp.regs[rd]),
 447         float64_val(env->vfp.regs[rd + 1])
 448     } };
 449     union CRYPTO_STATE n = { .l = {
 450         float64_val(env->vfp.regs[rn]),
 451         float64_val(env->vfp.regs[rn + 1])
 452     } };
 453     union CRYPTO_STATE m = { .l = {
 454         float64_val(env->vfp.regs[rm]),
 455         float64_val(env->vfp.regs[rm + 1])
 456     } };
 457
 458     CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1);
 459     CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2);
 460     CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3);
 461     CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0);
 462
 463     env->vfp.regs[rd] = make_float64(d.l[0]);
 464     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
 465 }