sys/crypto/aesni/intel_sha256.c

   1 /*******************************************************************************
   2 * Copyright (c) 2013, Intel Corporation
   3 *
   4 * All rights reserved.
   5 *
   6 * Redistribution and use in source and binary forms, with or without
   7 * modification, are permitted provided that the following conditions are
   8 * met:
   9 *
  10 * * Redistributions of source code must retain the above copyright
  11 *   notice, this list of conditions and the following disclaimer.
  12 *
  13 * * Redistributions in binary form must reproduce the above copyright
  14 *   notice, this list of conditions and the following disclaimer in the
  15 *   documentation and/or other materials provided with the
  16 *   distribution.
  17 *
  18 * * Neither the name of the Intel Corporation nor the names of its
  19 *   contributors may be used to endorse or promote products derived from
  20 *   this software without specific prior written permission.
  21 *
  22 *
  23 * THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
  24 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
  27 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  28 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  29 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34 ********************************************************************************
  35 *
  36 * Intel SHA Extensions optimized implementation of a SHA-256 update function
  37 *
  38 * The function takes a pointer to the current hash values, a pointer to the
  39 * input data, and a number of 64 byte blocks to process.  Once all blocks have
  40 * been processed, the digest pointer is  updated with the resulting hash value.
  41 * The function only processes complete blocks, there is no functionality to
  42 * store partial blocks.  All message padding and hash value initialization must
  43 * be done outside the update function.
  44 *
  45 * The indented lines in the loop are instructions related to rounds processing.
  46 * The non-indented lines are instructions related to the message schedule.
  47 *
  48 * Author: Sean Gulley <sean.m.gulley@intel.com>
  49 * Date:   July 2013
  50 *
  51 ********************************************************************************
  52 *
  53 * Example complier command line:
  54 * icc intel_sha_extensions_sha256_intrinsic.c
  55 * gcc -msha -msse4 intel_sha_extensions_sha256_intrinsic.c
  56 *
  57 *******************************************************************************/
  58
  59 #include <sys/types.h>
  60 #include <crypto/aesni/aesni_os.h>
  61 #include <crypto/aesni/sha_sse.h>
  62
  63 #include <immintrin.h>
  64
  65 void intel_sha256_step(uint32_t *digest, const char *data, uint32_t num_blks) {
  66    __m128i state0, state1;
  67    __m128i msg;
  68    __m128i msgtmp0, msgtmp1, msgtmp2, msgtmp3;
  69    __m128i tmp;
  70    __m128i shuf_mask;
  71    __m128i abef_save, cdgh_save;
  72
  73    // Load initial hash values
  74    // Need to reorder these appropriately
  75    // DCBA, HGFE -> ABEF, CDGH
  76    tmp    = _mm_loadu_si128((__m128i*) digest);
  77    state1 = _mm_loadu_si128((__m128i*) (digest+4));
  78
  79    tmp    = _mm_shuffle_epi32(tmp, 0xB1);       // CDAB
  80    state1 = _mm_shuffle_epi32(state1, 0x1B);    // EFGH
  81    state0 = _mm_alignr_epi8(tmp, state1, 8);    // ABEF
  82    state1 = _mm_blend_epi16(state1, tmp, 0xF0); // CDGH
  83
  84    shuf_mask = _mm_set_epi64x(0x0c0d0e0f08090a0bull, 0x0405060700010203ull);
  85
  86    while (num_blks > 0) {
  87       // Save hash values for addition after rounds
  88       abef_save = state0;
  89       cdgh_save = state1;
  90
  91       // Rounds 0-3
  92       msg     = _mm_loadu_si128((const __m128i*) data);
  93       msgtmp0 = _mm_shuffle_epi8(msg, shuf_mask);
  94          msg    = _mm_add_epi32(msgtmp0,
  95                   _mm_set_epi64x(0xE9B5DBA5B5C0FBCFull, 0x71374491428A2F98ull));
  96          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
  97          msg    = _mm_shuffle_epi32(msg, 0x0E);
  98          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
  99
 100       // Rounds 4-7
 101       msgtmp1 = _mm_loadu_si128((const __m128i*) (data+16));
 102       msgtmp1 = _mm_shuffle_epi8(msgtmp1, shuf_mask);
 103          msg    = _mm_add_epi32(msgtmp1,
 104                   _mm_set_epi64x(0xAB1C5ED5923F82A4ull, 0x59F111F13956C25Bull));
 105          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 106          msg    = _mm_shuffle_epi32(msg, 0x0E);
 107          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 108       msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
 109
 110       // Rounds 8-11
 111       msgtmp2 = _mm_loadu_si128((const __m128i*) (data+32));
 112       msgtmp2 = _mm_shuffle_epi8(msgtmp2, shuf_mask);
 113          msg    = _mm_add_epi32(msgtmp2,
 114                   _mm_set_epi64x(0x550C7DC3243185BEull, 0x12835B01D807AA98ull));
 115          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 116          msg    = _mm_shuffle_epi32(msg, 0x0E);
 117          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 118       msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
 119
 120       // Rounds 12-15
 121       msgtmp3 = _mm_loadu_si128((const __m128i*) (data+48));
 122       msgtmp3 = _mm_shuffle_epi8(msgtmp3, shuf_mask);
 123          msg    = _mm_add_epi32(msgtmp3,
 124                   _mm_set_epi64x(0xC19BF1749BDC06A7ull, 0x80DEB1FE72BE5D74ull));
 125          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 126       tmp     = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
 127       msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
 128       msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
 129          msg    = _mm_shuffle_epi32(msg, 0x0E);
 130          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 131       msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
 132
 133       // Rounds 16-19
 134          msg    = _mm_add_epi32(msgtmp0,
 135                   _mm_set_epi64x(0x240CA1CC0FC19DC6ull, 0xEFBE4786E49B69C1ull));
 136          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 137       tmp     = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
 138       msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
 139       msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
 140          msg    = _mm_shuffle_epi32(msg, 0x0E);
 141          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 142       msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
 143
 144       // Rounds 20-23
 145          msg    = _mm_add_epi32(msgtmp1,
 146                   _mm_set_epi64x(0x76F988DA5CB0A9DCull, 0x4A7484AA2DE92C6Full));
 147          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 148       tmp     = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
 149       msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
 150       msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
 151          msg    = _mm_shuffle_epi32(msg, 0x0E);
 152          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 153       msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
 154
 155       // Rounds 24-27
 156          msg    = _mm_add_epi32(msgtmp2,
 157                   _mm_set_epi64x(0xBF597FC7B00327C8ull, 0xA831C66D983E5152ull));
 158          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 159       tmp     = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
 160       msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
 161       msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
 162          msg    = _mm_shuffle_epi32(msg, 0x0E);
 163          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 164       msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
 165
 166       // Rounds 28-31
 167          msg    = _mm_add_epi32(msgtmp3,
 168                   _mm_set_epi64x(0x1429296706CA6351ull, 0xD5A79147C6E00BF3ull));
 169          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 170       tmp     = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
 171       msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
 172       msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
 173          msg    = _mm_shuffle_epi32(msg, 0x0E);
 174          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 175       msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
 176
 177       // Rounds 32-35
 178          msg    = _mm_add_epi32(msgtmp0,
 179                   _mm_set_epi64x(0x53380D134D2C6DFCull, 0x2E1B213827B70A85ull));
 180          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 181       tmp     = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
 182       msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
 183       msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
 184          msg    = _mm_shuffle_epi32(msg, 0x0E);
 185          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 186       msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
 187
 188       // Rounds 36-39
 189          msg    = _mm_add_epi32(msgtmp1,
 190                   _mm_set_epi64x(0x92722C8581C2C92Eull, 0x766A0ABB650A7354ull));
 191          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 192       tmp     = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
 193       msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
 194       msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
 195          msg    = _mm_shuffle_epi32(msg, 0x0E);
 196          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 197       msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
 198
 199       // Rounds 40-43
 200          msg    = _mm_add_epi32(msgtmp2,
 201                   _mm_set_epi64x(0xC76C51A3C24B8B70ull, 0xA81A664BA2BFE8A1ull));
 202          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 203       tmp     = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
 204       msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
 205       msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
 206          msg    = _mm_shuffle_epi32(msg, 0x0E);
 207          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 208       msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
 209
 210       // Rounds 44-47
 211          msg    = _mm_add_epi32(msgtmp3,
 212                   _mm_set_epi64x(0x106AA070F40E3585ull, 0xD6990624D192E819ull));
 213          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 214       tmp     = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
 215       msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
 216       msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
 217          msg    = _mm_shuffle_epi32(msg, 0x0E);
 218          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 219       msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
 220
 221       // Rounds 48-51
 222          msg    = _mm_add_epi32(msgtmp0,
 223                   _mm_set_epi64x(0x34B0BCB52748774Cull, 0x1E376C0819A4C116ull));
 224          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 225       tmp     = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
 226       msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
 227       msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
 228          msg    = _mm_shuffle_epi32(msg, 0x0E);
 229          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 230       msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
 231
 232       // Rounds 52-55
 233          msg    = _mm_add_epi32(msgtmp1,
 234                   _mm_set_epi64x(0x682E6FF35B9CCA4Full, 0x4ED8AA4A391C0CB3ull));
 235          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 236       tmp     = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
 237       msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
 238       msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
 239          msg    = _mm_shuffle_epi32(msg, 0x0E);
 240          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 241
 242       // Rounds 56-59
 243          msg    = _mm_add_epi32(msgtmp2,
 244                   _mm_set_epi64x(0x8CC7020884C87814ull, 0x78A5636F748F82EEull));
 245          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 246       tmp     = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
 247       msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
 248       msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
 249          msg    = _mm_shuffle_epi32(msg, 0x0E);
 250          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 251
 252       // Rounds 60-63
 253          msg    = _mm_add_epi32(msgtmp3,
 254                   _mm_set_epi64x(0xC67178F2BEF9A3F7ull, 0xA4506CEB90BEFFFAull));
 255          state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
 256          msg    = _mm_shuffle_epi32(msg, 0x0E);
 257          state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
 258
 259       // Add current hash values with previously saved
 260       state0 = _mm_add_epi32(state0, abef_save);
 261       state1 = _mm_add_epi32(state1, cdgh_save);
 262
 263       data += 64;
 264       num_blks--;
 265    }
 266
 267    // Write hash values back in the correct order
 268    tmp    = _mm_shuffle_epi32(state0, 0x1B);    // FEBA
 269    state1 = _mm_shuffle_epi32(state1, 0xB1);    // DCHG
 270    state0 = _mm_blend_epi16(tmp, state1, 0xF0); // DCBA
 271    state1 = _mm_alignr_epi8(state1, tmp, 8);    // ABEF
 272
 273    _mm_store_si128((__m128i*) digest, state0);
 274    _mm_store_si128((__m128i*) (digest+4), state1);
 275 }
 276