arch/x86/crypto/sha512_ssse3_glue.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Glue code for the SHA512 Secure Hash Algorithm assembler
   5  * implementation using supplemental SSE3 / AVX / AVX2 instructions.
   6  *
   7  * This file is based on sha512_generic.c
   8  *
   9  * Copyright (C) 2013 Intel Corporation
  10  * Author: Tim Chen <tim.c.chen@linux.intel.com>
  11  *
  12  * This program is free software; you can redistribute it and/or modify it
  13  * under the terms of the GNU General Public License as published by the Free
  14  * Software Foundation; either version 2 of the License, or (at your option)
  15  * any later version.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  21  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  22  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  23  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24  * SOFTWARE.
  25  *
  26  */
  27
  28 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  29
  30 #include <crypto/internal/hash.h>
  31 #include <crypto/internal/simd.h>
  32 #include <linux/init.h>
  33 #include <linux/module.h>
  34 #include <linux/mm.h>
  35 #include <linux/string.h>
  36 #include <linux/types.h>
  37 #include <crypto/sha2.h>
  38 #include <crypto/sha512_base.h>
  39 #include <asm/simd.h>
  40
  41 asmlinkage void sha512_transform_ssse3(struct sha512_state *state,
  42                                        const u8 *data, int blocks);
  43
  44 static int sha512_update(struct shash_desc *desc, const u8 *data,
  45                        unsigned int len, sha512_block_fn *sha512_xform)
  46 {
  47         struct sha512_state *sctx = shash_desc_ctx(desc);
  48
  49         if (!crypto_simd_usable() ||
  50             (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
  51                 return crypto_sha512_update(desc, data, len);
  52
  53         /*
  54          * Make sure struct sha512_state begins directly with the SHA512
  55          * 512-bit internal state, as this is what the asm functions expect.
  56          */
  57         BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
  58
  59         kernel_fpu_begin();
  60         sha512_base_do_update(desc, data, len, sha512_xform);
  61         kernel_fpu_end();
  62
  63         return 0;
  64 }
  65
  66 static int sha512_finup(struct shash_desc *desc, const u8 *data,
  67               unsigned int len, u8 *out, sha512_block_fn *sha512_xform)
  68 {
  69         if (!crypto_simd_usable())
  70                 return crypto_sha512_finup(desc, data, len, out);
  71
  72         kernel_fpu_begin();
  73         if (len)
  74                 sha512_base_do_update(desc, data, len, sha512_xform);
  75         sha512_base_do_finalize(desc, sha512_xform);
  76         kernel_fpu_end();
  77
  78         return sha512_base_finish(desc, out);
  79 }
  80
  81 static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
  82                        unsigned int len)
  83 {
  84         return sha512_update(desc, data, len, sha512_transform_ssse3);
  85 }
  86
  87 static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
  88               unsigned int len, u8 *out)
  89 {
  90         return sha512_finup(desc, data, len, out, sha512_transform_ssse3);
  91 }
  92
  93 /* Add padding and return the message digest. */
  94 static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
  95 {
  96         return sha512_ssse3_finup(desc, NULL, 0, out);
  97 }
  98
  99 static struct shash_alg sha512_ssse3_algs[] = { {
 100         .digestsize     =       SHA512_DIGEST_SIZE,
 101         .init           =       sha512_base_init,
 102         .update         =       sha512_ssse3_update,
 103         .final          =       sha512_ssse3_final,
 104         .finup          =       sha512_ssse3_finup,
 105         .descsize       =       sizeof(struct sha512_state),
 106         .base           =       {
 107                 .cra_name       =       "sha512",
 108                 .cra_driver_name =      "sha512-ssse3",
 109                 .cra_priority   =       150,
 110                 .cra_blocksize  =       SHA512_BLOCK_SIZE,
 111                 .cra_module     =       THIS_MODULE,
 112         }
 113 },  {
 114         .digestsize     =       SHA384_DIGEST_SIZE,
 115         .init           =       sha384_base_init,
 116         .update         =       sha512_ssse3_update,
 117         .final          =       sha512_ssse3_final,
 118         .finup          =       sha512_ssse3_finup,
 119         .descsize       =       sizeof(struct sha512_state),
 120         .base           =       {
 121                 .cra_name       =       "sha384",
 122                 .cra_driver_name =      "sha384-ssse3",
 123                 .cra_priority   =       150,
 124                 .cra_blocksize  =       SHA384_BLOCK_SIZE,
 125                 .cra_module     =       THIS_MODULE,
 126         }
 127 } };
 128
 129 static int register_sha512_ssse3(void)
 130 {
 131         if (boot_cpu_has(X86_FEATURE_SSSE3))
 132                 return crypto_register_shashes(sha512_ssse3_algs,
 133                         ARRAY_SIZE(sha512_ssse3_algs));
 134         return 0;
 135 }
 136
 137 static void unregister_sha512_ssse3(void)
 138 {
 139         if (boot_cpu_has(X86_FEATURE_SSSE3))
 140                 crypto_unregister_shashes(sha512_ssse3_algs,
 141                         ARRAY_SIZE(sha512_ssse3_algs));
 142 }
 143
 144 asmlinkage void sha512_transform_avx(struct sha512_state *state,
 145                                      const u8 *data, int blocks);
 146 static bool avx_usable(void)
 147 {
 148         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
 149                 if (boot_cpu_has(X86_FEATURE_AVX))
 150                         pr_info("AVX detected but unusable.\n");
 151                 return false;
 152         }
 153
 154         return true;
 155 }
 156
 157 static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
 158                        unsigned int len)
 159 {
 160         return sha512_update(desc, data, len, sha512_transform_avx);
 161 }
 162
 163 static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
 164               unsigned int len, u8 *out)
 165 {
 166         return sha512_finup(desc, data, len, out, sha512_transform_avx);
 167 }
 168
 169 /* Add padding and return the message digest. */
 170 static int sha512_avx_final(struct shash_desc *desc, u8 *out)
 171 {
 172         return sha512_avx_finup(desc, NULL, 0, out);
 173 }
 174
 175 static struct shash_alg sha512_avx_algs[] = { {
 176         .digestsize     =       SHA512_DIGEST_SIZE,
 177         .init           =       sha512_base_init,
 178         .update         =       sha512_avx_update,
 179         .final          =       sha512_avx_final,
 180         .finup          =       sha512_avx_finup,
 181         .descsize       =       sizeof(struct sha512_state),
 182         .base           =       {
 183                 .cra_name       =       "sha512",
 184                 .cra_driver_name =      "sha512-avx",
 185                 .cra_priority   =       160,
 186                 .cra_blocksize  =       SHA512_BLOCK_SIZE,
 187                 .cra_module     =       THIS_MODULE,
 188         }
 189 },  {
 190         .digestsize     =       SHA384_DIGEST_SIZE,
 191         .init           =       sha384_base_init,
 192         .update         =       sha512_avx_update,
 193         .final          =       sha512_avx_final,
 194         .finup          =       sha512_avx_finup,
 195         .descsize       =       sizeof(struct sha512_state),
 196         .base           =       {
 197                 .cra_name       =       "sha384",
 198                 .cra_driver_name =      "sha384-avx",
 199                 .cra_priority   =       160,
 200                 .cra_blocksize  =       SHA384_BLOCK_SIZE,
 201                 .cra_module     =       THIS_MODULE,
 202         }
 203 } };
 204
 205 static int register_sha512_avx(void)
 206 {
 207         if (avx_usable())
 208                 return crypto_register_shashes(sha512_avx_algs,
 209                         ARRAY_SIZE(sha512_avx_algs));
 210         return 0;
 211 }
 212
 213 static void unregister_sha512_avx(void)
 214 {
 215         if (avx_usable())
 216                 crypto_unregister_shashes(sha512_avx_algs,
 217                         ARRAY_SIZE(sha512_avx_algs));
 218 }
 219
 220 asmlinkage void sha512_transform_rorx(struct sha512_state *state,
 221                                       const u8 *data, int blocks);
 222
 223 static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
 224                        unsigned int len)
 225 {
 226         return sha512_update(desc, data, len, sha512_transform_rorx);
 227 }
 228
 229 static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
 230               unsigned int len, u8 *out)
 231 {
 232         return sha512_finup(desc, data, len, out, sha512_transform_rorx);
 233 }
 234
 235 /* Add padding and return the message digest. */
 236 static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
 237 {
 238         return sha512_avx2_finup(desc, NULL, 0, out);
 239 }
 240
 241 static struct shash_alg sha512_avx2_algs[] = { {
 242         .digestsize     =       SHA512_DIGEST_SIZE,
 243         .init           =       sha512_base_init,
 244         .update         =       sha512_avx2_update,
 245         .final          =       sha512_avx2_final,
 246         .finup          =       sha512_avx2_finup,
 247         .descsize       =       sizeof(struct sha512_state),
 248         .base           =       {
 249                 .cra_name       =       "sha512",
 250                 .cra_driver_name =      "sha512-avx2",
 251                 .cra_priority   =       170,
 252                 .cra_blocksize  =       SHA512_BLOCK_SIZE,
 253                 .cra_module     =       THIS_MODULE,
 254         }
 255 },  {
 256         .digestsize     =       SHA384_DIGEST_SIZE,
 257         .init           =       sha384_base_init,
 258         .update         =       sha512_avx2_update,
 259         .final          =       sha512_avx2_final,
 260         .finup          =       sha512_avx2_finup,
 261         .descsize       =       sizeof(struct sha512_state),
 262         .base           =       {
 263                 .cra_name       =       "sha384",
 264                 .cra_driver_name =      "sha384-avx2",
 265                 .cra_priority   =       170,
 266                 .cra_blocksize  =       SHA384_BLOCK_SIZE,
 267                 .cra_module     =       THIS_MODULE,
 268         }
 269 } };
 270
 271 static bool avx2_usable(void)
 272 {
 273         if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
 274                     boot_cpu_has(X86_FEATURE_BMI2))
 275                 return true;
 276
 277         return false;
 278 }
 279
 280 static int register_sha512_avx2(void)
 281 {
 282         if (avx2_usable())
 283                 return crypto_register_shashes(sha512_avx2_algs,
 284                         ARRAY_SIZE(sha512_avx2_algs));
 285         return 0;
 286 }
 287
 288 static void unregister_sha512_avx2(void)
 289 {
 290         if (avx2_usable())
 291                 crypto_unregister_shashes(sha512_avx2_algs,
 292                         ARRAY_SIZE(sha512_avx2_algs));
 293 }
 294
 295 static int __init sha512_ssse3_mod_init(void)
 296 {
 297
 298         if (register_sha512_ssse3())
 299                 goto fail;
 300
 301         if (register_sha512_avx()) {
 302                 unregister_sha512_ssse3();
 303                 goto fail;
 304         }
 305
 306         if (register_sha512_avx2()) {
 307                 unregister_sha512_avx();
 308                 unregister_sha512_ssse3();
 309                 goto fail;
 310         }
 311
 312         return 0;
 313 fail:
 314         return -ENODEV;
 315 }
 316
 317 static void __exit sha512_ssse3_mod_fini(void)
 318 {
 319         unregister_sha512_avx2();
 320         unregister_sha512_avx();
 321         unregister_sha512_ssse3();
 322 }
 323
 324 module_init(sha512_ssse3_mod_init);
 325 module_exit(sha512_ssse3_mod_fini);
 326
 327 MODULE_LICENSE("GPL");
 328 MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 329
 330 MODULE_ALIAS_CRYPTO("sha512");
 331 MODULE_ALIAS_CRYPTO("sha512-ssse3");
 332 MODULE_ALIAS_CRYPTO("sha512-avx");
 333 MODULE_ALIAS_CRYPTO("sha512-avx2");
 334 MODULE_ALIAS_CRYPTO("sha384");
 335 MODULE_ALIAS_CRYPTO("sha384-ssse3");
 336 MODULE_ALIAS_CRYPTO("sha384-avx");
 337 MODULE_ALIAS_CRYPTO("sha384-avx2");