arch/x86/crypto/sha256_ssse3_glue.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Glue code for the SHA256 Secure Hash Algorithm assembler implementations
   5  * using SSSE3, AVX, AVX2, and SHA-NI instructions.
   6  *
   7  * This file is based on sha256_generic.c
   8  *
   9  * Copyright (C) 2013 Intel Corporation.
  10  *
  11  * Author:
  12  *     Tim Chen <tim.c.chen@linux.intel.com>
  13  *
  14  * This program is free software; you can redistribute it and/or modify it
  15  * under the terms of the GNU General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option)
  17  * any later version.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  22  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  23  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  24  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  25  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  26  * SOFTWARE.
  27  */
  28
  29
  30 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  31
  32 #include <crypto/internal/hash.h>
  33 #include <crypto/internal/simd.h>
  34 #include <linux/init.h>
  35 #include <linux/module.h>
  36 #include <linux/mm.h>
  37 #include <linux/types.h>
  38 #include <crypto/sha2.h>
  39 #include <crypto/sha256_base.h>
  40 #include <linux/string.h>
  41 #include <asm/cpu_device_id.h>
  42 #include <asm/simd.h>
  43
  44 asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
  45                                        const u8 *data, int blocks);
  46
  47 static const struct x86_cpu_id module_cpu_ids[] = {
  48 #ifdef CONFIG_AS_SHA256_NI
  49         X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
  50 #endif
  51         X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
  52         X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
  53         X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
  54         {}
  55 };
  56 MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
  57
  58 static int _sha256_update(struct shash_desc *desc, const u8 *data,
  59                           unsigned int len, sha256_block_fn *sha256_xform)
  60 {
  61         struct sha256_state *sctx = shash_desc_ctx(desc);
  62
  63         if (!crypto_simd_usable() ||
  64             (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
  65                 return crypto_sha256_update(desc, data, len);
  66
  67         /*
  68          * Make sure struct sha256_state begins directly with the SHA256
  69          * 256-bit internal state, as this is what the asm functions expect.
  70          */
  71         BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
  72
  73         kernel_fpu_begin();
  74         sha256_base_do_update(desc, data, len, sha256_xform);
  75         kernel_fpu_end();
  76
  77         return 0;
  78 }
  79
  80 static int sha256_finup(struct shash_desc *desc, const u8 *data,
  81               unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
  82 {
  83         if (!crypto_simd_usable())
  84                 return crypto_sha256_finup(desc, data, len, out);
  85
  86         kernel_fpu_begin();
  87         if (len)
  88                 sha256_base_do_update(desc, data, len, sha256_xform);
  89         sha256_base_do_finalize(desc, sha256_xform);
  90         kernel_fpu_end();
  91
  92         return sha256_base_finish(desc, out);
  93 }
  94
  95 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
  96                          unsigned int len)
  97 {
  98         return _sha256_update(desc, data, len, sha256_transform_ssse3);
  99 }
 100
 101 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
 102               unsigned int len, u8 *out)
 103 {
 104         return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
 105 }
 106
 107 /* Add padding and return the message digest. */
 108 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
 109 {
 110         return sha256_ssse3_finup(desc, NULL, 0, out);
 111 }
 112
 113 static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data,
 114               unsigned int len, u8 *out)
 115 {
 116         return sha256_base_init(desc) ?:
 117                sha256_ssse3_finup(desc, data, len, out);
 118 }
 119
 120 static struct shash_alg sha256_ssse3_algs[] = { {
 121         .digestsize     =       SHA256_DIGEST_SIZE,
 122         .init           =       sha256_base_init,
 123         .update         =       sha256_ssse3_update,
 124         .final          =       sha256_ssse3_final,
 125         .finup          =       sha256_ssse3_finup,
 126         .digest         =       sha256_ssse3_digest,
 127         .descsize       =       sizeof(struct sha256_state),
 128         .base           =       {
 129                 .cra_name       =       "sha256",
 130                 .cra_driver_name =      "sha256-ssse3",
 131                 .cra_priority   =       150,
 132                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 133                 .cra_module     =       THIS_MODULE,
 134         }
 135 }, {
 136         .digestsize     =       SHA224_DIGEST_SIZE,
 137         .init           =       sha224_base_init,
 138         .update         =       sha256_ssse3_update,
 139         .final          =       sha256_ssse3_final,
 140         .finup          =       sha256_ssse3_finup,
 141         .descsize       =       sizeof(struct sha256_state),
 142         .base           =       {
 143                 .cra_name       =       "sha224",
 144                 .cra_driver_name =      "sha224-ssse3",
 145                 .cra_priority   =       150,
 146                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 147                 .cra_module     =       THIS_MODULE,
 148         }
 149 } };
 150
 151 static int register_sha256_ssse3(void)
 152 {
 153         if (boot_cpu_has(X86_FEATURE_SSSE3))
 154                 return crypto_register_shashes(sha256_ssse3_algs,
 155                                 ARRAY_SIZE(sha256_ssse3_algs));
 156         return 0;
 157 }
 158
 159 static void unregister_sha256_ssse3(void)
 160 {
 161         if (boot_cpu_has(X86_FEATURE_SSSE3))
 162                 crypto_unregister_shashes(sha256_ssse3_algs,
 163                                 ARRAY_SIZE(sha256_ssse3_algs));
 164 }
 165
 166 asmlinkage void sha256_transform_avx(struct sha256_state *state,
 167                                      const u8 *data, int blocks);
 168
 169 static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
 170                          unsigned int len)
 171 {
 172         return _sha256_update(desc, data, len, sha256_transform_avx);
 173 }
 174
 175 static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
 176                       unsigned int len, u8 *out)
 177 {
 178         return sha256_finup(desc, data, len, out, sha256_transform_avx);
 179 }
 180
 181 static int sha256_avx_final(struct shash_desc *desc, u8 *out)
 182 {
 183         return sha256_avx_finup(desc, NULL, 0, out);
 184 }
 185
 186 static int sha256_avx_digest(struct shash_desc *desc, const u8 *data,
 187                       unsigned int len, u8 *out)
 188 {
 189         return sha256_base_init(desc) ?:
 190                sha256_avx_finup(desc, data, len, out);
 191 }
 192
 193 static struct shash_alg sha256_avx_algs[] = { {
 194         .digestsize     =       SHA256_DIGEST_SIZE,
 195         .init           =       sha256_base_init,
 196         .update         =       sha256_avx_update,
 197         .final          =       sha256_avx_final,
 198         .finup          =       sha256_avx_finup,
 199         .digest         =       sha256_avx_digest,
 200         .descsize       =       sizeof(struct sha256_state),
 201         .base           =       {
 202                 .cra_name       =       "sha256",
 203                 .cra_driver_name =      "sha256-avx",
 204                 .cra_priority   =       160,
 205                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 206                 .cra_module     =       THIS_MODULE,
 207         }
 208 }, {
 209         .digestsize     =       SHA224_DIGEST_SIZE,
 210         .init           =       sha224_base_init,
 211         .update         =       sha256_avx_update,
 212         .final          =       sha256_avx_final,
 213         .finup          =       sha256_avx_finup,
 214         .descsize       =       sizeof(struct sha256_state),
 215         .base           =       {
 216                 .cra_name       =       "sha224",
 217                 .cra_driver_name =      "sha224-avx",
 218                 .cra_priority   =       160,
 219                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 220                 .cra_module     =       THIS_MODULE,
 221         }
 222 } };
 223
 224 static bool avx_usable(void)
 225 {
 226         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
 227                 if (boot_cpu_has(X86_FEATURE_AVX))
 228                         pr_info("AVX detected but unusable.\n");
 229                 return false;
 230         }
 231
 232         return true;
 233 }
 234
 235 static int register_sha256_avx(void)
 236 {
 237         if (avx_usable())
 238                 return crypto_register_shashes(sha256_avx_algs,
 239                                 ARRAY_SIZE(sha256_avx_algs));
 240         return 0;
 241 }
 242
 243 static void unregister_sha256_avx(void)
 244 {
 245         if (avx_usable())
 246                 crypto_unregister_shashes(sha256_avx_algs,
 247                                 ARRAY_SIZE(sha256_avx_algs));
 248 }
 249
 250 asmlinkage void sha256_transform_rorx(struct sha256_state *state,
 251                                       const u8 *data, int blocks);
 252
 253 static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
 254                          unsigned int len)
 255 {
 256         return _sha256_update(desc, data, len, sha256_transform_rorx);
 257 }
 258
 259 static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
 260                       unsigned int len, u8 *out)
 261 {
 262         return sha256_finup(desc, data, len, out, sha256_transform_rorx);
 263 }
 264
 265 static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
 266 {
 267         return sha256_avx2_finup(desc, NULL, 0, out);
 268 }
 269
 270 static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data,
 271                       unsigned int len, u8 *out)
 272 {
 273         return sha256_base_init(desc) ?:
 274                sha256_avx2_finup(desc, data, len, out);
 275 }
 276
 277 static struct shash_alg sha256_avx2_algs[] = { {
 278         .digestsize     =       SHA256_DIGEST_SIZE,
 279         .init           =       sha256_base_init,
 280         .update         =       sha256_avx2_update,
 281         .final          =       sha256_avx2_final,
 282         .finup          =       sha256_avx2_finup,
 283         .digest         =       sha256_avx2_digest,
 284         .descsize       =       sizeof(struct sha256_state),
 285         .base           =       {
 286                 .cra_name       =       "sha256",
 287                 .cra_driver_name =      "sha256-avx2",
 288                 .cra_priority   =       170,
 289                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 290                 .cra_module     =       THIS_MODULE,
 291         }
 292 }, {
 293         .digestsize     =       SHA224_DIGEST_SIZE,
 294         .init           =       sha224_base_init,
 295         .update         =       sha256_avx2_update,
 296         .final          =       sha256_avx2_final,
 297         .finup          =       sha256_avx2_finup,
 298         .descsize       =       sizeof(struct sha256_state),
 299         .base           =       {
 300                 .cra_name       =       "sha224",
 301                 .cra_driver_name =      "sha224-avx2",
 302                 .cra_priority   =       170,
 303                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 304                 .cra_module     =       THIS_MODULE,
 305         }
 306 } };
 307
 308 static bool avx2_usable(void)
 309 {
 310         if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
 311                     boot_cpu_has(X86_FEATURE_BMI2))
 312                 return true;
 313
 314         return false;
 315 }
 316
 317 static int register_sha256_avx2(void)
 318 {
 319         if (avx2_usable())
 320                 return crypto_register_shashes(sha256_avx2_algs,
 321                                 ARRAY_SIZE(sha256_avx2_algs));
 322         return 0;
 323 }
 324
 325 static void unregister_sha256_avx2(void)
 326 {
 327         if (avx2_usable())
 328                 crypto_unregister_shashes(sha256_avx2_algs,
 329                                 ARRAY_SIZE(sha256_avx2_algs));
 330 }
 331
 332 #ifdef CONFIG_AS_SHA256_NI
 333 asmlinkage void sha256_ni_transform(struct sha256_state *digest,
 334                                     const u8 *data, int rounds);
 335
 336 static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
 337                          unsigned int len)
 338 {
 339         return _sha256_update(desc, data, len, sha256_ni_transform);
 340 }
 341
 342 static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
 343                       unsigned int len, u8 *out)
 344 {
 345         return sha256_finup(desc, data, len, out, sha256_ni_transform);
 346 }
 347
 348 static int sha256_ni_final(struct shash_desc *desc, u8 *out)
 349 {
 350         return sha256_ni_finup(desc, NULL, 0, out);
 351 }
 352
 353 static int sha256_ni_digest(struct shash_desc *desc, const u8 *data,
 354                       unsigned int len, u8 *out)
 355 {
 356         return sha256_base_init(desc) ?:
 357                sha256_ni_finup(desc, data, len, out);
 358 }
 359
 360 static struct shash_alg sha256_ni_algs[] = { {
 361         .digestsize     =       SHA256_DIGEST_SIZE,
 362         .init           =       sha256_base_init,
 363         .update         =       sha256_ni_update,
 364         .final          =       sha256_ni_final,
 365         .finup          =       sha256_ni_finup,
 366         .digest         =       sha256_ni_digest,
 367         .descsize       =       sizeof(struct sha256_state),
 368         .base           =       {
 369                 .cra_name       =       "sha256",
 370                 .cra_driver_name =      "sha256-ni",
 371                 .cra_priority   =       250,
 372                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 373                 .cra_module     =       THIS_MODULE,
 374         }
 375 }, {
 376         .digestsize     =       SHA224_DIGEST_SIZE,
 377         .init           =       sha224_base_init,
 378         .update         =       sha256_ni_update,
 379         .final          =       sha256_ni_final,
 380         .finup          =       sha256_ni_finup,
 381         .descsize       =       sizeof(struct sha256_state),
 382         .base           =       {
 383                 .cra_name       =       "sha224",
 384                 .cra_driver_name =      "sha224-ni",
 385                 .cra_priority   =       250,
 386                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 387                 .cra_module     =       THIS_MODULE,
 388         }
 389 } };
 390
 391 static int register_sha256_ni(void)
 392 {
 393         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 394                 return crypto_register_shashes(sha256_ni_algs,
 395                                 ARRAY_SIZE(sha256_ni_algs));
 396         return 0;
 397 }
 398
 399 static void unregister_sha256_ni(void)
 400 {
 401         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 402                 crypto_unregister_shashes(sha256_ni_algs,
 403                                 ARRAY_SIZE(sha256_ni_algs));
 404 }
 405
 406 #else
 407 static inline int register_sha256_ni(void) { return 0; }
 408 static inline void unregister_sha256_ni(void) { }
 409 #endif
 410
 411 static int __init sha256_ssse3_mod_init(void)
 412 {
 413         if (!x86_match_cpu(module_cpu_ids))
 414                 return -ENODEV;
 415
 416         if (register_sha256_ssse3())
 417                 goto fail;
 418
 419         if (register_sha256_avx()) {
 420                 unregister_sha256_ssse3();
 421                 goto fail;
 422         }
 423
 424         if (register_sha256_avx2()) {
 425                 unregister_sha256_avx();
 426                 unregister_sha256_ssse3();
 427                 goto fail;
 428         }
 429
 430         if (register_sha256_ni()) {
 431                 unregister_sha256_avx2();
 432                 unregister_sha256_avx();
 433                 unregister_sha256_ssse3();
 434                 goto fail;
 435         }
 436
 437         return 0;
 438 fail:
 439         return -ENODEV;
 440 }
 441
 442 static void __exit sha256_ssse3_mod_fini(void)
 443 {
 444         unregister_sha256_ni();
 445         unregister_sha256_avx2();
 446         unregister_sha256_avx();
 447         unregister_sha256_ssse3();
 448 }
 449
 450 module_init(sha256_ssse3_mod_init);
 451 module_exit(sha256_ssse3_mod_fini);
 452
 453 MODULE_LICENSE("GPL");
 454 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 455
 456 MODULE_ALIAS_CRYPTO("sha256");
 457 MODULE_ALIAS_CRYPTO("sha256-ssse3");
 458 MODULE_ALIAS_CRYPTO("sha256-avx");
 459 MODULE_ALIAS_CRYPTO("sha256-avx2");
 460 MODULE_ALIAS_CRYPTO("sha224");
 461 MODULE_ALIAS_CRYPTO("sha224-ssse3");
 462 MODULE_ALIAS_CRYPTO("sha224-avx");
 463 MODULE_ALIAS_CRYPTO("sha224-avx2");
 464 #ifdef CONFIG_AS_SHA256_NI
 465 MODULE_ALIAS_CRYPTO("sha256-ni");
 466 MODULE_ALIAS_CRYPTO("sha224-ni");
 467 #endif