arch/arm64/crypto/sha256-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
   4  *
   5  * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
   6  */
   7
   8 #include <asm/hwcap.h>
   9 #include <asm/neon.h>
  10 #include <asm/simd.h>
  11 #include <crypto/internal/hash.h>
  12 #include <crypto/internal/simd.h>
  13 #include <crypto/sha.h>
  14 #include <crypto/sha256_base.h>
  15 #include <linux/cryptohash.h>
  16 #include <linux/types.h>
  17 #include <linux/string.h>
  18
  19 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
  20 MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
  21 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  22 MODULE_LICENSE("GPL v2");
  23 MODULE_ALIAS_CRYPTO("sha224");
  24 MODULE_ALIAS_CRYPTO("sha256");
  25
  26 asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
  27                                         unsigned int num_blks);
  28 EXPORT_SYMBOL(sha256_block_data_order);
  29
  30 static void __sha256_block_data_order(struct sha256_state *sst, u8 const *src,
  31                                       int blocks)
  32 {
  33         sha256_block_data_order(sst->state, src, blocks);
  34 }
  35
  36 asmlinkage void sha256_block_neon(u32 *digest, const void *data,
  37                                   unsigned int num_blks);
  38
  39 static void __sha256_block_neon(struct sha256_state *sst, u8 const *src,
  40                                 int blocks)
  41 {
  42         sha256_block_neon(sst->state, src, blocks);
  43 }
  44
  45 static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data,
  46                                       unsigned int len)
  47 {
  48         return sha256_base_do_update(desc, data, len,
  49                                      __sha256_block_data_order);
  50 }
  51
  52 static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data,
  53                                      unsigned int len, u8 *out)
  54 {
  55         if (len)
  56                 sha256_base_do_update(desc, data, len,
  57                                       __sha256_block_data_order);
  58         sha256_base_do_finalize(desc, __sha256_block_data_order);
  59
  60         return sha256_base_finish(desc, out);
  61 }
  62
  63 static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out)
  64 {
  65         return crypto_sha256_arm64_finup(desc, NULL, 0, out);
  66 }
  67
  68 static struct shash_alg algs[] = { {
  69         .digestsize             = SHA256_DIGEST_SIZE,
  70         .init                   = sha256_base_init,
  71         .update                 = crypto_sha256_arm64_update,
  72         .final                  = crypto_sha256_arm64_final,
  73         .finup                  = crypto_sha256_arm64_finup,
  74         .descsize               = sizeof(struct sha256_state),
  75         .base.cra_name          = "sha256",
  76         .base.cra_driver_name   = "sha256-arm64",
  77         .base.cra_priority      = 125,
  78         .base.cra_blocksize     = SHA256_BLOCK_SIZE,
  79         .base.cra_module        = THIS_MODULE,
  80 }, {
  81         .digestsize             = SHA224_DIGEST_SIZE,
  82         .init                   = sha224_base_init,
  83         .update                 = crypto_sha256_arm64_update,
  84         .final                  = crypto_sha256_arm64_final,
  85         .finup                  = crypto_sha256_arm64_finup,
  86         .descsize               = sizeof(struct sha256_state),
  87         .base.cra_name          = "sha224",
  88         .base.cra_driver_name   = "sha224-arm64",
  89         .base.cra_priority      = 125,
  90         .base.cra_blocksize     = SHA224_BLOCK_SIZE,
  91         .base.cra_module        = THIS_MODULE,
  92 } };
  93
  94 static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
  95                               unsigned int len)
  96 {
  97         struct sha256_state *sctx = shash_desc_ctx(desc);
  98
  99         if (!crypto_simd_usable())
 100                 return sha256_base_do_update(desc, data, len,
 101                                 __sha256_block_data_order);
 102
 103         while (len > 0) {
 104                 unsigned int chunk = len;
 105
 106                 /*
 107                  * Don't hog the CPU for the entire time it takes to process all
 108                  * input when running on a preemptible kernel, but process the
 109                  * data block by block instead.
 110                  */
 111                 if (IS_ENABLED(CONFIG_PREEMPTION) &&
 112                     chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE)
 113                         chunk = SHA256_BLOCK_SIZE -
 114                                 sctx->count % SHA256_BLOCK_SIZE;
 115
 116                 kernel_neon_begin();
 117                 sha256_base_do_update(desc, data, chunk, __sha256_block_neon);
 118                 kernel_neon_end();
 119                 data += chunk;
 120                 len -= chunk;
 121         }
 122         return 0;
 123 }
 124
 125 static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
 126                              unsigned int len, u8 *out)
 127 {
 128         if (!crypto_simd_usable()) {
 129                 if (len)
 130                         sha256_base_do_update(desc, data, len,
 131                                 __sha256_block_data_order);
 132                 sha256_base_do_finalize(desc, __sha256_block_data_order);
 133         } else {
 134                 if (len)
 135                         sha256_update_neon(desc, data, len);
 136                 kernel_neon_begin();
 137                 sha256_base_do_finalize(desc, __sha256_block_neon);
 138                 kernel_neon_end();
 139         }
 140         return sha256_base_finish(desc, out);
 141 }
 142
 143 static int sha256_final_neon(struct shash_desc *desc, u8 *out)
 144 {
 145         return sha256_finup_neon(desc, NULL, 0, out);
 146 }
 147
 148 static struct shash_alg neon_algs[] = { {
 149         .digestsize             = SHA256_DIGEST_SIZE,
 150         .init                   = sha256_base_init,
 151         .update                 = sha256_update_neon,
 152         .final                  = sha256_final_neon,
 153         .finup                  = sha256_finup_neon,
 154         .descsize               = sizeof(struct sha256_state),
 155         .base.cra_name          = "sha256",
 156         .base.cra_driver_name   = "sha256-arm64-neon",
 157         .base.cra_priority      = 150,
 158         .base.cra_blocksize     = SHA256_BLOCK_SIZE,
 159         .base.cra_module        = THIS_MODULE,
 160 }, {
 161         .digestsize             = SHA224_DIGEST_SIZE,
 162         .init                   = sha224_base_init,
 163         .update                 = sha256_update_neon,
 164         .final                  = sha256_final_neon,
 165         .finup                  = sha256_finup_neon,
 166         .descsize               = sizeof(struct sha256_state),
 167         .base.cra_name          = "sha224",
 168         .base.cra_driver_name   = "sha224-arm64-neon",
 169         .base.cra_priority      = 150,
 170         .base.cra_blocksize     = SHA224_BLOCK_SIZE,
 171         .base.cra_module        = THIS_MODULE,
 172 } };
 173
 174 static int __init sha256_mod_init(void)
 175 {
 176         int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
 177         if (ret)
 178                 return ret;
 179
 180         if (cpu_have_named_feature(ASIMD)) {
 181                 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
 182                 if (ret)
 183                         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 184         }
 185         return ret;
 186 }
 187
 188 static void __exit sha256_mod_fini(void)
 189 {
 190         if (cpu_have_named_feature(ASIMD))
 191                 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
 192         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 193 }
 194
 195 module_init(sha256_mod_init);
 196 module_exit(sha256_mod_fini);