arch/arm64/crypto/crc32-ce-glue.c

   1 /*
   2  * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/cpufeature.h>
  12 #include <linux/crc32.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/string.h>
  17
  18 #include <crypto/internal/hash.h>
  19
  20 #include <asm/hwcap.h>
  21 #include <asm/neon.h>
  22 #include <asm/simd.h>
  23 #include <asm/unaligned.h>
  24
  25 #define PMULL_MIN_LEN           64L     /* minimum size of buffer
  26                                          * for crc32_pmull_le_16 */
  27 #define SCALE_F                 16L     /* size of NEON register */
  28
  29 asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
  30 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);
  31
  32 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
  33 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);
  34
  35 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
  36 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);
  37
  38 static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
  39 {
  40         u32 *key = crypto_tfm_ctx(tfm);
  41
  42         *key = 0;
  43         return 0;
  44 }
  45
  46 static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
  47 {
  48         u32 *key = crypto_tfm_ctx(tfm);
  49
  50         *key = ~0;
  51         return 0;
  52 }
  53
  54 static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
  55                               unsigned int keylen)
  56 {
  57         u32 *mctx = crypto_shash_ctx(hash);
  58
  59         if (keylen != sizeof(u32)) {
  60                 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
  61                 return -EINVAL;
  62         }
  63         *mctx = le32_to_cpup((__le32 *)key);
  64         return 0;
  65 }
  66
  67 static int crc32_pmull_init(struct shash_desc *desc)
  68 {
  69         u32 *mctx = crypto_shash_ctx(desc->tfm);
  70         u32 *crc = shash_desc_ctx(desc);
  71
  72         *crc = *mctx;
  73         return 0;
  74 }
  75
  76 static int crc32_update(struct shash_desc *desc, const u8 *data,
  77                         unsigned int length)
  78 {
  79         u32 *crc = shash_desc_ctx(desc);
  80
  81         *crc = crc32_armv8_le(*crc, data, length);
  82         return 0;
  83 }
  84
  85 static int crc32c_update(struct shash_desc *desc, const u8 *data,
  86                          unsigned int length)
  87 {
  88         u32 *crc = shash_desc_ctx(desc);
  89
  90         *crc = crc32c_armv8_le(*crc, data, length);
  91         return 0;
  92 }
  93
  94 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
  95                          unsigned int length)
  96 {
  97         u32 *crc = shash_desc_ctx(desc);
  98         unsigned int l;
  99
 100         if ((u64)data % SCALE_F) {
 101                 l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
 102
 103                 *crc = fallback_crc32(*crc, data, l);
 104
 105                 data += l;
 106                 length -= l;
 107         }
 108
 109         if (length >= PMULL_MIN_LEN && may_use_simd()) {
 110                 l = round_down(length, SCALE_F);
 111
 112                 kernel_neon_begin();
 113                 *crc = crc32_pmull_le(data, l, *crc);
 114                 kernel_neon_end();
 115
 116                 data += l;
 117                 length -= l;
 118         }
 119
 120         if (length > 0)
 121                 *crc = fallback_crc32(*crc, data, length);
 122
 123         return 0;
 124 }
 125
 126 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
 127                          unsigned int length)
 128 {
 129         u32 *crc = shash_desc_ctx(desc);
 130         unsigned int l;
 131
 132         if ((u64)data % SCALE_F) {
 133                 l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
 134
 135                 *crc = fallback_crc32c(*crc, data, l);
 136
 137                 data += l;
 138                 length -= l;
 139         }
 140
 141         if (length >= PMULL_MIN_LEN && may_use_simd()) {
 142                 l = round_down(length, SCALE_F);
 143
 144                 kernel_neon_begin();
 145                 *crc = crc32c_pmull_le(data, l, *crc);
 146                 kernel_neon_end();
 147
 148                 data += l;
 149                 length -= l;
 150         }
 151
 152         if (length > 0) {
 153                 *crc = fallback_crc32c(*crc, data, length);
 154         }
 155
 156         return 0;
 157 }
 158
 159 static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
 160 {
 161         u32 *crc = shash_desc_ctx(desc);
 162
 163         put_unaligned_le32(*crc, out);
 164         return 0;
 165 }
 166
 167 static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
 168 {
 169         u32 *crc = shash_desc_ctx(desc);
 170
 171         put_unaligned_le32(~*crc, out);
 172         return 0;
 173 }
 174
 175 static struct shash_alg crc32_pmull_algs[] = { {
 176         .setkey                 = crc32_pmull_setkey,
 177         .init                   = crc32_pmull_init,
 178         .update                 = crc32_update,
 179         .final                  = crc32_pmull_final,
 180         .descsize               = sizeof(u32),
 181         .digestsize             = sizeof(u32),
 182
 183         .base.cra_ctxsize       = sizeof(u32),
 184         .base.cra_init          = crc32_pmull_cra_init,
 185         .base.cra_name          = "crc32",
 186         .base.cra_driver_name   = "crc32-arm64-ce",
 187         .base.cra_priority      = 200,
 188         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
 189         .base.cra_blocksize     = 1,
 190         .base.cra_module        = THIS_MODULE,
 191 }, {
 192         .setkey                 = crc32_pmull_setkey,
 193         .init                   = crc32_pmull_init,
 194         .update                 = crc32c_update,
 195         .final                  = crc32c_pmull_final,
 196         .descsize               = sizeof(u32),
 197         .digestsize             = sizeof(u32),
 198
 199         .base.cra_ctxsize       = sizeof(u32),
 200         .base.cra_init          = crc32c_pmull_cra_init,
 201         .base.cra_name          = "crc32c",
 202         .base.cra_driver_name   = "crc32c-arm64-ce",
 203         .base.cra_priority      = 200,
 204         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
 205         .base.cra_blocksize     = 1,
 206         .base.cra_module        = THIS_MODULE,
 207 } };
 208
 209 static int __init crc32_pmull_mod_init(void)
 210 {
 211         if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
 212                 crc32_pmull_algs[0].update = crc32_pmull_update;
 213                 crc32_pmull_algs[1].update = crc32c_pmull_update;
 214
 215                 if (elf_hwcap & HWCAP_CRC32) {
 216                         fallback_crc32 = crc32_armv8_le;
 217                         fallback_crc32c = crc32c_armv8_le;
 218                 } else {
 219                         fallback_crc32 = crc32_le;
 220                         fallback_crc32c = __crc32c_le;
 221                 }
 222         } else if (!(elf_hwcap & HWCAP_CRC32)) {
 223                 return -ENODEV;
 224         }
 225         return crypto_register_shashes(crc32_pmull_algs,
 226                                        ARRAY_SIZE(crc32_pmull_algs));
 227 }
 228
 229 static void __exit crc32_pmull_mod_exit(void)
 230 {
 231         crypto_unregister_shashes(crc32_pmull_algs,
 232                                   ARRAY_SIZE(crc32_pmull_algs));
 233 }
 234
 235 static const struct cpu_feature crc32_cpu_feature[] = {
 236         { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
 237 };
 238 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
 239
 240 module_init(crc32_pmull_mod_init);
 241 module_exit(crc32_pmull_mod_exit);
 242
 243 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 244 MODULE_LICENSE("GPL v2");