arch/arm/crypto/speck-neon-glue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
   4  *
   5  * Copyright (c) 2018 Google, Inc
   6  *
   7  * Note: the NIST recommendation for XTS only specifies a 128-bit block size,
   8  * but a 64-bit version (needed for Speck64) is fairly straightforward; the math
   9  * is just done in GF(2^64) instead of GF(2^128), with the reducing polynomial
  10  * x^64 + x^4 + x^3 + x + 1 from the original XEX paper (Rogaway, 2004:
  11  * "Efficient Instantiations of Tweakable Blockciphers and Refinements to Modes
  12  * OCB and PMAC"), represented as 0x1B.
  13  */
  14
  15 #include <asm/hwcap.h>
  16 #include <asm/neon.h>
  17 #include <asm/simd.h>
  18 #include <crypto/algapi.h>
  19 #include <crypto/gf128mul.h>
  20 #include <crypto/internal/skcipher.h>
  21 #include <crypto/speck.h>
  22 #include <crypto/xts.h>
  23 #include <linux/kernel.h>
  24 #include <linux/module.h>
  25
  26 /* The assembly functions only handle multiples of 128 bytes */
  27 #define SPECK_NEON_CHUNK_SIZE   128
  28
  29 /* Speck128 */
  30
  31 struct speck128_xts_tfm_ctx {
  32         struct speck128_tfm_ctx main_key;
  33         struct speck128_tfm_ctx tweak_key;
  34 };
  35
  36 asmlinkage void speck128_xts_encrypt_neon(const u64 *round_keys, int nrounds,
  37                                           void *dst, const void *src,
  38                                           unsigned int nbytes, void *tweak);
  39
  40 asmlinkage void speck128_xts_decrypt_neon(const u64 *round_keys, int nrounds,
  41                                           void *dst, const void *src,
  42                                           unsigned int nbytes, void *tweak);
  43
  44 typedef void (*speck128_crypt_one_t)(const struct speck128_tfm_ctx *,
  45                                      u8 *, const u8 *);
  46 typedef void (*speck128_xts_crypt_many_t)(const u64 *, int, void *,
  47                                           const void *, unsigned int, void *);
  48
  49 static __always_inline int
  50 __speck128_xts_crypt(struct skcipher_request *req,
  51                      speck128_crypt_one_t crypt_one,
  52                      speck128_xts_crypt_many_t crypt_many)
  53 {
  54         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  55         const struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  56         struct skcipher_walk walk;
  57         le128 tweak;
  58         int err;
  59
  60         err = skcipher_walk_virt(&walk, req, true);
  61
  62         crypto_speck128_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
  63
  64         while (walk.nbytes > 0) {
  65                 unsigned int nbytes = walk.nbytes;
  66                 u8 *dst = walk.dst.virt.addr;
  67                 const u8 *src = walk.src.virt.addr;
  68
  69                 if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
  70                         unsigned int count;
  71
  72                         count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
  73                         kernel_neon_begin();
  74                         (*crypt_many)(ctx->main_key.round_keys,
  75                                       ctx->main_key.nrounds,
  76                                       dst, src, count, &tweak);
  77                         kernel_neon_end();
  78                         dst += count;
  79                         src += count;
  80                         nbytes -= count;
  81                 }
  82
  83                 /* Handle any remainder with generic code */
  84                 while (nbytes >= sizeof(tweak)) {
  85                         le128_xor((le128 *)dst, (const le128 *)src, &tweak);
  86                         (*crypt_one)(&ctx->main_key, dst, dst);
  87                         le128_xor((le128 *)dst, (const le128 *)dst, &tweak);
  88                         gf128mul_x_ble(&tweak, &tweak);
  89
  90                         dst += sizeof(tweak);
  91                         src += sizeof(tweak);
  92                         nbytes -= sizeof(tweak);
  93                 }
  94                 err = skcipher_walk_done(&walk, nbytes);
  95         }
  96
  97         return err;
  98 }
  99
 100 static int speck128_xts_encrypt(struct skcipher_request *req)
 101 {
 102         return __speck128_xts_crypt(req, crypto_speck128_encrypt,
 103                                     speck128_xts_encrypt_neon);
 104 }
 105
 106 static int speck128_xts_decrypt(struct skcipher_request *req)
 107 {
 108         return __speck128_xts_crypt(req, crypto_speck128_decrypt,
 109                                     speck128_xts_decrypt_neon);
 110 }
 111
 112 static int speck128_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
 113                                unsigned int keylen)
 114 {
 115         struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 116         int err;
 117
 118         err = xts_verify_key(tfm, key, keylen);
 119         if (err)
 120                 return err;
 121
 122         keylen /= 2;
 123
 124         err = crypto_speck128_setkey(&ctx->main_key, key, keylen);
 125         if (err)
 126                 return err;
 127
 128         return crypto_speck128_setkey(&ctx->tweak_key, key + keylen, keylen);
 129 }
 130
 131 /* Speck64 */
 132
 133 struct speck64_xts_tfm_ctx {
 134         struct speck64_tfm_ctx main_key;
 135         struct speck64_tfm_ctx tweak_key;
 136 };
 137
 138 asmlinkage void speck64_xts_encrypt_neon(const u32 *round_keys, int nrounds,
 139                                          void *dst, const void *src,
 140                                          unsigned int nbytes, void *tweak);
 141
 142 asmlinkage void speck64_xts_decrypt_neon(const u32 *round_keys, int nrounds,
 143                                          void *dst, const void *src,
 144                                          unsigned int nbytes, void *tweak);
 145
 146 typedef void (*speck64_crypt_one_t)(const struct speck64_tfm_ctx *,
 147                                     u8 *, const u8 *);
 148 typedef void (*speck64_xts_crypt_many_t)(const u32 *, int, void *,
 149                                          const void *, unsigned int, void *);
 150
 151 static __always_inline int
 152 __speck64_xts_crypt(struct skcipher_request *req, speck64_crypt_one_t crypt_one,
 153                     speck64_xts_crypt_many_t crypt_many)
 154 {
 155         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 156         const struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 157         struct skcipher_walk walk;
 158         __le64 tweak;
 159         int err;
 160
 161         err = skcipher_walk_virt(&walk, req, true);
 162
 163         crypto_speck64_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
 164
 165         while (walk.nbytes > 0) {
 166                 unsigned int nbytes = walk.nbytes;
 167                 u8 *dst = walk.dst.virt.addr;
 168                 const u8 *src = walk.src.virt.addr;
 169
 170                 if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
 171                         unsigned int count;
 172
 173                         count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
 174                         kernel_neon_begin();
 175                         (*crypt_many)(ctx->main_key.round_keys,
 176                                       ctx->main_key.nrounds,
 177                                       dst, src, count, &tweak);
 178                         kernel_neon_end();
 179                         dst += count;
 180                         src += count;
 181                         nbytes -= count;
 182                 }
 183
 184                 /* Handle any remainder with generic code */
 185                 while (nbytes >= sizeof(tweak)) {
 186                         *(__le64 *)dst = *(__le64 *)src ^ tweak;
 187                         (*crypt_one)(&ctx->main_key, dst, dst);
 188                         *(__le64 *)dst ^= tweak;
 189                         tweak = cpu_to_le64((le64_to_cpu(tweak) << 1) ^
 190                                             ((tweak & cpu_to_le64(1ULL << 63)) ?
 191                                              0x1B : 0));
 192                         dst += sizeof(tweak);
 193                         src += sizeof(tweak);
 194                         nbytes -= sizeof(tweak);
 195                 }
 196                 err = skcipher_walk_done(&walk, nbytes);
 197         }
 198
 199         return err;
 200 }
 201
 202 static int speck64_xts_encrypt(struct skcipher_request *req)
 203 {
 204         return __speck64_xts_crypt(req, crypto_speck64_encrypt,
 205                                    speck64_xts_encrypt_neon);
 206 }
 207
 208 static int speck64_xts_decrypt(struct skcipher_request *req)
 209 {
 210         return __speck64_xts_crypt(req, crypto_speck64_decrypt,
 211                                    speck64_xts_decrypt_neon);
 212 }
 213
 214 static int speck64_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
 215                               unsigned int keylen)
 216 {
 217         struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 218         int err;
 219
 220         err = xts_verify_key(tfm, key, keylen);
 221         if (err)
 222                 return err;
 223
 224         keylen /= 2;
 225
 226         err = crypto_speck64_setkey(&ctx->main_key, key, keylen);
 227         if (err)
 228                 return err;
 229
 230         return crypto_speck64_setkey(&ctx->tweak_key, key + keylen, keylen);
 231 }
 232
 233 static struct skcipher_alg speck_algs[] = {
 234         {
 235                 .base.cra_name          = "xts(speck128)",
 236                 .base.cra_driver_name   = "xts-speck128-neon",
 237                 .base.cra_priority      = 300,
 238                 .base.cra_blocksize     = SPECK128_BLOCK_SIZE,
 239                 .base.cra_ctxsize       = sizeof(struct speck128_xts_tfm_ctx),
 240                 .base.cra_alignmask     = 7,
 241                 .base.cra_module        = THIS_MODULE,
 242                 .min_keysize            = 2 * SPECK128_128_KEY_SIZE,
 243                 .max_keysize            = 2 * SPECK128_256_KEY_SIZE,
 244                 .ivsize                 = SPECK128_BLOCK_SIZE,
 245                 .walksize               = SPECK_NEON_CHUNK_SIZE,
 246                 .setkey                 = speck128_xts_setkey,
 247                 .encrypt                = speck128_xts_encrypt,
 248                 .decrypt                = speck128_xts_decrypt,
 249         }, {
 250                 .base.cra_name          = "xts(speck64)",
 251                 .base.cra_driver_name   = "xts-speck64-neon",
 252                 .base.cra_priority      = 300,
 253                 .base.cra_blocksize     = SPECK64_BLOCK_SIZE,
 254                 .base.cra_ctxsize       = sizeof(struct speck64_xts_tfm_ctx),
 255                 .base.cra_alignmask     = 7,
 256                 .base.cra_module        = THIS_MODULE,
 257                 .min_keysize            = 2 * SPECK64_96_KEY_SIZE,
 258                 .max_keysize            = 2 * SPECK64_128_KEY_SIZE,
 259                 .ivsize                 = SPECK64_BLOCK_SIZE,
 260                 .walksize               = SPECK_NEON_CHUNK_SIZE,
 261                 .setkey                 = speck64_xts_setkey,
 262                 .encrypt                = speck64_xts_encrypt,
 263                 .decrypt                = speck64_xts_decrypt,
 264         }
 265 };
 266
 267 static int __init speck_neon_module_init(void)
 268 {
 269         if (!(elf_hwcap & HWCAP_NEON))
 270                 return -ENODEV;
 271         return crypto_register_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
 272 }
 273
 274 static void __exit speck_neon_module_exit(void)
 275 {
 276         crypto_unregister_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
 277 }
 278
 279 module_init(speck_neon_module_init);
 280 module_exit(speck_neon_module_exit);
 281
 282 MODULE_DESCRIPTION("Speck block cipher (NEON-accelerated)");
 283 MODULE_LICENSE("GPL");
 284 MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
 285 MODULE_ALIAS_CRYPTO("xts(speck128)");
 286 MODULE_ALIAS_CRYPTO("xts-speck128-neon");
 287 MODULE_ALIAS_CRYPTO("xts(speck64)");
 288 MODULE_ALIAS_CRYPTO("xts-speck64-neon");