arch/arm64/crypto/chacha-neon-glue.c

   1 /*
   2  * ARM NEON and scalar accelerated ChaCha and XChaCha stream ciphers,
   3  * including ChaCha20 (RFC7539)
   4  *
   5  * Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * Based on:
  12  * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
  13  *
  14  * Copyright (C) 2015 Martin Willi
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <crypto/algapi.h>
  23 #include <crypto/internal/chacha.h>
  24 #include <crypto/internal/simd.h>
  25 #include <crypto/internal/skcipher.h>
  26 #include <linux/jump_label.h>
  27 #include <linux/kernel.h>
  28 #include <linux/module.h>
  29
  30 #include <asm/hwcap.h>
  31 #include <asm/neon.h>
  32 #include <asm/simd.h>
  33
  34 asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src,
  35                                       int nrounds);
  36 asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src,
  37                                        int nrounds, int bytes);
  38 asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
  39
  40 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
  41
  42 static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
  43                           int bytes, int nrounds)
  44 {
  45         while (bytes > 0) {
  46                 int l = min(bytes, CHACHA_BLOCK_SIZE * 5);
  47
  48                 if (l <= CHACHA_BLOCK_SIZE) {
  49                         u8 buf[CHACHA_BLOCK_SIZE];
  50
  51                         memcpy(buf, src, l);
  52                         chacha_block_xor_neon(state, buf, buf, nrounds);
  53                         memcpy(dst, buf, l);
  54                         state[12] += 1;
  55                         break;
  56                 }
  57                 chacha_4block_xor_neon(state, dst, src, nrounds, l);
  58                 bytes -= CHACHA_BLOCK_SIZE * 5;
  59                 src += CHACHA_BLOCK_SIZE * 5;
  60                 dst += CHACHA_BLOCK_SIZE * 5;
  61                 state[12] += 5;
  62         }
  63 }
  64
  65 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
  66 {
  67         if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {
  68                 hchacha_block_generic(state, stream, nrounds);
  69         } else {
  70                 kernel_neon_begin();
  71                 hchacha_block_neon(state, stream, nrounds);
  72                 kernel_neon_end();
  73         }
  74 }
  75 EXPORT_SYMBOL(hchacha_block_arch);
  76
  77 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
  78 {
  79         chacha_init_generic(state, key, iv);
  80 }
  81 EXPORT_SYMBOL(chacha_init_arch);
  82
  83 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
  84                        int nrounds)
  85 {
  86         if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE ||
  87             !crypto_simd_usable())
  88                 return chacha_crypt_generic(state, dst, src, bytes, nrounds);
  89
  90         kernel_neon_begin();
  91         chacha_doneon(state, dst, src, bytes, nrounds);
  92         kernel_neon_end();
  93 }
  94 EXPORT_SYMBOL(chacha_crypt_arch);
  95
  96 static int chacha_neon_stream_xor(struct skcipher_request *req,
  97                                   const struct chacha_ctx *ctx, const u8 *iv)
  98 {
  99         struct skcipher_walk walk;
 100         u32 state[16];
 101         int err;
 102
 103         err = skcipher_walk_virt(&walk, req, false);
 104
 105         chacha_init_generic(state, ctx->key, iv);
 106
 107         while (walk.nbytes > 0) {
 108                 unsigned int nbytes = walk.nbytes;
 109
 110                 if (nbytes < walk.total)
 111                         nbytes = rounddown(nbytes, walk.stride);
 112
 113                 if (!static_branch_likely(&have_neon) ||
 114                     !crypto_simd_usable()) {
 115                         chacha_crypt_generic(state, walk.dst.virt.addr,
 116                                              walk.src.virt.addr, nbytes,
 117                                              ctx->nrounds);
 118                 } else {
 119                         kernel_neon_begin();
 120                         chacha_doneon(state, walk.dst.virt.addr,
 121                                       walk.src.virt.addr, nbytes, ctx->nrounds);
 122                         kernel_neon_end();
 123                 }
 124                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 125         }
 126
 127         return err;
 128 }
 129
 130 static int chacha_neon(struct skcipher_request *req)
 131 {
 132         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 133         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 134
 135         return chacha_neon_stream_xor(req, ctx, req->iv);
 136 }
 137
 138 static int xchacha_neon(struct skcipher_request *req)
 139 {
 140         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 141         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 142         struct chacha_ctx subctx;
 143         u32 state[16];
 144         u8 real_iv[16];
 145
 146         chacha_init_generic(state, ctx->key, req->iv);
 147         hchacha_block_arch(state, subctx.key, ctx->nrounds);
 148         subctx.nrounds = ctx->nrounds;
 149
 150         memcpy(&real_iv[0], req->iv + 24, 8);
 151         memcpy(&real_iv[8], req->iv + 16, 8);
 152         return chacha_neon_stream_xor(req, &subctx, real_iv);
 153 }
 154
 155 static struct skcipher_alg algs[] = {
 156         {
 157                 .base.cra_name          = "chacha20",
 158                 .base.cra_driver_name   = "chacha20-neon",
 159                 .base.cra_priority      = 300,
 160                 .base.cra_blocksize     = 1,
 161                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 162                 .base.cra_module        = THIS_MODULE,
 163
 164                 .min_keysize            = CHACHA_KEY_SIZE,
 165                 .max_keysize            = CHACHA_KEY_SIZE,
 166                 .ivsize                 = CHACHA_IV_SIZE,
 167                 .chunksize              = CHACHA_BLOCK_SIZE,
 168                 .walksize               = 5 * CHACHA_BLOCK_SIZE,
 169                 .setkey                 = chacha20_setkey,
 170                 .encrypt                = chacha_neon,
 171                 .decrypt                = chacha_neon,
 172         }, {
 173                 .base.cra_name          = "xchacha20",
 174                 .base.cra_driver_name   = "xchacha20-neon",
 175                 .base.cra_priority      = 300,
 176                 .base.cra_blocksize     = 1,
 177                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 178                 .base.cra_module        = THIS_MODULE,
 179
 180                 .min_keysize            = CHACHA_KEY_SIZE,
 181                 .max_keysize            = CHACHA_KEY_SIZE,
 182                 .ivsize                 = XCHACHA_IV_SIZE,
 183                 .chunksize              = CHACHA_BLOCK_SIZE,
 184                 .walksize               = 5 * CHACHA_BLOCK_SIZE,
 185                 .setkey                 = chacha20_setkey,
 186                 .encrypt                = xchacha_neon,
 187                 .decrypt                = xchacha_neon,
 188         }, {
 189                 .base.cra_name          = "xchacha12",
 190                 .base.cra_driver_name   = "xchacha12-neon",
 191                 .base.cra_priority      = 300,
 192                 .base.cra_blocksize     = 1,
 193                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 194                 .base.cra_module        = THIS_MODULE,
 195
 196                 .min_keysize            = CHACHA_KEY_SIZE,
 197                 .max_keysize            = CHACHA_KEY_SIZE,
 198                 .ivsize                 = XCHACHA_IV_SIZE,
 199                 .chunksize              = CHACHA_BLOCK_SIZE,
 200                 .walksize               = 5 * CHACHA_BLOCK_SIZE,
 201                 .setkey                 = chacha12_setkey,
 202                 .encrypt                = xchacha_neon,
 203                 .decrypt                = xchacha_neon,
 204         }
 205 };
 206
 207 static int __init chacha_simd_mod_init(void)
 208 {
 209         if (!cpu_have_named_feature(ASIMD))
 210                 return 0;
 211
 212         static_branch_enable(&have_neon);
 213
 214         return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ?
 215                 crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0;
 216 }
 217
 218 static void __exit chacha_simd_mod_fini(void)
 219 {
 220         if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && cpu_have_named_feature(ASIMD))
 221                 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 222 }
 223
 224 module_init(chacha_simd_mod_init);
 225 module_exit(chacha_simd_mod_fini);
 226
 227 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
 228 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 229 MODULE_LICENSE("GPL v2");
 230 MODULE_ALIAS_CRYPTO("chacha20");
 231 MODULE_ALIAS_CRYPTO("chacha20-neon");
 232 MODULE_ALIAS_CRYPTO("xchacha20");
 233 MODULE_ALIAS_CRYPTO("xchacha20-neon");
 234 MODULE_ALIAS_CRYPTO("xchacha12");
 235 MODULE_ALIAS_CRYPTO("xchacha12-neon");