arch/x86/crypto/twofish_avx_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for AVX assembler version of Twofish Cipher
   4  *
   5  * Copyright (C) 2012 Johannes Goetzfried
   6  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   7  *
   8  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/types.h>
  13 #include <linux/crypto.h>
  14 #include <linux/err.h>
  15 #include <crypto/algapi.h>
  16 #include <crypto/internal/simd.h>
  17 #include <crypto/twofish.h>
  18 #include <crypto/xts.h>
  19 #include <asm/crypto/glue_helper.h>
  20 #include <asm/crypto/twofish.h>
  21
  22 #define TWOFISH_PARALLEL_BLOCKS 8
  23
  24 /* 8-way parallel cipher functions */
  25 asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src);
  26 asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src);
  27
  28 asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src);
  29 asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src,
  30                                  le128 *iv);
  31
  32 asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src,
  33                                      le128 *iv);
  34 asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src,
  35                                      le128 *iv);
  36
  37 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
  38                                    const u8 *key, unsigned int keylen)
  39 {
  40         return twofish_setkey(&tfm->base, key, keylen);
  41 }
  42
  43 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
  44 {
  45         __twofish_enc_blk_3way(ctx, dst, src, false);
  46 }
  47
  48 static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  49 {
  50         glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk);
  51 }
  52
  53 static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  54 {
  55         glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk);
  56 }
  57
  58 struct twofish_xts_ctx {
  59         struct twofish_ctx tweak_ctx;
  60         struct twofish_ctx crypt_ctx;
  61 };
  62
  63 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
  64                               unsigned int keylen)
  65 {
  66         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  67         int err;
  68
  69         err = xts_verify_key(tfm, key, keylen);
  70         if (err)
  71                 return err;
  72
  73         /* first half of xts-key is for crypt */
  74         err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2);
  75         if (err)
  76                 return err;
  77
  78         /* second half of xts-key is for tweak */
  79         return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
  80 }
  81
  82 static const struct common_glue_ctx twofish_enc = {
  83         .num_funcs = 3,
  84         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
  85
  86         .funcs = { {
  87                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
  88                 .fn_u = { .ecb = twofish_ecb_enc_8way }
  89         }, {
  90                 .num_blocks = 3,
  91                 .fn_u = { .ecb = twofish_enc_blk_3way }
  92         }, {
  93                 .num_blocks = 1,
  94                 .fn_u = { .ecb = twofish_enc_blk }
  95         } }
  96 };
  97
  98 static const struct common_glue_ctx twofish_ctr = {
  99         .num_funcs = 3,
 100         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 101
 102         .funcs = { {
 103                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 104                 .fn_u = { .ctr = twofish_ctr_8way }
 105         }, {
 106                 .num_blocks = 3,
 107                 .fn_u = { .ctr = twofish_enc_blk_ctr_3way }
 108         }, {
 109                 .num_blocks = 1,
 110                 .fn_u = { .ctr = twofish_enc_blk_ctr }
 111         } }
 112 };
 113
 114 static const struct common_glue_ctx twofish_enc_xts = {
 115         .num_funcs = 2,
 116         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 117
 118         .funcs = { {
 119                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 120                 .fn_u = { .xts = twofish_xts_enc_8way }
 121         }, {
 122                 .num_blocks = 1,
 123                 .fn_u = { .xts = twofish_xts_enc }
 124         } }
 125 };
 126
 127 static const struct common_glue_ctx twofish_dec = {
 128         .num_funcs = 3,
 129         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 130
 131         .funcs = { {
 132                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 133                 .fn_u = { .ecb = twofish_ecb_dec_8way }
 134         }, {
 135                 .num_blocks = 3,
 136                 .fn_u = { .ecb = twofish_dec_blk_3way }
 137         }, {
 138                 .num_blocks = 1,
 139                 .fn_u = { .ecb = twofish_dec_blk }
 140         } }
 141 };
 142
 143 static const struct common_glue_ctx twofish_dec_cbc = {
 144         .num_funcs = 3,
 145         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 146
 147         .funcs = { {
 148                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 149                 .fn_u = { .cbc = twofish_cbc_dec_8way }
 150         }, {
 151                 .num_blocks = 3,
 152                 .fn_u = { .cbc = twofish_dec_blk_cbc_3way }
 153         }, {
 154                 .num_blocks = 1,
 155                 .fn_u = { .cbc = twofish_dec_blk }
 156         } }
 157 };
 158
 159 static const struct common_glue_ctx twofish_dec_xts = {
 160         .num_funcs = 2,
 161         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 162
 163         .funcs = { {
 164                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 165                 .fn_u = { .xts = twofish_xts_dec_8way }
 166         }, {
 167                 .num_blocks = 1,
 168                 .fn_u = { .xts = twofish_xts_dec }
 169         } }
 170 };
 171
 172 static int ecb_encrypt(struct skcipher_request *req)
 173 {
 174         return glue_ecb_req_128bit(&twofish_enc, req);
 175 }
 176
 177 static int ecb_decrypt(struct skcipher_request *req)
 178 {
 179         return glue_ecb_req_128bit(&twofish_dec, req);
 180 }
 181
 182 static int cbc_encrypt(struct skcipher_request *req)
 183 {
 184         return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req);
 185 }
 186
 187 static int cbc_decrypt(struct skcipher_request *req)
 188 {
 189         return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
 190 }
 191
 192 static int ctr_crypt(struct skcipher_request *req)
 193 {
 194         return glue_ctr_req_128bit(&twofish_ctr, req);
 195 }
 196
 197 static int xts_encrypt(struct skcipher_request *req)
 198 {
 199         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 200         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 201
 202         return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk,
 203                                    &ctx->tweak_ctx, &ctx->crypt_ctx, false);
 204 }
 205
 206 static int xts_decrypt(struct skcipher_request *req)
 207 {
 208         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 209         struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 210
 211         return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk,
 212                                    &ctx->tweak_ctx, &ctx->crypt_ctx, true);
 213 }
 214
 215 static struct skcipher_alg twofish_algs[] = {
 216         {
 217                 .base.cra_name          = "__ecb(twofish)",
 218                 .base.cra_driver_name   = "__ecb-twofish-avx",
 219                 .base.cra_priority      = 400,
 220                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 221                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 222                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 223                 .base.cra_module        = THIS_MODULE,
 224                 .min_keysize            = TF_MIN_KEY_SIZE,
 225                 .max_keysize            = TF_MAX_KEY_SIZE,
 226                 .setkey                 = twofish_setkey_skcipher,
 227                 .encrypt                = ecb_encrypt,
 228                 .decrypt                = ecb_decrypt,
 229         }, {
 230                 .base.cra_name          = "__cbc(twofish)",
 231                 .base.cra_driver_name   = "__cbc-twofish-avx",
 232                 .base.cra_priority      = 400,
 233                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 234                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 235                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 236                 .base.cra_module        = THIS_MODULE,
 237                 .min_keysize            = TF_MIN_KEY_SIZE,
 238                 .max_keysize            = TF_MAX_KEY_SIZE,
 239                 .ivsize                 = TF_BLOCK_SIZE,
 240                 .setkey                 = twofish_setkey_skcipher,
 241                 .encrypt                = cbc_encrypt,
 242                 .decrypt                = cbc_decrypt,
 243         }, {
 244                 .base.cra_name          = "__ctr(twofish)",
 245                 .base.cra_driver_name   = "__ctr-twofish-avx",
 246                 .base.cra_priority      = 400,
 247                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 248                 .base.cra_blocksize     = 1,
 249                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
 250                 .base.cra_module        = THIS_MODULE,
 251                 .min_keysize            = TF_MIN_KEY_SIZE,
 252                 .max_keysize            = TF_MAX_KEY_SIZE,
 253                 .ivsize                 = TF_BLOCK_SIZE,
 254                 .chunksize              = TF_BLOCK_SIZE,
 255                 .setkey                 = twofish_setkey_skcipher,
 256                 .encrypt                = ctr_crypt,
 257                 .decrypt                = ctr_crypt,
 258         }, {
 259                 .base.cra_name          = "__xts(twofish)",
 260                 .base.cra_driver_name   = "__xts-twofish-avx",
 261                 .base.cra_priority      = 400,
 262                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 263                 .base.cra_blocksize     = TF_BLOCK_SIZE,
 264                 .base.cra_ctxsize       = sizeof(struct twofish_xts_ctx),
 265                 .base.cra_module        = THIS_MODULE,
 266                 .min_keysize            = 2 * TF_MIN_KEY_SIZE,
 267                 .max_keysize            = 2 * TF_MAX_KEY_SIZE,
 268                 .ivsize                 = TF_BLOCK_SIZE,
 269                 .setkey                 = xts_twofish_setkey,
 270                 .encrypt                = xts_encrypt,
 271                 .decrypt                = xts_decrypt,
 272         },
 273 };
 274
 275 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
 276
 277 static int __init twofish_init(void)
 278 {
 279         const char *feature_name;
 280
 281         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
 282                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 283                 return -ENODEV;
 284         }
 285
 286         return simd_register_skciphers_compat(twofish_algs,
 287                                               ARRAY_SIZE(twofish_algs),
 288                                               twofish_simd_algs);
 289 }
 290
 291 static void __exit twofish_exit(void)
 292 {
 293         simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
 294                                   twofish_simd_algs);
 295 }
 296
 297 module_init(twofish_init);
 298 module_exit(twofish_exit);
 299
 300 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
 301 MODULE_LICENSE("GPL");
 302 MODULE_ALIAS_CRYPTO("twofish");