arch/x86/crypto/serpent_avx_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for AVX assembler versions of Serpent Cipher
   4  *
   5  * Copyright (C) 2012 Johannes Goetzfried
   6  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   7  *
   8  * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/types.h>
  13 #include <linux/crypto.h>
  14 #include <linux/err.h>
  15 #include <crypto/algapi.h>
  16 #include <crypto/internal/simd.h>
  17 #include <crypto/serpent.h>
  18 #include <crypto/xts.h>
  19 #include <asm/crypto/glue_helper.h>
  20 #include <asm/crypto/serpent-avx.h>
  21
  22 /* 8-way parallel cipher functions */
  23 asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst,
  24                                          const u8 *src);
  25 EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx);
  26
  27 asmlinkage void serpent_ecb_dec_8way_avx(const void *ctx, u8 *dst,
  28                                          const u8 *src);
  29 EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx);
  30
  31 asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst,
  32                                          const u8 *src);
  33 EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx);
  34
  35 asmlinkage void serpent_ctr_8way_avx(const void *ctx, u8 *dst, const u8 *src,
  36                                      le128 *iv);
  37 EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx);
  38
  39 asmlinkage void serpent_xts_enc_8way_avx(const void *ctx, u8 *dst,
  40                                          const u8 *src, le128 *iv);
  41 EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx);
  42
  43 asmlinkage void serpent_xts_dec_8way_avx(const void *ctx, u8 *dst,
  44                                          const u8 *src, le128 *iv);
  45 EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx);
  46
  47 void __serpent_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv)
  48 {
  49         be128 ctrblk;
  50         u128 *dst = (u128 *)d;
  51         const u128 *src = (const u128 *)s;
  52
  53         le128_to_be128(&ctrblk, iv);
  54         le128_inc(iv);
  55
  56         __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
  57         u128_xor(dst, src, (u128 *)&ctrblk);
  58 }
  59 EXPORT_SYMBOL_GPL(__serpent_crypt_ctr);
  60
  61 void serpent_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  62 {
  63         glue_xts_crypt_128bit_one(ctx, dst, src, iv, __serpent_encrypt);
  64 }
  65 EXPORT_SYMBOL_GPL(serpent_xts_enc);
  66
  67 void serpent_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  68 {
  69         glue_xts_crypt_128bit_one(ctx, dst, src, iv, __serpent_decrypt);
  70 }
  71 EXPORT_SYMBOL_GPL(serpent_xts_dec);
  72
  73 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
  74                                    const u8 *key, unsigned int keylen)
  75 {
  76         return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
  77 }
  78
  79 int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key,
  80                        unsigned int keylen)
  81 {
  82         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  83         int err;
  84
  85         err = xts_verify_key(tfm, key, keylen);
  86         if (err)
  87                 return err;
  88
  89         /* first half of xts-key is for crypt */
  90         err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
  91         if (err)
  92                 return err;
  93
  94         /* second half of xts-key is for tweak */
  95         return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
  96 }
  97 EXPORT_SYMBOL_GPL(xts_serpent_setkey);
  98
  99 static const struct common_glue_ctx serpent_enc = {
 100         .num_funcs = 2,
 101         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 102
 103         .funcs = { {
 104                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 105                 .fn_u = { .ecb = serpent_ecb_enc_8way_avx }
 106         }, {
 107                 .num_blocks = 1,
 108                 .fn_u = { .ecb = __serpent_encrypt }
 109         } }
 110 };
 111
 112 static const struct common_glue_ctx serpent_ctr = {
 113         .num_funcs = 2,
 114         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 115
 116         .funcs = { {
 117                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 118                 .fn_u = { .ctr = serpent_ctr_8way_avx }
 119         }, {
 120                 .num_blocks = 1,
 121                 .fn_u = { .ctr = __serpent_crypt_ctr }
 122         } }
 123 };
 124
 125 static const struct common_glue_ctx serpent_enc_xts = {
 126         .num_funcs = 2,
 127         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 128
 129         .funcs = { {
 130                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 131                 .fn_u = { .xts = serpent_xts_enc_8way_avx }
 132         }, {
 133                 .num_blocks = 1,
 134                 .fn_u = { .xts = serpent_xts_enc }
 135         } }
 136 };
 137
 138 static const struct common_glue_ctx serpent_dec = {
 139         .num_funcs = 2,
 140         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 141
 142         .funcs = { {
 143                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 144                 .fn_u = { .ecb = serpent_ecb_dec_8way_avx }
 145         }, {
 146                 .num_blocks = 1,
 147                 .fn_u = { .ecb = __serpent_decrypt }
 148         } }
 149 };
 150
 151 static const struct common_glue_ctx serpent_dec_cbc = {
 152         .num_funcs = 2,
 153         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 154
 155         .funcs = { {
 156                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 157                 .fn_u = { .cbc = serpent_cbc_dec_8way_avx }
 158         }, {
 159                 .num_blocks = 1,
 160                 .fn_u = { .cbc = __serpent_decrypt }
 161         } }
 162 };
 163
 164 static const struct common_glue_ctx serpent_dec_xts = {
 165         .num_funcs = 2,
 166         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 167
 168         .funcs = { {
 169                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 170                 .fn_u = { .xts = serpent_xts_dec_8way_avx }
 171         }, {
 172                 .num_blocks = 1,
 173                 .fn_u = { .xts = serpent_xts_dec }
 174         } }
 175 };
 176
 177 static int ecb_encrypt(struct skcipher_request *req)
 178 {
 179         return glue_ecb_req_128bit(&serpent_enc, req);
 180 }
 181
 182 static int ecb_decrypt(struct skcipher_request *req)
 183 {
 184         return glue_ecb_req_128bit(&serpent_dec, req);
 185 }
 186
 187 static int cbc_encrypt(struct skcipher_request *req)
 188 {
 189         return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req);
 190 }
 191
 192 static int cbc_decrypt(struct skcipher_request *req)
 193 {
 194         return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
 195 }
 196
 197 static int ctr_crypt(struct skcipher_request *req)
 198 {
 199         return glue_ctr_req_128bit(&serpent_ctr, req);
 200 }
 201
 202 static int xts_encrypt(struct skcipher_request *req)
 203 {
 204         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 205         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 206
 207         return glue_xts_req_128bit(&serpent_enc_xts, req,
 208                                    __serpent_encrypt, &ctx->tweak_ctx,
 209                                    &ctx->crypt_ctx, false);
 210 }
 211
 212 static int xts_decrypt(struct skcipher_request *req)
 213 {
 214         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 215         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 216
 217         return glue_xts_req_128bit(&serpent_dec_xts, req,
 218                                    __serpent_encrypt, &ctx->tweak_ctx,
 219                                    &ctx->crypt_ctx, true);
 220 }
 221
 222 static struct skcipher_alg serpent_algs[] = {
 223         {
 224                 .base.cra_name          = "__ecb(serpent)",
 225                 .base.cra_driver_name   = "__ecb-serpent-avx",
 226                 .base.cra_priority      = 500,
 227                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 228                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 229                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 230                 .base.cra_module        = THIS_MODULE,
 231                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 232                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 233                 .setkey                 = serpent_setkey_skcipher,
 234                 .encrypt                = ecb_encrypt,
 235                 .decrypt                = ecb_decrypt,
 236         }, {
 237                 .base.cra_name          = "__cbc(serpent)",
 238                 .base.cra_driver_name   = "__cbc-serpent-avx",
 239                 .base.cra_priority      = 500,
 240                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 241                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 242                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 243                 .base.cra_module        = THIS_MODULE,
 244                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 245                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 246                 .ivsize                 = SERPENT_BLOCK_SIZE,
 247                 .setkey                 = serpent_setkey_skcipher,
 248                 .encrypt                = cbc_encrypt,
 249                 .decrypt                = cbc_decrypt,
 250         }, {
 251                 .base.cra_name          = "__ctr(serpent)",
 252                 .base.cra_driver_name   = "__ctr-serpent-avx",
 253                 .base.cra_priority      = 500,
 254                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 255                 .base.cra_blocksize     = 1,
 256                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 257                 .base.cra_module        = THIS_MODULE,
 258                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 259                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 260                 .ivsize                 = SERPENT_BLOCK_SIZE,
 261                 .chunksize              = SERPENT_BLOCK_SIZE,
 262                 .setkey                 = serpent_setkey_skcipher,
 263                 .encrypt                = ctr_crypt,
 264                 .decrypt                = ctr_crypt,
 265         }, {
 266                 .base.cra_name          = "__xts(serpent)",
 267                 .base.cra_driver_name   = "__xts-serpent-avx",
 268                 .base.cra_priority      = 500,
 269                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 270                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 271                 .base.cra_ctxsize       = sizeof(struct serpent_xts_ctx),
 272                 .base.cra_module        = THIS_MODULE,
 273                 .min_keysize            = 2 * SERPENT_MIN_KEY_SIZE,
 274                 .max_keysize            = 2 * SERPENT_MAX_KEY_SIZE,
 275                 .ivsize                 = SERPENT_BLOCK_SIZE,
 276                 .setkey                 = xts_serpent_setkey,
 277                 .encrypt                = xts_encrypt,
 278                 .decrypt                = xts_decrypt,
 279         },
 280 };
 281
 282 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
 283
 284 static int __init serpent_init(void)
 285 {
 286         const char *feature_name;
 287
 288         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 289                                 &feature_name)) {
 290                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 291                 return -ENODEV;
 292         }
 293
 294         return simd_register_skciphers_compat(serpent_algs,
 295                                               ARRAY_SIZE(serpent_algs),
 296                                               serpent_simd_algs);
 297 }
 298
 299 static void __exit serpent_exit(void)
 300 {
 301         simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
 302                                   serpent_simd_algs);
 303 }
 304
 305 module_init(serpent_init);
 306 module_exit(serpent_exit);
 307
 308 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
 309 MODULE_LICENSE("GPL");
 310 MODULE_ALIAS_CRYPTO("serpent");