arch/x86/crypto/serpent_avx2_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
   4  *
   5  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/types.h>
  10 #include <linux/crypto.h>
  11 #include <linux/err.h>
  12 #include <crypto/algapi.h>
  13 #include <crypto/internal/simd.h>
  14 #include <crypto/serpent.h>
  15
  16 #include "serpent-avx.h"
  17 #include "ecb_cbc_helpers.h"
  18
  19 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
  20
  21 /* 16-way AVX2 parallel cipher functions */
  22 asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
  23 asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
  24 asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
  25
  26 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
  27                                    const u8 *key, unsigned int keylen)
  28 {
  29         return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
  30 }
  31
  32 static int ecb_encrypt(struct skcipher_request *req)
  33 {
  34         ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
  35         ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way);
  36         ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx);
  37         ECB_BLOCK(1, __serpent_encrypt);
  38         ECB_WALK_END();
  39 }
  40
  41 static int ecb_decrypt(struct skcipher_request *req)
  42 {
  43         ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
  44         ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way);
  45         ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx);
  46         ECB_BLOCK(1, __serpent_decrypt);
  47         ECB_WALK_END();
  48 }
  49
  50 static int cbc_encrypt(struct skcipher_request *req)
  51 {
  52         CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
  53         CBC_ENC_BLOCK(__serpent_encrypt);
  54         CBC_WALK_END();
  55 }
  56
  57 static int cbc_decrypt(struct skcipher_request *req)
  58 {
  59         CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
  60         CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way);
  61         CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx);
  62         CBC_DEC_BLOCK(1, __serpent_decrypt);
  63         CBC_WALK_END();
  64 }
  65
  66 static struct skcipher_alg serpent_algs[] = {
  67         {
  68                 .base.cra_name          = "__ecb(serpent)",
  69                 .base.cra_driver_name   = "__ecb-serpent-avx2",
  70                 .base.cra_priority      = 600,
  71                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
  72                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
  73                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
  74                 .base.cra_module        = THIS_MODULE,
  75                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
  76                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
  77                 .setkey                 = serpent_setkey_skcipher,
  78                 .encrypt                = ecb_encrypt,
  79                 .decrypt                = ecb_decrypt,
  80         }, {
  81                 .base.cra_name          = "__cbc(serpent)",
  82                 .base.cra_driver_name   = "__cbc-serpent-avx2",
  83                 .base.cra_priority      = 600,
  84                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
  85                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
  86                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
  87                 .base.cra_module        = THIS_MODULE,
  88                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
  89                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
  90                 .ivsize                 = SERPENT_BLOCK_SIZE,
  91                 .setkey                 = serpent_setkey_skcipher,
  92                 .encrypt                = cbc_encrypt,
  93                 .decrypt                = cbc_decrypt,
  94         },
  95 };
  96
  97 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
  98
  99 static int __init serpent_avx2_init(void)
 100 {
 101         const char *feature_name;
 102
 103         if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 104                 pr_info("AVX2 instructions are not detected.\n");
 105                 return -ENODEV;
 106         }
 107         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 108                                 &feature_name)) {
 109                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 110                 return -ENODEV;
 111         }
 112
 113         return simd_register_skciphers_compat(serpent_algs,
 114                                               ARRAY_SIZE(serpent_algs),
 115                                               serpent_simd_algs);
 116 }
 117
 118 static void __exit serpent_avx2_fini(void)
 119 {
 120         simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
 121                                   serpent_simd_algs);
 122 }
 123
 124 module_init(serpent_avx2_init);
 125 module_exit(serpent_avx2_fini);
 126
 127 MODULE_LICENSE("GPL");
 128 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
 129 MODULE_ALIAS_CRYPTO("serpent");
 130 MODULE_ALIAS_CRYPTO("serpent-asm");