arch/x86/crypto/twofish_glue_3way.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for 3-way parallel assembler optimized version of Twofish
   4  *
   5  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   6  */
   7
   8 #include <asm/cpu_device_id.h>
   9 #include <crypto/algapi.h>
  10 #include <crypto/twofish.h>
  11 #include <linux/crypto.h>
  12 #include <linux/init.h>
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15
  16 #include "twofish.h"
  17 #include "ecb_cbc_helpers.h"
  18
  19 EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
  20 EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
  21
  22 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
  23                                    const u8 *key, unsigned int keylen)
  24 {
  25         return twofish_setkey(&tfm->base, key, keylen);
  26 }
  27
  28 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
  29 {
  30         __twofish_enc_blk_3way(ctx, dst, src, false);
  31 }
  32
  33 void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src)
  34 {
  35         u8 buf[2][TF_BLOCK_SIZE];
  36         const u8 *s = src;
  37
  38         if (dst == src)
  39                 s = memcpy(buf, src, sizeof(buf));
  40         twofish_dec_blk_3way(ctx, dst, src);
  41         crypto_xor(dst + TF_BLOCK_SIZE, s, sizeof(buf));
  42
  43 }
  44 EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
  45
  46 static int ecb_encrypt(struct skcipher_request *req)
  47 {
  48         ECB_WALK_START(req, TF_BLOCK_SIZE, -1);
  49         ECB_BLOCK(3, twofish_enc_blk_3way);
  50         ECB_BLOCK(1, twofish_enc_blk);
  51         ECB_WALK_END();
  52 }
  53
  54 static int ecb_decrypt(struct skcipher_request *req)
  55 {
  56         ECB_WALK_START(req, TF_BLOCK_SIZE, -1);
  57         ECB_BLOCK(3, twofish_dec_blk_3way);
  58         ECB_BLOCK(1, twofish_dec_blk);
  59         ECB_WALK_END();
  60 }
  61
  62 static int cbc_encrypt(struct skcipher_request *req)
  63 {
  64         CBC_WALK_START(req, TF_BLOCK_SIZE, -1);
  65         CBC_ENC_BLOCK(twofish_enc_blk);
  66         CBC_WALK_END();
  67 }
  68
  69 static int cbc_decrypt(struct skcipher_request *req)
  70 {
  71         CBC_WALK_START(req, TF_BLOCK_SIZE, -1);
  72         CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way);
  73         CBC_DEC_BLOCK(1, twofish_dec_blk);
  74         CBC_WALK_END();
  75 }
  76
  77 static struct skcipher_alg tf_skciphers[] = {
  78         {
  79                 .base.cra_name          = "ecb(twofish)",
  80                 .base.cra_driver_name   = "ecb-twofish-3way",
  81                 .base.cra_priority      = 300,
  82                 .base.cra_blocksize     = TF_BLOCK_SIZE,
  83                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
  84                 .base.cra_module        = THIS_MODULE,
  85                 .min_keysize            = TF_MIN_KEY_SIZE,
  86                 .max_keysize            = TF_MAX_KEY_SIZE,
  87                 .setkey                 = twofish_setkey_skcipher,
  88                 .encrypt                = ecb_encrypt,
  89                 .decrypt                = ecb_decrypt,
  90         }, {
  91                 .base.cra_name          = "cbc(twofish)",
  92                 .base.cra_driver_name   = "cbc-twofish-3way",
  93                 .base.cra_priority      = 300,
  94                 .base.cra_blocksize     = TF_BLOCK_SIZE,
  95                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
  96                 .base.cra_module        = THIS_MODULE,
  97                 .min_keysize            = TF_MIN_KEY_SIZE,
  98                 .max_keysize            = TF_MAX_KEY_SIZE,
  99                 .ivsize                 = TF_BLOCK_SIZE,
 100                 .setkey                 = twofish_setkey_skcipher,
 101                 .encrypt                = cbc_encrypt,
 102                 .decrypt                = cbc_decrypt,
 103         },
 104 };
 105
 106 static bool is_blacklisted_cpu(void)
 107 {
 108         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
 109                 return false;
 110
 111         switch (boot_cpu_data.x86_vfm) {
 112         case INTEL_ATOM_BONNELL:
 113         case INTEL_ATOM_BONNELL_MID:
 114         case INTEL_ATOM_SALTWELL:
 115                 /*
 116                  * On Atom, twofish-3way is slower than original assembler
 117                  * implementation. Twofish-3way trades off some performance in
 118                  * storing blocks in 64bit registers to allow three blocks to
 119                  * be processed parallel. Parallel operation then allows gaining
 120                  * more performance than was trade off, on out-of-order CPUs.
 121                  * However Atom does not benefit from this parallelism and
 122                  * should be blacklisted.
 123                  */
 124                 return true;
 125         }
 126
 127         if (boot_cpu_data.x86 == 0x0f) {
 128                 /*
 129                  * On Pentium 4, twofish-3way is slower than original assembler
 130                  * implementation because excessive uses of 64bit rotate and
 131                  * left-shifts (which are really slow on P4) needed to store and
 132                  * handle 128bit block in two 64bit registers.
 133                  */
 134                 return true;
 135         }
 136
 137         return false;
 138 }
 139
 140 static int force;
 141 module_param(force, int, 0);
 142 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
 143
 144 static int __init twofish_3way_init(void)
 145 {
 146         if (!force && is_blacklisted_cpu()) {
 147                 printk(KERN_INFO
 148                         "twofish-x86_64-3way: performance on this CPU "
 149                         "would be suboptimal: disabling "
 150                         "twofish-x86_64-3way.\n");
 151                 return -ENODEV;
 152         }
 153
 154         return crypto_register_skciphers(tf_skciphers,
 155                                          ARRAY_SIZE(tf_skciphers));
 156 }
 157
 158 static void __exit twofish_3way_fini(void)
 159 {
 160         crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers));
 161 }
 162
 163 module_init(twofish_3way_init);
 164 module_exit(twofish_3way_fini);
 165
 166 MODULE_LICENSE("GPL");
 167 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
 168 MODULE_ALIAS_CRYPTO("twofish");
 169 MODULE_ALIAS_CRYPTO("twofish-asm");