arch/x86/crypto/aria_gfni_avx512_glue.c

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * Glue Code for the AVX512/GFNI assembler implementation of the ARIA Cipher
   4  *
   5  * Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com>
   6  */
   7
   8 #include <crypto/algapi.h>
   9 #include <crypto/internal/simd.h>
  10 #include <crypto/aria.h>
  11 #include <linux/crypto.h>
  12 #include <linux/err.h>
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15
  16 #include "ecb_cbc_helpers.h"
  17 #include "aria-avx.h"
  18
  19 asmlinkage void aria_gfni_avx512_encrypt_64way(const void *ctx, u8 *dst,
  20                                                const u8 *src);
  21 asmlinkage void aria_gfni_avx512_decrypt_64way(const void *ctx, u8 *dst,
  22                                                const u8 *src);
  23 asmlinkage void aria_gfni_avx512_ctr_crypt_64way(const void *ctx, u8 *dst,
  24                                                  const u8 *src,
  25                                                  u8 *keystream, u8 *iv);
  26
  27 static struct aria_avx_ops aria_ops;
  28
  29 struct aria_avx512_request_ctx {
  30         u8 keystream[ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE];
  31 };
  32
  33 static int ecb_do_encrypt(struct skcipher_request *req, const u32 *rkey)
  34 {
  35         ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS);
  36         ECB_BLOCK(ARIA_GFNI_AVX512_PARALLEL_BLOCKS, aria_ops.aria_encrypt_64way);
  37         ECB_BLOCK(ARIA_AESNI_AVX2_PARALLEL_BLOCKS, aria_ops.aria_encrypt_32way);
  38         ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_encrypt_16way);
  39         ECB_BLOCK(1, aria_encrypt);
  40         ECB_WALK_END();
  41 }
  42
  43 static int ecb_do_decrypt(struct skcipher_request *req, const u32 *rkey)
  44 {
  45         ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS);
  46         ECB_BLOCK(ARIA_GFNI_AVX512_PARALLEL_BLOCKS, aria_ops.aria_decrypt_64way);
  47         ECB_BLOCK(ARIA_AESNI_AVX2_PARALLEL_BLOCKS, aria_ops.aria_decrypt_32way);
  48         ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_decrypt_16way);
  49         ECB_BLOCK(1, aria_decrypt);
  50         ECB_WALK_END();
  51 }
  52
  53 static int aria_avx512_ecb_encrypt(struct skcipher_request *req)
  54 {
  55         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  56         struct aria_ctx *ctx = crypto_skcipher_ctx(tfm);
  57
  58         return ecb_do_encrypt(req, ctx->enc_key[0]);
  59 }
  60
  61 static int aria_avx512_ecb_decrypt(struct skcipher_request *req)
  62 {
  63         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  64         struct aria_ctx *ctx = crypto_skcipher_ctx(tfm);
  65
  66         return ecb_do_decrypt(req, ctx->dec_key[0]);
  67 }
  68
  69 static int aria_avx512_set_key(struct crypto_skcipher *tfm, const u8 *key,
  70                             unsigned int keylen)
  71 {
  72         return aria_set_key(&tfm->base, key, keylen);
  73 }
  74
  75 static int aria_avx512_ctr_encrypt(struct skcipher_request *req)
  76 {
  77         struct aria_avx512_request_ctx *req_ctx = skcipher_request_ctx(req);
  78         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  79         struct aria_ctx *ctx = crypto_skcipher_ctx(tfm);
  80         struct skcipher_walk walk;
  81         unsigned int nbytes;
  82         int err;
  83
  84         err = skcipher_walk_virt(&walk, req, false);
  85
  86         while ((nbytes = walk.nbytes) > 0) {
  87                 const u8 *src = walk.src.virt.addr;
  88                 u8 *dst = walk.dst.virt.addr;
  89
  90                 while (nbytes >= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE) {
  91                         kernel_fpu_begin();
  92                         aria_ops.aria_ctr_crypt_64way(ctx, dst, src,
  93                                                       &req_ctx->keystream[0],
  94                                                       walk.iv);
  95                         kernel_fpu_end();
  96                         dst += ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE;
  97                         src += ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE;
  98                         nbytes -= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE;
  99                 }
 100
 101                 while (nbytes >= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE) {
 102                         kernel_fpu_begin();
 103                         aria_ops.aria_ctr_crypt_32way(ctx, dst, src,
 104                                                       &req_ctx->keystream[0],
 105                                                       walk.iv);
 106                         kernel_fpu_end();
 107                         dst += ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE;
 108                         src += ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE;
 109                         nbytes -= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE;
 110                 }
 111
 112                 while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) {
 113                         kernel_fpu_begin();
 114                         aria_ops.aria_ctr_crypt_16way(ctx, dst, src,
 115                                                       &req_ctx->keystream[0],
 116                                                       walk.iv);
 117                         kernel_fpu_end();
 118                         dst += ARIA_AESNI_PARALLEL_BLOCK_SIZE;
 119                         src += ARIA_AESNI_PARALLEL_BLOCK_SIZE;
 120                         nbytes -= ARIA_AESNI_PARALLEL_BLOCK_SIZE;
 121                 }
 122
 123                 while (nbytes >= ARIA_BLOCK_SIZE) {
 124                         memcpy(&req_ctx->keystream[0], walk.iv,
 125                                ARIA_BLOCK_SIZE);
 126                         crypto_inc(walk.iv, ARIA_BLOCK_SIZE);
 127
 128                         aria_encrypt(ctx, &req_ctx->keystream[0],
 129                                      &req_ctx->keystream[0]);
 130
 131                         crypto_xor_cpy(dst, src, &req_ctx->keystream[0],
 132                                        ARIA_BLOCK_SIZE);
 133                         dst += ARIA_BLOCK_SIZE;
 134                         src += ARIA_BLOCK_SIZE;
 135                         nbytes -= ARIA_BLOCK_SIZE;
 136                 }
 137
 138                 if (walk.nbytes == walk.total && nbytes > 0) {
 139                         memcpy(&req_ctx->keystream[0], walk.iv,
 140                                ARIA_BLOCK_SIZE);
 141                         crypto_inc(walk.iv, ARIA_BLOCK_SIZE);
 142
 143                         aria_encrypt(ctx, &req_ctx->keystream[0],
 144                                      &req_ctx->keystream[0]);
 145
 146                         crypto_xor_cpy(dst, src, &req_ctx->keystream[0],
 147                                        nbytes);
 148                         dst += nbytes;
 149                         src += nbytes;
 150                         nbytes = 0;
 151                 }
 152                 err = skcipher_walk_done(&walk, nbytes);
 153         }
 154
 155         return err;
 156 }
 157
 158 static int aria_avx512_init_tfm(struct crypto_skcipher *tfm)
 159 {
 160         crypto_skcipher_set_reqsize(tfm,
 161                                     sizeof(struct aria_avx512_request_ctx));
 162
 163         return 0;
 164 }
 165
 166 static struct skcipher_alg aria_algs[] = {
 167         {
 168                 .base.cra_name          = "__ecb(aria)",
 169                 .base.cra_driver_name   = "__ecb-aria-avx512",
 170                 .base.cra_priority      = 600,
 171                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 172                 .base.cra_blocksize     = ARIA_BLOCK_SIZE,
 173                 .base.cra_ctxsize       = sizeof(struct aria_ctx),
 174                 .base.cra_module        = THIS_MODULE,
 175                 .min_keysize            = ARIA_MIN_KEY_SIZE,
 176                 .max_keysize            = ARIA_MAX_KEY_SIZE,
 177                 .setkey                 = aria_avx512_set_key,
 178                 .encrypt                = aria_avx512_ecb_encrypt,
 179                 .decrypt                = aria_avx512_ecb_decrypt,
 180         }, {
 181                 .base.cra_name          = "__ctr(aria)",
 182                 .base.cra_driver_name   = "__ctr-aria-avx512",
 183                 .base.cra_priority      = 600,
 184                 .base.cra_flags         = CRYPTO_ALG_INTERNAL |
 185                                           CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE,
 186                 .base.cra_blocksize     = 1,
 187                 .base.cra_ctxsize       = sizeof(struct aria_ctx),
 188                 .base.cra_module        = THIS_MODULE,
 189                 .min_keysize            = ARIA_MIN_KEY_SIZE,
 190                 .max_keysize            = ARIA_MAX_KEY_SIZE,
 191                 .ivsize                 = ARIA_BLOCK_SIZE,
 192                 .chunksize              = ARIA_BLOCK_SIZE,
 193                 .setkey                 = aria_avx512_set_key,
 194                 .encrypt                = aria_avx512_ctr_encrypt,
 195                 .decrypt                = aria_avx512_ctr_encrypt,
 196                 .init                   = aria_avx512_init_tfm,
 197         }
 198 };
 199
 200 static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)];
 201
 202 static int __init aria_avx512_init(void)
 203 {
 204         const char *feature_name;
 205
 206         if (!boot_cpu_has(X86_FEATURE_AVX) ||
 207             !boot_cpu_has(X86_FEATURE_AVX2) ||
 208             !boot_cpu_has(X86_FEATURE_AVX512F) ||
 209             !boot_cpu_has(X86_FEATURE_AVX512VL) ||
 210             !boot_cpu_has(X86_FEATURE_GFNI) ||
 211             !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 212                 pr_info("AVX512/GFNI instructions are not detected.\n");
 213                 return -ENODEV;
 214         }
 215
 216         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
 217                                XFEATURE_MASK_AVX512, &feature_name)) {
 218                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 219                 return -ENODEV;
 220         }
 221
 222         aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way;
 223         aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way;
 224         aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way;
 225         aria_ops.aria_encrypt_32way = aria_aesni_avx2_gfni_encrypt_32way;
 226         aria_ops.aria_decrypt_32way = aria_aesni_avx2_gfni_decrypt_32way;
 227         aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_gfni_ctr_crypt_32way;
 228         aria_ops.aria_encrypt_64way = aria_gfni_avx512_encrypt_64way;
 229         aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way;
 230         aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way;
 231
 232         return simd_register_skciphers_compat(aria_algs,
 233                                               ARRAY_SIZE(aria_algs),
 234                                               aria_simd_algs);
 235 }
 236
 237 static void __exit aria_avx512_exit(void)
 238 {
 239         simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs),
 240                                   aria_simd_algs);
 241 }
 242
 243 module_init(aria_avx512_init);
 244 module_exit(aria_avx512_exit);
 245
 246 MODULE_LICENSE("GPL");
 247 MODULE_AUTHOR("Taehee Yoo <ap420073@gmail.com>");
 248 MODULE_DESCRIPTION("ARIA Cipher Algorithm, AVX512/GFNI optimized");
 249 MODULE_ALIAS_CRYPTO("aria");
 250 MODULE_ALIAS_CRYPTO("aria-gfni-avx512");