arch/riscv/crypto/ghash-riscv64-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * GHASH using the RISC-V vector crypto extensions
   4  *
   5  * Copyright (C) 2023 VRULL GmbH
   6  * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
   7  *
   8  * Copyright (C) 2023 SiFive, Inc.
   9  * Author: Jerry Shih <jerry.shih@sifive.com>
  10  */
  11
  12 #include <asm/simd.h>
  13 #include <asm/vector.h>
  14 #include <crypto/ghash.h>
  15 #include <crypto/internal/hash.h>
  16 #include <crypto/internal/simd.h>
  17 #include <linux/linkage.h>
  18 #include <linux/module.h>
  19
  20 asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
  21                            size_t len);
  22
  23 struct riscv64_ghash_tfm_ctx {
  24         be128 key;
  25 };
  26
  27 struct riscv64_ghash_desc_ctx {
  28         be128 accumulator;
  29         u8 buffer[GHASH_BLOCK_SIZE];
  30         u32 bytes;
  31 };
  32
  33 static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
  34                                 unsigned int keylen)
  35 {
  36         struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
  37
  38         if (keylen != GHASH_BLOCK_SIZE)
  39                 return -EINVAL;
  40
  41         memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
  42
  43         return 0;
  44 }
  45
  46 static int riscv64_ghash_init(struct shash_desc *desc)
  47 {
  48         struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
  49
  50         *dctx = (struct riscv64_ghash_desc_ctx){};
  51
  52         return 0;
  53 }
  54
  55 static inline void
  56 riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
  57                      struct riscv64_ghash_desc_ctx *dctx,
  58                      const u8 *src, size_t srclen)
  59 {
  60         /* The srclen is nonzero and a multiple of 16. */
  61         if (crypto_simd_usable()) {
  62                 kernel_vector_begin();
  63                 ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
  64                 kernel_vector_end();
  65         } else {
  66                 do {
  67                         crypto_xor((u8 *)&dctx->accumulator, src,
  68                                    GHASH_BLOCK_SIZE);
  69                         gf128mul_lle(&dctx->accumulator, &tctx->key);
  70                         src += GHASH_BLOCK_SIZE;
  71                         srclen -= GHASH_BLOCK_SIZE;
  72                 } while (srclen);
  73         }
  74 }
  75
  76 static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
  77                                 unsigned int srclen)
  78 {
  79         const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
  80         struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
  81         unsigned int len;
  82
  83         if (dctx->bytes) {
  84                 if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) {
  85                         memcpy(dctx->buffer + dctx->bytes, src, srclen);
  86                         dctx->bytes += srclen;
  87                         return 0;
  88                 }
  89                 memcpy(dctx->buffer + dctx->bytes, src,
  90                        GHASH_BLOCK_SIZE - dctx->bytes);
  91                 riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
  92                                      GHASH_BLOCK_SIZE);
  93                 src += GHASH_BLOCK_SIZE - dctx->bytes;
  94                 srclen -= GHASH_BLOCK_SIZE - dctx->bytes;
  95                 dctx->bytes = 0;
  96         }
  97
  98         len = round_down(srclen, GHASH_BLOCK_SIZE);
  99         if (len) {
 100                 riscv64_ghash_blocks(tctx, dctx, src, len);
 101                 src += len;
 102                 srclen -= len;
 103         }
 104
 105         if (srclen) {
 106                 memcpy(dctx->buffer, src, srclen);
 107                 dctx->bytes = srclen;
 108         }
 109
 110         return 0;
 111 }
 112
 113 static int riscv64_ghash_final(struct shash_desc *desc, u8 *out)
 114 {
 115         const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 116         struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 117         int i;
 118
 119         if (dctx->bytes) {
 120                 for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++)
 121                         dctx->buffer[i] = 0;
 122
 123                 riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
 124                                      GHASH_BLOCK_SIZE);
 125         }
 126
 127         memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
 128         return 0;
 129 }
 130
 131 static struct shash_alg riscv64_ghash_alg = {
 132         .init = riscv64_ghash_init,
 133         .update = riscv64_ghash_update,
 134         .final = riscv64_ghash_final,
 135         .setkey = riscv64_ghash_setkey,
 136         .descsize = sizeof(struct riscv64_ghash_desc_ctx),
 137         .digestsize = GHASH_DIGEST_SIZE,
 138         .base = {
 139                 .cra_blocksize = GHASH_BLOCK_SIZE,
 140                 .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
 141                 .cra_priority = 300,
 142                 .cra_name = "ghash",
 143                 .cra_driver_name = "ghash-riscv64-zvkg",
 144                 .cra_module = THIS_MODULE,
 145         },
 146 };
 147
 148 static int __init riscv64_ghash_mod_init(void)
 149 {
 150         if (riscv_isa_extension_available(NULL, ZVKG) &&
 151             riscv_vector_vlen() >= 128)
 152                 return crypto_register_shash(&riscv64_ghash_alg);
 153
 154         return -ENODEV;
 155 }
 156
 157 static void __exit riscv64_ghash_mod_exit(void)
 158 {
 159         crypto_unregister_shash(&riscv64_ghash_alg);
 160 }
 161
 162 module_init(riscv64_ghash_mod_init);
 163 module_exit(riscv64_ghash_mod_exit);
 164
 165 MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
 166 MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
 167 MODULE_LICENSE("GPL");
 168 MODULE_ALIAS_CRYPTO("ghash");