arch/loongarch/crypto/crc32-loongarch.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * crc32.c - CRC32 and CRC32C using LoongArch crc* instructions
   4  *
   5  * Module based on mips/crypto/crc32-mips.c
   6  *
   7  * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
   8  * Copyright (C) 2018 MIPS Tech, LLC
   9  * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
  10  */
  11
  12 #include <linux/module.h>
  13 #include <crypto/internal/hash.h>
  14
  15 #include <asm/cpu-features.h>
  16 #include <linux/unaligned.h>
  17
  18 #define _CRC32(crc, value, size, type)                  \
  19 do {                                                    \
  20         __asm__ __volatile__(                           \
  21                 #type ".w." #size ".w" " %0, %1, %0\n\t"\
  22                 : "+r" (crc)                            \
  23                 : "r" (value)                           \
  24                 : "memory");                            \
  25 } while (0)
  26
  27 #define CRC32(crc, value, size)         _CRC32(crc, value, size, crc)
  28 #define CRC32C(crc, value, size)        _CRC32(crc, value, size, crcc)
  29
  30 static u32 crc32_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
  31 {
  32         u32 crc = crc_;
  33
  34         while (len >= sizeof(u64)) {
  35                 u64 value = get_unaligned_le64(p);
  36
  37                 CRC32(crc, value, d);
  38                 p += sizeof(u64);
  39                 len -= sizeof(u64);
  40         }
  41
  42         if (len & sizeof(u32)) {
  43                 u32 value = get_unaligned_le32(p);
  44
  45                 CRC32(crc, value, w);
  46                 p += sizeof(u32);
  47         }
  48
  49         if (len & sizeof(u16)) {
  50                 u16 value = get_unaligned_le16(p);
  51
  52                 CRC32(crc, value, h);
  53                 p += sizeof(u16);
  54         }
  55
  56         if (len & sizeof(u8)) {
  57                 u8 value = *p++;
  58
  59                 CRC32(crc, value, b);
  60         }
  61
  62         return crc;
  63 }
  64
  65 static u32 crc32c_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
  66 {
  67         u32 crc = crc_;
  68
  69         while (len >= sizeof(u64)) {
  70                 u64 value = get_unaligned_le64(p);
  71
  72                 CRC32C(crc, value, d);
  73                 p += sizeof(u64);
  74                 len -= sizeof(u64);
  75         }
  76
  77         if (len & sizeof(u32)) {
  78                 u32 value = get_unaligned_le32(p);
  79
  80                 CRC32C(crc, value, w);
  81                 p += sizeof(u32);
  82         }
  83
  84         if (len & sizeof(u16)) {
  85                 u16 value = get_unaligned_le16(p);
  86
  87                 CRC32C(crc, value, h);
  88                 p += sizeof(u16);
  89         }
  90
  91         if (len & sizeof(u8)) {
  92                 u8 value = *p++;
  93
  94                 CRC32C(crc, value, b);
  95         }
  96
  97         return crc;
  98 }
  99
 100 #define CHKSUM_BLOCK_SIZE       1
 101 #define CHKSUM_DIGEST_SIZE      4
 102
 103 struct chksum_ctx {
 104         u32 key;
 105 };
 106
 107 struct chksum_desc_ctx {
 108         u32 crc;
 109 };
 110
 111 static int chksum_init(struct shash_desc *desc)
 112 {
 113         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 114         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 115
 116         ctx->crc = mctx->key;
 117
 118         return 0;
 119 }
 120
 121 /*
 122  * Setting the seed allows arbitrary accumulators and flexible XOR policy
 123  * If your algorithm starts with ~0, then XOR with ~0 before you set the seed.
 124  */
 125 static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen)
 126 {
 127         struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
 128
 129         if (keylen != sizeof(mctx->key))
 130                 return -EINVAL;
 131
 132         mctx->key = get_unaligned_le32(key);
 133
 134         return 0;
 135 }
 136
 137 static int chksum_update(struct shash_desc *desc, const u8 *data, unsigned int length)
 138 {
 139         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 140
 141         ctx->crc = crc32_loongarch_hw(ctx->crc, data, length);
 142         return 0;
 143 }
 144
 145 static int chksumc_update(struct shash_desc *desc, const u8 *data, unsigned int length)
 146 {
 147         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 148
 149         ctx->crc = crc32c_loongarch_hw(ctx->crc, data, length);
 150         return 0;
 151 }
 152
 153 static int chksum_final(struct shash_desc *desc, u8 *out)
 154 {
 155         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 156
 157         put_unaligned_le32(ctx->crc, out);
 158         return 0;
 159 }
 160
 161 static int chksumc_final(struct shash_desc *desc, u8 *out)
 162 {
 163         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 164
 165         put_unaligned_le32(~ctx->crc, out);
 166         return 0;
 167 }
 168
 169 static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 170 {
 171         put_unaligned_le32(crc32_loongarch_hw(crc, data, len), out);
 172         return 0;
 173 }
 174
 175 static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 176 {
 177         put_unaligned_le32(~crc32c_loongarch_hw(crc, data, len), out);
 178         return 0;
 179 }
 180
 181 static int chksum_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
 182 {
 183         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 184
 185         return __chksum_finup(ctx->crc, data, len, out);
 186 }
 187
 188 static int chksumc_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
 189 {
 190         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 191
 192         return __chksumc_finup(ctx->crc, data, len, out);
 193 }
 194
 195 static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
 196 {
 197         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 198
 199         return __chksum_finup(mctx->key, data, length, out);
 200 }
 201
 202 static int chksumc_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
 203 {
 204         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 205
 206         return __chksumc_finup(mctx->key, data, length, out);
 207 }
 208
 209 static int chksum_cra_init(struct crypto_tfm *tfm)
 210 {
 211         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 212
 213         mctx->key = 0;
 214         return 0;
 215 }
 216
 217 static int chksumc_cra_init(struct crypto_tfm *tfm)
 218 {
 219         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 220
 221         mctx->key = ~0;
 222         return 0;
 223 }
 224
 225 static struct shash_alg crc32_alg = {
 226         .digestsize             =       CHKSUM_DIGEST_SIZE,
 227         .setkey                 =       chksum_setkey,
 228         .init                   =       chksum_init,
 229         .update                 =       chksum_update,
 230         .final                  =       chksum_final,
 231         .finup                  =       chksum_finup,
 232         .digest                 =       chksum_digest,
 233         .descsize               =       sizeof(struct chksum_desc_ctx),
 234         .base                   =       {
 235                 .cra_name               =       "crc32",
 236                 .cra_driver_name        =       "crc32-loongarch",
 237                 .cra_priority           =       300,
 238                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 239                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 240                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 241                 .cra_module             =       THIS_MODULE,
 242                 .cra_init               =       chksum_cra_init,
 243         }
 244 };
 245
 246 static struct shash_alg crc32c_alg = {
 247         .digestsize             =       CHKSUM_DIGEST_SIZE,
 248         .setkey                 =       chksum_setkey,
 249         .init                   =       chksum_init,
 250         .update                 =       chksumc_update,
 251         .final                  =       chksumc_final,
 252         .finup                  =       chksumc_finup,
 253         .digest                 =       chksumc_digest,
 254         .descsize               =       sizeof(struct chksum_desc_ctx),
 255         .base                   =       {
 256                 .cra_name               =       "crc32c",
 257                 .cra_driver_name        =       "crc32c-loongarch",
 258                 .cra_priority           =       300,
 259                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 260                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 261                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 262                 .cra_module             =       THIS_MODULE,
 263                 .cra_init               =       chksumc_cra_init,
 264         }
 265 };
 266
 267 static int __init crc32_mod_init(void)
 268 {
 269         int err;
 270
 271         if (!cpu_has(CPU_FEATURE_CRC32))
 272                 return 0;
 273
 274         err = crypto_register_shash(&crc32_alg);
 275         if (err)
 276                 return err;
 277
 278         err = crypto_register_shash(&crc32c_alg);
 279         if (err)
 280                 return err;
 281
 282         return 0;
 283 }
 284
 285 static void __exit crc32_mod_exit(void)
 286 {
 287         if (!cpu_has(CPU_FEATURE_CRC32))
 288                 return;
 289
 290         crypto_unregister_shash(&crc32_alg);
 291         crypto_unregister_shash(&crc32c_alg);
 292 }
 293
 294 module_init(crc32_mod_init);
 295 module_exit(crc32_mod_exit);
 296
 297 MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
 298 MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
 299 MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
 300 MODULE_LICENSE("GPL v2");