arch/mips/crypto/crc32-mips.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
   4  *
   5  * Module based on arm64/crypto/crc32-arm.c
   6  *
   7  * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
   8  * Copyright (C) 2018 MIPS Tech, LLC
   9  */
  10
  11 #include <linux/cpufeature.h>
  12 #include <linux/init.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/string.h>
  16 #include <asm/mipsregs.h>
  17 #include <linux/unaligned.h>
  18
  19 #include <crypto/internal/hash.h>
  20
  21 enum crc_op_size {
  22         b, h, w, d,
  23 };
  24
  25 enum crc_type {
  26         crc32,
  27         crc32c,
  28 };
  29
  30 #ifndef TOOLCHAIN_SUPPORTS_CRC
  31 #define _ASM_SET_CRC(OP, SZ, TYPE)                                        \
  32 _ASM_MACRO_3R(OP, rt, rs, rt2,                                            \
  33         ".ifnc  \\rt, \\rt2\n\t"                                          \
  34         ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
  35         ".endif\n\t"                                                      \
  36         _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) |      \
  37                           ((SZ) <<  6) | ((TYPE) << 8))                   \
  38         _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) |      \
  39                           ((SZ) << 14) | ((TYPE) << 3)))
  40 #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
  41 #else /* !TOOLCHAIN_SUPPORTS_CRC */
  42 #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
  43 #define _ASM_UNSET_CRC(op, SZ, TYPE)
  44 #endif
  45
  46 #define __CRC32(crc, value, op, SZ, TYPE)               \
  47 do {                                                    \
  48         __asm__ __volatile__(                           \
  49                 ".set   push\n\t"                       \
  50                 _ASM_SET_CRC(op, SZ, TYPE)              \
  51                 #op "   %0, %1, %0\n\t"                 \
  52                 _ASM_UNSET_CRC(op, SZ, TYPE)            \
  53                 ".set   pop"                            \
  54                 : "+r" (crc)                            \
  55                 : "r" (value));                         \
  56 } while (0)
  57
  58 #define _CRC32_crc32b(crc, value)       __CRC32(crc, value, crc32b, 0, 0)
  59 #define _CRC32_crc32h(crc, value)       __CRC32(crc, value, crc32h, 1, 0)
  60 #define _CRC32_crc32w(crc, value)       __CRC32(crc, value, crc32w, 2, 0)
  61 #define _CRC32_crc32d(crc, value)       __CRC32(crc, value, crc32d, 3, 0)
  62 #define _CRC32_crc32cb(crc, value)      __CRC32(crc, value, crc32cb, 0, 1)
  63 #define _CRC32_crc32ch(crc, value)      __CRC32(crc, value, crc32ch, 1, 1)
  64 #define _CRC32_crc32cw(crc, value)      __CRC32(crc, value, crc32cw, 2, 1)
  65 #define _CRC32_crc32cd(crc, value)      __CRC32(crc, value, crc32cd, 3, 1)
  66
  67 #define _CRC32(crc, value, size, op) \
  68         _CRC32_##op##size(crc, value)
  69
  70 #define CRC32(crc, value, size) \
  71         _CRC32(crc, value, size, crc32)
  72
  73 #define CRC32C(crc, value, size) \
  74         _CRC32(crc, value, size, crc32c)
  75
  76 static u32 crc32_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
  77 {
  78         u32 crc = crc_;
  79
  80         if (IS_ENABLED(CONFIG_64BIT)) {
  81                 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
  82                         u64 value = get_unaligned_le64(p);
  83
  84                         CRC32(crc, value, d);
  85                 }
  86
  87                 if (len & sizeof(u32)) {
  88                         u32 value = get_unaligned_le32(p);
  89
  90                         CRC32(crc, value, w);
  91                         p += sizeof(u32);
  92                 }
  93         } else {
  94                 for (; len >= sizeof(u32); len -= sizeof(u32)) {
  95                         u32 value = get_unaligned_le32(p);
  96
  97                         CRC32(crc, value, w);
  98                         p += sizeof(u32);
  99                 }
 100         }
 101
 102         if (len & sizeof(u16)) {
 103                 u16 value = get_unaligned_le16(p);
 104
 105                 CRC32(crc, value, h);
 106                 p += sizeof(u16);
 107         }
 108
 109         if (len & sizeof(u8)) {
 110                 u8 value = *p++;
 111
 112                 CRC32(crc, value, b);
 113         }
 114
 115         return crc;
 116 }
 117
 118 static u32 crc32c_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
 119 {
 120         u32 crc = crc_;
 121
 122         if (IS_ENABLED(CONFIG_64BIT)) {
 123                 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
 124                         u64 value = get_unaligned_le64(p);
 125
 126                         CRC32C(crc, value, d);
 127                 }
 128
 129                 if (len & sizeof(u32)) {
 130                         u32 value = get_unaligned_le32(p);
 131
 132                         CRC32C(crc, value, w);
 133                         p += sizeof(u32);
 134                 }
 135         } else {
 136                 for (; len >= sizeof(u32); len -= sizeof(u32)) {
 137                         u32 value = get_unaligned_le32(p);
 138
 139                         CRC32C(crc, value, w);
 140                         p += sizeof(u32);
 141                 }
 142         }
 143
 144         if (len & sizeof(u16)) {
 145                 u16 value = get_unaligned_le16(p);
 146
 147                 CRC32C(crc, value, h);
 148                 p += sizeof(u16);
 149         }
 150
 151         if (len & sizeof(u8)) {
 152                 u8 value = *p++;
 153
 154                 CRC32C(crc, value, b);
 155         }
 156         return crc;
 157 }
 158
 159 #define CHKSUM_BLOCK_SIZE       1
 160 #define CHKSUM_DIGEST_SIZE      4
 161
 162 struct chksum_ctx {
 163         u32 key;
 164 };
 165
 166 struct chksum_desc_ctx {
 167         u32 crc;
 168 };
 169
 170 static int chksum_init(struct shash_desc *desc)
 171 {
 172         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 173         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 174
 175         ctx->crc = mctx->key;
 176
 177         return 0;
 178 }
 179
 180 /*
 181  * Setting the seed allows arbitrary accumulators and flexible XOR policy
 182  * If your algorithm starts with ~0, then XOR with ~0 before you set
 183  * the seed.
 184  */
 185 static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
 186                          unsigned int keylen)
 187 {
 188         struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
 189
 190         if (keylen != sizeof(mctx->key))
 191                 return -EINVAL;
 192         mctx->key = get_unaligned_le32(key);
 193         return 0;
 194 }
 195
 196 static int chksum_update(struct shash_desc *desc, const u8 *data,
 197                          unsigned int length)
 198 {
 199         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 200
 201         ctx->crc = crc32_mips_le_hw(ctx->crc, data, length);
 202         return 0;
 203 }
 204
 205 static int chksumc_update(struct shash_desc *desc, const u8 *data,
 206                          unsigned int length)
 207 {
 208         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 209
 210         ctx->crc = crc32c_mips_le_hw(ctx->crc, data, length);
 211         return 0;
 212 }
 213
 214 static int chksum_final(struct shash_desc *desc, u8 *out)
 215 {
 216         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 217
 218         put_unaligned_le32(ctx->crc, out);
 219         return 0;
 220 }
 221
 222 static int chksumc_final(struct shash_desc *desc, u8 *out)
 223 {
 224         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 225
 226         put_unaligned_le32(~ctx->crc, out);
 227         return 0;
 228 }
 229
 230 static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 231 {
 232         put_unaligned_le32(crc32_mips_le_hw(crc, data, len), out);
 233         return 0;
 234 }
 235
 236 static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 237 {
 238         put_unaligned_le32(~crc32c_mips_le_hw(crc, data, len), out);
 239         return 0;
 240 }
 241
 242 static int chksum_finup(struct shash_desc *desc, const u8 *data,
 243                         unsigned int len, u8 *out)
 244 {
 245         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 246
 247         return __chksum_finup(ctx->crc, data, len, out);
 248 }
 249
 250 static int chksumc_finup(struct shash_desc *desc, const u8 *data,
 251                         unsigned int len, u8 *out)
 252 {
 253         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 254
 255         return __chksumc_finup(ctx->crc, data, len, out);
 256 }
 257
 258 static int chksum_digest(struct shash_desc *desc, const u8 *data,
 259                          unsigned int length, u8 *out)
 260 {
 261         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 262
 263         return __chksum_finup(mctx->key, data, length, out);
 264 }
 265
 266 static int chksumc_digest(struct shash_desc *desc, const u8 *data,
 267                          unsigned int length, u8 *out)
 268 {
 269         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 270
 271         return __chksumc_finup(mctx->key, data, length, out);
 272 }
 273
 274 static int chksum_cra_init(struct crypto_tfm *tfm)
 275 {
 276         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 277
 278         mctx->key = ~0;
 279         return 0;
 280 }
 281
 282 static struct shash_alg crc32_alg = {
 283         .digestsize             =       CHKSUM_DIGEST_SIZE,
 284         .setkey                 =       chksum_setkey,
 285         .init                   =       chksum_init,
 286         .update                 =       chksum_update,
 287         .final                  =       chksum_final,
 288         .finup                  =       chksum_finup,
 289         .digest                 =       chksum_digest,
 290         .descsize               =       sizeof(struct chksum_desc_ctx),
 291         .base                   =       {
 292                 .cra_name               =       "crc32",
 293                 .cra_driver_name        =       "crc32-mips-hw",
 294                 .cra_priority           =       300,
 295                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 296                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 297                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 298                 .cra_module             =       THIS_MODULE,
 299                 .cra_init               =       chksum_cra_init,
 300         }
 301 };
 302
 303 static struct shash_alg crc32c_alg = {
 304         .digestsize             =       CHKSUM_DIGEST_SIZE,
 305         .setkey                 =       chksum_setkey,
 306         .init                   =       chksum_init,
 307         .update                 =       chksumc_update,
 308         .final                  =       chksumc_final,
 309         .finup                  =       chksumc_finup,
 310         .digest                 =       chksumc_digest,
 311         .descsize               =       sizeof(struct chksum_desc_ctx),
 312         .base                   =       {
 313                 .cra_name               =       "crc32c",
 314                 .cra_driver_name        =       "crc32c-mips-hw",
 315                 .cra_priority           =       300,
 316                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 317                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 318                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 319                 .cra_module             =       THIS_MODULE,
 320                 .cra_init               =       chksum_cra_init,
 321         }
 322 };
 323
 324 static int __init crc32_mod_init(void)
 325 {
 326         int err;
 327
 328         err = crypto_register_shash(&crc32_alg);
 329
 330         if (err)
 331                 return err;
 332
 333         err = crypto_register_shash(&crc32c_alg);
 334
 335         if (err) {
 336                 crypto_unregister_shash(&crc32_alg);
 337                 return err;
 338         }
 339
 340         return 0;
 341 }
 342
 343 static void __exit crc32_mod_exit(void)
 344 {
 345         crypto_unregister_shash(&crc32_alg);
 346         crypto_unregister_shash(&crc32c_alg);
 347 }
 348
 349 MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com");
 350 MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions");
 351 MODULE_LICENSE("GPL v2");
 352
 353 module_cpu_feature_match(MIPS_CRC32, crc32_mod_init);
 354 module_exit(crc32_mod_exit);