arch/arm64/lib/crc32-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 #include <linux/crc32.h>
   4 #include <linux/linkage.h>
   5
   6 #include <asm/alternative.h>
   7 #include <asm/cpufeature.h>
   8 #include <asm/neon.h>
   9 #include <asm/simd.h>
  10
  11 #include <crypto/internal/simd.h>
  12
  13 // The minimum input length to consider the 4-way interleaved code path
  14 static const size_t min_len = 1024;
  15
  16 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len);
  17 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len);
  18 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len);
  19
  20 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
  21 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
  22 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
  23
  24 u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len)
  25 {
  26         if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
  27                 return crc32_le_base(crc, p, len);
  28
  29         if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
  30                 kernel_neon_begin();
  31                 crc = crc32_le_arm64_4way(crc, p, len);
  32                 kernel_neon_end();
  33
  34                 p += round_down(len, 64);
  35                 len %= 64;
  36
  37                 if (!len)
  38                         return crc;
  39         }
  40
  41         return crc32_le_arm64(crc, p, len);
  42 }
  43
  44 u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
  45 {
  46         if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
  47                 return __crc32c_le_base(crc, p, len);
  48
  49         if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
  50                 kernel_neon_begin();
  51                 crc = crc32c_le_arm64_4way(crc, p, len);
  52                 kernel_neon_end();
  53
  54                 p += round_down(len, 64);
  55                 len %= 64;
  56
  57                 if (!len)
  58                         return crc;
  59         }
  60
  61         return crc32c_le_arm64(crc, p, len);
  62 }
  63
  64 u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
  65 {
  66         if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
  67                 return crc32_be_base(crc, p, len);
  68
  69         if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
  70                 kernel_neon_begin();
  71                 crc = crc32_be_arm64_4way(crc, p, len);
  72                 kernel_neon_end();
  73
  74                 p += round_down(len, 64);
  75                 len %= 64;
  76
  77                 if (!len)
  78                         return crc;
  79         }
  80
  81         return crc32_be_arm64(crc, p, len);
  82 }