module/icp/algs/sha2/sha256_impl.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
  24  */
  25
  26 #include <sys/simd.h>
  27 #include <sys/zfs_context.h>
  28 #include <sys/zfs_impl.h>
  29 #include <sys/sha2.h>
  30
  31 #include <sha2/sha2_impl.h>
  32 #include <sys/asm_linkage.h>
  33
  34 #define TF(E, N) \
  35         extern void ASMABI E(uint32_t s[8], const void *, size_t); \
  36         static inline void N(uint32_t s[8], const void *d, size_t b) { \
  37         kfpu_begin(); E(s, d, b); kfpu_end(); \
  38 }
  39
  40 /* some implementation is always okay */
  41 static inline boolean_t sha2_is_supported(void)
  42 {
  43         return (B_TRUE);
  44 }
  45
  46 #if defined(__x86_64)
  47
  48 /* Users of ASMABI requires all calls to be from wrappers */
  49 extern void ASMABI
  50 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
  51
  52 static inline void
  53 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
  54 {
  55         zfs_sha256_transform_x64(s, d, b);
  56 }
  57
  58 const sha256_ops_t sha256_x64_impl = {
  59         .is_supported = sha2_is_supported,
  60         .transform = tf_sha256_transform_x64,
  61         .name = "x64"
  62 };
  63
  64 #if defined(HAVE_SSSE3)
  65 static boolean_t sha2_have_ssse3(void)
  66 {
  67         return (kfpu_allowed() && zfs_ssse3_available());
  68 }
  69
  70 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
  71 const sha256_ops_t sha256_ssse3_impl = {
  72         .is_supported = sha2_have_ssse3,
  73         .transform = tf_sha256_ssse3,
  74         .name = "ssse3"
  75 };
  76 #endif
  77
  78 #if defined(HAVE_AVX)
  79 static boolean_t sha2_have_avx(void)
  80 {
  81         return (kfpu_allowed() && zfs_avx_available());
  82 }
  83
  84 TF(zfs_sha256_transform_avx, tf_sha256_avx);
  85 const sha256_ops_t sha256_avx_impl = {
  86         .is_supported = sha2_have_avx,
  87         .transform = tf_sha256_avx,
  88         .name = "avx"
  89 };
  90 #endif
  91
  92 #if defined(HAVE_AVX2)
  93 static boolean_t sha2_have_avx2(void)
  94 {
  95         return (kfpu_allowed() && zfs_avx2_available());
  96 }
  97
  98 TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
  99 const sha256_ops_t sha256_avx2_impl = {
 100         .is_supported = sha2_have_avx2,
 101         .transform = tf_sha256_avx2,
 102         .name = "avx2"
 103 };
 104 #endif
 105
 106 #if defined(HAVE_SSE4_1)
 107 static boolean_t sha2_have_shani(void)
 108 {
 109         return (kfpu_allowed() && zfs_sse4_1_available() && \
 110             zfs_shani_available());
 111 }
 112
 113 TF(zfs_sha256_transform_shani, tf_sha256_shani);
 114 const sha256_ops_t sha256_shani_impl = {
 115         .is_supported = sha2_have_shani,
 116         .transform = tf_sha256_shani,
 117         .name = "shani"
 118 };
 119 #endif
 120
 121 #elif defined(__aarch64__) || defined(__arm__)
 122 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
 123 const sha256_ops_t sha256_armv7_impl = {
 124         .is_supported = sha2_is_supported,
 125         .transform = zfs_sha256_block_armv7,
 126         .name = "armv7"
 127 };
 128
 129 #if __ARM_ARCH > 6
 130 static boolean_t sha256_have_neon(void)
 131 {
 132         return (kfpu_allowed() && zfs_neon_available());
 133 }
 134
 135 static boolean_t sha256_have_armv8ce(void)
 136 {
 137         return (kfpu_allowed() && zfs_sha256_available());
 138 }
 139
 140 TF(zfs_sha256_block_neon, tf_sha256_neon);
 141 const sha256_ops_t sha256_neon_impl = {
 142         .is_supported = sha256_have_neon,
 143         .transform = tf_sha256_neon,
 144         .name = "neon"
 145 };
 146
 147 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
 148 const sha256_ops_t sha256_armv8_impl = {
 149         .is_supported = sha256_have_armv8ce,
 150         .transform = tf_sha256_armv8ce,
 151         .name = "armv8-ce"
 152 };
 153 #endif
 154
 155 #elif defined(__PPC64__)
 156 static boolean_t sha256_have_isa207(void)
 157 {
 158         return (kfpu_allowed() && zfs_isa207_available());
 159 }
 160
 161 TF(zfs_sha256_ppc, tf_sha256_ppc);
 162 const sha256_ops_t sha256_ppc_impl = {
 163         .is_supported = sha2_is_supported,
 164         .transform = tf_sha256_ppc,
 165         .name = "ppc"
 166 };
 167
 168 TF(zfs_sha256_power8, tf_sha256_power8);
 169 const sha256_ops_t sha256_power8_impl = {
 170         .is_supported = sha256_have_isa207,
 171         .transform = tf_sha256_power8,
 172         .name = "power8"
 173 };
 174 #endif /* __PPC64__ */
 175
 176 /* the two generic ones */
 177 extern const sha256_ops_t sha256_generic_impl;
 178
 179 /* array with all sha256 implementations */
 180 static const sha256_ops_t *const sha256_impls[] = {
 181         &sha256_generic_impl,
 182 #if defined(__x86_64)
 183         &sha256_x64_impl,
 184 #endif
 185 #if defined(__x86_64) && defined(HAVE_SSSE3)
 186         &sha256_ssse3_impl,
 187 #endif
 188 #if defined(__x86_64) && defined(HAVE_AVX)
 189         &sha256_avx_impl,
 190 #endif
 191 #if defined(__x86_64) && defined(HAVE_AVX2)
 192         &sha256_avx2_impl,
 193 #endif
 194 #if defined(__x86_64) && defined(HAVE_SSE4_1)
 195         &sha256_shani_impl,
 196 #endif
 197 #if defined(__aarch64__) || defined(__arm__)
 198         &sha256_armv7_impl,
 199 #if __ARM_ARCH > 6
 200         &sha256_neon_impl,
 201         &sha256_armv8_impl,
 202 #endif
 203 #endif
 204 #if defined(__PPC64__)
 205         &sha256_ppc_impl,
 206         &sha256_power8_impl,
 207 #endif /* __PPC64__ */
 208 };
 209
 210 /* use the generic implementation functions */
 211 #define IMPL_NAME               "sha256"
 212 #define IMPL_OPS_T              sha256_ops_t
 213 #define IMPL_ARRAY              sha256_impls
 214 #define IMPL_GET_OPS            sha256_get_ops
 215 #define ZFS_IMPL_OPS            zfs_sha256_ops
 216 #include <generic_impl.c>
 217
 218 #ifdef _KERNEL
 219
 220 #define IMPL_FMT(impl, i)       (((impl) == (i)) ? "[%s] " : "%s ")
 221
 222 #if defined(__linux__)
 223
 224 static int
 225 sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
 226 {
 227         const uint32_t impl = IMPL_READ(generic_impl_chosen);
 228         char *fmt;
 229         int cnt = 0;
 230
 231         /* cycling */
 232         fmt = IMPL_FMT(impl, IMPL_CYCLE);
 233         cnt += sprintf(buffer + cnt, fmt, "cycle");
 234
 235         /* list fastest */
 236         fmt = IMPL_FMT(impl, IMPL_FASTEST);
 237         cnt += sprintf(buffer + cnt, fmt, "fastest");
 238
 239         /* list all supported implementations */
 240         generic_impl_init();
 241         for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
 242                 fmt = IMPL_FMT(impl, i);
 243                 cnt += sprintf(buffer + cnt, fmt,
 244                     generic_supp_impls[i]->name);
 245         }
 246
 247         return (cnt);
 248 }
 249
 250 static int
 251 sha256_param_set(const char *val, zfs_kernel_param_t *unused)
 252 {
 253         (void) unused;
 254         return (generic_impl_setname(val));
 255 }
 256
 257 #elif defined(__FreeBSD__)
 258
 259 #include <sys/sbuf.h>
 260
 261 static int
 262 sha256_param(ZFS_MODULE_PARAM_ARGS)
 263 {
 264         int err;
 265
 266         generic_impl_init();
 267         if (req->newptr == NULL) {
 268                 const uint32_t impl = IMPL_READ(generic_impl_chosen);
 269                 const int init_buflen = 64;
 270                 const char *fmt;
 271                 struct sbuf *s;
 272
 273                 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
 274
 275                 /* cycling */
 276                 fmt = IMPL_FMT(impl, IMPL_CYCLE);
 277                 (void) sbuf_printf(s, fmt, "cycle");
 278
 279                 /* list fastest */
 280                 fmt = IMPL_FMT(impl, IMPL_FASTEST);
 281                 (void) sbuf_printf(s, fmt, "fastest");
 282
 283                 /* list all supported implementations */
 284                 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
 285                         fmt = IMPL_FMT(impl, i);
 286                         (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
 287                 }
 288
 289                 err = sbuf_finish(s);
 290                 sbuf_delete(s);
 291
 292                 return (err);
 293         }
 294
 295         char buf[16];
 296
 297         err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
 298         if (err) {
 299                 return (err);
 300         }
 301
 302         return (-generic_impl_setname(buf));
 303 }
 304 #endif
 305
 306 #undef IMPL_FMT
 307
 308 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
 309     sha256_param_set, sha256_param_get, ZMOD_RW, \
 310         "Select SHA256 implementation.");
 311 #endif
 312
 313 #undef TF