module/zcommon/zfs_fletcher_avx512.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
  23  */
  24
  25 #if defined(__x86_64) && defined(HAVE_AVX512F)
  26
  27 #include <sys/byteorder.h>
  28 #include <sys/frame.h>
  29 #include <sys/spa_checksum.h>
  30 #include <sys/string.h>
  31 #include <sys/simd.h>
  32 #include <zfs_fletcher.h>
  33
  34 #ifdef __linux__
  35 #define __asm __asm__ __volatile__
  36 #endif
  37
  38 ZFS_NO_SANITIZE_UNDEFINED
  39 static void
  40 fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx)
  41 {
  42         memset(ctx->avx512, 0, 4 * sizeof (zfs_fletcher_avx512_t));
  43 }
  44
  45 ZFS_NO_SANITIZE_UNDEFINED
  46 static void
  47 fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
  48 {
  49         static const uint64_t
  50         CcA[] = {   0,   0,   1,   3,   6,  10,  15,  21 },
  51         CcB[] = {  28,  36,  44,  52,  60,  68,  76,  84 },
  52         DcA[] = {   0,   0,   0,   1,   4,  10,  20,  35 },
  53         DcB[] = {  56,  84, 120, 164, 216, 276, 344, 420 },
  54         DcC[] = { 448, 512, 576, 640, 704, 768, 832, 896 };
  55
  56         uint64_t A, B, C, D;
  57         uint64_t i;
  58
  59         A = ctx->avx512[0].v[0];
  60         B = 8 * ctx->avx512[1].v[0];
  61         C = 64 * ctx->avx512[2].v[0] - CcB[0] * ctx->avx512[1].v[0];
  62         D = 512 * ctx->avx512[3].v[0] - DcC[0] * ctx->avx512[2].v[0] +
  63             DcB[0] * ctx->avx512[1].v[0];
  64
  65         for (i = 1; i < 8; i++) {
  66                 A += ctx->avx512[0].v[i];
  67                 B += 8 * ctx->avx512[1].v[i] - i * ctx->avx512[0].v[i];
  68                 C += 64 * ctx->avx512[2].v[i] - CcB[i] * ctx->avx512[1].v[i] +
  69                     CcA[i] * ctx->avx512[0].v[i];
  70                 D += 512 * ctx->avx512[3].v[i] - DcC[i] * ctx->avx512[2].v[i] +
  71                     DcB[i] * ctx->avx512[1].v[i] - DcA[i] * ctx->avx512[0].v[i];
  72         }
  73
  74         ZIO_SET_CHECKSUM(zcp, A, B, C, D);
  75 }
  76
  77 #define FLETCHER_4_AVX512_RESTORE_CTX(ctx)                              \
  78 {                                                                       \
  79         __asm("vmovdqu64 %0, %%zmm0" :: "m" ((ctx)->avx512[0]));        \
  80         __asm("vmovdqu64 %0, %%zmm1" :: "m" ((ctx)->avx512[1]));        \
  81         __asm("vmovdqu64 %0, %%zmm2" :: "m" ((ctx)->avx512[2]));        \
  82         __asm("vmovdqu64 %0, %%zmm3" :: "m" ((ctx)->avx512[3]));        \
  83 }
  84
  85 #define FLETCHER_4_AVX512_SAVE_CTX(ctx)                                 \
  86 {                                                                       \
  87         __asm("vmovdqu64 %%zmm0, %0" : "=m" ((ctx)->avx512[0]));        \
  88         __asm("vmovdqu64 %%zmm1, %0" : "=m" ((ctx)->avx512[1]));        \
  89         __asm("vmovdqu64 %%zmm2, %0" : "=m" ((ctx)->avx512[2]));        \
  90         __asm("vmovdqu64 %%zmm3, %0" : "=m" ((ctx)->avx512[3]));        \
  91 }
  92
  93 static void
  94 fletcher_4_avx512f_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
  95 {
  96         const uint32_t *ip = buf;
  97         const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);
  98
  99         kfpu_begin();
 100
 101         FLETCHER_4_AVX512_RESTORE_CTX(ctx);
 102
 103         for (; ip < ipend; ip += 8) {
 104                 __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip));
 105                 __asm("vpaddq %zmm4, %zmm0, %zmm0");
 106                 __asm("vpaddq %zmm0, %zmm1, %zmm1");
 107                 __asm("vpaddq %zmm1, %zmm2, %zmm2");
 108                 __asm("vpaddq %zmm2, %zmm3, %zmm3");
 109         }
 110
 111         FLETCHER_4_AVX512_SAVE_CTX(ctx);
 112
 113         kfpu_end();
 114 }
 115 STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_native);
 116
 117 static void
 118 fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
 119     uint64_t size)
 120 {
 121         static const uint64_t byteswap_mask = 0xFFULL;
 122         const uint32_t *ip = buf;
 123         const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);
 124
 125         kfpu_begin();
 126
 127         FLETCHER_4_AVX512_RESTORE_CTX(ctx);
 128
 129         __asm("vpbroadcastq %0, %%zmm8" :: "r" (byteswap_mask));
 130         __asm("vpsllq $8, %zmm8, %zmm9");
 131         __asm("vpsllq $16, %zmm8, %zmm10");
 132         __asm("vpsllq $24, %zmm8, %zmm11");
 133
 134         for (; ip < ipend; ip += 8) {
 135                 __asm("vpmovzxdq %0, %%zmm5"::"m" (*ip));
 136
 137                 __asm("vpsrlq $24, %zmm5, %zmm6");
 138                 __asm("vpandd %zmm8, %zmm6, %zmm6");
 139                 __asm("vpsrlq $8, %zmm5, %zmm7");
 140                 __asm("vpandd %zmm9, %zmm7, %zmm7");
 141                 __asm("vpord %zmm6, %zmm7, %zmm4");
 142                 __asm("vpsllq $8, %zmm5, %zmm6");
 143                 __asm("vpandd %zmm10, %zmm6, %zmm6");
 144                 __asm("vpord %zmm6, %zmm4, %zmm4");
 145                 __asm("vpsllq $24, %zmm5, %zmm5");
 146                 __asm("vpandd %zmm11, %zmm5, %zmm5");
 147                 __asm("vpord %zmm5, %zmm4, %zmm4");
 148
 149                 __asm("vpaddq %zmm4, %zmm0, %zmm0");
 150                 __asm("vpaddq %zmm0, %zmm1, %zmm1");
 151                 __asm("vpaddq %zmm1, %zmm2, %zmm2");
 152                 __asm("vpaddq %zmm2, %zmm3, %zmm3");
 153         }
 154
 155         FLETCHER_4_AVX512_SAVE_CTX(ctx)
 156
 157         kfpu_end();
 158 }
 159 STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_byteswap);
 160
 161 static boolean_t
 162 fletcher_4_avx512f_valid(void)
 163 {
 164         return (kfpu_allowed() && zfs_avx512f_available());
 165 }
 166
 167 const fletcher_4_ops_t fletcher_4_avx512f_ops = {
 168         .init_native = fletcher_4_avx512f_init,
 169         .fini_native = fletcher_4_avx512f_fini,
 170         .compute_native = fletcher_4_avx512f_native,
 171         .init_byteswap = fletcher_4_avx512f_init,
 172         .fini_byteswap = fletcher_4_avx512f_fini,
 173         .compute_byteswap = fletcher_4_avx512f_byteswap,
 174         .valid = fletcher_4_avx512f_valid,
 175         .name = "avx512f"
 176 };
 177
 178 #if defined(HAVE_AVX512BW)
 179 static void
 180 fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
 181     uint64_t size)
 182 {
 183         static const zfs_fletcher_avx512_t mask = {
 184                 .v = { 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B,
 185                 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B,
 186                 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B,
 187                 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B }
 188         };
 189         const uint32_t *ip = buf;
 190         const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);
 191
 192         kfpu_begin();
 193
 194         FLETCHER_4_AVX512_RESTORE_CTX(ctx);
 195
 196         __asm("vmovdqu64 %0, %%zmm5" :: "m" (mask));
 197
 198         for (; ip < ipend; ip += 8) {
 199                 __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip));
 200
 201                 __asm("vpshufb %zmm5, %zmm4, %zmm4");
 202
 203                 __asm("vpaddq %zmm4, %zmm0, %zmm0");
 204                 __asm("vpaddq %zmm0, %zmm1, %zmm1");
 205                 __asm("vpaddq %zmm1, %zmm2, %zmm2");
 206                 __asm("vpaddq %zmm2, %zmm3, %zmm3");
 207         }
 208
 209         FLETCHER_4_AVX512_SAVE_CTX(ctx)
 210
 211         kfpu_end();
 212 }
 213 STACK_FRAME_NON_STANDARD(fletcher_4_avx512bw_byteswap);
 214
 215 static boolean_t
 216 fletcher_4_avx512bw_valid(void)
 217 {
 218         return (fletcher_4_avx512f_valid() && zfs_avx512bw_available());
 219 }
 220
 221 const fletcher_4_ops_t fletcher_4_avx512bw_ops = {
 222         .init_native = fletcher_4_avx512f_init,
 223         .fini_native = fletcher_4_avx512f_fini,
 224         .compute_native = fletcher_4_avx512f_native,
 225         .init_byteswap = fletcher_4_avx512f_init,
 226         .fini_byteswap = fletcher_4_avx512f_fini,
 227         .compute_byteswap = fletcher_4_avx512bw_byteswap,
 228         .valid = fletcher_4_avx512bw_valid,
 229         .name = "avx512bw"
 230 };
 231 #endif
 232
 233 #endif /* defined(__x86_64) && defined(HAVE_AVX512F) */