?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ABD: Adapt avx512bw raidz assembly
[zfs.git] / module / zfs / vdev_raidz_math_avx512bw.c
blob3d5326b9e6e161aae77b61461d1af7e300f267df
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (C) 2016 Romain Dolbeau. All rights reserved.
23 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
24 */
25
26 #include <sys/isa_defs.h>
27
28 #if defined(__x86_64) && defined(HAVE_AVX512BW)
29
30 #include <sys/types.h>
31 #include <linux/simd_x86.h>
32
33 #define __asm __asm__ __volatile__
34
35 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
36 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
37
38 #define VR0_(REG, ...) "zmm"#REG
39 #define VR1_(_1, REG, ...) "zmm"#REG
40 #define VR2_(_1, _2, REG, ...) "zmm"#REG
41 #define VR3_(_1, _2, _3, REG, ...) "zmm"#REG
42 #define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG
43 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG
44 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG
45 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG
46
47 #define VR0(r...) VR0_(r)
48 #define VR1(r...) VR1_(r)
49 #define VR2(r...) VR2_(r, 1)
50 #define VR3(r...) VR3_(r, 1, 2)
51 #define VR4(r...) VR4_(r, 1, 2)
52 #define VR5(r...) VR5_(r, 1, 2, 3)
53 #define VR6(r...) VR6_(r, 1, 2, 3, 4)
54 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
55
56 #define R_01(REG1, REG2, ...) REG1, REG2
57 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
58 #define R_23(REG...) _R_23(REG, 1, 2, 3)
59
60 #define ASM_BUG() ASSERT(0)
61
62 extern const uint8_t gf_clmul_mod_lt[4*256][16];
63
64 #define ELEM_SIZE 64
65
66 typedef struct v {
67 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
68 } v_t;
69
70 #define XOR_ACC(src, r...) \
71 { \
72 switch (REG_CNT(r)) { \
73 case 4: \
74 __asm( \
75 "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
76 "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
77 "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \
78 "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \
79 : : [SRC] "r" (src)); \
80 break; \
81 case 2: \
82 __asm( \
83 "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
84 "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
85 : : [SRC] "r" (src)); \
86 break; \
87 default: \
88 ASM_BUG(); \
89 } \
90 }
91
92 #define XOR(r...) \
93 { \
94 switch (REG_CNT(r)) { \
95 case 8: \
96 __asm( \
97 "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \
98 "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \
99 "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \
100 "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \
101 break; \
102 case 4: \
103 __asm( \
104 "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \
105 "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \
106 break; \
107 default: \
108 ASM_BUG(); \
109 } \
110 }
111
112 #define ZERO(r...) XOR(r, r)
113
114 #define COPY(r...) \
115 { \
116 switch (REG_CNT(r)) { \
117 case 8: \
118 __asm( \
119 "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \
120 "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \
121 "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \
122 "vmovdqa64 %" VR3(r) ", %" VR7(r)); \
123 break; \
124 case 4: \
125 __asm( \
126 "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \
127 "vmovdqa64 %" VR1(r) ", %" VR3(r)); \
128 break; \
129 default: \
130 ASM_BUG(); \
131 } \
132 }
133
134 #define LOAD(src, r...) \
135 { \
136 switch (REG_CNT(r)) { \
137 case 4: \
138 __asm( \
139 "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \
140 "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \
141 "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \
142 "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \
143 : : [SRC] "r" (src)); \
144 break; \
145 case 2: \
146 __asm( \
147 "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \
148 "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \
149 : : [SRC] "r" (src)); \
150 break; \
151 default: \
152 ASM_BUG(); \
153 } \
154 }
155
156 #define STORE(dst, r...) \
157 { \
158 switch (REG_CNT(r)) { \
159 case 4: \
160 __asm( \
161 "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \
162 "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \
163 "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \
164 "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \
165 : : [DST] "r" (dst)); \
166 break; \
167 case 2: \
168 __asm( \
169 "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \
170 "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \
171 : : [DST] "r" (dst)); \
172 break; \
173 default: \
174 ASM_BUG(); \
175 } \
176 }
177
178 #define MUL2_SETUP() \
179 { \
180 __asm("vmovq %0, %%xmm22" :: "r"(0x1d1d1d1d1d1d1d1d)); \
181 __asm("vpbroadcastq %xmm22, %zmm22"); \
182 __asm("vpxord %zmm23, %zmm23 ,%zmm23"); \
183 }
184
185 #define _MUL2(r...) \
186 { \
187 switch (REG_CNT(r)) { \
188 case 2: \
189 __asm( \
190 "vpcmpb $1, %zmm23, %" VR0(r)", %k1\n" \
191 "vpcmpb $1, %zmm23, %" VR1(r)", %k2\n" \
192 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \
193 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \
194 "vpxord %zmm22, %" VR0(r)", %zmm12\n" \
195 "vpxord %zmm22, %" VR1(r)", %zmm13\n" \
196 "vmovdqu8 %zmm12, %" VR0(r) "{%k1}\n" \
197 "vmovdqu8 %zmm13, %" VR1(r) "{%k2}"); \
198 break; \
199 default: \
200 ASM_BUG(); \
201 } \
202 }
203
204 #define MUL2(r...) \
205 { \
206 switch (REG_CNT(r)) { \
207 case 4: \
208 _MUL2(R_01(r)); \
209 _MUL2(R_23(r)); \
210 break; \
211 case 2: \
212 _MUL2(r); \
213 break; \
214 default: \
215 ASM_BUG(); \
216 } \
217 }
218
219 #define MUL4(r...) \
220 { \
221 MUL2(r); \
222 MUL2(r); \
223 }
224
225 #define _0f "zmm15"
226 #define _as "zmm14"
227 #define _bs "zmm13"
228 #define _ltmod "zmm12"
229 #define _ltmul "zmm11"
230 #define _ta "zmm10"
231 #define _tb "zmm15"
232
233 static const uint8_t __attribute__((aligned(64))) _mul_mask = 0x0F;
234
235 #define _MULx2(c, r...) \
236 { \
237 switch (REG_CNT(r)) { \
238 case 2: \
239 __asm( \
240 "vpbroadcastb (%[mask]), %%" _0f "\n" \
241 /* upper bits */ \
242 "vbroadcasti32x4 0x00(%[lt]), %%" _ltmod "\n" \
243 "vbroadcasti32x4 0x10(%[lt]), %%" _ltmul "\n" \
244 \
245 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \
246 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \
247 "vpandq %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \
248 "vpandq %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \
249 "vpandq %%" _0f ", %%" _as ", %%" _as "\n" \
250 "vpandq %%" _0f ", %%" _bs ", %%" _bs "\n" \
251 \
252 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \
253 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \
254 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \
255 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \
256 /* lower bits */ \
257 "vbroadcasti32x4 0x20(%[lt]), %%" _ltmod "\n" \
258 "vbroadcasti32x4 0x30(%[lt]), %%" _ltmul "\n" \
259 \
260 "vpxorq %%" _ta ", %%" _as ", %%" _as "\n" \
261 "vpxorq %%" _tb ", %%" _bs ", %%" _bs "\n" \
262 \
263 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \
264 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \
265 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
266 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
267 \
268 "vpxorq %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \
269 "vpxorq %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \
270 "vpxorq %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \
271 "vpxorq %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \
272 : : [mask] "r" (&_mul_mask), \
273 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \
274 break; \
275 default: \
276 ASM_BUG(); \
277 } \
278 }
279
280 #define MUL(c, r...) \
281 { \
282 switch (REG_CNT(r)) { \
283 case 4: \
284 _MULx2(c, R_01(r)); \
285 _MULx2(c, R_23(r)); \
286 break; \
287 case 2: \
288 _MULx2(c, R_01(r)); \
289 break; \
290 default: \
291 ASM_BUG(); \
292 } \
293 }
294
295 #define raidz_math_begin() kfpu_begin()
296 #define raidz_math_end() kfpu_end()
297
298 /*
299 * ZERO, COPY, and MUL operations are already 2x unrolled, which means that
300 * the stride of these operations for avx512 must not exceed 4. Otherwise, a
301 * single step would exceed 512B block size.
302 */
303
304 #define SYN_STRIDE 4
305
306 #define ZERO_STRIDE 4
307 #define ZERO_DEFINE() {}
308 #define ZERO_D 0, 1, 2, 3
309
310 #define COPY_STRIDE 4
311 #define COPY_DEFINE() {}
312 #define COPY_D 0, 1, 2, 3
313
314 #define ADD_STRIDE 4
315 #define ADD_DEFINE() {}
316 #define ADD_D 0, 1, 2, 3
317
318 #define MUL_STRIDE 4
319 #define MUL_DEFINE() {}
320 #define MUL_D 0, 1, 2, 3
321
322 #define GEN_P_STRIDE 4
323 #define GEN_P_DEFINE() {}
324 #define GEN_P_P 0, 1, 2, 3
325
326 #define GEN_PQ_STRIDE 4
327 #define GEN_PQ_DEFINE() {}
328 #define GEN_PQ_D 0, 1, 2, 3
329 #define GEN_PQ_C 4, 5, 6, 7
330
331 #define GEN_PQR_STRIDE 4
332 #define GEN_PQR_DEFINE() {}
333 #define GEN_PQR_D 0, 1, 2, 3
334 #define GEN_PQR_C 4, 5, 6, 7
335
336 #define SYN_Q_DEFINE() {}
337 #define SYN_Q_D 0, 1, 2, 3
338 #define SYN_Q_X 4, 5, 6, 7
339
340 #define SYN_R_DEFINE() {}
341 #define SYN_R_D 0, 1, 2, 3
342 #define SYN_R_X 4, 5, 6, 7
343
344 #define SYN_PQ_DEFINE() {}
345 #define SYN_PQ_D 0, 1, 2, 3
346 #define SYN_PQ_X 4, 5, 6, 7
347
348 #define REC_PQ_STRIDE 2
349 #define REC_PQ_DEFINE() {}
350 #define REC_PQ_X 0, 1
351 #define REC_PQ_Y 2, 3
352 #define REC_PQ_T 4, 5
353
354 #define SYN_PR_DEFINE() {}
355 #define SYN_PR_D 0, 1, 2, 3
356 #define SYN_PR_X 4, 5, 6, 7
357
358 #define REC_PR_STRIDE 2
359 #define REC_PR_DEFINE() {}
360 #define REC_PR_X 0, 1
361 #define REC_PR_Y 2, 3
362 #define REC_PR_T 4, 5
363
364 #define SYN_QR_DEFINE() {}
365 #define SYN_QR_D 0, 1, 2, 3
366 #define SYN_QR_X 4, 5, 6, 7
367
368 #define REC_QR_STRIDE 2
369 #define REC_QR_DEFINE() {}
370 #define REC_QR_X 0, 1
371 #define REC_QR_Y 2, 3
372 #define REC_QR_T 4, 5
373
374 #define SYN_PQR_DEFINE() {}
375 #define SYN_PQR_D 0, 1, 2, 3
376 #define SYN_PQR_X 4, 5, 6, 7
377
378 #define REC_PQR_STRIDE 2
379 #define REC_PQR_DEFINE() {}
380 #define REC_PQR_X 0, 1
381 #define REC_PQR_Y 2, 3
382 #define REC_PQR_Z 4, 5
383 #define REC_PQR_XS 6, 7
384 #define REC_PQR_YS 8, 9
385
386
387 #include <sys/vdev_raidz_impl.h>
388 #include "vdev_raidz_math_impl.h"
389
390 DEFINE_GEN_METHODS(avx512bw);
391 DEFINE_REC_METHODS(avx512bw);
392
393 static boolean_t
394 raidz_will_avx512bw_work(void)
395 {
396 return (zfs_avx_available() &&
397 zfs_avx512f_available() &&
398 zfs_avx512bw_available());
399 }
400
401 const raidz_impl_ops_t vdev_raidz_avx512bw_impl = {
402 .init = NULL,
403 .fini = NULL,
404 .gen = RAIDZ_GEN_METHODS(avx512bw),
405 .rec = RAIDZ_REC_METHODS(avx512bw),
406 .is_supported = &raidz_will_avx512bw_work,
407 .name = "avx512bw"
408 };
409
410 #endif /* defined(__x86_64) && defined(HAVE_AVX512BW) */
OpenZFS on Linux and FreeBSD