?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
CI: Automate some GitHub PR status labels manipulations
[zfs.git] / module / os / linux / zfs / qat_compress.c
blob6d0595dd5f7604fc14fcddaf4bd80b1fccc1fbe3
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 #if defined(_KERNEL) && defined(HAVE_QAT)
23 #include <linux/slab.h>
24 #include <linux/vmalloc.h>
25 #include <linux/pagemap.h>
26 #include <linux/completion.h>
27 #include <sys/zfs_context.h>
28 #include <sys/byteorder.h>
29 #include <sys/zio.h>
30 #include <sys/qat.h>
31
32 /*
33 * Max instances in a QAT device, each instance is a channel to submit
34 * jobs to QAT hardware, this is only for pre-allocating instance and
35 * session arrays; the actual number of instances are defined in the
36 * QAT driver's configuration file.
37 */
38 #define QAT_DC_MAX_INSTANCES 48
39
40 /*
41 * ZLIB head and foot size
42 */
43 #define ZLIB_HEAD_SZ 2
44 #define ZLIB_FOOT_SZ 4
45
46 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
47 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
48 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
49 static Cpa16U num_inst = 0;
50 static Cpa32U inst_num = 0;
51 static boolean_t qat_dc_init_done = B_FALSE;
52 int zfs_qat_compress_disable = 0;
53
54 boolean_t
55 qat_dc_use_accel(size_t s_len)
56 {
57 return (!zfs_qat_compress_disable &&
58 qat_dc_init_done &&
59 s_len >= QAT_MIN_BUF_SIZE &&
60 s_len <= QAT_MAX_BUF_SIZE);
61 }
62
63 static void
64 qat_dc_callback(void *p_callback, CpaStatus status)
65 {
66 if (p_callback != NULL)
67 complete((struct completion *)p_callback);
68 }
69
70 static void
71 qat_dc_clean(void)
72 {
73 Cpa16U buff_num = 0;
74 Cpa16U num_inter_buff_lists = 0;
75
76 for (Cpa16U i = 0; i < num_inst; i++) {
77 cpaDcStopInstance(dc_inst_handles[i]);
78 QAT_PHYS_CONTIG_FREE(session_handles[i]);
79 /* free intermediate buffers */
80 if (buffer_array[i] != NULL) {
81 cpaDcGetNumIntermediateBuffers(
82 dc_inst_handles[i], &num_inter_buff_lists);
83 for (buff_num = 0; buff_num < num_inter_buff_lists;
84 buff_num++) {
85 CpaBufferList *buffer_inter =
86 buffer_array[i][buff_num];
87 if (buffer_inter->pBuffers) {
88 QAT_PHYS_CONTIG_FREE(
89 buffer_inter->pBuffers->pData);
90 QAT_PHYS_CONTIG_FREE(
91 buffer_inter->pBuffers);
92 }
93 QAT_PHYS_CONTIG_FREE(
94 buffer_inter->pPrivateMetaData);
95 QAT_PHYS_CONTIG_FREE(buffer_inter);
96 }
97 }
98 }
99
100 num_inst = 0;
101 qat_dc_init_done = B_FALSE;
102 }
103
104 int
105 qat_dc_init(void)
106 {
107 CpaStatus status = CPA_STATUS_SUCCESS;
108 Cpa32U sess_size = 0;
109 Cpa32U ctx_size = 0;
110 Cpa16U num_inter_buff_lists = 0;
111 Cpa16U buff_num = 0;
112 Cpa32U buff_meta_size = 0;
113 CpaDcSessionSetupData sd = {0};
114
115 if (qat_dc_init_done)
116 return (0);
117
118 status = cpaDcGetNumInstances(&num_inst);
119 if (status != CPA_STATUS_SUCCESS)
120 return (-1);
121
122 /* if the user has configured no QAT compression units just return */
123 if (num_inst == 0)
124 return (0);
125
126 if (num_inst > QAT_DC_MAX_INSTANCES)
127 num_inst = QAT_DC_MAX_INSTANCES;
128
129 status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
130 if (status != CPA_STATUS_SUCCESS)
131 return (-1);
132
133 for (Cpa16U i = 0; i < num_inst; i++) {
134 cpaDcSetAddressTranslation(dc_inst_handles[i],
135 (void*)virt_to_phys);
136
137 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
138 1, &buff_meta_size);
139
140 if (status == CPA_STATUS_SUCCESS)
141 status = cpaDcGetNumIntermediateBuffers(
142 dc_inst_handles[i], &num_inter_buff_lists);
143
144 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
145 status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
146 num_inter_buff_lists *
147 sizeof (CpaBufferList *));
148
149 for (buff_num = 0; buff_num < num_inter_buff_lists;
150 buff_num++) {
151 if (status == CPA_STATUS_SUCCESS)
152 status = QAT_PHYS_CONTIG_ALLOC(
153 &buffer_array[i][buff_num],
154 sizeof (CpaBufferList));
155
156 if (status == CPA_STATUS_SUCCESS)
157 status = QAT_PHYS_CONTIG_ALLOC(
158 &buffer_array[i][buff_num]->
159 pPrivateMetaData,
160 buff_meta_size);
161
162 if (status == CPA_STATUS_SUCCESS)
163 status = QAT_PHYS_CONTIG_ALLOC(
164 &buffer_array[i][buff_num]->pBuffers,
165 sizeof (CpaFlatBuffer));
166
167 if (status == CPA_STATUS_SUCCESS) {
168 /*
169 * implementation requires an intermediate
170 * buffer approximately twice the size of
171 * output buffer, which is 2x max buffer
172 * size here.
173 */
174 status = QAT_PHYS_CONTIG_ALLOC(
175 &buffer_array[i][buff_num]->pBuffers->
176 pData, 2 * QAT_MAX_BUF_SIZE);
177 if (status != CPA_STATUS_SUCCESS)
178 goto fail;
179
180 buffer_array[i][buff_num]->numBuffers = 1;
181 buffer_array[i][buff_num]->pBuffers->
182 dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
183 }
184 }
185
186 status = cpaDcStartInstance(dc_inst_handles[i],
187 num_inter_buff_lists, buffer_array[i]);
188 if (status != CPA_STATUS_SUCCESS)
189 goto fail;
190
191 sd.compLevel = CPA_DC_L1;
192 sd.compType = CPA_DC_DEFLATE;
193 sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
194 sd.sessDirection = CPA_DC_DIR_COMBINED;
195 sd.sessState = CPA_DC_STATELESS;
196 #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6)
197 sd.deflateWindowSize = 7;
198 #endif
199 sd.checksum = CPA_DC_ADLER32;
200 status = cpaDcGetSessionSize(dc_inst_handles[i],
201 &sd, &sess_size, &ctx_size);
202 if (status != CPA_STATUS_SUCCESS)
203 goto fail;
204
205 QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
206 if (session_handles[i] == NULL)
207 goto fail;
208
209 status = cpaDcInitSession(dc_inst_handles[i],
210 session_handles[i],
211 &sd, NULL, qat_dc_callback);
212 if (status != CPA_STATUS_SUCCESS)
213 goto fail;
214 }
215
216 qat_dc_init_done = B_TRUE;
217 return (0);
218 fail:
219 qat_dc_clean();
220 return (-1);
221 }
222
223 void
224 qat_dc_fini(void)
225 {
226 if (!qat_dc_init_done)
227 return;
228
229 qat_dc_clean();
230 }
231
232 /*
233 * The "add" parameter is an additional buffer which is passed
234 * to QAT as a scratch buffer alongside the destination buffer
235 * in case the "compressed" data ends up being larger than the
236 * original source data. This is necessary to prevent QAT from
237 * generating buffer overflow warnings for incompressible data.
238 */
239 static int
240 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
241 char *dst, int dst_len, char *add, int add_len, size_t *c_len)
242 {
243 CpaInstanceHandle dc_inst_handle;
244 CpaDcSessionHandle session_handle;
245 CpaBufferList *buf_list_src = NULL;
246 CpaBufferList *buf_list_dst = NULL;
247 CpaFlatBuffer *flat_buf_src = NULL;
248 CpaFlatBuffer *flat_buf_dst = NULL;
249 Cpa8U *buffer_meta_src = NULL;
250 Cpa8U *buffer_meta_dst = NULL;
251 Cpa32U buffer_meta_size = 0;
252 CpaDcRqResults dc_results = {.checksum = 1};
253 CpaStatus status = CPA_STATUS_FAIL;
254 Cpa32U hdr_sz = 0;
255 Cpa32U compressed_sz;
256 Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
257 Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
258 Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
259 Cpa32U bytes_left;
260 Cpa32U dst_pages = 0;
261 Cpa32U adler32 = 0;
262 char *data;
263 struct page *page;
264 struct page **in_pages = NULL;
265 struct page **out_pages = NULL;
266 struct page **add_pages = NULL;
267 Cpa32U page_off = 0;
268 struct completion complete;
269 Cpa32U page_num = 0;
270 Cpa16U i;
271
272 /*
273 * We increment num_src_buf and num_dst_buf by 2 to allow
274 * us to handle non page-aligned buffer addresses and buffers
275 * whose sizes are not divisible by PAGE_SIZE.
276 */
277 Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
278 (num_src_buf * sizeof (CpaFlatBuffer));
279 Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
280 ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));
281
282 status = QAT_PHYS_CONTIG_ALLOC(&in_pages,
283 num_src_buf * sizeof (struct page *));
284 if (status != CPA_STATUS_SUCCESS)
285 goto fail;
286
287 status = QAT_PHYS_CONTIG_ALLOC(&out_pages,
288 num_dst_buf * sizeof (struct page *));
289 if (status != CPA_STATUS_SUCCESS)
290 goto fail;
291
292 status = QAT_PHYS_CONTIG_ALLOC(&add_pages,
293 num_add_buf * sizeof (struct page *));
294 if (status != CPA_STATUS_SUCCESS)
295 goto fail;
296
297 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
298 dc_inst_handle = dc_inst_handles[i];
299 session_handle = session_handles[i];
300
301 cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
302 &buffer_meta_size);
303 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size);
304 if (status != CPA_STATUS_SUCCESS)
305 goto fail;
306
307 cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
308 &buffer_meta_size);
309 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size);
310 if (status != CPA_STATUS_SUCCESS)
311 goto fail;
312
313 /* build source buffer list */
314 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size);
315 if (status != CPA_STATUS_SUCCESS)
316 goto fail;
317
318 flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
319
320 buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
321
322 /* build destination buffer list */
323 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size);
324 if (status != CPA_STATUS_SUCCESS)
325 goto fail;
326
327 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
328
329 buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
330
331 buf_list_src->numBuffers = 0;
332 buf_list_src->pPrivateMetaData = buffer_meta_src;
333 bytes_left = src_len;
334 data = src;
335 page_num = 0;
336 while (bytes_left > 0) {
337 page_off = ((long)data & ~PAGE_MASK);
338 page = qat_mem_to_page(data);
339 in_pages[page_num] = page;
340 flat_buf_src->pData = kmap(page) + page_off;
341 flat_buf_src->dataLenInBytes =
342 min((long)PAGE_SIZE - page_off, (long)bytes_left);
343
344 bytes_left -= flat_buf_src->dataLenInBytes;
345 data += flat_buf_src->dataLenInBytes;
346 flat_buf_src++;
347 buf_list_src->numBuffers++;
348 page_num++;
349 }
350
351 buf_list_dst->numBuffers = 0;
352 buf_list_dst->pPrivateMetaData = buffer_meta_dst;
353 bytes_left = dst_len;
354 data = dst;
355 page_num = 0;
356 while (bytes_left > 0) {
357 page_off = ((long)data & ~PAGE_MASK);
358 page = qat_mem_to_page(data);
359 flat_buf_dst->pData = kmap(page) + page_off;
360 out_pages[page_num] = page;
361 flat_buf_dst->dataLenInBytes =
362 min((long)PAGE_SIZE - page_off, (long)bytes_left);
363
364 bytes_left -= flat_buf_dst->dataLenInBytes;
365 data += flat_buf_dst->dataLenInBytes;
366 flat_buf_dst++;
367 buf_list_dst->numBuffers++;
368 page_num++;
369 dst_pages++;
370 }
371
372 /* map additional scratch pages into the destination buffer list */
373 bytes_left = add_len;
374 data = add;
375 page_num = 0;
376 while (bytes_left > 0) {
377 page_off = ((long)data & ~PAGE_MASK);
378 page = qat_mem_to_page(data);
379 flat_buf_dst->pData = kmap(page) + page_off;
380 add_pages[page_num] = page;
381 flat_buf_dst->dataLenInBytes =
382 min((long)PAGE_SIZE - page_off, (long)bytes_left);
383
384 bytes_left -= flat_buf_dst->dataLenInBytes;
385 data += flat_buf_dst->dataLenInBytes;
386 flat_buf_dst++;
387 buf_list_dst->numBuffers++;
388 page_num++;
389 }
390
391 init_completion(&complete);
392
393 if (dir == QAT_COMPRESS) {
394 QAT_STAT_BUMP(comp_requests);
395 QAT_STAT_INCR(comp_total_in_bytes, src_len);
396
397 cpaDcGenerateHeader(session_handle,
398 buf_list_dst->pBuffers, &hdr_sz);
399 buf_list_dst->pBuffers->pData += hdr_sz;
400 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
401 status = cpaDcCompressData(
402 dc_inst_handle, session_handle,
403 buf_list_src, buf_list_dst,
404 &dc_results, CPA_DC_FLUSH_FINAL,
405 &complete);
406 if (status != CPA_STATUS_SUCCESS) {
407 goto fail;
408 }
409
410 /* we now wait until the completion of the operation. */
411 wait_for_completion(&complete);
412
413 if (dc_results.status != CPA_STATUS_SUCCESS) {
414 status = CPA_STATUS_FAIL;
415 goto fail;
416 }
417
418 compressed_sz = dc_results.produced;
419 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
420 status = CPA_STATUS_INCOMPRESSIBLE;
421 goto fail;
422 }
423
424 /* get adler32 checksum and append footer */
425 *(Cpa32U*)(dst + hdr_sz + compressed_sz) =
426 BSWAP_32(dc_results.checksum);
427
428 *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ;
429 QAT_STAT_INCR(comp_total_out_bytes, *c_len);
430 } else {
431 ASSERT3U(dir, ==, QAT_DECOMPRESS);
432 QAT_STAT_BUMP(decomp_requests);
433 QAT_STAT_INCR(decomp_total_in_bytes, src_len);
434
435 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
436 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
437 status = cpaDcDecompressData(dc_inst_handle, session_handle,
438 buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
439 &complete);
440
441 if (CPA_STATUS_SUCCESS != status) {
442 status = CPA_STATUS_FAIL;
443 goto fail;
444 }
445
446 /* we now wait until the completion of the operation. */
447 wait_for_completion(&complete);
448
449 if (dc_results.status != CPA_STATUS_SUCCESS) {
450 status = CPA_STATUS_FAIL;
451 goto fail;
452 }
453
454 /* verify adler checksum */
455 adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
456 if (adler32 != BSWAP_32(dc_results.checksum)) {
457 status = CPA_STATUS_FAIL;
458 goto fail;
459 }
460 *c_len = dc_results.produced;
461 QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
462 }
463
464 fail:
465 if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
466 QAT_STAT_BUMP(dc_fails);
467
468 if (in_pages) {
469 for (page_num = 0;
470 page_num < buf_list_src->numBuffers;
471 page_num++) {
472 kunmap(in_pages[page_num]);
473 }
474 QAT_PHYS_CONTIG_FREE(in_pages);
475 }
476
477 if (out_pages) {
478 for (page_num = 0; page_num < dst_pages; page_num++) {
479 kunmap(out_pages[page_num]);
480 }
481 QAT_PHYS_CONTIG_FREE(out_pages);
482 }
483
484 if (add_pages) {
485 for (page_num = 0;
486 page_num < buf_list_dst->numBuffers - dst_pages;
487 page_num++) {
488 kunmap(add_pages[page_num]);
489 }
490 QAT_PHYS_CONTIG_FREE(add_pages);
491 }
492
493 QAT_PHYS_CONTIG_FREE(buffer_meta_src);
494 QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
495 QAT_PHYS_CONTIG_FREE(buf_list_src);
496 QAT_PHYS_CONTIG_FREE(buf_list_dst);
497
498 return (status);
499 }
500
501 /*
502 * Entry point for QAT accelerated compression / decompression.
503 */
504 int
505 qat_compress(qat_compress_dir_t dir, char *src, int src_len,
506 char *dst, int dst_len, size_t *c_len)
507 {
508 int ret;
509 size_t add_len = 0;
510 void *add = NULL;
511
512 if (dir == QAT_COMPRESS) {
513 add_len = dst_len;
514 add = zio_data_buf_alloc(add_len);
515 }
516
517 ret = qat_compress_impl(dir, src, src_len, dst,
518 dst_len, add, add_len, c_len);
519
520 if (dir == QAT_COMPRESS)
521 zio_data_buf_free(add, add_len);
522
523 return (ret);
524 }
525
526 static int
527 param_set_qat_compress(const char *val, zfs_kernel_param_t *kp)
528 {
529 int ret;
530 int *pvalue = kp->arg;
531 ret = param_set_int(val, kp);
532 if (ret)
533 return (ret);
534 /*
535 * zfs_qat_compress_disable = 0: enable qat compress
536 * try to initialize qat instance if it has not been done
537 */
538 if (*pvalue == 0 && !qat_dc_init_done) {
539 ret = qat_dc_init();
540 if (ret != 0) {
541 zfs_qat_compress_disable = 1;
542 return (ret);
543 }
544 }
545 return (ret);
546 }
547
548 module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
549 param_get_int, &zfs_qat_compress_disable, 0644);
550 MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");
551
552 #endif
OpenZFS on Linux and FreeBSD