crypto/async_tx/async_raid6_recov.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
   4  * Copyright(c) 2009 Intel Corporation
   5  *
   6  * based on raid6recov.c:
   7  *   Copyright 2002 H. Peter Anvin
   8  */
   9 #include <linux/kernel.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/module.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/raid/pq.h>
  14 #include <linux/async_tx.h>
  15 #include <linux/dmaengine.h>
  16
  17 static struct dma_async_tx_descriptor *
  18 async_sum_product(struct page *dest, unsigned int d_off,
  19                 struct page **srcs, unsigned int *src_offs, unsigned char *coef,
  20                 size_t len, struct async_submit_ctl *submit)
  21 {
  22         struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
  23                                                       &dest, 1, srcs, 2, len);
  24         struct dma_device *dma = chan ? chan->device : NULL;
  25         struct dmaengine_unmap_data *unmap = NULL;
  26         const u8 *amul, *bmul;
  27         u8 ax, bx;
  28         u8 *a, *b, *c;
  29
  30         if (dma)
  31                 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
  32
  33         if (unmap) {
  34                 struct device *dev = dma->dev;
  35                 dma_addr_t pq[2];
  36                 struct dma_async_tx_descriptor *tx;
  37                 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
  38
  39                 if (submit->flags & ASYNC_TX_FENCE)
  40                         dma_flags |= DMA_PREP_FENCE;
  41                 unmap->addr[0] = dma_map_page(dev, srcs[0], src_offs[0],
  42                                                 len, DMA_TO_DEVICE);
  43                 unmap->addr[1] = dma_map_page(dev, srcs[1], src_offs[1],
  44                                                 len, DMA_TO_DEVICE);
  45                 unmap->to_cnt = 2;
  46
  47                 unmap->addr[2] = dma_map_page(dev, dest, d_off,
  48                                                 len, DMA_BIDIRECTIONAL);
  49                 unmap->bidi_cnt = 1;
  50                 /* engine only looks at Q, but expects it to follow P */
  51                 pq[1] = unmap->addr[2];
  52
  53                 unmap->len = len;
  54                 tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
  55                                              len, dma_flags);
  56                 if (tx) {
  57                         dma_set_unmap(tx, unmap);
  58                         async_tx_submit(chan, tx, submit);
  59                         dmaengine_unmap_put(unmap);
  60                         return tx;
  61                 }
  62
  63                 /* could not get a descriptor, unmap and fall through to
  64                  * the synchronous path
  65                  */
  66                 dmaengine_unmap_put(unmap);
  67         }
  68
  69         /* run the operation synchronously */
  70         async_tx_quiesce(&submit->depend_tx);
  71         amul = raid6_gfmul[coef[0]];
  72         bmul = raid6_gfmul[coef[1]];
  73         a = page_address(srcs[0]) + src_offs[0];
  74         b = page_address(srcs[1]) + src_offs[1];
  75         c = page_address(dest) + d_off;
  76
  77         while (len--) {
  78                 ax    = amul[*a++];
  79                 bx    = bmul[*b++];
  80                 *c++ = ax ^ bx;
  81         }
  82
  83         return NULL;
  84 }
  85
  86 static struct dma_async_tx_descriptor *
  87 async_mult(struct page *dest, unsigned int d_off, struct page *src,
  88                 unsigned int s_off, u8 coef, size_t len,
  89                 struct async_submit_ctl *submit)
  90 {
  91         struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
  92                                                       &dest, 1, &src, 1, len);
  93         struct dma_device *dma = chan ? chan->device : NULL;
  94         struct dmaengine_unmap_data *unmap = NULL;
  95         const u8 *qmul; /* Q multiplier table */
  96         u8 *d, *s;
  97
  98         if (dma)
  99                 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
 100
 101         if (unmap) {
 102                 dma_addr_t dma_dest[2];
 103                 struct device *dev = dma->dev;
 104                 struct dma_async_tx_descriptor *tx;
 105                 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
 106
 107                 if (submit->flags & ASYNC_TX_FENCE)
 108                         dma_flags |= DMA_PREP_FENCE;
 109                 unmap->addr[0] = dma_map_page(dev, src, s_off,
 110                                                 len, DMA_TO_DEVICE);
 111                 unmap->to_cnt++;
 112                 unmap->addr[1] = dma_map_page(dev, dest, d_off,
 113                                                 len, DMA_BIDIRECTIONAL);
 114                 dma_dest[1] = unmap->addr[1];
 115                 unmap->bidi_cnt++;
 116                 unmap->len = len;
 117
 118                 /* this looks funny, but the engine looks for Q at
 119                  * dma_dest[1] and ignores dma_dest[0] as a dest
 120                  * due to DMA_PREP_PQ_DISABLE_P
 121                  */
 122                 tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
 123                                              1, &coef, len, dma_flags);
 124
 125                 if (tx) {
 126                         dma_set_unmap(tx, unmap);
 127                         dmaengine_unmap_put(unmap);
 128                         async_tx_submit(chan, tx, submit);
 129                         return tx;
 130                 }
 131
 132                 /* could not get a descriptor, unmap and fall through to
 133                  * the synchronous path
 134                  */
 135                 dmaengine_unmap_put(unmap);
 136         }
 137
 138         /* no channel available, or failed to allocate a descriptor, so
 139          * perform the operation synchronously
 140          */
 141         async_tx_quiesce(&submit->depend_tx);
 142         qmul  = raid6_gfmul[coef];
 143         d = page_address(dest) + d_off;
 144         s = page_address(src) + s_off;
 145
 146         while (len--)
 147                 *d++ = qmul[*s++];
 148
 149         return NULL;
 150 }
 151
 152 static struct dma_async_tx_descriptor *
 153 __2data_recov_4(int disks, size_t bytes, int faila, int failb,
 154                 struct page **blocks, unsigned int *offs,
 155                 struct async_submit_ctl *submit)
 156 {
 157         struct dma_async_tx_descriptor *tx = NULL;
 158         struct page *p, *q, *a, *b;
 159         unsigned int p_off, q_off, a_off, b_off;
 160         struct page *srcs[2];
 161         unsigned int src_offs[2];
 162         unsigned char coef[2];
 163         enum async_tx_flags flags = submit->flags;
 164         dma_async_tx_callback cb_fn = submit->cb_fn;
 165         void *cb_param = submit->cb_param;
 166         void *scribble = submit->scribble;
 167
 168         p = blocks[disks-2];
 169         p_off = offs[disks-2];
 170         q = blocks[disks-1];
 171         q_off = offs[disks-1];
 172
 173         a = blocks[faila];
 174         a_off = offs[faila];
 175         b = blocks[failb];
 176         b_off = offs[failb];
 177
 178         /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
 179         /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 180         srcs[0] = p;
 181         src_offs[0] = p_off;
 182         srcs[1] = q;
 183         src_offs[1] = q_off;
 184         coef[0] = raid6_gfexi[failb-faila];
 185         coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 186         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 187         tx = async_sum_product(b, b_off, srcs, src_offs, coef, bytes, submit);
 188
 189         /* Dy = P+Pxy+Dx */
 190         srcs[0] = p;
 191         src_offs[0] = p_off;
 192         srcs[1] = b;
 193         src_offs[1] = b_off;
 194         init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
 195                           cb_param, scribble);
 196         tx = async_xor_offs(a, a_off, srcs, src_offs, 2, bytes, submit);
 197
 198         return tx;
 199
 200 }
 201
 202 static struct dma_async_tx_descriptor *
 203 __2data_recov_5(int disks, size_t bytes, int faila, int failb,
 204                 struct page **blocks, unsigned int *offs,
 205                 struct async_submit_ctl *submit)
 206 {
 207         struct dma_async_tx_descriptor *tx = NULL;
 208         struct page *p, *q, *g, *dp, *dq;
 209         unsigned int p_off, q_off, g_off, dp_off, dq_off;
 210         struct page *srcs[2];
 211         unsigned int src_offs[2];
 212         unsigned char coef[2];
 213         enum async_tx_flags flags = submit->flags;
 214         dma_async_tx_callback cb_fn = submit->cb_fn;
 215         void *cb_param = submit->cb_param;
 216         void *scribble = submit->scribble;
 217         int good_srcs, good, i;
 218
 219         good_srcs = 0;
 220         good = -1;
 221         for (i = 0; i < disks-2; i++) {
 222                 if (blocks[i] == NULL)
 223                         continue;
 224                 if (i == faila || i == failb)
 225                         continue;
 226                 good = i;
 227                 good_srcs++;
 228         }
 229         BUG_ON(good_srcs > 1);
 230
 231         p = blocks[disks-2];
 232         p_off = offs[disks-2];
 233         q = blocks[disks-1];
 234         q_off = offs[disks-1];
 235         g = blocks[good];
 236         g_off = offs[good];
 237
 238         /* Compute syndrome with zero for the missing data pages
 239          * Use the dead data pages as temporary storage for delta p and
 240          * delta q
 241          */
 242         dp = blocks[faila];
 243         dp_off = offs[faila];
 244         dq = blocks[failb];
 245         dq_off = offs[failb];
 246
 247         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 248         tx = async_memcpy(dp, g, dp_off, g_off, bytes, submit);
 249         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 250         tx = async_mult(dq, dq_off, g, g_off,
 251                         raid6_gfexp[good], bytes, submit);
 252
 253         /* compute P + Pxy */
 254         srcs[0] = dp;
 255         src_offs[0] = dp_off;
 256         srcs[1] = p;
 257         src_offs[1] = p_off;
 258         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 259                           NULL, NULL, scribble);
 260         tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
 261
 262         /* compute Q + Qxy */
 263         srcs[0] = dq;
 264         src_offs[0] = dq_off;
 265         srcs[1] = q;
 266         src_offs[1] = q_off;
 267         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 268                           NULL, NULL, scribble);
 269         tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
 270
 271         /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 272         srcs[0] = dp;
 273         src_offs[0] = dp_off;
 274         srcs[1] = dq;
 275         src_offs[1] = dq_off;
 276         coef[0] = raid6_gfexi[failb-faila];
 277         coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 278         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 279         tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
 280
 281         /* Dy = P+Pxy+Dx */
 282         srcs[0] = dp;
 283         src_offs[0] = dp_off;
 284         srcs[1] = dq;
 285         src_offs[1] = dq_off;
 286         init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 287                           cb_param, scribble);
 288         tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
 289
 290         return tx;
 291 }
 292
 293 static struct dma_async_tx_descriptor *
 294 __2data_recov_n(int disks, size_t bytes, int faila, int failb,
 295               struct page **blocks, unsigned int *offs,
 296                   struct async_submit_ctl *submit)
 297 {
 298         struct dma_async_tx_descriptor *tx = NULL;
 299         struct page *p, *q, *dp, *dq;
 300         unsigned int p_off, q_off, dp_off, dq_off;
 301         struct page *srcs[2];
 302         unsigned int src_offs[2];
 303         unsigned char coef[2];
 304         enum async_tx_flags flags = submit->flags;
 305         dma_async_tx_callback cb_fn = submit->cb_fn;
 306         void *cb_param = submit->cb_param;
 307         void *scribble = submit->scribble;
 308
 309         p = blocks[disks-2];
 310         p_off = offs[disks-2];
 311         q = blocks[disks-1];
 312         q_off = offs[disks-1];
 313
 314         /* Compute syndrome with zero for the missing data pages
 315          * Use the dead data pages as temporary storage for
 316          * delta p and delta q
 317          */
 318         dp = blocks[faila];
 319         dp_off = offs[faila];
 320         blocks[faila] = NULL;
 321         blocks[disks-2] = dp;
 322         offs[disks-2] = dp_off;
 323         dq = blocks[failb];
 324         dq_off = offs[failb];
 325         blocks[failb] = NULL;
 326         blocks[disks-1] = dq;
 327         offs[disks-1] = dq_off;
 328
 329         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 330         tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
 331
 332         /* Restore pointer table */
 333         blocks[faila]   = dp;
 334         offs[faila] = dp_off;
 335         blocks[failb]   = dq;
 336         offs[failb] = dq_off;
 337         blocks[disks-2] = p;
 338         offs[disks-2] = p_off;
 339         blocks[disks-1] = q;
 340         offs[disks-1] = q_off;
 341
 342         /* compute P + Pxy */
 343         srcs[0] = dp;
 344         src_offs[0] = dp_off;
 345         srcs[1] = p;
 346         src_offs[1] = p_off;
 347         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 348                           NULL, NULL, scribble);
 349         tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
 350
 351         /* compute Q + Qxy */
 352         srcs[0] = dq;
 353         src_offs[0] = dq_off;
 354         srcs[1] = q;
 355         src_offs[1] = q_off;
 356         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 357                           NULL, NULL, scribble);
 358         tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
 359
 360         /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 361         srcs[0] = dp;
 362         src_offs[0] = dp_off;
 363         srcs[1] = dq;
 364         src_offs[1] = dq_off;
 365         coef[0] = raid6_gfexi[failb-faila];
 366         coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 367         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 368         tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
 369
 370         /* Dy = P+Pxy+Dx */
 371         srcs[0] = dp;
 372         src_offs[0] = dp_off;
 373         srcs[1] = dq;
 374         src_offs[1] = dq_off;
 375         init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 376                           cb_param, scribble);
 377         tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
 378
 379         return tx;
 380 }
 381
 382 /**
 383  * async_raid6_2data_recov - asynchronously calculate two missing data blocks
 384  * @disks: number of disks in the RAID-6 array
 385  * @bytes: block size
 386  * @faila: first failed drive index
 387  * @failb: second failed drive index
 388  * @blocks: array of source pointers where the last two entries are p and q
 389  * @offs: array of offset for pages in blocks
 390  * @submit: submission/completion modifiers
 391  */
 392 struct dma_async_tx_descriptor *
 393 async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
 394                         struct page **blocks, unsigned int *offs,
 395                         struct async_submit_ctl *submit)
 396 {
 397         void *scribble = submit->scribble;
 398         int non_zero_srcs, i;
 399
 400         BUG_ON(faila == failb);
 401         if (failb < faila)
 402                 swap(faila, failb);
 403
 404         pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
 405
 406         /* if a dma resource is not available or a scribble buffer is not
 407          * available punt to the synchronous path.  In the 'dma not
 408          * available' case be sure to use the scribble buffer to
 409          * preserve the content of 'blocks' as the caller intended.
 410          */
 411         if (!async_dma_find_channel(DMA_PQ) || !scribble) {
 412                 void **ptrs = scribble ? scribble : (void **) blocks;
 413
 414                 async_tx_quiesce(&submit->depend_tx);
 415                 for (i = 0; i < disks; i++)
 416                         if (blocks[i] == NULL)
 417                                 ptrs[i] = (void *) raid6_empty_zero_page;
 418                         else
 419                                 ptrs[i] = page_address(blocks[i]) + offs[i];
 420
 421                 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
 422
 423                 async_tx_sync_epilog(submit);
 424
 425                 return NULL;
 426         }
 427
 428         non_zero_srcs = 0;
 429         for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
 430                 if (blocks[i])
 431                         non_zero_srcs++;
 432         switch (non_zero_srcs) {
 433         case 0:
 434         case 1:
 435                 /* There must be at least 2 sources - the failed devices. */
 436                 BUG();
 437
 438         case 2:
 439                 /* dma devices do not uniformly understand a zero source pq
 440                  * operation (in contrast to the synchronous case), so
 441                  * explicitly handle the special case of a 4 disk array with
 442                  * both data disks missing.
 443                  */
 444                 return __2data_recov_4(disks, bytes, faila, failb,
 445                                 blocks, offs, submit);
 446         case 3:
 447                 /* dma devices do not uniformly understand a single
 448                  * source pq operation (in contrast to the synchronous
 449                  * case), so explicitly handle the special case of a 5 disk
 450                  * array with 2 of 3 data disks missing.
 451                  */
 452                 return __2data_recov_5(disks, bytes, faila, failb,
 453                                 blocks, offs, submit);
 454         default:
 455                 return __2data_recov_n(disks, bytes, faila, failb,
 456                                 blocks, offs, submit);
 457         }
 458 }
 459 EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
 460
 461 /**
 462  * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
 463  * @disks: number of disks in the RAID-6 array
 464  * @bytes: block size
 465  * @faila: failed drive index
 466  * @blocks: array of source pointers where the last two entries are p and q
 467  * @offs: array of offset for pages in blocks
 468  * @submit: submission/completion modifiers
 469  */
 470 struct dma_async_tx_descriptor *
 471 async_raid6_datap_recov(int disks, size_t bytes, int faila,
 472                         struct page **blocks, unsigned int *offs,
 473                         struct async_submit_ctl *submit)
 474 {
 475         struct dma_async_tx_descriptor *tx = NULL;
 476         struct page *p, *q, *dq;
 477         unsigned int p_off, q_off, dq_off;
 478         u8 coef;
 479         enum async_tx_flags flags = submit->flags;
 480         dma_async_tx_callback cb_fn = submit->cb_fn;
 481         void *cb_param = submit->cb_param;
 482         void *scribble = submit->scribble;
 483         int good_srcs, good, i;
 484         struct page *srcs[2];
 485         unsigned int src_offs[2];
 486
 487         pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
 488
 489         /* if a dma resource is not available or a scribble buffer is not
 490          * available punt to the synchronous path.  In the 'dma not
 491          * available' case be sure to use the scribble buffer to
 492          * preserve the content of 'blocks' as the caller intended.
 493          */
 494         if (!async_dma_find_channel(DMA_PQ) || !scribble) {
 495                 void **ptrs = scribble ? scribble : (void **) blocks;
 496
 497                 async_tx_quiesce(&submit->depend_tx);
 498                 for (i = 0; i < disks; i++)
 499                         if (blocks[i] == NULL)
 500                                 ptrs[i] = (void*)raid6_empty_zero_page;
 501                         else
 502                                 ptrs[i] = page_address(blocks[i]) + offs[i];
 503
 504                 raid6_datap_recov(disks, bytes, faila, ptrs);
 505
 506                 async_tx_sync_epilog(submit);
 507
 508                 return NULL;
 509         }
 510
 511         good_srcs = 0;
 512         good = -1;
 513         for (i = 0; i < disks-2; i++) {
 514                 if (i == faila)
 515                         continue;
 516                 if (blocks[i]) {
 517                         good = i;
 518                         good_srcs++;
 519                         if (good_srcs > 1)
 520                                 break;
 521                 }
 522         }
 523         BUG_ON(good_srcs == 0);
 524
 525         p = blocks[disks-2];
 526         p_off = offs[disks-2];
 527         q = blocks[disks-1];
 528         q_off = offs[disks-1];
 529
 530         /* Compute syndrome with zero for the missing data page
 531          * Use the dead data page as temporary storage for delta q
 532          */
 533         dq = blocks[faila];
 534         dq_off = offs[faila];
 535         blocks[faila] = NULL;
 536         blocks[disks-1] = dq;
 537         offs[disks-1] = dq_off;
 538
 539         /* in the 4-disk case we only need to perform a single source
 540          * multiplication with the one good data block.
 541          */
 542         if (good_srcs == 1) {
 543                 struct page *g = blocks[good];
 544                 unsigned int g_off = offs[good];
 545
 546                 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 547                                   scribble);
 548                 tx = async_memcpy(p, g, p_off, g_off, bytes, submit);
 549
 550                 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 551                                   scribble);
 552                 tx = async_mult(dq, dq_off, g, g_off,
 553                                 raid6_gfexp[good], bytes, submit);
 554         } else {
 555                 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 556                                   scribble);
 557                 tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
 558         }
 559
 560         /* Restore pointer table */
 561         blocks[faila]   = dq;
 562         offs[faila] = dq_off;
 563         blocks[disks-1] = q;
 564         offs[disks-1] = q_off;
 565
 566         /* calculate g^{-faila} */
 567         coef = raid6_gfinv[raid6_gfexp[faila]];
 568
 569         srcs[0] = dq;
 570         src_offs[0] = dq_off;
 571         srcs[1] = q;
 572         src_offs[1] = q_off;
 573         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 574                           NULL, NULL, scribble);
 575         tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
 576
 577         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 578         tx = async_mult(dq, dq_off, dq, dq_off, coef, bytes, submit);
 579
 580         srcs[0] = p;
 581         src_offs[0] = p_off;
 582         srcs[1] = dq;
 583         src_offs[1] = dq_off;
 584         init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 585                           cb_param, scribble);
 586         tx = async_xor_offs(p, p_off, srcs, src_offs, 2, bytes, submit);
 587
 588         return tx;
 589 }
 590 EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
 591
 592 MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
 593 MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
 594 MODULE_LICENSE("GPL");