crypto/async_tx/async_raid6_recov.c

   1 /*
   2  * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
   3  * Copyright(c) 2009 Intel Corporation
   4  *
   5  * based on raid6recov.c:
   6  *   Copyright 2002 H. Peter Anvin
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along with
  19  * this program; if not, write to the Free Software Foundation, Inc., 51
  20  * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  21  *
  22  */
  23 #include <linux/kernel.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/dma-mapping.h>
  27 #include <linux/raid/pq.h>
  28 #include <linux/async_tx.h>
  29 #include <linux/dmaengine.h>
  30
  31 static struct dma_async_tx_descriptor *
  32 async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
  33                   size_t len, struct async_submit_ctl *submit)
  34 {
  35         struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
  36                                                       &dest, 1, srcs, 2, len);
  37         struct dma_device *dma = chan ? chan->device : NULL;
  38         struct dmaengine_unmap_data *unmap = NULL;
  39         const u8 *amul, *bmul;
  40         u8 ax, bx;
  41         u8 *a, *b, *c;
  42
  43         if (dma)
  44                 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
  45
  46         if (unmap) {
  47                 struct device *dev = dma->dev;
  48                 dma_addr_t pq[2];
  49                 struct dma_async_tx_descriptor *tx;
  50                 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
  51
  52                 if (submit->flags & ASYNC_TX_FENCE)
  53                         dma_flags |= DMA_PREP_FENCE;
  54                 unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
  55                 unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
  56                 unmap->to_cnt = 2;
  57
  58                 unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
  59                 unmap->bidi_cnt = 1;
  60                 /* engine only looks at Q, but expects it to follow P */
  61                 pq[1] = unmap->addr[2];
  62
  63                 unmap->len = len;
  64                 tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
  65                                              len, dma_flags);
  66                 if (tx) {
  67                         dma_set_unmap(tx, unmap);
  68                         async_tx_submit(chan, tx, submit);
  69                         dmaengine_unmap_put(unmap);
  70                         return tx;
  71                 }
  72
  73                 /* could not get a descriptor, unmap and fall through to
  74                  * the synchronous path
  75                  */
  76                 dmaengine_unmap_put(unmap);
  77         }
  78
  79         /* run the operation synchronously */
  80         async_tx_quiesce(&submit->depend_tx);
  81         amul = raid6_gfmul[coef[0]];
  82         bmul = raid6_gfmul[coef[1]];
  83         a = page_address(srcs[0]);
  84         b = page_address(srcs[1]);
  85         c = page_address(dest);
  86
  87         while (len--) {
  88                 ax    = amul[*a++];
  89                 bx    = bmul[*b++];
  90                 *c++ = ax ^ bx;
  91         }
  92
  93         return NULL;
  94 }
  95
  96 static struct dma_async_tx_descriptor *
  97 async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
  98            struct async_submit_ctl *submit)
  99 {
 100         struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
 101                                                       &dest, 1, &src, 1, len);
 102         struct dma_device *dma = chan ? chan->device : NULL;
 103         struct dmaengine_unmap_data *unmap = NULL;
 104         const u8 *qmul; /* Q multiplier table */
 105         u8 *d, *s;
 106
 107         if (dma)
 108                 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
 109
 110         if (unmap) {
 111                 dma_addr_t dma_dest[2];
 112                 struct device *dev = dma->dev;
 113                 struct dma_async_tx_descriptor *tx;
 114                 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
 115
 116                 if (submit->flags & ASYNC_TX_FENCE)
 117                         dma_flags |= DMA_PREP_FENCE;
 118                 unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
 119                 unmap->to_cnt++;
 120                 unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
 121                 dma_dest[1] = unmap->addr[1];
 122                 unmap->bidi_cnt++;
 123                 unmap->len = len;
 124
 125                 /* this looks funny, but the engine looks for Q at
 126                  * dma_dest[1] and ignores dma_dest[0] as a dest
 127                  * due to DMA_PREP_PQ_DISABLE_P
 128                  */
 129                 tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
 130                                              1, &coef, len, dma_flags);
 131
 132                 if (tx) {
 133                         dma_set_unmap(tx, unmap);
 134                         dmaengine_unmap_put(unmap);
 135                         async_tx_submit(chan, tx, submit);
 136                         return tx;
 137                 }
 138
 139                 /* could not get a descriptor, unmap and fall through to
 140                  * the synchronous path
 141                  */
 142                 dmaengine_unmap_put(unmap);
 143         }
 144
 145         /* no channel available, or failed to allocate a descriptor, so
 146          * perform the operation synchronously
 147          */
 148         async_tx_quiesce(&submit->depend_tx);
 149         qmul  = raid6_gfmul[coef];
 150         d = page_address(dest);
 151         s = page_address(src);
 152
 153         while (len--)
 154                 *d++ = qmul[*s++];
 155
 156         return NULL;
 157 }
 158
 159 static struct dma_async_tx_descriptor *
 160 __2data_recov_4(int disks, size_t bytes, int faila, int failb,
 161                 struct page **blocks, struct async_submit_ctl *submit)
 162 {
 163         struct dma_async_tx_descriptor *tx = NULL;
 164         struct page *p, *q, *a, *b;
 165         struct page *srcs[2];
 166         unsigned char coef[2];
 167         enum async_tx_flags flags = submit->flags;
 168         dma_async_tx_callback cb_fn = submit->cb_fn;
 169         void *cb_param = submit->cb_param;
 170         void *scribble = submit->scribble;
 171
 172         p = blocks[disks-2];
 173         q = blocks[disks-1];
 174
 175         a = blocks[faila];
 176         b = blocks[failb];
 177
 178         /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
 179         /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 180         srcs[0] = p;
 181         srcs[1] = q;
 182         coef[0] = raid6_gfexi[failb-faila];
 183         coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 184         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 185         tx = async_sum_product(b, srcs, coef, bytes, submit);
 186
 187         /* Dy = P+Pxy+Dx */
 188         srcs[0] = p;
 189         srcs[1] = b;
 190         init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
 191                           cb_param, scribble);
 192         tx = async_xor(a, srcs, 0, 2, bytes, submit);
 193
 194         return tx;
 195
 196 }
 197
 198 static struct dma_async_tx_descriptor *
 199 __2data_recov_5(int disks, size_t bytes, int faila, int failb,
 200                 struct page **blocks, struct async_submit_ctl *submit)
 201 {
 202         struct dma_async_tx_descriptor *tx = NULL;
 203         struct page *p, *q, *g, *dp, *dq;
 204         struct page *srcs[2];
 205         unsigned char coef[2];
 206         enum async_tx_flags flags = submit->flags;
 207         dma_async_tx_callback cb_fn = submit->cb_fn;
 208         void *cb_param = submit->cb_param;
 209         void *scribble = submit->scribble;
 210         int good_srcs, good, i;
 211
 212         good_srcs = 0;
 213         good = -1;
 214         for (i = 0; i < disks-2; i++) {
 215                 if (blocks[i] == NULL)
 216                         continue;
 217                 if (i == faila || i == failb)
 218                         continue;
 219                 good = i;
 220                 good_srcs++;
 221         }
 222         BUG_ON(good_srcs > 1);
 223
 224         p = blocks[disks-2];
 225         q = blocks[disks-1];
 226         g = blocks[good];
 227
 228         /* Compute syndrome with zero for the missing data pages
 229          * Use the dead data pages as temporary storage for delta p and
 230          * delta q
 231          */
 232         dp = blocks[faila];
 233         dq = blocks[failb];
 234
 235         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 236         tx = async_memcpy(dp, g, 0, 0, bytes, submit);
 237         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 238         tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
 239
 240         /* compute P + Pxy */
 241         srcs[0] = dp;
 242         srcs[1] = p;
 243         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 244                           NULL, NULL, scribble);
 245         tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 246
 247         /* compute Q + Qxy */
 248         srcs[0] = dq;
 249         srcs[1] = q;
 250         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 251                           NULL, NULL, scribble);
 252         tx = async_xor(dq, srcs, 0, 2, bytes, submit);
 253
 254         /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 255         srcs[0] = dp;
 256         srcs[1] = dq;
 257         coef[0] = raid6_gfexi[failb-faila];
 258         coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 259         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 260         tx = async_sum_product(dq, srcs, coef, bytes, submit);
 261
 262         /* Dy = P+Pxy+Dx */
 263         srcs[0] = dp;
 264         srcs[1] = dq;
 265         init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 266                           cb_param, scribble);
 267         tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 268
 269         return tx;
 270 }
 271
 272 static struct dma_async_tx_descriptor *
 273 __2data_recov_n(int disks, size_t bytes, int faila, int failb,
 274               struct page **blocks, struct async_submit_ctl *submit)
 275 {
 276         struct dma_async_tx_descriptor *tx = NULL;
 277         struct page *p, *q, *dp, *dq;
 278         struct page *srcs[2];
 279         unsigned char coef[2];
 280         enum async_tx_flags flags = submit->flags;
 281         dma_async_tx_callback cb_fn = submit->cb_fn;
 282         void *cb_param = submit->cb_param;
 283         void *scribble = submit->scribble;
 284
 285         p = blocks[disks-2];
 286         q = blocks[disks-1];
 287
 288         /* Compute syndrome with zero for the missing data pages
 289          * Use the dead data pages as temporary storage for
 290          * delta p and delta q
 291          */
 292         dp = blocks[faila];
 293         blocks[faila] = NULL;
 294         blocks[disks-2] = dp;
 295         dq = blocks[failb];
 296         blocks[failb] = NULL;
 297         blocks[disks-1] = dq;
 298
 299         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 300         tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
 301
 302         /* Restore pointer table */
 303         blocks[faila]   = dp;
 304         blocks[failb]   = dq;
 305         blocks[disks-2] = p;
 306         blocks[disks-1] = q;
 307
 308         /* compute P + Pxy */
 309         srcs[0] = dp;
 310         srcs[1] = p;
 311         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 312                           NULL, NULL, scribble);
 313         tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 314
 315         /* compute Q + Qxy */
 316         srcs[0] = dq;
 317         srcs[1] = q;
 318         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 319                           NULL, NULL, scribble);
 320         tx = async_xor(dq, srcs, 0, 2, bytes, submit);
 321
 322         /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 323         srcs[0] = dp;
 324         srcs[1] = dq;
 325         coef[0] = raid6_gfexi[failb-faila];
 326         coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 327         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 328         tx = async_sum_product(dq, srcs, coef, bytes, submit);
 329
 330         /* Dy = P+Pxy+Dx */
 331         srcs[0] = dp;
 332         srcs[1] = dq;
 333         init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 334                           cb_param, scribble);
 335         tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 336
 337         return tx;
 338 }
 339
 340 /**
 341  * async_raid6_2data_recov - asynchronously calculate two missing data blocks
 342  * @disks: number of disks in the RAID-6 array
 343  * @bytes: block size
 344  * @faila: first failed drive index
 345  * @failb: second failed drive index
 346  * @blocks: array of source pointers where the last two entries are p and q
 347  * @submit: submission/completion modifiers
 348  */
 349 struct dma_async_tx_descriptor *
 350 async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
 351                         struct page **blocks, struct async_submit_ctl *submit)
 352 {
 353         void *scribble = submit->scribble;
 354         int non_zero_srcs, i;
 355
 356         BUG_ON(faila == failb);
 357         if (failb < faila)
 358                 swap(faila, failb);
 359
 360         pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
 361
 362         /* if a dma resource is not available or a scribble buffer is not
 363          * available punt to the synchronous path.  In the 'dma not
 364          * available' case be sure to use the scribble buffer to
 365          * preserve the content of 'blocks' as the caller intended.
 366          */
 367         if (!async_dma_find_channel(DMA_PQ) || !scribble) {
 368                 void **ptrs = scribble ? scribble : (void **) blocks;
 369
 370                 async_tx_quiesce(&submit->depend_tx);
 371                 for (i = 0; i < disks; i++)
 372                         if (blocks[i] == NULL)
 373                                 ptrs[i] = (void *) raid6_empty_zero_page;
 374                         else
 375                                 ptrs[i] = page_address(blocks[i]);
 376
 377                 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
 378
 379                 async_tx_sync_epilog(submit);
 380
 381                 return NULL;
 382         }
 383
 384         non_zero_srcs = 0;
 385         for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
 386                 if (blocks[i])
 387                         non_zero_srcs++;
 388         switch (non_zero_srcs) {
 389         case 0:
 390         case 1:
 391                 /* There must be at least 2 sources - the failed devices. */
 392                 BUG();
 393
 394         case 2:
 395                 /* dma devices do not uniformly understand a zero source pq
 396                  * operation (in contrast to the synchronous case), so
 397                  * explicitly handle the special case of a 4 disk array with
 398                  * both data disks missing.
 399                  */
 400                 return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
 401         case 3:
 402                 /* dma devices do not uniformly understand a single
 403                  * source pq operation (in contrast to the synchronous
 404                  * case), so explicitly handle the special case of a 5 disk
 405                  * array with 2 of 3 data disks missing.
 406                  */
 407                 return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
 408         default:
 409                 return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
 410         }
 411 }
 412 EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
 413
 414 /**
 415  * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
 416  * @disks: number of disks in the RAID-6 array
 417  * @bytes: block size
 418  * @faila: failed drive index
 419  * @blocks: array of source pointers where the last two entries are p and q
 420  * @submit: submission/completion modifiers
 421  */
 422 struct dma_async_tx_descriptor *
 423 async_raid6_datap_recov(int disks, size_t bytes, int faila,
 424                         struct page **blocks, struct async_submit_ctl *submit)
 425 {
 426         struct dma_async_tx_descriptor *tx = NULL;
 427         struct page *p, *q, *dq;
 428         u8 coef;
 429         enum async_tx_flags flags = submit->flags;
 430         dma_async_tx_callback cb_fn = submit->cb_fn;
 431         void *cb_param = submit->cb_param;
 432         void *scribble = submit->scribble;
 433         int good_srcs, good, i;
 434         struct page *srcs[2];
 435
 436         pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
 437
 438         /* if a dma resource is not available or a scribble buffer is not
 439          * available punt to the synchronous path.  In the 'dma not
 440          * available' case be sure to use the scribble buffer to
 441          * preserve the content of 'blocks' as the caller intended.
 442          */
 443         if (!async_dma_find_channel(DMA_PQ) || !scribble) {
 444                 void **ptrs = scribble ? scribble : (void **) blocks;
 445
 446                 async_tx_quiesce(&submit->depend_tx);
 447                 for (i = 0; i < disks; i++)
 448                         if (blocks[i] == NULL)
 449                                 ptrs[i] = (void*)raid6_empty_zero_page;
 450                         else
 451                                 ptrs[i] = page_address(blocks[i]);
 452
 453                 raid6_datap_recov(disks, bytes, faila, ptrs);
 454
 455                 async_tx_sync_epilog(submit);
 456
 457                 return NULL;
 458         }
 459
 460         good_srcs = 0;
 461         good = -1;
 462         for (i = 0; i < disks-2; i++) {
 463                 if (i == faila)
 464                         continue;
 465                 if (blocks[i]) {
 466                         good = i;
 467                         good_srcs++;
 468                         if (good_srcs > 1)
 469                                 break;
 470                 }
 471         }
 472         BUG_ON(good_srcs == 0);
 473
 474         p = blocks[disks-2];
 475         q = blocks[disks-1];
 476
 477         /* Compute syndrome with zero for the missing data page
 478          * Use the dead data page as temporary storage for delta q
 479          */
 480         dq = blocks[faila];
 481         blocks[faila] = NULL;
 482         blocks[disks-1] = dq;
 483
 484         /* in the 4-disk case we only need to perform a single source
 485          * multiplication with the one good data block.
 486          */
 487         if (good_srcs == 1) {
 488                 struct page *g = blocks[good];
 489
 490                 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 491                                   scribble);
 492                 tx = async_memcpy(p, g, 0, 0, bytes, submit);
 493
 494                 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 495                                   scribble);
 496                 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
 497         } else {
 498                 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 499                                   scribble);
 500                 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
 501         }
 502
 503         /* Restore pointer table */
 504         blocks[faila]   = dq;
 505         blocks[disks-1] = q;
 506
 507         /* calculate g^{-faila} */
 508         coef = raid6_gfinv[raid6_gfexp[faila]];
 509
 510         srcs[0] = dq;
 511         srcs[1] = q;
 512         init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 513                           NULL, NULL, scribble);
 514         tx = async_xor(dq, srcs, 0, 2, bytes, submit);
 515
 516         init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 517         tx = async_mult(dq, dq, coef, bytes, submit);
 518
 519         srcs[0] = p;
 520         srcs[1] = dq;
 521         init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 522                           cb_param, scribble);
 523         tx = async_xor(p, srcs, 0, 2, bytes, submit);
 524
 525         return tx;
 526 }
 527 EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
 528
 529 MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
 530 MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
 531 MODULE_LICENSE("GPL");